LLVM 19.0.0git
TargetLowering.cpp
Go to the documentation of this file.
1//===-- TargetLowering.cpp - Implement the TargetLowering class -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the TargetLowering class.
10//
11//===----------------------------------------------------------------------===//
12
14#include "llvm/ADT/STLExtras.h"
25#include "llvm/IR/DataLayout.h"
28#include "llvm/IR/LLVMContext.h"
29#include "llvm/MC/MCAsmInfo.h"
30#include "llvm/MC/MCExpr.h"
36#include <cctype>
37using namespace llvm;
38
39/// NOTE: The TargetMachine owns TLOF.
41 : TargetLoweringBase(tm) {}
42
43const char *TargetLowering::getTargetNodeName(unsigned Opcode) const {
44 return nullptr;
45}
46
49}
50
51/// Check whether a given call node is in tail position within its function. If
52/// so, it sets Chain to the input chain of the tail call.
54 SDValue &Chain) const {
56
57 // First, check if tail calls have been disabled in this function.
58 if (F.getFnAttribute("disable-tail-calls").getValueAsBool())
59 return false;
60
61 // Conservatively require the attributes of the call to match those of
62 // the return. Ignore following attributes because they don't affect the
63 // call sequence.
64 AttrBuilder CallerAttrs(F.getContext(), F.getAttributes().getRetAttrs());
65 for (const auto &Attr : {Attribute::Alignment, Attribute::Dereferenceable,
66 Attribute::DereferenceableOrNull, Attribute::NoAlias,
67 Attribute::NonNull, Attribute::NoUndef})
68 CallerAttrs.removeAttribute(Attr);
69
70 if (CallerAttrs.hasAttributes())
71 return false;
72
73 // It's not safe to eliminate the sign / zero extension of the return value.
74 if (CallerAttrs.contains(Attribute::ZExt) ||
75 CallerAttrs.contains(Attribute::SExt))
76 return false;
77
78 // Check if the only use is a function return node.
79 return isUsedByReturnOnly(Node, Chain);
80}
81
83 const uint32_t *CallerPreservedMask,
84 const SmallVectorImpl<CCValAssign> &ArgLocs,
85 const SmallVectorImpl<SDValue> &OutVals) const {
86 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
87 const CCValAssign &ArgLoc = ArgLocs[I];
88 if (!ArgLoc.isRegLoc())
89 continue;
90 MCRegister Reg = ArgLoc.getLocReg();
91 // Only look at callee saved registers.
92 if (MachineOperand::clobbersPhysReg(CallerPreservedMask, Reg))
93 continue;
94 // Check that we pass the value used for the caller.
95 // (We look for a CopyFromReg reading a virtual register that is used
96 // for the function live-in value of register Reg)
97 SDValue Value = OutVals[I];
98 if (Value->getOpcode() == ISD::AssertZext)
99 Value = Value.getOperand(0);
100 if (Value->getOpcode() != ISD::CopyFromReg)
101 return false;
102 Register ArgReg = cast<RegisterSDNode>(Value->getOperand(1))->getReg();
103 if (MRI.getLiveInPhysReg(ArgReg) != Reg)
104 return false;
105 }
106 return true;
107}
108
109/// Set CallLoweringInfo attribute flags based on a call instruction
110/// and called function attributes.
112 unsigned ArgIdx) {
113 IsSExt = Call->paramHasAttr(ArgIdx, Attribute::SExt);
114 IsZExt = Call->paramHasAttr(ArgIdx, Attribute::ZExt);
115 IsInReg = Call->paramHasAttr(ArgIdx, Attribute::InReg);
116 IsSRet = Call->paramHasAttr(ArgIdx, Attribute::StructRet);
117 IsNest = Call->paramHasAttr(ArgIdx, Attribute::Nest);
118 IsByVal = Call->paramHasAttr(ArgIdx, Attribute::ByVal);
119 IsPreallocated = Call->paramHasAttr(ArgIdx, Attribute::Preallocated);
120 IsInAlloca = Call->paramHasAttr(ArgIdx, Attribute::InAlloca);
121 IsReturned = Call->paramHasAttr(ArgIdx, Attribute::Returned);
122 IsSwiftSelf = Call->paramHasAttr(ArgIdx, Attribute::SwiftSelf);
123 IsSwiftAsync = Call->paramHasAttr(ArgIdx, Attribute::SwiftAsync);
124 IsSwiftError = Call->paramHasAttr(ArgIdx, Attribute::SwiftError);
125 Alignment = Call->getParamStackAlign(ArgIdx);
126 IndirectType = nullptr;
128 "multiple ABI attributes?");
129 if (IsByVal) {
130 IndirectType = Call->getParamByValType(ArgIdx);
131 if (!Alignment)
132 Alignment = Call->getParamAlign(ArgIdx);
133 }
134 if (IsPreallocated)
135 IndirectType = Call->getParamPreallocatedType(ArgIdx);
136 if (IsInAlloca)
137 IndirectType = Call->getParamInAllocaType(ArgIdx);
138 if (IsSRet)
139 IndirectType = Call->getParamStructRetType(ArgIdx);
140}
141
142/// Generate a libcall taking the given operands as arguments and returning a
143/// result of type RetVT.
144std::pair<SDValue, SDValue>
147 MakeLibCallOptions CallOptions,
148 const SDLoc &dl,
149 SDValue InChain) const {
150 if (!InChain)
151 InChain = DAG.getEntryNode();
152
154 Args.reserve(Ops.size());
155
157 for (unsigned i = 0; i < Ops.size(); ++i) {
158 SDValue NewOp = Ops[i];
159 Entry.Node = NewOp;
160 Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
161 Entry.IsSExt = shouldSignExtendTypeInLibCall(NewOp.getValueType(),
162 CallOptions.IsSExt);
163 Entry.IsZExt = !Entry.IsSExt;
164
165 if (CallOptions.IsSoften &&
167 Entry.IsSExt = Entry.IsZExt = false;
168 }
169 Args.push_back(Entry);
170 }
171
172 if (LC == RTLIB::UNKNOWN_LIBCALL)
173 report_fatal_error("Unsupported library call operation!");
176
177 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
179 bool signExtend = shouldSignExtendTypeInLibCall(RetVT, CallOptions.IsSExt);
180 bool zeroExtend = !signExtend;
181
182 if (CallOptions.IsSoften &&
184 signExtend = zeroExtend = false;
185 }
186
187 CLI.setDebugLoc(dl)
188 .setChain(InChain)
189 .setLibCallee(getLibcallCallingConv(LC), RetTy, Callee, std::move(Args))
190 .setNoReturn(CallOptions.DoesNotReturn)
193 .setSExtResult(signExtend)
194 .setZExtResult(zeroExtend);
195 return LowerCallTo(CLI);
196}
197
199 std::vector<EVT> &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS,
200 unsigned SrcAS, const AttributeList &FuncAttributes) const {
201 if (Limit != ~unsigned(0) && Op.isMemcpyWithFixedDstAlign() &&
202 Op.getSrcAlign() < Op.getDstAlign())
203 return false;
204
205 EVT VT = getOptimalMemOpType(Op, FuncAttributes);
206
207 if (VT == MVT::Other) {
208 // Use the largest integer type whose alignment constraints are satisfied.
209 // We only need to check DstAlign here as SrcAlign is always greater or
210 // equal to DstAlign (or zero).
211 VT = MVT::i64;
212 if (Op.isFixedDstAlign())
213 while (Op.getDstAlign() < (VT.getSizeInBits() / 8) &&
214 !allowsMisalignedMemoryAccesses(VT, DstAS, Op.getDstAlign()))
216 assert(VT.isInteger());
217
218 // Find the largest legal integer type.
219 MVT LVT = MVT::i64;
220 while (!isTypeLegal(LVT))
221 LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
222 assert(LVT.isInteger());
223
224 // If the type we've chosen is larger than the largest legal integer type
225 // then use that instead.
226 if (VT.bitsGT(LVT))
227 VT = LVT;
228 }
229
230 unsigned NumMemOps = 0;
231 uint64_t Size = Op.size();
232 while (Size) {
233 unsigned VTSize = VT.getSizeInBits() / 8;
234 while (VTSize > Size) {
235 // For now, only use non-vector load / store's for the left-over pieces.
236 EVT NewVT = VT;
237 unsigned NewVTSize;
238
239 bool Found = false;
240 if (VT.isVector() || VT.isFloatingPoint()) {
241 NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;
244 Found = true;
245 else if (NewVT == MVT::i64 &&
247 isSafeMemOpType(MVT::f64)) {
248 // i64 is usually not legal on 32-bit targets, but f64 may be.
249 NewVT = MVT::f64;
250 Found = true;
251 }
252 }
253
254 if (!Found) {
255 do {
256 NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
257 if (NewVT == MVT::i8)
258 break;
259 } while (!isSafeMemOpType(NewVT.getSimpleVT()));
260 }
261 NewVTSize = NewVT.getSizeInBits() / 8;
262
263 // If the new VT cannot cover all of the remaining bits, then consider
264 // issuing a (or a pair of) unaligned and overlapping load / store.
265 unsigned Fast;
266 if (NumMemOps && Op.allowOverlap() && NewVTSize < Size &&
268 VT, DstAS, Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1),
270 Fast)
271 VTSize = Size;
272 else {
273 VT = NewVT;
274 VTSize = NewVTSize;
275 }
276 }
277
278 if (++NumMemOps > Limit)
279 return false;
280
281 MemOps.push_back(VT);
282 Size -= VTSize;
283 }
284
285 return true;
286}
287
288/// Soften the operands of a comparison. This code is shared among BR_CC,
289/// SELECT_CC, and SETCC handlers.
291 SDValue &NewLHS, SDValue &NewRHS,
292 ISD::CondCode &CCCode,
293 const SDLoc &dl, const SDValue OldLHS,
294 const SDValue OldRHS) const {
295 SDValue Chain;
296 return softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, dl, OldLHS,
297 OldRHS, Chain);
298}
299
301 SDValue &NewLHS, SDValue &NewRHS,
302 ISD::CondCode &CCCode,
303 const SDLoc &dl, const SDValue OldLHS,
304 const SDValue OldRHS,
305 SDValue &Chain,
306 bool IsSignaling) const {
307 // FIXME: Currently we cannot really respect all IEEE predicates due to libgcc
308 // not supporting it. We can update this code when libgcc provides such
309 // functions.
310
311 assert((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128)
312 && "Unsupported setcc type!");
313
314 // Expand into one or more soft-fp libcall(s).
315 RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL;
316 bool ShouldInvertCC = false;
317 switch (CCCode) {
318 case ISD::SETEQ:
319 case ISD::SETOEQ:
320 LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
321 (VT == MVT::f64) ? RTLIB::OEQ_F64 :
322 (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
323 break;
324 case ISD::SETNE:
325 case ISD::SETUNE:
326 LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 :
327 (VT == MVT::f64) ? RTLIB::UNE_F64 :
328 (VT == MVT::f128) ? RTLIB::UNE_F128 : RTLIB::UNE_PPCF128;
329 break;
330 case ISD::SETGE:
331 case ISD::SETOGE:
332 LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
333 (VT == MVT::f64) ? RTLIB::OGE_F64 :
334 (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
335 break;
336 case ISD::SETLT:
337 case ISD::SETOLT:
338 LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
339 (VT == MVT::f64) ? RTLIB::OLT_F64 :
340 (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
341 break;
342 case ISD::SETLE:
343 case ISD::SETOLE:
344 LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
345 (VT == MVT::f64) ? RTLIB::OLE_F64 :
346 (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
347 break;
348 case ISD::SETGT:
349 case ISD::SETOGT:
350 LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
351 (VT == MVT::f64) ? RTLIB::OGT_F64 :
352 (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
353 break;
354 case ISD::SETO:
355 ShouldInvertCC = true;
356 [[fallthrough]];
357 case ISD::SETUO:
358 LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
359 (VT == MVT::f64) ? RTLIB::UO_F64 :
360 (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
361 break;
362 case ISD::SETONE:
363 // SETONE = O && UNE
364 ShouldInvertCC = true;
365 [[fallthrough]];
366 case ISD::SETUEQ:
367 LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
368 (VT == MVT::f64) ? RTLIB::UO_F64 :
369 (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
370 LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
371 (VT == MVT::f64) ? RTLIB::OEQ_F64 :
372 (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
373 break;
374 default:
375 // Invert CC for unordered comparisons
376 ShouldInvertCC = true;
377 switch (CCCode) {
378 case ISD::SETULT:
379 LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
380 (VT == MVT::f64) ? RTLIB::OGE_F64 :
381 (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
382 break;
383 case ISD::SETULE:
384 LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
385 (VT == MVT::f64) ? RTLIB::OGT_F64 :
386 (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
387 break;
388 case ISD::SETUGT:
389 LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
390 (VT == MVT::f64) ? RTLIB::OLE_F64 :
391 (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
392 break;
393 case ISD::SETUGE:
394 LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
395 (VT == MVT::f64) ? RTLIB::OLT_F64 :
396 (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
397 break;
398 default: llvm_unreachable("Do not know how to soften this setcc!");
399 }
400 }
401
402 // Use the target specific return value for comparison lib calls.
404 SDValue Ops[2] = {NewLHS, NewRHS};
406 EVT OpsVT[2] = { OldLHS.getValueType(),
407 OldRHS.getValueType() };
408 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT, true);
409 auto Call = makeLibCall(DAG, LC1, RetVT, Ops, CallOptions, dl, Chain);
410 NewLHS = Call.first;
411 NewRHS = DAG.getConstant(0, dl, RetVT);
412
413 CCCode = getCmpLibcallCC(LC1);
414 if (ShouldInvertCC) {
415 assert(RetVT.isInteger());
416 CCCode = getSetCCInverse(CCCode, RetVT);
417 }
418
419 if (LC2 == RTLIB::UNKNOWN_LIBCALL) {
420 // Update Chain.
421 Chain = Call.second;
422 } else {
423 EVT SetCCVT =
424 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), RetVT);
425 SDValue Tmp = DAG.getSetCC(dl, SetCCVT, NewLHS, NewRHS, CCCode);
426 auto Call2 = makeLibCall(DAG, LC2, RetVT, Ops, CallOptions, dl, Chain);
427 CCCode = getCmpLibcallCC(LC2);
428 if (ShouldInvertCC)
429 CCCode = getSetCCInverse(CCCode, RetVT);
430 NewLHS = DAG.getSetCC(dl, SetCCVT, Call2.first, NewRHS, CCCode);
431 if (Chain)
432 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Call.second,
433 Call2.second);
434 NewLHS = DAG.getNode(ShouldInvertCC ? ISD::AND : ISD::OR, dl,
435 Tmp.getValueType(), Tmp, NewLHS);
436 NewRHS = SDValue();
437 }
438}
439
440/// Return the entry encoding for a jump table in the current function. The
441/// returned value is a member of the MachineJumpTableInfo::JTEntryKind enum.
443 // In non-pic modes, just use the address of a block.
444 if (!isPositionIndependent())
446
447 // In PIC mode, if the target supports a GPRel32 directive, use it.
448 if (getTargetMachine().getMCAsmInfo()->getGPRel32Directive() != nullptr)
450
451 // Otherwise, use a label difference.
453}
454
456 SelectionDAG &DAG) const {
457 // If our PIC model is GP relative, use the global offset table as the base.
458 unsigned JTEncoding = getJumpTableEncoding();
459
463
464 return Table;
465}
466
467/// This returns the relocation base for the given PIC jumptable, the same as
468/// getPICJumpTableRelocBase, but as an MCExpr.
469const MCExpr *
471 unsigned JTI,MCContext &Ctx) const{
472 // The normal PIC reloc base is the label at the start of the jump table.
473 return MCSymbolRefExpr::create(MF->getJTISymbol(JTI, Ctx), Ctx);
474}
475
477 SDValue Addr, int JTI,
478 SelectionDAG &DAG) const {
479 SDValue Chain = Value;
480 // Jump table debug info is only needed if CodeView is enabled.
482 Chain = DAG.getJumpTableDebugInfo(JTI, Chain, dl);
483 }
484 return DAG.getNode(ISD::BRIND, dl, MVT::Other, Chain, Addr);
485}
486
487bool
489 const TargetMachine &TM = getTargetMachine();
490 const GlobalValue *GV = GA->getGlobal();
491
492 // If the address is not even local to this DSO we will have to load it from
493 // a got and then add the offset.
494 if (!TM.shouldAssumeDSOLocal(GV))
495 return false;
496
497 // If the code is position independent we will have to add a base register.
498 if (isPositionIndependent())
499 return false;
500
501 // Otherwise we can do it.
502 return true;
503}
504
505//===----------------------------------------------------------------------===//
506// Optimization Methods
507//===----------------------------------------------------------------------===//
508
509/// If the specified instruction has a constant integer operand and there are
510/// bits set in that constant that are not demanded, then clear those bits and
511/// return true.
513 const APInt &DemandedBits,
514 const APInt &DemandedElts,
515 TargetLoweringOpt &TLO) const {
516 SDLoc DL(Op);
517 unsigned Opcode = Op.getOpcode();
518
519 // Early-out if we've ended up calling an undemanded node, leave this to
520 // constant folding.
521 if (DemandedBits.isZero() || DemandedElts.isZero())
522 return false;
523
524 // Do target-specific constant optimization.
525 if (targetShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
526 return TLO.New.getNode();
527
528 // FIXME: ISD::SELECT, ISD::SELECT_CC
529 switch (Opcode) {
530 default:
531 break;
532 case ISD::XOR:
533 case ISD::AND:
534 case ISD::OR: {
535 auto *Op1C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
536 if (!Op1C || Op1C->isOpaque())
537 return false;
538
539 // If this is a 'not' op, don't touch it because that's a canonical form.
540 const APInt &C = Op1C->getAPIntValue();
541 if (Opcode == ISD::XOR && DemandedBits.isSubsetOf(C))
542 return false;
543
544 if (!C.isSubsetOf(DemandedBits)) {
545 EVT VT = Op.getValueType();
546 SDValue NewC = TLO.DAG.getConstant(DemandedBits & C, DL, VT);
547 SDValue NewOp = TLO.DAG.getNode(Opcode, DL, VT, Op.getOperand(0), NewC);
548 return TLO.CombineTo(Op, NewOp);
549 }
550
551 break;
552 }
553 }
554
555 return false;
556}
557
559 const APInt &DemandedBits,
560 TargetLoweringOpt &TLO) const {
561 EVT VT = Op.getValueType();
562 APInt DemandedElts = VT.isVector()
564 : APInt(1, 1);
565 return ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO);
566}
567
568/// Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.
569/// This uses isTruncateFree/isZExtFree and ANY_EXTEND for the widening cast,
570/// but it could be generalized for targets with other types of implicit
571/// widening casts.
573 const APInt &DemandedBits,
574 TargetLoweringOpt &TLO) const {
575 assert(Op.getNumOperands() == 2 &&
576 "ShrinkDemandedOp only supports binary operators!");
577 assert(Op.getNode()->getNumValues() == 1 &&
578 "ShrinkDemandedOp only supports nodes with one result!");
579
580 EVT VT = Op.getValueType();
581 SelectionDAG &DAG = TLO.DAG;
582 SDLoc dl(Op);
583
584 // Early return, as this function cannot handle vector types.
585 if (VT.isVector())
586 return false;
587
588 // Don't do this if the node has another user, which may require the
589 // full value.
590 if (!Op.getNode()->hasOneUse())
591 return false;
592
593 // Search for the smallest integer type with free casts to and from
594 // Op's type. For expedience, just check power-of-2 integer types.
595 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
596 unsigned DemandedSize = DemandedBits.getActiveBits();
597 for (unsigned SmallVTBits = llvm::bit_ceil(DemandedSize);
598 SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(SmallVTBits)) {
599 EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), SmallVTBits);
600 if (TLI.isTruncateFree(VT, SmallVT) && TLI.isZExtFree(SmallVT, VT)) {
601 // We found a type with free casts.
602 SDValue X = DAG.getNode(
603 Op.getOpcode(), dl, SmallVT,
604 DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(0)),
605 DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(1)));
606 assert(DemandedSize <= SmallVTBits && "Narrowed below demanded bits?");
607 SDValue Z = DAG.getNode(ISD::ANY_EXTEND, dl, VT, X);
608 return TLO.CombineTo(Op, Z);
609 }
610 }
611 return false;
612}
613
615 DAGCombinerInfo &DCI) const {
616 SelectionDAG &DAG = DCI.DAG;
617 TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
618 !DCI.isBeforeLegalizeOps());
619 KnownBits Known;
620
621 bool Simplified = SimplifyDemandedBits(Op, DemandedBits, Known, TLO);
622 if (Simplified) {
623 DCI.AddToWorklist(Op.getNode());
625 }
626 return Simplified;
627}
628
630 const APInt &DemandedElts,
631 DAGCombinerInfo &DCI) const {
632 SelectionDAG &DAG = DCI.DAG;
633 TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
634 !DCI.isBeforeLegalizeOps());
635 KnownBits Known;
636
637 bool Simplified =
638 SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO);
639 if (Simplified) {
640 DCI.AddToWorklist(Op.getNode());
642 }
643 return Simplified;
644}
645
647 KnownBits &Known,
649 unsigned Depth,
650 bool AssumeSingleUse) const {
651 EVT VT = Op.getValueType();
652
653 // Since the number of lanes in a scalable vector is unknown at compile time,
654 // we track one bit which is implicitly broadcast to all lanes. This means
655 // that all lanes in a scalable vector are considered demanded.
656 APInt DemandedElts = VT.isFixedLengthVector()
658 : APInt(1, 1);
659 return SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO, Depth,
660 AssumeSingleUse);
661}
662
663// TODO: Under what circumstances can we create nodes? Constant folding?
665 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
666 SelectionDAG &DAG, unsigned Depth) const {
667 EVT VT = Op.getValueType();
668
669 // Limit search depth.
671 return SDValue();
672
673 // Ignore UNDEFs.
674 if (Op.isUndef())
675 return SDValue();
676
677 // Not demanding any bits/elts from Op.
678 if (DemandedBits == 0 || DemandedElts == 0)
679 return DAG.getUNDEF(VT);
680
681 bool IsLE = DAG.getDataLayout().isLittleEndian();
682 unsigned NumElts = DemandedElts.getBitWidth();
683 unsigned BitWidth = DemandedBits.getBitWidth();
684 KnownBits LHSKnown, RHSKnown;
685 switch (Op.getOpcode()) {
686 case ISD::BITCAST: {
687 if (VT.isScalableVector())
688 return SDValue();
689
690 SDValue Src = peekThroughBitcasts(Op.getOperand(0));
691 EVT SrcVT = Src.getValueType();
692 EVT DstVT = Op.getValueType();
693 if (SrcVT == DstVT)
694 return Src;
695
696 unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
697 unsigned NumDstEltBits = DstVT.getScalarSizeInBits();
698 if (NumSrcEltBits == NumDstEltBits)
699 if (SDValue V = SimplifyMultipleUseDemandedBits(
700 Src, DemandedBits, DemandedElts, DAG, Depth + 1))
701 return DAG.getBitcast(DstVT, V);
702
703 if (SrcVT.isVector() && (NumDstEltBits % NumSrcEltBits) == 0) {
704 unsigned Scale = NumDstEltBits / NumSrcEltBits;
705 unsigned NumSrcElts = SrcVT.getVectorNumElements();
706 APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);
707 APInt DemandedSrcElts = APInt::getZero(NumSrcElts);
708 for (unsigned i = 0; i != Scale; ++i) {
709 unsigned EltOffset = IsLE ? i : (Scale - 1 - i);
710 unsigned BitOffset = EltOffset * NumSrcEltBits;
711 APInt Sub = DemandedBits.extractBits(NumSrcEltBits, BitOffset);
712 if (!Sub.isZero()) {
713 DemandedSrcBits |= Sub;
714 for (unsigned j = 0; j != NumElts; ++j)
715 if (DemandedElts[j])
716 DemandedSrcElts.setBit((j * Scale) + i);
717 }
718 }
719
720 if (SDValue V = SimplifyMultipleUseDemandedBits(
721 Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))
722 return DAG.getBitcast(DstVT, V);
723 }
724
725 // TODO - bigendian once we have test coverage.
726 if (IsLE && (NumSrcEltBits % NumDstEltBits) == 0) {
727 unsigned Scale = NumSrcEltBits / NumDstEltBits;
728 unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
729 APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);
730 APInt DemandedSrcElts = APInt::getZero(NumSrcElts);
731 for (unsigned i = 0; i != NumElts; ++i)
732 if (DemandedElts[i]) {
733 unsigned Offset = (i % Scale) * NumDstEltBits;
734 DemandedSrcBits.insertBits(DemandedBits, Offset);
735 DemandedSrcElts.setBit(i / Scale);
736 }
737
738 if (SDValue V = SimplifyMultipleUseDemandedBits(
739 Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))
740 return DAG.getBitcast(DstVT, V);
741 }
742
743 break;
744 }
745 case ISD::AND: {
746 LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
747 RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
748
749 // If all of the demanded bits are known 1 on one side, return the other.
750 // These bits cannot contribute to the result of the 'and' in this
751 // context.
752 if (DemandedBits.isSubsetOf(LHSKnown.Zero | RHSKnown.One))
753 return Op.getOperand(0);
754 if (DemandedBits.isSubsetOf(RHSKnown.Zero | LHSKnown.One))
755 return Op.getOperand(1);
756 break;
757 }
758 case ISD::OR: {
759 LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
760 RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
761
762 // If all of the demanded bits are known zero on one side, return the
763 // other. These bits cannot contribute to the result of the 'or' in this
764 // context.
765 if (DemandedBits.isSubsetOf(LHSKnown.One | RHSKnown.Zero))
766 return Op.getOperand(0);
767 if (DemandedBits.isSubsetOf(RHSKnown.One | LHSKnown.Zero))
768 return Op.getOperand(1);
769 break;
770 }
771 case ISD::XOR: {
772 LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
773 RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
774
775 // If all of the demanded bits are known zero on one side, return the
776 // other.
777 if (DemandedBits.isSubsetOf(RHSKnown.Zero))
778 return Op.getOperand(0);
779 if (DemandedBits.isSubsetOf(LHSKnown.Zero))
780 return Op.getOperand(1);
781 break;
782 }
783 case ISD::SHL: {
784 // If we are only demanding sign bits then we can use the shift source
785 // directly.
786 if (const APInt *MaxSA =
787 DAG.getValidMaximumShiftAmountConstant(Op, DemandedElts)) {
788 SDValue Op0 = Op.getOperand(0);
789 unsigned ShAmt = MaxSA->getZExtValue();
790 unsigned NumSignBits =
791 DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
792 unsigned UpperDemandedBits = BitWidth - DemandedBits.countr_zero();
793 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
794 return Op0;
795 }
796 break;
797 }
798 case ISD::SETCC: {
799 SDValue Op0 = Op.getOperand(0);
800 SDValue Op1 = Op.getOperand(1);
801 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
802 // If (1) we only need the sign-bit, (2) the setcc operands are the same
803 // width as the setcc result, and (3) the result of a setcc conforms to 0 or
804 // -1, we may be able to bypass the setcc.
805 if (DemandedBits.isSignMask() &&
809 // If we're testing X < 0, then this compare isn't needed - just use X!
810 // FIXME: We're limiting to integer types here, but this should also work
811 // if we don't care about FP signed-zero. The use of SETLT with FP means
812 // that we don't care about NaNs.
813 if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
815 return Op0;
816 }
817 break;
818 }
820 // If none of the extended bits are demanded, eliminate the sextinreg.
821 SDValue Op0 = Op.getOperand(0);
822 EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
823 unsigned ExBits = ExVT.getScalarSizeInBits();
824 if (DemandedBits.getActiveBits() <= ExBits &&
826 return Op0;
827 // If the input is already sign extended, just drop the extension.
828 unsigned NumSignBits = DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
829 if (NumSignBits >= (BitWidth - ExBits + 1))
830 return Op0;
831 break;
832 }
836 if (VT.isScalableVector())
837 return SDValue();
838
839 // If we only want the lowest element and none of extended bits, then we can
840 // return the bitcasted source vector.
841 SDValue Src = Op.getOperand(0);
842 EVT SrcVT = Src.getValueType();
843 EVT DstVT = Op.getValueType();
844 if (IsLE && DemandedElts == 1 &&
845 DstVT.getSizeInBits() == SrcVT.getSizeInBits() &&
846 DemandedBits.getActiveBits() <= SrcVT.getScalarSizeInBits()) {
847 return DAG.getBitcast(DstVT, Src);
848 }
849 break;
850 }
852 if (VT.isScalableVector())
853 return SDValue();
854
855 // If we don't demand the inserted element, return the base vector.
856 SDValue Vec = Op.getOperand(0);
857 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
858 EVT VecVT = Vec.getValueType();
859 if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements()) &&
860 !DemandedElts[CIdx->getZExtValue()])
861 return Vec;
862 break;
863 }
865 if (VT.isScalableVector())
866 return SDValue();
867
868 SDValue Vec = Op.getOperand(0);
869 SDValue Sub = Op.getOperand(1);
870 uint64_t Idx = Op.getConstantOperandVal(2);
871 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
872 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
873 // If we don't demand the inserted subvector, return the base vector.
874 if (DemandedSubElts == 0)
875 return Vec;
876 break;
877 }
878 case ISD::VECTOR_SHUFFLE: {
880 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
881
882 // If all the demanded elts are from one operand and are inline,
883 // then we can use the operand directly.
884 bool AllUndef = true, IdentityLHS = true, IdentityRHS = true;
885 for (unsigned i = 0; i != NumElts; ++i) {
886 int M = ShuffleMask[i];
887 if (M < 0 || !DemandedElts[i])
888 continue;
889 AllUndef = false;
890 IdentityLHS &= (M == (int)i);
891 IdentityRHS &= ((M - NumElts) == i);
892 }
893
894 if (AllUndef)
895 return DAG.getUNDEF(Op.getValueType());
896 if (IdentityLHS)
897 return Op.getOperand(0);
898 if (IdentityRHS)
899 return Op.getOperand(1);
900 break;
901 }
902 default:
903 // TODO: Probably okay to remove after audit; here to reduce change size
904 // in initial enablement patch for scalable vectors
905 if (VT.isScalableVector())
906 return SDValue();
907
908 if (Op.getOpcode() >= ISD::BUILTIN_OP_END)
909 if (SDValue V = SimplifyMultipleUseDemandedBitsForTargetNode(
910 Op, DemandedBits, DemandedElts, DAG, Depth))
911 return V;
912 break;
913 }
914 return SDValue();
915}
916
919 unsigned Depth) const {
920 EVT VT = Op.getValueType();
921 // Since the number of lanes in a scalable vector is unknown at compile time,
922 // we track one bit which is implicitly broadcast to all lanes. This means
923 // that all lanes in a scalable vector are considered demanded.
924 APInt DemandedElts = VT.isFixedLengthVector()
926 : APInt(1, 1);
927 return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
928 Depth);
929}
930
932 SDValue Op, const APInt &DemandedElts, SelectionDAG &DAG,
933 unsigned Depth) const {
934 APInt DemandedBits = APInt::getAllOnes(Op.getScalarValueSizeInBits());
935 return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
936 Depth);
937}
938
939// Attempt to form ext(avgfloor(A, B)) from shr(add(ext(A), ext(B)), 1).
940// or to form ext(avgceil(A, B)) from shr(add(ext(A), ext(B), 1), 1).
942 const TargetLowering &TLI,
943 const APInt &DemandedBits,
944 const APInt &DemandedElts,
945 unsigned Depth) {
946 assert((Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SRA) &&
947 "SRL or SRA node is required here!");
948 // Is the right shift using an immediate value of 1?
949 ConstantSDNode *N1C = isConstOrConstSplat(Op.getOperand(1), DemandedElts);
950 if (!N1C || !N1C->isOne())
951 return SDValue();
952
953 // We are looking for an avgfloor
954 // add(ext, ext)
955 // or one of these as a avgceil
956 // add(add(ext, ext), 1)
957 // add(add(ext, 1), ext)
958 // add(ext, add(ext, 1))
959 SDValue Add = Op.getOperand(0);
960 if (Add.getOpcode() != ISD::ADD)
961 return SDValue();
962
963 SDValue ExtOpA = Add.getOperand(0);
964 SDValue ExtOpB = Add.getOperand(1);
965 SDValue Add2;
966 auto MatchOperands = [&](SDValue Op1, SDValue Op2, SDValue Op3, SDValue A) {
967 ConstantSDNode *ConstOp;
968 if ((ConstOp = isConstOrConstSplat(Op2, DemandedElts)) &&
969 ConstOp->isOne()) {
970 ExtOpA = Op1;
971 ExtOpB = Op3;
972 Add2 = A;
973 return true;
974 }
975 if ((ConstOp = isConstOrConstSplat(Op3, DemandedElts)) &&
976 ConstOp->isOne()) {
977 ExtOpA = Op1;
978 ExtOpB = Op2;
979 Add2 = A;
980 return true;
981 }
982 return false;
983 };
984 bool IsCeil =
985 (ExtOpA.getOpcode() == ISD::ADD &&
986 MatchOperands(ExtOpA.getOperand(0), ExtOpA.getOperand(1), ExtOpB, ExtOpA)) ||
987 (ExtOpB.getOpcode() == ISD::ADD &&
988 MatchOperands(ExtOpB.getOperand(0), ExtOpB.getOperand(1), ExtOpA, ExtOpB));
989
990 // If the shift is signed (sra):
991 // - Needs >= 2 sign bit for both operands.
992 // - Needs >= 2 zero bits.
993 // If the shift is unsigned (srl):
994 // - Needs >= 1 zero bit for both operands.
995 // - Needs 1 demanded bit zero and >= 2 sign bits.
996 unsigned ShiftOpc = Op.getOpcode();
997 bool IsSigned = false;
998 unsigned KnownBits;
999 unsigned NumSignedA = DAG.ComputeNumSignBits(ExtOpA, DemandedElts, Depth);
1000 unsigned NumSignedB = DAG.ComputeNumSignBits(ExtOpB, DemandedElts, Depth);
1001 unsigned NumSigned = std::min(NumSignedA, NumSignedB) - 1;
1002 unsigned NumZeroA =
1003 DAG.computeKnownBits(ExtOpA, DemandedElts, Depth).countMinLeadingZeros();
1004 unsigned NumZeroB =
1005 DAG.computeKnownBits(ExtOpB, DemandedElts, Depth).countMinLeadingZeros();
1006 unsigned NumZero = std::min(NumZeroA, NumZeroB);
1007
1008 switch (ShiftOpc) {
1009 default:
1010 llvm_unreachable("Unexpected ShiftOpc in combineShiftToAVG");
1011 case ISD::SRA: {
1012 if (NumZero >= 2 && NumSigned < NumZero) {
1013 IsSigned = false;
1014 KnownBits = NumZero;
1015 break;
1016 }
1017 if (NumSigned >= 1) {
1018 IsSigned = true;
1019 KnownBits = NumSigned;
1020 break;
1021 }
1022 return SDValue();
1023 }
1024 case ISD::SRL: {
1025 if (NumZero >= 1 && NumSigned < NumZero) {
1026 IsSigned = false;
1027 KnownBits = NumZero;
1028 break;
1029 }
1030 if (NumSigned >= 1 && DemandedBits.isSignBitClear()) {
1031 IsSigned = true;
1032 KnownBits = NumSigned;
1033 break;
1034 }
1035 return SDValue();
1036 }
1037 }
1038
1039 unsigned AVGOpc = IsCeil ? (IsSigned ? ISD::AVGCEILS : ISD::AVGCEILU)
1040 : (IsSigned ? ISD::AVGFLOORS : ISD::AVGFLOORU);
1041
1042 // Find the smallest power-2 type that is legal for this vector size and
1043 // operation, given the original type size and the number of known sign/zero
1044 // bits.
1045 EVT VT = Op.getValueType();
1046 unsigned MinWidth =
1047 std::max<unsigned>(VT.getScalarSizeInBits() - KnownBits, 8);
1048 EVT NVT = EVT::getIntegerVT(*DAG.getContext(), llvm::bit_ceil(MinWidth));
1049 if (VT.isVector())
1050 NVT = EVT::getVectorVT(*DAG.getContext(), NVT, VT.getVectorElementCount());
1051 if (!TLI.isOperationLegalOrCustom(AVGOpc, NVT)) {
1052 // If we could not transform, and (both) adds are nuw/nsw, we can use the
1053 // larger type size to do the transform.
1054 if (!TLI.isOperationLegalOrCustom(AVGOpc, VT))
1055 return SDValue();
1056 if (DAG.willNotOverflowAdd(IsSigned, Add.getOperand(0),
1057 Add.getOperand(1)) &&
1058 (!Add2 || DAG.willNotOverflowAdd(IsSigned, Add2.getOperand(0),
1059 Add2.getOperand(1))))
1060 NVT = VT;
1061 else
1062 return SDValue();
1063 }
1064
1065 SDLoc DL(Op);
1066 SDValue ResultAVG =
1067 DAG.getNode(AVGOpc, DL, NVT, DAG.getExtOrTrunc(IsSigned, ExtOpA, DL, NVT),
1068 DAG.getExtOrTrunc(IsSigned, ExtOpB, DL, NVT));
1069 return DAG.getExtOrTrunc(IsSigned, ResultAVG, DL, VT);
1070}
1071
1072/// Look at Op. At this point, we know that only the OriginalDemandedBits of the
1073/// result of Op are ever used downstream. If we can use this information to
1074/// simplify Op, create a new simplified DAG node and return true, returning the
1075/// original and new nodes in Old and New. Otherwise, analyze the expression and
1076/// return a mask of Known bits for the expression (used to simplify the
1077/// caller). The Known bits may only be accurate for those bits in the
1078/// OriginalDemandedBits and OriginalDemandedElts.
1080 SDValue Op, const APInt &OriginalDemandedBits,
1081 const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
1082 unsigned Depth, bool AssumeSingleUse) const {
1083 unsigned BitWidth = OriginalDemandedBits.getBitWidth();
1084 assert(Op.getScalarValueSizeInBits() == BitWidth &&
1085 "Mask size mismatches value type size!");
1086
1087 // Don't know anything.
1088 Known = KnownBits(BitWidth);
1089
1090 EVT VT = Op.getValueType();
1091 bool IsLE = TLO.DAG.getDataLayout().isLittleEndian();
1092 unsigned NumElts = OriginalDemandedElts.getBitWidth();
1093 assert((!VT.isFixedLengthVector() || NumElts == VT.getVectorNumElements()) &&
1094 "Unexpected vector size");
1095
1096 APInt DemandedBits = OriginalDemandedBits;
1097 APInt DemandedElts = OriginalDemandedElts;
1098 SDLoc dl(Op);
1099
1100 // Undef operand.
1101 if (Op.isUndef())
1102 return false;
1103
1104 // We can't simplify target constants.
1105 if (Op.getOpcode() == ISD::TargetConstant)
1106 return false;
1107
1108 if (Op.getOpcode() == ISD::Constant) {
1109 // We know all of the bits for a constant!
1110 Known = KnownBits::makeConstant(Op->getAsAPIntVal());
1111 return false;
1112 }
1113
1114 if (Op.getOpcode() == ISD::ConstantFP) {
1115 // We know all of the bits for a floating point constant!
1117 cast<ConstantFPSDNode>(Op)->getValueAPF().bitcastToAPInt());
1118 return false;
1119 }
1120
1121 // Other users may use these bits.
1122 bool HasMultiUse = false;
1123 if (!AssumeSingleUse && !Op.getNode()->hasOneUse()) {
1125 // Limit search depth.
1126 return false;
1127 }
1128 // Allow multiple uses, just set the DemandedBits/Elts to all bits.
1130 DemandedElts = APInt::getAllOnes(NumElts);
1131 HasMultiUse = true;
1132 } else if (OriginalDemandedBits == 0 || OriginalDemandedElts == 0) {
1133 // Not demanding any bits/elts from Op.
1134 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
1135 } else if (Depth >= SelectionDAG::MaxRecursionDepth) {
1136 // Limit search depth.
1137 return false;
1138 }
1139
1140 KnownBits Known2;
1141 switch (Op.getOpcode()) {
1142 case ISD::SCALAR_TO_VECTOR: {
1143 if (VT.isScalableVector())
1144 return false;
1145 if (!DemandedElts[0])
1146 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
1147
1148 KnownBits SrcKnown;
1149 SDValue Src = Op.getOperand(0);
1150 unsigned SrcBitWidth = Src.getScalarValueSizeInBits();
1151 APInt SrcDemandedBits = DemandedBits.zext(SrcBitWidth);
1152 if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcKnown, TLO, Depth + 1))
1153 return true;
1154
1155 // Upper elements are undef, so only get the knownbits if we just demand
1156 // the bottom element.
1157 if (DemandedElts == 1)
1158 Known = SrcKnown.anyextOrTrunc(BitWidth);
1159 break;
1160 }
1161 case ISD::BUILD_VECTOR:
1162 // Collect the known bits that are shared by every demanded element.
1163 // TODO: Call SimplifyDemandedBits for non-constant demanded elements.
1164 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
1165 return false; // Don't fall through, will infinitely loop.
1166 case ISD::SPLAT_VECTOR: {
1167 SDValue Scl = Op.getOperand(0);
1168 APInt DemandedSclBits = DemandedBits.zextOrTrunc(Scl.getValueSizeInBits());
1169 KnownBits KnownScl;
1170 if (SimplifyDemandedBits(Scl, DemandedSclBits, KnownScl, TLO, Depth + 1))
1171 return true;
1172
1173 // Implicitly truncate the bits to match the official semantics of
1174 // SPLAT_VECTOR.
1175 Known = KnownScl.trunc(BitWidth);
1176 break;
1177 }
1178 case ISD::LOAD: {
1179 auto *LD = cast<LoadSDNode>(Op);
1180 if (getTargetConstantFromLoad(LD)) {
1181 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
1182 return false; // Don't fall through, will infinitely loop.
1183 }
1184 if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
1185 // If this is a ZEXTLoad and we are looking at the loaded value.
1186 EVT MemVT = LD->getMemoryVT();
1187 unsigned MemBits = MemVT.getScalarSizeInBits();
1188 Known.Zero.setBitsFrom(MemBits);
1189 return false; // Don't fall through, will infinitely loop.
1190 }
1191 break;
1192 }
1194 if (VT.isScalableVector())
1195 return false;
1196 SDValue Vec = Op.getOperand(0);
1197 SDValue Scl = Op.getOperand(1);
1198 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
1199 EVT VecVT = Vec.getValueType();
1200
1201 // If index isn't constant, assume we need all vector elements AND the
1202 // inserted element.
1203 APInt DemandedVecElts(DemandedElts);
1204 if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements())) {
1205 unsigned Idx = CIdx->getZExtValue();
1206 DemandedVecElts.clearBit(Idx);
1207
1208 // Inserted element is not required.
1209 if (!DemandedElts[Idx])
1210 return TLO.CombineTo(Op, Vec);
1211 }
1212
1213 KnownBits KnownScl;
1214 unsigned NumSclBits = Scl.getScalarValueSizeInBits();
1215 APInt DemandedSclBits = DemandedBits.zextOrTrunc(NumSclBits);
1216 if (SimplifyDemandedBits(Scl, DemandedSclBits, KnownScl, TLO, Depth + 1))
1217 return true;
1218
1219 Known = KnownScl.anyextOrTrunc(BitWidth);
1220
1221 KnownBits KnownVec;
1222 if (SimplifyDemandedBits(Vec, DemandedBits, DemandedVecElts, KnownVec, TLO,
1223 Depth + 1))
1224 return true;
1225
1226 if (!!DemandedVecElts)
1227 Known = Known.intersectWith(KnownVec);
1228
1229 return false;
1230 }
1231 case ISD::INSERT_SUBVECTOR: {
1232 if (VT.isScalableVector())
1233 return false;
1234 // Demand any elements from the subvector and the remainder from the src its
1235 // inserted into.
1236 SDValue Src = Op.getOperand(0);
1237 SDValue Sub = Op.getOperand(1);
1238 uint64_t Idx = Op.getConstantOperandVal(2);
1239 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
1240 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
1241 APInt DemandedSrcElts = DemandedElts;
1242 DemandedSrcElts.insertBits(APInt::getZero(NumSubElts), Idx);
1243
1244 KnownBits KnownSub, KnownSrc;
1245 if (SimplifyDemandedBits(Sub, DemandedBits, DemandedSubElts, KnownSub, TLO,
1246 Depth + 1))
1247 return true;
1248 if (SimplifyDemandedBits(Src, DemandedBits, DemandedSrcElts, KnownSrc, TLO,
1249 Depth + 1))
1250 return true;
1251
1252 Known.Zero.setAllBits();
1253 Known.One.setAllBits();
1254 if (!!DemandedSubElts)
1255 Known = Known.intersectWith(KnownSub);
1256 if (!!DemandedSrcElts)
1257 Known = Known.intersectWith(KnownSrc);
1258
1259 // Attempt to avoid multi-use src if we don't need anything from it.
1260 if (!DemandedBits.isAllOnes() || !DemandedSubElts.isAllOnes() ||
1261 !DemandedSrcElts.isAllOnes()) {
1262 SDValue NewSub = SimplifyMultipleUseDemandedBits(
1263 Sub, DemandedBits, DemandedSubElts, TLO.DAG, Depth + 1);
1264 SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1265 Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);
1266 if (NewSub || NewSrc) {
1267 NewSub = NewSub ? NewSub : Sub;
1268 NewSrc = NewSrc ? NewSrc : Src;
1269 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc, NewSub,
1270 Op.getOperand(2));
1271 return TLO.CombineTo(Op, NewOp);
1272 }
1273 }
1274 break;
1275 }
1277 if (VT.isScalableVector())
1278 return false;
1279 // Offset the demanded elts by the subvector index.
1280 SDValue Src = Op.getOperand(0);
1281 if (Src.getValueType().isScalableVector())
1282 break;
1283 uint64_t Idx = Op.getConstantOperandVal(1);
1284 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
1285 APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx);
1286
1287 if (SimplifyDemandedBits(Src, DemandedBits, DemandedSrcElts, Known, TLO,
1288 Depth + 1))
1289 return true;
1290
1291 // Attempt to avoid multi-use src if we don't need anything from it.
1292 if (!DemandedBits.isAllOnes() || !DemandedSrcElts.isAllOnes()) {
1293 SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
1294 Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);
1295 if (DemandedSrc) {
1296 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc,
1297 Op.getOperand(1));
1298 return TLO.CombineTo(Op, NewOp);
1299 }
1300 }
1301 break;
1302 }
1303 case ISD::CONCAT_VECTORS: {
1304 if (VT.isScalableVector())
1305 return false;
1306 Known.Zero.setAllBits();
1307 Known.One.setAllBits();
1308 EVT SubVT = Op.getOperand(0).getValueType();
1309 unsigned NumSubVecs = Op.getNumOperands();
1310 unsigned NumSubElts = SubVT.getVectorNumElements();
1311 for (unsigned i = 0; i != NumSubVecs; ++i) {
1312 APInt DemandedSubElts =
1313 DemandedElts.extractBits(NumSubElts, i * NumSubElts);
1314 if (SimplifyDemandedBits(Op.getOperand(i), DemandedBits, DemandedSubElts,
1315 Known2, TLO, Depth + 1))
1316 return true;
1317 // Known bits are shared by every demanded subvector element.
1318 if (!!DemandedSubElts)
1319 Known = Known.intersectWith(Known2);
1320 }
1321 break;
1322 }
1323 case ISD::VECTOR_SHUFFLE: {
1324 assert(!VT.isScalableVector());
1325 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
1326
1327 // Collect demanded elements from shuffle operands..
1328 APInt DemandedLHS, DemandedRHS;
1329 if (!getShuffleDemandedElts(NumElts, ShuffleMask, DemandedElts, DemandedLHS,
1330 DemandedRHS))
1331 break;
1332
1333 if (!!DemandedLHS || !!DemandedRHS) {
1334 SDValue Op0 = Op.getOperand(0);
1335 SDValue Op1 = Op.getOperand(1);
1336
1337 Known.Zero.setAllBits();
1338 Known.One.setAllBits();
1339 if (!!DemandedLHS) {
1340 if (SimplifyDemandedBits(Op0, DemandedBits, DemandedLHS, Known2, TLO,
1341 Depth + 1))
1342 return true;
1343 Known = Known.intersectWith(Known2);
1344 }
1345 if (!!DemandedRHS) {
1346 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedRHS, Known2, TLO,
1347 Depth + 1))
1348 return true;
1349 Known = Known.intersectWith(Known2);
1350 }
1351
1352 // Attempt to avoid multi-use ops if we don't need anything from them.
1353 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1354 Op0, DemandedBits, DemandedLHS, TLO.DAG, Depth + 1);
1355 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1356 Op1, DemandedBits, DemandedRHS, TLO.DAG, Depth + 1);
1357 if (DemandedOp0 || DemandedOp1) {
1358 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1359 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1360 SDValue NewOp = TLO.DAG.getVectorShuffle(VT, dl, Op0, Op1, ShuffleMask);
1361 return TLO.CombineTo(Op, NewOp);
1362 }
1363 }
1364 break;
1365 }
1366 case ISD::AND: {
1367 SDValue Op0 = Op.getOperand(0);
1368 SDValue Op1 = Op.getOperand(1);
1369
1370 // If the RHS is a constant, check to see if the LHS would be zero without
1371 // using the bits from the RHS. Below, we use knowledge about the RHS to
1372 // simplify the LHS, here we're using information from the LHS to simplify
1373 // the RHS.
1374 if (ConstantSDNode *RHSC = isConstOrConstSplat(Op1)) {
1375 // Do not increment Depth here; that can cause an infinite loop.
1376 KnownBits LHSKnown = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth);
1377 // If the LHS already has zeros where RHSC does, this 'and' is dead.
1378 if ((LHSKnown.Zero & DemandedBits) ==
1379 (~RHSC->getAPIntValue() & DemandedBits))
1380 return TLO.CombineTo(Op, Op0);
1381
1382 // If any of the set bits in the RHS are known zero on the LHS, shrink
1383 // the constant.
1384 if (ShrinkDemandedConstant(Op, ~LHSKnown.Zero & DemandedBits,
1385 DemandedElts, TLO))
1386 return true;
1387
1388 // Bitwise-not (xor X, -1) is a special case: we don't usually shrink its
1389 // constant, but if this 'and' is only clearing bits that were just set by
1390 // the xor, then this 'and' can be eliminated by shrinking the mask of
1391 // the xor. For example, for a 32-bit X:
1392 // and (xor (srl X, 31), -1), 1 --> xor (srl X, 31), 1
1393 if (isBitwiseNot(Op0) && Op0.hasOneUse() &&
1394 LHSKnown.One == ~RHSC->getAPIntValue()) {
1395 SDValue Xor = TLO.DAG.getNode(ISD::XOR, dl, VT, Op0.getOperand(0), Op1);
1396 return TLO.CombineTo(Op, Xor);
1397 }
1398 }
1399
1400 // AND(INSERT_SUBVECTOR(C,X,I),M) -> INSERT_SUBVECTOR(AND(C,M),X,I)
1401 // iff 'C' is Undef/Constant and AND(X,M) == X (for DemandedBits).
1402 if (Op0.getOpcode() == ISD::INSERT_SUBVECTOR && !VT.isScalableVector() &&
1403 (Op0.getOperand(0).isUndef() ||
1405 Op0->hasOneUse()) {
1406 unsigned NumSubElts =
1408 unsigned SubIdx = Op0.getConstantOperandVal(2);
1409 APInt DemandedSub =
1410 APInt::getBitsSet(NumElts, SubIdx, SubIdx + NumSubElts);
1411 KnownBits KnownSubMask =
1412 TLO.DAG.computeKnownBits(Op1, DemandedSub & DemandedElts, Depth + 1);
1413 if (DemandedBits.isSubsetOf(KnownSubMask.One)) {
1414 SDValue NewAnd =
1415 TLO.DAG.getNode(ISD::AND, dl, VT, Op0.getOperand(0), Op1);
1416 SDValue NewInsert =
1417 TLO.DAG.getNode(ISD::INSERT_SUBVECTOR, dl, VT, NewAnd,
1418 Op0.getOperand(1), Op0.getOperand(2));
1419 return TLO.CombineTo(Op, NewInsert);
1420 }
1421 }
1422
1423 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1424 Depth + 1))
1425 return true;
1426 assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1427 if (SimplifyDemandedBits(Op0, ~Known.Zero & DemandedBits, DemandedElts,
1428 Known2, TLO, Depth + 1))
1429 return true;
1430 assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
1431
1432 // If all of the demanded bits are known one on one side, return the other.
1433 // These bits cannot contribute to the result of the 'and'.
1434 if (DemandedBits.isSubsetOf(Known2.Zero | Known.One))
1435 return TLO.CombineTo(Op, Op0);
1436 if (DemandedBits.isSubsetOf(Known.Zero | Known2.One))
1437 return TLO.CombineTo(Op, Op1);
1438 // If all of the demanded bits in the inputs are known zeros, return zero.
1439 if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
1440 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, dl, VT));
1441 // If the RHS is a constant, see if we can simplify it.
1442 if (ShrinkDemandedConstant(Op, ~Known2.Zero & DemandedBits, DemandedElts,
1443 TLO))
1444 return true;
1445 // If the operation can be done in a smaller type, do so.
1446 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1447 return true;
1448
1449 // Attempt to avoid multi-use ops if we don't need anything from them.
1450 if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
1451 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1452 Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1453 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1454 Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1455 if (DemandedOp0 || DemandedOp1) {
1456 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1457 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1458 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1459 return TLO.CombineTo(Op, NewOp);
1460 }
1461 }
1462
1463 Known &= Known2;
1464 break;
1465 }
1466 case ISD::OR: {
1467 SDValue Op0 = Op.getOperand(0);
1468 SDValue Op1 = Op.getOperand(1);
1469 SDNodeFlags Flags = Op.getNode()->getFlags();
1470 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1471 Depth + 1)) {
1472 if (Flags.hasDisjoint()) {
1473 Flags.setDisjoint(false);
1474 Op->setFlags(Flags);
1475 }
1476 return true;
1477 }
1478 assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1479 if (SimplifyDemandedBits(Op0, ~Known.One & DemandedBits, DemandedElts,
1480 Known2, TLO, Depth + 1)) {
1481 if (Flags.hasDisjoint()) {
1482 Flags.setDisjoint(false);
1483 Op->setFlags(Flags);
1484 }
1485 return true;
1486 }
1487 assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
1488
1489 // If all of the demanded bits are known zero on one side, return the other.
1490 // These bits cannot contribute to the result of the 'or'.
1491 if (DemandedBits.isSubsetOf(Known2.One | Known.Zero))
1492 return TLO.CombineTo(Op, Op0);
1493 if (DemandedBits.isSubsetOf(Known.One | Known2.Zero))
1494 return TLO.CombineTo(Op, Op1);
1495 // If the RHS is a constant, see if we can simplify it.
1496 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1497 return true;
1498 // If the operation can be done in a smaller type, do so.
1499 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1500 return true;
1501
1502 // Attempt to avoid multi-use ops if we don't need anything from them.
1503 if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
1504 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1505 Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1506 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1507 Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1508 if (DemandedOp0 || DemandedOp1) {
1509 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1510 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1511 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1512 return TLO.CombineTo(Op, NewOp);
1513 }
1514 }
1515
1516 // (or (and X, C1), (and (or X, Y), C2)) -> (or (and X, C1|C2), (and Y, C2))
1517 // TODO: Use SimplifyMultipleUseDemandedBits to peek through masks.
1518 if (Op0.getOpcode() == ISD::AND && Op1.getOpcode() == ISD::AND &&
1519 Op0->hasOneUse() && Op1->hasOneUse()) {
1520 // Attempt to match all commutations - m_c_Or would've been useful!
1521 for (int I = 0; I != 2; ++I) {
1522 SDValue X = Op.getOperand(I).getOperand(0);
1523 SDValue C1 = Op.getOperand(I).getOperand(1);
1524 SDValue Alt = Op.getOperand(1 - I).getOperand(0);
1525 SDValue C2 = Op.getOperand(1 - I).getOperand(1);
1526 if (Alt.getOpcode() == ISD::OR) {
1527 for (int J = 0; J != 2; ++J) {
1528 if (X == Alt.getOperand(J)) {
1529 SDValue Y = Alt.getOperand(1 - J);
1530 if (SDValue C12 = TLO.DAG.FoldConstantArithmetic(ISD::OR, dl, VT,
1531 {C1, C2})) {
1532 SDValue MaskX = TLO.DAG.getNode(ISD::AND, dl, VT, X, C12);
1533 SDValue MaskY = TLO.DAG.getNode(ISD::AND, dl, VT, Y, C2);
1534 return TLO.CombineTo(
1535 Op, TLO.DAG.getNode(ISD::OR, dl, VT, MaskX, MaskY));
1536 }
1537 }
1538 }
1539 }
1540 }
1541 }
1542
1543 Known |= Known2;
1544 break;
1545 }
1546 case ISD::XOR: {
1547 SDValue Op0 = Op.getOperand(0);
1548 SDValue Op1 = Op.getOperand(1);
1549
1550 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1551 Depth + 1))
1552 return true;
1553 assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1554 if (SimplifyDemandedBits(Op0, DemandedBits, DemandedElts, Known2, TLO,
1555 Depth + 1))
1556 return true;
1557 assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
1558
1559 // If all of the demanded bits are known zero on one side, return the other.
1560 // These bits cannot contribute to the result of the 'xor'.
1561 if (DemandedBits.isSubsetOf(Known.Zero))
1562 return TLO.CombineTo(Op, Op0);
1563 if (DemandedBits.isSubsetOf(Known2.Zero))
1564 return TLO.CombineTo(Op, Op1);
1565 // If the operation can be done in a smaller type, do so.
1566 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1567 return true;
1568
1569 // If all of the unknown bits are known to be zero on one side or the other
1570 // turn this into an *inclusive* or.
1571 // e.g. (A & C1)^(B & C2) -> (A & C1)|(B & C2) iff C1&C2 == 0
1572 if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
1573 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::OR, dl, VT, Op0, Op1));
1574
1575 ConstantSDNode *C = isConstOrConstSplat(Op1, DemandedElts);
1576 if (C) {
1577 // If one side is a constant, and all of the set bits in the constant are
1578 // also known set on the other side, turn this into an AND, as we know
1579 // the bits will be cleared.
1580 // e.g. (X | C1) ^ C2 --> (X | C1) & ~C2 iff (C1&C2) == C2
1581 // NB: it is okay if more bits are known than are requested
1582 if (C->getAPIntValue() == Known2.One) {
1583 SDValue ANDC =
1584 TLO.DAG.getConstant(~C->getAPIntValue() & DemandedBits, dl, VT);
1585 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, dl, VT, Op0, ANDC));
1586 }
1587
1588 // If the RHS is a constant, see if we can change it. Don't alter a -1
1589 // constant because that's a 'not' op, and that is better for combining
1590 // and codegen.
1591 if (!C->isAllOnes() && DemandedBits.isSubsetOf(C->getAPIntValue())) {
1592 // We're flipping all demanded bits. Flip the undemanded bits too.
1593 SDValue New = TLO.DAG.getNOT(dl, Op0, VT);
1594 return TLO.CombineTo(Op, New);
1595 }
1596
1597 unsigned Op0Opcode = Op0.getOpcode();
1598 if ((Op0Opcode == ISD::SRL || Op0Opcode == ISD::SHL) && Op0.hasOneUse()) {
1599 if (ConstantSDNode *ShiftC =
1600 isConstOrConstSplat(Op0.getOperand(1), DemandedElts)) {
1601 // Don't crash on an oversized shift. We can not guarantee that a
1602 // bogus shift has been simplified to undef.
1603 if (ShiftC->getAPIntValue().ult(BitWidth)) {
1604 uint64_t ShiftAmt = ShiftC->getZExtValue();
1606 Ones = Op0Opcode == ISD::SHL ? Ones.shl(ShiftAmt)
1607 : Ones.lshr(ShiftAmt);
1608 const TargetLowering &TLI = TLO.DAG.getTargetLoweringInfo();
1609 if ((DemandedBits & C->getAPIntValue()) == (DemandedBits & Ones) &&
1610 TLI.isDesirableToCommuteXorWithShift(Op.getNode())) {
1611 // If the xor constant is a demanded mask, do a 'not' before the
1612 // shift:
1613 // xor (X << ShiftC), XorC --> (not X) << ShiftC
1614 // xor (X >> ShiftC), XorC --> (not X) >> ShiftC
1615 SDValue Not = TLO.DAG.getNOT(dl, Op0.getOperand(0), VT);
1616 return TLO.CombineTo(Op, TLO.DAG.getNode(Op0Opcode, dl, VT, Not,
1617 Op0.getOperand(1)));
1618 }
1619 }
1620 }
1621 }
1622 }
1623
1624 // If we can't turn this into a 'not', try to shrink the constant.
1625 if (!C || !C->isAllOnes())
1626 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1627 return true;
1628
1629 // Attempt to avoid multi-use ops if we don't need anything from them.
1630 if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
1631 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1632 Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1633 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1634 Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1635 if (DemandedOp0 || DemandedOp1) {
1636 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1637 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1638 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1639 return TLO.CombineTo(Op, NewOp);
1640 }
1641 }
1642
1643 Known ^= Known2;
1644 break;
1645 }
1646 case ISD::SELECT:
1647 if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, DemandedElts,
1648 Known, TLO, Depth + 1))
1649 return true;
1650 if (SimplifyDemandedBits(Op.getOperand(1), DemandedBits, DemandedElts,
1651 Known2, TLO, Depth + 1))
1652 return true;
1653 assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1654 assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
1655
1656 // If the operands are constants, see if we can simplify them.
1657 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1658 return true;
1659
1660 // Only known if known in both the LHS and RHS.
1661 Known = Known.intersectWith(Known2);
1662 break;
1663 case ISD::VSELECT:
1664 if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, DemandedElts,
1665 Known, TLO, Depth + 1))
1666 return true;
1667 if (SimplifyDemandedBits(Op.getOperand(1), DemandedBits, DemandedElts,
1668 Known2, TLO, Depth + 1))
1669 return true;
1670 assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1671 assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
1672
1673 // Only known if known in both the LHS and RHS.
1674 Known = Known.intersectWith(Known2);
1675 break;
1676 case ISD::SELECT_CC:
1677 if (SimplifyDemandedBits(Op.getOperand(3), DemandedBits, DemandedElts,
1678 Known, TLO, Depth + 1))
1679 return true;
1680 if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, DemandedElts,
1681 Known2, TLO, Depth + 1))
1682 return true;
1683 assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1684 assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
1685
1686 // If the operands are constants, see if we can simplify them.
1687 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1688 return true;
1689
1690 // Only known if known in both the LHS and RHS.
1691 Known = Known.intersectWith(Known2);
1692 break;
1693 case ISD::SETCC: {
1694 SDValue Op0 = Op.getOperand(0);
1695 SDValue Op1 = Op.getOperand(1);
1696 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
1697 // If (1) we only need the sign-bit, (2) the setcc operands are the same
1698 // width as the setcc result, and (3) the result of a setcc conforms to 0 or
1699 // -1, we may be able to bypass the setcc.
1700 if (DemandedBits.isSignMask() &&
1704 // If we're testing X < 0, then this compare isn't needed - just use X!
1705 // FIXME: We're limiting to integer types here, but this should also work
1706 // if we don't care about FP signed-zero. The use of SETLT with FP means
1707 // that we don't care about NaNs.
1708 if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
1710 return TLO.CombineTo(Op, Op0);
1711
1712 // TODO: Should we check for other forms of sign-bit comparisons?
1713 // Examples: X <= -1, X >= 0
1714 }
1715 if (getBooleanContents(Op0.getValueType()) ==
1717 BitWidth > 1)
1718 Known.Zero.setBitsFrom(1);
1719 break;
1720 }
1721 case ISD::SHL: {
1722 SDValue Op0 = Op.getOperand(0);
1723 SDValue Op1 = Op.getOperand(1);
1724 EVT ShiftVT = Op1.getValueType();
1725
1726 if (const APInt *SA =
1727 TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
1728 unsigned ShAmt = SA->getZExtValue();
1729 if (ShAmt == 0)
1730 return TLO.CombineTo(Op, Op0);
1731
1732 // If this is ((X >>u C1) << ShAmt), see if we can simplify this into a
1733 // single shift. We can do this if the bottom bits (which are shifted
1734 // out) are never demanded.
1735 // TODO - support non-uniform vector amounts.
1736 if (Op0.getOpcode() == ISD::SRL) {
1737 if (!DemandedBits.intersects(APInt::getLowBitsSet(BitWidth, ShAmt))) {
1738 if (const APInt *SA2 =
1739 TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {
1740 unsigned C1 = SA2->getZExtValue();
1741 unsigned Opc = ISD::SHL;
1742 int Diff = ShAmt - C1;
1743 if (Diff < 0) {
1744 Diff = -Diff;
1745 Opc = ISD::SRL;
1746 }
1747 SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);
1748 return TLO.CombineTo(
1749 Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));
1750 }
1751 }
1752 }
1753
1754 // Convert (shl (anyext x, c)) to (anyext (shl x, c)) if the high bits
1755 // are not demanded. This will likely allow the anyext to be folded away.
1756 // TODO - support non-uniform vector amounts.
1757 if (Op0.getOpcode() == ISD::ANY_EXTEND) {
1758 SDValue InnerOp = Op0.getOperand(0);
1759 EVT InnerVT = InnerOp.getValueType();
1760 unsigned InnerBits = InnerVT.getScalarSizeInBits();
1761 if (ShAmt < InnerBits && DemandedBits.getActiveBits() <= InnerBits &&
1762 isTypeDesirableForOp(ISD::SHL, InnerVT)) {
1763 SDValue NarrowShl = TLO.DAG.getNode(
1764 ISD::SHL, dl, InnerVT, InnerOp,
1765 TLO.DAG.getShiftAmountConstant(ShAmt, InnerVT, dl));
1766 return TLO.CombineTo(
1767 Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, NarrowShl));
1768 }
1769
1770 // Repeat the SHL optimization above in cases where an extension
1771 // intervenes: (shl (anyext (shr x, c1)), c2) to
1772 // (shl (anyext x), c2-c1). This requires that the bottom c1 bits
1773 // aren't demanded (as above) and that the shifted upper c1 bits of
1774 // x aren't demanded.
1775 // TODO - support non-uniform vector amounts.
1776 if (InnerOp.getOpcode() == ISD::SRL && Op0.hasOneUse() &&
1777 InnerOp.hasOneUse()) {
1778 if (const APInt *SA2 =
1779 TLO.DAG.getValidShiftAmountConstant(InnerOp, DemandedElts)) {
1780 unsigned InnerShAmt = SA2->getZExtValue();
1781 if (InnerShAmt < ShAmt && InnerShAmt < InnerBits &&
1782 DemandedBits.getActiveBits() <=
1783 (InnerBits - InnerShAmt + ShAmt) &&
1784 DemandedBits.countr_zero() >= ShAmt) {
1785 SDValue NewSA =
1786 TLO.DAG.getConstant(ShAmt - InnerShAmt, dl, ShiftVT);
1787 SDValue NewExt = TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT,
1788 InnerOp.getOperand(0));
1789 return TLO.CombineTo(
1790 Op, TLO.DAG.getNode(ISD::SHL, dl, VT, NewExt, NewSA));
1791 }
1792 }
1793 }
1794 }
1795
1796 APInt InDemandedMask = DemandedBits.lshr(ShAmt);
1797 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1798 Depth + 1)) {
1799 SDNodeFlags Flags = Op.getNode()->getFlags();
1800 if (Flags.hasNoSignedWrap() || Flags.hasNoUnsignedWrap()) {
1801 // Disable the nsw and nuw flags. We can no longer guarantee that we
1802 // won't wrap after simplification.
1803 Flags.setNoSignedWrap(false);
1804 Flags.setNoUnsignedWrap(false);
1805 Op->setFlags(Flags);
1806 }
1807 return true;
1808 }
1809 assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1810 Known.Zero <<= ShAmt;
1811 Known.One <<= ShAmt;
1812 // low bits known zero.
1813 Known.Zero.setLowBits(ShAmt);
1814
1815 // Attempt to avoid multi-use ops if we don't need anything from them.
1816 if (!InDemandedMask.isAllOnes() || !DemandedElts.isAllOnes()) {
1817 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1818 Op0, InDemandedMask, DemandedElts, TLO.DAG, Depth + 1);
1819 if (DemandedOp0) {
1820 SDValue NewOp = TLO.DAG.getNode(ISD::SHL, dl, VT, DemandedOp0, Op1);
1821 return TLO.CombineTo(Op, NewOp);
1822 }
1823 }
1824
1825 // Try shrinking the operation as long as the shift amount will still be
1826 // in range.
1827 if ((ShAmt < DemandedBits.getActiveBits()) &&
1828 ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1829 return true;
1830
1831 // Narrow shift to lower half - similar to ShrinkDemandedOp.
1832 // (shl i64:x, K) -> (i64 zero_extend (shl (i32 (trunc i64:x)), K))
1833 // Only do this if we demand the upper half so the knownbits are correct.
1834 unsigned HalfWidth = BitWidth / 2;
1835 if ((BitWidth % 2) == 0 && !VT.isVector() && ShAmt < HalfWidth &&
1836 DemandedBits.countLeadingOnes() >= HalfWidth) {
1837 EVT HalfVT = EVT::getIntegerVT(*TLO.DAG.getContext(), HalfWidth);
1838 if (isNarrowingProfitable(VT, HalfVT) &&
1839 isTypeDesirableForOp(ISD::SHL, HalfVT) &&
1840 isTruncateFree(VT, HalfVT) && isZExtFree(HalfVT, VT) &&
1841 (!TLO.LegalOperations() || isOperationLegal(ISD::SHL, HalfVT))) {
1842 // If we're demanding the upper bits at all, we must ensure
1843 // that the upper bits of the shift result are known to be zero,
1844 // which is equivalent to the narrow shift being NUW.
1845 if (bool IsNUW = (Known.countMinLeadingZeros() >= HalfWidth)) {
1846 bool IsNSW = Known.countMinSignBits() > HalfWidth;
1847 SDNodeFlags Flags;
1848 Flags.setNoSignedWrap(IsNSW);
1849 Flags.setNoUnsignedWrap(IsNUW);
1850 SDValue NewOp = TLO.DAG.getNode(ISD::TRUNCATE, dl, HalfVT, Op0);
1851 SDValue NewShiftAmt = TLO.DAG.getShiftAmountConstant(
1852 ShAmt, HalfVT, dl, TLO.LegalTypes());
1853 SDValue NewShift = TLO.DAG.getNode(ISD::SHL, dl, HalfVT, NewOp,
1854 NewShiftAmt, Flags);
1855 SDValue NewExt =
1856 TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, NewShift);
1857 return TLO.CombineTo(Op, NewExt);
1858 }
1859 }
1860 }
1861 } else {
1862 // This is a variable shift, so we can't shift the demand mask by a known
1863 // amount. But if we are not demanding high bits, then we are not
1864 // demanding those bits from the pre-shifted operand either.
1865 if (unsigned CTLZ = DemandedBits.countl_zero()) {
1866 APInt DemandedFromOp(APInt::getLowBitsSet(BitWidth, BitWidth - CTLZ));
1867 if (SimplifyDemandedBits(Op0, DemandedFromOp, DemandedElts, Known, TLO,
1868 Depth + 1)) {
1869 SDNodeFlags Flags = Op.getNode()->getFlags();
1870 if (Flags.hasNoSignedWrap() || Flags.hasNoUnsignedWrap()) {
1871 // Disable the nsw and nuw flags. We can no longer guarantee that we
1872 // won't wrap after simplification.
1873 Flags.setNoSignedWrap(false);
1874 Flags.setNoUnsignedWrap(false);
1875 Op->setFlags(Flags);
1876 }
1877 return true;
1878 }
1879 Known.resetAll();
1880 }
1881 }
1882
1883 // If we are only demanding sign bits then we can use the shift source
1884 // directly.
1885 if (const APInt *MaxSA =
1886 TLO.DAG.getValidMaximumShiftAmountConstant(Op, DemandedElts)) {
1887 unsigned ShAmt = MaxSA->getZExtValue();
1888 unsigned NumSignBits =
1889 TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
1890 unsigned UpperDemandedBits = BitWidth - DemandedBits.countr_zero();
1891 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
1892 return TLO.CombineTo(Op, Op0);
1893 }
1894 break;
1895 }
1896 case ISD::SRL: {
1897 SDValue Op0 = Op.getOperand(0);
1898 SDValue Op1 = Op.getOperand(1);
1899 EVT ShiftVT = Op1.getValueType();
1900
1901 // Try to match AVG patterns.
1902 if (SDValue AVG = combineShiftToAVG(Op, TLO.DAG, *this, DemandedBits,
1903 DemandedElts, Depth + 1))
1904 return TLO.CombineTo(Op, AVG);
1905
1906 if (const APInt *SA =
1907 TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
1908 unsigned ShAmt = SA->getZExtValue();
1909 if (ShAmt == 0)
1910 return TLO.CombineTo(Op, Op0);
1911
1912 // If this is ((X << C1) >>u ShAmt), see if we can simplify this into a
1913 // single shift. We can do this if the top bits (which are shifted out)
1914 // are never demanded.
1915 // TODO - support non-uniform vector amounts.
1916 if (Op0.getOpcode() == ISD::SHL) {
1917 if (!DemandedBits.intersects(APInt::getHighBitsSet(BitWidth, ShAmt))) {
1918 if (const APInt *SA2 =
1919 TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {
1920 unsigned C1 = SA2->getZExtValue();
1921 unsigned Opc = ISD::SRL;
1922 int Diff = ShAmt - C1;
1923 if (Diff < 0) {
1924 Diff = -Diff;
1925 Opc = ISD::SHL;
1926 }
1927 SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);
1928 return TLO.CombineTo(
1929 Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));
1930 }
1931 }
1932 }
1933
1934 APInt InDemandedMask = (DemandedBits << ShAmt);
1935
1936 // If the shift is exact, then it does demand the low bits (and knows that
1937 // they are zero).
1938 if (Op->getFlags().hasExact())
1939 InDemandedMask.setLowBits(ShAmt);
1940
1941 // Narrow shift to lower half - similar to ShrinkDemandedOp.
1942 // (srl i64:x, K) -> (i64 zero_extend (srl (i32 (trunc i64:x)), K))
1943 if ((BitWidth % 2) == 0 && !VT.isVector()) {
1945 EVT HalfVT = EVT::getIntegerVT(*TLO.DAG.getContext(), BitWidth / 2);
1946 if (isNarrowingProfitable(VT, HalfVT) &&
1947 isTypeDesirableForOp(ISD::SRL, HalfVT) &&
1948 isTruncateFree(VT, HalfVT) && isZExtFree(HalfVT, VT) &&
1949 (!TLO.LegalOperations() || isOperationLegal(ISD::SRL, HalfVT)) &&
1950 ((InDemandedMask.countLeadingZeros() >= (BitWidth / 2)) ||
1951 TLO.DAG.MaskedValueIsZero(Op0, HiBits))) {
1952 SDValue NewOp = TLO.DAG.getNode(ISD::TRUNCATE, dl, HalfVT, Op0);
1953 SDValue NewShiftAmt = TLO.DAG.getShiftAmountConstant(
1954 ShAmt, HalfVT, dl, TLO.LegalTypes());
1955 SDValue NewShift =
1956 TLO.DAG.getNode(ISD::SRL, dl, HalfVT, NewOp, NewShiftAmt);
1957 return TLO.CombineTo(
1958 Op, TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, NewShift));
1959 }
1960 }
1961
1962 // Compute the new bits that are at the top now.
1963 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1964 Depth + 1))
1965 return true;
1966 assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1967 Known.Zero.lshrInPlace(ShAmt);
1968 Known.One.lshrInPlace(ShAmt);
1969 // High bits known zero.
1970 Known.Zero.setHighBits(ShAmt);
1971
1972 // Attempt to avoid multi-use ops if we don't need anything from them.
1973 if (!InDemandedMask.isAllOnes() || !DemandedElts.isAllOnes()) {
1974 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1975 Op0, InDemandedMask, DemandedElts, TLO.DAG, Depth + 1);
1976 if (DemandedOp0) {
1977 SDValue NewOp = TLO.DAG.getNode(ISD::SRL, dl, VT, DemandedOp0, Op1);
1978 return TLO.CombineTo(Op, NewOp);
1979 }
1980 }
1981 } else {
1982 // Use generic knownbits computation as it has support for non-uniform
1983 // shift amounts.
1984 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
1985 }
1986 break;
1987 }
1988 case ISD::SRA: {
1989 SDValue Op0 = Op.getOperand(0);
1990 SDValue Op1 = Op.getOperand(1);
1991 EVT ShiftVT = Op1.getValueType();
1992
1993 // If we only want bits that already match the signbit then we don't need
1994 // to shift.
1995 unsigned NumHiDemandedBits = BitWidth - DemandedBits.countr_zero();
1996 if (TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1) >=
1997 NumHiDemandedBits)
1998 return TLO.CombineTo(Op, Op0);
1999
2000 // If this is an arithmetic shift right and only the low-bit is set, we can
2001 // always convert this into a logical shr, even if the shift amount is
2002 // variable. The low bit of the shift cannot be an input sign bit unless
2003 // the shift amount is >= the size of the datatype, which is undefined.
2004 if (DemandedBits.isOne())
2005 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1));
2006
2007 // Try to match AVG patterns.
2008 if (SDValue AVG = combineShiftToAVG(Op, TLO.DAG, *this, DemandedBits,
2009 DemandedElts, Depth + 1))
2010 return TLO.CombineTo(Op, AVG);
2011
2012 if (const APInt *SA =
2013 TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
2014 unsigned ShAmt = SA->getZExtValue();
2015 if (ShAmt == 0)
2016 return TLO.CombineTo(Op, Op0);
2017
2018 // fold (sra (shl x, c1), c1) -> sext_inreg for some c1 and target
2019 // supports sext_inreg.
2020 if (Op0.getOpcode() == ISD::SHL) {
2021 if (const APInt *InnerSA =
2022 TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {
2023 unsigned LowBits = BitWidth - ShAmt;
2024 EVT ExtVT = EVT::getIntegerVT(*TLO.DAG.getContext(), LowBits);
2025 if (VT.isVector())
2026 ExtVT = EVT::getVectorVT(*TLO.DAG.getContext(), ExtVT,
2028
2029 if (*InnerSA == ShAmt) {
2030 if (!TLO.LegalOperations() ||
2032 return TLO.CombineTo(
2033 Op, TLO.DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, VT,
2034 Op0.getOperand(0),
2035 TLO.DAG.getValueType(ExtVT)));
2036
2037 // Even if we can't convert to sext_inreg, we might be able to
2038 // remove this shift pair if the input is already sign extended.
2039 unsigned NumSignBits =
2040 TLO.DAG.ComputeNumSignBits(Op0.getOperand(0), DemandedElts);
2041 if (NumSignBits > ShAmt)
2042 return TLO.CombineTo(Op, Op0.getOperand(0));
2043 }
2044 }
2045 }
2046
2047 APInt InDemandedMask = (DemandedBits << ShAmt);
2048
2049 // If the shift is exact, then it does demand the low bits (and knows that
2050 // they are zero).
2051 if (Op->getFlags().hasExact())
2052 InDemandedMask.setLowBits(ShAmt);
2053
2054 // If any of the demanded bits are produced by the sign extension, we also
2055 // demand the input sign bit.
2056 if (DemandedBits.countl_zero() < ShAmt)
2057 InDemandedMask.setSignBit();
2058
2059 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
2060 Depth + 1))
2061 return true;
2062 assert(!Known.hasConflict() && "Bits known to be one AND zero?");
2063 Known.Zero.lshrInPlace(ShAmt);
2064 Known.One.lshrInPlace(ShAmt);
2065
2066 // If the input sign bit is known to be zero, or if none of the top bits
2067 // are demanded, turn this into an unsigned shift right.
2068 if (Known.Zero[BitWidth - ShAmt - 1] ||
2069 DemandedBits.countl_zero() >= ShAmt) {
2070 SDNodeFlags Flags;
2071 Flags.setExact(Op->getFlags().hasExact());
2072 return TLO.CombineTo(
2073 Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1, Flags));
2074 }
2075
2076 int Log2 = DemandedBits.exactLogBase2();
2077 if (Log2 >= 0) {
2078 // The bit must come from the sign.
2079 SDValue NewSA = TLO.DAG.getConstant(BitWidth - 1 - Log2, dl, ShiftVT);
2080 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, NewSA));
2081 }
2082
2083 if (Known.One[BitWidth - ShAmt - 1])
2084 // New bits are known one.
2085 Known.One.setHighBits(ShAmt);
2086
2087 // Attempt to avoid multi-use ops if we don't need anything from them.
2088 if (!InDemandedMask.isAllOnes() || !DemandedElts.isAllOnes()) {
2089 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2090 Op0, InDemandedMask, DemandedElts, TLO.DAG, Depth + 1);
2091 if (DemandedOp0) {
2092 SDValue NewOp = TLO.DAG.getNode(ISD::SRA, dl, VT, DemandedOp0, Op1);
2093 return TLO.CombineTo(Op, NewOp);
2094 }
2095 }
2096 }
2097 break;
2098 }
2099 case ISD::FSHL:
2100 case ISD::FSHR: {
2101 SDValue Op0 = Op.getOperand(0);
2102 SDValue Op1 = Op.getOperand(1);
2103 SDValue Op2 = Op.getOperand(2);
2104 bool IsFSHL = (Op.getOpcode() == ISD::FSHL);
2105
2106 if (ConstantSDNode *SA = isConstOrConstSplat(Op2, DemandedElts)) {
2107 unsigned Amt = SA->getAPIntValue().urem(BitWidth);
2108
2109 // For fshl, 0-shift returns the 1st arg.
2110 // For fshr, 0-shift returns the 2nd arg.
2111 if (Amt == 0) {
2112 if (SimplifyDemandedBits(IsFSHL ? Op0 : Op1, DemandedBits, DemandedElts,
2113 Known, TLO, Depth + 1))
2114 return true;
2115 break;
2116 }
2117
2118 // fshl: (Op0 << Amt) | (Op1 >> (BW - Amt))
2119 // fshr: (Op0 << (BW - Amt)) | (Op1 >> Amt)
2120 APInt Demanded0 = DemandedBits.lshr(IsFSHL ? Amt : (BitWidth - Amt));
2121 APInt Demanded1 = DemandedBits << (IsFSHL ? (BitWidth - Amt) : Amt);
2122 if (SimplifyDemandedBits(Op0, Demanded0, DemandedElts, Known2, TLO,
2123 Depth + 1))
2124 return true;
2125 if (SimplifyDemandedBits(Op1, Demanded1, DemandedElts, Known, TLO,
2126 Depth + 1))
2127 return true;
2128
2129 Known2.One <<= (IsFSHL ? Amt : (BitWidth - Amt));
2130 Known2.Zero <<= (IsFSHL ? Amt : (BitWidth - Amt));
2131 Known.One.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);
2132 Known.Zero.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);
2133 Known = Known.unionWith(Known2);
2134
2135 // Attempt to avoid multi-use ops if we don't need anything from them.
2136 if (!Demanded0.isAllOnes() || !Demanded1.isAllOnes() ||
2137 !DemandedElts.isAllOnes()) {
2138 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2139 Op0, Demanded0, DemandedElts, TLO.DAG, Depth + 1);
2140 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
2141 Op1, Demanded1, DemandedElts, TLO.DAG, Depth + 1);
2142 if (DemandedOp0 || DemandedOp1) {
2143 DemandedOp0 = DemandedOp0 ? DemandedOp0 : Op0;
2144 DemandedOp1 = DemandedOp1 ? DemandedOp1 : Op1;
2145 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedOp0,
2146 DemandedOp1, Op2);
2147 return TLO.CombineTo(Op, NewOp);
2148 }
2149 }
2150 }
2151
2152 // For pow-2 bitwidths we only demand the bottom modulo amt bits.
2153 if (isPowerOf2_32(BitWidth)) {
2154 APInt DemandedAmtBits(Op2.getScalarValueSizeInBits(), BitWidth - 1);
2155 if (SimplifyDemandedBits(Op2, DemandedAmtBits, DemandedElts,
2156 Known2, TLO, Depth + 1))
2157 return true;
2158 }
2159 break;
2160 }
2161 case ISD::ROTL:
2162 case ISD::ROTR: {
2163 SDValue Op0 = Op.getOperand(0);
2164 SDValue Op1 = Op.getOperand(1);
2165 bool IsROTL = (Op.getOpcode() == ISD::ROTL);
2166
2167 // If we're rotating an 0/-1 value, then it stays an 0/-1 value.
2168 if (BitWidth == TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1))
2169 return TLO.CombineTo(Op, Op0);
2170
2171 if (ConstantSDNode *SA = isConstOrConstSplat(Op1, DemandedElts)) {
2172 unsigned Amt = SA->getAPIntValue().urem(BitWidth);
2173 unsigned RevAmt = BitWidth - Amt;
2174
2175 // rotl: (Op0 << Amt) | (Op0 >> (BW - Amt))
2176 // rotr: (Op0 << (BW - Amt)) | (Op0 >> Amt)
2177 APInt Demanded0 = DemandedBits.rotr(IsROTL ? Amt : RevAmt);
2178 if (SimplifyDemandedBits(Op0, Demanded0, DemandedElts, Known2, TLO,
2179 Depth + 1))
2180 return true;
2181
2182 // rot*(x, 0) --> x
2183 if (Amt == 0)
2184 return TLO.CombineTo(Op, Op0);
2185
2186 // See if we don't demand either half of the rotated bits.
2187 if ((!TLO.LegalOperations() || isOperationLegal(ISD::SHL, VT)) &&
2188 DemandedBits.countr_zero() >= (IsROTL ? Amt : RevAmt)) {
2189 Op1 = TLO.DAG.getConstant(IsROTL ? Amt : RevAmt, dl, Op1.getValueType());
2190 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, Op1));
2191 }
2192 if ((!TLO.LegalOperations() || isOperationLegal(ISD::SRL, VT)) &&
2193 DemandedBits.countl_zero() >= (IsROTL ? RevAmt : Amt)) {
2194 Op1 = TLO.DAG.getConstant(IsROTL ? RevAmt : Amt, dl, Op1.getValueType());
2195 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1));
2196 }
2197 }
2198
2199 // For pow-2 bitwidths we only demand the bottom modulo amt bits.
2200 if (isPowerOf2_32(BitWidth)) {
2201 APInt DemandedAmtBits(Op1.getScalarValueSizeInBits(), BitWidth - 1);
2202 if (SimplifyDemandedBits(Op1, DemandedAmtBits, DemandedElts, Known2, TLO,
2203 Depth + 1))
2204 return true;
2205 }
2206 break;
2207 }
2208 case ISD::SMIN:
2209 case ISD::SMAX:
2210 case ISD::UMIN:
2211 case ISD::UMAX: {
2212 unsigned Opc = Op.getOpcode();
2213 SDValue Op0 = Op.getOperand(0);
2214 SDValue Op1 = Op.getOperand(1);
2215
2216 // If we're only demanding signbits, then we can simplify to OR/AND node.
2217 unsigned BitOp =
2218 (Opc == ISD::SMIN || Opc == ISD::UMAX) ? ISD::OR : ISD::AND;
2219 unsigned NumSignBits =
2220 std::min(TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1),
2221 TLO.DAG.ComputeNumSignBits(Op1, DemandedElts, Depth + 1));
2222 unsigned NumDemandedUpperBits = BitWidth - DemandedBits.countr_zero();
2223 if (NumSignBits >= NumDemandedUpperBits)
2224 return TLO.CombineTo(Op, TLO.DAG.getNode(BitOp, SDLoc(Op), VT, Op0, Op1));
2225
2226 // Check if one arg is always less/greater than (or equal) to the other arg.
2227 KnownBits Known0 = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth + 1);
2228 KnownBits Known1 = TLO.DAG.computeKnownBits(Op1, DemandedElts, Depth + 1);
2229 switch (Opc) {
2230 case ISD::SMIN:
2231 if (std::optional<bool> IsSLE = KnownBits::sle(Known0, Known1))
2232 return TLO.CombineTo(Op, *IsSLE ? Op0 : Op1);
2233 if (std::optional<bool> IsSLT = KnownBits::slt(Known0, Known1))
2234 return TLO.CombineTo(Op, *IsSLT ? Op0 : Op1);
2235 Known = KnownBits::smin(Known0, Known1);
2236 break;
2237 case ISD::SMAX:
2238 if (std::optional<bool> IsSGE = KnownBits::sge(Known0, Known1))
2239 return TLO.CombineTo(Op, *IsSGE ? Op0 : Op1);
2240 if (std::optional<bool> IsSGT = KnownBits::sgt(Known0, Known1))
2241 return TLO.CombineTo(Op, *IsSGT ? Op0 : Op1);
2242 Known = KnownBits::smax(Known0, Known1);
2243 break;
2244 case ISD::UMIN:
2245 if (std::optional<bool> IsULE = KnownBits::ule(Known0, Known1))
2246 return TLO.CombineTo(Op, *IsULE ? Op0 : Op1);
2247 if (std::optional<bool> IsULT = KnownBits::ult(Known0, Known1))
2248 return TLO.CombineTo(Op, *IsULT ? Op0 : Op1);
2249 Known = KnownBits::umin(Known0, Known1);
2250 break;
2251 case ISD::UMAX:
2252 if (std::optional<bool> IsUGE = KnownBits::uge(Known0, Known1))
2253 return TLO.CombineTo(Op, *IsUGE ? Op0 : Op1);
2254 if (std::optional<bool> IsUGT = KnownBits::ugt(Known0, Known1))
2255 return TLO.CombineTo(Op, *IsUGT ? Op0 : Op1);
2256 Known = KnownBits::umax(Known0, Known1);
2257 break;
2258 }
2259 break;
2260 }
2261 case ISD::BITREVERSE: {
2262 SDValue Src = Op.getOperand(0);
2263 APInt DemandedSrcBits = DemandedBits.reverseBits();
2264 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
2265 Depth + 1))
2266 return true;
2267 Known.One = Known2.One.reverseBits();
2268 Known.Zero = Known2.Zero.reverseBits();
2269 break;
2270 }
2271 case ISD::BSWAP: {
2272 SDValue Src = Op.getOperand(0);
2273
2274 // If the only bits demanded come from one byte of the bswap result,
2275 // just shift the input byte into position to eliminate the bswap.
2276 unsigned NLZ = DemandedBits.countl_zero();
2277 unsigned NTZ = DemandedBits.countr_zero();
2278
2279 // Round NTZ down to the next byte. If we have 11 trailing zeros, then
2280 // we need all the bits down to bit 8. Likewise, round NLZ. If we
2281 // have 14 leading zeros, round to 8.
2282 NLZ = alignDown(NLZ, 8);
2283 NTZ = alignDown(NTZ, 8);
2284 // If we need exactly one byte, we can do this transformation.
2285 if (BitWidth - NLZ - NTZ == 8) {
2286 // Replace this with either a left or right shift to get the byte into
2287 // the right place.
2288 unsigned ShiftOpcode = NLZ > NTZ ? ISD::SRL : ISD::SHL;
2289 if (!TLO.LegalOperations() || isOperationLegal(ShiftOpcode, VT)) {
2290 unsigned ShiftAmount = NLZ > NTZ ? NLZ - NTZ : NTZ - NLZ;
2291 SDValue ShAmt = TLO.DAG.getShiftAmountConstant(ShiftAmount, VT, dl);
2292 SDValue NewOp = TLO.DAG.getNode(ShiftOpcode, dl, VT, Src, ShAmt);
2293 return TLO.CombineTo(Op, NewOp);
2294 }
2295 }
2296
2297 APInt DemandedSrcBits = DemandedBits.byteSwap();
2298 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
2299 Depth + 1))
2300 return true;
2301 Known.One = Known2.One.byteSwap();
2302 Known.Zero = Known2.Zero.byteSwap();
2303 break;
2304 }
2305 case ISD::CTPOP: {
2306 // If only 1 bit is demanded, replace with PARITY as long as we're before
2307 // op legalization.
2308 // FIXME: Limit to scalars for now.
2309 if (DemandedBits.isOne() && !TLO.LegalOps && !VT.isVector())
2310 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::PARITY, dl, VT,
2311 Op.getOperand(0)));
2312
2313 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2314 break;
2315 }
2317 SDValue Op0 = Op.getOperand(0);
2318 EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2319 unsigned ExVTBits = ExVT.getScalarSizeInBits();
2320
2321 // If we only care about the highest bit, don't bother shifting right.
2322 if (DemandedBits.isSignMask()) {
2323 unsigned MinSignedBits =
2324 TLO.DAG.ComputeMaxSignificantBits(Op0, DemandedElts, Depth + 1);
2325 bool AlreadySignExtended = ExVTBits >= MinSignedBits;
2326 // However if the input is already sign extended we expect the sign
2327 // extension to be dropped altogether later and do not simplify.
2328 if (!AlreadySignExtended) {
2329 // Compute the correct shift amount type, which must be getShiftAmountTy
2330 // for scalar types after legalization.
2331 SDValue ShiftAmt =
2332 TLO.DAG.getShiftAmountConstant(BitWidth - ExVTBits, VT, dl);
2333 return TLO.CombineTo(Op,
2334 TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, ShiftAmt));
2335 }
2336 }
2337
2338 // If none of the extended bits are demanded, eliminate the sextinreg.
2339 if (DemandedBits.getActiveBits() <= ExVTBits)
2340 return TLO.CombineTo(Op, Op0);
2341
2342 APInt InputDemandedBits = DemandedBits.getLoBits(ExVTBits);
2343
2344 // Since the sign extended bits are demanded, we know that the sign
2345 // bit is demanded.
2346 InputDemandedBits.setBit(ExVTBits - 1);
2347
2348 if (SimplifyDemandedBits(Op0, InputDemandedBits, DemandedElts, Known, TLO,
2349 Depth + 1))
2350 return true;
2351 assert(!Known.hasConflict() && "Bits known to be one AND zero?");
2352
2353 // If the sign bit of the input is known set or clear, then we know the
2354 // top bits of the result.
2355
2356 // If the input sign bit is known zero, convert this into a zero extension.
2357 if (Known.Zero[ExVTBits - 1])
2358 return TLO.CombineTo(Op, TLO.DAG.getZeroExtendInReg(Op0, dl, ExVT));
2359
2360 APInt Mask = APInt::getLowBitsSet(BitWidth, ExVTBits);
2361 if (Known.One[ExVTBits - 1]) { // Input sign bit known set
2362 Known.One.setBitsFrom(ExVTBits);
2363 Known.Zero &= Mask;
2364 } else { // Input sign bit unknown
2365 Known.Zero &= Mask;
2366 Known.One &= Mask;
2367 }
2368 break;
2369 }
2370 case ISD::BUILD_PAIR: {
2371 EVT HalfVT = Op.getOperand(0).getValueType();
2372 unsigned HalfBitWidth = HalfVT.getScalarSizeInBits();
2373
2374 APInt MaskLo = DemandedBits.getLoBits(HalfBitWidth).trunc(HalfBitWidth);
2375 APInt MaskHi = DemandedBits.getHiBits(HalfBitWidth).trunc(HalfBitWidth);
2376
2377 KnownBits KnownLo, KnownHi;
2378
2379 if (SimplifyDemandedBits(Op.getOperand(0), MaskLo, KnownLo, TLO, Depth + 1))
2380 return true;
2381
2382 if (SimplifyDemandedBits(Op.getOperand(1), MaskHi, KnownHi, TLO, Depth + 1))
2383 return true;
2384
2385 Known = KnownHi.concat(KnownLo);
2386 break;
2387 }
2389 if (VT.isScalableVector())
2390 return false;
2391 [[fallthrough]];
2392 case ISD::ZERO_EXTEND: {
2393 SDValue Src = Op.getOperand(0);
2394 EVT SrcVT = Src.getValueType();
2395 unsigned InBits = SrcVT.getScalarSizeInBits();
2396 unsigned InElts = SrcVT.isFixedLengthVector() ? SrcVT.getVectorNumElements() : 1;
2397 bool IsVecInReg = Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG;
2398
2399 // If none of the top bits are demanded, convert this into an any_extend.
2400 if (DemandedBits.getActiveBits() <= InBits) {
2401 // If we only need the non-extended bits of the bottom element
2402 // then we can just bitcast to the result.
2403 if (IsLE && IsVecInReg && DemandedElts == 1 &&
2404 VT.getSizeInBits() == SrcVT.getSizeInBits())
2405 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
2406
2407 unsigned Opc =
2409 if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
2410 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
2411 }
2412
2413 SDNodeFlags Flags = Op->getFlags();
2414 APInt InDemandedBits = DemandedBits.trunc(InBits);
2415 APInt InDemandedElts = DemandedElts.zext(InElts);
2416 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
2417 Depth + 1)) {
2418 if (Flags.hasNonNeg()) {
2419 Flags.setNonNeg(false);
2420 Op->setFlags(Flags);
2421 }
2422 return true;
2423 }
2424 assert(!Known.hasConflict() && "Bits known to be one AND zero?");
2425 assert(Known.getBitWidth() == InBits && "Src width has changed?");
2426 Known = Known.zext(BitWidth);
2427
2428 // Attempt to avoid multi-use ops if we don't need anything from them.
2429 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2430 Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
2431 return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
2432 break;
2433 }
2435 if (VT.isScalableVector())
2436 return false;
2437 [[fallthrough]];
2438 case ISD::SIGN_EXTEND: {
2439 SDValue Src = Op.getOperand(0);
2440 EVT SrcVT = Src.getValueType();
2441 unsigned InBits = SrcVT.getScalarSizeInBits();
2442 unsigned InElts = SrcVT.isFixedLengthVector() ? SrcVT.getVectorNumElements() : 1;
2443 bool IsVecInReg = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG;
2444
2445 APInt InDemandedElts = DemandedElts.zext(InElts);
2446 APInt InDemandedBits = DemandedBits.trunc(InBits);
2447
2448 // Since some of the sign extended bits are demanded, we know that the sign
2449 // bit is demanded.
2450 InDemandedBits.setBit(InBits - 1);
2451
2452 // If none of the top bits are demanded, convert this into an any_extend.
2453 if (DemandedBits.getActiveBits() <= InBits) {
2454 // If we only need the non-extended bits of the bottom element
2455 // then we can just bitcast to the result.
2456 if (IsLE && IsVecInReg && DemandedElts == 1 &&
2457 VT.getSizeInBits() == SrcVT.getSizeInBits())
2458 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
2459
2460 // Don't lose an all signbits 0/-1 splat on targets with 0/-1 booleans.
2462 TLO.DAG.ComputeNumSignBits(Src, InDemandedElts, Depth + 1) !=
2463 InBits) {
2464 unsigned Opc =
2466 if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
2467 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
2468 }
2469 }
2470
2471 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
2472 Depth + 1))
2473 return true;
2474 assert(!Known.hasConflict() && "Bits known to be one AND zero?");
2475 assert(Known.getBitWidth() == InBits && "Src width has changed?");
2476
2477 // If the sign bit is known one, the top bits match.
2478 Known = Known.sext(BitWidth);
2479
2480 // If the sign bit is known zero, convert this to a zero extend.
2481 if (Known.isNonNegative()) {
2482 unsigned Opc =
2484 if (!TLO.LegalOperations() || isOperationLegal(Opc, VT)) {
2485 SDNodeFlags Flags;
2486 if (!IsVecInReg)
2487 Flags.setNonNeg(true);
2488 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src, Flags));
2489 }
2490 }
2491
2492 // Attempt to avoid multi-use ops if we don't need anything from them.
2493 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2494 Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
2495 return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
2496 break;
2497 }
2499 if (VT.isScalableVector())
2500 return false;
2501 [[fallthrough]];
2502 case ISD::ANY_EXTEND: {
2503 SDValue Src = Op.getOperand(0);
2504 EVT SrcVT = Src.getValueType();
2505 unsigned InBits = SrcVT.getScalarSizeInBits();
2506 unsigned InElts = SrcVT.isFixedLengthVector() ? SrcVT.getVectorNumElements() : 1;
2507 bool IsVecInReg = Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG;
2508
2509 // If we only need the bottom element then we can just bitcast.
2510 // TODO: Handle ANY_EXTEND?
2511 if (IsLE && IsVecInReg && DemandedElts == 1 &&
2512 VT.getSizeInBits() == SrcVT.getSizeInBits())
2513 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
2514
2515 APInt InDemandedBits = DemandedBits.trunc(InBits);
2516 APInt InDemandedElts = DemandedElts.zext(InElts);
2517 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
2518 Depth + 1))
2519 return true;
2520 assert(!Known.hasConflict() && "Bits known to be one AND zero?");
2521 assert(Known.getBitWidth() == InBits && "Src width has changed?");
2522 Known = Known.anyext(BitWidth);
2523
2524 // Attempt to avoid multi-use ops if we don't need anything from them.
2525 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2526 Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
2527 return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
2528 break;
2529 }
2530 case ISD::TRUNCATE: {
2531 SDValue Src = Op.getOperand(0);
2532
2533 // Simplify the input, using demanded bit information, and compute the known
2534 // zero/one bits live out.
2535 unsigned OperandBitWidth = Src.getScalarValueSizeInBits();
2536 APInt TruncMask = DemandedBits.zext(OperandBitWidth);
2537 if (SimplifyDemandedBits(Src, TruncMask, DemandedElts, Known, TLO,
2538 Depth + 1))
2539 return true;
2540 Known = Known.trunc(BitWidth);
2541
2542 // Attempt to avoid multi-use ops if we don't need anything from them.
2543 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2544 Src, TruncMask, DemandedElts, TLO.DAG, Depth + 1))
2545 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, NewSrc));
2546
2547 // If the input is only used by this truncate, see if we can shrink it based
2548 // on the known demanded bits.
2549 switch (Src.getOpcode()) {
2550 default:
2551 break;
2552 case ISD::SRL:
2553 // Shrink SRL by a constant if none of the high bits shifted in are
2554 // demanded.
2555 if (TLO.LegalTypes() && !isTypeDesirableForOp(ISD::SRL, VT))
2556 // Do not turn (vt1 truncate (vt2 srl)) into (vt1 srl) if vt1 is
2557 // undesirable.
2558 break;
2559
2560 if (Src.getNode()->hasOneUse()) {
2561 const APInt *ShAmtC =
2562 TLO.DAG.getValidShiftAmountConstant(Src, DemandedElts);
2563 if (!ShAmtC || ShAmtC->uge(BitWidth))
2564 break;
2565 uint64_t ShVal = ShAmtC->getZExtValue();
2566
2567 APInt HighBits =
2568 APInt::getHighBitsSet(OperandBitWidth, OperandBitWidth - BitWidth);
2569 HighBits.lshrInPlace(ShVal);
2570 HighBits = HighBits.trunc(BitWidth);
2571
2572 if (!(HighBits & DemandedBits)) {
2573 // None of the shifted in bits are needed. Add a truncate of the
2574 // shift input, then shift it.
2575 SDValue NewShAmt =
2576 TLO.DAG.getShiftAmountConstant(ShVal, VT, dl, TLO.LegalTypes());
2577 SDValue NewTrunc =
2578 TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, Src.getOperand(0));
2579 return TLO.CombineTo(
2580 Op, TLO.DAG.getNode(ISD::SRL, dl, VT, NewTrunc, NewShAmt));
2581 }
2582 }
2583 break;
2584 }
2585
2586 assert(!Known.hasConflict() && "Bits known to be one AND zero?");
2587 break;
2588 }
2589 case ISD::AssertZext: {
2590 // AssertZext demands all of the high bits, plus any of the low bits
2591 // demanded by its users.
2592 EVT ZVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2594 if (SimplifyDemandedBits(Op.getOperand(0), ~InMask | DemandedBits, Known,
2595 TLO, Depth + 1))
2596 return true;
2597 assert(!Known.hasConflict() && "Bits known to be one AND zero?");
2598
2599 Known.Zero |= ~InMask;
2600 Known.One &= (~Known.Zero);
2601 break;
2602 }
2604 SDValue Src = Op.getOperand(0);
2605 SDValue Idx = Op.getOperand(1);
2606 ElementCount SrcEltCnt = Src.getValueType().getVectorElementCount();
2607 unsigned EltBitWidth = Src.getScalarValueSizeInBits();
2608
2609 if (SrcEltCnt.isScalable())
2610 return false;
2611
2612 // Demand the bits from every vector element without a constant index.
2613 unsigned NumSrcElts = SrcEltCnt.getFixedValue();
2614 APInt DemandedSrcElts = APInt::getAllOnes(NumSrcElts);
2615 if (auto *CIdx = dyn_cast<ConstantSDNode>(Idx))
2616 if (CIdx->getAPIntValue().ult(NumSrcElts))
2617 DemandedSrcElts = APInt::getOneBitSet(NumSrcElts, CIdx->getZExtValue());
2618
2619 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
2620 // anything about the extended bits.
2621 APInt DemandedSrcBits = DemandedBits;
2622 if (BitWidth > EltBitWidth)
2623 DemandedSrcBits = DemandedSrcBits.trunc(EltBitWidth);
2624
2625 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts, Known2, TLO,
2626 Depth + 1))
2627 return true;
2628
2629 // Attempt to avoid multi-use ops if we don't need anything from them.
2630 if (!DemandedSrcBits.isAllOnes() || !DemandedSrcElts.isAllOnes()) {
2631 if (SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
2632 Src, DemandedSrcBits, DemandedSrcElts, TLO.DAG, Depth + 1)) {
2633 SDValue NewOp =
2634 TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc, Idx);
2635 return TLO.CombineTo(Op, NewOp);
2636 }
2637 }
2638
2639 Known = Known2;
2640 if (BitWidth > EltBitWidth)
2641 Known = Known.anyext(BitWidth);
2642 break;
2643 }
2644 case ISD::BITCAST: {
2645 if (VT.isScalableVector())
2646 return false;
2647 SDValue Src = Op.getOperand(0);
2648 EVT SrcVT = Src.getValueType();
2649 unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
2650
2651 // If this is an FP->Int bitcast and if the sign bit is the only
2652 // thing demanded, turn this into a FGETSIGN.
2653 if (!TLO.LegalOperations() && !VT.isVector() && !SrcVT.isVector() &&
2654 DemandedBits == APInt::getSignMask(Op.getValueSizeInBits()) &&
2655 SrcVT.isFloatingPoint()) {
2656 bool OpVTLegal = isOperationLegalOrCustom(ISD::FGETSIGN, VT);
2657 bool i32Legal = isOperationLegalOrCustom(ISD::FGETSIGN, MVT::i32);
2658 if ((OpVTLegal || i32Legal) && VT.isSimple() && SrcVT != MVT::f16 &&
2659 SrcVT != MVT::f128) {
2660 // Cannot eliminate/lower SHL for f128 yet.
2661 EVT Ty = OpVTLegal ? VT : MVT::i32;
2662 // Make a FGETSIGN + SHL to move the sign bit into the appropriate
2663 // place. We expect the SHL to be eliminated by other optimizations.
2664 SDValue Sign = TLO.DAG.getNode(ISD::FGETSIGN, dl, Ty, Src);
2665 unsigned OpVTSizeInBits = Op.getValueSizeInBits();
2666 if (!OpVTLegal && OpVTSizeInBits > 32)
2667 Sign = TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Sign);
2668 unsigned ShVal = Op.getValueSizeInBits() - 1;
2669 SDValue ShAmt = TLO.DAG.getConstant(ShVal, dl, VT);
2670 return TLO.CombineTo(Op,
2671 TLO.DAG.getNode(ISD::SHL, dl, VT, Sign, ShAmt));
2672 }
2673 }
2674
2675 // Bitcast from a vector using SimplifyDemanded Bits/VectorElts.
2676 // Demand the elt/bit if any of the original elts/bits are demanded.
2677 if (SrcVT.isVector() && (BitWidth % NumSrcEltBits) == 0) {
2678 unsigned Scale = BitWidth / NumSrcEltBits;
2679 unsigned NumSrcElts = SrcVT.getVectorNumElements();
2680 APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);
2681 APInt DemandedSrcElts = APInt::getZero(NumSrcElts);
2682 for (unsigned i = 0; i != Scale; ++i) {
2683 unsigned EltOffset = IsLE ? i : (Scale - 1 - i);
2684 unsigned BitOffset = EltOffset * NumSrcEltBits;
2685 APInt Sub = DemandedBits.extractBits(NumSrcEltBits, BitOffset);
2686 if (!Sub.isZero()) {
2687 DemandedSrcBits |= Sub;
2688 for (unsigned j = 0; j != NumElts; ++j)
2689 if (DemandedElts[j])
2690 DemandedSrcElts.setBit((j * Scale) + i);
2691 }
2692 }
2693
2694 APInt KnownSrcUndef, KnownSrcZero;
2695 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
2696 KnownSrcZero, TLO, Depth + 1))
2697 return true;
2698
2699 KnownBits KnownSrcBits;
2700 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
2701 KnownSrcBits, TLO, Depth + 1))
2702 return true;
2703 } else if (IsLE && (NumSrcEltBits % BitWidth) == 0) {
2704 // TODO - bigendian once we have test coverage.
2705 unsigned Scale = NumSrcEltBits / BitWidth;
2706 unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
2707 APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);
2708 APInt DemandedSrcElts = APInt::getZero(NumSrcElts);
2709 for (unsigned i = 0; i != NumElts; ++i)
2710 if (DemandedElts[i]) {
2711 unsigned Offset = (i % Scale) * BitWidth;
2712 DemandedSrcBits.insertBits(DemandedBits, Offset);
2713 DemandedSrcElts.setBit(i / Scale);
2714 }
2715
2716 if (SrcVT.isVector()) {
2717 APInt KnownSrcUndef, KnownSrcZero;
2718 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
2719 KnownSrcZero, TLO, Depth + 1))
2720 return true;
2721 }
2722
2723 KnownBits KnownSrcBits;
2724 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
2725 KnownSrcBits, TLO, Depth + 1))
2726 return true;
2727
2728 // Attempt to avoid multi-use ops if we don't need anything from them.
2729 if (!DemandedSrcBits.isAllOnes() || !DemandedSrcElts.isAllOnes()) {
2730 if (SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
2731 Src, DemandedSrcBits, DemandedSrcElts, TLO.DAG, Depth + 1)) {
2732 SDValue NewOp = TLO.DAG.getBitcast(VT, DemandedSrc);
2733 return TLO.CombineTo(Op, NewOp);
2734 }
2735 }
2736 }
2737
2738 // If this is a bitcast, let computeKnownBits handle it. Only do this on a
2739 // recursive call where Known may be useful to the caller.
2740 if (Depth > 0) {
2741 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2742 return false;
2743 }
2744 break;
2745 }
2746 case ISD::MUL:
2747 if (DemandedBits.isPowerOf2()) {
2748 // The LSB of X*Y is set only if (X & 1) == 1 and (Y & 1) == 1.
2749 // If we demand exactly one bit N and we have "X * (C' << N)" where C' is
2750 // odd (has LSB set), then the left-shifted low bit of X is the answer.
2751 unsigned CTZ = DemandedBits.countr_zero();
2752 ConstantSDNode *C = isConstOrConstSplat(Op.getOperand(1), DemandedElts);
2753 if (C && C->getAPIntValue().countr_zero() == CTZ) {
2754 SDValue AmtC = TLO.DAG.getShiftAmountConstant(CTZ, VT, dl);
2755 SDValue Shl = TLO.DAG.getNode(ISD::SHL, dl, VT, Op.getOperand(0), AmtC);
2756 return TLO.CombineTo(Op, Shl);
2757 }
2758 }
2759 // For a squared value "X * X", the bottom 2 bits are 0 and X[0] because:
2760 // X * X is odd iff X is odd.
2761 // 'Quadratic Reciprocity': X * X -> 0 for bit[1]
2762 if (Op.getOperand(0) == Op.getOperand(1) && DemandedBits.ult(4)) {
2763 SDValue One = TLO.DAG.getConstant(1, dl, VT);
2764 SDValue And1 = TLO.DAG.getNode(ISD::AND, dl, VT, Op.getOperand(0), One);
2765 return TLO.CombineTo(Op, And1);
2766 }
2767 [[fallthrough]];
2768 case ISD::ADD:
2769 case ISD::SUB: {
2770 // Add, Sub, and Mul don't demand any bits in positions beyond that
2771 // of the highest bit demanded of them.
2772 SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1);
2773 SDNodeFlags Flags = Op.getNode()->getFlags();
2774 unsigned DemandedBitsLZ = DemandedBits.countl_zero();
2775 APInt LoMask = APInt::getLowBitsSet(BitWidth, BitWidth - DemandedBitsLZ);
2776 KnownBits KnownOp0, KnownOp1;
2777 if (SimplifyDemandedBits(Op0, LoMask, DemandedElts, KnownOp0, TLO,
2778 Depth + 1) ||
2779 SimplifyDemandedBits(Op1, LoMask, DemandedElts, KnownOp1, TLO,
2780 Depth + 1) ||
2781 // See if the operation should be performed at a smaller bit width.
2782 ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO)) {
2783 if (Flags.hasNoSignedWrap() || Flags.hasNoUnsignedWrap()) {
2784 // Disable the nsw and nuw flags. We can no longer guarantee that we
2785 // won't wrap after simplification.
2786 Flags.setNoSignedWrap(false);
2787 Flags.setNoUnsignedWrap(false);
2788 Op->setFlags(Flags);
2789 }
2790 return true;
2791 }
2792
2793 // neg x with only low bit demanded is simply x.
2794 if (Op.getOpcode() == ISD::SUB && DemandedBits.isOne() &&
2795 isNullConstant(Op0))
2796 return TLO.CombineTo(Op, Op1);
2797
2798 // Attempt to avoid multi-use ops if we don't need anything from them.
2799 if (!LoMask.isAllOnes() || !DemandedElts.isAllOnes()) {
2800 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2801 Op0, LoMask, DemandedElts, TLO.DAG, Depth + 1);
2802 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
2803 Op1, LoMask, DemandedElts, TLO.DAG, Depth + 1);
2804 if (DemandedOp0 || DemandedOp1) {
2805 Flags.setNoSignedWrap(false);
2806 Flags.setNoUnsignedWrap(false);
2807 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
2808 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
2809 SDValue NewOp =
2810 TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);
2811 return TLO.CombineTo(Op, NewOp);
2812 }
2813 }
2814
2815 // If we have a constant operand, we may be able to turn it into -1 if we
2816 // do not demand the high bits. This can make the constant smaller to
2817 // encode, allow more general folding, or match specialized instruction
2818 // patterns (eg, 'blsr' on x86). Don't bother changing 1 to -1 because that
2819 // is probably not useful (and could be detrimental).
2821 APInt HighMask = APInt::getHighBitsSet(BitWidth, DemandedBitsLZ);
2822 if (C && !C->isAllOnes() && !C->isOne() &&
2823 (C->getAPIntValue() | HighMask).isAllOnes()) {
2824 SDValue Neg1 = TLO.DAG.getAllOnesConstant(dl, VT);
2825 // Disable the nsw and nuw flags. We can no longer guarantee that we
2826 // won't wrap after simplification.
2827 Flags.setNoSignedWrap(false);
2828 Flags.setNoUnsignedWrap(false);
2829 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Neg1, Flags);
2830 return TLO.CombineTo(Op, NewOp);
2831 }
2832
2833 // Match a multiply with a disguised negated-power-of-2 and convert to a
2834 // an equivalent shift-left amount.
2835 // Example: (X * MulC) + Op1 --> Op1 - (X << log2(-MulC))
2836 auto getShiftLeftAmt = [&HighMask](SDValue Mul) -> unsigned {
2837 if (Mul.getOpcode() != ISD::MUL || !Mul.hasOneUse())
2838 return 0;
2839
2840 // Don't touch opaque constants. Also, ignore zero and power-of-2
2841 // multiplies. Those will get folded later.
2842 ConstantSDNode *MulC = isConstOrConstSplat(Mul.getOperand(1));
2843 if (MulC && !MulC->isOpaque() && !MulC->isZero() &&
2844 !MulC->getAPIntValue().isPowerOf2()) {
2845 APInt UnmaskedC = MulC->getAPIntValue() | HighMask;
2846 if (UnmaskedC.isNegatedPowerOf2())
2847 return (-UnmaskedC).logBase2();
2848 }
2849 return 0;
2850 };
2851
2852 auto foldMul = [&](ISD::NodeType NT, SDValue X, SDValue Y,
2853 unsigned ShlAmt) {
2854 SDValue ShlAmtC = TLO.DAG.getShiftAmountConstant(ShlAmt, VT, dl);
2855 SDValue Shl = TLO.DAG.getNode(ISD::SHL, dl, VT, X, ShlAmtC);
2856 SDValue Res = TLO.DAG.getNode(NT, dl, VT, Y, Shl);
2857 return TLO.CombineTo(Op, Res);
2858 };
2859
2861 if (Op.getOpcode() == ISD::ADD) {
2862 // (X * MulC) + Op1 --> Op1 - (X << log2(-MulC))
2863 if (unsigned ShAmt = getShiftLeftAmt(Op0))
2864 return foldMul(ISD::SUB, Op0.getOperand(0), Op1, ShAmt);
2865 // Op0 + (X * MulC) --> Op0 - (X << log2(-MulC))
2866 if (unsigned ShAmt = getShiftLeftAmt(Op1))
2867 return foldMul(ISD::SUB, Op1.getOperand(0), Op0, ShAmt);
2868 }
2869 if (Op.getOpcode() == ISD::SUB) {
2870 // Op0 - (X * MulC) --> Op0 + (X << log2(-MulC))
2871 if (unsigned ShAmt = getShiftLeftAmt(Op1))
2872 return foldMul(ISD::ADD, Op1.getOperand(0), Op0, ShAmt);
2873 }
2874 }
2875
2876 if (Op.getOpcode() == ISD::MUL) {
2877 Known = KnownBits::mul(KnownOp0, KnownOp1);
2878 } else { // Op.getOpcode() is either ISD::ADD or ISD::SUB.
2880 Op.getOpcode() == ISD::ADD, Flags.hasNoSignedWrap(),
2881 Flags.hasNoUnsignedWrap(), KnownOp0, KnownOp1);
2882 }
2883 break;
2884 }
2885 default:
2886 // We also ask the target about intrinsics (which could be specific to it).
2887 if (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
2888 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN) {
2889 // TODO: Probably okay to remove after audit; here to reduce change size
2890 // in initial enablement patch for scalable vectors
2891 if (Op.getValueType().isScalableVector())
2892 break;
2893 if (SimplifyDemandedBitsForTargetNode(Op, DemandedBits, DemandedElts,
2894 Known, TLO, Depth))
2895 return true;
2896 break;
2897 }
2898
2899 // Just use computeKnownBits to compute output bits.
2900 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2901 break;
2902 }
2903
2904 // If we know the value of all of the demanded bits, return this as a
2905 // constant.
2906 if (!isTargetCanonicalConstantNode(Op) &&
2907 DemandedBits.isSubsetOf(Known.Zero | Known.One)) {
2908 // Avoid folding to a constant if any OpaqueConstant is involved.
2909 const SDNode *N = Op.getNode();
2910 for (SDNode *Op :
2912 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op))
2913 if (C->isOpaque())
2914 return false;
2915 }
2916 if (VT.isInteger())
2917 return TLO.CombineTo(Op, TLO.DAG.getConstant(Known.One, dl, VT));
2918 if (VT.isFloatingPoint())
2919 return TLO.CombineTo(
2920 Op,
2921 TLO.DAG.getConstantFP(
2922 APFloat(TLO.DAG.EVTToAPFloatSemantics(VT), Known.One), dl, VT));
2923 }
2924
2925 // A multi use 'all demanded elts' simplify failed to find any knownbits.
2926 // Try again just for the original demanded elts.
2927 // Ensure we do this AFTER constant folding above.
2928 if (HasMultiUse && Known.isUnknown() && !OriginalDemandedElts.isAllOnes())
2929 Known = TLO.DAG.computeKnownBits(Op, OriginalDemandedElts, Depth);
2930
2931 return false;
2932}
2933
2935 const APInt &DemandedElts,
2936 DAGCombinerInfo &DCI) const {
2937 SelectionDAG &DAG = DCI.DAG;
2938 TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
2939 !DCI.isBeforeLegalizeOps());
2940
2941 APInt KnownUndef, KnownZero;
2942 bool Simplified =
2943 SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero, TLO);
2944 if (Simplified) {
2945 DCI.AddToWorklist(Op.getNode());
2946 DCI.CommitTargetLoweringOpt(TLO);
2947 }
2948
2949 return Simplified;
2950}
2951
2952/// Given a vector binary operation and known undefined elements for each input
2953/// operand, compute whether each element of the output is undefined.
2955 const APInt &UndefOp0,
2956 const APInt &UndefOp1) {
2957 EVT VT = BO.getValueType();
2959 "Vector binop only");
2960
2961 EVT EltVT = VT.getVectorElementType();
2962 unsigned NumElts = VT.isFixedLengthVector() ? VT.getVectorNumElements() : 1;
2963 assert(UndefOp0.getBitWidth() == NumElts &&
2964 UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis");
2965
2966 auto getUndefOrConstantElt = [&](SDValue V, unsigned Index,
2967 const APInt &UndefVals) {
2968 if (UndefVals[Index])
2969 return DAG.getUNDEF(EltVT);
2970
2971 if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
2972 // Try hard to make sure that the getNode() call is not creating temporary
2973 // nodes. Ignore opaque integers because they do not constant fold.
2974 SDValue Elt = BV->getOperand(Index);
2975 auto *C = dyn_cast<ConstantSDNode>(Elt);
2976 if (isa<ConstantFPSDNode>(Elt) || Elt.isUndef() || (C && !C->isOpaque()))
2977 return Elt;
2978 }
2979
2980 return SDValue();
2981 };
2982
2983 APInt KnownUndef = APInt::getZero(NumElts);
2984 for (unsigned i = 0; i != NumElts; ++i) {
2985 // If both inputs for this element are either constant or undef and match
2986 // the element type, compute the constant/undef result for this element of
2987 // the vector.
2988 // TODO: Ideally we would use FoldConstantArithmetic() here, but that does
2989 // not handle FP constants. The code within getNode() should be refactored
2990 // to avoid the danger of creating a bogus temporary node here.
2991 SDValue C0 = getUndefOrConstantElt(BO.getOperand(0), i, UndefOp0);
2992 SDValue C1 = getUndefOrConstantElt(BO.getOperand(1), i, UndefOp1);
2993 if (C0 && C1 && C0.getValueType() == EltVT && C1.getValueType() == EltVT)
2994 if (DAG.getNode(BO.getOpcode(), SDLoc(BO), EltVT, C0, C1).isUndef())
2995 KnownUndef.setBit(i);
2996 }
2997 return KnownUndef;
2998}
2999
3001 SDValue Op, const APInt &OriginalDemandedElts, APInt &KnownUndef,
3002 APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth,
3003 bool AssumeSingleUse) const {
3004 EVT VT = Op.getValueType();
3005 unsigned Opcode = Op.getOpcode();
3006 APInt DemandedElts = OriginalDemandedElts;
3007 unsigned NumElts = DemandedElts.getBitWidth();
3008 assert(VT.isVector() && "Expected vector op");
3009
3010 KnownUndef = KnownZero = APInt::getZero(NumElts);
3011
3012 const TargetLowering &TLI = TLO.DAG.getTargetLoweringInfo();
3013 if (!TLI.shouldSimplifyDemandedVectorElts(Op, TLO))
3014 return false;
3015
3016 // TODO: For now we assume we know nothing about scalable vectors.
3017 if (VT.isScalableVector())
3018 return false;
3019
3020 assert(VT.getVectorNumElements() == NumElts &&
3021 "Mask size mismatches value type element count!");
3022
3023 // Undef operand.
3024 if (Op.isUndef()) {
3025 KnownUndef.setAllBits();
3026 return false;
3027 }
3028
3029 // If Op has other users, assume that all elements are needed.
3030 if (!AssumeSingleUse && !Op.getNode()->hasOneUse())
3031 DemandedElts.setAllBits();
3032
3033 // Not demanding any elements from Op.
3034 if (DemandedElts == 0) {
3035 KnownUndef.setAllBits();
3036 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
3037 }
3038
3039 // Limit search depth.
3041 return false;
3042
3043 SDLoc DL(Op);
3044 unsigned EltSizeInBits = VT.getScalarSizeInBits();
3045 bool IsLE = TLO.DAG.getDataLayout().isLittleEndian();
3046
3047 // Helper for demanding the specified elements and all the bits of both binary
3048 // operands.
3049 auto SimplifyDemandedVectorEltsBinOp = [&](SDValue Op0, SDValue Op1) {
3050 SDValue NewOp0 = SimplifyMultipleUseDemandedVectorElts(Op0, DemandedElts,
3051 TLO.DAG, Depth + 1);
3052 SDValue NewOp1 = SimplifyMultipleUseDemandedVectorElts(Op1, DemandedElts,
3053 TLO.DAG, Depth + 1);
3054 if (NewOp0 || NewOp1) {
3055 SDValue NewOp =
3056 TLO.DAG.getNode(Opcode, SDLoc(Op), VT, NewOp0 ? NewOp0 : Op0,
3057 NewOp1 ? NewOp1 : Op1, Op->getFlags());
3058 return TLO.CombineTo(Op, NewOp);
3059 }
3060 return false;
3061 };
3062
3063 switch (Opcode) {
3064 case ISD::SCALAR_TO_VECTOR: {
3065 if (!DemandedElts[0]) {
3066 KnownUndef.setAllBits();
3067 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
3068 }
3069 SDValue ScalarSrc = Op.getOperand(0);
3070 if (ScalarSrc.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
3071 SDValue Src = ScalarSrc.getOperand(0);
3072 SDValue Idx = ScalarSrc.getOperand(1);
3073 EVT SrcVT = Src.getValueType();
3074
3075 ElementCount SrcEltCnt = SrcVT.getVectorElementCount();
3076
3077 if (SrcEltCnt.isScalable())
3078 return false;
3079
3080 unsigned NumSrcElts = SrcEltCnt.getFixedValue();
3081 if (isNullConstant(Idx)) {
3082 APInt SrcDemandedElts = APInt::getOneBitSet(NumSrcElts, 0);
3083 APInt SrcUndef = KnownUndef.zextOrTrunc(NumSrcElts);
3084 APInt SrcZero = KnownZero.zextOrTrunc(NumSrcElts);
3085 if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
3086 TLO, Depth + 1))
3087 return true;
3088 }
3089 }
3090 KnownUndef.setHighBits(NumElts - 1);
3091 break;
3092 }
3093 case ISD::BITCAST: {
3094 SDValue Src = Op.getOperand(0);
3095 EVT SrcVT = Src.getValueType();
3096
3097 // We only handle vectors here.
3098 // TODO - investigate calling SimplifyDemandedBits/ComputeKnownBits?
3099 if (!SrcVT.isVector())
3100 break;
3101
3102 // Fast handling of 'identity' bitcasts.
3103 unsigned NumSrcElts = SrcVT.getVectorNumElements();
3104 if (NumSrcElts == NumElts)
3105 return SimplifyDemandedVectorElts(Src, DemandedElts, KnownUndef,
3106 KnownZero, TLO, Depth + 1);
3107
3108 APInt SrcDemandedElts, SrcZero, SrcUndef;
3109
3110 // Bitcast from 'large element' src vector to 'small element' vector, we
3111 // must demand a source element if any DemandedElt maps to it.
3112 if ((NumElts % NumSrcElts) == 0) {
3113 unsigned Scale = NumElts / NumSrcElts;
3114 SrcDemandedElts = APIntOps::ScaleBitMask(DemandedElts, NumSrcElts);
3115 if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
3116 TLO, Depth + 1))
3117 return true;
3118
3119 // Try calling SimplifyDemandedBits, converting demanded elts to the bits
3120 // of the large element.
3121 // TODO - bigendian once we have test coverage.
3122 if (IsLE) {
3123 unsigned SrcEltSizeInBits = SrcVT.getScalarSizeInBits();
3124 APInt SrcDemandedBits = APInt::getZero(SrcEltSizeInBits);
3125 for (unsigned i = 0; i != NumElts; ++i)
3126 if (DemandedElts[i]) {
3127 unsigned Ofs = (i % Scale) * EltSizeInBits;
3128 SrcDemandedBits.setBits(Ofs, Ofs + EltSizeInBits);
3129 }
3130
3131 KnownBits Known;
3132 if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcDemandedElts, Known,
3133 TLO, Depth + 1))
3134 return true;
3135
3136 // The bitcast has split each wide element into a number of
3137 // narrow subelements. We have just computed the Known bits
3138 // for wide elements. See if element splitting results in
3139 // some subelements being zero. Only for demanded elements!
3140 for (unsigned SubElt = 0; SubElt != Scale; ++SubElt) {
3141 if (!Known.Zero.extractBits(EltSizeInBits, SubElt * EltSizeInBits)
3142 .isAllOnes())
3143 continue;
3144 for (unsigned SrcElt = 0; SrcElt != NumSrcElts; ++SrcElt) {
3145 unsigned Elt = Scale * SrcElt + SubElt;
3146 if (DemandedElts[Elt])
3147 KnownZero.setBit(Elt);
3148 }
3149 }
3150 }
3151
3152 // If the src element is zero/undef then all the output elements will be -
3153 // only demanded elements are guaranteed to be correct.
3154 for (unsigned i = 0; i != NumSrcElts; ++i) {
3155 if (SrcDemandedElts[i]) {
3156 if (SrcZero[i])
3157 KnownZero.setBits(i * Scale, (i + 1) * Scale);
3158 if (SrcUndef[i])
3159 KnownUndef.setBits(i * Scale, (i + 1) * Scale);
3160 }
3161 }
3162 }
3163
3164 // Bitcast from 'small element' src vector to 'large element' vector, we
3165 // demand all smaller source elements covered by the larger demanded element
3166 // of this vector.
3167 if ((NumSrcElts % NumElts) == 0) {
3168 unsigned Scale = NumSrcElts / NumElts;
3169 SrcDemandedElts = APIntOps::ScaleBitMask(DemandedElts, NumSrcElts);
3170 if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
3171 TLO, Depth + 1))
3172 return true;
3173
3174 // If all the src elements covering an output element are zero/undef, then
3175 // the output element will be as well, assuming it was demanded.
3176 for (unsigned i = 0; i != NumElts; ++i) {
3177 if (DemandedElts[i]) {
3178 if (SrcZero.extractBits(Scale, i * Scale).isAllOnes())
3179 KnownZero.setBit(i);
3180 if (SrcUndef.extractBits(Scale, i * Scale).isAllOnes())
3181 KnownUndef.setBit(i);
3182 }
3183 }
3184 }
3185 break;
3186 }
3187 case ISD::BUILD_VECTOR: {
3188 // Check all elements and simplify any unused elements with UNDEF.
3189 if (!DemandedElts.isAllOnes()) {
3190 // Don't simplify BROADCASTS.
3191 if (llvm::any_of(Op->op_values(),
3192 [&](SDValue Elt) { return Op.getOperand(0) != Elt; })) {
3193 SmallVector<SDValue, 32> Ops(Op->op_begin(), Op->op_end());
3194 bool Updated = false;
3195 for (unsigned i = 0; i != NumElts; ++i) {
3196 if (!DemandedElts[i] && !Ops[i].isUndef()) {
3197 Ops[i] = TLO.DAG.getUNDEF(Ops[0].getValueType());
3198 KnownUndef.setBit(i);
3199 Updated = true;
3200 }
3201 }
3202 if (Updated)
3203 return TLO.CombineTo(Op, TLO.DAG.getBuildVector(VT, DL, Ops));
3204 }
3205 }
3206 for (unsigned i = 0; i != NumElts; ++i) {
3207 SDValue SrcOp = Op.getOperand(i);
3208 if (SrcOp.isUndef()) {
3209 KnownUndef.setBit(i);
3210 } else if (EltSizeInBits == SrcOp.getScalarValueSizeInBits() &&
3212 KnownZero.setBit(i);
3213 }
3214 }
3215 break;
3216 }
3217 case ISD::CONCAT_VECTORS: {
3218 EVT SubVT = Op.getOperand(0).getValueType();
3219 unsigned NumSubVecs = Op.getNumOperands();
3220 unsigned NumSubElts = SubVT.getVectorNumElements();
3221 for (unsigned i = 0; i != NumSubVecs; ++i) {
3222 SDValue SubOp = Op.getOperand(i);
3223 APInt SubElts = DemandedElts.extractBits(NumSubElts, i * NumSubElts);
3224 APInt SubUndef, SubZero;
3225 if (SimplifyDemandedVectorElts(SubOp, SubElts, SubUndef, SubZero, TLO,
3226 Depth + 1))
3227 return true;
3228 KnownUndef.insertBits(SubUndef, i * NumSubElts);
3229 KnownZero.insertBits(SubZero, i * NumSubElts);
3230 }
3231
3232 // Attempt to avoid multi-use ops if we don't need anything from them.
3233 if (!DemandedElts.isAllOnes()) {
3234 bool FoundNewSub = false;
3235 SmallVector<SDValue, 2> DemandedSubOps;
3236 for (unsigned i = 0; i != NumSubVecs; ++i) {
3237 SDValue SubOp = Op.getOperand(i);
3238 APInt SubElts = DemandedElts.extractBits(NumSubElts, i * NumSubElts);
3239 SDValue NewSubOp = SimplifyMultipleUseDemandedVectorElts(
3240 SubOp, SubElts, TLO.DAG, Depth + 1);
3241 DemandedSubOps.push_back(NewSubOp ? NewSubOp : SubOp);
3242 FoundNewSub = NewSubOp ? true : FoundNewSub;
3243 }
3244 if (FoundNewSub) {
3245 SDValue NewOp =
3246 TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, DemandedSubOps);
3247 return TLO.CombineTo(Op, NewOp);
3248 }
3249 }
3250 break;
3251 }
3252 case ISD::INSERT_SUBVECTOR: {
3253 // Demand any elements from the subvector and the remainder from the src its
3254 // inserted into.
3255 SDValue Src = Op.getOperand(0);
3256 SDValue Sub = Op.getOperand(1);
3257 uint64_t Idx = Op.getConstantOperandVal(2);
3258 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
3259 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
3260 APInt DemandedSrcElts = DemandedElts;
3261 DemandedSrcElts.insertBits(APInt::getZero(NumSubElts), Idx);
3262
3263 APInt SubUndef, SubZero;
3264 if (SimplifyDemandedVectorElts(Sub, DemandedSubElts, SubUndef, SubZero, TLO,
3265 Depth + 1))
3266 return true;
3267
3268 // If none of the src operand elements are demanded, replace it with undef.
3269 if (!DemandedSrcElts && !Src.isUndef())
3270 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
3271 TLO.DAG.getUNDEF(VT), Sub,
3272 Op.getOperand(2)));
3273
3274 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownUndef, KnownZero,
3275 TLO, Depth + 1))
3276 return true;
3277 KnownUndef.insertBits(SubUndef, Idx);
3278 KnownZero.insertBits(SubZero, Idx);
3279
3280 // Attempt to avoid multi-use ops if we don't need anything from them.
3281 if (!DemandedSrcElts.isAllOnes() || !DemandedSubElts.isAllOnes()) {
3282 SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
3283 Src, DemandedSrcElts, TLO.DAG, Depth + 1);
3284 SDValue NewSub = SimplifyMultipleUseDemandedVectorElts(
3285 Sub, DemandedSubElts, TLO.DAG, Depth + 1);
3286 if (NewSrc || NewSub) {
3287 NewSrc = NewSrc ? NewSrc : Src;
3288 NewSub = NewSub ? NewSub : Sub;
3289 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,
3290 NewSub, Op.getOperand(2));
3291 return TLO.CombineTo(Op, NewOp);
3292 }
3293 }
3294 break;
3295 }
3297 // Offset the demanded elts by the subvector index.
3298 SDValue Src = Op.getOperand(0);
3299 if (Src.getValueType().isScalableVector())
3300 break;
3301 uint64_t Idx = Op.getConstantOperandVal(1);
3302 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3303 APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx);
3304
3305 APInt SrcUndef, SrcZero;
3306 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
3307 Depth + 1))
3308 return true;
3309 KnownUndef = SrcUndef.extractBits(NumElts, Idx);
3310 KnownZero = SrcZero.extractBits(NumElts, Idx);
3311
3312 // Attempt to avoid multi-use ops if we don't need anything from them.
3313 if (!DemandedElts.isAllOnes()) {
3314 SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
3315 Src, DemandedSrcElts, TLO.DAG, Depth + 1);
3316 if (NewSrc) {
3317 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,
3318 Op.getOperand(1));
3319 return TLO.CombineTo(Op, NewOp);
3320 }
3321 }
3322 break;
3323 }
3325 SDValue Vec = Op.getOperand(0);
3326 SDValue Scl = Op.getOperand(1);
3327 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
3328
3329 // For a legal, constant insertion index, if we don't need this insertion
3330 // then strip it, else remove it from the demanded elts.
3331 if (CIdx && CIdx->getAPIntValue().ult(NumElts)) {
3332 unsigned Idx = CIdx->getZExtValue();
3333 if (!DemandedElts[Idx])
3334 return TLO.CombineTo(Op, Vec);
3335
3336 APInt DemandedVecElts(DemandedElts);
3337 DemandedVecElts.clearBit(Idx);
3338 if (SimplifyDemandedVectorElts(Vec, DemandedVecElts, KnownUndef,
3339 KnownZero, TLO, Depth + 1))
3340 return true;
3341
3342 KnownUndef.setBitVal(Idx, Scl.isUndef());
3343
3344 KnownZero.setBitVal(Idx, isNullConstant(Scl) || isNullFPConstant(Scl));
3345 break;
3346 }
3347
3348 APInt VecUndef, VecZero;
3349 if (SimplifyDemandedVectorElts(Vec, DemandedElts, VecUndef, VecZero, TLO,
3350 Depth + 1))
3351 return true;
3352 // Without knowing the insertion index we can't set KnownUndef/KnownZero.
3353 break;
3354 }
3355 case ISD::VSELECT: {
3356 SDValue Sel = Op.getOperand(0);
3357 SDValue LHS = Op.getOperand(1);
3358 SDValue RHS = Op.getOperand(2);
3359
3360 // Try to transform the select condition based on the current demanded
3361 // elements.
3362 APInt UndefSel, ZeroSel;
3363 if (SimplifyDemandedVectorElts(Sel, DemandedElts, UndefSel, ZeroSel, TLO,
3364 Depth + 1))
3365 return true;
3366
3367 // See if we can simplify either vselect operand.
3368 APInt DemandedLHS(DemandedElts);
3369 APInt DemandedRHS(DemandedElts);
3370 APInt UndefLHS, ZeroLHS;
3371 APInt UndefRHS, ZeroRHS;
3372 if (SimplifyDemandedVectorElts(LHS, DemandedLHS, UndefLHS, ZeroLHS, TLO,
3373 Depth + 1))
3374 return true;
3375 if (SimplifyDemandedVectorElts(RHS, DemandedRHS, UndefRHS, ZeroRHS, TLO,
3376 Depth + 1))
3377 return true;
3378
3379 KnownUndef = UndefLHS & UndefRHS;
3380 KnownZero = ZeroLHS & ZeroRHS;
3381
3382 // If we know that the selected element is always zero, we don't need the
3383 // select value element.
3384 APInt DemandedSel = DemandedElts & ~KnownZero;
3385 if (DemandedSel != DemandedElts)
3386 if (SimplifyDemandedVectorElts(Sel, DemandedSel, UndefSel, ZeroSel, TLO,
3387 Depth + 1))
3388 return true;
3389
3390 break;
3391 }
3392 case ISD::VECTOR_SHUFFLE: {
3393 SDValue LHS = Op.getOperand(0);
3394 SDValue RHS = Op.getOperand(1);
3395 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
3396
3397 // Collect demanded elements from shuffle operands..
3398 APInt DemandedLHS(NumElts, 0);
3399 APInt DemandedRHS(NumElts, 0);
3400 for (unsigned i = 0; i != NumElts; ++i) {
3401 int M = ShuffleMask[i];
3402 if (M < 0 || !DemandedElts[i])
3403 continue;
3404 assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range");
3405 if (M < (int)NumElts)
3406 DemandedLHS.setBit(M);
3407 else
3408 DemandedRHS.setBit(M - NumElts);
3409 }
3410
3411 // See if we can simplify either shuffle operand.
3412 APInt UndefLHS, ZeroLHS;
3413 APInt UndefRHS, ZeroRHS;
3414 if (SimplifyDemandedVectorElts(LHS, DemandedLHS, UndefLHS, ZeroLHS, TLO,
3415 Depth + 1))
3416 return true;
3417 if (SimplifyDemandedVectorElts(RHS, DemandedRHS, UndefRHS, ZeroRHS, TLO,
3418 Depth + 1))
3419 return true;
3420
3421 // Simplify mask using undef elements from LHS/RHS.
3422 bool Updated = false;
3423 bool IdentityLHS = true, IdentityRHS = true;
3424 SmallVector<int, 32> NewMask(ShuffleMask);
3425 for (unsigned i = 0; i != NumElts; ++i) {
3426 int &M = NewMask[i];
3427 if (M < 0)
3428 continue;
3429 if (!DemandedElts[i] || (M < (int)NumElts && UndefLHS[M]) ||
3430 (M >= (int)NumElts && UndefRHS[M - NumElts])) {
3431 Updated = true;
3432 M = -1;
3433 }
3434 IdentityLHS &= (M < 0) || (M == (int)i);
3435 IdentityRHS &= (M < 0) || ((M - NumElts) == i);
3436 }
3437
3438 // Update legal shuffle masks based on demanded elements if it won't reduce
3439 // to Identity which can cause premature removal of the shuffle mask.
3440 if (Updated && !IdentityLHS && !IdentityRHS && !TLO.LegalOps) {
3441 SDValue LegalShuffle =
3442 buildLegalVectorShuffle(VT, DL, LHS, RHS, NewMask, TLO.DAG);
3443 if (LegalShuffle)
3444 return TLO.CombineTo(Op, LegalShuffle);
3445 }
3446
3447 // Propagate undef/zero elements from LHS/RHS.
3448 for (unsigned i = 0; i != NumElts; ++i) {
3449 int M = ShuffleMask[i];
3450 if (M < 0) {
3451 KnownUndef.setBit(i);
3452 } else if (M < (int)NumElts) {
3453 if (UndefLHS[M])
3454 KnownUndef.setBit(i);
3455 if (ZeroLHS[M])
3456 KnownZero.setBit(i);
3457 } else {
3458 if (UndefRHS[M - NumElts])
3459 KnownUndef.setBit(i);
3460 if (ZeroRHS[M - NumElts])
3461 KnownZero.setBit(i);
3462 }
3463 }
3464 break;
3465 }
3469 APInt SrcUndef, SrcZero;
3470 SDValue Src = Op.getOperand(0);
3471 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3472 APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts);
3473 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
3474 Depth + 1))
3475 return true;
3476 KnownZero = SrcZero.zextOrTrunc(NumElts);
3477 KnownUndef = SrcUndef.zextOrTrunc(NumElts);
3478
3479 if (IsLE && Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG &&
3480 Op.getValueSizeInBits() == Src.getValueSizeInBits() &&
3481 DemandedSrcElts == 1) {
3482 // aext - if we just need the bottom element then we can bitcast.
3483 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
3484 }
3485
3486 if (Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) {
3487 // zext(undef) upper bits are guaranteed to be zero.
3488 if (DemandedElts.isSubsetOf(KnownUndef))
3489 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
3490 KnownUndef.clearAllBits();
3491
3492 // zext - if we just need the bottom element then we can mask:
3493 // zext(and(x,c)) -> and(x,c') iff the zext is the only user of the and.
3494 if (IsLE && DemandedSrcElts == 1 && Src.getOpcode() == ISD::AND &&
3495 Op->isOnlyUserOf(Src.getNode()) &&
3496 Op.getValueSizeInBits() == Src.getValueSizeInBits()) {
3497 SDLoc DL(Op);
3498 EVT SrcVT = Src.getValueType();
3499 EVT SrcSVT = SrcVT.getScalarType();
3500 SmallVector<SDValue> MaskElts;
3501 MaskElts.push_back(TLO.DAG.getAllOnesConstant(DL, SrcSVT));
3502 MaskElts.append(NumSrcElts - 1, TLO.DAG.getConstant(0, DL, SrcSVT));
3503 SDValue Mask = TLO.DAG.getBuildVector(SrcVT, DL, MaskElts);
3504 if (SDValue Fold = TLO.DAG.FoldConstantArithmetic(
3505 ISD::AND, DL, SrcVT, {Src.getOperand(1), Mask})) {
3506 Fold = TLO.DAG.getNode(ISD::AND, DL, SrcVT, Src.getOperand(0), Fold);
3507 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Fold));
3508 }
3509 }
3510 }
3511 break;
3512 }
3513
3514 // TODO: There are more binop opcodes that could be handled here - MIN,
3515 // MAX, saturated math, etc.
3516 case ISD::ADD: {
3517 SDValue Op0 = Op.getOperand(0);
3518 SDValue Op1 = Op.getOperand(1);
3519 if (Op0 == Op1 && Op->isOnlyUserOf(Op0.getNode())) {
3520 APInt UndefLHS, ZeroLHS;
3521 if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
3522 Depth + 1, /*AssumeSingleUse*/ true))
3523 return true;
3524 }
3525 [[fallthrough]];
3526 }
3527 case ISD::OR:
3528 case ISD::XOR:
3529 case ISD::SUB:
3530 case ISD::FADD:
3531 case ISD::FSUB:
3532 case ISD::FMUL:
3533 case ISD::FDIV:
3534 case ISD::FREM: {
3535 SDValue Op0 = Op.getOperand(0);
3536 SDValue Op1 = Op.getOperand(1);
3537
3538 APInt UndefRHS, ZeroRHS;
3539 if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,
3540 Depth + 1))
3541 return true;
3542 APInt UndefLHS, ZeroLHS;
3543 if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
3544 Depth + 1))
3545 return true;
3546
3547 KnownZero = ZeroLHS & ZeroRHS;
3548 KnownUndef = getKnownUndefForVectorBinop(Op, TLO.DAG, UndefLHS, UndefRHS);
3549
3550 // Attempt to avoid multi-use ops if we don't need anything from them.
3551 // TODO - use KnownUndef to relax the demandedelts?
3552 if (!DemandedElts.isAllOnes())
3553 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
3554 return true;
3555 break;
3556 }
3557 case ISD::SHL:
3558 case ISD::SRL:
3559 case ISD::SRA:
3560 case ISD::ROTL:
3561 case ISD::ROTR: {
3562 SDValue Op0 = Op.getOperand(0);
3563 SDValue Op1 = Op.getOperand(1);
3564
3565 APInt UndefRHS, ZeroRHS;
3566 if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,
3567 Depth + 1))
3568 return true;
3569 APInt UndefLHS, ZeroLHS;
3570 if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
3571 Depth + 1))
3572 return true;
3573
3574 KnownZero = ZeroLHS;
3575 KnownUndef = UndefLHS & UndefRHS; // TODO: use getKnownUndefForVectorBinop?
3576
3577 // Attempt to avoid multi-use ops if we don't need anything from them.
3578 // TODO - use KnownUndef to relax the demandedelts?
3579 if (!DemandedElts.isAllOnes())
3580 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
3581 return true;
3582 break;
3583 }
3584 case ISD::MUL:
3585 case ISD::MULHU:
3586 case ISD::MULHS:
3587 case ISD::AND: {
3588 SDValue Op0 = Op.getOperand(0);
3589 SDValue Op1 = Op.getOperand(1);
3590
3591 APInt SrcUndef, SrcZero;
3592 if (SimplifyDemandedVectorElts(Op1, DemandedElts, SrcUndef, SrcZero, TLO,
3593 Depth + 1))
3594 return true;
3595 // If we know that a demanded element was zero in Op1 we don't need to
3596 // demand it in Op0 - its guaranteed to be zero.
3597 APInt DemandedElts0 = DemandedElts & ~SrcZero;
3598 if (SimplifyDemandedVectorElts(Op0, DemandedElts0, KnownUndef, KnownZero,
3599 TLO, Depth + 1))
3600 return true;
3601
3602 KnownUndef &= DemandedElts0;
3603 KnownZero &= DemandedElts0;
3604
3605 // If every element pair has a zero/undef then just fold to zero.
3606 // fold (and x, undef) -> 0 / (and x, 0) -> 0
3607 // fold (mul x, undef) -> 0 / (mul x, 0) -> 0
3608 if (DemandedElts.isSubsetOf(SrcZero | KnownZero | SrcUndef | KnownUndef))
3609 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
3610
3611 // If either side has a zero element, then the result element is zero, even
3612 // if the other is an UNDEF.
3613 // TODO: Extend getKnownUndefForVectorBinop to also deal with known zeros
3614 // and then handle 'and' nodes with the rest of the binop opcodes.
3615 KnownZero |= SrcZero;
3616 KnownUndef &= SrcUndef;
3617 KnownUndef &= ~KnownZero;
3618
3619 // Attempt to avoid multi-use ops if we don't need anything from them.
3620 if (!DemandedElts.isAllOnes())
3621 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
3622 return true;
3623 break;
3624 }
3625 case ISD::TRUNCATE:
3626 case ISD::SIGN_EXTEND:
3627 case ISD::ZERO_EXTEND:
3628 if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,
3629 KnownZero, TLO, Depth + 1))
3630 return true;
3631
3632 if (Op.getOpcode() == ISD::ZERO_EXTEND) {
3633 // zext(undef) upper bits are guaranteed to be zero.
3634 if (DemandedElts.isSubsetOf(KnownUndef))
3635 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
3636 KnownUndef.clearAllBits();
3637 }
3638 break;
3639 default: {
3640 if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
3641 if (SimplifyDemandedVectorEltsForTargetNode(Op, DemandedElts, KnownUndef,
3642 KnownZero, TLO, Depth))
3643 return true;
3644 } else {
3645 KnownBits Known;
3646 APInt DemandedBits = APInt::getAllOnes(EltSizeInBits);
3647 if (SimplifyDemandedBits(Op, DemandedBits, OriginalDemandedElts, Known,
3648 TLO, Depth, AssumeSingleUse))
3649 return true;
3650 }
3651 break;
3652 }
3653 }
3654 assert((KnownUndef & KnownZero) == 0 && "Elements flagged as undef AND zero");
3655
3656 // Constant fold all undef cases.
3657 // TODO: Handle zero cases as well.
3658 if (DemandedElts.isSubsetOf(KnownUndef))
3659 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
3660
3661 return false;
3662}
3663
3664/// Determine which of the bits specified in Mask are known to be either zero or
3665/// one and return them in the Known.
3667 KnownBits &Known,
3668 const APInt &DemandedElts,
3669 const SelectionDAG &DAG,
3670 unsigned Depth) const {
3671 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3672 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3673 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
3674 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
3675 "Should use MaskedValueIsZero if you don't know whether Op"
3676 " is a target node!");
3677 Known.resetAll();
3678}
3679
3682 const APInt &DemandedElts, const MachineRegisterInfo &MRI,
3683 unsigned Depth) const {
3684 Known.resetAll();
3685}
3686
3688 const int FrameIdx, KnownBits &Known, const MachineFunction &MF) const {
3689 // The low bits are known zero if the pointer is aligned.
3690 Known.Zero.setLowBits(Log2(MF.getFrameInfo().getObjectAlign(FrameIdx)));
3691}
3692
3695 unsigned Depth) const {
3696 return Align(1);
3697}
3698
3699/// This method can be implemented by targets that want to expose additional
3700/// information about sign bits to the DAG Combiner.
3702 const APInt &,
3703 const SelectionDAG &,
3704 unsigned Depth) const {
3705 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3706 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3707 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
3708 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
3709 "Should use ComputeNumSignBits if you don't know whether Op"
3710 " is a target node!");
3711 return 1;
3712}
3713
3715 GISelKnownBits &Analysis, Register R, const APInt &DemandedElts,
3716 const MachineRegisterInfo &MRI, unsigned Depth) const {
3717 return 1;
3718}
3719
3721 SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,
3722 TargetLoweringOpt &TLO, unsigned Depth) const {
3723 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3724 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3725 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
3726 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
3727 "Should use SimplifyDemandedVectorElts if you don't know whether Op"
3728 " is a target node!");
3729 return false;
3730}
3731
3733 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
3734 KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const {
3735 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3736 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3737 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
3738 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
3739 "Should use SimplifyDemandedBits if you don't know whether Op"
3740 " is a target node!");
3741 computeKnownBitsForTargetNode(Op, Known, DemandedElts, TLO.DAG, Depth);
3742 return false;
3743}
3744
3746 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
3747 SelectionDAG &DAG, unsigned Depth) const {
3748 assert(
3749 (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3750 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3751 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
3752 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
3753 "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
3754 " is a target node!");
3755 return SDValue();
3756}
3757
3758SDValue
3761 SelectionDAG &DAG) const {
3762 bool LegalMask = isShuffleMaskLegal(Mask, VT);
3763 if (!LegalMask) {
3764 std::swap(N0, N1);
3766 LegalMask = isShuffleMaskLegal(Mask, VT);
3767 }
3768
3769 if (!LegalMask)
3770 return SDValue();
3771
3772 return DAG.getVectorShuffle(VT, DL, N0, N1, Mask);
3773}
3774
3776 return nullptr;
3777}
3778
3780 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
3781 bool PoisonOnly, unsigned Depth) const {
3782 assert(
3783 (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3784 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3785 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
3786 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
3787 "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
3788 " is a target node!");
3789 return false;
3790}
3791
3793 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
3794 bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const {
3795 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3796 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3797 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
3798 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
3799 "Should use canCreateUndefOrPoison if you don't know whether Op"
3800 " is a target node!");
3801 // Be conservative and return true.
3802 return true;
3803}
3804
3806 const SelectionDAG &DAG,
3807 bool SNaN,
3808 unsigned Depth) const {
3809 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3810 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3811 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
3812 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
3813 "Should use isKnownNeverNaN if you don't know whether Op"
3814 " is a target node!");
3815 return false;
3816}
3817
3819 const APInt &DemandedElts,
3820 APInt &UndefElts,
3821 const SelectionDAG &DAG,
3822 unsigned Depth) const {
3823 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3824 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3825 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
3826 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
3827 "Should use isSplatValue if you don't know whether Op"
3828 " is a target node!");
3829 return false;
3830}
3831
3832// FIXME: Ideally, this would use ISD::isConstantSplatVector(), but that must
3833// work with truncating build vectors and vectors with elements of less than
3834// 8 bits.
3836 if (!N)
3837 return false;
3838
3839 unsigned EltWidth;
3840 APInt CVal;
3841 if (ConstantSDNode *CN = isConstOrConstSplat(N, /*AllowUndefs=*/false,
3842 /*AllowTruncation=*/true)) {
3843 CVal = CN->getAPIntValue();
3844 EltWidth = N.getValueType().getScalarSizeInBits();
3845 } else
3846 return false;
3847
3848 // If this is a truncating splat, truncate the splat value.
3849 // Otherwise, we may fail to match the expected values below.
3850 if (EltWidth < CVal.getBitWidth())
3851 CVal = CVal.trunc(EltWidth);
3852
3853 switch (getBooleanContents(N.getValueType())) {
3855 return CVal[0];
3857 return CVal.isOne();
3859 return CVal.isAllOnes();
3860 }
3861
3862 llvm_unreachable("Invalid boolean contents");
3863}
3864
3866 if (!N)
3867 return false;
3868
3869 const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N);
3870 if (!CN) {
3871 const BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N);
3872 if (!BV)
3873 return false;
3874
3875 // Only interested in constant splats, we don't care about undef
3876 // elements in identifying boolean constants and getConstantSplatNode
3877 // returns NULL if all ops are undef;
3878 CN = BV->getConstantSplatNode();
3879 if (!CN)
3880 return false;
3881 }
3882
3883 if (getBooleanContents(N->getValueType(0)) == UndefinedBooleanContent)
3884 return !CN->getAPIntValue()[0];
3885
3886 return CN->isZero();
3887}
3888
3890 bool SExt) const {
3891 if (VT == MVT::i1)
3892 return N->isOne();
3893
3895 switch (Cnt) {
3897 // An extended value of 1 is always true, unless its original type is i1,
3898 // in which case it will be sign extended to -1.
3899 return (N->isOne() && !SExt) || (SExt && (N->getValueType(0) != MVT::i1));
3902 return N->isAllOnes() && SExt;
3903 }
3904 llvm_unreachable("Unexpected enumeration.");
3905}
3906
3907/// This helper function of SimplifySetCC tries to optimize the comparison when
3908/// either operand of the SetCC node is a bitwise-and instruction.
3909SDValue TargetLowering::foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1,
3910 ISD::CondCode Cond, const SDLoc &DL,
3911 DAGCombinerInfo &DCI) const {
3912 if (N1.getOpcode() == ISD::AND && N0.getOpcode() != ISD::AND)
3913 std::swap(N0, N1);
3914
3915 SelectionDAG &DAG = DCI.DAG;
3916 EVT OpVT = N0.getValueType();
3917 if (N0.getOpcode() != ISD::AND || !OpVT.isInteger() ||
3918 (Cond != ISD::SETEQ && Cond != ISD::SETNE))
3919 return SDValue();
3920
3921 // (X & Y) != 0 --> zextOrTrunc(X & Y)
3922 // iff everything but LSB is known zero:
3923 if (Cond == ISD::SETNE && isNullConstant(N1) &&
3926 unsigned NumEltBits = OpVT.getScalarSizeInBits();
3927 APInt UpperBits = APInt::getHighBitsSet(NumEltBits, NumEltBits - 1);
3928 if (DAG.MaskedValueIsZero(N0, UpperBits))
3929 return DAG.getBoolExtOrTrunc(N0, DL, VT, OpVT);
3930 }
3931
3932 // Try to eliminate a power-of-2 mask constant by converting to a signbit
3933 // test in a narrow type that we can truncate to with no cost. Examples:
3934 // (i32 X & 32768) == 0 --> (trunc X to i16) >= 0
3935 // (i32 X & 32768) != 0 --> (trunc X to i16) < 0
3936 // TODO: This conservatively checks for type legality on the source and
3937 // destination types. That may inhibit optimizations, but it also
3938 // allows setcc->shift transforms that may be more beneficial.
3939 auto *AndC = dyn_cast<ConstantSDNode>(N0.getOperand(1));
3940 if (AndC && isNullConstant(N1) && AndC->getAPIntValue().isPowerOf2() &&
3941 isTypeLegal(OpVT) && N0.hasOneUse()) {
3942 EVT NarrowVT = EVT::getIntegerVT(*DAG.getContext(),
3943 AndC->getAPIntValue().getActiveBits());
3944 if (isTruncateFree(OpVT, NarrowVT) && isTypeLegal(NarrowVT)) {
3945 SDValue Trunc = DAG.getZExtOrTrunc(N0.getOperand(0), DL, NarrowVT);
3946 SDValue Zero = DAG.getConstant(0, DL, NarrowVT);
3947 return DAG.getSetCC(DL, VT, Trunc, Zero,
3949 }
3950 }
3951
3952 // Match these patterns in any of their permutations:
3953 // (X & Y) == Y
3954 // (X & Y) != Y
3955 SDValue X, Y;
3956 if (N0.getOperand(0) == N1) {
3957 X = N0.getOperand(1);
3958 Y = N0.getOperand(0);
3959 } else if (N0.getOperand(1) == N1) {
3960 X = N0.getOperand(0);
3961 Y = N0.getOperand(1);
3962 } else {
3963 return SDValue();
3964 }
3965
3966 // TODO: We should invert (X & Y) eq/ne 0 -> (X & Y) ne/eq Y if
3967 // `isXAndYEqZeroPreferableToXAndYEqY` is false. This is a bit difficult as
3968 // its liable to create and infinite loop.
3969 SDValue Zero = DAG.getConstant(0, DL, OpVT);
3970 if (isXAndYEqZeroPreferableToXAndYEqY(Cond, OpVT) &&
3972 // Simplify X & Y == Y to X & Y != 0 if Y has exactly one bit set.
3973 // Note that where Y is variable and is known to have at most one bit set
3974 // (for example, if it is Z & 1) we cannot do this; the expressions are not
3975 // equivalent when Y == 0.
3976 assert(OpVT.isInteger());
3978 if (DCI.isBeforeLegalizeOps() ||
3980 return DAG.getSetCC(DL, VT, N0, Zero, Cond);
3981 } else if (N0.hasOneUse() && hasAndNotCompare(Y)) {
3982 // If the target supports an 'and-not' or 'and-complement' logic operation,
3983 // try to use that to make a comparison operation more efficient.
3984 // But don't do this transform if the mask is a single bit because there are
3985 // more efficient ways to deal with that case (for example, 'bt' on x86 or
3986 // 'rlwinm' on PPC).
3987
3988 // Bail out if the compare operand that we want to turn into a zero is
3989 // already a zero (otherwise, infinite loop).
3990 if (isNullConstant(Y))
3991 return SDValue();
3992
3993 // Transform this into: ~X & Y == 0.
3994 SDValue NotX = DAG.getNOT(SDLoc(X), X, OpVT);
3995 SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, NotX, Y);
3996 return DAG.getSetCC(DL, VT, NewAnd, Zero, Cond);
3997 }
3998
3999 return SDValue();
4000}
4001
4002/// There are multiple IR patterns that could be checking whether certain
4003/// truncation of a signed number would be lossy or not. The pattern which is
4004/// best at IR level, may not lower optimally. Thus, we want to unfold it.
4005/// We are looking for the following pattern: (KeptBits is a constant)
4006/// (add %x, (1 << (KeptBits-1))) srccond (1 << KeptBits)
4007/// KeptBits won't be bitwidth(x), that will be constant-folded to true/false.
4008/// KeptBits also can't be 1, that would have been folded to %x dstcond 0
4009/// We will unfold it into the natural trunc+sext pattern:
4010/// ((%x << C) a>> C) dstcond %x
4011/// Where C = bitwidth(x) - KeptBits and C u< bitwidth(x)
4012SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(
4013 EVT SCCVT, SDValue N0, SDValue N1, ISD::CondCode Cond, DAGCombinerInfo &DCI,
4014 const SDLoc &DL) const {
4015 // We must be comparing with a constant.
4016 ConstantSDNode *C1;
4017 if (!(C1 = dyn_cast<ConstantSDNode>(N1)))
4018 return SDValue();
4019
4020 // N0 should be: add %x, (1 << (KeptBits-1))
4021 if (N0->getOpcode() != ISD::ADD)
4022 return SDValue();
4023
4024 // And we must be 'add'ing a constant.
4025 ConstantSDNode *C01;
4026 if (!(C01 = dyn_cast<ConstantSDNode>(N0->getOperand(1))))
4027 return SDValue();
4028
4029 SDValue X = N0->getOperand(0);
4030 EVT XVT = X.getValueType();
4031
4032 // Validate constants ...
4033
4034 APInt I1 = C1->getAPIntValue();
4035
4036 ISD::CondCode NewCond;
4037 if (Cond == ISD::CondCode::SETULT) {
4038 NewCond = ISD::CondCode::SETEQ;
4039 } else if (Cond == ISD::CondCode::SETULE) {
4040 NewCond = ISD::CondCode::SETEQ;
4041 // But need to 'canonicalize' the constant.
4042 I1 += 1;
4043 } else if (Cond == ISD::CondCode::SETUGT) {
4044 NewCond = ISD::CondCode::SETNE;
4045 // But need to 'canonicalize' the constant.
4046 I1 += 1;
4047 } else if (Cond == ISD::CondCode::SETUGE) {
4048 NewCond = ISD::CondCode::SETNE;
4049 } else
4050 return SDValue();
4051
4052 APInt I01 = C01->getAPIntValue();
4053
4054 auto checkConstants = [&I1, &I01]() -> bool {
4055 // Both of them must be power-of-two, and the constant from setcc is bigger.
4056 return I1.ugt(I01) && I1.isPowerOf2() && I01.isPowerOf2();
4057 };
4058
4059 if (checkConstants()) {
4060 // Great, e.g. got icmp ult i16 (add i16 %x, 128), 256
4061 } else {
4062 // What if we invert constants? (and the target predicate)
4063 I1.negate();
4064 I01.negate();
4065 assert(XVT.isInteger());
4066 NewCond = getSetCCInverse(NewCond, XVT);
4067 if (!checkConstants())
4068 return SDValue();
4069 // Great, e.g. got icmp uge i16 (add i16 %x, -128), -256
4070 }
4071
4072 // They are power-of-two, so which bit is set?
4073 const unsigned KeptBits = I1.logBase2();
4074 const unsigned KeptBitsMinusOne = I01.logBase2();
4075
4076 // Magic!
4077 if (KeptBits != (KeptBitsMinusOne + 1))
4078 return SDValue();
4079 assert(KeptBits > 0 && KeptBits < XVT.getSizeInBits() && "unreachable");
4080
4081 // We don't want to do this in every single case.
4082 SelectionDAG &DAG = DCI.DAG;
4084 XVT, KeptBits))
4085 return SDValue();
4086
4087 // Unfold into: sext_inreg(%x) cond %x
4088 // Where 'cond' will be either 'eq' or 'ne'.
4089 SDValue SExtInReg = DAG.getNode(
4091 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), KeptBits)));
4092 return DAG.getSetCC(DL, SCCVT, SExtInReg, X, NewCond);
4093}
4094
4095// (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
4096SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(
4097 EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond,
4098 DAGCombinerInfo &DCI, const SDLoc &DL) const {
4100 "Should be a comparison with 0.");
4101 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4102 "Valid only for [in]equality comparisons.");
4103
4104 unsigned NewShiftOpcode;
4105 SDValue X, C, Y;
4106
4107 SelectionDAG &DAG = DCI.DAG;
4108 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4109
4110 // Look for '(C l>>/<< Y)'.
4111 auto Match = [&NewShiftOpcode, &X, &C, &Y, &TLI, &DAG](SDValue V) {
4112 // The shift should be one-use.
4113 if (!V.hasOneUse())
4114 return false;
4115 unsigned OldShiftOpcode = V.getOpcode();
4116 switch (OldShiftOpcode) {
4117 case ISD::SHL:
4118 NewShiftOpcode = ISD::SRL;
4119 break;
4120 case ISD::SRL:
4121 NewShiftOpcode = ISD::SHL;
4122 break;
4123 default:
4124 return false; // must be a logical shift.
4125 }
4126 // We should be shifting a constant.
4127 // FIXME: best to use isConstantOrConstantVector().
4128 C = V.getOperand(0);
4130 isConstOrConstSplat(C, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
4131 if (!CC)
4132 return false;
4133 Y = V.getOperand(1);
4134
4136 isConstOrConstSplat(X, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
4137 return TLI.shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
4138 X, XC, CC, Y, OldShiftOpcode, NewShiftOpcode, DAG);
4139 };
4140
4141 // LHS of comparison should be an one-use 'and'.
4142 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())
4143 return SDValue();
4144
4145 X = N0.getOperand(0);
4146 SDValue Mask = N0.getOperand(1);
4147
4148 // 'and' is commutative!
4149 if (!Match(Mask)) {
4150 std::swap(X, Mask);
4151 if (!Match(Mask))
4152 return SDValue();
4153 }
4154
4155 EVT VT = X.getValueType();
4156
4157 // Produce:
4158 // ((X 'OppositeShiftOpcode' Y) & C) Cond 0
4159 SDValue T0 = DAG.getNode(NewShiftOpcode, DL, VT, X, Y);
4160 SDValue T1 = DAG.getNode(ISD::AND, DL, VT, T0, C);
4161 SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, N1C, Cond);
4162 return T2;
4163}
4164
4165/// Try to fold an equality comparison with a {add/sub/xor} binary operation as
4166/// the 1st operand (N0). Callers are expected to swap the N0/N1 parameters to
4167/// handle the commuted versions of these patterns.
4168SDValue TargetLowering::foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1,
4169 ISD::CondCode Cond, const SDLoc &DL,
4170 DAGCombinerInfo &DCI) const {
4171 unsigned BOpcode = N0.getOpcode();
4172 assert((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) &&
4173 "Unexpected binop");
4174 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Unexpected condcode");
4175
4176 // (X + Y) == X --> Y == 0
4177 // (X - Y) == X --> Y == 0
4178 // (X ^ Y) == X --> Y == 0
4179 SelectionDAG &DAG = DCI.DAG;
4180 EVT OpVT = N0.getValueType();
4181 SDValue X = N0.getOperand(0);
4182 SDValue Y = N0.getOperand(1);
4183 if (X == N1)
4184 return DAG.getSetCC(DL, VT, Y, DAG.getConstant(0, DL, OpVT), Cond);
4185
4186 if (Y != N1)
4187 return SDValue();
4188
4189 // (X + Y) == Y --> X == 0
4190 // (X ^ Y) == Y --> X == 0
4191 if (BOpcode == ISD::ADD || BOpcode == ISD::XOR)
4192 return DAG.getSetCC(DL, VT, X, DAG.getConstant(0, DL, OpVT), Cond);
4193
4194 // The shift would not be valid if the operands are boolean (i1).
4195 if (!N0.hasOneUse() || OpVT.getScalarSizeInBits() == 1)
4196 return SDValue();
4197
4198 // (X - Y) == Y --> X == Y << 1
4199 SDValue One =
4200 DAG.getShiftAmountConstant(1, OpVT, DL, !DCI.isBeforeLegalize());
4201 SDValue YShl1 = DAG.getNode(ISD::SHL, DL, N1.getValueType(), Y, One);
4202 if (!DCI.isCalledByLegalizer())
4203 DCI.AddToWorklist(YShl1.getNode());
4204 return DAG.getSetCC(DL, VT, X, YShl1, Cond);
4205}
4206
4208 SDValue N0, const APInt &C1,
4209 ISD::CondCode Cond, const SDLoc &dl,
4210 SelectionDAG &DAG) {
4211 // Look through truncs that don't change the value of a ctpop.
4212 // FIXME: Add vector support? Need to be careful with setcc result type below.
4213 SDValue CTPOP = N0;
4214 if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && !VT.isVector() &&
4216 CTPOP = N0.getOperand(0);
4217
4218 if (CTPOP.getOpcode() != ISD::CTPOP || !CTPOP.hasOneUse())
4219 return SDValue();
4220
4221 EVT CTVT = CTPOP.getValueType();
4222 SDValue CTOp = CTPOP.getOperand(0);
4223
4224 // Expand a power-of-2-or-zero comparison based on ctpop:
4225 // (ctpop x) u< 2 -> (x & x-1) == 0
4226 // (ctpop x) u> 1 -> (x & x-1) != 0
4227 if (Cond == ISD::SETULT || Cond == ISD::SETUGT) {
4228 // Keep the CTPOP if it is a cheap vector op.
4229 if (CTVT.isVector() && TLI.isCtpopFast(CTVT))
4230 return SDValue();
4231
4232 unsigned CostLimit = TLI.getCustomCtpopCost(CTVT, Cond);
4233 if (C1.ugt(CostLimit + (Cond == ISD::SETULT)))
4234 return SDValue();
4235 if (C1 == 0 && (Cond == ISD::SETULT))
4236 return SDValue(); // This is handled elsewhere.
4237
4238 unsigned Passes = C1.getLimitedValue() - (Cond == ISD::SETULT);
4239
4240 SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
4241 SDValue Result = CTOp;
4242 for (unsigned i = 0; i < Passes; i++) {
4243 SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, Result, NegOne);
4244 Result = DAG.getNode(ISD::AND, dl, CTVT, Result, Add);
4245 }
4247 return DAG.getSetCC(dl, VT, Result, DAG.getConstant(0, dl, CTVT), CC);
4248 }
4249
4250 // Expand a power-of-2 comparison based on ctpop
4251 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && C1 == 1) {
4252 // Keep the CTPOP if it is cheap.
4253 if (TLI.isCtpopFast(CTVT))
4254 return SDValue();
4255
4256 SDValue Zero = DAG.getConstant(0, dl, CTVT);
4257 SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
4258 assert(CTVT.isInteger());
4259 SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, CTOp, NegOne);
4260
4261 // Its not uncommon for known-never-zero X to exist in (ctpop X) eq/ne 1, so
4262 // check before emitting a potentially unnecessary op.
4263 if (DAG.isKnownNeverZero(CTOp)) {
4264 // (ctpop x) == 1 --> (x & x-1) == 0
4265 // (ctpop x) != 1 --> (x & x-1) != 0
4266 SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Add);
4267 SDValue RHS = DAG.getSetCC(dl, VT, And, Zero, Cond);
4268 return RHS;
4269 }
4270
4271 // (ctpop x) == 1 --> (x ^ x-1) > x-1
4272 // (ctpop x) != 1 --> (x ^ x-1) <= x-1
4273 SDValue Xor = DAG.getNode(ISD::XOR, dl, CTVT, CTOp, Add);
4275 return DAG.getSetCC(dl, VT, Xor, Add, CmpCond);
4276 }
4277
4278 return SDValue();
4279}
4280
4282 ISD::CondCode Cond, const SDLoc &dl,
4283 SelectionDAG &DAG) {
4284 if (Cond != ISD::SETEQ && Cond != ISD::SETNE)
4285 return SDValue();
4286
4287 auto *C1 = isConstOrConstSplat(N1, /* AllowUndefs */ true);
4288 if (!C1 || !(C1->isZero() || C1->isAllOnes()))
4289 return SDValue();
4290
4291 auto getRotateSource = [](SDValue X) {
4292 if (X.getOpcode() == ISD::ROTL || X.getOpcode() == ISD::ROTR)
4293 return X.getOperand(0);
4294 return SDValue();
4295 };
4296
4297 // Peek through a rotated value compared against 0 or -1:
4298 // (rot X, Y) == 0/-1 --> X == 0/-1
4299 // (rot X, Y) != 0/-1 --> X != 0/-1
4300 if (SDValue R = getRotateSource(N0))
4301 return DAG.getSetCC(dl, VT, R, N1, Cond);
4302
4303 // Peek through an 'or' of a rotated value compared against 0:
4304 // or (rot X, Y), Z ==/!= 0 --> (or X, Z) ==/!= 0
4305 // or Z, (rot X, Y) ==/!= 0 --> (or X, Z) ==/!= 0
4306 //
4307 // TODO: Add the 'and' with -1 sibling.
4308 // TODO: Recurse through a series of 'or' ops to find the rotate.
4309 EVT OpVT = N0.getValueType();
4310 if (N0.hasOneUse() && N0.getOpcode() == ISD::OR && C1->isZero()) {
4311 if (SDValue R = getRotateSource(N0.getOperand(0))) {
4312 SDValue NewOr = DAG.getNode(ISD::OR, dl, OpVT, R, N0.getOperand(1));
4313 return DAG.getSetCC(dl, VT, NewOr, N1, Cond);
4314 }
4315 if (SDValue R = getRotateSource(N0.getOperand(1))) {
4316 SDValue NewOr = DAG.getNode(ISD::OR, dl, OpVT, R, N0.getOperand(0));
4317 return DAG.getSetCC(dl, VT, NewOr, N1, Cond);
4318 }
4319 }
4320
4321 return SDValue();
4322}
4323
4325 ISD::CondCode Cond, const SDLoc &dl,
4326 SelectionDAG &DAG) {
4327 // If we are testing for all-bits-clear, we might be able to do that with
4328 // less shifting since bit-order does not matter.
4329 if (Cond != ISD::SETEQ && Cond != ISD::SETNE)
4330 return SDValue();
4331
4332 auto *C1 = isConstOrConstSplat(N1, /* AllowUndefs */ true);
4333 if (!C1 || !C1->isZero())
4334 return SDValue();
4335
4336 if (!N0.hasOneUse() ||
4337 (N0.getOpcode() != ISD::FSHL && N0.getOpcode() != ISD::FSHR))
4338 return SDValue();
4339
4340 unsigned BitWidth = N0.getScalarValueSizeInBits();
4341 auto *ShAmtC = isConstOrConstSplat(N0.getOperand(2));
4342 if (!ShAmtC || ShAmtC->getAPIntValue().uge(BitWidth))
4343 return SDValue();
4344
4345 // Canonicalize fshr as fshl to reduce pattern-matching.
4346 unsigned ShAmt = ShAmtC->getZExtValue();
4347 if (N0.getOpcode() == ISD::FSHR)
4348 ShAmt = BitWidth - ShAmt;
4349
4350 // Match an 'or' with a specific operand 'Other' in either commuted variant.
4351 SDValue X, Y;
4352 auto matchOr = [&X, &Y](SDValue Or, SDValue Other) {
4353 if (Or.getOpcode() != ISD::OR || !Or.hasOneUse())
4354 return false;
4355 if (Or.getOperand(0) == Other) {
4356 X = Or.getOperand(0);
4357 Y = Or.getOperand(1);
4358 return true;
4359 }
4360 if (Or.getOperand(1) == Other) {
4361 X = Or.getOperand(1);
4362 Y = Or.getOperand(0);
4363 return true;
4364 }
4365 return false;
4366 };
4367
4368 EVT OpVT = N0.getValueType();
4369 EVT ShAmtVT = N0.getOperand(2).getValueType();
4370 SDValue F0 = N0.getOperand(0);
4371 SDValue F1 = N0.getOperand(1);
4372 if (matchOr(F0, F1)) {
4373 // fshl (or X, Y), X, C ==/!= 0 --> or (shl Y, C), X ==/!= 0
4374 SDValue NewShAmt = DAG.getConstant(ShAmt, dl, ShAmtVT);
4375 SDValue Shift = DAG.getNode(ISD::SHL, dl, OpVT, Y, NewShAmt);
4376 SDValue NewOr = DAG.getNode(ISD::OR, dl, OpVT, Shift, X);
4377 return DAG.getSetCC(dl, VT, NewOr, N1, Cond);
4378 }
4379 if (matchOr(F1, F0)) {
4380 // fshl X, (or X, Y), C ==/!= 0 --> or (srl Y, BW-C), X ==/!= 0
4381 SDValue NewShAmt = DAG.getConstant(BitWidth - ShAmt, dl, ShAmtVT);
4382 SDValue Shift = DAG.getNode(ISD::SRL, dl, OpVT, Y, NewShAmt);
4383 SDValue NewOr = DAG.getNode(ISD::OR, dl, OpVT, Shift, X);
4384 return DAG.getSetCC(dl, VT, NewOr, N1, Cond);
4385 }
4386
4387 return SDValue();
4388}
4389
4390/// Try to simplify a setcc built with the specified operands and cc. If it is
4391/// unable to simplify it, return a null SDValue.
4393 ISD::CondCode Cond, bool foldBooleans,
4394 DAGCombinerInfo &DCI,
4395 const SDLoc &dl) const {
4396 SelectionDAG &DAG = DCI.DAG;
4397 const DataLayout &Layout = DAG.getDataLayout();
4398 EVT OpVT = N0.getValueType();
4400
4401 // Constant fold or commute setcc.
4402 if (SDValue Fold = DAG.FoldSetCC(VT, N0, N1, Cond, dl))
4403 return Fold;
4404
4405 bool N0ConstOrSplat =
4406 isConstOrConstSplat(N0, /*AllowUndefs*/ false, /*AllowTruncate*/ true);
4407 bool N1ConstOrSplat =
4408 isConstOrConstSplat(N1, /*AllowUndefs*/ false, /*AllowTruncate*/ true);
4409
4410 // Canonicalize toward having the constant on the RHS.
4411 // TODO: Handle non-splat vector constants. All undef causes trouble.
4412 // FIXME: We can't yet fold constant scalable vector splats, so avoid an
4413 // infinite loop here when we encounter one.
4415 if (N0ConstOrSplat && !N1ConstOrSplat &&
4416 (DCI.isBeforeLegalizeOps() ||
4417 isCondCodeLegal(SwappedCC, N0.getSimpleValueType())))
4418 return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);
4419
4420 // If we have a subtract with the same 2 non-constant operands as this setcc
4421 // -- but in reverse order -- then try to commute the operands of this setcc
4422 // to match. A matching pair of setcc (cmp) and sub may be combined into 1
4423 // instruction on some targets.
4424 if (!N0ConstOrSplat && !N1ConstOrSplat &&
4425 (DCI.isBeforeLegalizeOps() ||
4426 isCondCodeLegal(SwappedCC, N0.getSimpleValueType())) &&
4427 DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N1, N0}) &&
4428 !DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N0, N1}))
4429 return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);
4430
4431 if (SDValue V = foldSetCCWithRotate(VT, N0, N1, Cond, dl, DAG))
4432 return V;
4433
4434 if (SDValue V = foldSetCCWithFunnelShift(VT, N0, N1, Cond, dl, DAG))
4435 return V;
4436
4437 if (auto *N1C = isConstOrConstSplat(N1)) {
4438 const APInt &C1 = N1C->getAPIntValue();
4439
4440 // Optimize some CTPOP cases.
4441 if (SDValue V = simplifySetCCWithCTPOP(*this, VT, N0, C1, Cond, dl, DAG))
4442 return V;
4443
4444 // For equality to 0 of a no-wrap multiply, decompose and test each op:
4445 // X * Y == 0 --> (X == 0) || (Y == 0)
4446 // X * Y != 0 --> (X != 0) && (Y != 0)
4447 // TODO: This bails out if minsize is set, but if the target doesn't have a
4448 // single instruction multiply for this type, it would likely be
4449 // smaller to decompose.
4450 if (C1.isZero() && (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4451 N0.getOpcode() == ISD::MUL && N0.hasOneUse() &&
4452 (N0->getFlags().hasNoUnsignedWrap() ||
4453 N0->getFlags().hasNoSignedWrap()) &&
4454 !Attr.hasFnAttr(Attribute::MinSize)) {
4455 SDValue IsXZero = DAG.getSetCC(dl, VT, N0.getOperand(0), N1, Cond);
4456 SDValue IsYZero = DAG.getSetCC(dl, VT, N0.getOperand(1), N1, Cond);
4457 unsigned LogicOp = Cond == ISD::SETEQ ? ISD::OR : ISD::AND;
4458 return DAG.getNode(LogicOp, dl, VT, IsXZero, IsYZero);
4459 }
4460
4461 // If the LHS is '(srl (ctlz x), 5)', the RHS is 0/1, and this is an
4462 // equality comparison, then we're just comparing whether X itself is
4463 // zero.
4464 if (N0.getOpcode() == ISD::SRL && (C1.isZero() || C1.isOne()) &&
4465 N0.getOperand(0).getOpcode() == ISD::CTLZ &&
4466 llvm::has_single_bit<uint32_t>(N0.getScalarValueSizeInBits())) {
4467 if (ConstantSDNode *ShAmt = isConstOrConstSplat(N0.getOperand(1))) {
4468 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4469 ShAmt->getAPIntValue() == Log2_32(N0.getScalarValueSizeInBits())) {
4470 if ((C1 == 0) == (Cond == ISD::SETEQ)) {
4471 // (srl (ctlz x), 5) == 0 -> X != 0
4472 // (srl (ctlz x), 5) != 1 -> X != 0
4473 Cond = ISD::SETNE;
4474 } else {
4475 // (srl (ctlz x), 5) != 0 -> X == 0
4476 // (srl (ctlz x), 5) == 1 -> X == 0
4477 Cond = ISD::SETEQ;
4478 }
4479 SDValue Zero = DAG.getConstant(0, dl, N0.getValueType());
4480 return DAG.getSetCC(dl, VT, N0.getOperand(0).getOperand(0), Zero,
4481 Cond);
4482 }
4483 }
4484 }
4485 }
4486
4487 // FIXME: Support vectors.
4488 if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
4489 const APInt &C1 = N1C->getAPIntValue();
4490
4491 // (zext x) == C --> x == (trunc C)
4492 // (sext x) == C --> x == (trunc C)
4493 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4494 DCI.isBeforeLegalize() && N0->hasOneUse()) {
4495 unsigned MinBits = N0.getValueSizeInBits();
4496 SDValue PreExt;
4497 bool Signed = false;
4498 if (N0->getOpcode() == ISD::ZERO_EXTEND) {
4499 // ZExt
4500 MinBits = N0->getOperand(0).getValueSizeInBits();
4501 PreExt = N0->getOperand(0);
4502 } else if (N0->getOpcode() == ISD::AND) {
4503 // DAGCombine turns costly ZExts into ANDs
4504 if (auto *C = dyn_cast<ConstantSDNode>(N0->getOperand(1)))
4505 if ((C->getAPIntValue()+1).isPowerOf2()) {
4506 MinBits = C->getAPIntValue().countr_one();
4507 PreExt = N0->getOperand(0);
4508 }
4509 } else if (N0->getOpcode() == ISD::SIGN_EXTEND) {
4510 // SExt
4511 MinBits = N0->getOperand(0).getValueSizeInBits();
4512 PreExt = N0->getOperand(0);
4513 Signed = true;
4514 } else if (auto *LN0 = dyn_cast<LoadSDNode>(N0)) {
4515 // ZEXTLOAD / SEXTLOAD
4516 if (LN0->getExtensionType() == ISD::ZEXTLOAD) {
4517 MinBits = LN0->getMemoryVT().getSizeInBits();
4518 PreExt = N0;
4519 } else if (LN0->getExtensionType() == ISD::SEXTLOAD) {
4520 Signed = true;
4521 MinBits = LN0->getMemoryVT().getSizeInBits();
4522 PreExt = N0;
4523 }
4524 }
4525
4526 // Figure out how many bits we need to preserve this constant.
4527 unsigned ReqdBits = Signed ? C1.getSignificantBits() : C1.getActiveBits();
4528
4529 // Make sure we're not losing bits from the constant.
4530 if (MinBits > 0 &&
4531 MinBits < C1.getBitWidth() &&
4532 MinBits >= ReqdBits) {
4533 EVT MinVT = EVT::getIntegerVT(*DAG.getContext(), MinBits);
4534 if (isTypeDesirableForOp(ISD::SETCC, MinVT)) {
4535 // Will get folded away.
4536 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MinVT, PreExt);
4537 if (MinBits == 1 && C1 == 1)
4538 // Invert the condition.
4539 return DAG.getSetCC(dl, VT, Trunc, DAG.getConstant(0, dl, MVT::i1),
4541 SDValue C = DAG.getConstant(C1.trunc(MinBits), dl, MinVT);
4542 return DAG.getSetCC(dl, VT, Trunc, C, Cond);
4543 }
4544
4545 // If truncating the setcc operands is not desirable, we can still
4546 // simplify the expression in some cases:
4547 // setcc ([sz]ext (setcc x, y, cc)), 0, setne) -> setcc (x, y, cc)
4548 // setcc ([sz]ext (setcc x, y, cc)), 0, seteq) -> setcc (x, y, inv(cc))
4549 // setcc (zext (setcc x, y, cc)), 1, setne) -> setcc (x, y, inv(cc))
4550 // setcc (zext (setcc x, y, cc)), 1, seteq) -> setcc (x, y, cc)
4551 // setcc (sext (setcc x, y, cc)), -1, setne) -> setcc (x, y, inv(cc))
4552 // setcc (sext (setcc x, y, cc)), -1, seteq) -> setcc (x, y, cc)
4553 SDValue TopSetCC = N0->getOperand(0);
4554 unsigned N0Opc = N0->getOpcode();
4555 bool SExt = (N0Opc == ISD::SIGN_EXTEND);
4556 if (TopSetCC.getValueType() == MVT::i1 && VT == MVT::i1 &&
4557 TopSetCC.getOpcode() == ISD::SETCC &&
4558 (N0Opc == ISD::ZERO_EXTEND || N0Opc == ISD::SIGN_EXTEND) &&
4559 (isConstFalseVal(N1) ||
4560 isExtendedTrueVal(N1C, N0->getValueType(0), SExt))) {
4561
4562 bool Inverse = (N1C->isZero() && Cond == ISD::SETEQ) ||
4563 (!N1C->isZero() && Cond == ISD::SETNE);
4564
4565 if (!Inverse)
4566 return TopSetCC;
4567
4569 cast<CondCodeSDNode>(TopSetCC.getOperand(2))->get(),
4570 TopSetCC.getOperand(0).getValueType());
4571 return DAG.getSetCC(dl, VT, TopSetCC.getOperand(0),
4572 TopSetCC.getOperand(1),
4573 InvCond);
4574 }
4575 }
4576 }
4577
4578 // If the LHS is '(and load, const)', the RHS is 0, the test is for
4579 // equality or unsigned, and all 1 bits of the const are in the same
4580 // partial word, see if we can shorten the load.
4581 if (DCI.isBeforeLegalize() &&
4583 N0.getOpcode() == ISD::AND && C1 == 0 &&
4584 N0.getNode()->hasOneUse() &&
4585 isa<LoadSDNode>(N0.getOperand(0)) &&
4586 N0.getOperand(0).getNode()->hasOneUse() &&
4587 isa<ConstantSDNode>(N0.getOperand(1))) {
4588 LoadSDNode *Lod = cast<LoadSDNode>(N0.getOperand(0));
4589 APInt bestMask;
4590 unsigned bestWidth = 0, bestOffset = 0;
4591 if (Lod->isSimple() && Lod->isUnindexed()) {
4592 unsigned origWidth = N0.getValueSizeInBits();
4593 unsigned maskWidth = origWidth;
4594 // We can narrow (e.g.) 16-bit extending loads on 32-bit target to
4595 // 8 bits, but have to be careful...
4596 if (Lod->getExtensionType() != ISD::NON_EXTLOAD)
4597 origWidth = Lod->getMemoryVT().getSizeInBits();
4598 const APInt &Mask = N0.getConstantOperandAPInt(1);
4599 for (unsigned width = origWidth / 2; width>=8; width /= 2) {
4600 APInt newMask = APInt::getLowBitsSet(maskWidth, width);
4601 for (unsigned offset=0; offset<origWidth/width; offset++) {
4602 if (Mask.isSubsetOf(newMask)) {
4603 if (Layout.isLittleEndian())
4604 bestOffset = (uint64_t)offset * (width/8);
4605 else
4606 bestOffset = (origWidth/width - offset - 1) * (width/8);
4607 bestMask = Mask.lshr(offset * (width/8) * 8);
4608 bestWidth = width;
4609 break;
4610 }
4611 newMask <<= width;
4612 }
4613 }
4614 }
4615 if (bestWidth) {
4616 EVT newVT = EVT::getIntegerVT(*DAG.getContext(), bestWidth);
4617 if (newVT.isRound() &&
4619 SDValue Ptr = Lod->getBasePtr();
4620 if (bestOffset != 0)
4622 dl);
4623 SDValue NewLoad =
4624 DAG.getLoad(newVT, dl, Lod->getChain(), Ptr,
4625 Lod->getPointerInfo().getWithOffset(bestOffset),
4626 Lod->getOriginalAlign());
4627 return DAG.getSetCC(dl, VT,
4628 DAG.getNode(ISD::AND, dl, newVT, NewLoad,
4629 DAG.getConstant(bestMask.trunc(bestWidth),
4630 dl, newVT)),
4631 DAG.getConstant(0LL, dl, newVT), Cond);
4632 }
4633 }
4634 }
4635
4636 // If the LHS is a ZERO_EXTEND, perform the comparison on the input.
4637 if (N0.getOpcode() == ISD::ZERO_EXTEND) {
4638 unsigned InSize = N0.getOperand(0).getValueSizeInBits();
4639
4640 // If the comparison constant has bits in the upper part, the
4641 // zero-extended value could never match.
4643 C1.getBitWidth() - InSize))) {
4644 switch (Cond) {
4645 case ISD::SETUGT:
4646 case ISD::SETUGE:
4647 case ISD::SETEQ:
4648 return DAG.getConstant(0, dl, VT);
4649 case ISD::SETULT:
4650 case ISD::SETULE:
4651 case ISD::SETNE:
4652 return DAG.getConstant(1, dl, VT);
4653 case ISD::SETGT:
4654 case ISD::SETGE:
4655 // True if the sign bit of C1 is set.
4656 return DAG.getConstant(C1.isNegative(), dl, VT);
4657 case ISD::SETLT:
4658 case ISD::SETLE:
4659 // True if the sign bit of C1 isn't set.
4660 return DAG.getConstant(C1.isNonNegative(), dl, VT);
4661 default:
4662 break;
4663 }
4664 }
4665
4666 // Otherwise, we can perform the comparison with the low bits.
4667 switch (Cond) {
4668 case ISD::SETEQ:
4669 case ISD::SETNE:
4670 case ISD::SETUGT:
4671 case ISD::SETUGE:
4672 case ISD::SETULT:
4673 case ISD::SETULE: {
4674 EVT newVT = N0.getOperand(0).getValueType();
4675 if (DCI.isBeforeLegalizeOps() ||
4676 (isOperationLegal(ISD::SETCC, newVT) &&
4677 isCondCodeLegal(Cond, newVT.getSimpleVT()))) {
4678 EVT NewSetCCVT = getSetCCResultType(Layout, *DAG.getContext(), newVT);
4679 SDValue NewConst = DAG.getConstant(C1.trunc(InSize), dl, newVT);
4680
4681 SDValue NewSetCC = DAG.getSetCC(dl, NewSetCCVT, N0.getOperand(0),
4682 NewConst, Cond);
4683 return DAG.getBoolExtOrTrunc(NewSetCC, dl, VT, N0.getValueType());
4684 }
4685 break;
4686 }
4687 default:
4688 break; // todo, be more careful with signed comparisons
4689 }
4690 } else if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
4691 (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4692 !isSExtCheaperThanZExt(cast<VTSDNode>(N0.getOperand(1))->getVT(),
4693 OpVT)) {
4694 EVT ExtSrcTy = cast<VTSDNode>(N0.getOperand(1))->getVT();
4695 unsigned ExtSrcTyBits = ExtSrcTy.getSizeInBits();
4696 EVT ExtDstTy = N0.getValueType();
4697 unsigned ExtDstTyBits = ExtDstTy.getSizeInBits();
4698
4699 // If the constant doesn't fit into the number of bits for the source of
4700 // the sign extension, it is impossible for both sides to be equal.
4701 if (C1.getSignificantBits() > ExtSrcTyBits)
4702 return DAG.getBoolConstant(Cond == ISD::SETNE, dl, VT, OpVT);
4703
4704 assert(ExtDstTy == N0.getOperand(0).getValueType() &&
4705 ExtDstTy != ExtSrcTy && "Unexpected types!");
4706 APInt Imm = APInt::getLowBitsSet(ExtDstTyBits, ExtSrcTyBits);
4707 SDValue ZextOp = DAG.getNode(ISD::AND, dl, ExtDstTy, N0.getOperand(0),
4708 DAG.getConstant(Imm, dl, ExtDstTy));
4709 if (!DCI.isCalledByLegalizer())
4710 DCI.AddToWorklist(ZextOp.getNode());
4711 // Otherwise, make this a use of a zext.
4712 return DAG.getSetCC(dl, VT, ZextOp,
4713 DAG.getConstant(C1 & Imm, dl, ExtDstTy), Cond);
4714 } else if ((N1C->isZero() || N1C->isOne()) &&
4715 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
4716 // SETCC (X), [0|1], [EQ|NE] -> X if X is known 0/1. i1 types are
4717 // excluded as they are handled below whilst checking for foldBooleans.
4718 if ((N0.getOpcode() == ISD::SETCC || VT.getScalarType() != MVT::i1) &&
4719 isTypeLegal(VT) && VT.bitsLE(N0.getValueType()) &&
4720 (N0.getValueType() == MVT::i1 ||
4724 bool TrueWhenTrue = (Cond == ISD::SETEQ) ^ (!N1C->isOne());
4725 if (TrueWhenTrue)
4726 return DAG.getNode(ISD::TRUNCATE, dl, VT, N0);
4727 // Invert the condition.
4728 if (N0.getOpcode() == ISD::SETCC) {
4729 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
4731 if (DCI.isBeforeLegalizeOps() ||
4733 return DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC);
4734 }
4735 }
4736
4737 if ((N0.getOpcode() == ISD::XOR ||
4738 (N0.getOpcode() == ISD::AND &&
4739 N0.getOperand(0).getOpcode() == ISD::XOR &&
4740 N0.getOperand(1) == N0.getOperand(0).getOperand(1))) &&
4741 isOneConstant(N0.getOperand(1))) {
4742 // If this is (X^1) == 0/1, swap the RHS and eliminate the xor. We
4743 // can only do this if the top bits are known zero.
4744 unsigned BitWidth = N0.getValueSizeInBits();
4745 if (DAG.MaskedValueIsZero(N0,
4747 BitWidth-1))) {
4748 // Okay, get the un-inverted input value.
4749 SDValue Val;
4750 if (N0.getOpcode() == ISD::XOR) {
4751 Val = N0.getOperand(0);
4752 } else {
4753 assert(N0.getOpcode() == ISD::AND &&
4754 N0.getOperand(0).getOpcode() == ISD::XOR);
4755 // ((X^1)&1)^1 -> X & 1
4756 Val = DAG.getNode(ISD::AND, dl, N0.getValueType(),
4757 N0.getOperand(0).getOperand(0),
4758 N0.getOperand(1));
4759 }
4760
4761 return DAG.getSetCC(dl, VT, Val, N1,
4763 }
4764 } else if (N1C->isOne()) {
4765 SDValue Op0 = N0;
4766 if (Op0.getOpcode() == ISD::TRUNCATE)
4767 Op0 = Op0.getOperand(0);
4768
4769 if ((Op0.getOpcode() == ISD::XOR) &&
4770 Op0.getOperand(0).getOpcode() == ISD::SETCC &&
4771 Op0.getOperand(1).getOpcode() == ISD::SETCC) {
4772 SDValue XorLHS = Op0.getOperand(0);
4773 SDValue XorRHS = Op0.getOperand(1);
4774 // Ensure that the input setccs return an i1 type or 0/1 value.
4775 if (Op0.getValueType() == MVT::i1 ||
4780 // (xor (setcc), (setcc)) == / != 1 -> (setcc) != / == (setcc)
4782 return DAG.getSetCC(dl, VT, XorLHS, XorRHS, Cond);
4783 }
4784 }
4785 if (Op0.getOpcode() == ISD::AND && isOneConstant(Op0.getOperand(1))) {
4786 // If this is (X&1) == / != 1, normalize it to (X&1) != / == 0.
4787 if (Op0.getValueType().bitsGT(VT))
4788 Op0 = DAG.getNode(ISD::AND, dl, VT,
4789 DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)),
4790 DAG.getConstant(1, dl, VT));
4791 else if (Op0.getValueType().bitsLT(VT))
4792 Op0 = DAG.getNode(ISD::AND, dl, VT,
4793 DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)),
4794 DAG.getConstant(1, dl, VT));
4795
4796 return DAG.getSetCC(dl, VT, Op0,
4797 DAG.getConstant(0, dl, Op0.getValueType()),
4799 }
4800 if (Op0.getOpcode() == ISD::AssertZext &&
4801 cast<VTSDNode>(Op0.getOperand(1))->getVT() == MVT::i1)
4802 return DAG.getSetCC(dl, VT, Op0,
4803 DAG.getConstant(0, dl, Op0.getValueType()),
4805 }
4806 }
4807
4808 // Given:
4809 // icmp eq/ne (urem %x, %y), 0
4810 // Iff %x has 0 or 1 bits set, and %y has at least 2 bits set, omit 'urem':
4811 // icmp eq/ne %x, 0
4812 if (N0.getOpcode() == ISD::UREM && N1C->isZero() &&
4813 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
4814 KnownBits XKnown = DAG.computeKnownBits(N0.getOperand(0));
4815 KnownBits YKnown = DAG.computeKnownBits(N0.getOperand(1));
4816 if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2)
4817 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1, Cond);
4818 }
4819
4820 // Fold set_cc seteq (ashr X, BW-1), -1 -> set_cc setlt X, 0
4821 // and set_cc setne (ashr X, BW-1), -1 -> set_cc setge X, 0
4822 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4823 N0.getOpcode() == ISD::SRA && isa<ConstantSDNode>(N0.getOperand(1)) &&
4824 N0.getConstantOperandAPInt(1) == OpVT.getScalarSizeInBits() - 1 &&
4825 N1C && N1C->isAllOnes()) {
4826 return DAG.getSetCC(dl, VT, N0.getOperand(0),
4827 DAG.getConstant(0, dl, OpVT),
4829 }
4830
4831 if (SDValue V =
4832 optimizeSetCCOfSignedTruncationCheck(VT, N0, N1, Cond, DCI, dl))
4833 return V;
4834 }
4835
4836 // These simplifications apply to splat vectors as well.
4837 // TODO: Handle more splat vector cases.
4838 if (auto *N1C = isConstOrConstSplat(N1)) {
4839 const APInt &C1 = N1C->getAPIntValue();
4840
4841 APInt MinVal, MaxVal;
4842 unsigned OperandBitSize = N1C->getValueType(0).getScalarSizeInBits();
4844 MinVal = APInt::getSignedMinValue(OperandBitSize);
4845 MaxVal = APInt::getSignedMaxValue(OperandBitSize);
4846 } else {
4847 MinVal = APInt::getMinValue(OperandBitSize);
4848 MaxVal = APInt::getMaxValue(OperandBitSize);
4849 }
4850
4851 // Canonicalize GE/LE comparisons to use GT/LT comparisons.
4852 if (Cond == ISD::SETGE || Cond == ISD::SETUGE) {
4853 // X >= MIN --> true
4854 if (C1 == MinVal)
4855 return DAG.getBoolConstant(true, dl, VT, OpVT);
4856
4857 if (!VT.isVector()) { // TODO: Support this for vectors.
4858 // X >= C0 --> X > (C0 - 1)
4859 APInt C = C1 - 1;
4861 if ((DCI.isBeforeLegalizeOps() ||
4862 isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
4863 (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
4864 isLegalICmpImmediate(C.getSExtValue())))) {
4865 return DAG.getSetCC(dl, VT, N0,
4866 DAG.getConstant(C, dl, N1.getValueType()),
4867 NewCC);
4868 }
4869 }
4870 }
4871
4872 if (Cond == ISD::SETLE || Cond == ISD::SETULE) {
4873 // X <= MAX --> true
4874 if (C1 == MaxVal)
4875 return DAG.getBoolConstant(true, dl, VT, OpVT);
4876
4877 // X <= C0 --> X < (C0 + 1)
4878 if (!VT.isVector()) { // TODO: Support this for vectors.
4879 APInt C = C1 + 1;
4881 if ((DCI.isBeforeLegalizeOps() ||
4882 isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
4883 (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
4884 isLegalICmpImmediate(C.getSExtValue())))) {
4885 return DAG.getSetCC(dl, VT, N0,
4886 DAG.getConstant(C, dl, N1.getValueType()),
4887 NewCC);
4888 }
4889 }
4890 }
4891
4892 if (Cond == ISD::SETLT || Cond == ISD::SETULT) {
4893 if (C1 == MinVal)
4894 return DAG.getBoolConstant(false, dl, VT, OpVT); // X < MIN --> false
4895
4896 // TODO: Support this for vectors after legalize ops.
4897 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
4898 // Canonicalize setlt X, Max --> setne X, Max
4899 if (C1 == MaxVal)
4900 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
4901
4902 // If we have setult X, 1, turn it into seteq X, 0
4903 if (C1 == MinVal+1)
4904 return DAG.getSetCC(dl, VT, N0,
4905 DAG.getConstant(MinVal, dl, N0.getValueType()),
4906 ISD::SETEQ);
4907 }
4908 }
4909
4910 if (Cond == ISD::SETGT || Cond == ISD::SETUGT) {
4911 if (C1 == MaxVal)
4912 return DAG.getBoolConstant(false, dl, VT, OpVT); // X > MAX --> false
4913
4914 // TODO: Support this for vectors after legalize ops.
4915 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
4916 // Canonicalize setgt X, Min --> setne X, Min
4917 if (C1 == MinVal)
4918 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
4919
4920 // If we have setugt X, Max-1, turn it into seteq X, Max
4921 if (C1 == MaxVal-1)
4922 return DAG.getSetCC(dl, VT, N0,
4923 DAG.getConstant(MaxVal, dl, N0.getValueType()),
4924 ISD::SETEQ);
4925 }
4926 }
4927
4928 if (Cond == ISD::SETEQ || Cond == ISD::SETNE) {
4929 // (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
4930 if (C1.isZero())
4931 if (SDValue CC = optimizeSetCCByHoistingAndByConstFromLogicalShift(
4932 VT, N0, N1, Cond, DCI, dl))
4933 return CC;
4934
4935 // For all/any comparisons, replace or(x,shl(y,bw/2)) with and/or(x,y).
4936 // For example, when high 32-bits of i64 X are known clear:
4937 // all bits clear: (X | (Y<<32)) == 0 --> (X | Y) == 0
4938 // all bits set: (X | (Y<<32)) == -1 --> (X & Y) == -1
4939 bool CmpZero = N1C->isZero();
4940 bool CmpNegOne = N1C->isAllOnes();
4941 if ((CmpZero || CmpNegOne) && N0.hasOneUse()) {
4942 // Match or(lo,shl(hi,bw/2)) pattern.
4943 auto IsConcat = [&](SDValue V, SDValue &Lo, SDValue &Hi) {
4944 unsigned EltBits = V.getScalarValueSizeInBits();
4945 if (V.getOpcode() != ISD::OR || (EltBits % 2) != 0)
4946 return false;
4947 SDValue LHS = V.getOperand(0);
4948 SDValue RHS = V.getOperand(1);
4949 APInt HiBits = APInt::getHighBitsSet(EltBits, EltBits / 2);
4950 // Unshifted element must have zero upperbits.
4951 if (RHS.getOpcode() == ISD::SHL &&
4952 isa<ConstantSDNode>(RHS.getOperand(1)) &&
4953 RHS.getConstantOperandAPInt(1) == (EltBits / 2) &&
4954 DAG.MaskedValueIsZero(LHS, HiBits)) {
4955 Lo = LHS;
4956 Hi = RHS.getOperand(0);
4957 return true;
4958 }
4959 if (LHS.getOpcode() == ISD::SHL &&
4960 isa<ConstantSDNode>(LHS.getOperand(1)) &&
4961 LHS.getConstantOperandAPInt(1) == (EltBits / 2) &&
4962 DAG.MaskedValueIsZero(RHS, HiBits)) {
4963 Lo = RHS;
4964 Hi = LHS.getOperand(0);
4965 return true;
4966 }
4967 return false;
4968 };
4969
4970 auto MergeConcat = [&](SDValue Lo, SDValue Hi) {
4971 unsigned EltBits = N0.getScalarValueSizeInBits();
4972 unsigned HalfBits = EltBits / 2;
4973 APInt HiBits = APInt::getHighBitsSet(EltBits, HalfBits);
4974 SDValue LoBits = DAG.getConstant(~HiBits, dl, OpVT);
4975 SDValue HiMask = DAG.getNode(ISD::AND, dl, OpVT, Hi, LoBits);
4976 SDValue NewN0 =
4977 DAG.getNode(CmpZero ? ISD::OR : ISD::AND, dl, OpVT, Lo, HiMask);
4978 SDValue NewN1 = CmpZero ? DAG.getConstant(0, dl, OpVT) : LoBits;
4979 return DAG.getSetCC(dl, VT, NewN0, NewN1, Cond);
4980 };
4981
4982 SDValue Lo, Hi;
4983 if (IsConcat(N0, Lo, Hi))
4984 return MergeConcat(Lo, Hi);
4985
4986 if (N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR) {
4987 SDValue Lo0, Lo1, Hi0, Hi1;
4988 if (IsConcat(N0.getOperand(0), Lo0, Hi0) &&
4989 IsConcat(N0.getOperand(1), Lo1, Hi1)) {
4990 return MergeConcat(DAG.getNode(N0.getOpcode(), dl, OpVT, Lo0, Lo1),
4991 DAG.getNode(N0.getOpcode(), dl, OpVT, Hi0, Hi1));
4992 }
4993 }
4994 }
4995 }
4996
4997 // If we have "setcc X, C0", check to see if we can shrink the immediate
4998 // by changing cc.
4999 // TODO: Support this for vectors after legalize ops.
5000 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
5001 // SETUGT X, SINTMAX -> SETLT X, 0
5002 // SETUGE X, SINTMIN -> SETLT X, 0
5003 if ((Cond == ISD::SETUGT && C1.isMaxSignedValue()) ||
5004 (Cond == ISD::SETUGE && C1.isMinSignedValue()))
5005 return DAG.getSetCC(dl, VT, N0,
5006 DAG.getConstant(0, dl, N1.getValueType()),
5007 ISD::SETLT);
5008
5009 // SETULT X, SINTMIN -> SETGT X, -1
5010 // SETULE X, SINTMAX -> SETGT X, -1
5011 if ((Cond == ISD::SETULT && C1.isMinSignedValue()) ||
5012 (Cond == ISD::SETULE && C1.isMaxSignedValue()))
5013 return DAG.getSetCC(dl, VT, N0,
5014 DAG.getAllOnesConstant(dl, N1.getValueType()),
5015 ISD::SETGT);
5016 }
5017 }
5018
5019 // Back to non-vector simplifications.
5020 // TODO: Can we do these for vector splats?
5021 if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
5022 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5023 const APInt &C1 = N1C->getAPIntValue();
5024 EVT ShValTy = N0.getValueType();
5025
5026 // Fold bit comparisons when we can. This will result in an
5027 // incorrect value when boolean false is negative one, unless
5028 // the bitsize is 1 in which case the false value is the same
5029 // in practice regardless of the representation.
5030 if ((VT.getSizeInBits() == 1 ||
5032 (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5033 (VT == ShValTy || (isTypeLegal(VT) && VT.bitsLE(ShValTy))) &&
5034 N0.getOpcode() == ISD::AND) {
5035 if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
5036 if (Cond == ISD::SETNE && C1 == 0) {// (X & 8) != 0 --> (X & 8) >> 3
5037 // Perform the xform if the AND RHS is a single bit.
5038 unsigned ShCt = AndRHS->getAPIntValue().logBase2();
5039 if (AndRHS->getAPIntValue().isPowerOf2() &&
5040 !TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {
5041 return DAG.getNode(
5042 ISD::TRUNCATE, dl, VT,
5043 DAG.getNode(ISD::SRL, dl, ShValTy, N0,
5045 ShCt, ShValTy, dl, !DCI.isBeforeLegalize())));
5046 }
5047 } else if (Cond == ISD::SETEQ && C1 == AndRHS->getAPIntValue()) {
5048 // (X & 8) == 8 --> (X & 8) >> 3
5049 // Perform the xform if C1 is a single bit.
5050 unsigned ShCt = C1.logBase2();
5051 if (C1.isPowerOf2() &&
5052 !TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {
5053 return DAG.getNode(
5054 ISD::TRUNCATE, dl, VT,
5055 DAG.getNode(ISD::SRL, dl, ShValTy, N0,
5057 ShCt, ShValTy, dl, !DCI.isBeforeLegalize())));
5058 }
5059 }
5060 }
5061 }
5062
5063 if (C1.getSignificantBits() <= 64 &&
5065 // (X & -256) == 256 -> (X >> 8) == 1
5066 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5067 N0.getOpcode() == ISD::AND && N0.hasOneUse()) {
5068 if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
5069 const APInt &AndRHSC = AndRHS->getAPIntValue();
5070 if (AndRHSC.isNegatedPowerOf2() && (AndRHSC & C1) == C1) {
5071 unsigned ShiftBits = AndRHSC.countr_zero();
5072 if (!TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
5073 SDValue Shift = DAG.getNode(
5074 ISD::SRL, dl, ShValTy, N0.getOperand(0),
5075 DAG.getShiftAmountConstant(ShiftBits, ShValTy, dl,
5076 !DCI.isBeforeLegalize()));
5077 SDValue CmpRHS = DAG.getConstant(C1.lshr(ShiftBits), dl, ShValTy);
5078 return DAG.getSetCC(dl, VT, Shift, CmpRHS, Cond);
5079 }
5080 }
5081 }
5082 } else if (Cond == ISD::SETULT || Cond == ISD::SETUGE ||
5083 Cond == ISD::SETULE || Cond == ISD::SETUGT) {
5084 bool AdjOne = (Cond == ISD::SETULE || Cond == ISD::SETUGT);
5085 // X < 0x100000000 -> (X >> 32) < 1
5086 // X >= 0x100000000 -> (X >> 32) >= 1
5087 // X <= 0x0ffffffff -> (X >> 32) < 1
5088 // X > 0x0ffffffff -> (X >> 32) >= 1
5089 unsigned ShiftBits;
5090 APInt NewC = C1;
5091 ISD::CondCode NewCond = Cond;
5092 if (AdjOne) {
5093 ShiftBits = C1.countr_one();
5094 NewC = NewC + 1;
5095 NewCond = (Cond == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
5096 } else {
5097 ShiftBits = C1.countr_zero();
5098 }
5099 NewC.lshrInPlace(ShiftBits);
5100 if (ShiftBits && NewC.getSignificantBits() <= 64 &&
5102 !TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
5103 SDValue Shift =
5104 DAG.getNode(ISD::SRL, dl, ShValTy, N0,
5105 DAG.getShiftAmountConstant(ShiftBits, ShValTy, dl,
5106 !DCI.isBeforeLegalize()));
5107 SDValue CmpRHS = DAG.getConstant(NewC, dl, ShValTy);
5108 return DAG.getSetCC(dl, VT, Shift, CmpRHS, NewCond);
5109 }
5110 }
5111 }
5112 }
5113
5114 if (!isa<ConstantFPSDNode>(N0) && isa<ConstantFPSDNode>(N1)) {
5115 auto *CFP = cast<ConstantFPSDNode>(N1);
5116 assert(!CFP->getValueAPF().isNaN() && "Unexpected NaN value");
5117
5118 // Otherwise, we know the RHS is not a NaN. Simplify the node to drop the
5119 // constant if knowing that the operand is non-nan is enough. We prefer to
5120 // have SETO(x,x) instead of SETO(x, 0.0) because this avoids having to
5121 // materialize 0.0.
5122 if (Cond == ISD::SETO || Cond == ISD::SETUO)
5123 return DAG.getSetCC(dl, VT, N0, N0, Cond);
5124
5125 // setcc (fneg x), C -> setcc swap(pred) x, -C
5126 if (N0.getOpcode() == ISD::FNEG) {
5128 if (DCI.isBeforeLegalizeOps() ||
5129 isCondCodeLegal(SwapCond, N0.getSimpleValueType())) {
5130 SDValue NegN1 = DAG.getNode(ISD::FNEG, dl, N0.getValueType(), N1);
5131 return DAG.getSetCC(dl, VT, N0.getOperand(0), NegN1, SwapCond);
5132 }
5133 }
5134
5135 // setueq/setoeq X, (fabs Inf) -> is_fpclass X, fcInf
5137 !isFPImmLegal(CFP->getValueAPF(), CFP->getValueType(0))) {
5138 bool IsFabs = N0.getOpcode() == ISD::FABS;
5139 SDValue Op = IsFabs ? N0.getOperand(0) : N0;
5140 if ((Cond == ISD::SETOEQ || Cond == ISD::SETUEQ) && CFP->isInfinity()) {
5141 FPClassTest Flag = CFP->isNegative() ? (IsFabs ? fcNone : fcNegInf)
5142 : (IsFabs ? fcInf : fcPosInf);
5143 if (Cond == ISD::SETUEQ)
5144 Flag |= fcNan;
5145 return DAG.getNode(ISD::IS_FPCLASS, dl, VT, Op,
5146 DAG.getTargetConstant(Flag, dl, MVT::i32));
5147 }
5148 }
5149
5150 // If the condition is not legal, see if we can find an equivalent one
5151 // which is legal.
5153 // If the comparison was an awkward floating-point == or != and one of
5154 // the comparison operands is infinity or negative infinity, convert the
5155 // condition to a less-awkward <= or >=.
5156 if (CFP->getValueAPF().isInfinity()) {
5157 bool IsNegInf = CFP->getValueAPF().isNegative();
5159 switch (Cond) {
5160 case ISD::SETOEQ: NewCond = IsNegInf ? ISD::SETOLE : ISD::SETOGE; break;
5161 case ISD::SETUEQ: NewCond = IsNegInf ? ISD::SETULE : ISD::SETUGE; break;
5162 case ISD::SETUNE: NewCond = IsNegInf ? ISD::SETUGT : ISD::SETULT; break;
5163 case ISD::SETONE: NewCond = IsNegInf ? ISD::SETOGT : ISD::SETOLT; break;
5164 default: break;
5165 }
5166 if (NewCond != ISD::SETCC_INVALID &&
5167 isCondCodeLegal(NewCond, N0.getSimpleValueType()))
5168 return DAG.getSetCC(dl, VT, N0, N1, NewCond);
5169 }
5170 }
5171 }
5172
5173 if (N0 == N1) {
5174 // The sext(setcc()) => setcc() optimization relies on the appropriate
5175 // constant being emitted.
5176 assert(!N0.getValueType().isInteger() &&
5177 "Integer types should be handled by FoldSetCC");
5178
5179 bool EqTrue = ISD::isTrueWhenEqual(Cond);
5180 unsigned UOF = ISD::getUnorderedFlavor(Cond);
5181 if (UOF == 2) // FP operators that are undefined on NaNs.
5182 return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
5183 if (UOF == unsigned(EqTrue))
5184 return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
5185 // Otherwise, we can't fold it. However, we can simplify it to SETUO/SETO
5186 // if it is not already.
5187 ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO;
5188 if (NewCond != Cond &&
5189 (DCI.isBeforeLegalizeOps() ||
5190 isCondCodeLegal(NewCond, N0.getSimpleValueType())))
5191 return DAG.getSetCC(dl, VT, N0, N1, NewCond);
5192 }
5193
5194 // ~X > ~Y --> Y > X
5195 // ~X < ~Y --> Y < X
5196 // ~X < C --> X > ~C
5197 // ~X > C --> X < ~C
5198 if ((isSignedIntSetCC(Cond) || isUnsignedIntSetCC(Cond)) &&
5199 N0.getValueType().isInteger()) {
5200 if (isBitwiseNot(N0)) {
5201 if (isBitwiseNot(N1))
5202 return DAG.getSetCC(dl, VT, N1.getOperand(0), N0.getOperand(0), Cond);
5203
5206 SDValue Not = DAG.getNOT(dl, N1, OpVT);
5207 return DAG.getSetCC(dl, VT, Not, N0.getOperand(0), Cond);
5208 }
5209 }
5210 }
5211
5212 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5213 N0.getValueType().isInteger()) {
5214 if (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB ||
5215 N0.getOpcode() == ISD::XOR) {
5216 // Simplify (X+Y) == (X+Z) --> Y == Z
5217 if (N0.getOpcode() == N1.getOpcode()) {
5218 if (N0.getOperand(0) == N1.getOperand(0))
5219 return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(1), Cond);
5220 if (N0.getOperand(1) == N1.getOperand(1))
5221 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(0), Cond);
5222 if (isCommutativeBinOp(N0.getOpcode())) {
5223 // If X op Y == Y op X, try other combinations.
5224 if (N0.getOperand(0) == N1.getOperand(1))
5225 return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(0),
5226 Cond);
5227 if (N0.getOperand(1) == N1.getOperand(0))
5228 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(1),
5229 Cond);
5230 }
5231 }
5232
5233 // If RHS is a legal immediate value for a compare instruction, we need
5234 // to be careful about increasing register pressure needlessly.
5235 bool LegalRHSImm = false;
5236
5237 if (auto *RHSC = dyn_cast<ConstantSDNode>(N1)) {
5238 if (auto *LHSR = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
5239 // Turn (X+C1) == C2 --> X == C2-C1
5240 if (N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse())
5241 return DAG.getSetCC(
5242 dl, VT, N0.getOperand(0),
5243 DAG.getConstant(RHSC->getAPIntValue() - LHSR->getAPIntValue(),
5244 dl, N0.getValueType()),
5245 Cond);
5246
5247 // Turn (X^C1) == C2 --> X == C1^C2
5248 if (N0.getOpcode() == ISD::XOR && N0.getNode()->hasOneUse())
5249 return DAG.getSetCC(
5250 dl, VT, N0.getOperand(0),
5251 DAG.getConstant(LHSR->getAPIntValue() ^ RHSC->getAPIntValue(),
5252 dl, N0.getValueType()),
5253 Cond);
5254 }
5255
5256 // Turn (C1-X) == C2 --> X == C1-C2
5257 if (auto *SUBC = dyn_cast<ConstantSDNode>(N0.getOperand(0)))
5258 if (N0.getOpcode() == ISD::SUB && N0.getNode()->hasOneUse())
5259 return DAG.getSetCC(
5260 dl, VT, N0.getOperand(1),
5261 DAG.getConstant(SUBC->getAPIntValue() - RHSC->getAPIntValue(),
5262 dl, N0.getValueType()),
5263 Cond);
5264
5265 // Could RHSC fold directly into a compare?
5266 if (RHSC->getValueType(0).getSizeInBits() <= 64)
5267 LegalRHSImm = isLegalICmpImmediate(RHSC->getSExtValue());
5268 }
5269
5270 // (X+Y) == X --> Y == 0 and similar folds.
5271 // Don't do this if X is an immediate that can fold into a cmp
5272 // instruction and X+Y has other uses. It could be an induction variable
5273 // chain, and the transform would increase register pressure.
5274 if (!LegalRHSImm || N0.hasOneUse())
5275 if (SDValue V = foldSetCCWithBinOp(VT, N0, N1, Cond, dl, DCI))
5276 return V;
5277 }
5278
5279 if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB ||
5280 N1.getOpcode() == ISD::XOR)
5281 if (SDValue V = foldSetCCWithBinOp(VT, N1, N0, Cond, dl, DCI))
5282 return V;
5283
5284 if (SDValue V = foldSetCCWithAnd(VT, N0, N1, Cond, dl, DCI))
5285 return V;
5286 }
5287
5288 // Fold remainder of division by a constant.
5289 if ((N0.getOpcode() == ISD::UREM || N0.getOpcode() == ISD::SREM) &&
5290 N0.hasOneUse() && (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
5291 // When division is cheap or optimizing for minimum size,
5292 // fall through to DIVREM creation by skipping this fold.
5293 if (!isIntDivCheap(VT, Attr) && !Attr.hasFnAttr(Attribute::MinSize)) {
5294 if (N0.getOpcode() == ISD::UREM) {
5295 if (SDValue Folded = buildUREMEqFold(VT, N0, N1, Cond, DCI, dl))
5296 return Folded;
5297 } else if (N0.getOpcode() == ISD::SREM) {
5298 if (SDValue Folded = buildSREMEqFold(VT, N0, N1, Cond, DCI, dl))
5299 return Folded;
5300 }
5301 }
5302 }
5303
5304 // Fold away ALL boolean setcc's.
5305 if (N0.getValueType().getScalarType() == MVT::i1 && foldBooleans) {
5306 SDValue Temp;
5307 switch (Cond) {
5308 default: llvm_unreachable("Unknown integer setcc!");
5309 case ISD::SETEQ: // X == Y -> ~(X^Y)
5310 Temp = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
5311 N0 = DAG.getNOT(dl, Temp, OpVT);
5312 if (!DCI.isCalledByLegalizer())
5313 DCI.AddToWorklist(Temp.getNode());
5314 break;
5315 case ISD::SETNE: // X != Y --> (X^Y)
5316 N0 = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
5317 break;
5318 case ISD::SETGT: // X >s Y --> X == 0 & Y == 1 --> ~X & Y
5319 case ISD::SETULT: // X <u Y --> X == 0 & Y == 1 --> ~X & Y
5320 Temp = DAG.getNOT(dl, N0, OpVT);
5321 N0 = DAG.getNode(ISD::AND, dl, OpVT, N1, Temp);
5322 if (!DCI.isCalledByLegalizer())
5323 DCI.AddToWorklist(Temp.getNode());
5324 break;
5325 case ISD::SETLT: // X <s Y --> X == 1 & Y == 0 --> ~Y & X
5326 case ISD::SETUGT: // X >u Y --> X == 1 & Y == 0 --> ~Y & X
5327 Temp = DAG.getNOT(dl, N1, OpVT);
5328 N0 = DAG.getNode(ISD::AND, dl, OpVT, N0, Temp);
5329 if (!DCI.isCalledByLegalizer())
5330 DCI.AddToWorklist(Temp.getNode());
5331 break;
5332 case ISD::SETULE: // X <=u Y --> X == 0 | Y == 1 --> ~X | Y
5333 case ISD::SETGE: // X >=s Y --> X == 0 | Y == 1 --> ~X | Y
5334 Temp = DAG.getNOT(dl, N0, OpVT);
5335 N0 = DAG.getNode(ISD::OR, dl, OpVT, N1, Temp);
5336 if (!DCI.isCalledByLegalizer())
5337 DCI.AddToWorklist(Temp.getNode());
5338 break;
5339 case ISD::SETUGE: // X >=u Y --> X == 1 | Y == 0 --> ~Y | X
5340 case ISD::SETLE: // X <=s Y --> X == 1 | Y == 0 --> ~Y | X
5341 Temp = DAG.getNOT(dl, N1, OpVT);
5342 N0 = DAG.getNode(ISD::OR, dl, OpVT, N0, Temp);
5343 break;
5344 }
5345 if (VT.getScalarType() != MVT::i1) {
5346 if (!DCI.isCalledByLegalizer())
5347 DCI.AddToWorklist(N0.getNode());
5348 // FIXME: If running after legalize, we probably can't do this.
5350 N0 = DAG.getNode(ExtendCode, dl, VT, N0);
5351 }
5352 return N0;
5353 }
5354
5355 // Could not fold it.
5356 return SDValue();
5357}
5358
5359/// Returns true (and the GlobalValue and the offset) if the node is a
5360/// GlobalAddress + offset.
5362 int64_t &Offset) const {
5363
5364 SDNode *N = unwrapAddress(SDValue(WN, 0)).getNode();
5365
5366 if (auto *GASD = dyn_cast<GlobalAddressSDNode>(N)) {
5367 GA = GASD->getGlobal();
5368 Offset += GASD->getOffset();
5369 return true;
5370 }
5371
5372 if (N->getOpcode() == ISD::ADD) {
5373 SDValue N1 = N->getOperand(0);
5374 SDValue N2 = N->getOperand(1);
5375 if (isGAPlusOffset(N1.getNode(), GA, Offset)) {
5376 if (auto *V = dyn_cast<ConstantSDNode>(N2)) {
5377 Offset += V->getSExtValue();
5378 return true;
5379 }
5380 } else if (isGAPlusOffset(N2.getNode(), GA, Offset)) {
5381 if (auto *V = dyn_cast<ConstantSDNode>(N1)) {
5382 Offset += V->getSExtValue();
5383 return true;
5384 }
5385 }
5386 }
5387
5388 return false;
5389}
5390
5392 DAGCombinerInfo &DCI) const {
5393 // Default implementation: no optimization.
5394 return SDValue();
5395}
5396
5397//===----------------------------------------------------------------------===//
5398// Inline Assembler Implementation Methods
5399//===----------------------------------------------------------------------===//
5400
5403 unsigned S = Constraint.size();
5404
5405 if (S == 1) {
5406 switch (Constraint[0]) {
5407 default: break;
5408 case 'r':
5409 return C_RegisterClass;
5410 case 'm': // memory
5411 case 'o': // offsetable
5412 case 'V': // not offsetable
5413 return C_Memory;
5414 case 'p': // Address.
5415 return C_Address;
5416 case 'n': // Simple Integer
5417 case 'E': // Floating Point Constant
5418 case 'F': // Floating Point Constant
5419 return C_Immediate;
5420 case 'i': // Simple Integer or Relocatable Constant
5421 case 's': // Relocatable Constant
5422 case 'X': // Allow ANY value.
5423 case 'I': // Target registers.
5424 case 'J':
5425 case 'K':
5426 case 'L':
5427 case 'M':
5428 case 'N':
5429 case 'O':
5430 case 'P':
5431 case '<':
5432 case '>':
5433 return C_Other;
5434 }
5435 }
5436
5437 if (S > 1 && Constraint[0] == '{' && Constraint[S - 1] == '}') {
5438 if (S == 8 && Constraint.substr(1, 6) == "memory") // "{memory}"
5439 return C_Memory;
5440 return C_Register;
5441 }
5442 return C_Unknown;
5443}
5444
5445/// Try to replace an X constraint, which matches anything, with another that
5446/// has more specific requirements based on the type of the corresponding
5447/// operand.
5448const char *TargetLowering::LowerXConstraint(EVT ConstraintVT) const {
5449 if (ConstraintVT.isInteger())
5450 return "r";
5451 if (ConstraintVT.isFloatingPoint())
5452 return "f"; // works for many targets
5453 return nullptr;
5454}
5455
5457 SDValue &Chain, SDValue &Glue, const SDLoc &DL,
5458 const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const {
5459 return SDValue();
5460}
5461
5462/// Lower the specified operand into the Ops vector.
5463/// If it is invalid, don't add anything to Ops.
5465 StringRef Constraint,
5466 std::vector<SDValue> &Ops,
5467 SelectionDAG &DAG) const {
5468
5469 if (Constraint.size() > 1)
5470 return;
5471
5472 char ConstraintLetter = Constraint[0];
5473 switch (ConstraintLetter) {
5474 default: break;
5475 case 'X': // Allows any operand
5476 case 'i': // Simple Integer or Relocatable Constant
5477 case 'n': // Simple Integer
5478 case 's': { // Relocatable Constant
5479
5481 uint64_t Offset = 0;
5482
5483 // Match (GA) or (C) or (GA+C) or (GA-C) or ((GA+C)+C) or (((GA+C)+C)+C),
5484 // etc., since getelementpointer is variadic. We can't use
5485 // SelectionDAG::FoldSymbolOffset because it expects the GA to be accessible
5486 // while in this case the GA may be furthest from the root node which is
5487 // likely an ISD::ADD.
5488 while (true) {
5489 if ((C = dyn_cast<ConstantSDNode>(Op)) && ConstraintLetter != 's') {
5490 // gcc prints these as sign extended. Sign extend value to 64 bits
5491 // now; without this it would get ZExt'd later in
5492 // ScheduleDAGSDNodes::EmitNode, which is very generic.
5493 bool IsBool = C->getConstantIntValue()->getBitWidth() == 1;
5494 BooleanContent BCont = getBooleanContents(MVT::i64);
5495 ISD::NodeType ExtOpc =
5496 IsBool ? getExtendForContent(BCont) : ISD::SIGN_EXTEND;
5497 int64_t ExtVal =
5498 ExtOpc == ISD::ZERO_EXTEND ? C->getZExtValue() : C->getSExtValue();
5499 Ops.push_back(
5500 DAG.getTargetConstant(Offset + ExtVal, SDLoc(C), MVT::i64));
5501 return;
5502 }
5503 if (ConstraintLetter != 'n') {
5504 if (const auto *GA = dyn_cast<GlobalAddressSDNode>(Op)) {
5505 Ops.push_back(DAG.getTargetGlobalAddress(GA->getGlobal(), SDLoc(Op),
5506 GA->getValueType(0),
5507 Offset + GA->getOffset()));
5508 return;
5509 }
5510 if (const auto *BA = dyn_cast<BlockAddressSDNode>(Op)) {
5511 Ops.push_back(DAG.getTargetBlockAddress(
5512 BA->getBlockAddress(), BA->getValueType(0),
5513 Offset + BA->getOffset(), BA->getTargetFlags()));
5514 return;
5515 }
5516 if (isa<BasicBlockSDNode>(Op)) {
5517 Ops.push_back(Op);
5518 return;
5519 }
5520 }
5521 const unsigned OpCode = Op.getOpcode();
5522 if (OpCode == ISD::ADD || OpCode == ISD::SUB) {
5523 if ((C = dyn_cast<ConstantSDNode>(Op.getOperand(0))))
5524 Op = Op.getOperand(1);
5525 // Subtraction is not commutative.
5526 else if (OpCode == ISD::ADD &&
5527 (C = dyn_cast<ConstantSDNode>(Op.getOperand(1))))
5528 Op = Op.getOperand(0);
5529 else
5530 return;
5531 Offset += (OpCode == ISD::ADD ? 1 : -1) * C->getSExtValue();
5532 continue;
5533 }
5534 return;
5535 }
5536 break;
5537 }
5538 }
5539}
5540
5542 const CallInst &I, SmallVectorImpl<SDValue> &Ops, SelectionDAG &DAG) const {
5543}
5544
5545std::pair<unsigned, const TargetRegisterClass *>
5547 StringRef Constraint,
5548 MVT VT) const {
5549 if (!Constraint.starts_with("{"))
5550 return std::make_pair(0u, static_cast<TargetRegisterClass *>(nullptr));
5551 assert(*(Constraint.end() - 1) == '}' && "Not a brace enclosed constraint?");
5552
5553 // Remove the braces from around the name.
5554 StringRef RegName(Constraint.data() + 1, Constraint.size() - 2);
5555
5556 std::pair<unsigned, const TargetRegisterClass *> R =
5557 std::make_pair(0u, static_cast<const TargetRegisterClass *>(nullptr));
5558
5559 // Figure out which register class contains this reg.
5560 for (const TargetRegisterClass *RC : RI->regclasses()) {
5561 // If none of the value types for this register class are valid, we
5562 // can't use it. For example, 64-bit reg classes on 32-bit targets.
5563 if (!isLegalRC(*RI, *RC))
5564 continue;
5565
5566 for (const MCPhysReg &PR : *RC) {
5567 if (RegName.equals_insensitive(RI->getRegAsmName(PR))) {
5568 std::pair<unsigned, const TargetRegisterClass *> S =
5569 std::make_pair(PR, RC);
5570
5571 // If this register class has the requested value type, return it,
5572 // otherwise keep searching and return the first class found
5573 // if no other is found which explicitly has the requested type.
5574 if (RI->isTypeLegalForClass(*RC, VT))
5575 return S;
5576 if (!R.second)
5577 R = S;
5578 }
5579 }
5580 }
5581
5582 return R;
5583}
5584
5585//===----------------------------------------------------------------------===//
5586// Constraint Selection.
5587
5588/// Return true of this is an input operand that is a matching constraint like
5589/// "4".
5591 assert(!ConstraintCode.empty() && "No known constraint!");
5592 return isdigit(static_cast<unsigned char>(ConstraintCode[0]));
5593}
5594
5595/// If this is an input matching constraint, this method returns the output
5596/// operand it matches.
5598 assert(!ConstraintCode.empty() && "No known constraint!");
5599 return atoi(ConstraintCode.c_str());
5600}
5601
5602/// Split up the constraint string from the inline assembly value into the
5603/// specific constraints and their prefixes, and also tie in the associated
5604/// operand values.
5605/// If this returns an empty vector, and if the constraint string itself
5606/// isn't empty, there was an error parsing.
5609 const TargetRegisterInfo *TRI,
5610 const CallBase &Call) const {
5611 /// Information about all of the constraints.
5612 AsmOperandInfoVector ConstraintOperands;
5613 const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
5614 unsigned maCount = 0; // Largest number of multiple alternative constraints.
5615
5616 // Do a prepass over the constraints, canonicalizing them, and building up the
5617 // ConstraintOperands list.
5618 unsigned ArgNo = 0; // ArgNo - The argument of the CallInst.
5619 unsigned ResNo = 0; // ResNo - The result number of the next output.
5620 unsigned LabelNo = 0; // LabelNo - CallBr indirect dest number.
5621
5622 for (InlineAsm::ConstraintInfo &CI : IA->ParseConstraints()) {
5623 ConstraintOperands.emplace_back(std::move(CI));
5624 AsmOperandInfo &OpInfo = ConstraintOperands.back();
5625
5626 // Update multiple alternative constraint count.
5627 if (OpInfo.multipleAlternatives.size() > maCount)
5628 maCount = OpInfo.multipleAlternatives.size();
5629
5630 OpInfo.ConstraintVT = MVT::Other;
5631
5632 // Compute the value type for each operand.
5633 switch (OpInfo.Type) {
5635 // Indirect outputs just consume an argument.
5636 if (OpInfo.isIndirect) {
5637 OpInfo.CallOperandVal = Call.getArgOperand(ArgNo);
5638 break;
5639 }
5640
5641 // The return value of the call is this value. As such, there is no
5642 // corresponding argument.
5643 assert(!Call.getType()->isVoidTy() && "Bad inline asm!");
5644 if (StructType *STy = dyn_cast<StructType>(Call.getType())) {
5645 OpInfo.ConstraintVT =
5646 getSimpleValueType(DL, STy->getElementType(ResNo));
5647 } else {
5648 assert(ResNo == 0 && "Asm only has one result!");
5649 OpInfo.ConstraintVT =
5650 getAsmOperandValueType(DL, Call.getType()).getSimpleVT();
5651 }
5652 ++ResNo;
5653 break;
5654 case InlineAsm::isInput:
5655 OpInfo.CallOperandVal = Call.getArgOperand(ArgNo);
5656 break;
5657 case InlineAsm::isLabel:
5658 OpInfo.CallOperandVal = cast<CallBrInst>(&Call)->getIndirectDest(LabelNo);
5659 ++LabelNo;
5660 continue;
5662 // Nothing to do.
5663 break;
5664 }
5665
5666 if (OpInfo.CallOperandVal) {
5667 llvm::Type *OpTy = OpInfo.CallOperandVal->getType();
5668 if (OpInfo.isIndirect) {
5669 OpTy = Call.getParamElementType(ArgNo);
5670 assert(OpTy && "Indirect operand must have elementtype attribute");
5671 }
5672
5673 // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
5674 if (StructType *STy = dyn_cast<StructType>(OpTy))
5675 if (STy->getNumElements() == 1)
5676 OpTy = STy->getElementType(0);
5677
5678 // If OpTy is not a single value, it may be a struct/union that we
5679 // can tile with integers.
5680 if (!OpTy->isSingleValueType() && OpTy->isSized()) {
5681 unsigned BitSize = DL.getTypeSizeInBits(OpTy);
5682 switch (BitSize) {
5683 default: break;
5684 case 1:
5685 case 8:
5686 case 16:
5687 case 32:
5688 case 64:
5689 case 128:
5690 OpTy = IntegerType::get(OpTy->getContext(), BitSize);
5691 break;
5692 }
5693 }
5694
5695 EVT VT = getAsmOperandValueType(DL, OpTy, true);
5696 OpInfo.ConstraintVT = VT.isSimple() ? VT.getSimpleVT() : MVT::Other;
5697 ArgNo++;
5698 }
5699 }
5700
5701 // If we have multiple alternative constraints, select the best alternative.
5702 if (!ConstraintOperands.empty()) {
5703 if (maCount) {
5704 unsigned bestMAIndex = 0;
5705 int bestWeight = -1;
5706 // weight: -1 = invalid match, and 0 = so-so match to 5 = good match.
5707 int weight = -1;
5708 unsigned maIndex;
5709 // Compute the sums of the weights for each alternative, keeping track
5710 // of the best (highest weight) one so far.
5711 for (maIndex = 0; maIndex < maCount; ++maIndex) {
5712 int weightSum = 0;
5713 for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
5714 cIndex != eIndex; ++cIndex) {
5715 AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];
5716 if (OpInfo.Type == InlineAsm::isClobber)
5717 continue;
5718
5719 // If this is an output operand with a matching input operand,
5720 // look up the matching input. If their types mismatch, e.g. one
5721 // is an integer, the other is floating point, or their sizes are
5722 // different, flag it as an maCantMatch.
5723 if (OpInfo.hasMatchingInput()) {
5724 AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
5725 if (OpInfo.ConstraintVT != Input.ConstraintVT) {
5726 if ((OpInfo.ConstraintVT.isInteger() !=
5727 Input.ConstraintVT.isInteger()) ||
5728 (OpInfo.ConstraintVT.getSizeInBits() !=
5729 Input.ConstraintVT.getSizeInBits())) {
5730 weightSum = -1; // Can't match.
5731 break;
5732 }
5733 }
5734 }
5735 weight = getMultipleConstraintMatchWeight(OpInfo, maIndex);
5736 if (weight == -1) {
5737 weightSum = -1;
5738 break;
5739 }
5740 weightSum += weight;
5741 }
5742 // Update best.
5743 if (weightSum > bestWeight) {
5744 bestWeight = weightSum;
5745 bestMAIndex = maIndex;
5746 }
5747 }
5748
5749 // Now select chosen alternative in each constraint.
5750 for (AsmOperandInfo &cInfo : ConstraintOperands)
5751 if (cInfo.Type != InlineAsm::isClobber)
5752 cInfo.selectAlternative(bestMAIndex);
5753 }
5754 }
5755
5756 // Check and hook up tied operands, choose constraint code to use.
5757 for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
5758 cIndex != eIndex; ++cIndex) {
5759 AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];
5760
5761 // If this is an output operand with a matching input operand, look up the
5762 // matching input. If their types mismatch, e.g. one is an integer, the
5763 // other is floating point, or their sizes are different, flag it as an
5764 // error.
5765 if (OpInfo.hasMatchingInput()) {
5766 AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
5767
5768 if (OpInfo.ConstraintVT != Input.ConstraintVT) {
5769 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
5770 getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,
5771 OpInfo.ConstraintVT);
5772 std::pair<unsigned, const TargetRegisterClass *> InputRC =
5773 getRegForInlineAsmConstraint(TRI, Input.ConstraintCode,
5774 Input.ConstraintVT);
5775 if ((OpInfo.ConstraintVT.isInteger() !=
5776 Input.ConstraintVT.isInteger()) ||
5777 (MatchRC.second != InputRC.second)) {
5778 report_fatal_error("Unsupported asm: input constraint"
5779 " with a matching output constraint of"
5780 " incompatible type!");
5781 }
5782 }
5783 }
5784 }
5785
5786 return ConstraintOperands;
5787}
5788
5789/// Return a number indicating our preference for chosing a type of constraint
5790/// over another, for the purpose of sorting them. Immediates are almost always
5791/// preferrable (when they can be emitted). A higher return value means a
5792/// stronger preference for one constraint type relative to another.
5793/// FIXME: We should prefer registers over memory but doing so may lead to
5794/// unrecoverable register exhaustion later.
5795/// https://github.com/llvm/llvm-project/issues/20571
5797 switch (CT) {
5800 return 4;
5803 return 3;
5805 return 2;
5807 return 1;
5809 return 0;
5810 }
5811 llvm_unreachable("Invalid constraint type");
5812}
5813
5814/// Examine constraint type and operand type and determine a weight value.
5815/// This object must already have been set up with the operand type
5816/// and the current alternative constraint selected.
5819 AsmOperandInfo &info, int maIndex) const {
5821 if (maIndex >= (int)info.multipleAlternatives.size())
5822 rCodes = &info.Codes;
5823 else
5824 rCodes = &info.multipleAlternatives[maIndex].Codes;
5825 ConstraintWeight BestWeight = CW_Invalid;
5826
5827 // Loop over the options, keeping track of the most general one.
5828 for (const std::string &rCode : *rCodes) {
5829 ConstraintWeight weight =
5830 getSingleConstraintMatchWeight(info, rCode.c_str());
5831 if (weight > BestWeight)
5832 BestWeight = weight;
5833 }
5834
5835 return BestWeight;
5836}
5837
5838/// Examine constraint type and operand type and determine a weight value.
5839/// This object must already have been set up with the operand type
5840/// and the current alternative constraint selected.
5843 AsmOperandInfo &info, const char *constraint) const {
5844 ConstraintWeight weight = CW_Invalid;
5845 Value *CallOperandVal = info.CallOperandVal;
5846 // If we don't have a value, we can't do a match,
5847 // but allow it at the lowest weight.
5848 if (!CallOperandVal)
5849 return CW_Default;
5850 // Look at the constraint type.
5851 switch (*constraint) {
5852 case 'i': // immediate integer.
5853 case 'n': // immediate integer with a known value.
5854 if (isa<ConstantInt>(CallOperandVal))
5855 weight = CW_Constant;
5856 break;
5857 case 's': // non-explicit intregal immediate.
5858 if (isa<GlobalValue>(CallOperandVal))
5859 weight = CW_Constant;
5860 break;
5861 case 'E': // immediate float if host format.
5862 case 'F': // immediate float.
5863 if (isa<ConstantFP>(CallOperandVal))
5864 weight = CW_Constant;
5865 break;
5866 case '<': // memory operand with autodecrement.
5867 case '>': // memory operand with autoincrement.
5868 case 'm': // memory operand.
5869 case 'o': // offsettable memory operand
5870 case 'V': // non-offsettable memory operand
5871 weight = CW_Memory;
5872 break;
5873 case 'r': // general register.
5874 case 'g': // general register, memory operand or immediate integer.
5875 // note: Clang converts "g" to "imr".
5876 if (CallOperandVal->getType()->isIntegerTy())
5877 weight = CW_Register;
5878 break;
5879 case 'X': // any operand.
5880 default:
5881 weight = CW_Default;
5882 break;
5883 }
5884 return weight;
5885}
5886
5887/// If there are multiple different constraints that we could pick for this
5888/// operand (e.g. "imr") try to pick the 'best' one.
5889/// This is somewhat tricky: constraints (TargetLowering::ConstraintType) fall
5890/// into seven classes:
5891/// Register -> one specific register
5892/// RegisterClass -> a group of regs
5893/// Memory -> memory
5894/// Address -> a symbolic memory reference
5895/// Immediate -> immediate values
5896/// Other -> magic values (such as "Flag Output Operands")
5897/// Unknown -> something we don't recognize yet and can't handle
5898/// Ideally, we would pick the most specific constraint possible: if we have
5899/// something that fits into a register, we would pick it. The problem here
5900/// is that if we have something that could either be in a register or in
5901/// memory that use of the register could cause selection of *other*
5902/// operands to fail: they might only succeed if we pick memory. Because of
5903/// this the heuristic we use is:
5904///
5905/// 1) If there is an 'other' constraint, and if the operand is valid for
5906/// that constraint, use it. This makes us take advantage of 'i'
5907/// constraints when available.
5908/// 2) Otherwise, pick the most general constraint present. This prefers
5909/// 'm' over 'r', for example.
5910///
5912 TargetLowering::AsmOperandInfo &OpInfo) const {
5913 ConstraintGroup Ret;
5914
5915 Ret.reserve(OpInfo.Codes.size());
5916 for (StringRef Code : OpInfo.Codes) {
5917 TargetLowering::ConstraintType CType = getConstraintType(Code);
5918
5919 // Indirect 'other' or 'immediate' constraints are not allowed.
5920 if (OpInfo.isIndirect && !(CType == TargetLowering::C_Memory ||
5921 CType == TargetLowering::C_Register ||
5923 continue;
5924
5925 // Things with matching constraints can only be registers, per gcc
5926 // documentation. This mainly affects "g" constraints.
5927 if (CType == TargetLowering::C_Memory && OpInfo.hasMatchingInput())
5928 continue;
5929
5930 Ret.emplace_back(Code, CType);
5931 }
5932
5933 std::stable_sort(
5934 Ret.begin(), Ret.end(), [](ConstraintPair a, ConstraintPair b) {
5935 return getConstraintPiority(a.second) > getConstraintPiority(b.second);
5936 });
5937
5938 return Ret;
5939}
5940
5941/// If we have an immediate, see if we can lower it. Return true if we can,
5942/// false otherwise.
5944 SDValue Op, SelectionDAG *DAG,
5945 const TargetLowering &TLI) {
5946
5947 assert((P.second == TargetLowering::C_Other ||
5948 P.second == TargetLowering::C_Immediate) &&
5949 "need immediate or other");
5950
5951 if (!Op.getNode())
5952 return false;
5953
5954 std::vector<SDValue> ResultOps;
5955 TLI.LowerAsmOperandForConstraint(Op, P.first, ResultOps, *DAG);
5956 return !ResultOps.empty();
5957}
5958
5959/// Determines the constraint code and constraint type to use for the specific
5960/// AsmOperandInfo, setting OpInfo.ConstraintCode and OpInfo.ConstraintType.
5962 SDValue Op,
5963 SelectionDAG *DAG) const {
5964 assert(!OpInfo.Codes.empty() && "Must have at least one constraint");
5965
5966 // Single-letter constraints ('r') are very common.
5967 if (OpInfo.Codes.size() == 1) {
5968 OpInfo.ConstraintCode = OpInfo.Codes[0];
5969 OpInfo.ConstraintType = getConstraintType(OpInfo.ConstraintCode);
5970 } else {
5971 ConstraintGroup G = getConstraintPreferences(OpInfo);
5972 if (G.empty())
5973 return;
5974
5975 unsigned BestIdx = 0;
5976 for (const unsigned E = G.size();
5977 BestIdx < E && (G[BestIdx].second == TargetLowering::C_Other ||
5978 G[BestIdx].second == TargetLowering::C_Immediate);
5979 ++BestIdx) {
5980 if (lowerImmediateIfPossible(G[BestIdx], Op, DAG, *this))
5981 break;
5982 // If we're out of constraints, just pick the first one.
5983 if (BestIdx + 1 == E) {
5984 BestIdx = 0;
5985 break;
5986 }
5987 }
5988
5989 OpInfo.ConstraintCode = G[BestIdx].first;
5990 OpInfo.ConstraintType = G[BestIdx].second;
5991 }
5992
5993 // 'X' matches anything.
5994 if (OpInfo.ConstraintCode == "X" && OpInfo.CallOperandVal) {
5995 // Constants are handled elsewhere. For Functions, the type here is the
5996 // type of the result, which is not what we want to look at; leave them
5997 // alone.
5998 Value *v = OpInfo.CallOperandVal;
5999 if (isa<ConstantInt>(v) || isa<Function>(v)) {
6000 return;
6001 }
6002
6003 if (isa<BasicBlock>(v) || isa<BlockAddress>(v)) {
6004 OpInfo.ConstraintCode = "i";
6005 return;
6006 }
6007
6008 // Otherwise, try to resolve it to something we know about by looking at
6009 // the actual operand type.
6010 if (const char *Repl = LowerXConstraint(OpInfo.ConstraintVT)) {
6011 OpInfo.ConstraintCode = Repl;
6012 OpInfo.ConstraintType = getConstraintType(OpInfo.ConstraintCode);
6013 }
6014 }
6015}
6016
6017/// Given an exact SDIV by a constant, create a multiplication
6018/// with the multiplicative inverse of the constant.
6020 const SDLoc &dl, SelectionDAG &DAG,
6021 SmallVectorImpl<SDNode *> &Created) {
6022 SDValue Op0 = N->getOperand(0);
6023 SDValue Op1 = N->getOperand(1);
6024 EVT VT = N->getValueType(0);
6025 EVT SVT = VT.getScalarType();
6026 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
6027 EVT ShSVT = ShVT.getScalarType();
6028
6029 bool UseSRA = false;
6030 SmallVector<SDValue, 16> Shifts, Factors;
6031
6032 auto BuildSDIVPattern = [&](ConstantSDNode *C) {
6033 if (C->isZero())
6034 return false;
6035 APInt Divisor = C->getAPIntValue();
6036 unsigned Shift = Divisor.countr_zero();
6037 if (Shift) {
6038 Divisor.ashrInPlace(Shift);
6039 UseSRA = true;
6040 }
6041 // Calculate the multiplicative inverse, using Newton's method.
6042 APInt t;
6043 APInt Factor = Divisor;
6044 while ((t = Divisor * Factor) != 1)
6045 Factor *= APInt(Divisor.getBitWidth(), 2) - t;
6046 Shifts.push_back(DAG.getConstant(Shift, dl, ShSVT));
6047 Factors.push_back(DAG.getConstant(Factor, dl, SVT));
6048 return true;
6049 };
6050
6051 // Collect all magic values from the build vector.
6052 if (!ISD::matchUnaryPredicate(Op1, BuildSDIVPattern))
6053 return SDValue();
6054
6055 SDValue Shift, Factor;
6056 if (Op1.getOpcode() == ISD::BUILD_VECTOR) {
6057 Shift = DAG.getBuildVector(ShVT, dl, Shifts);
6058 Factor = DAG.getBuildVector(VT, dl, Factors);
6059 } else if (Op1.getOpcode() == ISD::SPLAT_VECTOR) {
6060 assert(Shifts.size() == 1 && Factors.size() == 1 &&
6061 "Expected matchUnaryPredicate to return one element for scalable "
6062 "vectors");
6063 Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]);
6064 Factor = DAG.getSplatVector(VT, dl, Factors[0]);
6065 } else {
6066 assert(isa<ConstantSDNode>(Op1) && "Expected a constant");
6067 Shift = Shifts[0];
6068 Factor = Factors[0];
6069 }
6070
6071 SDValue Res = Op0;
6072
6073 // Shift the value upfront if it is even, so the LSB is one.
6074 if (UseSRA) {
6075 // TODO: For UDIV use SRL instead of SRA.
6076 SDNodeFlags Flags;
6077 Flags.setExact(true);
6078 Res = DAG.getNode(ISD::SRA, dl, VT, Res, Shift, Flags);
6079 Created.push_back(Res.getNode());
6080 }
6081
6082 return DAG.getNode(ISD::MUL, dl, VT, Res, Factor);
6083}
6084
6086 SelectionDAG &DAG,
6087 SmallVectorImpl<SDNode *> &Created) const {
6089 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6090 if (TLI.isIntDivCheap(N->getValueType(0), Attr))
6091 return SDValue(N, 0); // Lower SDIV as SDIV
6092 return SDValue();
6093}
6094
6095SDValue
6097 SelectionDAG &DAG,
6098 SmallVectorImpl<SDNode *> &Created) const {
6100 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6101 if (TLI.isIntDivCheap(N->getValueType(0), Attr))
6102 return SDValue(N, 0); // Lower SREM as SREM
6103 return SDValue();
6104}
6105
6106/// Build sdiv by power-of-2 with conditional move instructions
6107/// Ref: "Hacker's Delight" by Henry Warren 10-1
6108/// If conditional move/branch is preferred, we lower sdiv x, +/-2**k into:
6109/// bgez x, label
6110/// add x, x, 2**k-1
6111/// label:
6112/// sra res, x, k
6113/// neg res, res (when the divisor is negative)
6115 SDNode *N, const APInt &Divisor, SelectionDAG &DAG,
6116 SmallVectorImpl<SDNode *> &Created) const {
6117 unsigned Lg2 = Divisor.countr_zero();
6118 EVT VT = N->getValueType(0);
6119
6120 SDLoc DL(N);
6121 SDValue N0 = N->getOperand(0);
6122 SDValue Zero = DAG.getConstant(0, DL, VT);
6123 APInt Lg2Mask = APInt::getLowBitsSet(VT.getSizeInBits(), Lg2);
6124 SDValue Pow2MinusOne = DAG.getConstant(Lg2Mask, DL, VT);
6125
6126 // If N0 is negative, we need to add (Pow2 - 1) to it before shifting right.
6127 EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
6128 SDValue Cmp = DAG.getSetCC(DL, CCVT, N0, Zero, ISD::SETLT);
6129 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Pow2MinusOne);
6130 SDValue CMov = DAG.getNode(ISD::SELECT, DL, VT, Cmp, Add, N0);
6131
6132 Created.push_back(Cmp.getNode());
6133 Created.push_back(Add.getNode());
6134 Created.push_back(CMov.getNode());
6135
6136 // Divide by pow2.
6137 SDValue SRA =
6138 DAG.getNode(ISD::SRA, DL, VT, CMov, DAG.getConstant(Lg2, DL, VT));
6139
6140 // If we're dividing by a positive value, we're done. Otherwise, we must
6141 // negate the result.
6142 if (Divisor.isNonNegative())
6143 return SRA;
6144
6145 Created.push_back(SRA.getNode());
6146 return DAG.getNode(ISD::SUB, DL, VT, Zero, SRA);
6147}
6148
6149/// Given an ISD::SDIV node expressing a divide by constant,
6150/// return a DAG expression to select that will generate the same value by
6151/// multiplying by a magic number.
6152/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
6154 bool IsAfterLegalization,
6155 SmallVectorImpl<SDNode *> &Created) const {
6156 SDLoc dl(N);
6157 EVT VT = N->getValueType(0);
6158 EVT SVT = VT.getScalarType();
6159 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
6160 EVT ShSVT = ShVT.getScalarType();
6161 unsigned EltBits = VT.getScalarSizeInBits();
6162 EVT MulVT;
6163
6164 // Check to see if we can do this.
6165 // FIXME: We should be more aggressive here.
6166 if (!isTypeLegal(VT)) {
6167 // Limit this to simple scalars for now.
6168 if (VT.isVector() || !VT.isSimple())
6169 return SDValue();
6170
6171 // If this type will be promoted to a large enough type with a legal
6172 // multiply operation, we can go ahead and do this transform.
6174 return SDValue();
6175
6176 MulVT = getTypeToTransformTo(*DAG.getContext(), VT);
6177 if (MulVT.getSizeInBits() < (2 * EltBits) ||
6178 !isOperationLegal(ISD::MUL, MulVT))
6179 return SDValue();
6180 }
6181
6182 // If the sdiv has an 'exact' bit we can use a simpler lowering.
6183 if (N->getFlags().hasExact())
6184 return BuildExactSDIV(*this, N, dl, DAG, Created);
6185
6186 SmallVector<SDValue, 16> MagicFactors, Factors, Shifts, ShiftMasks;
6187
6188 auto BuildSDIVPattern = [&](ConstantSDNode *C) {
6189 if (C->isZero())
6190 return false;
6191
6192 const APInt &Divisor = C->getAPIntValue();
6194 int NumeratorFactor = 0;
6195 int ShiftMask = -1;
6196
6197 if (Divisor.isOne() || Divisor.isAllOnes()) {
6198 // If d is +1/-1, we just multiply the numerator by +1/-1.
6199 NumeratorFactor = Divisor.getSExtValue();
6200 magics.Magic = 0;
6201 magics.ShiftAmount = 0;
6202 ShiftMask = 0;
6203 } else if (Divisor.isStrictlyPositive() && magics.Magic.isNegative()) {
6204 // If d > 0 and m < 0, add the numerator.
6205 NumeratorFactor = 1;
6206 } else if (Divisor.isNegative() && magics.Magic.isStrictlyPositive()) {
6207 // If d < 0 and m > 0, subtract the numerator.
6208 NumeratorFactor = -1;
6209 }
6210
6211 MagicFactors.push_back(DAG.getConstant(magics.Magic, dl, SVT));
6212 Factors.push_back(DAG.getConstant(NumeratorFactor, dl, SVT));
6213 Shifts.push_back(DAG.getConstant(magics.ShiftAmount, dl, ShSVT));
6214 ShiftMasks.push_back(DAG.getConstant(ShiftMask, dl, SVT));
6215 return true;
6216 };
6217
6218 SDValue N0 = N->getOperand(0);
6219 SDValue N1 = N->getOperand(1);
6220
6221 // Collect the shifts / magic values from each element.
6222 if (!ISD::matchUnaryPredicate(N1, BuildSDIVPattern))
6223 return SDValue();
6224
6225 SDValue MagicFactor, Factor, Shift, ShiftMask;
6226 if (N1.getOpcode() == ISD::BUILD_VECTOR) {
6227 MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);
6228 Factor = DAG.getBuildVector(VT, dl, Factors);
6229 Shift = DAG.getBuildVector(ShVT, dl, Shifts);
6230 ShiftMask = DAG.getBuildVector(VT, dl, ShiftMasks);
6231 } else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {
6232 assert(MagicFactors.size() == 1 && Factors.size() == 1 &&
6233 Shifts.size() == 1 && ShiftMasks.size() == 1 &&
6234 "Expected matchUnaryPredicate to return one element for scalable "
6235 "vectors");
6236 MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]);
6237 Factor = DAG.getSplatVector(VT, dl, Factors[0]);
6238 Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]);
6239 ShiftMask = DAG.getSplatVector(VT, dl, ShiftMasks[0]);
6240 } else {
6241 assert(isa<ConstantSDNode>(N1) && "Expected a constant");
6242 MagicFactor = MagicFactors[0];
6243 Factor = Factors[0];
6244 Shift = Shifts[0];
6245 ShiftMask = ShiftMasks[0];
6246 }
6247
6248 // Multiply the numerator (operand 0) by the magic value.
6249 // FIXME: We should support doing a MUL in a wider type.
6250 auto GetMULHS = [&](SDValue X, SDValue Y) {
6251 // If the type isn't legal, use a wider mul of the type calculated
6252 // earlier.
6253 if (!isTypeLegal(VT)) {
6254 X = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, X);
6255 Y = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, Y);
6256 Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y);
6257 Y = DAG.getNode(ISD::SRL, dl, MulVT, Y,
6258 DAG.getShiftAmountConstant(EltBits, MulVT, dl));
6259 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);
6260 }
6261
6262 if (isOperationLegalOrCustom(ISD::MULHS, VT, IsAfterLegalization))
6263 return DAG.getNode(ISD::MULHS, dl, VT, X, Y);
6264 if (isOperationLegalOrCustom(ISD::SMUL_LOHI, VT, IsAfterLegalization)) {
6265 SDValue LoHi =
6266 DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);
6267 return SDValue(LoHi.getNode(), 1);
6268 }
6269 // If type twice as wide legal, widen and use a mul plus a shift.
6270 unsigned Size = VT.getScalarSizeInBits();
6271 EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), Size * 2);
6272 if (VT.isVector())
6273 WideVT = EVT::getVectorVT(*DAG.getContext(), WideVT,
6275 if (isOperationLegalOrCustom(ISD::MUL, WideVT)) {
6276 X = DAG.getNode(ISD::SIGN_EXTEND, dl, WideVT, X);
6277 Y = DAG.getNode(ISD::SIGN_EXTEND, dl, WideVT, Y);
6278 Y = DAG.getNode(ISD::MUL, dl, WideVT, X, Y);
6279 Y = DAG.getNode(ISD::SRL, dl, WideVT, Y,
6280 DAG.getShiftAmountConstant(EltBits, WideVT, dl));
6281 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);
6282 }
6283 return SDValue();
6284 };
6285
6286 SDValue Q = GetMULHS(N0, MagicFactor);
6287 if (!Q)
6288 return SDValue();
6289
6290 Created.push_back(Q.getNode());
6291
6292 // (Optionally) Add/subtract the numerator using Factor.
6293 Factor = DAG.getNode(ISD::MUL, dl, VT, N0, Factor);
6294 Created.push_back(Factor.getNode());
6295 Q = DAG.getNode(ISD::ADD, dl, VT, Q, Factor);
6296 Created.push_back(Q.getNode());
6297
6298 // Shift right algebraic by shift value.
6299 Q = DAG.getNode(ISD::SRA, dl, VT, Q, Shift);
6300 Created.push_back(Q.getNode());
6301
6302 // Extract the sign bit, mask it and add it to the quotient.
6303 SDValue SignShift = DAG.getConstant(EltBits - 1, dl, ShVT);
6304 SDValue T = DAG.getNode(ISD::SRL, dl, VT, Q, SignShift);
6305 Created.push_back(T.getNode());
6306 T = DAG.getNode(ISD::AND, dl, VT, T, ShiftMask);
6307 Created.push_back(T.getNode());
6308 return DAG.getNode(ISD::ADD, dl, VT, Q, T);
6309}
6310
6311/// Given an ISD::UDIV node expressing a divide by constant,
6312/// return a DAG expression to select that will generate the same value by
6313/// multiplying by a magic number.
6314/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
6316 bool IsAfterLegalization,
6317 SmallVectorImpl<SDNode *> &Created) const {
6318 SDLoc dl(N);
6319 EVT VT = N->getValueType(0);
6320 EVT SVT = VT.getScalarType();
6321 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
6322 EVT ShSVT = ShVT.getScalarType();
6323 unsigned EltBits = VT.getScalarSizeInBits();
6324 EVT MulVT;
6325
6326 // Check to see if we can do this.
6327 // FIXME: We should be more aggressive here.
6328 if (!isTypeLegal(VT)) {
6329 // Limit this to simple scalars for now.
6330 if (VT.isVector() || !VT.isSimple())
6331 return SDValue();
6332
6333 // If this type will be promoted to a large enough type with a legal
6334 // multiply operation, we can go ahead and do this transform.
6336 return SDValue();
6337
6338 MulVT = getTypeToTransformTo(*DAG.getContext(), VT);
6339 if (MulVT.getSizeInBits() < (2 * EltBits) ||
6340 !isOperationLegal(ISD::MUL, MulVT))
6341 return SDValue();
6342 }
6343
6344 SDValue N0 = N->getOperand(0);
6345 SDValue N1 = N->getOperand(1);
6346
6347 // Try to use leading zeros of the dividend to reduce the multiplier and
6348 // avoid expensive fixups.
6349 // TODO: Support vectors.
6350 unsigned LeadingZeros = 0;
6351 if (!VT.isVector() && isa<ConstantSDNode>(N1)) {
6352 assert(!isOneConstant(N1) && "Unexpected divisor");
6353 LeadingZeros = DAG.computeKnownBits(N0).countMinLeadingZeros();
6354 // UnsignedDivisionByConstantInfo doesn't work correctly if leading zeros in
6355 // the dividend exceeds the leading zeros for the divisor.
6356 LeadingZeros = std::min(LeadingZeros, N1->getAsAPIntVal().countl_zero());
6357 }
6358
6359 bool UseNPQ = false, UsePreShift = false, UsePostShift = false;
6360 SmallVector<SDValue, 16> PreShifts, PostShifts, MagicFactors, NPQFactors;
6361
6362 auto BuildUDIVPattern = [&](ConstantSDNode *C) {
6363 if (C->isZero())
6364 return false;
6365 const APInt& Divisor = C->getAPIntValue();
6366
6367 SDValue PreShift, MagicFactor, NPQFactor, PostShift;
6368
6369 // Magic algorithm doesn't work for division by 1. We need to emit a select
6370 // at the end.
6371 if (Divisor.isOne()) {
6372 PreShift = PostShift = DAG.getUNDEF(ShSVT);
6373 MagicFactor = NPQFactor = DAG.getUNDEF(SVT);
6374 } else {
6376 UnsignedDivisionByConstantInfo::get(Divisor, LeadingZeros);
6377
6378 MagicFactor = DAG.getConstant(magics.Magic, dl, SVT);
6379
6380 assert(magics.PreShift < Divisor.getBitWidth() &&
6381 "We shouldn't generate an undefined shift!");
6382 assert(magics.PostShift < Divisor.getBitWidth() &&
6383 "We shouldn't generate an undefined shift!");
6384 assert((!magics.IsAdd || magics.PreShift == 0) &&
6385 "Unexpected pre-shift");
6386 PreShift = DAG.getConstant(magics.PreShift, dl, ShSVT);
6387 PostShift = DAG.getConstant(magics.PostShift, dl, ShSVT);
6388 NPQFactor = DAG.getConstant(
6389 magics.IsAdd ? APInt::getOneBitSet(EltBits, EltBits - 1)
6390 : APInt::getZero(EltBits),
6391 dl, SVT);
6392 UseNPQ |= magics.IsAdd;
6393 UsePreShift |= magics.PreShift != 0;
6394 UsePostShift |= magics.PostShift != 0;
6395 }
6396
6397 PreShifts.push_back(PreShift);
6398 MagicFactors.push_back(MagicFactor);
6399 NPQFactors.push_back(NPQFactor);
6400 PostShifts.push_back(PostShift);
6401 return true;
6402 };
6403
6404 // Collect the shifts/magic values from each element.
6405 if (!ISD::matchUnaryPredicate(N1, BuildUDIVPattern))
6406 return SDValue();
6407
6408 SDValue PreShift, PostShift, MagicFactor, NPQFactor;
6409 if (N1.getOpcode() == ISD::BUILD_VECTOR) {
6410 PreShift = DAG.getBuildVector(ShVT, dl, PreShifts);
6411 MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);
6412 NPQFactor = DAG.getBuildVector(VT, dl, NPQFactors);
6413 PostShift = DAG.getBuildVector(ShVT, dl, PostShifts);
6414 } else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {
6415 assert(PreShifts.size() == 1 && MagicFactors.size() == 1 &&
6416 NPQFactors.size() == 1 && PostShifts.size() == 1 &&
6417 "Expected matchUnaryPredicate to return one for scalable vectors");
6418 PreShift = DAG.getSplatVector(ShVT, dl, PreShifts[0]);
6419 MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]);
6420 NPQFactor = DAG.getSplatVector(VT, dl, NPQFactors[0]);
6421 PostShift = DAG.getSplatVector(ShVT, dl, PostShifts[0]);
6422 } else {
6423 assert(isa<ConstantSDNode>(N1) && "Expected a constant");
6424 PreShift = PreShifts[0];
6425 MagicFactor = MagicFactors[0];
6426 PostShift = PostShifts[0];
6427 }
6428
6429 SDValue Q = N0;
6430 if (UsePreShift) {
6431 Q = DAG.getNode(ISD::SRL, dl, VT, Q, PreShift);
6432 Created.push_back(Q.getNode());
6433 }
6434
6435 // FIXME: We should support doing a MUL in a wider type.
6436 auto GetMULHU = [&](SDValue X, SDValue Y) {
6437 // If the type isn't legal, use a wider mul of the type calculated
6438 // earlier.
6439 if (!isTypeLegal(VT)) {
6440 X = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, X);
6441 Y = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, Y);
6442 Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y);
6443 Y = DAG.getNode(ISD::SRL, dl, MulVT, Y,
6444 DAG.getShiftAmountConstant(EltBits, MulVT, dl));
6445 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);
6446 }
6447
6448 if (isOperationLegalOrCustom(ISD::MULHU, VT, IsAfterLegalization))
6449 return DAG.getNode(ISD::MULHU, dl, VT, X, Y);
6450 if (isOperationLegalOrCustom(ISD::UMUL_LOHI, VT, IsAfterLegalization)) {
6451 SDValue LoHi =
6452 DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);
6453 return SDValue(LoHi.getNode(), 1);
6454 }
6455 // If type twice as wide legal, widen and use a mul plus a shift.
6456 unsigned Size = VT.getScalarSizeInBits();
6457 EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), Size * 2);
6458 if (VT.isVector())
6459 WideVT = EVT::getVectorVT(*DAG.getContext(), WideVT,
6461 if (isOperationLegalOrCustom(ISD::MUL, WideVT)) {
6462 X = DAG.getNode(ISD::ZERO_EXTEND, dl, WideVT, X);
6463 Y = DAG.getNode(ISD::ZERO_EXTEND, dl, WideVT, Y);
6464 Y = DAG.getNode(ISD::MUL, dl, WideVT, X, Y);
6465 Y = DAG.getNode(ISD::SRL, dl, WideVT, Y,
6466 DAG.getShiftAmountConstant(EltBits, WideVT, dl));
6467 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);
6468 }
6469 return SDValue(); // No mulhu or equivalent
6470 };
6471
6472 // Multiply the numerator (operand 0) by the magic value.
6473 Q = GetMULHU(Q, MagicFactor);
6474 if (!Q)
6475 return SDValue();
6476
6477 Created.push_back(Q.getNode());
6478
6479 if (UseNPQ) {
6480 SDValue NPQ = DAG.getNode(ISD::SUB, dl, VT, N0, Q);
6481 Created.push_back(NPQ.getNode());
6482
6483 // For vectors we might have a mix of non-NPQ/NPQ paths, so use
6484 // MULHU to act as a SRL-by-1 for NPQ, else multiply by zero.
6485 if (VT.isVector())
6486 NPQ = GetMULHU(NPQ, NPQFactor);
6487 else
6488 NPQ = DAG.getNode(ISD::SRL, dl, VT, NPQ, DAG.getConstant(1, dl, ShVT));
6489
6490 Created.push_back(NPQ.getNode());
6491
6492 Q = DAG.getNode(ISD::ADD, dl, VT, NPQ, Q);
6493 Created.push_back(Q.getNode());
6494 }
6495
6496 if (UsePostShift) {
6497 Q = DAG.getNode(ISD::SRL, dl, VT, Q, PostShift);
6498 Created.push_back(Q.getNode());
6499 }
6500
6501 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
6502
6503 SDValue One = DAG.getConstant(1, dl, VT);
6504 SDValue IsOne = DAG.getSetCC(dl, SetCCVT, N1, One, ISD::SETEQ);
6505 return DAG.getSelect(dl, VT, IsOne, N0, Q);
6506}
6507
6508/// If all values in Values that *don't* match the predicate are same 'splat'
6509/// value, then replace all values with that splat value.
6510/// Else, if AlternativeReplacement was provided, then replace all values that
6511/// do match predicate with AlternativeReplacement value.
6512static void
6514 std::function<bool(SDValue)> Predicate,
6515 SDValue AlternativeReplacement = SDValue()) {
6516 SDValue Replacement;
6517 // Is there a value for which the Predicate does *NOT* match? What is it?
6518 auto SplatValue = llvm::find_if_not(Values, Predicate);
6519 if (SplatValue != Values.end()) {
6520 // Does Values consist only of SplatValue's and values matching Predicate?
6521 if (llvm::all_of(Values, [Predicate, SplatValue](SDValue Value) {
6522 return Value == *SplatValue || Predicate(Value);
6523 })) // Then we shall replace values matching predicate with SplatValue.
6524 Replacement = *SplatValue;
6525 }
6526 if (!Replacement) {
6527 // Oops, we did not find the "baseline" splat value.
6528 if (!AlternativeReplacement)
6529 return; // Nothing to do.
6530 // Let's replace with provided value then.
6531 Replacement = AlternativeReplacement;
6532 }
6533 std::replace_if(Values.begin(), Values.end(), Predicate, Replacement);
6534}
6535
6536/// Given an ISD::UREM used only by an ISD::SETEQ or ISD::SETNE
6537/// where the divisor is constant and the comparison target is zero,
6538/// return a DAG expression that will generate the same comparison result
6539/// using only multiplications, additions and shifts/rotations.
6540/// Ref: "Hacker's Delight" 10-17.
6541SDValue TargetLowering::buildUREMEqFold(EVT SETCCVT, SDValue REMNode,
6542 SDValue CompTargetNode,
6544 DAGCombinerInfo &DCI,
6545 const SDLoc &DL) const {
6547 if (SDValue Folded = prepareUREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
6548 DCI, DL, Built)) {
6549 for (SDNode *N : Built)
6550 DCI.AddToWorklist(N);
6551 return Folded;
6552 }
6553
6554 return SDValue();
6555}
6556
6557SDValue
6558TargetLowering::prepareUREMEqFold(EVT SETCCVT, SDValue REMNode,
6559 SDValue CompTargetNode, ISD::CondCode Cond,
6560 DAGCombinerInfo &DCI, const SDLoc &DL,
6561 SmallVectorImpl<SDNode *> &Created) const {
6562 // fold (seteq/ne (urem N, D), 0) -> (setule/ugt (rotr (mul N, P), K), Q)
6563 // - D must be constant, with D = D0 * 2^K where D0 is odd
6564 // - P is the multiplicative inverse of D0 modulo 2^W
6565 // - Q = floor(((2^W) - 1) / D)
6566 // where W is the width of the common type of N and D.
6567 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
6568 "Only applicable for (in)equality comparisons.");
6569
6570 SelectionDAG &DAG = DCI.DAG;
6571
6572 EVT VT = REMNode.getValueType();
6573 EVT SVT = VT.getScalarType();
6574 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout(), !DCI.isBeforeLegalize());
6575 EVT ShSVT = ShVT.getScalarType();
6576
6577 // If MUL is unavailable, we cannot proceed in any case.
6578 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT))
6579 return SDValue();
6580
6581 bool ComparingWithAllZeros = true;
6582 bool AllComparisonsWithNonZerosAreTautological = true;
6583 bool HadTautologicalLanes = false;
6584 bool AllLanesAreTautological = true;
6585 bool HadEvenDivisor = false;
6586 bool AllDivisorsArePowerOfTwo = true;
6587 bool HadTautologicalInvertedLanes = false;
6588 SmallVector<SDValue, 16> PAmts, KAmts, QAmts, IAmts;
6589
6590 auto BuildUREMPattern = [&](ConstantSDNode *CDiv, ConstantSDNode *CCmp) {
6591 // Division by 0 is UB. Leave it to be constant-folded elsewhere.
6592 if (CDiv->isZero())
6593 return false;
6594
6595 const APInt &D = CDiv->getAPIntValue();
6596 const APInt &Cmp = CCmp->getAPIntValue();
6597
6598 ComparingWithAllZeros &= Cmp.isZero();
6599
6600 // x u% C1` is *always* less than C1. So given `x u% C1 == C2`,
6601 // if C2 is not less than C1, the comparison is always false.
6602 // But we will only be able to produce the comparison that will give the
6603 // opposive tautological answer. So this lane would need to be fixed up.
6604 bool TautologicalInvertedLane = D.ule(Cmp);
6605 HadTautologicalInvertedLanes |= TautologicalInvertedLane;
6606
6607 // If all lanes are tautological (either all divisors are ones, or divisor
6608 // is not greater than the constant we are comparing with),
6609 // we will prefer to avoid the fold.
6610 bool TautologicalLane = D.isOne() || TautologicalInvertedLane;
6611 HadTautologicalLanes |= TautologicalLane;
6612 AllLanesAreTautological &= TautologicalLane;
6613
6614 // If we are comparing with non-zero, we need'll need to subtract said
6615 // comparison value from the LHS. But there is no point in doing that if
6616 // every lane where we are comparing with non-zero is tautological..
6617 if (!Cmp.isZero())
6618 AllComparisonsWithNonZerosAreTautological &= TautologicalLane;
6619
6620 // Decompose D into D0 * 2^K
6621 unsigned K = D.countr_zero();
6622 assert((!D.isOne() || (K == 0)) && "For divisor '1' we won't rotate.");
6623 APInt D0 = D.lshr(K);
6624
6625 // D is even if it has trailing zeros.
6626 HadEvenDivisor |= (K != 0);
6627 // D is a power-of-two if D0 is one.
6628 // If all divisors are power-of-two, we will prefer to avoid the fold.
6629 AllDivisorsArePowerOfTwo &= D0.isOne();
6630
6631 // P = inv(D0, 2^W)
6632 // 2^W requires W + 1 bits, so we have to extend and then truncate.
6633 unsigned W = D.getBitWidth();
6634 APInt P = D0.zext(W + 1)
6636 .trunc(W);
6637 assert(!P.isZero() && "No multiplicative inverse!"); // unreachable
6638 assert((D0 * P).isOne() && "Multiplicative inverse basic check failed.");
6639
6640 // Q = floor((2^W - 1) u/ D)
6641 // R = ((2^W - 1) u% D)
6642 APInt Q, R;
6644
6645 // If we are comparing with zero, then that comparison constant is okay,
6646 // else it may need to be one less than that.
6647 if (Cmp.ugt(R))
6648 Q -= 1;
6649
6651 "We are expecting that K is always less than all-ones for ShSVT");
6652
6653 // If the lane is tautological the result can be constant-folded.
6654 if (TautologicalLane) {
6655 // Set P and K amount to a bogus values so we can try to splat them.
6656 P = 0;
6657 K = -1;
6658 // And ensure that comparison constant is tautological,
6659 // it will always compare true/false.
6660 Q = -1;
6661 }
6662
6663 PAmts.push_back(DAG.getConstant(P, DL, SVT));
6664 KAmts.push_back(
6665 DAG.getConstant(APInt(ShSVT.getSizeInBits(), K), DL, ShSVT));
6666 QAmts.push_back(DAG.getConstant(Q, DL, SVT));
6667 return true;
6668 };
6669
6670 SDValue N = REMNode.getOperand(0);
6671 SDValue D = REMNode.getOperand(1);
6672
6673 // Collect the values from each element.
6674 if (!ISD::matchBinaryPredicate(D, CompTargetNode, BuildUREMPattern))
6675 return SDValue();
6676
6677 // If all lanes are tautological, the result can be constant-folded.
6678 if (AllLanesAreTautological)
6679 return SDValue();
6680
6681 // If this is a urem by a powers-of-two, avoid the fold since it can be
6682 // best implemented as a bit test.
6683 if (AllDivisorsArePowerOfTwo)
6684 return SDValue();
6685
6686 SDValue PVal, KVal, QVal;
6687 if (D.getOpcode() == ISD::BUILD_VECTOR) {
6688 if (HadTautologicalLanes) {
6689 // Try to turn PAmts into a splat, since we don't care about the values
6690 // that are currently '0'. If we can't, just keep '0'`s.
6692 // Try to turn KAmts into a splat, since we don't care about the values
6693 // that are currently '-1'. If we can't, change them to '0'`s.
6695 DAG.getConstant(0, DL, ShSVT));
6696 }
6697
6698 PVal = DAG.getBuildVector(VT, DL, PAmts);
6699 KVal = DAG.getBuildVector(ShVT, DL, KAmts);
6700 QVal = DAG.getBuildVector(VT, DL, QAmts);
6701 } else if (D.getOpcode() == ISD::SPLAT_VECTOR) {
6702 assert(PAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 &&
6703 "Expected matchBinaryPredicate to return one element for "
6704 "SPLAT_VECTORs");
6705 PVal = DAG.getSplatVector(VT, DL, PAmts[0]);
6706 KVal = DAG.getSplatVector(ShVT, DL, KAmts[0]);
6707 QVal = DAG.getSplatVector(VT, DL, QAmts[0]);
6708 } else {
6709 PVal = PAmts[0];
6710 KVal = KAmts[0];
6711 QVal = QAmts[0];
6712 }
6713
6714 if (!ComparingWithAllZeros && !AllComparisonsWithNonZerosAreTautological) {
6715 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::SUB, VT))
6716 return SDValue(); // FIXME: Could/should use `ISD::ADD`?
6717 assert(CompTargetNode.getValueType() == N.getValueType() &&
6718 "Expecting that the types on LHS and RHS of comparisons match.");
6719 N = DAG.getNode(ISD::SUB, DL, VT, N, CompTargetNode);
6720 }
6721
6722 // (mul N, P)
6723 SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal);
6724 Created.push_back(Op0.getNode());
6725
6726 // Rotate right only if any divisor was even. We avoid rotates for all-odd
6727 // divisors as a performance improvement, since rotating by 0 is a no-op.
6728 if (HadEvenDivisor) {
6729 // We need ROTR to do this.
6730 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ROTR, VT))
6731 return SDValue();
6732 // UREM: (rotr (mul N, P), K)
6733 Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal);
6734 Created.push_back(Op0.getNode());
6735 }
6736
6737 // UREM: (setule/setugt (rotr (mul N, P), K), Q)
6738 SDValue NewCC =
6739 DAG.getSetCC(DL, SETCCVT, Op0, QVal,
6741 if (!HadTautologicalInvertedLanes)
6742 return NewCC;
6743
6744 // If any lanes previously compared always-false, the NewCC will give
6745 // always-true result for them, so we need to fixup those lanes.
6746 // Or the other way around for inequality predicate.
6747 assert(VT.isVector() && "Can/should only get here for vectors.");
6748 Created.push_back(NewCC.getNode());
6749
6750 // x u% C1` is *always* less than C1. So given `x u% C1 == C2`,
6751 // if C2 is not less than C1, the comparison is always false.
6752 // But we have produced the comparison that will give the
6753 // opposive tautological answer. So these lanes would need to be fixed up.
6754 SDValue TautologicalInvertedChannels =
6755 DAG.getSetCC(DL, SETCCVT, D, CompTargetNode, ISD::SETULE);
6756 Created.push_back(TautologicalInvertedChannels.getNode());
6757
6758 // NOTE: we avoid letting illegal types through even if we're before legalize
6759 // ops – legalization has a hard time producing good code for this.
6760 if (isOperationLegalOrCustom(ISD::VSELECT, SETCCVT)) {
6761 // If we have a vector select, let's replace the comparison results in the
6762 // affected lanes with the correct tautological result.
6763 SDValue Replacement = DAG.getBoolConstant(Cond == ISD::SETEQ ? false : true,
6764 DL, SETCCVT, SETCCVT);
6765 return DAG.getNode(ISD::VSELECT, DL, SETCCVT, TautologicalInvertedChannels,
6766 Replacement, NewCC);
6767 }
6768
6769 // Else, we can just invert the comparison result in the appropriate lanes.
6770 //
6771 // NOTE: see the note above VSELECT above.
6772 if (isOperationLegalOrCustom(ISD::XOR, SETCCVT))
6773 return DAG.getNode(ISD::XOR, DL, SETCCVT, NewCC,
6774 TautologicalInvertedChannels);
6775
6776 return SDValue(); // Don't know how to lower.
6777}
6778
6779/// Given an ISD::SREM used only by an ISD::SETEQ or ISD::SETNE
6780/// where the divisor is constant and the comparison target is zero,
6781/// return a DAG expression that will generate the same comparison result
6782/// using only multiplications, additions and shifts/rotations.
6783/// Ref: "Hacker's Delight" 10-17.
6784SDValue TargetLowering::buildSREMEqFold(EVT SETCCVT, SDValue REMNode,
6785 SDValue CompTargetNode,
6787 DAGCombinerInfo &DCI,
6788 const SDLoc &DL) const {
6790 if (SDValue Folded = prepareSREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
6791 DCI, DL, Built)) {
6792 assert(Built.size() <= 7 && "Max size prediction failed.");
6793 for (SDNode *N : Built)
6794 DCI.AddToWorklist(N);
6795 return Folded;
6796 }
6797
6798 return SDValue();
6799}
6800
6801SDValue
6802TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode,
6803 SDValue CompTargetNode, ISD::CondCode Cond,
6804 DAGCombinerInfo &DCI, const SDLoc &DL,
6805 SmallVectorImpl<SDNode *> &Created) const {
6806 // Derived from Hacker's Delight, 2nd Edition, by Hank Warren. Section 10-17.
6807 // Fold:
6808 // (seteq/ne (srem N, D), 0)
6809 // To:
6810 // (setule/ugt (rotr (add (mul N, P), A), K), Q)
6811 //
6812 // - D must be constant, with D = D0 * 2^K where D0 is odd
6813 // - P is the multiplicative inverse of D0 modulo 2^W
6814 // - A = bitwiseand(floor((2^(W - 1) - 1) / D0), (-(2^k)))
6815 // - Q = floor((2 * A) / (2^K))
6816 // where W is the width of the common type of N and D.
6817 //
6818 // When D is a power of two (and thus D0 is 1), the normal
6819 // formula for A and Q don't apply, because the derivation
6820 // depends on D not dividing 2^(W-1), and thus theorem ZRS
6821 // does not apply. This specifically fails when N = INT_MIN.
6822 //
6823 // Instead, for power-of-two D, we use:
6824 // - A = 2^(W-1)
6825 // |-> Order-preserving map from [-2^(W-1), 2^(W-1) - 1] to [0,2^W - 1])
6826 // - Q = 2^(W-K) - 1
6827 // |-> Test that the top K bits are zero after rotation
6828 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
6829 "Only applicable for (in)equality comparisons.");
6830
6831 SelectionDAG &DAG = DCI.DAG;
6832
6833 EVT VT = REMNode.getValueType();
6834 EVT SVT = VT.getScalarType();
6835 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout(), !DCI.isBeforeLegalize());
6836 EVT ShSVT = ShVT.getScalarType();
6837
6838 // If we are after ops legalization, and MUL is unavailable, we can not
6839 // proceed.
6840 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT))
6841 return SDValue();
6842
6843 // TODO: Could support comparing with non-zero too.
6844 ConstantSDNode *CompTarget = isConstOrConstSplat(CompTargetNode);
6845 if (!CompTarget || !CompTarget->isZero())
6846 return SDValue();
6847
6848 bool HadIntMinDivisor = false;
6849 bool HadOneDivisor = false;
6850 bool AllDivisorsAreOnes = true;
6851 bool HadEvenDivisor = false;
6852 bool NeedToApplyOffset = false;
6853 bool AllDivisorsArePowerOfTwo = true;
6854 SmallVector<SDValue, 16> PAmts, AAmts, KAmts, QAmts;
6855
6856 auto BuildSREMPattern = [&](ConstantSDNode *C) {
6857 // Division by 0 is UB. Leave it to be constant-folded elsewhere.
6858 if (C->isZero())
6859 return false;
6860
6861 // FIXME: we don't fold `rem %X, -C` to `rem %X, C` in DAGCombine.
6862
6863 // WARNING: this fold is only valid for positive divisors!
6864 APInt D = C->getAPIntValue();
6865 if (D.isNegative())
6866 D.negate(); // `rem %X, -C` is equivalent to `rem %X, C`
6867
6868 HadIntMinDivisor |= D.isMinSignedValue();
6869
6870 // If all divisors are ones, we will prefer to avoid the fold.
6871 HadOneDivisor |= D.isOne();
6872 AllDivisorsAreOnes &= D.isOne();
6873
6874 // Decompose D into D0 * 2^K
6875 unsigned K = D.countr_zero();
6876 assert((!D.isOne() || (K == 0)) && "For divisor '1' we won't rotate.");
6877 APInt D0 = D.lshr(K);
6878
6879 if (!D.isMinSignedValue()) {
6880 // D is even if it has trailing zeros; unless it's INT_MIN, in which case
6881 // we don't care about this lane in this fold, we'll special-handle it.
6882 HadEvenDivisor |= (K != 0);
6883 }
6884
6885 // D is a power-of-two if D0 is one. This includes INT_MIN.
6886 // If all divisors are power-of-two, we will prefer to avoid the fold.
6887 AllDivisorsArePowerOfTwo &= D0.isOne();
6888
6889 // P = inv(D0, 2^W)
6890 // 2^W requires W + 1 bits, so we have to extend and then truncate.
6891 unsigned W = D.getBitWidth();
6892 APInt P = D0.zext(W + 1)
6894 .trunc(W);
6895 assert(!P.isZero() && "No multiplicative inverse!"); // unreachable
6896 assert((D0 * P).isOne() && "Multiplicative inverse basic check failed.");
6897
6898 // A = floor((2^(W - 1) - 1) / D0) & -2^K
6900 A.clearLowBits(K);
6901
6902 if (!D.isMinSignedValue()) {
6903 // If divisor INT_MIN, then we don't care about this lane in this fold,
6904 // we'll special-handle it.
6905 NeedToApplyOffset |= A != 0;
6906 }
6907
6908 // Q = floor((2 * A) / (2^K))
6909 APInt Q = (2 * A).udiv(APInt::getOneBitSet(W, K));
6910
6911 // If D was a power of two, apply the alternate constant derivation.
6912 if (D0.isOne()) {
6913 // A = 2^(W-1)
6915 // - Q = 2^(W-K) - 1
6916 Q = APInt::getAllOnes(W - K).zext(W);
6917 }
6918
6920 "We are expecting that A is always less than all-ones for SVT");
6922 "We are expecting that K is always less than all-ones for ShSVT");
6923
6924 // If the divisor is 1 the result can be constant-folded. Likewise, we
6925 // don't care about INT_MIN lanes, those can be set to undef if appropriate.
6926 if (D.isOne()) {
6927 // Set P, A and K to a bogus values so we can try to splat them.
6928 P = 0;
6929 A = -1;
6930 K = -1;
6931
6932 // x ?% 1 == 0 <--> true <--> x u<= -1
6933 Q = -1;
6934 }
6935
6936 PAmts.push_back(DAG.getConstant(P, DL, SVT));
6937 AAmts.push_back(DAG.getConstant(A, DL, SVT));
6938 KAmts.push_back(
6939 DAG.getConstant(APInt(ShSVT.getSizeInBits(), K), DL, ShSVT));
6940 QAmts.push_back(DAG.getConstant(Q, DL, SVT));
6941 return true;
6942 };
6943
6944 SDValue N = REMNode.getOperand(0);
6945 SDValue D = REMNode.getOperand(1);
6946
6947 // Collect the values from each element.
6948 if (!ISD::matchUnaryPredicate(D, BuildSREMPattern))
6949 return SDValue();
6950
6951 // If this is a srem by a one, avoid the fold since it can be constant-folded.
6952 if (AllDivisorsAreOnes)
6953 return SDValue();
6954
6955 // If this is a srem by a powers-of-two (including INT_MIN), avoid the fold
6956 // since it can be best implemented as a bit test.
6957 if (AllDivisorsArePowerOfTwo)
6958 return SDValue();
6959
6960 SDValue PVal, AVal, KVal, QVal;
6961 if (D.getOpcode() == ISD::BUILD_VECTOR) {
6962 if (HadOneDivisor) {
6963 // Try to turn PAmts into a splat, since we don't care about the values
6964 // that are currently '0'. If we can't, just keep '0'`s.
6966 // Try to turn AAmts into a splat, since we don't care about the
6967 // values that are currently '-1'. If we can't, change them to '0'`s.
6969 DAG.getConstant(0, DL, SVT));
6970 // Try to turn KAmts into a splat, since we don't care about the values
6971 // that are currently '-1'. If we can't, change them to '0'`s.
6973 DAG.getConstant(0, DL, ShSVT));
6974 }
6975
6976 PVal = DAG.getBuildVector(VT, DL, PAmts);
6977 AVal = DAG.getBuildVector(VT, DL, AAmts);
6978 KVal = DAG.getBuildVector(ShVT, DL, KAmts);
6979 QVal = DAG.getBuildVector(VT, DL, QAmts);
6980 } else if (D.getOpcode() == ISD::SPLAT_VECTOR) {
6981 assert(PAmts.size() == 1 && AAmts.size() == 1 && KAmts.size() == 1 &&
6982 QAmts.size() == 1 &&
6983 "Expected matchUnaryPredicate to return one element for scalable "
6984 "vectors");
6985 PVal = DAG.getSplatVector(VT, DL, PAmts[0]);
6986 AVal = DAG.getSplatVector(VT, DL, AAmts[0]);
6987 KVal = DAG.getSplatVector(ShVT, DL, KAmts[0]);
6988 QVal = DAG.getSplatVector(VT, DL, QAmts[0]);
6989 } else {
6990 assert(isa<ConstantSDNode>(D) && "Expected a constant");
6991 PVal = PAmts[0];
6992 AVal = AAmts[0];
6993 KVal = KAmts[0];
6994 QVal = QAmts[0];
6995 }
6996
6997 // (mul N, P)
6998 SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal);
6999 Created.push_back(Op0.getNode());
7000
7001 if (NeedToApplyOffset) {
7002 // We need ADD to do this.
7003 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ADD, VT))
7004 return SDValue();
7005
7006 // (add (mul N, P), A)
7007 Op0 = DAG.getNode(ISD::ADD, DL, VT, Op0, AVal);
7008 Created.push_back(Op0.getNode());
7009 }
7010
7011 // Rotate right only if any divisor was even. We avoid rotates for all-odd
7012 // divisors as a performance improvement, since rotating by 0 is a no-op.
7013 if (HadEvenDivisor) {
7014 // We need ROTR to do this.
7015 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ROTR, VT))
7016 return SDValue();
7017 // SREM: (rotr (add (mul N, P), A), K)
7018 Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal);
7019 Created.push_back(Op0.getNode());
7020 }
7021
7022 // SREM: (setule/setugt (rotr (add (mul N, P), A), K), Q)
7023 SDValue Fold =
7024 DAG.getSetCC(DL, SETCCVT, Op0, QVal,
7026
7027 // If we didn't have lanes with INT_MIN divisor, then we're done.
7028 if (!HadIntMinDivisor)
7029 return Fold;
7030
7031 // That fold is only valid for positive divisors. Which effectively means,
7032 // it is invalid for INT_MIN divisors. So if we have such a lane,
7033 // we must fix-up results for said lanes.
7034 assert(VT.isVector() && "Can/should only get here for vectors.");
7035
7036 // NOTE: we avoid letting illegal types through even if we're before legalize
7037 // ops – legalization has a hard time producing good code for the code that
7038 // follows.
7039 if (!isOperationLegalOrCustom(ISD::SETCC, SETCCVT) ||
7043 return SDValue();
7044
7045 Created.push_back(Fold.getNode());
7046
7047 SDValue IntMin = DAG.getConstant(
7049 SDValue IntMax = DAG.getConstant(
7051 SDValue Zero =
7053
7054 // Which lanes had INT_MIN divisors? Divisor is constant, so const-folded.
7055 SDValue DivisorIsIntMin = DAG.getSetCC(DL, SETCCVT, D, IntMin, ISD::SETEQ);
7056 Created.push_back(DivisorIsIntMin.getNode());
7057
7058 // (N s% INT_MIN) ==/!= 0 <--> (N & INT_MAX) ==/!= 0
7059 SDValue Masked = DAG.getNode(ISD::AND, DL, VT, N, IntMax);
7060 Created.push_back(Masked.getNode());
7061 SDValue MaskedIsZero = DAG.getSetCC(DL, SETCCVT, Masked, Zero, Cond);
7062 Created.push_back(MaskedIsZero.getNode());
7063
7064 // To produce final result we need to blend 2 vectors: 'SetCC' and
7065 // 'MaskedIsZero'. If the divisor for channel was *NOT* INT_MIN, we pick
7066 // from 'Fold', else pick from 'MaskedIsZero'. Since 'DivisorIsIntMin' is
7067 // constant-folded, select can get lowered to a shuffle with constant mask.
7068 SDValue Blended = DAG.getNode(ISD::VSELECT, DL, SETCCVT, DivisorIsIntMin,
7069 MaskedIsZero, Fold);
7070
7071 return Blended;
7072}
7073
7076 if (!isa<ConstantSDNode>(Op.getOperand(0))) {
7077 DAG.getContext()->emitError("argument to '__builtin_return_address' must "
7078 "be a constant integer");
7079 return true;
7080 }
7081
7082 return false;
7083}
7084
7086 const DenormalMode &Mode) const {
7087 SDLoc DL(Op);
7088 EVT VT = Op.getValueType();
7089 EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
7090 SDValue FPZero = DAG.getConstantFP(0.0, DL, VT);
7091
7092 // This is specifically a check for the handling of denormal inputs, not the
7093 // result.
7094 if (Mode.Input == DenormalMode::PreserveSign ||
7095 Mode.Input == DenormalMode::PositiveZero) {
7096 // Test = X == 0.0
7097 return DAG.getSetCC(DL, CCVT, Op, FPZero, ISD::SETEQ);
7098 }
7099
7100 // Testing it with denormal inputs to avoid wrong estimate.
7101 //
7102 // Test = fabs(X) < SmallestNormal
7103 const fltSemantics &FltSem = DAG.EVTToAPFloatSemantics(VT);
7104 APFloat SmallestNorm = APFloat::getSmallestNormalized(FltSem);
7105 SDValue NormC = DAG.getConstantFP(SmallestNorm, DL, VT);
7106 SDValue Fabs = DAG.getNode(ISD::FABS, DL, VT, Op);
7107 return DAG.getSetCC(DL, CCVT, Fabs, NormC, ISD::SETLT);
7108}
7109
7111 bool LegalOps, bool OptForSize,
7113 unsigned Depth) const {
7114 // fneg is removable even if it has multiple uses.
7115 if (Op.getOpcode() == ISD::FNEG || Op.getOpcode() == ISD::VP_FNEG) {
7117 return Op.getOperand(0);
7118 }
7119
7120 // Don't recurse exponentially.
7122 return SDValue();
7123
7124 // Pre-increment recursion depth for use in recursive calls.
7125 ++Depth;
7126 const SDNodeFlags Flags = Op->getFlags();
7127 const TargetOptions &Options = DAG.getTarget().Options;
7128 EVT VT = Op.getValueType();
7129 unsigned Opcode = Op.getOpcode();
7130
7131 // Don't allow anything with multiple uses unless we know it is free.
7132 if (!Op.hasOneUse() && Opcode != ISD::ConstantFP) {
7133 bool IsFreeExtend = Opcode == ISD::FP_EXTEND &&
7134 isFPExtFree(VT, Op.getOperand(0).getValueType());
7135 if (!IsFreeExtend)
7136 return SDValue();
7137 }
7138
7139 auto RemoveDeadNode = [&](SDValue N) {
7140 if (N && N.getNode()->use_empty())
7141 DAG.RemoveDeadNode(N.getNode());
7142 };
7143
7144 SDLoc DL(Op);
7145
7146 // Because getNegatedExpression can delete nodes we need a handle to keep
7147 // temporary nodes alive in case the recursion manages to create an identical
7148 // node.
7149 std::list<HandleSDNode> Handles;
7150
7151 switch (Opcode) {
7152 case ISD::ConstantFP: {
7153 // Don't invert constant FP values after legalization unless the target says
7154 // the negated constant is legal.
7155 bool IsOpLegal =
7157 isFPImmLegal(neg(cast<ConstantFPSDNode>(Op)->getValueAPF()), VT,
7158 OptForSize);
7159
7160 if (LegalOps && !IsOpLegal)
7161 break;
7162
7163 APFloat V = cast<ConstantFPSDNode>(Op)->getValueAPF();
7164 V.changeSign();
7165 SDValue CFP = DAG.getConstantFP(V, DL, VT);
7166
7167 // If we already have the use of the negated floating constant, it is free
7168 // to negate it even it has multiple uses.
7169 if (!Op.hasOneUse() && CFP.use_empty())
7170 break;
7172 return CFP;
7173 }
7174 case ISD::BUILD_VECTOR: {
7175 // Only permit BUILD_VECTOR of constants.
7176 if (llvm::any_of(Op->op_values(), [&](SDValue N) {
7177 return !N.isUndef() && !isa<ConstantFPSDNode>(N);
7178 }))
7179 break;
7180
7181 bool IsOpLegal =
7184 llvm::all_of(Op->op_values(), [&](SDValue N) {
7185 return N.isUndef() ||
7186 isFPImmLegal(neg(cast<ConstantFPSDNode>(N)->getValueAPF()), VT,
7187 OptForSize);
7188 });
7189
7190 if (LegalOps && !IsOpLegal)
7191 break;
7192
7194 for (SDValue C : Op->op_values()) {
7195 if (C.isUndef()) {
7196 Ops.push_back(C);
7197 continue;
7198 }
7199 APFloat V = cast<ConstantFPSDNode>(C)->getValueAPF();
7200 V.changeSign();
7201 Ops.push_back(DAG.getConstantFP(V, DL, C.getValueType()));
7202 }
7204 return DAG.getBuildVector(VT, DL, Ops);
7205 }
7206 case ISD::FADD: {
7207 if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())
7208 break;
7209
7210 // After operation legalization, it might not be legal to create new FSUBs.
7211 if (LegalOps && !isOperationLegalOrCustom(ISD::FSUB, VT))
7212 break;
7213 SDValue X = Op.getOperand(0), Y = Op.getOperand(1);
7214
7215 // fold (fneg (fadd X, Y)) -> (fsub (fneg X), Y)
7217 SDValue NegX =
7218 getNegatedExpression(X, DAG, LegalOps, OptForSize, CostX, Depth);
7219 // Prevent this node from being deleted by the next call.
7220 if (NegX)
7221 Handles.emplace_back(NegX);
7222
7223 // fold (fneg (fadd X, Y)) -> (fsub (fneg Y), X)
7225 SDValue NegY =
7226 getNegatedExpression(Y, DAG, LegalOps, OptForSize, CostY, Depth);
7227
7228 // We're done with the handles.
7229 Handles.clear();
7230
7231 // Negate the X if its cost is less or equal than Y.
7232 if (NegX && (CostX <= CostY)) {
7233 Cost = CostX;
7234 SDValue N = DAG.getNode(ISD::FSUB, DL, VT, NegX, Y, Flags);
7235 if (NegY != N)
7236 RemoveDeadNode(NegY);
7237 return N;
7238 }
7239
7240 // Negate the Y if it is not expensive.
7241 if (NegY) {
7242 Cost = CostY;
7243 SDValue N = DAG.getNode(ISD::FSUB, DL, VT, NegY, X, Flags);
7244 if (NegX != N)
7245 RemoveDeadNode(NegX);
7246 return N;
7247 }
7248 break;
7249 }
7250 case ISD::FSUB: {
7251 // We can't turn -(A-B) into B-A when we honor signed zeros.
7252 if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())
7253 break;
7254
7255 SDValue X = Op.getOperand(0), Y = Op.getOperand(1);
7256 // fold (fneg (fsub 0, Y)) -> Y
7257 if (ConstantFPSDNode *C = isConstOrConstSplatFP(X, /*AllowUndefs*/ true))
7258 if (C->isZero()) {
7260 return Y;
7261 }
7262
7263 // fold (fneg (fsub X, Y)) -> (fsub Y, X)
7265 return DAG.getNode(ISD::FSUB, DL, VT, Y, X, Flags);
7266 }
7267 case ISD::FMUL:
7268 case ISD::FDIV: {
7269 SDValue X = Op.getOperand(0), Y = Op.getOperand(1);
7270
7271 // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y)
7273 SDValue NegX =
7274 getNegatedExpression(X, DAG, LegalOps, OptForSize, CostX, Depth);
7275 // Prevent this node from being deleted by the next call.
7276 if (NegX)
7277 Handles.emplace_back(NegX);
7278
7279 // fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y))
7281 SDValue NegY =
7282 getNegatedExpression(Y, DAG, LegalOps, OptForSize, CostY, Depth);
7283
7284 // We're done with the handles.
7285 Handles.clear();
7286
7287 // Negate the X if its cost is less or equal than Y.
7288 if (NegX && (CostX <= CostY)) {
7289 Cost = CostX;
7290 SDValue N = DAG.getNode(Opcode, DL, VT, NegX, Y, Flags);
7291 if (NegY != N)
7292 RemoveDeadNode(NegY);
7293 return N;
7294 }
7295
7296 // Ignore X * 2.0 because that is expected to be canonicalized to X + X.
7297 if (auto *C = isConstOrConstSplatFP(Op.getOperand(1)))
7298 if (C->isExactlyValue(2.0) && Op.getOpcode() == ISD::FMUL)
7299 break;
7300
7301 // Negate the Y if it is not expensive.
7302 if (NegY) {
7303 Cost = CostY;
7304 SDValue N = DAG.getNode(Opcode, DL, VT, X, NegY, Flags);
7305 if (NegX != N)
7306 RemoveDeadNode(NegX);
7307 return N;
7308 }
7309 break;
7310 }
7311 case ISD::FMA:
7312 case ISD::FMAD: {
7313 if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())
7314 break;
7315
7316 SDValue X = Op.getOperand(0), Y = Op.getOperand(1), Z = Op.getOperand(2);
7318 SDValue NegZ =
7319 getNegatedExpression(Z, DAG, LegalOps, OptForSize, CostZ, Depth);
7320 // Give up if fail to negate the Z.
7321 if (!NegZ)
7322 break;
7323
7324 // Prevent this node from being deleted by the next two calls.
7325 Handles.emplace_back(NegZ);
7326
7327 // fold (fneg (fma X, Y, Z)) -> (fma (fneg X), Y, (fneg Z))
7329 SDValue NegX =
7330 getNegatedExpression(X, DAG, LegalOps, OptForSize, CostX, Depth);
7331 // Prevent this node from being deleted by the next call.
7332 if (NegX)
7333 Handles.emplace_back(NegX);
7334
7335 // fold (fneg (fma X, Y, Z)) -> (fma X, (fneg Y), (fneg Z))
7337 SDValue NegY =
7338 getNegatedExpression(Y, DAG, LegalOps, OptForSize, CostY, Depth);
7339
7340 // We're done with the handles.
7341 Handles.clear();
7342
7343 // Negate the X if its cost is less or equal than Y.
7344 if (NegX && (CostX <= CostY)) {
7345 Cost = std::min(CostX, CostZ);
7346 SDValue N = DAG.getNode(Opcode, DL, VT, NegX, Y, NegZ, Flags);
7347 if (NegY != N)
7348 RemoveDeadNode(NegY);
7349 return N;
7350 }
7351
7352 // Negate the Y if it is not expensive.
7353 if (NegY) {
7354 Cost = std::min(CostY, CostZ);
7355 SDValue N = DAG.getNode(Opcode, DL, VT, X, NegY, NegZ, Flags);
7356 if (NegX != N)
7357 RemoveDeadNode(NegX);
7358 return N;
7359 }
7360 break;
7361 }
7362
7363 case ISD::FP_EXTEND:
7364 case ISD::FSIN:
7365 if (SDValue NegV = getNegatedExpression(Op.getOperand(0), DAG, LegalOps,
7366 OptForSize, Cost, Depth))
7367 return DAG.getNode(Opcode, DL, VT, NegV);
7368 break;
7369 case ISD::FP_ROUND:
7370 if (SDValue NegV = getNegatedExpression(Op.getOperand(0), DAG, LegalOps,
7371 OptForSize, Cost, Depth))
7372 return DAG.getNode(ISD::FP_ROUND, DL, VT, NegV, Op.getOperand(1));
7373 break;
7374 case ISD::SELECT:
7375 case ISD::VSELECT: {
7376 // fold (fneg (select C, LHS, RHS)) -> (select C, (fneg LHS), (fneg RHS))
7377 // iff at least one cost is cheaper and the other is neutral/cheaper
7378 SDValue LHS = Op.getOperand(1);
7380 SDValue NegLHS =
7381 getNegatedExpression(LHS, DAG, LegalOps, OptForSize, CostLHS, Depth);
7382 if (!NegLHS || CostLHS > NegatibleCost::Neutral) {
7383 RemoveDeadNode(NegLHS);
7384 break;
7385 }
7386
7387 // Prevent this node from being deleted by the next call.
7388 Handles.emplace_back(NegLHS);
7389
7390 SDValue RHS = Op.getOperand(2);
7392 SDValue NegRHS =
7393 getNegatedExpression(RHS, DAG, LegalOps, OptForSize, CostRHS, Depth);
7394
7395 // We're done with the handles.
7396 Handles.clear();
7397
7398 if (!NegRHS || CostRHS > NegatibleCost::Neutral ||
7399 (CostLHS != NegatibleCost::Cheaper &&
7400 CostRHS != NegatibleCost::Cheaper)) {
7401 RemoveDeadNode(NegLHS);
7402 RemoveDeadNode(NegRHS);
7403 break;
7404 }
7405
7406 Cost = std::min(CostLHS, CostRHS);
7407 return DAG.getSelect(DL, VT, Op.getOperand(0), NegLHS, NegRHS);
7408 }
7409 }
7410
7411 return SDValue();
7412}
7413
7414//===----------------------------------------------------------------------===//
7415// Legalization Utilities
7416//===----------------------------------------------------------------------===//
7417
7418bool TargetLowering::expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl,
7419 SDValue LHS, SDValue RHS,
7421 EVT HiLoVT, SelectionDAG &DAG,
7422 MulExpansionKind Kind, SDValue LL,
7423 SDValue LH, SDValue RL, SDValue RH) const {
7424 assert(Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI ||
7425 Opcode == ISD::SMUL_LOHI);
7426
7427 bool HasMULHS = (Kind == MulExpansionKind::Always) ||
7429 bool HasMULHU = (Kind == MulExpansionKind::Always) ||
7431 bool HasSMUL_LOHI = (Kind == MulExpansionKind::Always) ||
7433 bool HasUMUL_LOHI = (Kind == MulExpansionKind::Always) ||
7435
7436 if (!HasMULHU && !HasMULHS && !HasUMUL_LOHI && !HasSMUL_LOHI)
7437 return false;
7438
7439 unsigned OuterBitSize = VT.getScalarSizeInBits();
7440 unsigned InnerBitSize = HiLoVT.getScalarSizeInBits();
7441
7442 // LL, LH, RL, and RH must be either all NULL or all set to a value.
7443 assert((LL.getNode() && LH.getNode() && RL.getNode() && RH.getNode()) ||
7444 (!LL.getNode() && !LH.getNode() && !RL.getNode() && !RH.getNode()));
7445
7446 SDVTList VTs = DAG.getVTList(HiLoVT, HiLoVT);
7447 auto MakeMUL_LOHI = [&](SDValue L, SDValue R, SDValue &Lo, SDValue &Hi,
7448 bool Signed) -> bool {
7449 if ((Signed && HasSMUL_LOHI) || (!Signed && HasUMUL_LOHI)) {
7450 Lo = DAG.getNode(Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI, dl, VTs, L, R);
7451 Hi = SDValue(Lo.getNode(), 1);
7452 return true;
7453 }
7454 if ((Signed && HasMULHS) || (!Signed && HasMULHU)) {
7455 Lo = DAG.getNode(ISD::MUL, dl, HiLoVT, L, R);
7456 Hi = DAG.getNode(Signed ? ISD::MULHS : ISD::MULHU, dl, HiLoVT, L, R);
7457 return true;
7458 }
7459 return false;
7460 };
7461
7462 SDValue Lo, Hi;
7463
7464 if (!LL.getNode() && !RL.getNode() &&
7466 LL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LHS);
7467 RL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RHS);
7468 }
7469
7470 if (!LL.getNode())
7471 return false;
7472
7473 APInt HighMask = APInt::getHighBitsSet(OuterBitSize, InnerBitSize);
7474 if (DAG.MaskedValueIsZero(LHS, HighMask) &&
7475 DAG.MaskedValueIsZero(RHS, HighMask)) {
7476 // The inputs are both zero-extended.
7477 if (MakeMUL_LOHI(LL, RL, Lo, Hi, false)) {
7478 Result.push_back(Lo);
7479 Result.push_back(Hi);
7480 if (Opcode != ISD::MUL) {
7481 SDValue Zero = DAG.getConstant(0, dl, HiLoVT);
7482 Result.push_back(Zero);
7483 Result.push_back(Zero);
7484 }
7485 return true;
7486 }
7487 }
7488
7489 if (!VT.isVector() && Opcode == ISD::MUL &&
7490 DAG.ComputeMaxSignificantBits(LHS) <= InnerBitSize &&
7491 DAG.ComputeMaxSignificantBits(RHS) <= InnerBitSize) {
7492 // The input values are both sign-extended.
7493 // TODO non-MUL case?
7494 if (MakeMUL_LOHI(LL, RL, Lo, Hi, true)) {
7495 Result.push_back(Lo);
7496 Result.push_back(Hi);
7497 return true;
7498 }
7499 }
7500
7501 unsigned ShiftAmount = OuterBitSize - InnerBitSize;
7502 SDValue Shift = DAG.getShiftAmountConstant(ShiftAmount, VT, dl);
7503
7504 if (!LH.getNode() && !RH.getNode() &&
7507 LH = DAG.getNode(ISD::SRL, dl, VT, LHS, Shift);
7508 LH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LH);
7509 RH = DAG.getNode(ISD::SRL, dl, VT, RHS, Shift);
7510 RH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RH);
7511 }
7512
7513 if (!LH.getNode())
7514 return false;
7515
7516 if (!MakeMUL_LOHI(LL, RL, Lo, Hi, false))
7517 return false;
7518
7519 Result.push_back(Lo);
7520
7521 if (Opcode == ISD::MUL) {
7522 RH = DAG.getNode(ISD::MUL, dl, HiLoVT, LL, RH);
7523 LH = DAG.getNode(ISD::MUL, dl, HiLoVT, LH, RL);
7524 Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, RH);
7525 Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, LH);
7526 Result.push_back(Hi);
7527 return true;
7528 }
7529
7530 // Compute the full width result.
7531 auto Merge = [&](SDValue Lo, SDValue Hi) -> SDValue {
7532 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
7533 Hi = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi);
7534 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
7535 return DAG.getNode(ISD::OR, dl, VT, Lo, Hi);
7536 };
7537
7538 SDValue Next = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi);
7539 if (!MakeMUL_LOHI(LL, RH, Lo, Hi, false))
7540 return false;
7541
7542 // This is effectively the add part of a multiply-add of half-sized operands,
7543 // so it cannot overflow.
7544 Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi));
7545
7546 if (!MakeMUL_LOHI(LH, RL, Lo, Hi, false))
7547 return false;
7548
7549 SDValue Zero = DAG.getConstant(0, dl, HiLoVT);
7550 EVT BoolType = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
7551
7552 bool UseGlue = (isOperationLegalOrCustom(ISD::ADDC, VT) &&
7554 if (UseGlue)
7555 Next = DAG.getNode(ISD::ADDC, dl, DAG.getVTList(VT, MVT::Glue), Next,
7556 Merge(Lo, Hi));
7557 else
7558 Next = DAG.getNode(ISD::UADDO_CARRY, dl, DAG.getVTList(VT, BoolType), Next,
7559 Merge(Lo, Hi), DAG.getConstant(0, dl, BoolType));
7560
7561 SDValue Carry = Next.getValue(1);
7562 Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
7563 Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift);
7564
7565 if (!MakeMUL_LOHI(LH, RH, Lo, Hi, Opcode == ISD::SMUL_LOHI))
7566 return false;
7567
7568 if (UseGlue)
7569 Hi = DAG.getNode(ISD::ADDE, dl, DAG.getVTList(HiLoVT, MVT::Glue), Hi, Zero,
7570 Carry);
7571 else
7572 Hi = DAG.getNode(ISD::UADDO_CARRY, dl, DAG.getVTList(HiLoVT, BoolType), Hi,
7573 Zero, Carry);
7574
7575 Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi));
7576
7577 if (Opcode == ISD::SMUL_LOHI) {
7578 SDValue NextSub = DAG.getNode(ISD::SUB, dl, VT, Next,
7579 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RL));
7580 Next = DAG.getSelectCC(dl, LH, Zero, NextSub, Next, ISD::SETLT);
7581
7582 NextSub = DAG.getNode(ISD::SUB, dl, VT, Next,
7583 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LL));
7584 Next = DAG.getSelectCC(dl, RH, Zero, NextSub, Next, ISD::SETLT);
7585 }
7586
7587 Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
7588 Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift);
7589 Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
7590 return true;
7591}
7592
7594 SelectionDAG &DAG, MulExpansionKind Kind,
7595 SDValue LL, SDValue LH, SDValue RL,
7596 SDValue RH) const {
7598 bool Ok = expandMUL_LOHI(N->getOpcode(), N->getValueType(0), SDLoc(N),
7599 N->getOperand(0), N->getOperand(1), Result, HiLoVT,
7600 DAG, Kind, LL, LH, RL, RH);
7601 if (Ok) {
7602 assert(Result.size() == 2);
7603 Lo = Result[0];
7604 Hi = Result[1];
7605 }
7606 return Ok;
7607}
7608
7609// Optimize unsigned division or remainder by constants for types twice as large
7610// as a legal VT.
7611//
7612// If (1 << (BitWidth / 2)) % Constant == 1, then the remainder
7613// can be computed
7614// as:
7615// Sum += __builtin_uadd_overflow(Lo, High, &Sum);
7616// Remainder = Sum % Constant
7617// This is based on "Remainder by Summing Digits" from Hacker's Delight.
7618//
7619// For division, we can compute the remainder using the algorithm described
7620// above, subtract it from the dividend to get an exact multiple of Constant.
7621// Then multiply that extact multiply by the multiplicative inverse modulo
7622// (1 << (BitWidth / 2)) to get the quotient.
7623
7624// If Constant is even, we can shift right the dividend and the divisor by the
7625// number of trailing zeros in Constant before applying the remainder algorithm.
7626// If we're after the quotient, we can subtract this value from the shifted
7627// dividend and multiply by the multiplicative inverse of the shifted divisor.
7628// If we want the remainder, we shift the value left by the number of trailing
7629// zeros and add the bits that were shifted out of the dividend.
7632 EVT HiLoVT, SelectionDAG &DAG,
7633 SDValue LL, SDValue LH) const {
7634 unsigned Opcode = N->getOpcode();
7635 EVT VT = N->getValueType(0);
7636
7637 // TODO: Support signed division/remainder.
7638 if (Opcode == ISD::SREM || Opcode == ISD::SDIV || Opcode == ISD::SDIVREM)
7639 return false;
7640 assert(
7641 (Opcode == ISD::UREM || Opcode == ISD::UDIV || Opcode == ISD::UDIVREM) &&
7642 "Unexpected opcode");
7643
7644 auto *CN = dyn_cast<ConstantSDNode>(N->getOperand(1));
7645 if (!CN)
7646 return false;
7647
7648 APInt Divisor = CN->getAPIntValue();
7649 unsigned BitWidth = Divisor.getBitWidth();
7650 unsigned HBitWidth = BitWidth / 2;
7652 HiLoVT.getScalarSizeInBits() == HBitWidth && "Unexpected VTs");
7653
7654 // Divisor needs to less than (1 << HBitWidth).
7655 APInt HalfMaxPlus1 = APInt::getOneBitSet(BitWidth, HBitWidth);
7656 if (Divisor.uge(HalfMaxPlus1))
7657 return false;
7658
7659 // We depend on the UREM by constant optimization in DAGCombiner that requires
7660 // high multiply.
7661 if (!isOperationLegalOrCustom(ISD::MULHU, HiLoVT) &&
7663 return false;
7664
7665 // Don't expand if optimizing for size.
7666 if (DAG.shouldOptForSize())
7667 return false;
7668
7669 // Early out for 0 or 1 divisors.
7670 if (Divisor.ule(1))
7671 return false;
7672
7673 // If the divisor is even, shift it until it becomes odd.
7674 unsigned TrailingZeros = 0;
7675 if (!Divisor[0]) {
7676 TrailingZeros = Divisor.countr_zero();
7677 Divisor.lshrInPlace(TrailingZeros);
7678 }
7679
7680 SDLoc dl(N);
7681 SDValue Sum;
7682 SDValue PartialRem;
7683
7684 // If (1 << HBitWidth) % divisor == 1, we can add the two halves together and
7685 // then add in the carry.
7686 // TODO: If we can't split it in half, we might be able to split into 3 or
7687 // more pieces using a smaller bit width.
7688 if (HalfMaxPlus1.urem(Divisor).isOne()) {
7689 assert(!LL == !LH && "Expected both input halves or no input halves!");
7690 if (!LL)
7691 std::tie(LL, LH) = DAG.SplitScalar(N->getOperand(0), dl, HiLoVT, HiLoVT);
7692
7693 // Shift the input by the number of TrailingZeros in the divisor. The
7694 // shifted out bits will be added to the remainder later.
7695 if (TrailingZeros) {
7696 // Save the shifted off bits if we need the remainder.
7697 if (Opcode != ISD::UDIV) {
7698 APInt Mask = APInt::getLowBitsSet(HBitWidth, TrailingZeros);
7699 PartialRem = DAG.getNode(ISD::AND, dl, HiLoVT, LL,
7700 DAG.getConstant(Mask, dl, HiLoVT));
7701 }
7702
7703 LL = DAG.getNode(
7704 ISD::OR, dl, HiLoVT,
7705 DAG.getNode(ISD::SRL, dl, HiLoVT, LL,
7706 DAG.getShiftAmountConstant(TrailingZeros, HiLoVT, dl)),
7707 DAG.getNode(ISD::SHL, dl, HiLoVT, LH,
7708 DAG.getShiftAmountConstant(HBitWidth - TrailingZeros,
7709 HiLoVT, dl)));
7710 LH = DAG.getNode(ISD::SRL, dl, HiLoVT, LH,
7711 DAG.getShiftAmountConstant(TrailingZeros, HiLoVT, dl));
7712 }
7713
7714 // Use uaddo_carry if we can, otherwise use a compare to detect overflow.
7715 EVT SetCCType =
7716 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), HiLoVT);
7718 SDVTList VTList = DAG.getVTList(HiLoVT, SetCCType);
7719 Sum = DAG.getNode(ISD::UADDO, dl, VTList, LL, LH);
7720 Sum = DAG.getNode(ISD::UADDO_CARRY, dl, VTList, Sum,
7721 DAG.getConstant(0, dl, HiLoVT), Sum.getValue(1));
7722 } else {
7723 Sum = DAG.getNode(ISD::ADD, dl, HiLoVT, LL, LH);
7724 SDValue Carry = DAG.getSetCC(dl, SetCCType, Sum, LL, ISD::SETULT);
7725 // If the boolean for the target is 0 or 1, we can add the setcc result
7726 // directly.
7727 if (getBooleanContents(HiLoVT) ==
7729 Carry = DAG.getZExtOrTrunc(Carry, dl, HiLoVT);
7730 else
7731 Carry = DAG.getSelect(dl, HiLoVT, Carry, DAG.getConstant(1, dl, HiLoVT),
7732 DAG.getConstant(0, dl, HiLoVT));
7733 Sum = DAG.getNode(ISD::ADD, dl, HiLoVT, Sum, Carry);
7734 }
7735 }
7736
7737 // If we didn't find a sum, we can't do the expansion.
7738 if (!Sum)
7739 return false;
7740
7741 // Perform a HiLoVT urem on the Sum using truncated divisor.
7742 SDValue RemL =
7743 DAG.getNode(ISD::UREM, dl, HiLoVT, Sum,
7744 DAG.getConstant(Divisor.trunc(HBitWidth), dl, HiLoVT));
7745 SDValue RemH = DAG.getConstant(0, dl, HiLoVT);
7746
7747 if (Opcode != ISD::UREM) {
7748 // Subtract the remainder from the shifted dividend.
7749 SDValue Dividend = DAG.getNode(ISD::BUILD_PAIR, dl, VT, LL, LH);
7750 SDValue Rem = DAG.getNode(ISD::BUILD_PAIR, dl, VT, RemL, RemH);
7751
7752 Dividend = DAG.getNode(ISD::SUB, dl, VT, Dividend, Rem);
7753
7754 // Multiply by the multiplicative inverse of the divisor modulo
7755 // (1 << BitWidth).
7757 APInt MulFactor = Divisor.zext(BitWidth + 1);
7758 MulFactor = MulFactor.multiplicativeInverse(Mod);
7759 MulFactor = MulFactor.trunc(BitWidth);
7760
7761 SDValue Quotient = DAG.getNode(ISD::MUL, dl, VT, Dividend,
7762 DAG.getConstant(MulFactor, dl, VT));
7763
7764 // Split the quotient into low and high parts.
7765 SDValue QuotL, QuotH;
7766 std::tie(QuotL, QuotH) = DAG.SplitScalar(Quotient, dl, HiLoVT, HiLoVT);
7767 Result.push_back(QuotL);
7768 Result.push_back(QuotH);
7769 }
7770
7771 if (Opcode != ISD::UDIV) {
7772 // If we shifted the input, shift the remainder left and add the bits we
7773 // shifted off the input.
7774 if (TrailingZeros) {
7775 APInt Mask = APInt::getLowBitsSet(HBitWidth, TrailingZeros);
7776 RemL = DAG.getNode(ISD::SHL, dl, HiLoVT, RemL,
7777 DAG.getShiftAmountConstant(TrailingZeros, HiLoVT, dl));
7778 RemL = DAG.getNode(ISD::ADD, dl, HiLoVT, RemL, PartialRem);
7779 }
7780 Result.push_back(RemL);
7781 Result.push_back(DAG.getConstant(0, dl, HiLoVT));
7782 }
7783
7784 return true;
7785}
7786
7787// Check that (every element of) Z is undef or not an exact multiple of BW.
7788static bool isNonZeroModBitWidthOrUndef(SDValue Z, unsigned BW) {
7790 Z,
7791 [=](ConstantSDNode *C) { return !C || C->getAPIntValue().urem(BW) != 0; },
7792 true);
7793}
7794
7796 EVT VT = Node->getValueType(0);
7797 SDValue ShX, ShY;
7798 SDValue ShAmt, InvShAmt;
7799 SDValue X = Node->getOperand(0);
7800 SDValue Y = Node->getOperand(1);
7801 SDValue Z = Node->getOperand(2);
7802 SDValue Mask = Node->getOperand(3);
7803 SDValue VL = Node->getOperand(4);
7804
7805 unsigned BW = VT.getScalarSizeInBits();
7806 bool IsFSHL = Node->getOpcode() == ISD::VP_FSHL;
7807 SDLoc DL(SDValue(Node, 0));
7808
7809 EVT ShVT = Z.getValueType();
7810 if (isNonZeroModBitWidthOrUndef(Z, BW)) {
7811 // fshl: X << C | Y >> (BW - C)
7812 // fshr: X << (BW - C) | Y >> C
7813 // where C = Z % BW is not zero
7814 SDValue BitWidthC = DAG.getConstant(BW, DL, ShVT);
7815 ShAmt = DAG.getNode(ISD::VP_UREM, DL, ShVT, Z, BitWidthC, Mask, VL);
7816 InvShAmt = DAG.getNode(ISD::VP_SUB, DL, ShVT, BitWidthC, ShAmt, Mask, VL);
7817 ShX = DAG.getNode(ISD::VP_SHL, DL, VT, X, IsFSHL ? ShAmt : InvShAmt, Mask,
7818 VL);
7819 ShY = DAG.getNode(ISD::VP_LSHR, DL, VT, Y, IsFSHL ? InvShAmt : ShAmt, Mask,
7820 VL);
7821 } else {
7822 // fshl: X << (Z % BW) | Y >> 1 >> (BW - 1 - (Z % BW))
7823 // fshr: X << 1 << (BW - 1 - (Z % BW)) | Y >> (Z % BW)
7824 SDValue BitMask = DAG.getConstant(BW - 1, DL, ShVT);
7825 if (isPowerOf2_32(BW)) {
7826 // Z % BW -> Z & (BW - 1)
7827 ShAmt = DAG.getNode(ISD::VP_AND, DL, ShVT, Z, BitMask, Mask, VL);
7828 // (BW - 1) - (Z % BW) -> ~Z & (BW - 1)
7829 SDValue NotZ = DAG.getNode(ISD::VP_XOR, DL, ShVT, Z,
7830 DAG.getAllOnesConstant(DL, ShVT), Mask, VL);
7831 InvShAmt = DAG.getNode(ISD::VP_AND, DL, ShVT, NotZ, BitMask, Mask, VL);
7832 } else {
7833 SDValue BitWidthC = DAG.getConstant(BW, DL, ShVT);
7834 ShAmt = DAG.getNode(ISD::VP_UREM, DL, ShVT, Z, BitWidthC, Mask, VL);
7835 InvShAmt = DAG.getNode(ISD::VP_SUB, DL, ShVT, BitMask, ShAmt, Mask, VL);
7836 }
7837
7838 SDValue One = DAG.getConstant(1, DL, ShVT);
7839 if (IsFSHL) {
7840 ShX = DAG.getNode(ISD::VP_SHL, DL, VT, X, ShAmt, Mask, VL);
7841 SDValue ShY1 = DAG.getNode(ISD::VP_LSHR, DL, VT, Y, One, Mask, VL);
7842 ShY = DAG.getNode(ISD::VP_LSHR, DL, VT, ShY1, InvShAmt, Mask, VL);
7843 } else {
7844 SDValue ShX1 = DAG.getNode(ISD::VP_SHL, DL, VT, X, One, Mask, VL);
7845 ShX = DAG.getNode(ISD::VP_SHL, DL, VT, ShX1, InvShAmt, Mask, VL);
7846 ShY = DAG.getNode(ISD::VP_LSHR, DL, VT, Y, ShAmt, Mask, VL);
7847 }
7848 }
7849 return DAG.getNode(ISD::VP_OR, DL, VT, ShX, ShY, Mask, VL);
7850}
7851
7853 SelectionDAG &DAG) const {
7854 if (Node->isVPOpcode())
7855 return expandVPFunnelShift(Node, DAG);
7856
7857 EVT VT = Node->getValueType(0);
7858
7859 if (VT.isVector() && (!isOperationLegalOrCustom(ISD::SHL, VT) ||
7863 return SDValue();
7864
7865 SDValue X = Node->getOperand(0);
7866 SDValue Y = Node->getOperand(1);
7867 SDValue Z = Node->getOperand(2);
7868
7869 unsigned BW = VT.getScalarSizeInBits();
7870 bool IsFSHL = Node->getOpcode() == ISD::FSHL;
7871 SDLoc DL(SDValue(Node, 0));
7872
7873 EVT ShVT = Z.getValueType();
7874
7875 // If a funnel shift in the other direction is more supported, use it.
7876 unsigned RevOpcode = IsFSHL ? ISD::FSHR : ISD::FSHL;
7877 if (!isOperationLegalOrCustom(Node->getOpcode(), VT) &&
7878 isOperationLegalOrCustom(RevOpcode, VT) && isPowerOf2_32(BW)) {
7879 if (isNonZeroModBitWidthOrUndef(Z, BW)) {
7880 // fshl X, Y, Z -> fshr X, Y, -Z
7881 // fshr X, Y, Z -> fshl X, Y, -Z
7882 SDValue Zero = DAG.getConstant(0, DL, ShVT);
7883 Z = DAG.getNode(ISD::SUB, DL, VT, Zero, Z);
7884 } else {
7885 // fshl X, Y, Z -> fshr (srl X, 1), (fshr X, Y, 1), ~Z
7886 // fshr X, Y, Z -> fshl (fshl X, Y, 1), (shl Y, 1), ~Z
7887 SDValue One = DAG.getConstant(1, DL, ShVT);
7888 if (IsFSHL) {
7889 Y = DAG.getNode(RevOpcode, DL, VT, X, Y, One);
7890 X = DAG.getNode(ISD::SRL, DL, VT, X, One);
7891 } else {
7892 X = DAG.getNode(RevOpcode, DL, VT, X, Y, One);
7893 Y = DAG.getNode(ISD::SHL, DL, VT, Y, One);
7894 }
7895 Z = DAG.getNOT(DL, Z, ShVT);
7896 }
7897 return DAG.getNode(RevOpcode, DL, VT, X, Y, Z);
7898 }
7899
7900 SDValue ShX, ShY;
7901 SDValue ShAmt, InvShAmt;
7902 if (isNonZeroModBitWidthOrUndef(Z, BW)) {
7903 // fshl: X << C | Y >> (BW - C)
7904 // fshr: X << (BW - C) | Y >> C
7905 // where C = Z % BW is not zero
7906 SDValue BitWidthC = DAG.getConstant(BW, DL, ShVT);
7907 ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Z, BitWidthC);
7908 InvShAmt = DAG.getNode(ISD::SUB, DL, ShVT, BitWidthC, ShAmt);
7909 ShX = DAG.getNode(ISD::SHL, DL, VT, X, IsFSHL ? ShAmt : InvShAmt);
7910 ShY = DAG.getNode(ISD::SRL, DL, VT, Y, IsFSHL ? InvShAmt : ShAmt);
7911 } else {
7912 // fshl: X << (Z % BW) | Y >> 1 >> (BW - 1 - (Z % BW))
7913 // fshr: X << 1 << (BW - 1 - (Z % BW)) | Y >> (Z % BW)
7914 SDValue Mask = DAG.getConstant(BW - 1, DL, ShVT);
7915 if (isPowerOf2_32(BW)) {
7916 // Z % BW -> Z & (BW - 1)
7917 ShAmt = DAG.getNode(ISD::AND, DL, ShVT, Z, Mask);
7918 // (BW - 1) - (Z % BW) -> ~Z & (BW - 1)
7919 InvShAmt = DAG.getNode(ISD::AND, DL, ShVT, DAG.getNOT(DL, Z, ShVT), Mask);
7920 } else {
7921 SDValue BitWidthC = DAG.getConstant(BW, DL, ShVT);
7922 ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Z, BitWidthC);
7923 InvShAmt = DAG.getNode(ISD::SUB, DL, ShVT, Mask, ShAmt);
7924 }
7925
7926 SDValue One = DAG.getConstant(1, DL, ShVT);
7927 if (IsFSHL) {
7928 ShX = DAG.getNode(ISD::SHL, DL, VT, X, ShAmt);
7929 SDValue ShY1 = DAG.getNode(ISD::SRL, DL, VT, Y, One);
7930 ShY = DAG.getNode(ISD::SRL, DL, VT, ShY1, InvShAmt);
7931 } else {
7932 SDValue ShX1 = DAG.getNode(ISD::SHL, DL, VT, X, One);
7933 ShX = DAG.getNode(ISD::SHL, DL, VT, ShX1, InvShAmt);
7934 ShY = DAG.getNode(ISD::SRL, DL, VT, Y, ShAmt);
7935 }
7936 }
7937 return DAG.getNode(ISD::OR, DL, VT, ShX, ShY);
7938}
7939
7940// TODO: Merge with expandFunnelShift.
7941SDValue TargetLowering::expandROT(SDNode *Node, bool AllowVectorOps,
7942 SelectionDAG &DAG) const {
7943 EVT VT = Node->getValueType(0);
7944 unsigned EltSizeInBits = VT.getScalarSizeInBits();
7945 bool IsLeft = Node->getOpcode() == ISD::ROTL;
7946 SDValue Op0 = Node->getOperand(0);
7947 SDValue Op1 = Node->getOperand(1);
7948 SDLoc DL(SDValue(Node, 0));
7949
7950 EVT ShVT = Op1.getValueType();
7951 SDValue Zero = DAG.getConstant(0, DL, ShVT);
7952
7953 // If a rotate in the other direction is more supported, use it.
7954 unsigned RevRot = IsLeft ? ISD::ROTR : ISD::ROTL;
7955 if (!isOperationLegalOrCustom(Node->getOpcode(), VT) &&
7956 isOperationLegalOrCustom(RevRot, VT) && isPowerOf2_32(EltSizeInBits)) {
7957 SDValue Sub = DAG.getNode(ISD::SUB, DL, ShVT, Zero, Op1);
7958 return DAG.getNode(RevRot, DL, VT, Op0, Sub);
7959 }
7960
7961 if (!AllowVectorOps && VT.isVector() &&
7967 return SDValue();
7968
7969 unsigned ShOpc = IsLeft ? ISD::SHL : ISD::SRL;
7970 unsigned HsOpc = IsLeft ? ISD::SRL : ISD::SHL;
7971 SDValue BitWidthMinusOneC = DAG.getConstant(EltSizeInBits - 1, DL, ShVT);
7972 SDValue ShVal;
7973 SDValue HsVal;
7974 if (isPowerOf2_32(EltSizeInBits)) {
7975 // (rotl x, c) -> x << (c & (w - 1)) | x >> (-c & (w - 1))
7976 // (rotr x, c) -> x >> (c & (w - 1)) | x << (-c & (w - 1))
7977 SDValue NegOp1 = DAG.getNode(ISD::SUB, DL, ShVT, Zero, Op1);
7978 SDValue ShAmt = DAG.getNode(ISD::AND, DL, ShVT, Op1, BitWidthMinusOneC);
7979 ShVal = DAG.getNode(ShOpc, DL, VT, Op0, ShAmt);
7980 SDValue HsAmt = DAG.getNode(ISD::AND, DL, ShVT, NegOp1, BitWidthMinusOneC);
7981 HsVal = DAG.getNode(HsOpc, DL, VT, Op0, HsAmt);
7982 } else {
7983 // (rotl x, c) -> x << (c % w) | x >> 1 >> (w - 1 - (c % w))
7984 // (rotr x, c) -> x >> (c % w) | x << 1 << (w - 1 - (c % w))
7985 SDValue BitWidthC = DAG.getConstant(EltSizeInBits, DL, ShVT);
7986 SDValue ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Op1, BitWidthC);
7987 ShVal = DAG.getNode(ShOpc, DL, VT, Op0, ShAmt);
7988 SDValue HsAmt = DAG.getNode(ISD::SUB, DL, ShVT, BitWidthMinusOneC, ShAmt);
7989 SDValue One = DAG.getConstant(1, DL, ShVT);
7990 HsVal =
7991 DAG.getNode(HsOpc, DL, VT, DAG.getNode(HsOpc, DL, VT, Op0, One), HsAmt);
7992 }
7993 return DAG.getNode(ISD::OR, DL, VT, ShVal, HsVal);
7994}
7995
7997 SelectionDAG &DAG) const {
7998 assert(Node->getNumOperands() == 3 && "Not a double-shift!");
7999 EVT VT = Node->getValueType(0);
8000 unsigned VTBits = VT.getScalarSizeInBits();
8001 assert(isPowerOf2_32(VTBits) && "Power-of-two integer type expected");
8002
8003 bool IsSHL = Node->getOpcode() == ISD::SHL_PARTS;
8004 bool IsSRA = Node->getOpcode() == ISD::SRA_PARTS;
8005 SDValue ShOpLo = Node->getOperand(0);
8006 SDValue ShOpHi = Node->getOperand(1);
8007 SDValue ShAmt = Node->getOperand(2);
8008 EVT ShAmtVT = ShAmt.getValueType();
8009 EVT ShAmtCCVT =
8010 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), ShAmtVT);
8011 SDLoc dl(Node);
8012
8013 // ISD::FSHL and ISD::FSHR have defined overflow behavior but ISD::SHL and
8014 // ISD::SRA/L nodes haven't. Insert an AND to be safe, it's usually optimized
8015 // away during isel.
8016 SDValue SafeShAmt = DAG.getNode(ISD::AND, dl, ShAmtVT, ShAmt,
8017 DAG.getConstant(VTBits - 1, dl, ShAmtVT));
8018 SDValue Tmp1 = IsSRA ? DAG.getNode(ISD::SRA, dl, VT, ShOpHi,
8019 DAG.getConstant(VTBits - 1, dl, ShAmtVT))
8020 : DAG.getConstant(0, dl, VT);
8021
8022 SDValue Tmp2, Tmp3;
8023 if (IsSHL) {
8024 Tmp2 = DAG.getNode(ISD::FSHL, dl, VT, ShOpHi, ShOpLo, ShAmt);
8025 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, SafeShAmt);
8026 } else {
8027 Tmp2 = DAG.getNode(ISD::FSHR, dl, VT, ShOpHi, ShOpLo, ShAmt);
8028 Tmp3 = DAG.getNode(IsSRA ? ISD::SRA : ISD::SRL, dl, VT, ShOpHi, SafeShAmt);
8029 }
8030
8031 // If the shift amount is larger or equal than the width of a part we don't
8032 // use the result from the FSHL/FSHR. Insert a test and select the appropriate
8033 // values for large shift amounts.
8034 SDValue AndNode = DAG.getNode(ISD::AND, dl, ShAmtVT, ShAmt,
8035 DAG.getConstant(VTBits, dl, ShAmtVT));
8036 SDValue Cond = DAG.getSetCC(dl, ShAmtCCVT, AndNode,
8037 DAG.getConstant(0, dl, ShAmtVT), ISD::SETNE);
8038
8039 if (IsSHL) {
8040 Hi = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp3, Tmp2);
8041 Lo = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp1, Tmp3);
8042 } else {
8043 Lo = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp3, Tmp2);
8044 Hi = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp1, Tmp3);
8045 }
8046}
8047
8049 SelectionDAG &DAG) const {
8050 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
8051 SDValue Src = Node->getOperand(OpNo);
8052 EVT SrcVT = Src.getValueType();
8053 EVT DstVT = Node->getValueType(0);
8054 SDLoc dl(SDValue(Node, 0));
8055
8056 // FIXME: Only f32 to i64 conversions are supported.
8057 if (SrcVT != MVT::f32 || DstVT != MVT::i64)
8058 return false;
8059
8060 if (Node->isStrictFPOpcode())
8061 // When a NaN is converted to an integer a trap is allowed. We can't
8062 // use this expansion here because it would eliminate that trap. Other
8063 // traps are also allowed and cannot be eliminated. See
8064 // IEEE 754-2008 sec 5.8.
8065 return false;
8066
8067 // Expand f32 -> i64 conversion
8068 // This algorithm comes from compiler-rt's implementation of fixsfdi:
8069 // https://github.com/llvm/llvm-project/blob/main/compiler-rt/lib/builtins/fixsfdi.c
8070 unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
8071 EVT IntVT = SrcVT.changeTypeToInteger();
8072 EVT IntShVT = getShiftAmountTy(IntVT, DAG.getDataLayout());
8073
8074 SDValue ExponentMask = DAG.getConstant(0x7F800000, dl, IntVT);
8075 SDValue ExponentLoBit = DAG.getConstant(23, dl, IntVT);
8076 SDValue Bias = DAG.getConstant(127, dl, IntVT);
8077 SDValue SignMask = DAG.getConstant(APInt::getSignMask(SrcEltBits), dl, IntVT);
8078 SDValue SignLowBit = DAG.getConstant(SrcEltBits - 1, dl, IntVT);
8079 SDValue MantissaMask = DAG.getConstant(0x007FFFFF, dl, IntVT);
8080
8081 SDValue Bits = DAG.getNode(ISD::BITCAST, dl, IntVT, Src);
8082
8083 SDValue ExponentBits = DAG.getNode(
8084 ISD::SRL, dl, IntVT, DAG.getNode(ISD::AND, dl, IntVT, Bits, ExponentMask),
8085 DAG.getZExtOrTrunc(ExponentLoBit, dl, IntShVT));
8086 SDValue Exponent = DAG.getNode(ISD::SUB, dl, IntVT, ExponentBits, Bias);
8087
8088 SDValue Sign = DAG.getNode(ISD::SRA, dl, IntVT,
8089 DAG.getNode(ISD::AND, dl, IntVT, Bits, SignMask),
8090 DAG.getZExtOrTrunc(SignLowBit, dl, IntShVT));
8091 Sign = DAG.getSExtOrTrunc(Sign, dl, DstVT);
8092
8093 SDValue R = DAG.getNode(ISD::OR, dl, IntVT,
8094 DAG.getNode(ISD::AND, dl, IntVT, Bits, MantissaMask),
8095 DAG.getConstant(0x00800000, dl, IntVT));
8096
8097 R = DAG.getZExtOrTrunc(R, dl, DstVT);
8098
8099 R = DAG.getSelectCC(
8100 dl, Exponent, ExponentLoBit,
8101 DAG.getNode(ISD::SHL, dl, DstVT, R,
8102 DAG.getZExtOrTrunc(
8103 DAG.getNode(ISD::SUB, dl, IntVT, Exponent, ExponentLoBit),
8104 dl, IntShVT)),
8105 DAG.getNode(ISD::SRL, dl, DstVT, R,
8106 DAG.getZExtOrTrunc(
8107 DAG.getNode(ISD::SUB, dl, IntVT, ExponentLoBit, Exponent),
8108 dl, IntShVT)),
8109 ISD::SETGT);
8110
8111 SDValue Ret = DAG.getNode(ISD::SUB, dl, DstVT,
8112 DAG.getNode(ISD::XOR, dl, DstVT, R, Sign), Sign);
8113
8114 Result = DAG.getSelectCC(dl, Exponent, DAG.getConstant(0, dl, IntVT),
8115 DAG.getConstant(0, dl, DstVT), Ret, ISD::SETLT);
8116 return true;
8117}
8118
8120 SDValue &Chain,
8121 SelectionDAG &DAG) const {
8122 SDLoc dl(SDValue(Node, 0));
8123 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
8124 SDValue Src = Node->getOperand(OpNo);
8125
8126 EVT SrcVT = Src.getValueType();
8127 EVT DstVT = Node->getValueType(0);
8128 EVT SetCCVT =
8129 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), SrcVT);
8130 EVT DstSetCCVT =
8131 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), DstVT);
8132
8133 // Only expand vector types if we have the appropriate vector bit operations.
8134 unsigned SIntOpcode = Node->isStrictFPOpcode() ? ISD::STRICT_FP_TO_SINT :
8136 if (DstVT.isVector() && (!isOperationLegalOrCustom(SIntOpcode, DstVT) ||
8138 return false;
8139
8140 // If the maximum float value is smaller then the signed integer range,
8141 // the destination signmask can't be represented by the float, so we can
8142 // just use FP_TO_SINT directly.
8143 const fltSemantics &APFSem = DAG.EVTToAPFloatSemantics(SrcVT);
8144 APFloat APF(APFSem, APInt::getZero(SrcVT.getScalarSizeInBits()));
8145 APInt SignMask = APInt::getSignMask(DstVT.getScalarSizeInBits());
8147 APF.convertFromAPInt(SignMask, false, APFloat::rmNearestTiesToEven)) {
8148 if (Node->isStrictFPOpcode()) {
8149 Result = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, { DstVT, MVT::Other },
8150 { Node->getOperand(0), Src });
8151 Chain = Result.getValue(1);
8152 } else
8153 Result = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Src);
8154 return true;
8155 }
8156
8157 // Don't expand it if there isn't cheap fsub instruction.
8159 Node->isStrictFPOpcode() ? ISD::STRICT_FSUB : ISD::FSUB, SrcVT))
8160 return false;
8161
8162 SDValue Cst = DAG.getConstantFP(APF, dl, SrcVT);
8163 SDValue Sel;
8164
8165 if (Node->isStrictFPOpcode()) {
8166 Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT,
8167 Node->getOperand(0), /*IsSignaling*/ true);
8168 Chain = Sel.getValue(1);
8169 } else {
8170 Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT);
8171 }
8172
8173 bool Strict = Node->isStrictFPOpcode() ||
8174 shouldUseStrictFP_TO_INT(SrcVT, DstVT, /*IsSigned*/ false);
8175
8176 if (Strict) {
8177 // Expand based on maximum range of FP_TO_SINT, if the value exceeds the
8178 // signmask then offset (the result of which should be fully representable).
8179 // Sel = Src < 0x8000000000000000
8180 // FltOfs = select Sel, 0, 0x8000000000000000
8181 // IntOfs = select Sel, 0, 0x8000000000000000
8182 // Result = fp_to_sint(Src - FltOfs) ^ IntOfs
8183
8184 // TODO: Should any fast-math-flags be set for the FSUB?
8185 SDValue FltOfs = DAG.getSelect(dl, SrcVT, Sel,
8186 DAG.getConstantFP(0.0, dl, SrcVT), Cst);
8187 Sel = DAG.getBoolExtOrTrunc(Sel, dl, DstSetCCVT, DstVT);
8188 SDValue IntOfs = DAG.getSelect(dl, DstVT, Sel,
8189 DAG.getConstant(0, dl, DstVT),
8190 DAG.getConstant(SignMask, dl, DstVT));
8191 SDValue SInt;
8192 if (Node->isStrictFPOpcode()) {
8193 SDValue Val = DAG.getNode(ISD::STRICT_FSUB, dl, { SrcVT, MVT::Other },
8194 { Chain, Src, FltOfs });
8195 SInt = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, { DstVT, MVT::Other },
8196 { Val.getValue(1), Val });
8197 Chain = SInt.getValue(1);
8198 } else {
8199 SDValue Val = DAG.getNode(ISD::FSUB, dl, SrcVT, Src, FltOfs);
8200 SInt = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Val);
8201 }
8202 Result = DAG.getNode(ISD::XOR, dl, DstVT, SInt, IntOfs);
8203 } else {
8204 // Expand based on maximum range of FP_TO_SINT:
8205 // True = fp_to_sint(Src)
8206 // False = 0x8000000000000000 + fp_to_sint(Src - 0x8000000000000000)
8207 // Result = select (Src < 0x8000000000000000), True, False
8208
8209 SDValue True = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Src);
8210 // TODO: Should any fast-math-flags be set for the FSUB?
8211 SDValue False = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT,
8212 DAG.getNode(ISD::FSUB, dl, SrcVT, Src, Cst));
8213 False = DAG.getNode(ISD::XOR, dl, DstVT, False,
8214 DAG.getConstant(SignMask, dl, DstVT));
8215 Sel = DAG.getBoolExtOrTrunc(Sel, dl, DstSetCCVT, DstVT);
8216 Result = DAG.getSelect(dl, DstVT, Sel, True, False);
8217 }
8218 return true;
8219}
8220
8222 SDValue &Chain,
8223 SelectionDAG &DAG) const {
8224 // This transform is not correct for converting 0 when rounding mode is set
8225 // to round toward negative infinity which will produce -0.0. So disable under
8226 // strictfp.
8227 if (Node->isStrictFPOpcode())
8228 return false;
8229
8230 SDValue Src = Node->getOperand(0);
8231 EVT SrcVT = Src.getValueType();
8232 EVT DstVT = Node->getValueType(0);
8233
8234 if (SrcVT.getScalarType() != MVT::i64 || DstVT.getScalarType() != MVT::f64)
8235 return false;
8236
8237 // Only expand vector types if we have the appropriate vector bit operations.
8238 if (SrcVT.isVector() && (!isOperationLegalOrCustom(ISD::SRL, SrcVT) ||
8243 return false;
8244
8245 SDLoc dl(SDValue(Node, 0));
8246 EVT ShiftVT = getShiftAmountTy(SrcVT, DAG.getDataLayout());
8247
8248 // Implementation of unsigned i64 to f64 following the algorithm in
8249 // __floatundidf in compiler_rt. This implementation performs rounding
8250 // correctly in all rounding modes with the exception of converting 0
8251 // when rounding toward negative infinity. In that case the fsub will produce
8252 // -0.0. This will be added to +0.0 and produce -0.0 which is incorrect.
8253 SDValue TwoP52 = DAG.getConstant(UINT64_C(0x4330000000000000), dl, SrcVT);
8254 SDValue TwoP84PlusTwoP52 = DAG.getConstantFP(
8255 llvm::bit_cast<double>(UINT64_C(0x4530000000100000)), dl, DstVT);
8256 SDValue TwoP84 = DAG.getConstant(UINT64_C(0x4530000000000000), dl, SrcVT);
8257 SDValue LoMask = DAG.getConstant(UINT64_C(0x00000000FFFFFFFF), dl, SrcVT);
8258 SDValue HiShift = DAG.getConstant(32, dl, ShiftVT);
8259
8260 SDValue Lo = DAG.getNode(ISD::AND, dl, SrcVT, Src, LoMask);
8261 SDValue Hi = DAG.getNode(ISD::SRL, dl, SrcVT, Src, HiShift);
8262 SDValue LoOr = DAG.getNode(ISD::OR, dl, SrcVT, Lo, TwoP52);
8263 SDValue HiOr = DAG.getNode(ISD::OR, dl, SrcVT, Hi, TwoP84);
8264 SDValue LoFlt = DAG.getBitcast(DstVT, LoOr);
8265 SDValue HiFlt = DAG.getBitcast(DstVT, HiOr);
8266 SDValue HiSub =
8267 DAG.getNode(ISD::FSUB, dl, DstVT, HiFlt, TwoP84PlusTwoP52);
8268 Result = DAG.getNode(ISD::FADD, dl, DstVT, LoFlt, HiSub);
8269 return true;
8270}
8271
8272SDValue
8274 SelectionDAG &DAG) const {
8275 unsigned Opcode = Node->getOpcode();
8276 assert((Opcode == ISD::FMINNUM || Opcode == ISD::FMAXNUM ||
8277 Opcode == ISD::STRICT_FMINNUM || Opcode == ISD::STRICT_FMAXNUM) &&
8278 "Wrong opcode");
8279
8280 if (Node->getFlags().hasNoNaNs()) {
8281 ISD::CondCode Pred = Opcode == ISD::FMINNUM ? ISD::SETLT : ISD::SETGT;
8282 SDValue Op1 = Node->getOperand(0);
8283 SDValue Op2 = Node->getOperand(1);
8284 SDValue SelCC = DAG.getSelectCC(SDLoc(Node), Op1, Op2, Op1, Op2, Pred);
8285 // Copy FMF flags, but always set the no-signed-zeros flag
8286 // as this is implied by the FMINNUM/FMAXNUM semantics.
8287 SDNodeFlags Flags = Node->getFlags();
8288 Flags.setNoSignedZeros(true);
8289 SelCC->setFlags(Flags);
8290 return SelCC;
8291 }
8292
8293 return SDValue();
8294}
8295
8297 SelectionDAG &DAG) const {
8298 SDLoc dl(Node);
8299 unsigned NewOp = Node->getOpcode() == ISD::FMINNUM ?
8301 EVT VT = Node->getValueType(0);
8302
8303 if (VT.isScalableVector())
8305 "Expanding fminnum/fmaxnum for scalable vectors is undefined.");
8306
8307 if (isOperationLegalOrCustom(NewOp, VT)) {
8308 SDValue Quiet0 = Node->getOperand(0);
8309 SDValue Quiet1 = Node->getOperand(1);
8310
8311 if (!Node->getFlags().hasNoNaNs()) {
8312 // Insert canonicalizes if it's possible we need to quiet to get correct
8313 // sNaN behavior.
8314 if (!DAG.isKnownNeverSNaN(Quiet0)) {
8315 Quiet0 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet0,
8316 Node->getFlags());
8317 }
8318 if (!DAG.isKnownNeverSNaN(Quiet1)) {
8319 Quiet1 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet1,
8320 Node->getFlags());
8321 }
8322 }
8323
8324 return DAG.getNode(NewOp, dl, VT, Quiet0, Quiet1, Node->getFlags());
8325 }
8326
8327 // If the target has FMINIMUM/FMAXIMUM but not FMINNUM/FMAXNUM use that
8328 // instead if there are no NaNs and there can't be an incompatible zero
8329 // compare: at least one operand isn't +/-0, or there are no signed-zeros.
8330 if ((Node->getFlags().hasNoNaNs() ||
8331 (DAG.isKnownNeverNaN(Node->getOperand(0)) &&
8332 DAG.isKnownNeverNaN(Node->getOperand(1)))) &&
8333 (Node->getFlags().hasNoSignedZeros() ||
8334 DAG.isKnownNeverZeroFloat(Node->getOperand(0)) ||
8335 DAG.isKnownNeverZeroFloat(Node->getOperand(1)))) {
8336 unsigned IEEE2018Op =
8337 Node->getOpcode() == ISD::FMINNUM ? ISD::FMINIMUM : ISD::FMAXIMUM;
8338 if (isOperationLegalOrCustom(IEEE2018Op, VT))
8339 return DAG.getNode(IEEE2018Op, dl, VT, Node->getOperand(0),
8340 Node->getOperand(1), Node->getFlags());
8341 }
8342
8343 if (SDValue SelCC = createSelectForFMINNUM_FMAXNUM(Node, DAG))
8344 return SelCC;
8345
8346 return SDValue();
8347}
8348
8349/// Returns a true value if if this FPClassTest can be performed with an ordered
8350/// fcmp to 0, and a false value if it's an unordered fcmp to 0. Returns
8351/// std::nullopt if it cannot be performed as a compare with 0.
8352static std::optional<bool> isFCmpEqualZero(FPClassTest Test,
8353 const fltSemantics &Semantics,
8354 const MachineFunction &MF) {
8355 FPClassTest OrderedMask = Test & ~fcNan;
8356 FPClassTest NanTest = Test & fcNan;
8357 bool IsOrdered = NanTest == fcNone;
8358 bool IsUnordered = NanTest == fcNan;
8359
8360 // Skip cases that are testing for only a qnan or snan.
8361 if (!IsOrdered && !IsUnordered)
8362 return std::nullopt;
8363
8364 if (OrderedMask == fcZero &&
8365 MF.getDenormalMode(Semantics).Input == DenormalMode::IEEE)
8366 return IsOrdered;
8367 if (OrderedMask == (fcZero | fcSubnormal) &&
8368 MF.getDenormalMode(Semantics).inputsAreZero())
8369 return IsOrdered;
8370 return std::nullopt;
8371}
8372
8375 const SDLoc &DL,
8376 SelectionDAG &DAG) const {
8377 EVT OperandVT = Op.getValueType();
8378 assert(OperandVT.isFloatingPoint());
8379
8380 // Degenerated cases.
8381 if (Test == fcNone)
8382 return DAG.getBoolConstant(false, DL, ResultVT, OperandVT);
8383 if ((Test & fcAllFlags) == fcAllFlags)
8384 return DAG.getBoolConstant(true, DL, ResultVT, OperandVT);
8385
8386 // PPC double double is a pair of doubles, of which the higher part determines
8387 // the value class.
8388 if (OperandVT == MVT::ppcf128) {
8389 Op = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::f64, Op,
8390 DAG.getConstant(1, DL, MVT::i32));
8391 OperandVT = MVT::f64;
8392 }
8393
8394 // Some checks may be represented as inversion of simpler check, for example
8395 // "inf|normal|subnormal|zero" => !"nan".
8396 bool IsInverted = false;
8397 if (FPClassTest InvertedCheck = invertFPClassTestIfSimpler(Test)) {
8398 IsInverted = true;
8399 Test = InvertedCheck;
8400 }
8401
8402 // Floating-point type properties.
8403 EVT ScalarFloatVT = OperandVT.getScalarType();
8404 const Type *FloatTy = ScalarFloatVT.getTypeForEVT(*DAG.getContext());
8405 const llvm::fltSemantics &Semantics = FloatTy->getFltSemantics();
8406 bool IsF80 = (ScalarFloatVT == MVT::f80);
8407
8408 // Some checks can be implemented using float comparisons, if floating point
8409 // exceptions are ignored.
8410 if (Flags.hasNoFPExcept() &&
8412 ISD::CondCode OrderedCmpOpcode = IsInverted ? ISD::SETUNE : ISD::SETOEQ;
8413 ISD::CondCode UnorderedCmpOpcode = IsInverted ? ISD::SETONE : ISD::SETUEQ;
8414
8415 if (std::optional<bool> IsCmp0 =
8416 isFCmpEqualZero(Test, Semantics, DAG.getMachineFunction());
8417 IsCmp0 && (isCondCodeLegalOrCustom(
8418 *IsCmp0 ? OrderedCmpOpcode : UnorderedCmpOpcode,
8419 OperandVT.getScalarType().getSimpleVT()))) {
8420
8421 // If denormals could be implicitly treated as 0, this is not equivalent
8422 // to a compare with 0 since it will also be true for denormals.
8423 return DAG.getSetCC(DL, ResultVT, Op,
8424 DAG.getConstantFP(0.0, DL, OperandVT),
8425 *IsCmp0 ? OrderedCmpOpcode : UnorderedCmpOpcode);
8426 }
8427
8428 if (Test == fcNan &&
8430 OperandVT.getScalarType().getSimpleVT())) {
8431 return DAG.getSetCC(DL, ResultVT, Op, Op,
8432 IsInverted ? ISD::SETO : ISD::SETUO);
8433 }
8434
8435 if (Test == fcInf &&
8437 OperandVT.getScalarType().getSimpleVT()) &&
8439 // isinf(x) --> fabs(x) == inf
8440 SDValue Abs = DAG.getNode(ISD::FABS, DL, OperandVT, Op);
8441 SDValue Inf =
8442 DAG.getConstantFP(APFloat::getInf(Semantics), DL, OperandVT);
8443 return DAG.getSetCC(DL, ResultVT, Abs, Inf,
8444 IsInverted ? ISD::SETUNE : ISD::SETOEQ);
8445 }
8446 }
8447
8448 // In the general case use integer operations.
8449 unsigned BitSize = OperandVT.getScalarSizeInBits();
8450 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), BitSize);
8451 if (OperandVT.isVector())
8452 IntVT = EVT::getVectorVT(*DAG.getContext(), IntVT,
8453 OperandVT.getVectorElementCount());
8454 SDValue OpAsInt = DAG.getBitcast(IntVT, Op);
8455
8456 // Various masks.
8457 APInt SignBit = APInt::getSignMask(BitSize);
8458 APInt ValueMask = APInt::getSignedMaxValue(BitSize); // All bits but sign.
8459 APInt Inf = APFloat::getInf(Semantics).bitcastToAPInt(); // Exp and int bit.
8460 const unsigned ExplicitIntBitInF80 = 63;
8461 APInt ExpMask = Inf;
8462 if (IsF80)
8463 ExpMask.clearBit(ExplicitIntBitInF80);
8464 APInt AllOneMantissa = APFloat::getLargest(Semantics).bitcastToAPInt() & ~Inf;
8465 APInt QNaNBitMask =
8466 APInt::getOneBitSet(BitSize, AllOneMantissa.getActiveBits() - 1);
8467 APInt InvertionMask = APInt::getAllOnes(ResultVT.getScalarSizeInBits());
8468
8469 SDValue ValueMaskV = DAG.getConstant(ValueMask, DL, IntVT);
8470 SDValue SignBitV = DAG.getConstant(SignBit, DL, IntVT);
8471 SDValue ExpMaskV = DAG.getConstant(ExpMask, DL, IntVT);
8472 SDValue ZeroV = DAG.getConstant(0, DL, IntVT);
8473 SDValue InfV = DAG.getConstant(Inf, DL, IntVT);
8474 SDValue ResultInvertionMask = DAG.getConstant(InvertionMask, DL, ResultVT);
8475
8476 SDValue Res;
8477 const auto appendResult = [&](SDValue PartialRes) {
8478 if (PartialRes) {
8479 if (Res)
8480 Res = DAG.getNode(ISD::OR, DL, ResultVT, Res, PartialRes);
8481 else
8482 Res = PartialRes;
8483 }
8484 };
8485
8486 SDValue IntBitIsSetV; // Explicit integer bit in f80 mantissa is set.
8487 const auto getIntBitIsSet = [&]() -> SDValue {
8488 if (!IntBitIsSetV) {
8489 APInt IntBitMask(BitSize, 0);
8490 IntBitMask.setBit(ExplicitIntBitInF80);
8491 SDValue IntBitMaskV = DAG.getConstant(IntBitMask, DL, IntVT);
8492 SDValue IntBitV = DAG.getNode(ISD::AND, DL, IntVT, OpAsInt, IntBitMaskV);
8493 IntBitIsSetV = DAG.getSetCC(DL, ResultVT, IntBitV, ZeroV, ISD::SETNE);
8494 }
8495 return IntBitIsSetV;
8496 };
8497
8498 // Split the value into sign bit and absolute value.
8499 SDValue AbsV = DAG.getNode(ISD::AND, DL, IntVT, OpAsInt, ValueMaskV);
8500 SDValue SignV = DAG.getSetCC(DL, ResultVT, OpAsInt,
8501 DAG.getConstant(0.0, DL, IntVT), ISD::SETLT);
8502
8503 // Tests that involve more than one class should be processed first.
8504 SDValue PartialRes;
8505
8506 if (IsF80)
8507 ; // Detect finite numbers of f80 by checking individual classes because
8508 // they have different settings of the explicit integer bit.
8509 else if ((Test & fcFinite) == fcFinite) {
8510 // finite(V) ==> abs(V) < exp_mask
8511 PartialRes = DAG.getSetCC(DL, ResultVT, AbsV, ExpMaskV, ISD::SETLT);
8512 Test &= ~fcFinite;
8513 } else if ((Test & fcFinite) == fcPosFinite) {
8514 // finite(V) && V > 0 ==> V < exp_mask
8515 PartialRes = DAG.getSetCC(DL, ResultVT, OpAsInt, ExpMaskV, ISD::SETULT);
8516 Test &= ~fcPosFinite;
8517 } else if ((Test & fcFinite) == fcNegFinite) {
8518 // finite(V) && V < 0 ==> abs(V) < exp_mask && signbit == 1
8519 PartialRes = DAG.getSetCC(DL, ResultVT, AbsV, ExpMaskV, ISD::SETLT);
8520 PartialRes = DAG.getNode(ISD::AND, DL, ResultVT, PartialRes, SignV);
8521 Test &= ~fcNegFinite;
8522 }
8523 appendResult(PartialRes);
8524
8525 if (FPClassTest PartialCheck = Test & (fcZero | fcSubnormal)) {
8526 // fcZero | fcSubnormal => test all exponent bits are 0
8527 // TODO: Handle sign bit specific cases
8528 if (PartialCheck == (fcZero | fcSubnormal)) {
8529 SDValue ExpBits = DAG.getNode(ISD::AND, DL, IntVT, OpAsInt, ExpMaskV);
8530 SDValue ExpIsZero =
8531 DAG.getSetCC(DL, ResultVT, ExpBits, ZeroV, ISD::SETEQ);
8532 appendResult(ExpIsZero);
8533 Test &= ~PartialCheck & fcAllFlags;
8534 }
8535 }
8536
8537 // Check for individual classes.
8538
8539 if (unsigned PartialCheck = Test & fcZero) {
8540 if (PartialCheck == fcPosZero)
8541 PartialRes = DAG.getSetCC(DL, ResultVT, OpAsInt, ZeroV, ISD::SETEQ);
8542 else if (PartialCheck == fcZero)
8543 PartialRes = DAG.getSetCC(DL, ResultVT, AbsV, ZeroV, ISD::SETEQ);
8544 else // ISD::fcNegZero
8545 PartialRes = DAG.getSetCC(DL, ResultVT, OpAsInt, SignBitV, ISD::SETEQ);
8546 appendResult(PartialRes);
8547 }
8548
8549 if (unsigned PartialCheck = Test & fcSubnormal) {
8550 // issubnormal(V) ==> unsigned(abs(V) - 1) < (all mantissa bits set)
8551 // issubnormal(V) && V>0 ==> unsigned(V - 1) < (all mantissa bits set)
8552 SDValue V = (PartialCheck == fcPosSubnormal) ? OpAsInt : AbsV;
8553 SDValue MantissaV = DAG.getConstant(AllOneMantissa, DL, IntVT);
8554 SDValue VMinusOneV =
8555 DAG.getNode(ISD::SUB, DL, IntVT, V, DAG.getConstant(1, DL, IntVT));
8556 PartialRes = DAG.getSetCC(DL, ResultVT, VMinusOneV, MantissaV, ISD::SETULT);
8557 if (PartialCheck == fcNegSubnormal)
8558 PartialRes = DAG.getNode(ISD::AND, DL, ResultVT, PartialRes, SignV);
8559 appendResult(PartialRes);
8560 }
8561
8562 if (unsigned PartialCheck = Test & fcInf) {
8563 if (PartialCheck == fcPosInf)
8564 PartialRes = DAG.getSetCC(DL, ResultVT, OpAsInt, InfV, ISD::SETEQ);
8565 else if (PartialCheck == fcInf)
8566 PartialRes = DAG.getSetCC(DL, ResultVT, AbsV, InfV, ISD::SETEQ);
8567 else { // ISD::fcNegInf
8568 APInt NegInf = APFloat::getInf(Semantics, true).bitcastToAPInt();
8569 SDValue NegInfV = DAG.getConstant(NegInf, DL, IntVT);
8570 PartialRes = DAG.getSetCC(DL, ResultVT, OpAsInt, NegInfV, ISD::SETEQ);
8571 }
8572 appendResult(PartialRes);
8573 }
8574
8575 if (unsigned PartialCheck = Test & fcNan) {
8576 APInt InfWithQnanBit = Inf | QNaNBitMask;
8577 SDValue InfWithQnanBitV = DAG.getConstant(InfWithQnanBit, DL, IntVT);
8578 if (PartialCheck == fcNan) {
8579 // isnan(V) ==> abs(V) > int(inf)
8580 PartialRes = DAG.getSetCC(DL, ResultVT, AbsV, InfV, ISD::SETGT);
8581 if (IsF80) {
8582 // Recognize unsupported values as NaNs for compatibility with glibc.
8583 // In them (exp(V)==0) == int_bit.
8584 SDValue ExpBits = DAG.getNode(ISD::AND, DL, IntVT, AbsV, ExpMaskV);
8585 SDValue ExpIsZero =
8586 DAG.getSetCC(DL, ResultVT, ExpBits, ZeroV, ISD::SETEQ);
8587 SDValue IsPseudo =
8588 DAG.getSetCC(DL, ResultVT, getIntBitIsSet(), ExpIsZero, ISD::SETEQ);
8589 PartialRes = DAG.getNode(ISD::OR, DL, ResultVT, PartialRes, IsPseudo);
8590 }
8591 } else if (PartialCheck == fcQNan) {
8592 // isquiet(V) ==> abs(V) >= (unsigned(Inf) | quiet_bit)
8593 PartialRes =
8594 DAG.getSetCC(DL, ResultVT, AbsV, InfWithQnanBitV, ISD::SETGE);
8595 } else { // ISD::fcSNan
8596 // issignaling(V) ==> abs(V) > unsigned(Inf) &&
8597 // abs(V) < (unsigned(Inf) | quiet_bit)
8598 SDValue IsNan = DAG.getSetCC(DL, ResultVT, AbsV, InfV, ISD::SETGT);
8599 SDValue IsNotQnan =
8600 DAG.getSetCC(DL, ResultVT, AbsV, InfWithQnanBitV, ISD::SETLT);
8601 PartialRes = DAG.getNode(ISD::AND, DL, ResultVT, IsNan, IsNotQnan);
8602 }
8603 appendResult(PartialRes);
8604 }
8605
8606 if (unsigned PartialCheck = Test & fcNormal) {
8607 // isnormal(V) ==> (0 < exp < max_exp) ==> (unsigned(exp-1) < (max_exp-1))
8608 APInt ExpLSB = ExpMask & ~(ExpMask.shl(1));
8609 SDValue ExpLSBV = DAG.getConstant(ExpLSB, DL, IntVT);
8610 SDValue ExpMinus1 = DAG.getNode(ISD::SUB, DL, IntVT, AbsV, ExpLSBV);
8611 APInt ExpLimit = ExpMask - ExpLSB;
8612 SDValue ExpLimitV = DAG.getConstant(ExpLimit, DL, IntVT);
8613 PartialRes = DAG.getSetCC(DL, ResultVT, ExpMinus1, ExpLimitV, ISD::SETULT);
8614 if (PartialCheck == fcNegNormal)
8615 PartialRes = DAG.getNode(ISD::AND, DL, ResultVT, PartialRes, SignV);
8616 else if (PartialCheck == fcPosNormal) {
8617 SDValue PosSignV =
8618 DAG.getNode(ISD::XOR, DL, ResultVT, SignV, ResultInvertionMask);
8619 PartialRes = DAG.getNode(ISD::AND, DL, ResultVT, PartialRes, PosSignV);
8620 }
8621 if (IsF80)
8622 PartialRes =
8623 DAG.getNode(ISD::AND, DL, ResultVT, PartialRes, getIntBitIsSet());
8624 appendResult(PartialRes);
8625 }
8626
8627 if (!Res)
8628 return DAG.getConstant(IsInverted, DL, ResultVT);
8629 if (IsInverted)
8630 Res = DAG.getNode(ISD::XOR, DL, ResultVT, Res, ResultInvertionMask);
8631 return Res;
8632}
8633
8634// Only expand vector types if we have the appropriate vector bit operations.
8635static bool canExpandVectorCTPOP(const TargetLowering &TLI, EVT VT) {
8636 assert(VT.isVector() && "Expected vector type");
8637 unsigned Len = VT.getScalarSizeInBits();
8638 return TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
8641 (Len == 8 || TLI.isOperationLegalOrCustom(ISD::MUL, VT)) &&
8643}
8644
8646 SDLoc dl(Node);
8647 EVT VT = Node->getValueType(0);
8648 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
8649 SDValue Op = Node->getOperand(0);
8650 unsigned Len = VT.getScalarSizeInBits();
8651 assert(VT.isInteger() && "CTPOP not implemented for this type.");
8652
8653 // TODO: Add support for irregular type lengths.
8654 if (!(Len <= 128 && Len % 8 == 0))
8655 return SDValue();
8656
8657 // Only expand vector types if we have the appropriate vector bit operations.
8658 if (VT.isVector() && !canExpandVectorCTPOP(*this, VT))
8659 return SDValue();
8660
8661 // This is the "best" algorithm from
8662 // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
8663 SDValue Mask55 =
8664 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)), dl, VT);
8665 SDValue Mask33 =
8666 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)), dl, VT);
8667 SDValue Mask0F =
8668 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)), dl, VT);
8669
8670 // v = v - ((v >> 1) & 0x55555555...)
8671 Op = DAG.getNode(ISD::SUB, dl, VT, Op,
8672 DAG.getNode(ISD::AND, dl, VT,
8673 DAG.getNode(ISD::SRL, dl, VT, Op,
8674 DAG.getConstant(1, dl, ShVT)),
8675 Mask55));
8676 // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...)
8677 Op = DAG.getNode(ISD::ADD, dl, VT, DAG.getNode(ISD::AND, dl, VT, Op, Mask33),
8678 DAG.getNode(ISD::AND, dl, VT,
8679 DAG.getNode(ISD::SRL, dl, VT, Op,
8680 DAG.getConstant(2, dl, ShVT)),
8681 Mask33));
8682 // v = (v + (v >> 4)) & 0x0F0F0F0F...
8683 Op = DAG.getNode(ISD::AND, dl, VT,
8684 DAG.getNode(ISD::ADD, dl, VT, Op,
8685 DAG.getNode(ISD::SRL, dl, VT, Op,
8686 DAG.getConstant(4, dl, ShVT))),
8687 Mask0F);
8688
8689 if (Len <= 8)
8690 return Op;
8691
8692 // Avoid the multiply if we only have 2 bytes to add.
8693 // TODO: Only doing this for scalars because vectors weren't as obviously
8694 // improved.
8695 if (Len == 16 && !VT.isVector()) {
8696 // v = (v + (v >> 8)) & 0x00FF;
8697 return DAG.getNode(ISD::AND, dl, VT,
8698 DAG.getNode(ISD::ADD, dl, VT, Op,
8699 DAG.getNode(ISD::SRL, dl, VT, Op,
8700 DAG.getConstant(8, dl, ShVT))),
8701 DAG.getConstant(0xFF, dl, VT));
8702 }
8703
8704 // v = (v * 0x01010101...) >> (Len - 8)
8705 SDValue Mask01 =
8706 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)), dl, VT);
8707 return DAG.getNode(ISD::SRL, dl, VT,
8708 DAG.getNode(ISD::MUL, dl, VT, Op, Mask01),
8709 DAG.getConstant(Len - 8, dl, ShVT));
8710}
8711
8713 SDLoc dl(Node);
8714 EVT VT = Node->getValueType(0);
8715 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
8716 SDValue Op = Node->getOperand(0);
8717 SDValue Mask = Node->getOperand(1);
8718 SDValue VL = Node->getOperand(2);
8719 unsigned Len = VT.getScalarSizeInBits();
8720 assert(VT.isInteger() && "VP_CTPOP not implemented for this type.");
8721
8722 // TODO: Add support for irregular type lengths.
8723 if (!(Len <= 128 && Len % 8 == 0))
8724 return SDValue();
8725
8726 // This is same algorithm of expandCTPOP from
8727 // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
8728 SDValue Mask55 =
8729 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)), dl, VT);
8730 SDValue Mask33 =
8731 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)), dl, VT);
8732 SDValue Mask0F =
8733 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)), dl, VT);
8734
8735 SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5;
8736
8737 // v = v - ((v >> 1) & 0x55555555...)
8738 Tmp1 = DAG.getNode(ISD::VP_AND, dl, VT,
8739 DAG.getNode(ISD::VP_LSHR, dl, VT, Op,
8740 DAG.getConstant(1, dl, ShVT), Mask, VL),
8741 Mask55, Mask, VL);
8742 Op = DAG.getNode(ISD::VP_SUB, dl, VT, Op, Tmp1, Mask, VL);
8743
8744 // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...)
8745 Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Op, Mask33, Mask, VL);
8746 Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT,
8747 DAG.getNode(ISD::VP_LSHR, dl, VT, Op,
8748 DAG.getConstant(2, dl, ShVT), Mask, VL),
8749 Mask33, Mask, VL);
8750 Op = DAG.getNode(ISD::VP_ADD, dl, VT, Tmp2, Tmp3, Mask, VL);
8751
8752 // v = (v + (v >> 4)) & 0x0F0F0F0F...
8753 Tmp4 = DAG.getNode(ISD::VP_LSHR, dl, VT, Op, DAG.getConstant(4, dl, ShVT),
8754 Mask, VL),
8755 Tmp5 = DAG.getNode(ISD::VP_ADD, dl, VT, Op, Tmp4, Mask, VL);
8756 Op = DAG.getNode(ISD::VP_AND, dl, VT, Tmp5, Mask0F, Mask, VL);
8757
8758 if (Len <= 8)
8759 return Op;
8760
8761 // v = (v * 0x01010101...) >> (Len - 8)
8762 SDValue Mask01 =
8763 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)), dl, VT);
8764 return DAG.getNode(ISD::VP_LSHR, dl, VT,
8765 DAG.getNode(ISD::VP_MUL, dl, VT, Op, Mask01, Mask, VL),
8766 DAG.getConstant(Len - 8, dl, ShVT), Mask, VL);
8767}
8768
8770 SDLoc dl(Node);
8771 EVT VT = Node->getValueType(0);
8772 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
8773 SDValue Op = Node->getOperand(0);
8774 unsigned NumBitsPerElt = VT.getScalarSizeInBits();
8775
8776 // If the non-ZERO_UNDEF version is supported we can use that instead.
8777 if (Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF &&
8779 return DAG.getNode(ISD::CTLZ, dl, VT, Op);
8780
8781 // If the ZERO_UNDEF version is supported use that and handle the zero case.
8783 EVT SetCCVT =
8784 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
8785 SDValue CTLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, VT, Op);
8786 SDValue Zero = DAG.getConstant(0, dl, VT);
8787 SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
8788 return DAG.getSelect(dl, VT, SrcIsZero,
8789 DAG.getConstant(NumBitsPerElt, dl, VT), CTLZ);
8790 }
8791
8792 // Only expand vector types if we have the appropriate vector bit operations.
8793 // This includes the operations needed to expand CTPOP if it isn't supported.
8794 if (VT.isVector() && (!isPowerOf2_32(NumBitsPerElt) ||
8796 !canExpandVectorCTPOP(*this, VT)) ||
8799 return SDValue();
8800
8801 // for now, we do this:
8802 // x = x | (x >> 1);
8803 // x = x | (x >> 2);
8804 // ...
8805 // x = x | (x >>16);
8806 // x = x | (x >>32); // for 64-bit input
8807 // return popcount(~x);
8808 //
8809 // Ref: "Hacker's Delight" by Henry Warren
8810 for (unsigned i = 0; (1U << i) < NumBitsPerElt; ++i) {
8811 SDValue Tmp = DAG.getConstant(1ULL << i, dl, ShVT);
8812 Op = DAG.getNode(ISD::OR, dl, VT, Op,
8813 DAG.getNode(ISD::SRL, dl, VT, Op, Tmp));
8814 }
8815 Op = DAG.getNOT(dl, Op, VT);
8816 return DAG.getNode(ISD::CTPOP, dl, VT, Op);
8817}
8818
8820 SDLoc dl(Node);
8821 EVT VT = Node->getValueType(0);
8822 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
8823 SDValue Op = Node->getOperand(0);
8824 SDValue Mask = Node->getOperand(1);
8825 SDValue VL = Node->getOperand(2);
8826 unsigned NumBitsPerElt = VT.getScalarSizeInBits();
8827
8828 // do this:
8829 // x = x | (x >> 1);
8830 // x = x | (x >> 2);
8831 // ...
8832 // x = x | (x >>16);
8833 // x = x | (x >>32); // for 64-bit input
8834 // return popcount(~x);
8835 for (unsigned i = 0; (1U << i) < NumBitsPerElt; ++i) {
8836 SDValue Tmp = DAG.getConstant(1ULL << i, dl, ShVT);
8837 Op = DAG.getNode(ISD::VP_OR, dl, VT, Op,
8838 DAG.getNode(ISD::VP_LSHR, dl, VT, Op, Tmp, Mask, VL), Mask,
8839 VL);
8840 }
8841 Op = DAG.getNode(ISD::VP_XOR, dl, VT, Op, DAG.getConstant(-1, dl, VT), Mask,
8842 VL);
8843 return DAG.getNode(ISD::VP_CTPOP, dl, VT, Op, Mask, VL);
8844}
8845
8847 const SDLoc &DL, EVT VT, SDValue Op,
8848 unsigned BitWidth) const {
8849 if (BitWidth != 32 && BitWidth != 64)
8850 return SDValue();
8851 APInt DeBruijn = BitWidth == 32 ? APInt(32, 0x077CB531U)
8852 : APInt(64, 0x0218A392CD3D5DBFULL);
8853 const DataLayout &TD = DAG.getDataLayout();
8854 MachinePointerInfo PtrInfo =
8856 unsigned ShiftAmt = BitWidth - Log2_32(BitWidth);
8857 SDValue Neg = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Op);
8858 SDValue Lookup = DAG.getNode(
8859 ISD::SRL, DL, VT,
8860 DAG.getNode(ISD::MUL, DL, VT, DAG.getNode(ISD::AND, DL, VT, Op, Neg),
8861 DAG.getConstant(DeBruijn, DL, VT)),
8862 DAG.getConstant(ShiftAmt, DL, VT));
8864
8866 for (unsigned i = 0; i < BitWidth; i++) {
8867 APInt Shl = DeBruijn.shl(i);
8868 APInt Lshr = Shl.lshr(ShiftAmt);
8869 Table[Lshr.getZExtValue()] = i;
8870 }
8871
8872 // Create a ConstantArray in Constant Pool
8873 auto *CA = ConstantDataArray::get(*DAG.getContext(), Table);
8874 SDValue CPIdx = DAG.getConstantPool(CA, getPointerTy(TD),
8875 TD.getPrefTypeAlign(CA->getType()));
8876 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, DL, VT, DAG.getEntryNode(),
8877 DAG.getMemBasePlusOffset(CPIdx, Lookup, DL),
8878 PtrInfo, MVT::i8);
8879 if (Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF)
8880 return ExtLoad;
8881
8882 EVT SetCCVT =
8883 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
8884 SDValue Zero = DAG.getConstant(0, DL, VT);
8885 SDValue SrcIsZero = DAG.getSetCC(DL, SetCCVT, Op, Zero, ISD::SETEQ);
8886 return DAG.getSelect(DL, VT, SrcIsZero,
8887 DAG.getConstant(BitWidth, DL, VT), ExtLoad);
8888}
8889
8891 SDLoc dl(Node);
8892 EVT VT = Node->getValueType(0);
8893 SDValue Op = Node->getOperand(0);
8894 unsigned NumBitsPerElt = VT.getScalarSizeInBits();
8895
8896 // If the non-ZERO_UNDEF version is supported we can use that instead.
8897 if (Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF &&
8899 return DAG.getNode(ISD::CTTZ, dl, VT, Op);
8900
8901 // If the ZERO_UNDEF version is supported use that and handle the zero case.
8903 EVT SetCCVT =
8904 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
8905 SDValue CTTZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, VT, Op);
8906 SDValue Zero = DAG.getConstant(0, dl, VT);
8907 SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
8908 return DAG.getSelect(dl, VT, SrcIsZero,
8909 DAG.getConstant(NumBitsPerElt, dl, VT), CTTZ);
8910 }
8911
8912 // Only expand vector types if we have the appropriate vector bit operations.
8913 // This includes the operations needed to expand CTPOP if it isn't supported.
8914 if (VT.isVector() && (!isPowerOf2_32(NumBitsPerElt) ||
8917 !canExpandVectorCTPOP(*this, VT)) ||
8921 return SDValue();
8922
8923 // Emit Table Lookup if ISD::CTLZ and ISD::CTPOP are not legal.
8924 if (!VT.isVector() && isOperationExpand(ISD::CTPOP, VT) &&
8926 if (SDValue V = CTTZTableLookup(Node, DAG, dl, VT, Op, NumBitsPerElt))
8927 return V;
8928
8929 // for now, we use: { return popcount(~x & (x - 1)); }
8930 // unless the target has ctlz but not ctpop, in which case we use:
8931 // { return 32 - nlz(~x & (x-1)); }
8932 // Ref: "Hacker's Delight" by Henry Warren
8933 SDValue Tmp = DAG.getNode(
8934 ISD::AND, dl, VT, DAG.getNOT(dl, Op, VT),
8935 DAG.getNode(ISD::SUB, dl, VT, Op, DAG.getConstant(1, dl, VT)));
8936
8937 // If ISD::CTLZ is legal and CTPOP isn't, then do that instead.
8939 return DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(NumBitsPerElt, dl, VT),
8940 DAG.getNode(ISD::CTLZ, dl, VT, Tmp));
8941 }
8942
8943 return DAG.getNode(ISD::CTPOP, dl, VT, Tmp);
8944}
8945
8947 SDValue Op = Node->getOperand(0);
8948 SDValue Mask = Node->getOperand(1);
8949 SDValue VL = Node->getOperand(2);
8950 SDLoc dl(Node);
8951 EVT VT = Node->getValueType(0);
8952
8953 // Same as the vector part of expandCTTZ, use: popcount(~x & (x - 1))
8954 SDValue Not = DAG.getNode(ISD::VP_XOR, dl, VT, Op,
8955 DAG.getConstant(-1, dl, VT), Mask, VL);
8956 SDValue MinusOne = DAG.getNode(ISD::VP_SUB, dl, VT, Op,
8957 DAG.getConstant(1, dl, VT), Mask, VL);
8958 SDValue Tmp = DAG.getNode(ISD::VP_AND, dl, VT, Not, MinusOne, Mask, VL);
8959 return DAG.getNode(ISD::VP_CTPOP, dl, VT, Tmp, Mask, VL);
8960}
8961
8963 bool IsNegative) const {
8964 SDLoc dl(N);
8965 EVT VT = N->getValueType(0);
8966 SDValue Op = N->getOperand(0);
8967
8968 // abs(x) -> smax(x,sub(0,x))
8969 if (!IsNegative && isOperationLegal(ISD::SUB, VT) &&
8971 SDValue Zero = DAG.getConstant(0, dl, VT);
8972 return DAG.getNode(ISD::SMAX, dl, VT, Op,
8973 DAG.getNode(ISD::SUB, dl, VT, Zero, Op));
8974 }
8975
8976 // abs(x) -> umin(x,sub(0,x))
8977 if (!IsNegative && isOperationLegal(ISD::SUB, VT) &&
8979 SDValue Zero = DAG.getConstant(0, dl, VT);
8980 Op = DAG.getFreeze(Op);
8981 return DAG.getNode(ISD::UMIN, dl, VT, Op,
8982 DAG.getNode(ISD::SUB, dl, VT, Zero, Op));
8983 }
8984
8985 // 0 - abs(x) -> smin(x, sub(0,x))
8986 if (IsNegative && isOperationLegal(ISD::SUB, VT) &&
8988 Op = DAG.getFreeze(Op);
8989 SDValue Zero = DAG.getConstant(0, dl, VT);
8990 return DAG.getNode(ISD::SMIN, dl, VT, Op,
8991 DAG.getNode(ISD::SUB, dl, VT, Zero, Op));
8992 }
8993
8994 // Only expand vector types if we have the appropriate vector operations.
8995 if (VT.isVector() &&
8997 (!IsNegative && !isOperationLegalOrCustom(ISD::ADD, VT)) ||
8998 (IsNegative && !isOperationLegalOrCustom(ISD::SUB, VT)) ||
9000 return SDValue();
9001
9002 Op = DAG.getFreeze(Op);
9003 SDValue Shift = DAG.getNode(
9004 ISD::SRA, dl, VT, Op,
9005 DAG.getShiftAmountConstant(VT.getScalarSizeInBits() - 1, VT, dl));
9006 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, Op, Shift);
9007
9008 // abs(x) -> Y = sra (X, size(X)-1); sub (xor (X, Y), Y)
9009 if (!IsNegative)
9010 return DAG.getNode(ISD::SUB, dl, VT, Xor, Shift);
9011
9012 // 0 - abs(x) -> Y = sra (X, size(X)-1); sub (Y, xor (X, Y))
9013 return DAG.getNode(ISD::SUB, dl, VT, Shift, Xor);
9014}
9015
9017 SDLoc dl(N);
9018 EVT VT = N->getValueType(0);
9019 SDValue LHS = DAG.getFreeze(N->getOperand(0));
9020 SDValue RHS = DAG.getFreeze(N->getOperand(1));
9021 bool IsSigned = N->getOpcode() == ISD::ABDS;
9022
9023 // abds(lhs, rhs) -> sub(smax(lhs,rhs), smin(lhs,rhs))
9024 // abdu(lhs, rhs) -> sub(umax(lhs,rhs), umin(lhs,rhs))
9025 unsigned MaxOpc = IsSigned ? ISD::SMAX : ISD::UMAX;
9026 unsigned MinOpc = IsSigned ? ISD::SMIN : ISD::UMIN;
9027 if (isOperationLegal(MaxOpc, VT) && isOperationLegal(MinOpc, VT)) {
9028 SDValue Max = DAG.getNode(MaxOpc, dl, VT, LHS, RHS);
9029 SDValue Min = DAG.getNode(MinOpc, dl, VT, LHS, RHS);
9030 return DAG.getNode(ISD::SUB, dl, VT, Max, Min);
9031 }
9032
9033 // abdu(lhs, rhs) -> or(usubsat(lhs,rhs), usubsat(rhs,lhs))
9034 if (!IsSigned && isOperationLegal(ISD::USUBSAT, VT))
9035 return DAG.getNode(ISD::OR, dl, VT,
9036 DAG.getNode(ISD::USUBSAT, dl, VT, LHS, RHS),
9037 DAG.getNode(ISD::USUBSAT, dl, VT, RHS, LHS));
9038
9039 // abds(lhs, rhs) -> select(sgt(lhs,rhs), sub(lhs,rhs), sub(rhs,lhs))
9040 // abdu(lhs, rhs) -> select(ugt(lhs,rhs), sub(lhs,rhs), sub(rhs,lhs))
9041 EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
9043 SDValue Cmp = DAG.getSetCC(dl, CCVT, LHS, RHS, CC);
9044 return DAG.getSelect(dl, VT, Cmp, DAG.getNode(ISD::SUB, dl, VT, LHS, RHS),
9045 DAG.getNode(ISD::SUB, dl, VT, RHS, LHS));
9046}
9047
9049 SDLoc dl(N);
9050 EVT VT = N->getValueType(0);
9051 SDValue Op = N->getOperand(0);
9052
9053 if (!VT.isSimple())
9054 return SDValue();
9055
9056 EVT SHVT = getShiftAmountTy(VT, DAG.getDataLayout());
9057 SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
9058 switch (VT.getSimpleVT().getScalarType().SimpleTy) {
9059 default:
9060 return SDValue();
9061 case MVT::i16:
9062 // Use a rotate by 8. This can be further expanded if necessary.
9063 return DAG.getNode(ISD::ROTL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
9064 case MVT::i32:
9065 Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
9066 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Op,
9067 DAG.getConstant(0xFF00, dl, VT));
9068 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(8, dl, SHVT));
9069 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
9070 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT));
9071 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
9072 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
9073 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
9074 return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
9075 case MVT::i64:
9076 Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
9077 Tmp7 = DAG.getNode(ISD::AND, dl, VT, Op,
9078 DAG.getConstant(255ULL<<8, dl, VT));
9079 Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Tmp7, DAG.getConstant(40, dl, SHVT));
9080 Tmp6 = DAG.getNode(ISD::AND, dl, VT, Op,
9081 DAG.getConstant(255ULL<<16, dl, VT));
9082 Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Tmp6, DAG.getConstant(24, dl, SHVT));
9083 Tmp5 = DAG.getNode(ISD::AND, dl, VT, Op,
9084 DAG.getConstant(255ULL<<24, dl, VT));
9085 Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Tmp5, DAG.getConstant(8, dl, SHVT));
9086 Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
9087 Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4,
9088 DAG.getConstant(255ULL<<24, dl, VT));
9089 Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
9090 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
9091 DAG.getConstant(255ULL<<16, dl, VT));
9092 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
9093 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2,
9094 DAG.getConstant(255ULL<<8, dl, VT));
9095 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
9096 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7);
9097 Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5);
9098 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
9099 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
9100 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6);
9101 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
9102 return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4);
9103 }
9104}
9105
9107 SDLoc dl(N);
9108 EVT VT = N->getValueType(0);
9109 SDValue Op = N->getOperand(0);
9110 SDValue Mask = N->getOperand(1);
9111 SDValue EVL = N->getOperand(2);
9112
9113 if (!VT.isSimple())
9114 return SDValue();
9115
9116 EVT SHVT = getShiftAmountTy(VT, DAG.getDataLayout());
9117 SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
9118 switch (VT.getSimpleVT().getScalarType().SimpleTy) {
9119 default:
9120 return SDValue();
9121 case MVT::i16:
9122 Tmp1 = DAG.getNode(ISD::VP_SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT),
9123 Mask, EVL);
9124 Tmp2 = DAG.getNode(ISD::VP_LSHR, dl, VT, Op, DAG.getConstant(8, dl, SHVT),
9125 Mask, EVL);
9126 return DAG.getNode(ISD::VP_OR, dl, VT, Tmp1, Tmp2, Mask, EVL);
9127 case MVT::i32:
9128 Tmp4 = DAG.getNode(ISD::VP_SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT),
9129 Mask, EVL);
9130 Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, Op, DAG.getConstant(0xFF00, dl, VT),
9131 Mask, EVL);
9132 Tmp3 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp3, DAG.getConstant(8, dl, SHVT),
9133 Mask, EVL);
9134 Tmp2 = DAG.getNode(ISD::VP_LSHR, dl, VT, Op, DAG.getConstant(8, dl, SHVT),
9135 Mask, EVL);
9136 Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp2,
9137 DAG.getConstant(0xFF00, dl, VT), Mask, EVL);
9138 Tmp1 = DAG.getNode(ISD::VP_LSHR, dl, VT, Op, DAG.getConstant(24, dl, SHVT),
9139 Mask, EVL);
9140 Tmp4 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp4, Tmp3, Mask, EVL);
9141 Tmp2 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp1, Mask, EVL);
9142 return DAG.getNode(ISD::VP_OR, dl, VT, Tmp4, Tmp2, Mask, EVL);
9143 case MVT::i64:
9144 Tmp8 = DAG.getNode(ISD::VP_SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT),
9145 Mask, EVL);
9146 Tmp7 = DAG.getNode(ISD::VP_AND, dl, VT, Op,
9147 DAG.getConstant(255ULL << 8, dl, VT), Mask, EVL);
9148 Tmp7 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp7, DAG.getConstant(40, dl, SHVT),
9149 Mask, EVL);
9150 Tmp6 = DAG.getNode(ISD::VP_AND, dl, VT, Op,
9151 DAG.getConstant(255ULL << 16, dl, VT), Mask, EVL);
9152 Tmp6 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp6, DAG.getConstant(24, dl, SHVT),
9153 Mask, EVL);
9154 Tmp5 = DAG.getNode(ISD::VP_AND, dl, VT, Op,
9155 DAG.getConstant(255ULL << 24, dl, VT), Mask, EVL);
9156 Tmp5 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp5, DAG.getConstant(8, dl, SHVT),
9157 Mask, EVL);
9158 Tmp4 = DAG.getNode(ISD::VP_LSHR, dl, VT, Op, DAG.getConstant(8, dl, SHVT),
9159 Mask, EVL);
9160 Tmp4 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp4,
9161 DAG.getConstant(255ULL << 24, dl, VT), Mask, EVL);
9162 Tmp3 = DAG.getNode(ISD::VP_LSHR, dl, VT, Op, DAG.getConstant(24, dl, SHVT),
9163 Mask, EVL);
9164 Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp3,
9165 DAG.getConstant(255ULL << 16, dl, VT), Mask, EVL);
9166 Tmp2 = DAG.getNode(ISD::VP_LSHR, dl, VT, Op, DAG.getConstant(40, dl, SHVT),
9167 Mask, EVL);
9168 Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp2,
9169 DAG.getConstant(255ULL << 8, dl, VT), Mask, EVL);
9170 Tmp1 = DAG.getNode(ISD::VP_LSHR, dl, VT, Op, DAG.getConstant(56, dl, SHVT),
9171 Mask, EVL);
9172 Tmp8 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp8, Tmp7, Mask, EVL);
9173 Tmp6 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp6, Tmp5, Mask, EVL);
9174 Tmp4 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp4, Tmp3, Mask, EVL);
9175 Tmp2 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp1, Mask, EVL);
9176 Tmp8 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp8, Tmp6, Mask, EVL);
9177 Tmp4 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp4, Tmp2, Mask, EVL);
9178 return DAG.getNode(ISD::VP_OR, dl, VT, Tmp8, Tmp4, Mask, EVL);
9179 }
9180}
9181
9183 SDLoc dl(N);
9184 EVT VT = N->getValueType(0);
9185 SDValue Op = N->getOperand(0);
9186 EVT SHVT = getShiftAmountTy(VT, DAG.getDataLayout());
9187 unsigned Sz = VT.getScalarSizeInBits();
9188
9189 SDValue Tmp, Tmp2, Tmp3;
9190
9191 // If we can, perform BSWAP first and then the mask+swap the i4, then i2
9192 // and finally the i1 pairs.
9193 // TODO: We can easily support i4/i2 legal types if any target ever does.
9194 if (Sz >= 8 && isPowerOf2_32(Sz)) {
9195 // Create the masks - repeating the pattern every byte.
9196 APInt Mask4 = APInt::getSplat(Sz, APInt(8, 0x0F));
9197 APInt Mask2 = APInt::getSplat(Sz, APInt(8, 0x33));
9198 APInt Mask1 = APInt::getSplat(Sz, APInt(8, 0x55));
9199
9200 // BSWAP if the type is wider than a single byte.
9201 Tmp = (Sz > 8 ? DAG.getNode(ISD::BSWAP, dl, VT, Op) : Op);
9202
9203 // swap i4: ((V >> 4) & 0x0F) | ((V & 0x0F) << 4)
9204 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(4, dl, SHVT));
9205 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Mask4, dl, VT));
9206 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(Mask4, dl, VT));
9207 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, SHVT));
9208 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
9209
9210 // swap i2: ((V >> 2) & 0x33) | ((V & 0x33) << 2)
9211 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(2, dl, SHVT));
9212 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Mask2, dl, VT));
9213 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(Mask2, dl, VT));
9214 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, SHVT));
9215 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
9216
9217 // swap i1: ((V >> 1) & 0x55) | ((V & 0x55) << 1)
9218 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(1, dl, SHVT));
9219 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Mask1, dl, VT));
9220 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(Mask1, dl, VT));
9221 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, SHVT));
9222 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
9223 return Tmp;
9224 }
9225
9226 Tmp = DAG.getConstant(0, dl, VT);
9227 for (unsigned I = 0, J = Sz-1; I < Sz; ++I, --J) {
9228 if (I < J)
9229 Tmp2 =
9230 DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(J - I, dl, SHVT));
9231 else
9232 Tmp2 =
9233 DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(I - J, dl, SHVT));
9234
9235 APInt Shift = APInt::getOneBitSet(Sz, J);
9236 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Shift, dl, VT));
9237 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp, Tmp2);
9238 }
9239
9240 return Tmp;
9241}
9242
9244 assert(N->getOpcode() == ISD::VP_BITREVERSE);
9245
9246 SDLoc dl(N);
9247 EVT VT = N->getValueType(0);
9248 SDValue Op = N->getOperand(0);
9249 SDValue Mask = N->getOperand(1);
9250 SDValue EVL = N->getOperand(2);
9251 EVT SHVT = getShiftAmountTy(VT, DAG.getDataLayout());
9252 unsigned Sz = VT.getScalarSizeInBits();
9253
9254 SDValue Tmp, Tmp2, Tmp3;
9255
9256 // If we can, perform BSWAP first and then the mask+swap the i4, then i2
9257 // and finally the i1 pairs.
9258 // TODO: We can easily support i4/i2 legal types if any target ever does.
9259 if (Sz >= 8 && isPowerOf2_32(Sz)) {
9260 // Create the masks - repeating the pattern every byte.
9261 APInt Mask4 = APInt::getSplat(Sz, APInt(8, 0x0F));
9262 APInt Mask2 = APInt::getSplat(Sz, APInt(8, 0x33));
9263 APInt Mask1 = APInt::getSplat(Sz, APInt(8, 0x55));
9264
9265 // BSWAP if the type is wider than a single byte.
9266 Tmp = (Sz > 8 ? DAG.getNode(ISD::VP_BSWAP, dl, VT, Op, Mask, EVL) : Op);
9267
9268 // swap i4: ((V >> 4) & 0x0F) | ((V & 0x0F) << 4)
9269 Tmp2 = DAG.getNode(ISD::VP_LSHR, dl, VT, Tmp, DAG.getConstant(4, dl, SHVT),
9270 Mask, EVL);
9271 Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp2,
9272 DAG.getConstant(Mask4, dl, VT), Mask, EVL);
9273 Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp, DAG.getConstant(Mask4, dl, VT),
9274 Mask, EVL);
9275 Tmp3 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, SHVT),
9276 Mask, EVL);
9277 Tmp = DAG.getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp3, Mask, EVL);
9278
9279 // swap i2: ((V >> 2) & 0x33) | ((V & 0x33) << 2)
9280 Tmp2 = DAG.getNode(ISD::VP_LSHR, dl, VT, Tmp, DAG.getConstant(2, dl, SHVT),
9281 Mask, EVL);
9282 Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp2,
9283 DAG.getConstant(Mask2, dl, VT), Mask, EVL);
9284 Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp, DAG.getConstant(Mask2, dl, VT),
9285 Mask, EVL);
9286 Tmp3 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, SHVT),
9287 Mask, EVL);
9288 Tmp = DAG.getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp3, Mask, EVL);
9289
9290 // swap i1: ((V >> 1) & 0x55) | ((V & 0x55) << 1)
9291 Tmp2 = DAG.getNode(ISD::VP_LSHR, dl, VT, Tmp, DAG.getConstant(1, dl, SHVT),
9292 Mask, EVL);
9293 Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp2,
9294 DAG.getConstant(Mask1, dl, VT), Mask, EVL);
9295 Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp, DAG.getConstant(Mask1, dl, VT),
9296 Mask, EVL);
9297 Tmp3 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, SHVT),
9298 Mask, EVL);
9299 Tmp = DAG.getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp3, Mask, EVL);
9300 return Tmp;
9301 }
9302 return SDValue();
9303}
9304
9305std::pair<SDValue, SDValue>
9307 SelectionDAG &DAG) const {
9308 SDLoc SL(LD);
9309 SDValue Chain = LD->getChain();
9310 SDValue BasePTR = LD->getBasePtr();
9311 EVT SrcVT = LD->getMemoryVT();
9312 EVT DstVT = LD->getValueType(0);
9313 ISD::LoadExtType ExtType = LD->getExtensionType();
9314
9315 if (SrcVT.isScalableVector())
9316 report_fatal_error("Cannot scalarize scalable vector loads");
9317
9318 unsigned NumElem = SrcVT.getVectorNumElements();
9319
9320 EVT SrcEltVT = SrcVT.getScalarType();
9321 EVT DstEltVT = DstVT.getScalarType();
9322
9323 // A vector must always be stored in memory as-is, i.e. without any padding
9324 // between the elements, since various code depend on it, e.g. in the
9325 // handling of a bitcast of a vector type to int, which may be done with a
9326 // vector store followed by an integer load. A vector that does not have
9327 // elements that are byte-sized must therefore be stored as an integer
9328 // built out of the extracted vector elements.
9329 if (!SrcEltVT.isByteSized()) {
9330 unsigned NumLoadBits = SrcVT.getStoreSizeInBits();
9331 EVT LoadVT = EVT::getIntegerVT(*DAG.getContext(), NumLoadBits);
9332
9333 unsigned NumSrcBits = SrcVT.getSizeInBits();
9334 EVT SrcIntVT = EVT::getIntegerVT(*DAG.getContext(), NumSrcBits);
9335
9336 unsigned SrcEltBits = SrcEltVT.getSizeInBits();
9337 SDValue SrcEltBitMask = DAG.getConstant(
9338 APInt::getLowBitsSet(NumLoadBits, SrcEltBits), SL, LoadVT);
9339
9340 // Load the whole vector and avoid masking off the top bits as it makes
9341 // the codegen worse.
9342 SDValue Load =
9343 DAG.getExtLoad(ISD::EXTLOAD, SL, LoadVT, Chain, BasePTR,
9344 LD->getPointerInfo(), SrcIntVT, LD->getOriginalAlign(),
9345 LD->getMemOperand()->getFlags(), LD->getAAInfo());
9346
9348 for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
9349 unsigned ShiftIntoIdx =
9350 (DAG.getDataLayout().isBigEndian() ? (NumElem - 1) - Idx : Idx);
9351 SDValue ShiftAmount =
9352 DAG.getShiftAmountConstant(ShiftIntoIdx * SrcEltVT.getSizeInBits(),
9353 LoadVT, SL, /*LegalTypes=*/false);
9354 SDValue ShiftedElt = DAG.getNode(ISD::SRL, SL, LoadVT, Load, ShiftAmount);
9355 SDValue Elt =
9356 DAG.getNode(ISD::AND, SL, LoadVT, ShiftedElt, SrcEltBitMask);
9357 SDValue Scalar = DAG.getNode(ISD::TRUNCATE, SL, SrcEltVT, Elt);
9358
9359 if (ExtType != ISD::NON_EXTLOAD) {
9360 unsigned ExtendOp = ISD::getExtForLoadExtType(false, ExtType);
9361 Scalar = DAG.getNode(ExtendOp, SL, DstEltVT, Scalar);
9362 }
9363
9364 Vals.push_back(Scalar);
9365 }
9366
9367 SDValue Value = DAG.getBuildVector(DstVT, SL, Vals);
9368 return std::make_pair(Value, Load.getValue(1));
9369 }
9370
9371 unsigned Stride = SrcEltVT.getSizeInBits() / 8;
9372 assert(SrcEltVT.isByteSized());
9373
9375 SmallVector<SDValue, 8> LoadChains;
9376
9377 for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
9378 SDValue ScalarLoad =
9379 DAG.getExtLoad(ExtType, SL, DstEltVT, Chain, BasePTR,
9380 LD->getPointerInfo().getWithOffset(Idx * Stride),
9381 SrcEltVT, LD->getOriginalAlign(),
9382 LD->getMemOperand()->getFlags(), LD->getAAInfo());
9383
9384 BasePTR = DAG.getObjectPtrOffset(SL, BasePTR, TypeSize::getFixed(Stride));
9385
9386 Vals.push_back(ScalarLoad.getValue(0));
9387 LoadChains.push_back(ScalarLoad.getValue(1));
9388 }
9389
9390 SDValue NewChain = DAG.getNode(ISD::TokenFactor, SL, MVT::Other, LoadChains);
9391 SDValue Value = DAG.getBuildVector(DstVT, SL, Vals);
9392
9393 return std::make_pair(Value, NewChain);
9394}
9395
9397 SelectionDAG &DAG) const {
9398 SDLoc SL(ST);
9399
9400 SDValue Chain = ST->getChain();
9401 SDValue BasePtr = ST->getBasePtr();
9402 SDValue Value = ST->getValue();
9403 EVT StVT = ST->getMemoryVT();
9404
9405 if (StVT.isScalableVector())
9406 report_fatal_error("Cannot scalarize scalable vector stores");
9407
9408 // The type of the data we want to save
9409 EVT RegVT = Value.getValueType();
9410 EVT RegSclVT = RegVT.getScalarType();
9411
9412 // The type of data as saved in memory.
9413 EVT MemSclVT = StVT.getScalarType();
9414
9415 unsigned NumElem = StVT.getVectorNumElements();
9416
9417 // A vector must always be stored in memory as-is, i.e. without any padding
9418 // between the elements, since various code depend on it, e.g. in the
9419 // handling of a bitcast of a vector type to int, which may be done with a
9420 // vector store followed by an integer load. A vector that does not have
9421 // elements that are byte-sized must therefore be stored as an integer
9422 // built out of the extracted vector elements.
9423 if (!MemSclVT.isByteSized()) {
9424 unsigned NumBits = StVT.getSizeInBits();
9425 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);
9426
9427 SDValue CurrVal = DAG.getConstant(0, SL, IntVT);
9428
9429 for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
9430 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, RegSclVT, Value,
9431 DAG.getVectorIdxConstant(Idx, SL));
9432 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, MemSclVT, Elt);
9433 SDValue ExtElt = DAG.getNode(ISD::ZERO_EXTEND, SL, IntVT, Trunc);
9434 unsigned ShiftIntoIdx =
9435 (DAG.getDataLayout().isBigEndian() ? (NumElem - 1) - Idx : Idx);
9436 SDValue ShiftAmount =
9437 DAG.getConstant(ShiftIntoIdx * MemSclVT.getSizeInBits(), SL, IntVT);
9438 SDValue ShiftedElt =
9439 DAG.getNode(ISD::SHL, SL, IntVT, ExtElt, ShiftAmount);
9440 CurrVal = DAG.getNode(ISD::OR, SL, IntVT, CurrVal, ShiftedElt);
9441 }
9442
9443 return DAG.getStore(Chain, SL, CurrVal, BasePtr, ST->getPointerInfo(),
9444 ST->getOriginalAlign(), ST->getMemOperand()->getFlags(),
9445 ST->getAAInfo());
9446 }
9447
9448 // Store Stride in bytes
9449 unsigned Stride = MemSclVT.getSizeInBits() / 8;
9450 assert(Stride && "Zero stride!");
9451 // Extract each of the elements from the original vector and save them into
9452 // memory individually.
9454 for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
9455 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, RegSclVT, Value,
9456 DAG.getVectorIdxConstant(Idx, SL));
9457
9458 SDValue Ptr =
9459 DAG.getObjectPtrOffset(SL, BasePtr, TypeSize::getFixed(Idx * Stride));
9460
9461 // This scalar TruncStore may be illegal, but we legalize it later.
9462 SDValue Store = DAG.getTruncStore(
9463 Chain, SL, Elt, Ptr, ST->getPointerInfo().getWithOffset(Idx * Stride),
9464 MemSclVT, ST->getOriginalAlign(), ST->getMemOperand()->getFlags(),
9465 ST->getAAInfo());
9466
9467 Stores.push_back(Store);
9468 }
9469
9470 return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, Stores);
9471}
9472
9473std::pair<SDValue, SDValue>
9475 assert(LD->getAddressingMode() == ISD::UNINDEXED &&
9476 "unaligned indexed loads not implemented!");
9477 SDValue Chain = LD->getChain();
9478 SDValue Ptr = LD->getBasePtr();
9479 EVT VT = LD->getValueType(0);
9480 EVT LoadedVT = LD->getMemoryVT();
9481 SDLoc dl(LD);
9482 auto &MF = DAG.getMachineFunction();
9483
9484 if (VT.isFloatingPoint() || VT.isVector()) {
9485 EVT intVT = EVT::getIntegerVT(*DAG.getContext(), LoadedVT.getSizeInBits());
9486 if (isTypeLegal(intVT) && isTypeLegal(LoadedVT)) {
9487 if (!isOperationLegalOrCustom(ISD::LOAD, intVT) &&
9488 LoadedVT.isVector()) {
9489 // Scalarize the load and let the individual components be handled.
9490 return scalarizeVectorLoad(LD, DAG);
9491 }
9492
9493 // Expand to a (misaligned) integer load of the same size,
9494 // then bitconvert to floating point or vector.
9495 SDValue newLoad = DAG.getLoad(intVT, dl, Chain, Ptr,
9496 LD->getMemOperand());
9497 SDValue Result = DAG.getNode(ISD::BITCAST, dl, LoadedVT, newLoad);
9498 if (LoadedVT != VT)
9499 Result = DAG.getNode(VT.isFloatingPoint() ? ISD::FP_EXTEND :
9500 ISD::ANY_EXTEND, dl, VT, Result);
9501
9502 return std::make_pair(Result, newLoad.getValue(1));
9503 }
9504
9505 // Copy the value to a (aligned) stack slot using (unaligned) integer
9506 // loads and stores, then do a (aligned) load from the stack slot.
9507 MVT RegVT = getRegisterType(*DAG.getContext(), intVT);
9508 unsigned LoadedBytes = LoadedVT.getStoreSize();
9509 unsigned RegBytes = RegVT.getSizeInBits() / 8;
9510 unsigned NumRegs = (LoadedBytes + RegBytes - 1) / RegBytes;
9511
9512 // Make sure the stack slot is also aligned for the register type.
9513 SDValue StackBase = DAG.CreateStackTemporary(LoadedVT, RegVT);
9514 auto FrameIndex = cast<FrameIndexSDNode>(StackBase.getNode())->getIndex();
9516 SDValue StackPtr = StackBase;
9517 unsigned Offset = 0;
9518
9519 EVT PtrVT = Ptr.getValueType();
9520 EVT StackPtrVT = StackPtr.getValueType();
9521
9522 SDValue PtrIncrement = DAG.getConstant(RegBytes, dl, PtrVT);
9523 SDValue StackPtrIncrement = DAG.getConstant(RegBytes, dl, StackPtrVT);
9524
9525 // Do all but one copies using the full register width.
9526 for (unsigned i = 1; i < NumRegs; i++) {
9527 // Load one integer register's worth from the original location.
9528 SDValue Load = DAG.getLoad(
9529 RegVT, dl, Chain, Ptr, LD->getPointerInfo().getWithOffset(Offset),
9530 LD->getOriginalAlign(), LD->getMemOperand()->getFlags(),
9531 LD->getAAInfo());
9532 // Follow the load with a store to the stack slot. Remember the store.
9533 Stores.push_back(DAG.getStore(
9534 Load.getValue(1), dl, Load, StackPtr,
9535 MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset)));
9536 // Increment the pointers.
9537 Offset += RegBytes;
9538
9539 Ptr = DAG.getObjectPtrOffset(dl, Ptr, PtrIncrement);
9540 StackPtr = DAG.getObjectPtrOffset(dl, StackPtr, StackPtrIncrement);
9541 }
9542
9543 // The last copy may be partial. Do an extending load.
9544 EVT MemVT = EVT::getIntegerVT(*DAG.getContext(),
9545 8 * (LoadedBytes - Offset));
9546 SDValue Load =
9547 DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Chain, Ptr,
9548 LD->getPointerInfo().getWithOffset(Offset), MemVT,
9549 LD->getOriginalAlign(), LD->getMemOperand()->getFlags(),
9550 LD->getAAInfo());
9551 // Follow the load with a store to the stack slot. Remember the store.
9552 // On big-endian machines this requires a truncating store to ensure
9553 // that the bits end up in the right place.
9554 Stores.push_back(DAG.getTruncStore(
9555 Load.getValue(1), dl, Load, StackPtr,
9556 MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset), MemVT));
9557
9558 // The order of the stores doesn't matter - say it with a TokenFactor.
9559 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
9560
9561 // Finally, perform the original load only redirected to the stack slot.
9562 Load = DAG.getExtLoad(LD->getExtensionType(), dl, VT, TF, StackBase,
9563 MachinePointerInfo::getFixedStack(MF, FrameIndex, 0),
9564 LoadedVT);
9565
9566 // Callers expect a MERGE_VALUES node.
9567 return std::make_pair(Load, TF);
9568 }
9569
9570 assert(LoadedVT.isInteger() && !LoadedVT.isVector() &&
9571 "Unaligned load of unsupported type.");
9572
9573 // Compute the new VT that is half the size of the old one. This is an
9574 // integer MVT.
9575 unsigned NumBits = LoadedVT.getSizeInBits();
9576 EVT NewLoadedVT;
9577 NewLoadedVT = EVT::getIntegerVT(*DAG.getContext(), NumBits/2);
9578 NumBits >>= 1;
9579
9580 Align Alignment = LD->getOriginalAlign();
9581 unsigned IncrementSize = NumBits / 8;
9582 ISD::LoadExtType HiExtType = LD->getExtensionType();
9583
9584 // If the original load is NON_EXTLOAD, the hi part load must be ZEXTLOAD.
9585 if (HiExtType == ISD::NON_EXTLOAD)
9586 HiExtType = ISD::ZEXTLOAD;
9587
9588 // Load the value in two parts
9589 SDValue Lo, Hi;
9590 if (DAG.getDataLayout().isLittleEndian()) {
9591 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, LD->getPointerInfo(),
9592 NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
9593 LD->getAAInfo());
9594
9595 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::getFixed(IncrementSize));
9596 Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr,
9597 LD->getPointerInfo().getWithOffset(IncrementSize),
9598 NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
9599 LD->getAAInfo());
9600 } else {
9601 Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr, LD->getPointerInfo(),
9602 NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
9603 LD->getAAInfo());
9604
9605 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::getFixed(IncrementSize));
9606 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr,
9607 LD->getPointerInfo().getWithOffset(IncrementSize),
9608 NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
9609 LD->getAAInfo());
9610 }
9611
9612 // aggregate the two parts
9613 SDValue ShiftAmount = DAG.getShiftAmountConstant(NumBits, VT, dl);
9614 SDValue Result = DAG.getNode(ISD::SHL, dl, VT, Hi, ShiftAmount);
9615 Result = DAG.getNode(ISD::OR, dl, VT, Result, Lo);
9616
9617 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
9618 Hi.getValue(1));
9619
9620 return std::make_pair(Result, TF);
9621}
9622
9624 SelectionDAG &DAG) const {
9625 assert(ST->getAddressingMode() == ISD::UNINDEXED &&
9626 "unaligned indexed stores not implemented!");
9627 SDValue Chain = ST->getChain();
9628 SDValue Ptr = ST->getBasePtr();
9629 SDValue Val = ST->getValue();
9630 EVT VT = Val.getValueType();
9631 Align Alignment = ST->getOriginalAlign();
9632 auto &MF = DAG.getMachineFunction();
9633 EVT StoreMemVT = ST->getMemoryVT();
9634
9635 SDLoc dl(ST);
9636 if (StoreMemVT.isFloatingPoint() || StoreMemVT.isVector()) {
9637 EVT intVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
9638 if (isTypeLegal(intVT)) {
9639 if (!isOperationLegalOrCustom(ISD::STORE, intVT) &&
9640 StoreMemVT.isVector()) {
9641 // Scalarize the store and let the individual components be handled.
9642 SDValue Result = scalarizeVectorStore(ST, DAG);
9643 return Result;
9644 }
9645 // Expand to a bitconvert of the value to the integer type of the
9646 // same size, then a (misaligned) int store.
9647 // FIXME: Does not handle truncating floating point stores!
9648 SDValue Result = DAG.getNode(ISD::BITCAST, dl, intVT, Val);
9649 Result = DAG.getStore(Chain, dl, Result, Ptr, ST->getPointerInfo(),
9650 Alignment, ST->getMemOperand()->getFlags());
9651 return Result;
9652 }
9653 // Do a (aligned) store to a stack slot, then copy from the stack slot
9654 // to the final destination using (unaligned) integer loads and stores.
9655 MVT RegVT = getRegisterType(
9656 *DAG.getContext(),
9657 EVT::getIntegerVT(*DAG.getContext(), StoreMemVT.getSizeInBits()));
9658 EVT PtrVT = Ptr.getValueType();
9659 unsigned StoredBytes = StoreMemVT.getStoreSize();
9660 unsigned RegBytes = RegVT.getSizeInBits() / 8;
9661 unsigned NumRegs = (StoredBytes + RegBytes - 1) / RegBytes;
9662
9663 // Make sure the stack slot is also aligned for the register type.
9664 SDValue StackPtr = DAG.CreateStackTemporary(StoreMemVT, RegVT);
9665 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
9666
9667 // Perform the original store, only redirected to the stack slot.
9668 SDValue Store = DAG.getTruncStore(
9669 Chain, dl, Val, StackPtr,
9670 MachinePointerInfo::getFixedStack(MF, FrameIndex, 0), StoreMemVT);
9671
9672 EVT StackPtrVT = StackPtr.getValueType();
9673
9674 SDValue PtrIncrement = DAG.getConstant(RegBytes, dl, PtrVT);
9675 SDValue StackPtrIncrement = DAG.getConstant(RegBytes, dl, StackPtrVT);
9677 unsigned Offset = 0;
9678
9679 // Do all but one copies using the full register width.
9680 for (unsigned i = 1; i < NumRegs; i++) {
9681 // Load one integer register's worth from the stack slot.
9682 SDValue Load = DAG.getLoad(
9683 RegVT, dl, Store, StackPtr,
9684 MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset));
9685 // Store it to the final location. Remember the store.
9686 Stores.push_back(DAG.getStore(Load.getValue(1), dl, Load, Ptr,
9687 ST->getPointerInfo().getWithOffset(Offset),
9688 ST->getOriginalAlign(),
9689 ST->getMemOperand()->getFlags()));
9690 // Increment the pointers.
9691 Offset += RegBytes;
9692 StackPtr = DAG.getObjectPtrOffset(dl, StackPtr, StackPtrIncrement);
9693 Ptr = DAG.getObjectPtrOffset(dl, Ptr, PtrIncrement);
9694 }
9695
9696 // The last store may be partial. Do a truncating store. On big-endian
9697 // machines this requires an extending load from the stack slot to ensure
9698 // that the bits are in the right place.
9699 EVT LoadMemVT =
9700 EVT::getIntegerVT(*DAG.getContext(), 8 * (StoredBytes - Offset));
9701
9702 // Load from the stack slot.
9703 SDValue Load = DAG.getExtLoad(
9704 ISD::EXTLOAD, dl, RegVT, Store, StackPtr,
9705 MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset), LoadMemVT);
9706
9707 Stores.push_back(
9708 DAG.getTruncStore(Load.getValue(1), dl, Load, Ptr,
9709 ST->getPointerInfo().getWithOffset(Offset), LoadMemVT,
9710 ST->getOriginalAlign(),
9711 ST->getMemOperand()->getFlags(), ST->getAAInfo()));
9712 // The order of the stores doesn't matter - say it with a TokenFactor.
9713 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
9714 return Result;
9715 }
9716
9717 assert(StoreMemVT.isInteger() && !StoreMemVT.isVector() &&
9718 "Unaligned store of unknown type.");
9719 // Get the half-size VT
9720 EVT NewStoredVT = StoreMemVT.getHalfSizedIntegerVT(*DAG.getContext());
9721 unsigned NumBits = NewStoredVT.getFixedSizeInBits();
9722 unsigned IncrementSize = NumBits / 8;
9723
9724 // Divide the stored value in two parts.
9725 SDValue ShiftAmount =
9726 DAG.getShiftAmountConstant(NumBits, Val.getValueType(), dl);
9727 SDValue Lo = Val;
9728 // If Val is a constant, replace the upper bits with 0. The SRL will constant
9729 // fold and not use the upper bits. A smaller constant may be easier to
9730 // materialize.
9731 if (auto *C = dyn_cast<ConstantSDNode>(Lo); C && !C->isOpaque())
9732 Lo = DAG.getNode(
9733 ISD::AND, dl, VT, Lo,
9734 DAG.getConstant(APInt::getLowBitsSet(VT.getSizeInBits(), NumBits), dl,
9735 VT));
9736 SDValue Hi = DAG.getNode(ISD::SRL, dl, VT, Val, ShiftAmount);
9737
9738 // Store the two parts
9739 SDValue Store1, Store2;
9740 Store1 = DAG.getTruncStore(Chain, dl,
9741 DAG.getDataLayout().isLittleEndian() ? Lo : Hi,
9742 Ptr, ST->getPointerInfo(), NewStoredVT, Alignment,
9743 ST->getMemOperand()->getFlags());
9744
9745 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::getFixed(IncrementSize));
9746 Store2 = DAG.getTruncStore(
9747 Chain, dl, DAG.getDataLayout().isLittleEndian() ? Hi : Lo, Ptr,
9748 ST->getPointerInfo().getWithOffset(IncrementSize), NewStoredVT, Alignment,
9749 ST->getMemOperand()->getFlags(), ST->getAAInfo());
9750
9751 SDValue Result =
9752 DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2);
9753 return Result;
9754}
9755
9756SDValue
9758 const SDLoc &DL, EVT DataVT,
9759 SelectionDAG &DAG,
9760 bool IsCompressedMemory) const {
9761 SDValue Increment;
9762 EVT AddrVT = Addr.getValueType();
9763 EVT MaskVT = Mask.getValueType();
9764 assert(DataVT.getVectorElementCount() == MaskVT.getVectorElementCount() &&
9765 "Incompatible types of Data and Mask");
9766 if (IsCompressedMemory) {
9767 if (DataVT.isScalableVector())
9769 "Cannot currently handle compressed memory with scalable vectors");
9770 // Incrementing the pointer according to number of '1's in the mask.
9771 EVT MaskIntVT = EVT::getIntegerVT(*DAG.getContext(), MaskVT.getSizeInBits());
9772 SDValue MaskInIntReg = DAG.getBitcast(MaskIntVT, Mask);
9773 if (MaskIntVT.getSizeInBits() < 32) {
9774 MaskInIntReg = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, MaskInIntReg);
9775 MaskIntVT = MVT::i32;
9776 }
9777
9778 // Count '1's with POPCNT.
9779 Increment = DAG.getNode(ISD::CTPOP, DL, MaskIntVT, MaskInIntReg);
9780 Increment = DAG.getZExtOrTrunc(Increment, DL, AddrVT);
9781 // Scale is an element size in bytes.
9782 SDValue Scale = DAG.getConstant(DataVT.getScalarSizeInBits() / 8, DL,
9783 AddrVT);
9784 Increment = DAG.getNode(ISD::MUL, DL, AddrVT, Increment, Scale);
9785 } else if (DataVT.isScalableVector()) {
9786 Increment = DAG.getVScale(DL, AddrVT,
9787 APInt(AddrVT.getFixedSizeInBits(),
9788 DataVT.getStoreSize().getKnownMinValue()));
9789 } else
9790 Increment = DAG.getConstant(DataVT.getStoreSize(), DL, AddrVT);
9791
9792 return DAG.getNode(ISD::ADD, DL, AddrVT, Addr, Increment);
9793}
9794
9796 EVT VecVT, const SDLoc &dl,
9797 ElementCount SubEC) {
9798 assert(!(SubEC.isScalable() && VecVT.isFixedLengthVector()) &&
9799 "Cannot index a scalable vector within a fixed-width vector");
9800
9801 unsigned NElts = VecVT.getVectorMinNumElements();
9802 unsigned NumSubElts = SubEC.getKnownMinValue();
9803 EVT IdxVT = Idx.getValueType();
9804
9805 if (VecVT.isScalableVector() && !SubEC.isScalable()) {
9806 // If this is a constant index and we know the value plus the number of the
9807 // elements in the subvector minus one is less than the minimum number of
9808 // elements then it's safe to return Idx.
9809 if (auto *IdxCst = dyn_cast<ConstantSDNode>(Idx))
9810 if (IdxCst->getZExtValue() + (NumSubElts - 1) < NElts)
9811 return Idx;
9812 SDValue VS =
9813 DAG.getVScale(dl, IdxVT, APInt(IdxVT.getFixedSizeInBits(), NElts));
9814 unsigned SubOpcode = NumSubElts <= NElts ? ISD::SUB : ISD::USUBSAT;
9815 SDValue Sub = DAG.getNode(SubOpcode, dl, IdxVT, VS,
9816 DAG.getConstant(NumSubElts, dl, IdxVT));
9817 return DAG.getNode(ISD::UMIN, dl, IdxVT, Idx, Sub);
9818 }
9819 if (isPowerOf2_32(NElts) && NumSubElts == 1) {
9820 APInt Imm = APInt::getLowBitsSet(IdxVT.getSizeInBits(), Log2_32(NElts));
9821 return DAG.getNode(ISD::AND, dl, IdxVT, Idx,
9822 DAG.getConstant(Imm, dl, IdxVT));
9823 }
9824 unsigned MaxIndex = NumSubElts < NElts ? NElts - NumSubElts : 0;
9825 return DAG.getNode(ISD::UMIN, dl, IdxVT, Idx,
9826 DAG.getConstant(MaxIndex, dl, IdxVT));
9827}
9828
9830 SDValue VecPtr, EVT VecVT,
9831 SDValue Index) const {
9832 return getVectorSubVecPointer(
9833 DAG, VecPtr, VecVT,
9835 Index);
9836}
9837
9839 SDValue VecPtr, EVT VecVT,
9840 EVT SubVecVT,
9841 SDValue Index) const {
9842 SDLoc dl(Index);
9843 // Make sure the index type is big enough to compute in.
9844 Index = DAG.getZExtOrTrunc(Index, dl, VecPtr.getValueType());
9845
9846 EVT EltVT = VecVT.getVectorElementType();
9847
9848 // Calculate the element offset and add it to the pointer.
9849 unsigned EltSize = EltVT.getFixedSizeInBits() / 8; // FIXME: should be ABI size.
9850 assert(EltSize * 8 == EltVT.getFixedSizeInBits() &&
9851 "Converting bits to bytes lost precision");
9852 assert(SubVecVT.getVectorElementType() == EltVT &&
9853 "Sub-vector must be a vector with matching element type");
9854 Index = clampDynamicVectorIndex(DAG, Index, VecVT, dl,
9855 SubVecVT.getVectorElementCount());
9856
9857 EVT IdxVT = Index.getValueType();
9858 if (SubVecVT.isScalableVector())
9859 Index =
9860 DAG.getNode(ISD::MUL, dl, IdxVT, Index,
9861 DAG.getVScale(dl, IdxVT, APInt(IdxVT.getSizeInBits(), 1)));
9862
9863 Index = DAG.getNode(ISD::MUL, dl, IdxVT, Index,
9864 DAG.getConstant(EltSize, dl, IdxVT));
9865 return DAG.getMemBasePlusOffset(VecPtr, Index, dl);
9866}
9867
9868//===----------------------------------------------------------------------===//
9869// Implementation of Emulated TLS Model
9870//===----------------------------------------------------------------------===//
9871
9873 SelectionDAG &DAG) const {
9874 // Access to address of TLS varialbe xyz is lowered to a function call:
9875 // __emutls_get_address( address of global variable named "__emutls_v.xyz" )
9876 EVT PtrVT = getPointerTy(DAG.getDataLayout());
9877 PointerType *VoidPtrType = PointerType::get(*DAG.getContext(), 0);
9878 SDLoc dl(GA);
9879
9880 ArgListTy Args;
9881 ArgListEntry Entry;
9882 std::string NameString = ("__emutls_v." + GA->getGlobal()->getName()).str();
9883 Module *VariableModule = const_cast<Module*>(GA->getGlobal()->getParent());
9884 StringRef EmuTlsVarName(NameString);
9885 GlobalVariable *EmuTlsVar = VariableModule->getNamedGlobal(EmuTlsVarName);
9886 assert(EmuTlsVar && "Cannot find EmuTlsVar ");
9887 Entry.Node = DAG.getGlobalAddress(EmuTlsVar, dl, PtrVT);
9888 Entry.Ty = VoidPtrType;
9889 Args.push_back(Entry);
9890
9891 SDValue EmuTlsGetAddr = DAG.getExternalSymbol("__emutls_get_address", PtrVT);
9892
9894 CLI.setDebugLoc(dl).setChain(DAG.getEntryNode());
9895 CLI.setLibCallee(CallingConv::C, VoidPtrType, EmuTlsGetAddr, std::move(Args));
9896 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
9897
9898 // TLSADDR will be codegen'ed as call. Inform MFI that function has calls.
9899 // At last for X86 targets, maybe good for other targets too?
9901 MFI.setAdjustsStack(true); // Is this only for X86 target?
9902 MFI.setHasCalls(true);
9903
9904 assert((GA->getOffset() == 0) &&
9905 "Emulated TLS must have zero offset in GlobalAddressSDNode");
9906 return CallResult.first;
9907}
9908
9910 SelectionDAG &DAG) const {
9911 assert((Op->getOpcode() == ISD::SETCC) && "Input has to be a SETCC node.");
9912 if (!isCtlzFast())
9913 return SDValue();
9914 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
9915 SDLoc dl(Op);
9916 if (isNullConstant(Op.getOperand(1)) && CC == ISD::SETEQ) {
9917 EVT VT = Op.getOperand(0).getValueType();
9918 SDValue Zext = Op.getOperand(0);
9919 if (VT.bitsLT(MVT::i32)) {
9920 VT = MVT::i32;
9921 Zext = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Op.getOperand(0));
9922 }
9923 unsigned Log2b = Log2_32(VT.getSizeInBits());
9924 SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Zext);
9925 SDValue Scc = DAG.getNode(ISD::SRL, dl, VT, Clz,
9926 DAG.getConstant(Log2b, dl, MVT::i32));
9927 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Scc);
9928 }
9929 return SDValue();
9930}
9931
9933 SDValue Op0 = Node->getOperand(0);
9934 SDValue Op1 = Node->getOperand(1);
9935 EVT VT = Op0.getValueType();
9936 EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
9937 unsigned Opcode = Node->getOpcode();
9938 SDLoc DL(Node);
9939
9940 // umax(x,1) --> sub(x,cmpeq(x,0)) iff cmp result is allbits
9941 if (Opcode == ISD::UMAX && llvm::isOneOrOneSplat(Op1, true) && BoolVT == VT &&
9943 Op0 = DAG.getFreeze(Op0);
9944 SDValue Zero = DAG.getConstant(0, DL, VT);
9945 return DAG.getNode(ISD::SUB, DL, VT, Op0,
9946 DAG.getSetCC(DL, VT, Op0, Zero, ISD::SETEQ));
9947 }
9948
9949 // umin(x,y) -> sub(x,usubsat(x,y))
9950 // TODO: Missing freeze(Op0)?
9951 if (Opcode == ISD::UMIN && isOperationLegal(ISD::SUB, VT) &&
9953 return DAG.getNode(ISD::SUB, DL, VT, Op0,
9954 DAG.getNode(ISD::USUBSAT, DL, VT, Op0, Op1));
9955 }
9956
9957 // umax(x,y) -> add(x,usubsat(y,x))
9958 // TODO: Missing freeze(Op0)?
9959 if (Opcode == ISD::UMAX && isOperationLegal(ISD::ADD, VT) &&
9961 return DAG.getNode(ISD::ADD, DL, VT, Op0,
9962 DAG.getNode(ISD::USUBSAT, DL, VT, Op1, Op0));
9963 }
9964
9965 // FIXME: Should really try to split the vector in case it's legal on a
9966 // subvector.
9968 return DAG.UnrollVectorOp(Node);
9969
9970 // Attempt to find an existing SETCC node that we can reuse.
9971 // TODO: Do we need a generic doesSETCCNodeExist?
9972 // TODO: Missing freeze(Op0)/freeze(Op1)?
9973 auto buildMinMax = [&](ISD::CondCode PrefCC, ISD::CondCode AltCC,
9974 ISD::CondCode PrefCommuteCC,
9975 ISD::CondCode AltCommuteCC) {
9976 SDVTList BoolVTList = DAG.getVTList(BoolVT);
9977 for (ISD::CondCode CC : {PrefCC, AltCC}) {
9978 if (DAG.doesNodeExist(ISD::SETCC, BoolVTList,
9979 {Op0, Op1, DAG.getCondCode(CC)})) {
9980 SDValue Cond = DAG.getSetCC(DL, BoolVT, Op0, Op1, CC);
9981 return DAG.getSelect(DL, VT, Cond, Op0, Op1);
9982 }
9983 }
9984 for (ISD::CondCode CC : {PrefCommuteCC, AltCommuteCC}) {
9985 if (DAG.doesNodeExist(ISD::SETCC, BoolVTList,
9986 {Op0, Op1, DAG.getCondCode(CC)})) {
9987 SDValue Cond = DAG.getSetCC(DL, BoolVT, Op0, Op1, CC);
9988 return DAG.getSelect(DL, VT, Cond, Op1, Op0);
9989 }
9990 }
9991 SDValue Cond = DAG.getSetCC(DL, BoolVT, Op0, Op1, PrefCC);
9992 return DAG.getSelect(DL, VT, Cond, Op0, Op1);
9993 };
9994
9995 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
9996 // -> Y = (A < B) ? B : A
9997 // -> Y = (A >= B) ? A : B
9998 // -> Y = (A <= B) ? B : A
9999 switch (Opcode) {
10000 case ISD::SMAX:
10001 return buildMinMax(ISD::SETGT, ISD::SETGE, ISD::SETLT, ISD::SETLE);
10002 case ISD::SMIN:
10003 return buildMinMax(ISD::SETLT, ISD::SETLE, ISD::SETGT, ISD::SETGE);
10004 case ISD::UMAX:
10005 return buildMinMax(ISD::SETUGT, ISD::SETUGE, ISD::SETULT, ISD::SETULE);
10006 case ISD::UMIN:
10007 return buildMinMax(ISD::SETULT, ISD::SETULE, ISD::SETUGT, ISD::SETUGE);
10008 }
10009
10010 llvm_unreachable("How did we get here?");
10011}
10012
10014 unsigned Opcode = Node->getOpcode();
10015 SDValue LHS = Node->getOperand(0);
10016 SDValue RHS = Node->getOperand(1);
10017 EVT VT = LHS.getValueType();
10018 SDLoc dl(Node);
10019
10020 assert(VT == RHS.getValueType() && "Expected operands to be the same type");
10021 assert(VT.isInteger() && "Expected operands to be integers");
10022
10023 // usub.sat(a, b) -> umax(a, b) - b
10024 if (Opcode == ISD::USUBSAT && isOperationLegal(ISD::UMAX, VT)) {
10025 SDValue Max = DAG.getNode(ISD::UMAX, dl, VT, LHS, RHS);
10026 return DAG.getNode(ISD::SUB, dl, VT, Max, RHS);
10027 }
10028
10029 // uadd.sat(a, b) -> umin(a, ~b) + b
10030 if (Opcode == ISD::UADDSAT && isOperationLegal(ISD::UMIN, VT)) {
10031 SDValue InvRHS = DAG.getNOT(dl, RHS, VT);
10032 SDValue Min = DAG.getNode(ISD::UMIN, dl, VT, LHS, InvRHS);
10033 return DAG.getNode(ISD::ADD, dl, VT, Min, RHS);
10034 }
10035
10036 unsigned OverflowOp;
10037 switch (Opcode) {
10038 case ISD::SADDSAT:
10039 OverflowOp = ISD::SADDO;
10040 break;
10041 case ISD::UADDSAT:
10042 OverflowOp = ISD::UADDO;
10043 break;
10044 case ISD::SSUBSAT:
10045 OverflowOp = ISD::SSUBO;
10046 break;
10047 case ISD::USUBSAT:
10048 OverflowOp = ISD::USUBO;
10049 break;
10050 default:
10051 llvm_unreachable("Expected method to receive signed or unsigned saturation "
10052 "addition or subtraction node.");
10053 }
10054
10055 // FIXME: Should really try to split the vector in case it's legal on a
10056 // subvector.
10058 return DAG.UnrollVectorOp(Node);
10059
10060 unsigned BitWidth = LHS.getScalarValueSizeInBits();
10061 EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
10062 SDValue Result = DAG.getNode(OverflowOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
10063 SDValue SumDiff = Result.getValue(0);
10064 SDValue Overflow = Result.getValue(1);
10065 SDValue Zero = DAG.getConstant(0, dl, VT);
10066 SDValue AllOnes = DAG.getAllOnesConstant(dl, VT);
10067
10068 if (Opcode == ISD::UADDSAT) {
10070 // (LHS + RHS) | OverflowMask
10071 SDValue OverflowMask = DAG.getSExtOrTrunc(Overflow, dl, VT);
10072 return DAG.getNode(ISD::OR, dl, VT, SumDiff, OverflowMask);
10073 }
10074 // Overflow ? 0xffff.... : (LHS + RHS)
10075 return DAG.getSelect(dl, VT, Overflow, AllOnes, SumDiff);
10076 }
10077
10078 if (Opcode == ISD::USUBSAT) {
10080 // (LHS - RHS) & ~OverflowMask
10081 SDValue OverflowMask = DAG.getSExtOrTrunc(Overflow, dl, VT);
10082 SDValue Not = DAG.getNOT(dl, OverflowMask, VT);
10083 return DAG.getNode(ISD::AND, dl, VT, SumDiff, Not);
10084 }
10085 // Overflow ? 0 : (LHS - RHS)
10086 return DAG.getSelect(dl, VT, Overflow, Zero, SumDiff);
10087 }
10088
10089 if (Opcode == ISD::SADDSAT || Opcode == ISD::SSUBSAT) {
10092
10093 KnownBits KnownLHS = DAG.computeKnownBits(LHS);
10094 KnownBits KnownRHS = DAG.computeKnownBits(RHS);
10095
10096 // If either of the operand signs are known, then they are guaranteed to
10097 // only saturate in one direction. If non-negative they will saturate
10098 // towards SIGNED_MAX, if negative they will saturate towards SIGNED_MIN.
10099 //
10100 // In the case of ISD::SSUBSAT, 'x - y' is equivalent to 'x + (-y)', so the
10101 // sign of 'y' has to be flipped.
10102
10103 bool LHSIsNonNegative = KnownLHS.isNonNegative();
10104 bool RHSIsNonNegative = Opcode == ISD::SADDSAT ? KnownRHS.isNonNegative()
10105 : KnownRHS.isNegative();
10106 if (LHSIsNonNegative || RHSIsNonNegative) {
10107 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
10108 return DAG.getSelect(dl, VT, Overflow, SatMax, SumDiff);
10109 }
10110
10111 bool LHSIsNegative = KnownLHS.isNegative();
10112 bool RHSIsNegative = Opcode == ISD::SADDSAT ? KnownRHS.isNegative()
10113 : KnownRHS.isNonNegative();
10114 if (LHSIsNegative || RHSIsNegative) {
10115 SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
10116 return DAG.getSelect(dl, VT, Overflow, SatMin, SumDiff);
10117 }
10118 }
10119
10120 // Overflow ? (SumDiff >> BW) ^ MinVal : SumDiff
10122 SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
10123 SDValue Shift = DAG.getNode(ISD::SRA, dl, VT, SumDiff,
10124 DAG.getConstant(BitWidth - 1, dl, VT));
10125 Result = DAG.getNode(ISD::XOR, dl, VT, Shift, SatMin);
10126 return DAG.getSelect(dl, VT, Overflow, Result, SumDiff);
10127}
10128
10130 unsigned Opcode = Node->getOpcode();
10131 bool IsSigned = Opcode == ISD::SSHLSAT;
10132 SDValue LHS = Node->getOperand(0);
10133 SDValue RHS = Node->getOperand(1);
10134 EVT VT = LHS.getValueType();
10135 SDLoc dl(Node);
10136
10137 assert((Node->getOpcode() == ISD::SSHLSAT ||
10138 Node->getOpcode() == ISD::USHLSAT) &&
10139 "Expected a SHLSAT opcode");
10140 assert(VT == RHS.getValueType() && "Expected operands to be the same type");
10141 assert(VT.isInteger() && "Expected operands to be integers");
10142
10144 return DAG.UnrollVectorOp(Node);
10145
10146 // If LHS != (LHS << RHS) >> RHS, we have overflow and must saturate.
10147
10148 unsigned BW = VT.getScalarSizeInBits();
10149 EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
10150 SDValue Result = DAG.getNode(ISD::SHL, dl, VT, LHS, RHS);
10151 SDValue Orig =
10152 DAG.getNode(IsSigned ? ISD::SRA : ISD::SRL, dl, VT, Result, RHS);
10153
10154 SDValue SatVal;
10155 if (IsSigned) {
10156 SDValue SatMin = DAG.getConstant(APInt::getSignedMinValue(BW), dl, VT);
10157 SDValue SatMax = DAG.getConstant(APInt::getSignedMaxValue(BW), dl, VT);
10158 SDValue Cond =
10159 DAG.getSetCC(dl, BoolVT, LHS, DAG.getConstant(0, dl, VT), ISD::SETLT);
10160 SatVal = DAG.getSelect(dl, VT, Cond, SatMin, SatMax);
10161 } else {
10162 SatVal = DAG.getConstant(APInt::getMaxValue(BW), dl, VT);
10163 }
10164 SDValue Cond = DAG.getSetCC(dl, BoolVT, LHS, Orig, ISD::SETNE);
10165 return DAG.getSelect(dl, VT, Cond, SatVal, Result);
10166}
10167
10169 bool Signed, EVT WideVT,
10170 const SDValue LL, const SDValue LH,
10171 const SDValue RL, const SDValue RH,
10172 SDValue &Lo, SDValue &Hi) const {
10173 // We can fall back to a libcall with an illegal type for the MUL if we
10174 // have a libcall big enough.
10175 // Also, we can fall back to a division in some cases, but that's a big
10176 // performance hit in the general case.
10177 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
10178 if (WideVT == MVT::i16)
10179 LC = RTLIB::MUL_I16;
10180 else if (WideVT == MVT::i32)
10181 LC = RTLIB::MUL_I32;
10182 else if (WideVT == MVT::i64)
10183 LC = RTLIB::MUL_I64;
10184 else if (WideVT == MVT::i128)
10185 LC = RTLIB::MUL_I128;
10186
10187 if (LC == RTLIB::UNKNOWN_LIBCALL || !getLibcallName(LC)) {
10188 // We'll expand the multiplication by brute force because we have no other
10189 // options. This is a trivially-generalized version of the code from
10190 // Hacker's Delight (itself derived from Knuth's Algorithm M from section
10191 // 4.3.1).
10192 EVT VT = LL.getValueType();
10193 unsigned Bits = VT.getSizeInBits();
10194 unsigned HalfBits = Bits >> 1;
10195 SDValue Mask =
10196 DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl, VT);
10197 SDValue LLL = DAG.getNode(ISD::AND, dl, VT, LL, Mask);
10198 SDValue RLL = DAG.getNode(ISD::AND, dl, VT, RL, Mask);
10199
10200 SDValue T = DAG.getNode(ISD::MUL, dl, VT, LLL, RLL);
10201 SDValue TL = DAG.getNode(ISD::AND, dl, VT, T, Mask);
10202
10203 SDValue Shift = DAG.getShiftAmountConstant(HalfBits, VT, dl);
10204 SDValue TH = DAG.getNode(ISD::SRL, dl, VT, T, Shift);
10205 SDValue LLH = DAG.getNode(ISD::SRL, dl, VT, LL, Shift);
10206 SDValue RLH = DAG.getNode(ISD::SRL, dl, VT, RL, Shift);
10207
10208 SDValue U = DAG.getNode(ISD::ADD, dl, VT,
10209 DAG.getNode(ISD::MUL, dl, VT, LLH, RLL), TH);
10210 SDValue UL = DAG.getNode(ISD::AND, dl, VT, U, Mask);
10211 SDValue UH = DAG.getNode(ISD::SRL, dl, VT, U, Shift);
10212
10213 SDValue V = DAG.getNode(ISD::ADD, dl, VT,
10214 DAG.getNode(ISD::MUL, dl, VT, LLL, RLH), UL);
10215 SDValue VH = DAG.getNode(ISD::SRL, dl, VT, V, Shift);
10216
10217 SDValue W =
10218 DAG.getNode(ISD::ADD, dl, VT, DAG.getNode(ISD::MUL, dl, VT, LLH, RLH),
10219 DAG.getNode(ISD::ADD, dl, VT, UH, VH));
10220 Lo = DAG.getNode(ISD::ADD, dl, VT, TL,
10221 DAG.getNode(ISD::SHL, dl, VT, V, Shift));
10222
10223 Hi = DAG.getNode(ISD::ADD, dl, VT, W,
10224 DAG.getNode(ISD::ADD, dl, VT,
10225 DAG.getNode(ISD::MUL, dl, VT, RH, LL),
10226 DAG.getNode(ISD::MUL, dl, VT, RL, LH)));
10227 } else {
10228 // Attempt a libcall.
10229 SDValue Ret;
10231 CallOptions.setSExt(Signed);
10232 CallOptions.setIsPostTypeLegalization(true);
10233 if (shouldSplitFunctionArgumentsAsLittleEndian(DAG.getDataLayout())) {
10234 // Halves of WideVT are packed into registers in different order
10235 // depending on platform endianness. This is usually handled by
10236 // the C calling convention, but we can't defer to it in
10237 // the legalizer.
10238 SDValue Args[] = {LL, LH, RL, RH};
10239 Ret = makeLibCall(DAG, LC, WideVT, Args, CallOptions, dl).first;
10240 } else {
10241 SDValue Args[] = {LH, LL, RH, RL};
10242 Ret = makeLibCall(DAG, LC, WideVT, Args, CallOptions, dl).first;
10243 }
10244 assert(Ret.getOpcode() == ISD::MERGE_VALUES &&
10245 "Ret value is a collection of constituent nodes holding result.");
10246 if (DAG.getDataLayout().isLittleEndian()) {
10247 // Same as above.
10248 Lo = Ret.getOperand(0);
10249 Hi = Ret.getOperand(1);
10250 } else {
10251 Lo = Ret.getOperand(1);
10252 Hi = Ret.getOperand(0);
10253 }
10254 }
10255}
10256
10258 bool Signed, const SDValue LHS,
10259 const SDValue RHS, SDValue &Lo,
10260 SDValue &Hi) const {
10261 EVT VT = LHS.getValueType();
10262 assert(RHS.getValueType() == VT && "Mismatching operand types");
10263
10264 SDValue HiLHS;
10265 SDValue HiRHS;
10266 if (Signed) {
10267 // The high part is obtained by SRA'ing all but one of the bits of low
10268 // part.
10269 unsigned LoSize = VT.getFixedSizeInBits();
10270 HiLHS = DAG.getNode(
10271 ISD::SRA, dl, VT, LHS,
10272 DAG.getConstant(LoSize - 1, dl, getPointerTy(DAG.getDataLayout())));
10273 HiRHS = DAG.getNode(
10274 ISD::SRA, dl, VT, RHS,
10275 DAG.getConstant(LoSize - 1, dl, getPointerTy(DAG.getDataLayout())));
10276 } else {
10277 HiLHS = DAG.getConstant(0, dl, VT);
10278 HiRHS = DAG.getConstant(0, dl, VT);
10279 }
10280 EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits() * 2);
10281 forceExpandWideMUL(DAG, dl, Signed, WideVT, LHS, HiLHS, RHS, HiRHS, Lo, Hi);
10282}
10283
10284SDValue
10286 assert((Node->getOpcode() == ISD::SMULFIX ||
10287 Node->getOpcode() == ISD::UMULFIX ||
10288 Node->getOpcode() == ISD::SMULFIXSAT ||
10289 Node->getOpcode() == ISD::UMULFIXSAT) &&
10290 "Expected a fixed point multiplication opcode");
10291
10292 SDLoc dl(Node);
10293 SDValue LHS = Node->getOperand(0);
10294 SDValue RHS = Node->getOperand(1);
10295 EVT VT = LHS.getValueType();
10296 unsigned Scale = Node->getConstantOperandVal(2);
10297 bool Saturating = (Node->getOpcode() == ISD::SMULFIXSAT ||
10298 Node->getOpcode() == ISD::UMULFIXSAT);
10299 bool Signed = (Node->getOpcode() == ISD::SMULFIX ||
10300 Node->getOpcode() == ISD::SMULFIXSAT);
10301 EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
10302 unsigned VTSize = VT.getScalarSizeInBits();
10303
10304 if (!Scale) {
10305 // [us]mul.fix(a, b, 0) -> mul(a, b)
10306 if (!Saturating) {
10308 return DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
10309 } else if (Signed && isOperationLegalOrCustom(ISD::SMULO, VT)) {
10310 SDValue Result =
10311 DAG.getNode(ISD::SMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
10312 SDValue Product = Result.getValue(0);
10313 SDValue Overflow = Result.getValue(1);
10314 SDValue Zero = DAG.getConstant(0, dl, VT);
10315
10316 APInt MinVal = APInt::getSignedMinValue(VTSize);
10317 APInt MaxVal = APInt::getSignedMaxValue(VTSize);
10318 SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
10319 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
10320 // Xor the inputs, if resulting sign bit is 0 the product will be
10321 // positive, else negative.
10322 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS);
10323 SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Xor, Zero, ISD::SETLT);
10324 Result = DAG.getSelect(dl, VT, ProdNeg, SatMin, SatMax);
10325 return DAG.getSelect(dl, VT, Overflow, Result, Product);
10326 } else if (!Signed && isOperationLegalOrCustom(ISD::UMULO, VT)) {
10327 SDValue Result =
10328 DAG.getNode(ISD::UMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
10329 SDValue Product = Result.getValue(0);
10330 SDValue Overflow = Result.getValue(1);
10331
10332 APInt MaxVal = APInt::getMaxValue(VTSize);
10333 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
10334 return DAG.getSelect(dl, VT, Overflow, SatMax, Product);
10335 }
10336 }
10337
10338 assert(((Signed && Scale < VTSize) || (!Signed && Scale <= VTSize)) &&
10339 "Expected scale to be less than the number of bits if signed or at "
10340 "most the number of bits if unsigned.");
10341 assert(LHS.getValueType() == RHS.getValueType() &&
10342 "Expected both operands to be the same type");
10343
10344 // Get the upper and lower bits of the result.
10345 SDValue Lo, Hi;
10346 unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
10347 unsigned HiOp = Signed ? ISD::MULHS : ISD::MULHU;
10348 if (isOperationLegalOrCustom(LoHiOp, VT)) {
10349 SDValue Result = DAG.getNode(LoHiOp, dl, DAG.getVTList(VT, VT), LHS, RHS);
10350 Lo = Result.getValue(0);
10351 Hi = Result.getValue(1);
10352 } else if (isOperationLegalOrCustom(HiOp, VT)) {
10353 Lo = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
10354 Hi = DAG.getNode(HiOp, dl, VT, LHS, RHS);
10355 } else if (VT.isVector()) {
10356 return SDValue();
10357 } else {
10358 forceExpandWideMUL(DAG, dl, Signed, LHS, RHS, Lo, Hi);
10359 }
10360
10361 if (Scale == VTSize)
10362 // Result is just the top half since we'd be shifting by the width of the
10363 // operand. Overflow impossible so this works for both UMULFIX and
10364 // UMULFIXSAT.
10365 return Hi;
10366
10367 // The result will need to be shifted right by the scale since both operands
10368 // are scaled. The result is given to us in 2 halves, so we only want part of
10369 // both in the result.
10370 SDValue Result = DAG.getNode(ISD::FSHR, dl, VT, Hi, Lo,
10371 DAG.getShiftAmountConstant(Scale, VT, dl));
10372 if (!Saturating)
10373 return Result;
10374
10375 if (!Signed) {
10376 // Unsigned overflow happened if the upper (VTSize - Scale) bits (of the
10377 // widened multiplication) aren't all zeroes.
10378
10379 // Saturate to max if ((Hi >> Scale) != 0),
10380 // which is the same as if (Hi > ((1 << Scale) - 1))
10381 APInt MaxVal = APInt::getMaxValue(VTSize);
10382 SDValue LowMask = DAG.getConstant(APInt::getLowBitsSet(VTSize, Scale),
10383 dl, VT);
10384 Result = DAG.getSelectCC(dl, Hi, LowMask,
10385 DAG.getConstant(MaxVal, dl, VT), Result,
10386 ISD::SETUGT);
10387
10388 return Result;
10389 }
10390
10391 // Signed overflow happened if the upper (VTSize - Scale + 1) bits (of the
10392 // widened multiplication) aren't all ones or all zeroes.
10393
10394 SDValue SatMin = DAG.getConstant(APInt::getSignedMinValue(VTSize), dl, VT);
10395 SDValue SatMax = DAG.getConstant(APInt::getSignedMaxValue(VTSize), dl, VT);
10396
10397 if (Scale == 0) {
10398 SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, Lo,
10399 DAG.getShiftAmountConstant(VTSize - 1, VT, dl));
10400 SDValue Overflow = DAG.getSetCC(dl, BoolVT, Hi, Sign, ISD::SETNE);
10401 // Saturated to SatMin if wide product is negative, and SatMax if wide
10402 // product is positive ...
10403 SDValue Zero = DAG.getConstant(0, dl, VT);
10404 SDValue ResultIfOverflow = DAG.getSelectCC(dl, Hi, Zero, SatMin, SatMax,
10405 ISD::SETLT);
10406 // ... but only if we overflowed.
10407 return DAG.getSelect(dl, VT, Overflow, ResultIfOverflow, Result);
10408 }
10409
10410 // We handled Scale==0 above so all the bits to examine is in Hi.
10411
10412 // Saturate to max if ((Hi >> (Scale - 1)) > 0),
10413 // which is the same as if (Hi > (1 << (Scale - 1)) - 1)
10414 SDValue LowMask = DAG.getConstant(APInt::getLowBitsSet(VTSize, Scale - 1),
10415 dl, VT);
10416 Result = DAG.getSelectCC(dl, Hi, LowMask, SatMax, Result, ISD::SETGT);
10417 // Saturate to min if (Hi >> (Scale - 1)) < -1),
10418 // which is the same as if (HI < (-1 << (Scale - 1))
10419 SDValue HighMask =
10420 DAG.getConstant(APInt::getHighBitsSet(VTSize, VTSize - Scale + 1),
10421 dl, VT);
10422 Result = DAG.getSelectCC(dl, Hi, HighMask, SatMin, Result, ISD::SETLT);
10423 return Result;
10424}
10425
10426SDValue
10428 SDValue LHS, SDValue RHS,
10429 unsigned Scale, SelectionDAG &DAG) const {
10430 assert((Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT ||
10431 Opcode == ISD::UDIVFIX || Opcode == ISD::UDIVFIXSAT) &&
10432 "Expected a fixed point division opcode");
10433
10434 EVT VT = LHS.getValueType();
10435 bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;
10436 bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;
10437 EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
10438
10439 // If there is enough room in the type to upscale the LHS or downscale the
10440 // RHS before the division, we can perform it in this type without having to
10441 // resize. For signed operations, the LHS headroom is the number of
10442 // redundant sign bits, and for unsigned ones it is the number of zeroes.
10443 // The headroom for the RHS is the number of trailing zeroes.
10444 unsigned LHSLead = Signed ? DAG.ComputeNumSignBits(LHS) - 1
10446 unsigned RHSTrail = DAG.computeKnownBits(RHS).countMinTrailingZeros();
10447
10448 // For signed saturating operations, we need to be able to detect true integer
10449 // division overflow; that is, when you have MIN / -EPS. However, this
10450 // is undefined behavior and if we emit divisions that could take such
10451 // values it may cause undesired behavior (arithmetic exceptions on x86, for
10452 // example).
10453 // Avoid this by requiring an extra bit so that we never get this case.
10454 // FIXME: This is a bit unfortunate as it means that for an 8-bit 7-scale
10455 // signed saturating division, we need to emit a whopping 32-bit division.
10456 if (LHSLead + RHSTrail < Scale + (unsigned)(Saturating && Signed))
10457 return SDValue();
10458
10459 unsigned LHSShift = std::min(LHSLead, Scale);
10460 unsigned RHSShift = Scale - LHSShift;
10461
10462 // At this point, we know that if we shift the LHS up by LHSShift and the
10463 // RHS down by RHSShift, we can emit a regular division with a final scaling
10464 // factor of Scale.
10465
10466 if (LHSShift)
10467 LHS = DAG.getNode(ISD::SHL, dl, VT, LHS,
10468 DAG.getShiftAmountConstant(LHSShift, VT, dl));
10469 if (RHSShift)
10470 RHS = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, VT, RHS,
10471 DAG.getShiftAmountConstant(RHSShift, VT, dl));
10472
10473 SDValue Quot;
10474 if (Signed) {
10475 // For signed operations, if the resulting quotient is negative and the
10476 // remainder is nonzero, subtract 1 from the quotient to round towards
10477 // negative infinity.
10478 SDValue Rem;
10479 // FIXME: Ideally we would always produce an SDIVREM here, but if the
10480 // type isn't legal, SDIVREM cannot be expanded. There is no reason why
10481 // we couldn't just form a libcall, but the type legalizer doesn't do it.
10482 if (isTypeLegal(VT) &&
10484 Quot = DAG.getNode(ISD::SDIVREM, dl,
10485 DAG.getVTList(VT, VT),
10486 LHS, RHS);
10487 Rem = Quot.getValue(1);
10488 Quot = Quot.getValue(0);
10489 } else {
10490 Quot = DAG.getNode(ISD::SDIV, dl, VT,
10491 LHS, RHS);
10492 Rem = DAG.getNode(ISD::SREM, dl, VT,
10493 LHS, RHS);
10494 }
10495 SDValue Zero = DAG.getConstant(0, dl, VT);
10496 SDValue RemNonZero = DAG.getSetCC(dl, BoolVT, Rem, Zero, ISD::SETNE);
10497 SDValue LHSNeg = DAG.getSetCC(dl, BoolVT, LHS, Zero, ISD::SETLT);
10498 SDValue RHSNeg = DAG.getSetCC(dl, BoolVT, RHS, Zero, ISD::SETLT);
10499 SDValue QuotNeg = DAG.getNode(ISD::XOR, dl, BoolVT, LHSNeg, RHSNeg);
10500 SDValue Sub1 = DAG.getNode(ISD::SUB, dl, VT, Quot,
10501 DAG.getConstant(1, dl, VT));
10502 Quot = DAG.getSelect(dl, VT,
10503 DAG.getNode(ISD::AND, dl, BoolVT, RemNonZero, QuotNeg),
10504 Sub1, Quot);
10505 } else
10506 Quot = DAG.getNode(ISD::UDIV, dl, VT,
10507 LHS, RHS);
10508
10509 return Quot;
10510}
10511
10513 SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const {
10514 SDLoc dl(Node);
10515 SDValue LHS = Node->getOperand(0);
10516 SDValue RHS = Node->getOperand(1);
10517 bool IsAdd = Node->getOpcode() == ISD::UADDO;
10518
10519 // If UADDO_CARRY/SUBO_CARRY is legal, use that instead.
10520 unsigned OpcCarry = IsAdd ? ISD::UADDO_CARRY : ISD::USUBO_CARRY;
10521 if (isOperationLegalOrCustom(OpcCarry, Node->getValueType(0))) {
10522 SDValue CarryIn = DAG.getConstant(0, dl, Node->getValueType(1));
10523 SDValue NodeCarry = DAG.getNode(OpcCarry, dl, Node->getVTList(),
10524 { LHS, RHS, CarryIn });
10525 Result = SDValue(NodeCarry.getNode(), 0);
10526 Overflow = SDValue(NodeCarry.getNode(), 1);
10527 return;
10528 }
10529
10530 Result = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl,
10531 LHS.getValueType(), LHS, RHS);
10532
10533 EVT ResultType = Node->getValueType(1);
10534 EVT SetCCType = getSetCCResultType(
10535 DAG.getDataLayout(), *DAG.getContext(), Node->getValueType(0));
10536 SDValue SetCC;
10537 if (IsAdd && isOneConstant(RHS)) {
10538 // Special case: uaddo X, 1 overflowed if X+1 is 0. This potential reduces
10539 // the live range of X. We assume comparing with 0 is cheap.
10540 // The general case (X + C) < C is not necessarily beneficial. Although we
10541 // reduce the live range of X, we may introduce the materialization of
10542 // constant C.
10543 SetCC =
10544 DAG.getSetCC(dl, SetCCType, Result,
10545 DAG.getConstant(0, dl, Node->getValueType(0)), ISD::SETEQ);
10546 } else if (IsAdd && isAllOnesConstant(RHS)) {
10547 // Special case: uaddo X, -1 overflows if X != 0.
10548 SetCC =
10549 DAG.getSetCC(dl, SetCCType, LHS,
10550 DAG.getConstant(0, dl, Node->getValueType(0)), ISD::SETNE);
10551 } else {
10553 SetCC = DAG.getSetCC(dl, SetCCType, Result, LHS, CC);
10554 }
10555 Overflow = DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType);
10556}
10557
10559 SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const {
10560 SDLoc dl(Node);
10561 SDValue LHS = Node->getOperand(0);
10562 SDValue RHS = Node->getOperand(1);
10563 bool IsAdd = Node->getOpcode() == ISD::SADDO;
10564
10565 Result = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl,
10566 LHS.getValueType(), LHS, RHS);
10567
10568 EVT ResultType = Node->getValueType(1);
10569 EVT OType = getSetCCResultType(
10570 DAG.getDataLayout(), *DAG.getContext(), Node->getValueType(0));
10571
10572 // If SADDSAT/SSUBSAT is legal, compare results to detect overflow.
10573 unsigned OpcSat = IsAdd ? ISD::SADDSAT : ISD::SSUBSAT;
10574 if (isOperationLegal(OpcSat, LHS.getValueType())) {
10575 SDValue Sat = DAG.getNode(OpcSat, dl, LHS.getValueType(), LHS, RHS);
10576 SDValue SetCC = DAG.getSetCC(dl, OType, Result, Sat, ISD::SETNE);
10577 Overflow = DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType);
10578 return;
10579 }
10580
10581 SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
10582
10583 // For an addition, the result should be less than one of the operands (LHS)
10584 // if and only if the other operand (RHS) is negative, otherwise there will
10585 // be overflow.
10586 // For a subtraction, the result should be less than one of the operands
10587 // (LHS) if and only if the other operand (RHS) is (non-zero) positive,
10588 // otherwise there will be overflow.
10589 SDValue ResultLowerThanLHS = DAG.getSetCC(dl, OType, Result, LHS, ISD::SETLT);
10590 SDValue ConditionRHS =
10591 DAG.getSetCC(dl, OType, RHS, Zero, IsAdd ? ISD::SETLT : ISD::SETGT);
10592
10593 Overflow = DAG.getBoolExtOrTrunc(
10594 DAG.getNode(ISD::XOR, dl, OType, ConditionRHS, ResultLowerThanLHS), dl,
10595 ResultType, ResultType);
10596}
10597
10599 SDValue &Overflow, SelectionDAG &DAG) const {
10600 SDLoc dl(Node);
10601 EVT VT = Node->getValueType(0);
10602 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
10603 SDValue LHS = Node->getOperand(0);
10604 SDValue RHS = Node->getOperand(1);
10605 bool isSigned = Node->getOpcode() == ISD::SMULO;
10606
10607 // For power-of-two multiplications we can use a simpler shift expansion.
10608 if (ConstantSDNode *RHSC = isConstOrConstSplat(RHS)) {
10609 const APInt &C = RHSC->getAPIntValue();
10610 // mulo(X, 1 << S) -> { X << S, (X << S) >> S != X }
10611 if (C.isPowerOf2()) {
10612 // smulo(x, signed_min) is same as umulo(x, signed_min).
10613 bool UseArithShift = isSigned && !C.isMinSignedValue();
10614 SDValue ShiftAmt = DAG.getShiftAmountConstant(C.logBase2(), VT, dl);
10615 Result = DAG.getNode(ISD::SHL, dl, VT, LHS, ShiftAmt);
10616 Overflow = DAG.getSetCC(dl, SetCCVT,
10617 DAG.getNode(UseArithShift ? ISD::SRA : ISD::SRL,
10618 dl, VT, Result, ShiftAmt),
10619 LHS, ISD::SETNE);
10620 return true;
10621 }
10622 }
10623
10624 EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getScalarSizeInBits() * 2);
10625 if (VT.isVector())
10626 WideVT =
10628
10629 SDValue BottomHalf;
10630 SDValue TopHalf;
10631 static const unsigned Ops[2][3] =
10634 if (isOperationLegalOrCustom(Ops[isSigned][0], VT)) {
10635 BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
10636 TopHalf = DAG.getNode(Ops[isSigned][0], dl, VT, LHS, RHS);
10637 } else if (isOperationLegalOrCustom(Ops[isSigned][1], VT)) {
10638 BottomHalf = DAG.getNode(Ops[isSigned][1], dl, DAG.getVTList(VT, VT), LHS,
10639 RHS);
10640 TopHalf = BottomHalf.getValue(1);
10641 } else if (isTypeLegal(WideVT)) {
10642 LHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, LHS);
10643 RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS);
10644 SDValue Mul = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS);
10645 BottomHalf = DAG.getNode(ISD::TRUNCATE, dl, VT, Mul);
10646 SDValue ShiftAmt =
10647 DAG.getShiftAmountConstant(VT.getScalarSizeInBits(), WideVT, dl);
10648 TopHalf = DAG.getNode(ISD::TRUNCATE, dl, VT,
10649 DAG.getNode(ISD::SRL, dl, WideVT, Mul, ShiftAmt));
10650 } else {
10651 if (VT.isVector())
10652 return false;
10653
10654 forceExpandWideMUL(DAG, dl, isSigned, LHS, RHS, BottomHalf, TopHalf);
10655 }
10656
10657 Result = BottomHalf;
10658 if (isSigned) {
10659 SDValue ShiftAmt = DAG.getShiftAmountConstant(
10660 VT.getScalarSizeInBits() - 1, BottomHalf.getValueType(), dl);
10661 SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, ShiftAmt);
10662 Overflow = DAG.getSetCC(dl, SetCCVT, TopHalf, Sign, ISD::SETNE);
10663 } else {
10664 Overflow = DAG.getSetCC(dl, SetCCVT, TopHalf,
10665 DAG.getConstant(0, dl, VT), ISD::SETNE);
10666 }
10667
10668 // Truncate the result if SetCC returns a larger type than needed.
10669 EVT RType = Node->getValueType(1);
10670 if (RType.bitsLT(Overflow.getValueType()))
10671 Overflow = DAG.getNode(ISD::TRUNCATE, dl, RType, Overflow);
10672
10673 assert(RType.getSizeInBits() == Overflow.getValueSizeInBits() &&
10674 "Unexpected result type for S/UMULO legalization");
10675 return true;
10676}
10677
10679 SDLoc dl(Node);
10680 unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(Node->getOpcode());
10681 SDValue Op = Node->getOperand(0);
10682 EVT VT = Op.getValueType();
10683
10684 if (VT.isScalableVector())
10686 "Expanding reductions for scalable vectors is undefined.");
10687
10688 // Try to use a shuffle reduction for power of two vectors.
10689 if (VT.isPow2VectorType()) {
10690 while (VT.getVectorNumElements() > 1) {
10691 EVT HalfVT = VT.getHalfNumVectorElementsVT(*DAG.getContext());
10692 if (!isOperationLegalOrCustom(BaseOpcode, HalfVT))
10693 break;
10694
10695 SDValue Lo, Hi;
10696 std::tie(Lo, Hi) = DAG.SplitVector(Op, dl);
10697 Op = DAG.getNode(BaseOpcode, dl, HalfVT, Lo, Hi, Node->getFlags());
10698 VT = HalfVT;
10699 }
10700 }
10701
10702 EVT EltVT = VT.getVectorElementType();
10703 unsigned NumElts = VT.getVectorNumElements();
10704
10706 DAG.ExtractVectorElements(Op, Ops, 0, NumElts);
10707
10708 SDValue Res = Ops[0];
10709 for (unsigned i = 1; i < NumElts; i++)
10710 Res = DAG.getNode(BaseOpcode, dl, EltVT, Res, Ops[i], Node->getFlags());
10711
10712 // Result type may be wider than element type.
10713 if (EltVT != Node->getValueType(0))
10714 Res = DAG.getNode(ISD::ANY_EXTEND, dl, Node->getValueType(0), Res);
10715 return Res;
10716}
10717
10719 SDLoc dl(Node);
10720 SDValue AccOp = Node->getOperand(0);
10721 SDValue VecOp = Node->getOperand(1);
10722 SDNodeFlags Flags = Node->getFlags();
10723
10724 EVT VT = VecOp.getValueType();
10725 EVT EltVT = VT.getVectorElementType();
10726
10727 if (VT.isScalableVector())
10729 "Expanding reductions for scalable vectors is undefined.");
10730
10731 unsigned NumElts = VT.getVectorNumElements();
10732
10734 DAG.ExtractVectorElements(VecOp, Ops, 0, NumElts);
10735
10736 unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(Node->getOpcode());
10737
10738 SDValue Res = AccOp;
10739 for (unsigned i = 0; i < NumElts; i++)
10740 Res = DAG.getNode(BaseOpcode, dl, EltVT, Res, Ops[i], Flags);
10741
10742 return Res;
10743}
10744
10746 SelectionDAG &DAG) const {
10747 EVT VT = Node->getValueType(0);
10748 SDLoc dl(Node);
10749 bool isSigned = Node->getOpcode() == ISD::SREM;
10750 unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV;
10751 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
10752 SDValue Dividend = Node->getOperand(0);
10753 SDValue Divisor = Node->getOperand(1);
10754 if (isOperationLegalOrCustom(DivRemOpc, VT)) {
10755 SDVTList VTs = DAG.getVTList(VT, VT);
10756 Result = DAG.getNode(DivRemOpc, dl, VTs, Dividend, Divisor).getValue(1);
10757 return true;
10758 }
10759 if (isOperationLegalOrCustom(DivOpc, VT)) {
10760 // X % Y -> X-X/Y*Y
10761 SDValue Divide = DAG.getNode(DivOpc, dl, VT, Dividend, Divisor);
10762 SDValue Mul = DAG.getNode(ISD::MUL, dl, VT, Divide, Divisor);
10763 Result = DAG.getNode(ISD::SUB, dl, VT, Dividend, Mul);
10764 return true;
10765 }
10766 return false;
10767}
10768
10770 SelectionDAG &DAG) const {
10771 bool IsSigned = Node->getOpcode() == ISD::FP_TO_SINT_SAT;
10772 SDLoc dl(SDValue(Node, 0));
10773 SDValue Src = Node->getOperand(0);
10774
10775 // DstVT is the result type, while SatVT is the size to which we saturate
10776 EVT SrcVT = Src.getValueType();
10777 EVT DstVT = Node->getValueType(0);
10778
10779 EVT SatVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
10780 unsigned SatWidth = SatVT.getScalarSizeInBits();
10781 unsigned DstWidth = DstVT.getScalarSizeInBits();
10782 assert(SatWidth <= DstWidth &&
10783 "Expected saturation width smaller than result width");
10784
10785 // Determine minimum and maximum integer values and their corresponding
10786 // floating-point values.
10787 APInt MinInt, MaxInt;
10788 if (IsSigned) {
10789 MinInt = APInt::getSignedMinValue(SatWidth).sext(DstWidth);
10790 MaxInt = APInt::getSignedMaxValue(SatWidth).sext(DstWidth);
10791 } else {
10792 MinInt = APInt::getMinValue(SatWidth).zext(DstWidth);
10793 MaxInt = APInt::getMaxValue(SatWidth).zext(DstWidth);
10794 }
10795
10796 // We cannot risk emitting FP_TO_XINT nodes with a source VT of [b]f16, as
10797 // libcall emission cannot handle this. Large result types will fail.
10798 if (SrcVT == MVT::f16 || SrcVT == MVT::bf16) {
10799 Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, Src);
10800 SrcVT = Src.getValueType();
10801 }
10802
10803 APFloat MinFloat(DAG.EVTToAPFloatSemantics(SrcVT));
10804 APFloat MaxFloat(DAG.EVTToAPFloatSemantics(SrcVT));
10805
10806 APFloat::opStatus MinStatus =
10807 MinFloat.convertFromAPInt(MinInt, IsSigned, APFloat::rmTowardZero);
10808 APFloat::opStatus MaxStatus =
10809 MaxFloat.convertFromAPInt(MaxInt, IsSigned, APFloat::rmTowardZero);
10810 bool AreExactFloatBounds = !(MinStatus & APFloat::opStatus::opInexact) &&
10811 !(MaxStatus & APFloat::opStatus::opInexact);
10812
10813 SDValue MinFloatNode = DAG.getConstantFP(MinFloat, dl, SrcVT);
10814 SDValue MaxFloatNode = DAG.getConstantFP(MaxFloat, dl, SrcVT);
10815
10816 // If the integer bounds are exactly representable as floats and min/max are
10817 // legal, emit a min+max+fptoi sequence. Otherwise we have to use a sequence
10818 // of comparisons and selects.
10819 bool MinMaxLegal = isOperationLegal(ISD::FMINNUM, SrcVT) &&
10821 if (AreExactFloatBounds && MinMaxLegal) {
10822 SDValue Clamped = Src;
10823
10824 // Clamp Src by MinFloat from below. If Src is NaN the result is MinFloat.
10825 Clamped = DAG.getNode(ISD::FMAXNUM, dl, SrcVT, Clamped, MinFloatNode);
10826 // Clamp by MaxFloat from above. NaN cannot occur.
10827 Clamped = DAG.getNode(ISD::FMINNUM, dl, SrcVT, Clamped, MaxFloatNode);
10828 // Convert clamped value to integer.
10829 SDValue FpToInt = DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT,
10830 dl, DstVT, Clamped);
10831
10832 // In the unsigned case we're done, because we mapped NaN to MinFloat,
10833 // which will cast to zero.
10834 if (!IsSigned)
10835 return FpToInt;
10836
10837 // Otherwise, select 0 if Src is NaN.
10838 SDValue ZeroInt = DAG.getConstant(0, dl, DstVT);
10839 EVT SetCCVT =
10840 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), SrcVT);
10841 SDValue IsNan = DAG.getSetCC(dl, SetCCVT, Src, Src, ISD::CondCode::SETUO);
10842 return DAG.getSelect(dl, DstVT, IsNan, ZeroInt, FpToInt);
10843 }
10844
10845 SDValue MinIntNode = DAG.getConstant(MinInt, dl, DstVT);
10846 SDValue MaxIntNode = DAG.getConstant(MaxInt, dl, DstVT);
10847
10848 // Result of direct conversion. The assumption here is that the operation is
10849 // non-trapping and it's fine to apply it to an out-of-range value if we
10850 // select it away later.
10851 SDValue FpToInt =
10852 DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, dl, DstVT, Src);
10853
10854 SDValue Select = FpToInt;
10855
10856 EVT SetCCVT =
10857 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), SrcVT);
10858
10859 // If Src ULT MinFloat, select MinInt. In particular, this also selects
10860 // MinInt if Src is NaN.
10861 SDValue ULT = DAG.getSetCC(dl, SetCCVT, Src, MinFloatNode, ISD::SETULT);
10862 Select = DAG.getSelect(dl, DstVT, ULT, MinIntNode, Select);
10863 // If Src OGT MaxFloat, select MaxInt.
10864 SDValue OGT = DAG.getSetCC(dl, SetCCVT, Src, MaxFloatNode, ISD::SETOGT);
10865 Select = DAG.getSelect(dl, DstVT, OGT, MaxIntNode, Select);
10866
10867 // In the unsigned case we are done, because we mapped NaN to MinInt, which
10868 // is already zero.
10869 if (!IsSigned)
10870 return Select;
10871
10872 // Otherwise, select 0 if Src is NaN.
10873 SDValue ZeroInt = DAG.getConstant(0, dl, DstVT);
10874 SDValue IsNan = DAG.getSetCC(dl, SetCCVT, Src, Src, ISD::CondCode::SETUO);
10875 return DAG.getSelect(dl, DstVT, IsNan, ZeroInt, Select);
10876}
10877
10879 const SDLoc &dl,
10880 SelectionDAG &DAG) const {
10881 EVT OperandVT = Op.getValueType();
10882 if (OperandVT.getScalarType() == ResultVT.getScalarType())
10883 return Op;
10884 EVT ResultIntVT = ResultVT.changeTypeToInteger();
10885 // We are rounding binary64/binary128 -> binary32 -> bfloat16. This
10886 // can induce double-rounding which may alter the results. We can
10887 // correct for this using a trick explained in: Boldo, Sylvie, and
10888 // Guillaume Melquiond. "When double rounding is odd." 17th IMACS
10889 // World Congress. 2005.
10890 unsigned BitSize = OperandVT.getScalarSizeInBits();
10891 EVT WideIntVT = OperandVT.changeTypeToInteger();
10892 SDValue OpAsInt = DAG.getBitcast(WideIntVT, Op);
10893 SDValue SignBit =
10894 DAG.getNode(ISD::AND, dl, WideIntVT, OpAsInt,
10895 DAG.getConstant(APInt::getSignMask(BitSize), dl, WideIntVT));
10896 SDValue AbsWide;
10897 if (isOperationLegalOrCustom(ISD::FABS, OperandVT)) {
10898 AbsWide = DAG.getNode(ISD::FABS, dl, OperandVT, Op);
10899 } else {
10900 SDValue ClearedSign = DAG.getNode(
10901 ISD::AND, dl, WideIntVT, OpAsInt,
10902 DAG.getConstant(APInt::getSignedMaxValue(BitSize), dl, WideIntVT));
10903 AbsWide = DAG.getBitcast(OperandVT, ClearedSign);
10904 }
10905 SDValue AbsNarrow = DAG.getFPExtendOrRound(AbsWide, dl, ResultVT);
10906 SDValue AbsNarrowAsWide = DAG.getFPExtendOrRound(AbsNarrow, dl, OperandVT);
10907
10908 // We can keep the narrow value as-is if narrowing was exact (no
10909 // rounding error), the wide value was NaN (the narrow value is also
10910 // NaN and should be preserved) or if we rounded to the odd value.
10911 SDValue NarrowBits = DAG.getNode(ISD::BITCAST, dl, ResultIntVT, AbsNarrow);
10912 SDValue One = DAG.getConstant(1, dl, ResultIntVT);
10913 SDValue NegativeOne = DAG.getAllOnesConstant(dl, ResultIntVT);
10914 SDValue And = DAG.getNode(ISD::AND, dl, ResultIntVT, NarrowBits, One);
10915 EVT ResultIntVTCCVT = getSetCCResultType(
10916 DAG.getDataLayout(), *DAG.getContext(), And.getValueType());
10917 SDValue Zero = DAG.getConstant(0, dl, ResultIntVT);
10918 // The result is already odd so we don't need to do anything.
10919 SDValue AlreadyOdd = DAG.getSetCC(dl, ResultIntVTCCVT, And, Zero, ISD::SETNE);
10920
10921 EVT WideSetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
10922 AbsWide.getValueType());
10923 // We keep results which are exact, odd or NaN.
10924 SDValue KeepNarrow =
10925 DAG.getSetCC(dl, WideSetCCVT, AbsWide, AbsNarrowAsWide, ISD::SETUEQ);
10926 KeepNarrow = DAG.getNode(ISD::OR, dl, WideSetCCVT, KeepNarrow, AlreadyOdd);
10927 // We morally performed a round-down if AbsNarrow is smaller than
10928 // AbsWide.
10929 SDValue NarrowIsRd =
10930 DAG.getSetCC(dl, WideSetCCVT, AbsWide, AbsNarrowAsWide, ISD::SETOGT);
10931 // If the narrow value is odd or exact, pick it.
10932 // Otherwise, narrow is even and corresponds to either the rounded-up
10933 // or rounded-down value. If narrow is the rounded-down value, we want
10934 // the rounded-up value as it will be odd.
10935 SDValue Adjust = DAG.getSelect(dl, ResultIntVT, NarrowIsRd, One, NegativeOne);
10936 SDValue Adjusted = DAG.getNode(ISD::ADD, dl, ResultIntVT, NarrowBits, Adjust);
10937 Op = DAG.getSelect(dl, ResultIntVT, KeepNarrow, NarrowBits, Adjusted);
10938 int ShiftAmount = BitSize - ResultVT.getScalarSizeInBits();
10939 SDValue ShiftCnst = DAG.getShiftAmountConstant(ShiftAmount, WideIntVT, dl);
10940 SignBit = DAG.getNode(ISD::SRL, dl, WideIntVT, SignBit, ShiftCnst);
10941 SignBit = DAG.getNode(ISD::TRUNCATE, dl, ResultIntVT, SignBit);
10942 Op = DAG.getNode(ISD::OR, dl, ResultIntVT, Op, SignBit);
10943 return DAG.getNode(ISD::BITCAST, dl, ResultVT, Op);
10944}
10945
10947 assert(Node->getOpcode() == ISD::FP_ROUND && "Unexpected opcode!");
10948 SDValue Op = Node->getOperand(0);
10949 EVT VT = Node->getValueType(0);
10950 SDLoc dl(Node);
10951 if (VT.getScalarType() == MVT::bf16) {
10952 if (Node->getConstantOperandVal(1) == 1) {
10953 return DAG.getNode(ISD::FP_TO_BF16, dl, VT, Node->getOperand(0));
10954 }
10955 EVT OperandVT = Op.getValueType();
10956 SDValue IsNaN = DAG.getSetCC(
10957 dl,
10958 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), OperandVT),
10959 Op, Op, ISD::SETUO);
10960
10961 // We are rounding binary64/binary128 -> binary32 -> bfloat16. This
10962 // can induce double-rounding which may alter the results. We can
10963 // correct for this using a trick explained in: Boldo, Sylvie, and
10964 // Guillaume Melquiond. "When double rounding is odd." 17th IMACS
10965 // World Congress. 2005.
10966 EVT F32 = VT.isVector() ? VT.changeVectorElementType(MVT::f32) : MVT::f32;
10967 EVT I32 = F32.changeTypeToInteger();
10968 Op = expandRoundInexactToOdd(F32, Op, dl, DAG);
10969 Op = DAG.getNode(ISD::BITCAST, dl, I32, Op);
10970
10971 // Conversions should set NaN's quiet bit. This also prevents NaNs from
10972 // turning into infinities.
10973 SDValue NaN =
10974 DAG.getNode(ISD::OR, dl, I32, Op, DAG.getConstant(0x400000, dl, I32));
10975
10976 // Factor in the contribution of the low 16 bits.
10977 SDValue One = DAG.getConstant(1, dl, I32);
10978 SDValue Lsb = DAG.getNode(ISD::SRL, dl, I32, Op,
10979 DAG.getShiftAmountConstant(16, I32, dl));
10980 Lsb = DAG.getNode(ISD::AND, dl, I32, Lsb, One);
10981 SDValue RoundingBias =
10982 DAG.getNode(ISD::ADD, dl, I32, DAG.getConstant(0x7fff, dl, I32), Lsb);
10983 SDValue Add = DAG.getNode(ISD::ADD, dl, I32, Op, RoundingBias);
10984
10985 // Don't round if we had a NaN, we don't want to turn 0x7fffffff into
10986 // 0x80000000.
10987 Op = DAG.getSelect(dl, I32, IsNaN, NaN, Add);
10988
10989 // Now that we have rounded, shift the bits into position.
10990 Op = DAG.getNode(ISD::SRL, dl, I32, Op,
10991 DAG.getShiftAmountConstant(16, I32, dl));
10992 Op = DAG.getNode(ISD::BITCAST, dl, I32, Op);
10993 EVT I16 = I32.isVector() ? I32.changeVectorElementType(MVT::i16) : MVT::i16;
10994 Op = DAG.getNode(ISD::TRUNCATE, dl, I16, Op);
10995 return DAG.getNode(ISD::BITCAST, dl, VT, Op);
10996 }
10997 return SDValue();
10998}
10999
11001 SelectionDAG &DAG) const {
11002 assert(Node->getOpcode() == ISD::VECTOR_SPLICE && "Unexpected opcode!");
11003 assert(Node->getValueType(0).isScalableVector() &&
11004 "Fixed length vector types expected to use SHUFFLE_VECTOR!");
11005
11006 EVT VT = Node->getValueType(0);
11007 SDValue V1 = Node->getOperand(0);
11008 SDValue V2 = Node->getOperand(1);
11009 int64_t Imm = cast<ConstantSDNode>(Node->getOperand(2))->getSExtValue();
11010 SDLoc DL(Node);
11011
11012 // Expand through memory thusly:
11013 // Alloca CONCAT_VECTORS_TYPES(V1, V2) Ptr
11014 // Store V1, Ptr
11015 // Store V2, Ptr + sizeof(V1)
11016 // If (Imm < 0)
11017 // TrailingElts = -Imm
11018 // Ptr = Ptr + sizeof(V1) - (TrailingElts * sizeof(VT.Elt))
11019 // else
11020 // Ptr = Ptr + (Imm * sizeof(VT.Elt))
11021 // Res = Load Ptr
11022
11023 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
11024
11026 VT.getVectorElementCount() * 2);
11027 SDValue StackPtr = DAG.CreateStackTemporary(MemVT.getStoreSize(), Alignment);
11028 EVT PtrVT = StackPtr.getValueType();
11029 auto &MF = DAG.getMachineFunction();
11030 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
11031 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
11032
11033 // Store the lo part of CONCAT_VECTORS(V1, V2)
11034 SDValue StoreV1 = DAG.getStore(DAG.getEntryNode(), DL, V1, StackPtr, PtrInfo);
11035 // Store the hi part of CONCAT_VECTORS(V1, V2)
11036 SDValue OffsetToV2 = DAG.getVScale(
11037 DL, PtrVT,
11039 SDValue StackPtr2 = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, OffsetToV2);
11040 SDValue StoreV2 = DAG.getStore(StoreV1, DL, V2, StackPtr2, PtrInfo);
11041
11042 if (Imm >= 0) {
11043 // Load back the required element. getVectorElementPointer takes care of
11044 // clamping the index if it's out-of-bounds.
11045 StackPtr = getVectorElementPointer(DAG, StackPtr, VT, Node->getOperand(2));
11046 // Load the spliced result
11047 return DAG.getLoad(VT, DL, StoreV2, StackPtr,
11049 }
11050
11051 uint64_t TrailingElts = -Imm;
11052
11053 // NOTE: TrailingElts must be clamped so as not to read outside of V1:V2.
11054 TypeSize EltByteSize = VT.getVectorElementType().getStoreSize();
11055 SDValue TrailingBytes =
11056 DAG.getConstant(TrailingElts * EltByteSize, DL, PtrVT);
11057
11058 if (TrailingElts > VT.getVectorMinNumElements()) {
11059 SDValue VLBytes =
11060 DAG.getVScale(DL, PtrVT,
11061 APInt(PtrVT.getFixedSizeInBits(),
11063 TrailingBytes = DAG.getNode(ISD::UMIN, DL, PtrVT, TrailingBytes, VLBytes);
11064 }
11065
11066 // Calculate the start address of the spliced result.
11067 StackPtr2 = DAG.getNode(ISD::SUB, DL, PtrVT, StackPtr2, TrailingBytes);
11068
11069 // Load the spliced result
11070 return DAG.getLoad(VT, DL, StoreV2, StackPtr2,
11072}
11073
11075 SDValue &LHS, SDValue &RHS,
11076 SDValue &CC, SDValue Mask,
11077 SDValue EVL, bool &NeedInvert,
11078 const SDLoc &dl, SDValue &Chain,
11079 bool IsSignaling) const {
11080 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11081 MVT OpVT = LHS.getSimpleValueType();
11082 ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
11083 NeedInvert = false;
11084 assert(!EVL == !Mask && "VP Mask and EVL must either both be set or unset");
11085 bool IsNonVP = !EVL;
11086 switch (TLI.getCondCodeAction(CCCode, OpVT)) {
11087 default:
11088 llvm_unreachable("Unknown condition code action!");
11090 // Nothing to do.
11091 break;
11094 if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
11095 std::swap(LHS, RHS);
11096 CC = DAG.getCondCode(InvCC);
11097 return true;
11098 }
11099 // Swapping operands didn't work. Try inverting the condition.
11100 bool NeedSwap = false;
11101 InvCC = getSetCCInverse(CCCode, OpVT);
11102 if (!TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
11103 // If inverting the condition is not enough, try swapping operands
11104 // on top of it.
11105 InvCC = ISD::getSetCCSwappedOperands(InvCC);
11106 NeedSwap = true;
11107 }
11108 if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
11109 CC = DAG.getCondCode(InvCC);
11110 NeedInvert = true;
11111 if (NeedSwap)
11112 std::swap(LHS, RHS);
11113 return true;
11114 }
11115
11117 unsigned Opc = 0;
11118 switch (CCCode) {
11119 default:
11120 llvm_unreachable("Don't know how to expand this condition!");
11121 case ISD::SETUO:
11122 if (TLI.isCondCodeLegal(ISD::SETUNE, OpVT)) {
11123 CC1 = ISD::SETUNE;
11124 CC2 = ISD::SETUNE;
11125 Opc = ISD::OR;
11126 break;
11127 }
11128 assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) &&
11129 "If SETUE is expanded, SETOEQ or SETUNE must be legal!");
11130 NeedInvert = true;
11131 [[fallthrough]];
11132 case ISD::SETO:
11133 assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) &&
11134 "If SETO is expanded, SETOEQ must be legal!");
11135 CC1 = ISD::SETOEQ;
11136 CC2 = ISD::SETOEQ;
11137 Opc = ISD::AND;
11138 break;
11139 case ISD::SETONE:
11140 case ISD::SETUEQ:
11141 // If the SETUO or SETO CC isn't legal, we might be able to use
11142 // SETOGT || SETOLT, inverting the result for SETUEQ. We only need one
11143 // of SETOGT/SETOLT to be legal, the other can be emulated by swapping
11144 // the operands.
11145 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
11146 if (!TLI.isCondCodeLegal(CC2, OpVT) &&
11147 (TLI.isCondCodeLegal(ISD::SETOGT, OpVT) ||
11148 TLI.isCondCodeLegal(ISD::SETOLT, OpVT))) {
11149 CC1 = ISD::SETOGT;
11150 CC2 = ISD::SETOLT;
11151 Opc = ISD::OR;
11152 NeedInvert = ((unsigned)CCCode & 0x8U);
11153 break;
11154 }
11155 [[fallthrough]];
11156 case ISD::SETOEQ:
11157 case ISD::SETOGT:
11158 case ISD::SETOGE:
11159 case ISD::SETOLT:
11160 case ISD::SETOLE:
11161 case ISD::SETUNE:
11162 case ISD::SETUGT:
11163 case ISD::SETUGE:
11164 case ISD::SETULT:
11165 case ISD::SETULE:
11166 // If we are floating point, assign and break, otherwise fall through.
11167 if (!OpVT.isInteger()) {
11168 // We can use the 4th bit to tell if we are the unordered
11169 // or ordered version of the opcode.
11170 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
11171 Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND;
11172 CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10);
11173 break;
11174 }
11175 // Fallthrough if we are unsigned integer.
11176 [[fallthrough]];
11177 case ISD::SETLE:
11178 case ISD::SETGT:
11179 case ISD::SETGE:
11180 case ISD::SETLT:
11181 case ISD::SETNE:
11182 case ISD::SETEQ:
11183 // If all combinations of inverting the condition and swapping operands
11184 // didn't work then we have no means to expand the condition.
11185 llvm_unreachable("Don't know how to expand this condition!");
11186 }
11187
11188 SDValue SetCC1, SetCC2;
11189 if (CCCode != ISD::SETO && CCCode != ISD::SETUO) {
11190 // If we aren't the ordered or unorder operation,
11191 // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS).
11192 if (IsNonVP) {
11193 SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1, Chain, IsSignaling);
11194 SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2, Chain, IsSignaling);
11195 } else {
11196 SetCC1 = DAG.getSetCCVP(dl, VT, LHS, RHS, CC1, Mask, EVL);
11197 SetCC2 = DAG.getSetCCVP(dl, VT, LHS, RHS, CC2, Mask, EVL);
11198 }
11199 } else {
11200 // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS)
11201 if (IsNonVP) {
11202 SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1, Chain, IsSignaling);
11203 SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2, Chain, IsSignaling);
11204 } else {
11205 SetCC1 = DAG.getSetCCVP(dl, VT, LHS, LHS, CC1, Mask, EVL);
11206 SetCC2 = DAG.getSetCCVP(dl, VT, RHS, RHS, CC2, Mask, EVL);
11207 }
11208 }
11209 if (Chain)
11210 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, SetCC1.getValue(1),
11211 SetCC2.getValue(1));
11212 if (IsNonVP)
11213 LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2);
11214 else {
11215 // Transform the binary opcode to the VP equivalent.
11216 assert((Opc == ISD::OR || Opc == ISD::AND) && "Unexpected opcode");
11217 Opc = Opc == ISD::OR ? ISD::VP_OR : ISD::VP_AND;
11218 LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2, Mask, EVL);
11219 }
11220 RHS = SDValue();
11221 CC = SDValue();
11222 return true;
11223 }
11224 }
11225 return false;
11226}
unsigned const MachineRegisterInfo * MRI
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu AMDGPU Register Bank Select
basic Basic Alias true
block Block Frequency Analysis
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
return RetTy
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
uint64_t Addr
uint64_t Size
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static bool isSigned(unsigned int Opcode)
static bool isUndef(ArrayRef< int > Mask)
static bool ShrinkDemandedConstant(Instruction *I, unsigned OpNo, const APInt &Demanded)
Check to see if the specified operand of the specified instruction is a constant integer.
#define RegName(no)
static LVOptions Options
Definition: LVOptions.cpp:25
lazy value info
static bool isNonZeroModBitWidthOrUndef(const MachineRegisterInfo &MRI, Register Reg, unsigned BW)
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition: Lint.cpp:531
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
#define G(x, y, z)
Definition: MD5.cpp:56
unsigned const TargetRegisterInfo * TRI
#define T1
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
#define P(N)
const char LLVMTargetMachineRef TM
const char * Passes
R600 Clause Merge
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
static SDValue foldSetCCWithFunnelShift(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, const SDLoc &dl, SelectionDAG &DAG)
static bool lowerImmediateIfPossible(TargetLowering::ConstraintPair &P, SDValue Op, SelectionDAG *DAG, const TargetLowering &TLI)
If we have an immediate, see if we can lower it.
static SDValue expandVPFunnelShift(SDNode *Node, SelectionDAG &DAG)
static APInt getKnownUndefForVectorBinop(SDValue BO, SelectionDAG &DAG, const APInt &UndefOp0, const APInt &UndefOp1)
Given a vector binary operation and known undefined elements for each input operand,...
static SDValue clampDynamicVectorIndex(SelectionDAG &DAG, SDValue Idx, EVT VecVT, const SDLoc &dl, ElementCount SubEC)
static unsigned getConstraintPiority(TargetLowering::ConstraintType CT)
Return a number indicating our preference for chosing a type of constraint over another,...
static std::optional< bool > isFCmpEqualZero(FPClassTest Test, const fltSemantics &Semantics, const MachineFunction &MF)
Returns a true value if if this FPClassTest can be performed with an ordered fcmp to 0,...
static void turnVectorIntoSplatVector(MutableArrayRef< SDValue > Values, std::function< bool(SDValue)> Predicate, SDValue AlternativeReplacement=SDValue())
If all values in Values that don't match the predicate are same 'splat' value, then replace all value...
static bool canExpandVectorCTPOP(const TargetLowering &TLI, EVT VT)
static SDValue foldSetCCWithRotate(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, const SDLoc &dl, SelectionDAG &DAG)
static SDValue combineShiftToAVG(SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, const APInt &DemandedBits, const APInt &DemandedElts, unsigned Depth)
static SDValue BuildExactSDIV(const TargetLowering &TLI, SDNode *N, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl< SDNode * > &Created)
Given an exact SDIV by a constant, create a multiplication with the multiplicative inverse of the con...
static SDValue simplifySetCCWithCTPOP(const TargetLowering &TLI, EVT VT, SDValue N0, const APInt &C1, ISD::CondCode Cond, const SDLoc &dl, SelectionDAG &DAG)
This file describes how to lower LLVM code to machine code.
static int Lookup(ArrayRef< TableEntry > Table, unsigned Opcode)
static SDValue scalarizeVectorStore(StoreSDNode *Store, MVT StoreVT, SelectionDAG &DAG)
Scalarize a vector store, bitcasting to TargetVT to determine the scalar type.
Value * RHS
Value * LHS
opStatus convertFromAPInt(const APInt &Input, bool IsSigned, roundingMode RM)
Definition: APFloat.h:1193
static APFloat getSmallestNormalized(const fltSemantics &Sem, bool Negative=false)
Returns the smallest (by magnitude) normalized finite number in the given semantics.
Definition: APFloat.h:1026
APInt bitcastToAPInt() const
Definition: APFloat.h:1210
static APFloat getLargest(const fltSemantics &Sem, bool Negative=false)
Returns the largest finite number in the given semantics.
Definition: APFloat.h:1006
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
Definition: APFloat.h:966
Class for arbitrary precision integers.
Definition: APInt.h:76
APInt udiv(const APInt &RHS) const
Unsigned division operation.
Definition: APInt.cpp:1579
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition: APInt.h:212
static void udivrem(const APInt &LHS, const APInt &RHS, APInt &Quotient, APInt &Remainder)
Dual division/remainder interface.
Definition: APInt.cpp:1764
void clearBit(unsigned BitPosition)
Set a given bit to 0.
Definition: APInt.h:1385
bool isNegatedPowerOf2() const
Check if this APInt's negated value is a power of two greater than zero.
Definition: APInt.h:427
APInt zext(unsigned width) const
Zero extend to a new width.
Definition: APInt.cpp:981
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
Definition: APInt.h:207
bool isMinSignedValue() const
Determine if this is the smallest signed value.
Definition: APInt.h:401
uint64_t getZExtValue() const
Get zero extended value.
Definition: APInt.h:1491
void setHighBits(unsigned hiBits)
Set the top hiBits bits.
Definition: APInt.h:1370
void setBitsFrom(unsigned loBit)
Set the top bits starting from loBit.
Definition: APInt.h:1364
APInt multiplicativeInverse(const APInt &modulo) const
Computes the multiplicative inverse of this APInt for a given modulo.
Definition: APInt.cpp:1250
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
Definition: APInt.cpp:1002
unsigned getActiveBits() const
Compute the number of active bits in the value.
Definition: APInt.h:1463
APInt trunc(unsigned width) const
Truncate to new width.
Definition: APInt.cpp:906
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
Definition: APInt.h:184
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
Definition: APInt.h:1308
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
Definition: APInt.h:349
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
Definition: APInt.h:1160
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
Definition: APInt.h:236
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
Definition: APInt.h:358
APInt urem(const APInt &RHS) const
Unsigned remainder operation.
Definition: APInt.cpp:1672
void setSignBit()
Set the sign bit to 1.
Definition: APInt.h:1318
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition: APInt.h:1439
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
Definition: APInt.h:187
static APInt getMinValue(unsigned numBits)
Gets minimum unsigned value of APInt for a specific bit width.
Definition: APInt.h:194
bool isNegative() const
Determine sign of this APInt.
Definition: APInt.h:307
bool intersects(const APInt &RHS) const
This operation tests if there are any pairs of corresponding bits between this APInt and RHS that are...
Definition: APInt.h:1227
void clearAllBits()
Set every bit to 0.
Definition: APInt.h:1375
APInt reverseBits() const
Definition: APInt.cpp:737
void ashrInPlace(unsigned ShiftAmt)
Arithmetic right-shift this APInt by ShiftAmt in place.
Definition: APInt.h:812
void negate()
Negate this APInt in place.
Definition: APInt.h:1421
unsigned countr_zero() const
Count the number of trailing zero bits.
Definition: APInt.h:1589
unsigned countl_zero() const
The APInt version of std::countl_zero.
Definition: APInt.h:1548
static APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
Definition: APInt.cpp:620
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
Definition: APInt.h:197
unsigned getSignificantBits() const
Get the minimum bit size for this signed APInt.
Definition: APInt.h:1482
unsigned countLeadingZeros() const
Definition: APInt.h:1556
bool isStrictlyPositive() const
Determine if this APInt Value is positive.
Definition: APInt.h:334
void insertBits(const APInt &SubBits, unsigned bitPosition)
Insert the bits from a smaller APInt starting at bitPosition.
Definition: APInt.cpp:368
unsigned logBase2() const
Definition: APInt.h:1703
uint64_t getLimitedValue(uint64_t Limit=UINT64_MAX) const
If this value is smaller than the specified limit, return it, otherwise return the limit value.
Definition: APInt.h:453
void setAllBits()
Set every bit to 1.
Definition: APInt.h:1297
bool isMaxSignedValue() const
Determine if this is the largest signed value.
Definition: APInt.h:383
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
Definition: APInt.h:312
bool ule(const APInt &RHS) const
Unsigned less or equal comparison.
Definition: APInt.h:1128
APInt sext(unsigned width) const
Sign extend to a new width.
Definition: APInt.cpp:954
void setBits(unsigned loBit, unsigned hiBit)
Set the bits from loBit (inclusive) to hiBit (exclusive) to 1.
Definition: APInt.h:1345
APInt shl(unsigned shiftAmt) const
Left-shift function.
Definition: APInt.h:851
APInt byteSwap() const
Definition: APInt.cpp:715
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Definition: APInt.h:1235
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
Definition: APInt.h:418
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
Definition: APInt.h:284
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
Definition: APInt.h:274
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
Definition: APInt.h:178
void setLowBits(unsigned loBits)
Set the bottom loBits bits.
Definition: APInt.h:1367
APInt extractBits(unsigned numBits, unsigned bitPosition) const
Return an APInt with the extracted bits [bitPosition,bitPosition+numBits).
Definition: APInt.cpp:453
bool isOne() const
Determine if this is a value of 1.
Definition: APInt.h:367
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
Definition: APInt.h:264
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
Definition: APInt.h:217
int64_t getSExtValue() const
Get sign extended value.
Definition: APInt.h:1513
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
Definition: APInt.h:836
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
Definition: APInt.h:829
unsigned countr_one() const
Count the number of trailing one bits.
Definition: APInt.h:1606
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
Definition: APInt.h:1199
void setBitVal(unsigned BitPosition, bool BitValue)
Set a given bit to a given value.
Definition: APInt.h:1321
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:165
bool hasAttributes() const
Return true if the builder has IR-level attributes.
Definition: Attributes.h:1070
bool contains(Attribute::AttrKind A) const
Return true if the builder has the specified attribute.
AttrBuilder & removeAttribute(Attribute::AttrKind Val)
Remove an attribute from the builder.
bool hasFnAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the function.
A "pseudo-class" with methods for operating on BUILD_VECTORs.
ConstantSDNode * getConstantSplatNode(const APInt &DemandedElts, BitVector *UndefElements=nullptr) const
Returns the demanded splatted constant or null if this is not a constant splat.
CCValAssign - Represent assignment of one arg/retval to a location.
bool isRegLoc() const
Register getLocReg() const
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1455
This class represents a function call, abstracting a target machine's calling convention.
static Constant * get(LLVMContext &Context, ArrayRef< ElementTy > Elts)
get() constructor - Return a constant with array type with an element count and element type matching...
Definition: Constants.h:704
const APInt & getAPIntValue() const
This is an important base class in LLVM.
Definition: Constant.h:41
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
bool isLittleEndian() const
Layout endianness...
Definition: DataLayout.h:238
bool isBigEndian() const
Definition: DataLayout.h:239
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
Definition: DataLayout.cpp:874
AttributeList getAttributes() const
Return the attribute list for this Function.
Definition: Function.h:338
const GlobalValue * getGlobal() const
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:655
std::vector< std::string > ConstraintCodeVector
Definition: InlineAsm.h:102
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition: Type.cpp:278
void emitError(uint64_t LocCookie, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
Context object for machine code objects.
Definition: MCContext.h:76
Base class for the full range of assembler expressions which are needed for parsing.
Definition: MCExpr.h:35
Wrapper class representing physical registers. Should be passed by value.
Definition: MCRegister.h:33
static const MCSymbolRefExpr * create(const MCSymbol *Symbol, MCContext &Ctx)
Definition: MCExpr.h:397
Machine Value Type.
SimpleValueType SimpleTy
bool isInteger() const
Return true if this is an integer or a vector integer type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
DenormalMode getDenormalMode(const fltSemantics &FPType) const
Returns the denormal handling type for the default rounding mode of the function.
MCSymbol * getJTISymbol(unsigned JTI, MCContext &Ctx, bool isLinkerPrivate=false) const
getJTISymbol - Return the MCSymbol for the specified non-empty jump table.
Function & getFunction()
Return the LLVM function that this machine code represents.
@ EK_GPRel32BlockAddress
EK_GPRel32BlockAddress - Each entry is an address of block, encoded with a relocation as gp-relative,...
@ EK_LabelDifference32
EK_LabelDifference32 - Each entry is the address of the block minus the address of the jump table.
@ EK_BlockAddress
EK_BlockAddress - Each entry is a plain address of block, e.g.: .word LBB123.
@ EK_GPRel64BlockAddress
EK_GPRel64BlockAddress - Each entry is an address of block, encoded with a relocation as gp-relative,...
static bool clobbersPhysReg(const uint32_t *RegMask, MCRegister PhysReg)
clobbersPhysReg - Returns true if this RegMask clobbers PhysReg.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Align getOriginalAlign() const
Returns alignment and volatility of the memory access.
bool isSimple() const
Returns true if the memory operation is neither atomic or volatile.
const MachinePointerInfo & getPointerInfo() const
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
const GlobalVariable * getNamedGlobal(StringRef Name) const
Return the global variable in the module with the specified name, of arbitrary type.
Definition: Module.h:455
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Definition: ArrayRef.h:307
iterator end() const
Definition: ArrayRef.h:357
iterator begin() const
Definition: ArrayRef.h:356
Class to represent pointers.
Definition: DerivedTypes.h:646
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
static SDNodeIterator end(const SDNode *N)
static SDNodeIterator begin(const SDNode *N)
Represents one node in the SelectionDAG.
const APInt & getAsAPIntVal() const
Helper method returns the APInt value of a ConstantSDNode.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
SDNodeFlags getFlags() const
const SDValue & getOperand(unsigned Num) const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
void setFlags(SDNodeFlags NewFlags)
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
bool isUndef() const
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
bool use_empty() const
Return true if there are no nodes using value ResNo of Node.
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
uint64_t getConstantOperandVal(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:225
bool willNotOverflowAdd(bool IsSigned, SDValue N0, SDValue N1) const
Determine if the result of the addition of 2 nodes can never overflow.
Align getReducedAlign(EVT VT, bool UseABI)
In most cases this function returns the ABI alignment for a given type, except for illegal vector typ...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
Definition: SelectionDAG.h:722
SDValue getExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT, unsigned Opcode)
Convert Op, which must be of integer type, to the integer type VT, by either any/sign/zero-extending ...
Definition: SelectionDAG.h:954
unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
bool isKnownNeverSNaN(SDValue Op, unsigned Depth=0) const
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
const APInt * getValidMaximumShiftAmountConstant(SDValue V, const APInt &DemandedElts) const
If a SHL/SRA/SRL node V has constant shift amounts that are all less than the element bit-width of th...
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond, const SDLoc &dl)
Constant fold a setcc to true or false.
void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm, bool ConstantFold=true)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offs=0, bool isT=false, unsigned TargetFlags=0)
SDNode * isConstantIntBuildVectorOrConstantInt(SDValue N) const
Test whether the given value is a constant int or similar node.
SDValue getJumpTableDebugInfo(int JTI, SDValue Chain, const SDLoc &DL)
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getGLOBAL_OFFSET_TABLE(EVT VT)
Return a GLOBAL_OFFSET_TABLE node. This does not have a useful SDLoc.
bool shouldOptForSize() const
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:478
static constexpr unsigned MaxRecursionDepth
Definition: SelectionDAG.h:448
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
Definition: SelectionDAG.h:828
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:472
bool doesNodeExist(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops)
Check if a node exists without modifying its flags.
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, bool isTargetGA=false, unsigned TargetFlags=0)
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
Definition: SelectionDAG.h:659
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provided VTs and return the low/high part.
const APInt * getValidShiftAmountConstant(SDValue V, const APInt &DemandedElts) const
If a SHL/SRA/SRL node V has a constant or splat constant shift amount that is less than the element b...
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op)
Definition: SelectionDAG.h:862
void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
bool isKnownToBeAPowerOfTwo(SDValue Val, unsigned Depth=0) const
Test if the given value is known to have exactly one bit set.
bool isKnownNeverZero(SDValue Op, unsigned Depth=0) const
Test whether the given SDValue is known to contain non-zero value(s).
SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
Definition: SelectionDAG.h:473
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
bool isKnownNeverZeroFloat(SDValue Op) const
Test whether the given floating point SDValue is known to never be positive or negative zero.
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
bool isKnownNeverNaN(SDValue Op, bool SNaN=false, unsigned Depth=0) const
Test whether the given SDValue (or all elements of it, if it is a vector) is known to never be NaN.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:676
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT)
Create a true or false constant of type VT using the target's BooleanContent for type OpVT.
SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, unsigned TargetFlags=0)
Definition: SelectionDAG.h:768
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:469
SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops)
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
Definition: SelectionDAG.h:485
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes=true)
SDValue getSetCCVP(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Mask, SDValue EVL)
Helper function to make it easier to build VP_SETCCs if you just have an ISD::CondCode instead of an ...
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:554
std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
static void commuteMask(MutableArrayRef< int > Mask)
Change values in a shuffle permute mask assuming the two vector operands have swapped position.
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:696
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
This class is used to represent ISD::STORE nodes.
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition: StringRef.h:567
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
Definition: StringRef.h:257
constexpr size_t size() const
size - Get the string size.
Definition: StringRef.h:137
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Definition: StringRef.h:131
iterator end() const
Definition: StringRef.h:113
Class to represent struct types.
Definition: DerivedTypes.h:216
void setAttributes(const CallBase *Call, unsigned ArgIdx)
Set CallLoweringInfo attribute flags based on a call instruction and called function attributes.
This base class for TargetLowering contains the SelectionDAG-independent parts that can be used from ...
bool isOperationExpand(unsigned Op, EVT VT) const
Return true if the specified operation is illegal on this target or unlikely to be made legal with cu...
virtual bool shouldSignExtendTypeInLibCall(EVT Type, bool IsSigned) const
Returns true if arguments should be sign-extended in lib calls.
virtual bool isShuffleMaskLegal(ArrayRef< int >, EVT) const
Targets can use this to indicate that they only support some VECTOR_SHUFFLE operations,...
virtual bool shouldRemoveRedundantExtend(SDValue Op) const
Return true (the default) if it is profitable to remove a sext_inreg(x) where the sext is redundant,...
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
virtual bool isLegalICmpImmediate(int64_t) const
Return true if the specified immediate is legal icmp immediate, that is the target has icmp instructi...
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
virtual bool isSafeMemOpType(MVT) const
Returns true if it's safe to use load / store of the specified type to expand memcpy / memset inline.
const TargetMachine & getTargetMachine() const
virtual bool isCtpopFast(EVT VT) const
Return true if ctpop instruction is fast.
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
virtual EVT getOptimalMemOpType(const MemOp &Op, const AttributeList &) const
Returns the target specific optimal type for load and store operations as a result of memset,...
LegalizeAction getCondCodeAction(ISD::CondCode CC, MVT VT) const
Return how the condition code should be treated: either it is legal, needs to be expanded to some oth...
virtual bool isCommutativeBinOp(unsigned Opcode) const
Returns true if the opcode is a commutative binary operation.
virtual bool isFPImmLegal(const APFloat &, EVT, bool ForCodeSize=false) const
Returns true if the target can instruction select the specified FP immediate natively.
virtual MVT::SimpleValueType getCmpLibcallReturnType() const
Return the ValueType for comparison libcalls.
virtual bool shouldTransformSignedTruncationCheck(EVT XVT, unsigned KeptBits) const
Should we tranform the IR-optimal check for whether given truncation down into KeptBits would be trun...
bool isLegalRC(const TargetRegisterInfo &TRI, const TargetRegisterClass &RC) const
Return true if the value types that can be represented by the specified register class are all legal.
virtual bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *=nullptr) const
Determine if the target supports unaligned memory accesses.
virtual bool shouldExtendTypeInLibCall(EVT Type) const
Returns true if arguments should be extended in lib calls.
virtual bool isTruncateFree(Type *FromTy, Type *ToTy) const
Return true if it's free to truncate a value of type FromTy to type ToTy.
virtual bool shouldAvoidTransformToShift(EVT VT, unsigned Amount) const
Return true if creating a shift of the type by the given amount is not profitable.
virtual bool isFPExtFree(EVT DestVT, EVT SrcVT) const
Return true if an fpext operation is free (for instance, because single-precision floating-point numb...
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
Returns the type for the shift amount of a shift opcode.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
MVT getSimpleValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the MVT corresponding to this LLVM type. See getValueType.
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
bool isCondCodeLegal(ISD::CondCode CC, MVT VT) const
Return true if the specified condition code is legal on this target.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
virtual bool isNarrowingProfitable(EVT SrcVT, EVT DestVT) const
Return true if it's profitable to narrow operations of type SrcVT to DestVT.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
virtual bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT) const
Return true if it is profitable to reduce a load to a smaller type.
virtual unsigned getCustomCtpopCost(EVT VT, ISD::CondCode Cond) const
Return the maximum number of "x & (x - 1)" operations that can be done instead of deferring to a cust...
BooleanContent
Enum that describes how the target represents true/false values.
virtual bool isIntDivCheap(EVT VT, AttributeList Attr) const
Return true if integer divide is usually cheaper than a sequence of several shifts,...
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual bool hasAndNotCompare(SDValue Y) const
Return true if the target should transform: (X & Y) == Y —> (~X & Y) == 0 (X & Y) !...
virtual bool isBinOp(unsigned Opcode) const
Return true if the node is a math/logic binary operator.
virtual bool isCtlzFast() const
Return true if ctlz instruction is fast.
virtual bool shouldUseStrictFP_TO_INT(EVT FpVT, EVT IntVT, bool IsSigned) const
Return true if it is more correct/profitable to use strict FP_TO_INT conversion operations - canonica...
NegatibleCost
Enum that specifies when a float negation is beneficial.
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
ISD::CondCode getCmpLibcallCC(RTLIB::Libcall Call) const
Get the CondCode that's to be used to test the result of the comparison libcall against zero.
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
std::vector< ArgListEntry > ArgListTy
virtual EVT getAsmOperandValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
bool isCondCodeLegalOrCustom(ISD::CondCode CC, MVT VT) const
Return true if the specified condition code is legal or custom on this target.
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
MulExpansionKind
Enum that specifies when a multiplication should be expanded.
static ISD::NodeType getExtendForContent(BooleanContent Content)
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US][ADD|SUB]SAT.
SDValue buildSDIVPow2WithCMov(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl< SDNode * > &Created) const
Build sdiv by power-of-2 with conditional move instructions Ref: "Hacker's Delight" by Henry Warren 1...
virtual ConstraintWeight getMultipleConstraintMatchWeight(AsmOperandInfo &info, int maIndex) const
Examine constraint type and operand type and determine a weight value.
SDValue expandVPCTLZ(SDNode *N, SelectionDAG &DAG) const
Expand VP_CTLZ/VP_CTLZ_ZERO_UNDEF nodes.
bool expandMULO(SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]MULO.
bool expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL into two nodes.
virtual const MCExpr * getPICJumpTableRelocBaseExpr(const MachineFunction *MF, unsigned JTI, MCContext &Ctx) const
This returns the relocation base for the given PIC jumptable, the same as getPICJumpTableRelocBase,...
bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedEltMask, APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Vector Op.
virtual bool isUsedByReturnOnly(SDNode *, SDValue &) const
Return true if result of the specified node is used by a return node only.
virtual void computeKnownBitsForFrameIndex(int FIOp, KnownBits &Known, const MachineFunction &MF) const
Determine which of the bits of FrameIndex FIOp are known to be 0.
SDValue scalarizeVectorStore(StoreSDNode *ST, SelectionDAG &DAG) const
virtual unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const
This method can be implemented by targets that want to expose additional information about sign bits ...
SDValue lowerCmpEqZeroToCtlzSrl(SDValue Op, SelectionDAG &DAG) const
virtual unsigned computeNumSignBitsForTargetInstr(GISelKnownBits &Analysis, Register R, const APInt &DemandedElts, const MachineRegisterInfo &MRI, unsigned Depth=0) const
This method can be implemented by targets that want to expose additional information about sign bits ...
SDValue expandVPBSWAP(SDNode *N, SelectionDAG &DAG) const
Expand VP_BSWAP nodes.
void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &DL, const SDValue OldLHS, const SDValue OldRHS) const
Soften the operands of a comparison.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
SDValue expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_SEQ_* into an explicit ordered calculation.
SDValue expandCTLZ(SDNode *N, SelectionDAG &DAG) const
Expand CTLZ/CTLZ_ZERO_UNDEF nodes.
SDValue expandBITREVERSE(SDNode *N, SelectionDAG &DAG) const
Expand BITREVERSE nodes.
SDValue expandCTTZ(SDNode *N, SelectionDAG &DAG) const
Expand CTTZ/CTTZ_ZERO_UNDEF nodes.
virtual SDValue expandIndirectJTBranch(const SDLoc &dl, SDValue Value, SDValue Addr, int JTI, SelectionDAG &DAG) const
Expands target specific indirect branch for the case of JumpTable expansion.
SDValue expandABD(SDNode *N, SelectionDAG &DAG) const
Expand ABDS/ABDU nodes.
virtual Align computeKnownAlignForTargetInstr(GISelKnownBits &Analysis, Register R, const MachineRegisterInfo &MRI, unsigned Depth=0) const
Determine the known alignment for the pointer value R.
std::vector< AsmOperandInfo > AsmOperandInfoVector
SDValue expandShlSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]SHLSAT.
SDValue expandIS_FPCLASS(EVT ResultVT, SDValue Op, FPClassTest Test, SDNodeFlags Flags, const SDLoc &DL, SelectionDAG &DAG) const
Expand check for floating point class.
SDValue expandFP_TO_INT_SAT(SDNode *N, SelectionDAG &DAG) const
Expand FP_TO_[US]INT_SAT into FP_TO_[US]INT and selects or min/max.
SDValue SimplifyMultipleUseDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, SelectionDAG &DAG, unsigned Depth=0) const
More limited version of SimplifyDemandedBits that can be used to "look through" ops that don't contri...
SDValue expandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG) const
Expands an unaligned store to 2 half-size stores for integer values, and possibly more for vectors.
SDValue SimplifyMultipleUseDemandedVectorElts(SDValue Op, const APInt &DemandedElts, SelectionDAG &DAG, unsigned Depth=0) const
Helper wrapper around SimplifyMultipleUseDemandedBits, demanding all bits from only some vector eleme...
virtual bool findOptimalMemOpLowering(std::vector< EVT > &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS, unsigned SrcAS, const AttributeList &FuncAttributes) const
Determines the optimal series of memory ops to replace the memset / memcpy.
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
void expandSADDSUBO(SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::S(ADD|SUB)O.
SDValue expandVPBITREVERSE(SDNode *N, SelectionDAG &DAG) const
Expand VP_BITREVERSE nodes.
SDValue expandABS(SDNode *N, SelectionDAG &DAG, bool IsNegative=false) const
Expand ABS nodes.
SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_* into an explicit calculation.
bool ShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const
Check to see if the specified operand of the specified instruction is a constant integer.
virtual const char * getTargetNodeName(unsigned Opcode) const
This method returns the name of a target specific DAG node.
bool expandFP_TO_UINT(SDNode *N, SDValue &Result, SDValue &Chain, SelectionDAG &DAG) const
Expand float to UINT conversion.
bool parametersInCSRMatch(const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask, const SmallVectorImpl< CCValAssign > &ArgLocs, const SmallVectorImpl< SDValue > &OutVals) const
Check whether parameters to a call that are passed in callee saved registers are the same as from the...
virtual bool SimplifyDemandedVectorEltsForTargetNode(SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth=0) const
Attempt to simplify any target nodes based on the demanded vector elements, returning true on success...
bool expandREM(SDNode *Node, SDValue &Result, SelectionDAG &DAG) const
Expand an SREM or UREM using SDIV/UDIV or SDIVREM/UDIVREM, if legal.
std::pair< SDValue, SDValue > expandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG) const
Expands an unaligned load to 2 half-size loads for an integer, and possibly more for vectors.
virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, SelectionDAG &DAG) const
Lower TLS global address SDNode for target independent emulated TLS model.
SDValue expandVectorSplice(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::VECTOR_SPLICE.
virtual const char * LowerXConstraint(EVT ConstraintVT) const
Try to replace an X constraint, which matches anything, with another that has more specific requireme...
SDValue expandCTPOP(SDNode *N, SelectionDAG &DAG) const
Expand CTPOP nodes.
SDValue expandBSWAP(SDNode *N, SelectionDAG &DAG) const
Expand BSWAP nodes.
SDValue CTTZTableLookup(SDNode *N, SelectionDAG &DAG, const SDLoc &DL, EVT VT, SDValue Op, unsigned NumBitsPerElt) const
Expand CTTZ via Table Lookup.
virtual bool isKnownNeverNaNForTargetNode(SDValue Op, const SelectionDAG &DAG, bool SNaN=false, unsigned Depth=0) const
If SNaN is false,.
bool expandDIVREMByConstant(SDNode *N, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, SDValue LL=SDValue(), SDValue LH=SDValue()) const
Attempt to expand an n-bit div/rem/divrem by constant using a n/2-bit urem by constant and other arit...
SDValue getVectorSubVecPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT, EVT SubVecVT, SDValue Index) const
Get a pointer to a sub-vector of type SubVecVT at index Idx located in memory for a vector of type Ve...
virtual void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
bool isPositionIndependent() const
std::pair< StringRef, TargetLowering::ConstraintType > ConstraintPair
virtual SDValue getNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOps, bool OptForSize, NegatibleCost &Cost, unsigned Depth=0) const
Return the newly negated expression if the cost is not expensive and set the cost in Cost to indicate...
virtual ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &info, const char *constraint) const
Examine constraint string and operand type and determine a weight value.
virtual SDValue getSqrtInputTest(SDValue Operand, SelectionDAG &DAG, const DenormalMode &Mode) const
Return a target-dependent comparison result if the input operand is suitable for use with a square ro...
ConstraintGroup getConstraintPreferences(AsmOperandInfo &OpInfo) const
Given an OpInfo with list of constraints codes as strings, return a sorted Vector of pairs of constra...
bool expandFP_TO_SINT(SDNode *N, SDValue &Result, SelectionDAG &DAG) const
Expand float(f32) to SINT(i64) conversion.
virtual SDValue SimplifyMultipleUseDemandedBitsForTargetNode(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, SelectionDAG &DAG, unsigned Depth) const
More limited version of SimplifyDemandedBits that can be used to "look through" ops that don't contri...
virtual SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Glue, const SDLoc &DL, const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const
SDValue buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0, SDValue N1, MutableArrayRef< int > Mask, SelectionDAG &DAG) const
Tries to build a legal vector shuffle using the provided parameters or equivalent variations.
virtual SDValue getPICJumpTableRelocBase(SDValue Table, SelectionDAG &DAG) const
Returns relocation base for the given PIC jumptable.
std::pair< SDValue, SDValue > scalarizeVectorLoad(LoadSDNode *LD, SelectionDAG &DAG) const
Turn load of vector type into a load of the individual elements.
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
void forceExpandWideMUL(SelectionDAG &DAG, const SDLoc &dl, bool Signed, EVT WideVT, const SDValue LL, const SDValue LH, const SDValue RL, const SDValue RH, SDValue &Lo, SDValue &Hi) const
forceExpandWideMUL - Unconditionally expand a MUL into either a libcall or brute force via a wide mul...
virtual bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0) const
Attempt to simplify any target nodes based on the demanded bits/elts, returning true on success.
virtual bool isDesirableToCommuteXorWithShift(const SDNode *N) const
Return true if it is profitable to combine an XOR of a logical shift to create a logical shift of NOT...
TargetLowering(const TargetLowering &)=delete
virtual bool shouldSimplifyDemandedVectorElts(SDValue Op, const TargetLoweringOpt &TLO) const
Return true if the target supports simplifying demanded vector elements by converting them to undefs.
bool isConstFalseVal(SDValue N) const
Return if the N is a constant or constant vector equal to the false value from getBooleanContents().
SDValue BuildUDIV(SDNode *N, SelectionDAG &DAG, bool IsAfterLegalization, SmallVectorImpl< SDNode * > &Created) const
Given an ISD::UDIV node expressing a divide by constant, return a DAG expression to select that will ...
SDValue IncrementMemoryAddress(SDValue Addr, SDValue Mask, const SDLoc &DL, EVT DataVT, SelectionDAG &DAG, bool IsCompressedMemory) const
Increments memory address Addr according to the type of the value DataVT that should be stored.
bool verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const
bool isInTailCallPosition(SelectionDAG &DAG, SDNode *Node, SDValue &Chain) const
Check whether a given call node is in tail position within its function.
virtual AsmOperandInfoVector ParseConstraints(const DataLayout &DL, const TargetRegisterInfo *TRI, const CallBase &Call) const
Split up the constraint string from the inline assembly value into the specific constraints and their...
virtual bool isSplatValueForTargetNode(SDValue Op, const APInt &DemandedElts, APInt &UndefElts, const SelectionDAG &DAG, unsigned Depth=0) const
Return true if vector Op has the same value across all DemandedElts, indicating any elements which ma...
SDValue expandRoundInexactToOdd(EVT ResultVT, SDValue Op, const SDLoc &DL, SelectionDAG &DAG) const
Truncate Op to ResultVT.
SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, bool foldBooleans, DAGCombinerInfo &DCI, const SDLoc &dl) const
Try to simplify a setcc built with the specified operands and cc.
SDValue expandFunnelShift(SDNode *N, SelectionDAG &DAG) const
Expand funnel shift.
virtual bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const
Return true if folding a constant offset with the given GlobalAddress is legal.
bool LegalizeSetCCCondCode(SelectionDAG &DAG, EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC, SDValue Mask, SDValue EVL, bool &NeedInvert, const SDLoc &dl, SDValue &Chain, bool IsSignaling=false) const
Legalize a SETCC or VP_SETCC with given LHS and RHS and condition code CC on the current target.
bool isExtendedTrueVal(const ConstantSDNode *N, EVT VT, bool SExt) const
Return if N is a True value when extended to VT.
bool ShrinkDemandedOp(SDValue Op, unsigned BitWidth, const APInt &DemandedBits, TargetLoweringOpt &TLO) const
Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.
bool isConstTrueVal(SDValue N) const
Return if the N is a constant or constant vector equal to the true value from getBooleanContents().
SDValue expandVPCTPOP(SDNode *N, SelectionDAG &DAG) const
Expand VP_CTPOP nodes.
SDValue expandFixedPointDiv(unsigned Opcode, const SDLoc &dl, SDValue LHS, SDValue RHS, unsigned Scale, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]DIVFIX[SAT].
SDValue getVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT, SDValue Index) const
Get a pointer to vector element Idx located in memory for a vector of type VecVT starting at a base a...
virtual void ComputeConstraintToUse(AsmOperandInfo &OpInfo, SDValue Op, SelectionDAG *DAG=nullptr) const
Determines the constraint code and constraint type to use for the specific AsmOperandInfo,...
virtual void CollectTargetIntrinsicOperands(const CallInst &I, SmallVectorImpl< SDValue > &Ops, SelectionDAG &DAG) const
SDValue expandVPCTTZ(SDNode *N, SelectionDAG &DAG) const
Expand VP_CTTZ/VP_CTTZ_ZERO_UNDEF nodes.
virtual const Constant * getTargetConstantFromLoad(LoadSDNode *LD) const
This method returns the constant pool value that will be loaded by LD.
SDValue expandFP_ROUND(SDNode *Node, SelectionDAG &DAG) const
Expand round(fp) to fp conversion.
SDValue createSelectForFMINNUM_FMAXNUM(SDNode *Node, SelectionDAG &DAG) const
Try to convert the fminnum/fmaxnum to a compare/select sequence.
SDValue expandROT(SDNode *N, bool AllowVectorOps, SelectionDAG &DAG) const
Expand rotations.
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
SDValue expandFMINNUM_FMAXNUM(SDNode *N, SelectionDAG &DAG) const
Expand fminnum/fmaxnum into fminnum_ieee/fmaxnum_ieee with quieted inputs.
virtual bool isGAPlusOffset(SDNode *N, const GlobalValue *&GA, int64_t &Offset) const
Returns true (and the GlobalValue and the offset) if the node is a GlobalAddress + offset.
virtual bool isGuaranteedNotToBeUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, unsigned Depth) const
Return true if this function can prove that Op is never poison and, if PoisonOnly is false,...
virtual unsigned getJumpTableEncoding() const
Return the entry encoding for a jump table in the current function.
void expandShiftParts(SDNode *N, SDValue &Lo, SDValue &Hi, SelectionDAG &DAG) const
Expand shift-by-parts.
virtual SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const
This method will be invoked for all target nodes and for any target-independent nodes that the target...
virtual bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const
Return true if Op can create undef or poison from non-undef & non-poison operands.
SDValue expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[U|S]MULFIX[SAT].
SDValue expandIntMINMAX(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US][MIN|MAX].
virtual void computeKnownBitsForTargetInstr(GISelKnownBits &Analysis, Register R, KnownBits &Known, const APInt &DemandedElts, const MachineRegisterInfo &MRI, unsigned Depth=0) const
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
void expandUADDSUBO(SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::U(ADD|SUB)O.
SDValue BuildSDIV(SDNode *N, SelectionDAG &DAG, bool IsAfterLegalization, SmallVectorImpl< SDNode * > &Created) const
Given an ISD::SDIV node expressing a divide by constant, return a DAG expression to select that will ...
virtual SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl< SDNode * > &Created) const
Targets may override this function to provide custom SDIV lowering for power-of-2 denominators.
virtual SDValue BuildSREMPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl< SDNode * > &Created) const
Targets may override this function to provide custom SREM lowering for power-of-2 denominators.
bool expandUINT_TO_FP(SDNode *N, SDValue &Result, SDValue &Chain, SelectionDAG &DAG) const
Expand UINT(i64) to double(f64) conversion.
bool expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl, SDValue LHS, SDValue RHS, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL or [US]MUL_LOHI of n-bit values into two or four nodes, respectively,...
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:76
bool isPositionIndependent() const
const Triple & getTargetTriple() const
TargetOptions Options
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
iterator_range< regclass_iterator > regclasses() const
virtual StringRef getRegAsmName(MCRegister Reg) const
Return the assembly name for Reg.
bool isTypeLegalForClass(const TargetRegisterClass &RC, MVT T) const
Return true if the given TargetRegisterClass has the ValueType T.
bool isOSBinFormatCOFF() const
Tests whether the OS uses the COFF binary format.
Definition: Triple.h:708
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition: TypeSize.h:330
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
const fltSemantics & getFltSemantics() const
bool isSingleValueType() const
Return true if the type is a valid type for a register in codegen.
Definition: Type.h:287
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
Definition: Type.h:302
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:129
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:228
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
constexpr ScalarTy getFixedValue() const
Definition: TypeSize.h:187
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition: TypeSize.h:171
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition: TypeSize.h:168
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
APInt ScaleBitMask(const APInt &A, unsigned NewBitWidth, bool MatchAllBits=false)
Splat/Merge neighboring bits to widen/narrow the bitmask represented by.
Definition: APInt.cpp:3011
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
@ Fast
Attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:41
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:40
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:750
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:236
@ CTLZ_ZERO_UNDEF
Definition: ISDOpcodes.h:723
@ FGETSIGN
INT = FGETSIGN(FP) - Return the sign bit of the specified floating point value as an integer 0/1 valu...
Definition: ISDOpcodes.h:497
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition: ISDOpcodes.h:250
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition: ISDOpcodes.h:559
@ BSWAP
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:714
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
Definition: ISDOpcodes.h:367
@ ConstantFP
Definition: ISDOpcodes.h:77
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:269
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
Definition: ISDOpcodes.h:487
@ FMAXNUM_IEEE
Definition: ISDOpcodes.h:979
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:239
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:1031
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:373
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:783
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:483
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition: ISDOpcodes.h:199
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:543
@ FADD
Simple binary floating point operators.
Definition: ISDOpcodes.h:390
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:820
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:255
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:903
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:229
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
Definition: ISDOpcodes.h:380
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
Definition: ISDOpcodes.h:1400
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:774
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
Definition: ISDOpcodes.h:662
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
Definition: ISDOpcodes.h:620
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:722
@ FNEG
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:930
@ SSUBO
Same for subtraction.
Definition: ISDOpcodes.h:327
@ BRIND
BRIND - Indirect branch.
Definition: ISDOpcodes.h:1052
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition: ISDOpcodes.h:500
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
Definition: ISDOpcodes.h:507
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition: ISDOpcodes.h:349
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:727
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
Definition: ISDOpcodes.h:222
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition: ISDOpcodes.h:627
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition: ISDOpcodes.h:208
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:323
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition: ISDOpcodes.h:651
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:705
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:600
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition: ISDOpcodes.h:573
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimum or maximum on two values,...
Definition: ISDOpcodes.h:978
@ STRICT_FMAXNUM
Definition: ISDOpcodes.h:424
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:535
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:780
@ STRICT_FMINNUM
Definition: ISDOpcodes.h:425
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:742
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:971
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
Definition: ISDOpcodes.h:359
@ SMULO
Same for multiplication.
Definition: ISDOpcodes.h:331
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition: ISDOpcodes.h:809
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:798
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition: ISDOpcodes.h:674
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:386
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:888
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:736
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:303
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:442
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
Definition: ISDOpcodes.h:984
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:836
@ TargetConstant
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification,...
Definition: ISDOpcodes.h:158
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:680
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition: ISDOpcodes.h:184
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
Definition: ISDOpcodes.h:657
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:279
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition: ISDOpcodes.h:524
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:52
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
Definition: ISDOpcodes.h:612
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:869
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:831
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition: ISDOpcodes.h:855
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:786
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
Definition: ISDOpcodes.h:763
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition: ISDOpcodes.h:340
@ AssertZext
Definition: ISDOpcodes.h:62
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition: ISDOpcodes.h:192
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:515
bool isBuildVectorOfConstantSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantSDNode or undef.
NodeType getExtForLoadExtType(bool IsFP, LoadExtType)
bool matchUnaryPredicate(SDValue Op, std::function< bool(ConstantSDNode *)> Match, bool AllowUndefs=false)
Hook for matching ConstantSDNode predicate.
bool isZEXTLoad(const SDNode *N)
Returns true if the specified node is a ZEXTLOAD.
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
bool isTrueWhenEqual(CondCode Cond)
Return true if the specified condition returns true if the two operands to the condition are equal.
Definition: ISDOpcodes.h:1581
unsigned getUnorderedFlavor(CondCode Cond)
This function returns 0 if the condition is always false if an operand is a NaN, 1 if the condition i...
Definition: ISDOpcodes.h:1586
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
Definition: ISDOpcodes.h:1556
bool matchBinaryPredicate(SDValue LHS, SDValue RHS, std::function< bool(ConstantSDNode *, ConstantSDNode *)> Match, bool AllowUndefs=false, bool AllowTypeMismatch=false)
Attempt to match a binary predicate against a pair of scalar/splat constants or every element of a pa...
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1523
NodeType getVecReduceBaseOpcode(unsigned VecReduceOpcode)
Get underlying scalar opcode for VECREDUCE opcode.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:1503
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:456
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1731
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
FPClassTest invertFPClassTestIfSimpler(FPClassTest Test)
Evaluates if the specified FP class test is better performed as the inverse (i.e.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
T bit_ceil(T Value)
Returns the smallest integral power of two no smaller than Value if Value is nonzero.
Definition: bit.h:342
ConstantFPSDNode * isConstOrConstSplatFP(SDValue N, bool AllowUndefs=false)
Returns the SDNode if it is a constant splat BuildVector or constant float.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1738
bool getShuffleDemandedElts(int SrcWidth, ArrayRef< int > Mask, const APInt &DemandedElts, APInt &DemandedLHS, APInt &DemandedRHS, bool AllowUndefElts=false)
Transform a shuffle mask's output demanded element mask into demanded element masks for the 2 operand...
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:313
bool isBitwiseNot(SDValue V, bool AllowUndefs=false)
Returns true if V is a bitwise not operation.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:264
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
auto find_if_not(R &&Range, UnaryPredicate P)
Definition: STLExtras.h:1763
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:156
bool isOneOrOneSplat(SDValue V, bool AllowUndefs=false)
Return true if the value is a constant 1 integer or a splatted vector of a constant 1 integer (with n...
@ Mod
The access may modify the value stored in memory.
@ Other
Any other memory.
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
@ And
Bitwise or logical AND of integers.
@ Add
Sum of integers.
DWARFExpression::Operation Op
ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
bool isConstFalseVal(const TargetLowering &TLI, int64_t Val, bool IsVector, bool IsFP)
Definition: Utils.cpp:1516
constexpr unsigned BitWidth
Definition: BitmaskEnum.h:191
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
bool isNullFPConstant(SDValue V)
Returns true if V is an FP constant with a value of positive zero.
APFloat neg(APFloat X)
Returns the negated value of the argument.
Definition: APFloat.h:1387
unsigned Log2(Align A)
Returns the log2 of the alignment.
Definition: Alignment.h:208
uint64_t alignDown(uint64_t Value, uint64_t Align, uint64_t Skew=0)
Returns the largest uint64_t less than or equal to Value and is Skew mod Align.
Definition: MathExtras.h:428
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
constexpr uint64_t NextPowerOf2(uint64_t A)
Returns the next power of two (in 64-bits) that is strictly greater than A.
Definition: MathExtras.h:349
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define N
static constexpr roundingMode rmNearestTiesToEven
Definition: APFloat.h:230
static constexpr roundingMode rmTowardZero
Definition: APFloat.h:234
opStatus
IEEE-754R 7: Default exception handling.
Definition: APFloat.h:246
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Represent subnormal handling kind for floating point instruction inputs and outputs.
DenormalModeKind Input
Denormal treatment kind for floating point instruction inputs in the default floating-point environme...
@ PreserveSign
The sign of a flushed-to-zero number is preserved in the sign of 0.
@ PositiveZero
Denormals are flushed to positive zero.
@ IEEE
IEEE-754 denormal numbers preserved.
constexpr bool inputsAreZero() const
Return true if input denormals must be implicitly treated as 0.
Extended Value Type.
Definition: ValueTypes.h:34
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:380
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:136
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition: ValueTypes.h:73
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
Definition: ValueTypes.h:120
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
Definition: ValueTypes.h:274
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:290
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition: ValueTypes.h:146
ElementCount getVectorElementCount() const
Definition: ValueTypes.h:340
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:358
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:233
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
Definition: ValueTypes.h:349
uint64_t getScalarSizeInBits() const
Definition: ValueTypes.h:370
EVT getHalfSizedIntegerVT(LLVMContext &Context) const
Finds the smallest simple value type that is greater than or equal to half the width of this EVT.
Definition: ValueTypes.h:415
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
Definition: ValueTypes.h:455
TypeSize getStoreSizeInBits() const
Return the number of bits overwritten by a store of the specified value type.
Definition: ValueTypes.h:397
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:306
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
Definition: ValueTypes.h:366
bool isFixedLengthVector() const
Definition: ValueTypes.h:177
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:167
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:313
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:202
bool isRound() const
Return true if the size is a power-of-two number of bytes.
Definition: ValueTypes.h:238
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
Definition: ValueTypes.h:173
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:318
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
Definition: ValueTypes.h:101
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:326
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition: ValueTypes.h:298
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
Definition: ValueTypes.h:438
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:151
ConstraintPrefix Type
Type - The basic type of the constraint: input/output/clobber/label.
Definition: InlineAsm.h:126
int MatchingInput
MatchingInput - If this is not -1, this is an output constraint where an input constraint is required...
Definition: InlineAsm.h:136
ConstraintCodeVector Codes
Code - The constraint code, either the register name (in braces) or the constraint letter/number.
Definition: InlineAsm.h:154
SubConstraintInfoVector multipleAlternatives
multipleAlternatives - If there are multiple alternative constraints, this array will contain them.
Definition: InlineAsm.h:161
bool isIndirect
isIndirect - True if this operand is an indirect operand.
Definition: InlineAsm.h:150
bool hasMatchingInput() const
hasMatchingInput - Return true if this is an output constraint that has a matching input constraint.
Definition: InlineAsm.h:140
static KnownBits makeConstant(const APInt &C)
Create known bits from a known constant.
Definition: KnownBits.h:297
KnownBits anyextOrTrunc(unsigned BitWidth) const
Return known bits for an "any" extension or truncation of the value we're tracking.
Definition: KnownBits.h:182
unsigned countMinSignBits() const
Returns the number of times the sign bit is replicated into the other bits.
Definition: KnownBits.h:251
static KnownBits smax(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for smax(LHS, RHS).
Definition: KnownBits.cpp:208
bool isNonNegative() const
Returns true if this value is known to be non-negative.
Definition: KnownBits.h:104
unsigned countMinTrailingZeros() const
Returns the minimum number of trailing zero bits.
Definition: KnownBits.h:238
bool isUnknown() const
Returns true if we don't know any bits.
Definition: KnownBits.h:63
KnownBits trunc(unsigned BitWidth) const
Return known bits for a truncation of the value we're tracking.
Definition: KnownBits.h:157
bool hasConflict() const
Returns true if there is conflicting information.
Definition: KnownBits.h:47
static std::optional< bool > sge(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_SGE result.
Definition: KnownBits.cpp:530
unsigned countMaxPopulation() const
Returns the maximum number of bits that could be one.
Definition: KnownBits.h:285
KnownBits concat(const KnownBits &Lo) const
Concatenate the bits from Lo onto the bottom of *this.
Definition: KnownBits.h:229
unsigned getBitWidth() const
Get the bit width of this value.
Definition: KnownBits.h:40
static KnownBits umax(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for umax(LHS, RHS).
Definition: KnownBits.cpp:184
KnownBits zext(unsigned BitWidth) const
Return known bits for a zero extension of the value we're tracking.
Definition: KnownBits.h:168
void resetAll()
Resets the known state of all bits.
Definition: KnownBits.h:71
KnownBits unionWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for either this or RHS or both.
Definition: KnownBits.h:317
KnownBits intersectWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for both this and RHS.
Definition: KnownBits.h:307
KnownBits sext(unsigned BitWidth) const
Return known bits for a sign extension of the value we're tracking.
Definition: KnownBits.h:176
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
Definition: KnownBits.h:244
static KnownBits smin(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for smin(LHS, RHS).
Definition: KnownBits.cpp:221
static std::optional< bool > ugt(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_UGT result.
Definition: KnownBits.cpp:496
static std::optional< bool > slt(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_SLT result.
Definition: KnownBits.cpp:536
static KnownBits computeForAddSub(bool Add, bool NSW, bool NUW, const KnownBits &LHS, const KnownBits &RHS)
Compute known bits resulting from adding LHS and RHS.
Definition: KnownBits.cpp:57
static std::optional< bool > ult(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_ULT result.
Definition: KnownBits.cpp:512
static std::optional< bool > ule(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_ULE result.
Definition: KnownBits.cpp:516
bool isNegative() const
Returns true if this value is known to be negative.
Definition: KnownBits.h:101
static KnownBits mul(const KnownBits &LHS, const KnownBits &RHS, bool NoUndefSelfMultiply=false)
Compute known bits resulting from multiplying LHS and RHS.
Definition: KnownBits.cpp:765
KnownBits anyext(unsigned BitWidth) const
Return known bits for an "any" extension of the value we're tracking, where we don't know anything ab...
Definition: KnownBits.h:163
static std::optional< bool > sle(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_SLE result.
Definition: KnownBits.cpp:540
static std::optional< bool > sgt(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_SGT result.
Definition: KnownBits.cpp:520
unsigned countMinPopulation() const
Returns the number of bits known to be one.
Definition: KnownBits.h:282
static std::optional< bool > uge(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_UGE result.
Definition: KnownBits.cpp:506
static KnownBits umin(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for umin(LHS, RHS).
Definition: KnownBits.cpp:202
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
These are IR-level optimization flags that may be propagated to SDNodes.
bool hasNoUnsignedWrap() const
bool hasNoSignedWrap() const
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
Magic data for optimising signed division by a constant.
static SignedDivisionByConstantInfo get(const APInt &D)
Calculate the magic numbers required to implement a signed integer division by a constant as a sequen...
This contains information for each constraint that we are lowering.
MVT ConstraintVT
The ValueType for the operand value.
TargetLowering::ConstraintType ConstraintType
Information about the constraint code, e.g.
std::string ConstraintCode
This contains the actual string for the code, like "m".
Value * CallOperandVal
If this is the result output operand or a clobber, this is null, otherwise it is the incoming operand...
unsigned getMatchedOperand() const
If this is an input matching constraint, this method returns the output operand it matches.
bool isMatchingInputConstraint() const
Return true of this is an input operand that is a matching constraint like "4".
This structure contains all information that is necessary for lowering calls.
CallLoweringInfo & setIsPostTypeLegalization(bool Value=true)
CallLoweringInfo & setLibCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList)
CallLoweringInfo & setDiscardResult(bool Value=true)
CallLoweringInfo & setZExtResult(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setSExtResult(bool Value=true)
CallLoweringInfo & setNoReturn(bool Value=true)
CallLoweringInfo & setChain(SDValue InChain)
void CommitTargetLoweringOpt(const TargetLoweringOpt &TLO)
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setIsPostTypeLegalization(bool Value=true)
MakeLibCallOptions & setSExt(bool Value=true)
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT, bool Value=true)
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
bool CombineTo(SDValue O, SDValue N)
Magic data for optimising unsigned division by a constant.
static UnsignedDivisionByConstantInfo get(const APInt &D, unsigned LeadingZeros=0, bool AllowEvenDivisorOptimization=true)
Calculate the magic numbers required to implement an unsigned integer division by a constant as a seq...