LLVM 24.0.0git
TargetLowering.cpp
Go to the documentation of this file.
1//===-- TargetLowering.cpp - Implement the TargetLowering class -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the TargetLowering class.
10//
11//===----------------------------------------------------------------------===//
12
14#include "llvm/ADT/STLExtras.h"
27#include "llvm/IR/DataLayout.h"
30#include "llvm/IR/LLVMContext.h"
31#include "llvm/MC/MCAsmInfo.h"
32#include "llvm/MC/MCExpr.h"
38#include <cctype>
39#include <deque>
40#include <list>
41using namespace llvm;
42using namespace llvm::SDPatternMatch;
43
44/// NOTE: The TargetMachine owns TLOF.
48
49// Define the virtual destructor out-of-line for build efficiency.
51
52const char *TargetLowering::getTargetNodeName(unsigned Opcode) const {
53 return nullptr;
54}
55
59
60/// Check whether a given call node is in tail position within its function. If
61/// so, it sets Chain to the input chain of the tail call.
63 SDValue &Chain) const {
65
66 // First, check if tail calls have been disabled in this function.
67 if (F.getFnAttribute("disable-tail-calls").getValueAsBool())
68 return false;
69
70 // Conservatively require the attributes of the call to match those of
71 // the return. Ignore following attributes because they don't affect the
72 // call sequence.
73 AttrBuilder CallerAttrs(F.getContext(), F.getAttributes().getRetAttrs());
74 for (const auto &Attr : {Attribute::Alignment, Attribute::Dereferenceable,
75 Attribute::DereferenceableOrNull, Attribute::NoAlias,
76 Attribute::NonNull, Attribute::NoUndef,
77 Attribute::Range, Attribute::NoFPClass})
78 CallerAttrs.removeAttribute(Attr);
79
80 if (CallerAttrs.hasAttributes())
81 return false;
82
83 // It's not safe to eliminate the sign / zero extension of the return value.
84 if (CallerAttrs.contains(Attribute::ZExt) ||
85 CallerAttrs.contains(Attribute::SExt))
86 return false;
87
88 // Check if the only use is a function return node.
89 return isUsedByReturnOnly(Node, Chain);
90}
91
93 const uint32_t *CallerPreservedMask,
94 const SmallVectorImpl<CCValAssign> &ArgLocs,
95 const SmallVectorImpl<SDValue> &OutVals) const {
96 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
97 const CCValAssign &ArgLoc = ArgLocs[I];
98 if (!ArgLoc.isRegLoc())
99 continue;
100 MCRegister Reg = ArgLoc.getLocReg();
101 // Only look at callee saved registers.
102 if (MachineOperand::clobbersPhysReg(CallerPreservedMask, Reg))
103 continue;
104 // Check that we pass the value used for the caller.
105 // (We look for a CopyFromReg reading a virtual register that is used
106 // for the function live-in value of register Reg)
107 SDValue Value = OutVals[I];
108 if (Value->getOpcode() == ISD::AssertZext)
109 Value = Value.getOperand(0);
110 if (Value->getOpcode() != ISD::CopyFromReg)
111 return false;
112 Register ArgReg = cast<RegisterSDNode>(Value->getOperand(1))->getReg();
113 if (MRI.getLiveInPhysReg(ArgReg) != Reg)
114 return false;
115 }
116 return true;
117}
118
119static bool paramHasAttr(const CallBase &Call, unsigned ArgIdx,
120 Attribute::AttrKind Kind) {
121 return Call.paramHasAttr(ArgIdx, Kind);
122}
123
124static bool paramHasAttr(const AttributeList &Attrs, unsigned ArgIdx,
125 Attribute::AttrKind Kind) {
126 return Attrs.hasParamAttr(ArgIdx, Kind);
127}
128
129static MaybeAlign getParamStackAlign(const CallBase &Call, unsigned ArgIdx) {
130 return Call.getParamStackAlign(ArgIdx);
131}
132
134 unsigned ArgIdx) {
135 return Attrs.getParamStackAlignment(ArgIdx);
136}
137
138static MaybeAlign getParamAlign(const CallBase &Call, unsigned ArgIdx) {
139 return Call.getParamAlign(ArgIdx);
140}
141
142static MaybeAlign getParamAlign(const AttributeList &Attrs, unsigned ArgIdx) {
143 return Attrs.getParamAlignment(ArgIdx);
144}
145
146/// Set CallLoweringInfo attribute flags based on a call instruction
147/// and called function attributes.
148template <typename SourceT>
150 const SourceT &Src, unsigned ArgIdx) {
151 Entry.IsSExt = paramHasAttr(Src, ArgIdx, Attribute::SExt);
152 Entry.IsZExt = paramHasAttr(Src, ArgIdx, Attribute::ZExt);
153 Entry.IsNoExt = paramHasAttr(Src, ArgIdx, Attribute::NoExt);
154 Entry.IsInReg = paramHasAttr(Src, ArgIdx, Attribute::InReg);
155 Entry.IsSRet = paramHasAttr(Src, ArgIdx, Attribute::StructRet);
156 Entry.IsNest = paramHasAttr(Src, ArgIdx, Attribute::Nest);
157 Entry.IsByVal = paramHasAttr(Src, ArgIdx, Attribute::ByVal);
158 Entry.IsPreallocated = paramHasAttr(Src, ArgIdx, Attribute::Preallocated);
159 Entry.IsInAlloca = paramHasAttr(Src, ArgIdx, Attribute::InAlloca);
160 Entry.IsReturned = paramHasAttr(Src, ArgIdx, Attribute::Returned);
161 Entry.IsSwiftSelf = paramHasAttr(Src, ArgIdx, Attribute::SwiftSelf);
162 Entry.IsSwiftAsync = paramHasAttr(Src, ArgIdx, Attribute::SwiftAsync);
163 Entry.IsSwiftError = paramHasAttr(Src, ArgIdx, Attribute::SwiftError);
164 Entry.Alignment = getParamStackAlign(Src, ArgIdx);
165 Entry.IndirectType = nullptr;
166 assert(Entry.IsByVal + Entry.IsPreallocated + Entry.IsInAlloca +
167 Entry.IsSRet <=
168 1 &&
169 "multiple ABI attributes?");
170 if (Entry.IsByVal) {
171 Entry.IndirectType = Src.getParamByValType(ArgIdx);
172 if (!Entry.Alignment)
173 Entry.Alignment = getParamAlign(Src, ArgIdx);
174 }
175 if (Entry.IsPreallocated)
176 Entry.IndirectType = Src.getParamPreallocatedType(ArgIdx);
177 if (Entry.IsInAlloca)
178 Entry.IndirectType = Src.getParamInAllocaType(ArgIdx);
179 if (Entry.IsSRet)
180 Entry.IndirectType = Src.getParamStructRetType(ArgIdx);
181}
182
184 unsigned ArgIdx) {
185 setArgListEntryAttributes(*this, *Call, ArgIdx);
186}
187
189 unsigned ArgIdx) {
190 setArgListEntryAttributes(*this, Attrs, ArgIdx);
191}
192
194 Attribute::AttrKind Attr) {
195 if (Attr != Attribute::None) {
196 if (Attr == Attribute::SExt)
197 IsSExt = true;
198 else if (Attr == Attribute::ZExt)
199 IsZExt = true;
200 else if (Attr == Attribute::NoExt)
201 IsNoExt = true;
202 else
203 llvm_unreachable("Unexpected extension attribute!");
204 }
205}
206
209 const AttributeList &FuncAttrs,
211 // TODO: This assumes each parameter maps to exactly one operand node, which
212 // does not hold when an argument requires type splitting.
213 assert(Ops.size() == FuncTy->getNumParams() &&
214 "argument count does not match the function type");
215 ArgListTy Args;
216 Args.reserve(Ops.size());
217 for (unsigned I = 0, E = FuncTy->getNumParams(); I != E; ++I) {
218 ArgListEntry Entry(Ops[I], FuncTy->getParamType(I));
219 Entry.setAttributes(FuncAttrs, I);
220 Args.push_back(Entry);
221 }
222 return Args;
223}
224
225std::pair<SDValue, SDValue> TargetLowering::makeLibCall(
226 SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef<SDValue> Ops,
227 MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain) const {
228 return makeLibCall(DAG, DAG.getLibcalls().getLibcallImpl(LC), RetVT, Ops,
229 CallOptions, dl, Chain);
230}
231
232/// Generate a libcall taking the given operands as arguments and returning a
233/// result of type RetVT.
234std::pair<SDValue, SDValue>
235TargetLowering::makeLibCall(SelectionDAG &DAG, RTLIB::LibcallImpl LibcallImpl,
237 MakeLibCallOptions CallOptions, const SDLoc &dl,
238 SDValue InChain) const {
239 if (LibcallImpl == RTLIB::Unsupported)
240 reportFatalInternalError("unsupported library call operation");
241
242 if (!InChain)
243 InChain = DAG.getEntryNode();
244
246 Args.reserve(Ops.size());
247
248 ArrayRef<Type *> OpsTypeOverrides = CallOptions.OpsTypeOverrides;
249 for (unsigned i = 0; i < Ops.size(); ++i) {
250 SDValue NewOp = Ops[i];
251 Type *Ty = i < OpsTypeOverrides.size() && OpsTypeOverrides[i]
252 ? OpsTypeOverrides[i]
253 : NewOp.getValueType().getTypeForEVT(*DAG.getContext());
254 TargetLowering::ArgListEntry Entry(NewOp, Ty);
255 if (CallOptions.IsSoften)
256 Entry.OrigTy =
257 CallOptions.OpsVTBeforeSoften[i].getTypeForEVT(*DAG.getContext());
258
259 Entry.IsSExt =
260 shouldSignExtendTypeInLibCall(Entry.Ty, CallOptions.IsSigned);
261 Entry.IsZExt = !Entry.IsSExt;
262
263 if (CallOptions.IsSoften &&
265 Entry.IsSExt = Entry.IsZExt = false;
266 }
267 Args.push_back(Entry);
268 }
269
270 SDValue Callee =
271 DAG.getExternalSymbol(LibcallImpl, getPointerTy(DAG.getDataLayout()));
272
273 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
274 Type *OrigRetTy = RetTy;
276 bool signExtend = shouldSignExtendTypeInLibCall(RetTy, CallOptions.IsSigned);
277 bool zeroExtend = !signExtend;
278
279 if (CallOptions.IsSoften) {
280 OrigRetTy = CallOptions.RetVTBeforeSoften.getTypeForEVT(*DAG.getContext());
282 signExtend = zeroExtend = false;
283 }
284
285 CLI.setDebugLoc(dl)
286 .setChain(InChain)
288 RetTy, OrigRetTy, Callee, std::move(Args))
289 .setNoReturn(CallOptions.DoesNotReturn)
292 .setSExtResult(signExtend)
293 .setZExtResult(zeroExtend);
294 return LowerCallTo(CLI);
295}
296
298 LLVMContext &Context, std::vector<EVT> &MemOps, unsigned Limit,
299 const MemOp &Op, unsigned DstAS, unsigned SrcAS,
300 const AttributeList &FuncAttributes, EVT *LargestVT) const {
301 EVT VT = getOptimalMemOpType(Context, Op, FuncAttributes);
302
303 if (VT == MVT::Other) {
304 // Use the largest integer type whose alignment constraints are satisfied.
305 VT = MVT::LAST_INTEGER_VALUETYPE;
306 if (Op.isFixedDstAlign()) {
307 bool LoadsFromSrc = Op.isMemcpyOrMemmove() && !Op.isMemcpyStrSrc();
308 while (VT != MVT::i8) {
309 unsigned VTSize = VT.getSizeInBits() / 8;
310 bool DstOk =
311 Op.getDstAlign() >= VTSize ||
312 allowsMisalignedMemoryAccesses(VT, DstAS, Op.getDstAlign());
313 bool SrcOk =
314 !LoadsFromSrc || Op.getSrcAlign() >= VTSize ||
315 allowsMisalignedMemoryAccesses(VT, SrcAS, Op.getSrcAlign());
316 if (DstOk && SrcOk)
317 break;
319 }
320 }
321 assert(VT.isInteger());
322
323 // Find the largest legal integer type.
324 MVT LVT = MVT::LAST_INTEGER_VALUETYPE;
325 while (!isTypeLegal(LVT))
326 LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
327 assert(LVT.isInteger());
328
329 // If the type we've chosen is larger than the largest legal integer type
330 // then use the largest legal type.
331 if (VT.bitsGT(LVT))
332 VT = LVT;
333 }
334
335 unsigned NumMemOps = 0;
336 uint64_t Size = Op.size();
337 while (Size) {
338 unsigned VTSize = VT.getSizeInBits() / 8;
339 while (VTSize > Size) {
340 // For now, only use non-vector load / store's for the left-over pieces.
341 EVT NewVT = VT;
342 unsigned NewVTSize;
343
344 bool Found = false;
345 if (VT.isVector() || VT.isFloatingPoint()) {
346 NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;
349 Found = true;
350 else if (NewVT == MVT::i64 &&
352 isSafeMemOpType(MVT::f64)) {
353 // i64 is usually not legal on 32-bit targets, but f64 may be.
354 NewVT = MVT::f64;
355 Found = true;
356 }
357 }
358
359 if (!Found) {
360 do {
361 NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
362 if (NewVT == MVT::i8)
363 break;
364 } while (!isSafeMemOpType(NewVT.getSimpleVT()));
365 }
366 NewVTSize = NewVT.getSizeInBits() / 8;
367
368 // If the new VT cannot cover all of the remaining bits, then consider
369 // issuing a (or a pair of) unaligned and overlapping load / store.
370 unsigned Fast;
371 if (NumMemOps && !Op.isVolatile() && NewVTSize < Size &&
373 VT, DstAS, Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1),
375 Fast)
376 VTSize = Size;
377 else {
378 VT = NewVT;
379 VTSize = NewVTSize;
380 }
381 }
382
383 if (++NumMemOps > Limit)
384 return false;
385
386 MemOps.push_back(VT);
387 Size -= VTSize;
388 }
389
390 return true;
391}
392
393/// Soften the operands of a comparison. This code is shared among BR_CC,
394/// SELECT_CC, and SETCC handlers.
396 SDValue &NewLHS, SDValue &NewRHS,
397 ISD::CondCode &CCCode,
398 const SDLoc &dl, const SDValue OldLHS,
399 const SDValue OldRHS) const {
400 SDValue Chain;
401 return softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, dl, OldLHS,
402 OldRHS, Chain);
403}
404
405/// Select the libcall and the condition code to test its result against 0 for
406/// an ordered floating-point compare. \p BoolLC is the boolean helper (result
407/// is 0/1). \p TriStateLC is the per-predicate three-way helper and \p
408/// GenericLC the generic single-symbol three-way helper (both return -1/0/1,
409/// tested against 0 with \p TriStateCC). The boolean form is preferred, then
410/// the per-predicate three-way, then the generic three-way.
411static std::pair<RTLIB::Libcall, ISD::CondCode>
412selectFPCmpLibcall(const LibcallLoweringInfo &Libcalls, RTLIB::Libcall BoolLC,
413 RTLIB::Libcall TriStateLC, RTLIB::Libcall GenericLC,
414 ISD::CondCode TriStateCC) {
415 if (Libcalls.getLibcallImpl(BoolLC) != RTLIB::Unsupported)
416 return {BoolLC, ISD::SETNE};
417 if (Libcalls.getLibcallImpl(TriStateLC) != RTLIB::Unsupported)
418 return {TriStateLC, TriStateCC};
419 return {GenericLC, TriStateCC};
420}
421
423 SDValue &NewLHS, SDValue &NewRHS,
424 ISD::CondCode &CCCode,
425 const SDLoc &dl, const SDValue OldLHS,
426 const SDValue OldRHS,
427 SDValue &Chain,
428 bool IsSignaling) const {
429 // FIXME: Currently we cannot really respect all IEEE predicates due to libgcc
430 // not supporting it. We can update this code when libgcc provides such
431 // functions.
432
433 assert((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128)
434 && "Unsupported setcc type!");
435
436 // Expand into one or more soft-fp libcall(s).
437 ISD::CondCode OrigCCCode = CCCode;
438 RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL;
440 bool ShouldInvertCC = false;
441
442 // Expand a compare libcall family name (e.g. OEQ, FCMP3_PRED_OEQ) to the
443 // RTLIB::Libcall for VT.
444#define FP_CMP_LIBCALL(BASE) \
445 RTLIB::getFPLibCall(VT, RTLIB::BASE##_F32, RTLIB::BASE##_F64, \
446 RTLIB::UNKNOWN_LIBCALL, RTLIB::BASE##_F128, \
447 RTLIB::BASE##_PPCF128)
448
449 switch (CCCode) {
450 case ISD::SETEQ:
451 case ISD::SETOEQ:
452 std::tie(LC1, CC1) = selectFPCmpLibcall(
453 DAG.getLibcalls(), FP_CMP_LIBCALL(OEQ), FP_CMP_LIBCALL(FCMP3_PRED_OEQ),
454 FP_CMP_LIBCALL(FCMP3), ISD::SETEQ);
455 break;
456 case ISD::SETNE:
457 case ISD::SETUNE:
458 std::tie(LC1, CC1) = selectFPCmpLibcall(
459 DAG.getLibcalls(), FP_CMP_LIBCALL(UNE), FP_CMP_LIBCALL(FCMP3_PRED_UNE),
460 FP_CMP_LIBCALL(FCMP3), ISD::SETNE);
461 // Some ABIs (e.g. AEABI) provide neither a not-equal nor a three-way
462 // compare; obtain not-equal (UNE = !OEQ) by inverting ordered-equal.
463 if (DAG.getLibcalls().getLibcallImpl(LC1) == RTLIB::Unsupported) {
464 std::tie(LC1, CC1) = selectFPCmpLibcall(
465 DAG.getLibcalls(), FP_CMP_LIBCALL(OEQ),
466 FP_CMP_LIBCALL(FCMP3_PRED_OEQ), FP_CMP_LIBCALL(FCMP3), ISD::SETEQ);
467 ShouldInvertCC = true;
468 }
469 break;
470 case ISD::SETGE:
471 case ISD::SETOGE:
472 std::tie(LC1, CC1) = selectFPCmpLibcall(
473 DAG.getLibcalls(), FP_CMP_LIBCALL(OGE), FP_CMP_LIBCALL(FCMP3_PRED_OGE),
474 FP_CMP_LIBCALL(FCMP3), ISD::SETGE);
475 break;
476 case ISD::SETLT:
477 case ISD::SETOLT:
478 std::tie(LC1, CC1) = selectFPCmpLibcall(
479 DAG.getLibcalls(), FP_CMP_LIBCALL(OLT), FP_CMP_LIBCALL(FCMP3_PRED_OLT),
480 FP_CMP_LIBCALL(FCMP3), ISD::SETLT);
481 break;
482 case ISD::SETLE:
483 case ISD::SETOLE:
484 std::tie(LC1, CC1) = selectFPCmpLibcall(
485 DAG.getLibcalls(), FP_CMP_LIBCALL(OLE), FP_CMP_LIBCALL(FCMP3_PRED_OLE),
486 FP_CMP_LIBCALL(FCMP3), ISD::SETLE);
487 break;
488 case ISD::SETGT:
489 case ISD::SETOGT:
490 std::tie(LC1, CC1) = selectFPCmpLibcall(
491 DAG.getLibcalls(), FP_CMP_LIBCALL(OGT), FP_CMP_LIBCALL(FCMP3_PRED_OGT),
492 FP_CMP_LIBCALL(FCMP3), ISD::SETGT);
493 break;
494 case ISD::SETO:
495 ShouldInvertCC = true;
496 [[fallthrough]];
497 case ISD::SETUO:
498 // Unordered is a boolean everywhere (__unordXf2 returns 0/1).
499 LC1 = FP_CMP_LIBCALL(UO);
500 CC1 = ISD::SETNE;
501 break;
502 case ISD::SETONE:
503 // SETONE = O && UNE
504 ShouldInvertCC = true;
505 [[fallthrough]];
506 case ISD::SETUEQ:
507 LC1 = FP_CMP_LIBCALL(UO);
508 CC1 = ISD::SETNE;
509 std::tie(LC2, CC2) = selectFPCmpLibcall(
510 DAG.getLibcalls(), FP_CMP_LIBCALL(OEQ), FP_CMP_LIBCALL(FCMP3_PRED_OEQ),
511 FP_CMP_LIBCALL(FCMP3), ISD::SETEQ);
512 break;
513 default:
514 // Invert CC for unordered comparisons, handled by the ordered inverse.
515 ShouldInvertCC = true;
516 switch (CCCode) {
517 case ISD::SETULT:
518 std::tie(LC1, CC1) = selectFPCmpLibcall(
519 DAG.getLibcalls(), FP_CMP_LIBCALL(OGE),
520 FP_CMP_LIBCALL(FCMP3_PRED_OGE), FP_CMP_LIBCALL(FCMP3), ISD::SETGE);
521 break;
522 case ISD::SETULE:
523 std::tie(LC1, CC1) = selectFPCmpLibcall(
524 DAG.getLibcalls(), FP_CMP_LIBCALL(OGT),
525 FP_CMP_LIBCALL(FCMP3_PRED_OGT), FP_CMP_LIBCALL(FCMP3), ISD::SETGT);
526 break;
527 case ISD::SETUGT:
528 std::tie(LC1, CC1) = selectFPCmpLibcall(
529 DAG.getLibcalls(), FP_CMP_LIBCALL(OLE),
530 FP_CMP_LIBCALL(FCMP3_PRED_OLE), FP_CMP_LIBCALL(FCMP3), ISD::SETLE);
531 break;
532 case ISD::SETUGE:
533 std::tie(LC1, CC1) = selectFPCmpLibcall(
534 DAG.getLibcalls(), FP_CMP_LIBCALL(OLT),
535 FP_CMP_LIBCALL(FCMP3_PRED_OLT), FP_CMP_LIBCALL(FCMP3), ISD::SETLT);
536 break;
537 default:
538 llvm_unreachable("Do not know how to soften this setcc!");
539 }
540 }
541
542#undef FP_CMP_LIBCALL
543
544 // Use the target specific return value for comparison lib calls.
546 SDValue Ops[2] = {NewLHS, NewRHS};
548 EVT OpsVT[2] = { OldLHS.getValueType(),
549 OldRHS.getValueType() };
550 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT);
551
552 auto ReportNoLibcall = [&]() {
553 DAG.getContext()->emitError(
554 Twine("no libcall available to soften floating-point ") +
555 ISD::getCondCodeName(OrigCCCode) + " compare with type " +
556 VT.getEVTString());
557 NewLHS = DAG.getPOISON(RetVT);
558 NewRHS = DAG.getConstant(0, dl, RetVT);
559 CCCode = ISD::SETNE;
560 };
561
562 // Check availability before makeLibCall, which fatally errors otherwise.
563 RTLIB::LibcallImpl LC1Impl = DAG.getLibcalls().getLibcallImpl(LC1);
564 if (LC1Impl == RTLIB::Unsupported) {
565 ReportNoLibcall();
566 return;
567 }
568
569 auto Call = makeLibCall(DAG, LC1Impl, RetVT, Ops, CallOptions, dl, Chain);
570 NewLHS = Call.first;
571 NewRHS = DAG.getConstant(0, dl, RetVT);
572
573 CCCode = CC1;
574 if (ShouldInvertCC) {
575 assert(RetVT.isInteger());
576 CCCode = getSetCCInverse(CCCode, RetVT);
577 }
578
579 if (LC2 == RTLIB::UNKNOWN_LIBCALL) {
580 // Update Chain.
581 Chain = Call.second;
582 } else {
583 RTLIB::LibcallImpl LC2Impl = DAG.getLibcalls().getLibcallImpl(LC2);
584 if (LC2Impl == RTLIB::Unsupported) {
585 ReportNoLibcall();
586 return;
587 }
588
589 assert(CCCode == (ShouldInvertCC ? ISD::SETEQ : ISD::SETNE) &&
590 "unordered call should be simple boolean");
591
592 EVT SetCCVT =
593 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), RetVT);
595 NewLHS = DAG.getNode(ISD::AssertZext, dl, RetVT, Call.first,
596 DAG.getValueType(MVT::i1));
597 }
598
599 SDValue Tmp = DAG.getSetCC(dl, SetCCVT, NewLHS, NewRHS, CCCode);
600 auto Call2 = makeLibCall(DAG, LC2Impl, RetVT, Ops, CallOptions, dl, Chain);
601 CCCode = CC2;
602 if (ShouldInvertCC)
603 CCCode = getSetCCInverse(CCCode, RetVT);
604 NewLHS = DAG.getSetCC(dl, SetCCVT, Call2.first, NewRHS, CCCode);
605 if (Chain)
606 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Call.second,
607 Call2.second);
608 NewLHS = DAG.getNode(ShouldInvertCC ? ISD::AND : ISD::OR, dl,
609 Tmp.getValueType(), Tmp, NewLHS);
610 NewRHS = SDValue();
611 }
612}
613
614/// Return the entry encoding for a jump table in the current function. The
615/// returned value is a member of the MachineJumpTableInfo::JTEntryKind enum.
617 // In non-pic modes, just use the address of a block.
620
621 // Otherwise, use a label difference.
623}
624
629
630/// This returns the relocation base for the given PIC jumptable, the same as
631/// getPICJumpTableRelocBase, but as an MCExpr.
632const MCExpr *
634 unsigned JTI,MCContext &Ctx) const{
635 // The normal PIC reloc base is the label at the start of the jump table.
636 return MCSymbolRefExpr::create(MF->getJTISymbol(JTI, Ctx), Ctx);
637}
638
640 SDValue Addr, int JTI,
641 SelectionDAG &DAG) const {
642 SDValue Chain = Value;
643 // Jump table debug info is only needed if CodeView is enabled.
645 Chain = DAG.getJumpTableDebugInfo(JTI, Chain, dl);
646 }
647 return DAG.getNode(ISD::BRIND, dl, MVT::Other, Chain, Addr);
648}
649
650bool
652 const TargetMachine &TM = getTargetMachine();
653 const GlobalValue *GV = GA->getGlobal();
654
655 // If the address is not even local to this DSO we will have to load it from
656 // a got and then add the offset.
657 if (!TM.shouldAssumeDSOLocal(GV))
658 return false;
659
660 // If the code is position independent we will have to add a base register.
662 return false;
663
664 // Otherwise we can do it.
665 return true;
666}
667
668//===----------------------------------------------------------------------===//
669// Optimization Methods
670//===----------------------------------------------------------------------===//
671
672/// If the specified instruction has a constant integer operand and there are
673/// bits set in that constant that are not demanded, then clear those bits and
674/// return true.
676 const APInt &DemandedBits,
677 const APInt &DemandedElts,
678 TargetLoweringOpt &TLO) const {
679 SDLoc DL(Op);
680 unsigned Opcode = Op.getOpcode();
681
682 // Early-out if we've ended up calling an undemanded node, leave this to
683 // constant folding.
684 if (DemandedBits.isZero() || DemandedElts.isZero())
685 return false;
686
687 // Do target-specific constant optimization.
688 if (targetShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
689 return TLO.New.getNode();
690
691 // FIXME: ISD::SELECT, ISD::SELECT_CC
692 switch (Opcode) {
693 default:
694 break;
695 case ISD::XOR:
696 case ISD::AND:
697 case ISD::OR: {
698 auto *Op1C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
699 if (!Op1C || Op1C->isOpaque())
700 return false;
701
702 // If this is a 'not' op, don't touch it because that's a canonical form.
703 const APInt &C = Op1C->getAPIntValue();
704 if (Opcode == ISD::XOR && DemandedBits.isSubsetOf(C))
705 return false;
706
707 if (!C.isSubsetOf(DemandedBits)) {
708 EVT VT = Op.getValueType();
709 SDValue NewC = TLO.DAG.getConstant(DemandedBits & C, DL, VT);
710 SDValue NewOp = TLO.DAG.getNode(Opcode, DL, VT, Op.getOperand(0), NewC,
711 Op->getFlags());
712 return TLO.CombineTo(Op, NewOp);
713 }
714
715 break;
716 }
717 }
718
719 return false;
720}
721
723 const APInt &DemandedBits,
724 TargetLoweringOpt &TLO) const {
725 EVT VT = Op.getValueType();
726 APInt DemandedElts = VT.isVector()
728 : APInt(1, 1);
729 return ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO);
730}
731
732/// Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.
733/// This uses isTruncateFree/isZExtFree and ANY_EXTEND for the widening cast,
734/// but it could be generalized for targets with other types of implicit
735/// widening casts.
737 const APInt &DemandedBits,
738 TargetLoweringOpt &TLO) const {
739 assert(Op.getNumOperands() == 2 &&
740 "ShrinkDemandedOp only supports binary operators!");
741 assert(Op.getNode()->getNumValues() == 1 &&
742 "ShrinkDemandedOp only supports nodes with one result!");
743
744 EVT VT = Op.getValueType();
745 SelectionDAG &DAG = TLO.DAG;
746 SDLoc dl(Op);
747
748 // Early return, as this function cannot handle vector types.
749 if (VT.isVector())
750 return false;
751
752 assert(Op.getOperand(0).getValueType().getScalarSizeInBits() == BitWidth &&
753 Op.getOperand(1).getValueType().getScalarSizeInBits() == BitWidth &&
754 "ShrinkDemandedOp only supports operands that have the same size!");
755
756 // Don't do this if the node has another user, which may require the
757 // full value.
758 if (!Op.getNode()->hasOneUse())
759 return false;
760
761 // Search for the smallest integer type with free casts to and from
762 // Op's type. For expedience, just check power-of-2 integer types.
763 unsigned DemandedSize = DemandedBits.getActiveBits();
764 for (unsigned SmallVTBits = llvm::bit_ceil(DemandedSize);
765 SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(SmallVTBits)) {
766 EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), SmallVTBits);
767 if (isTruncateFree(Op, SmallVT) && isZExtFree(SmallVT, VT)) {
768 // We found a type with free casts.
769
770 // If the operation has the 'disjoint' flag, then the
771 // operands on the new node are also disjoint.
772 SDNodeFlags Flags(Op->getFlags().hasDisjoint() ? SDNodeFlags::Disjoint
774 unsigned Opcode = Op.getOpcode();
775 if (Opcode == ISD::PTRADD) {
776 // It isn't a ptradd anymore if it doesn't operate on the entire
777 // pointer.
778 Opcode = ISD::ADD;
779 }
780 SDValue X = DAG.getNode(
781 Opcode, dl, SmallVT,
782 DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(0)),
783 DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(1)), Flags);
784 assert(DemandedSize <= SmallVTBits && "Narrowed below demanded bits?");
785 SDValue Z = DAG.getNode(ISD::ANY_EXTEND, dl, VT, X);
786 return TLO.CombineTo(Op, Z);
787 }
788 }
789 return false;
790}
791
793 DAGCombinerInfo &DCI) const {
794 SelectionDAG &DAG = DCI.DAG;
795 TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
796 !DCI.isBeforeLegalizeOps());
798
799 bool Simplified = SimplifyDemandedBits(Op, DemandedBits, Known, TLO);
800 if (Simplified) {
801 DCI.AddToWorklist(Op.getNode());
803 }
804 return Simplified;
805}
806
808 const APInt &DemandedElts,
809 DAGCombinerInfo &DCI) const {
810 SelectionDAG &DAG = DCI.DAG;
811 TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
812 !DCI.isBeforeLegalizeOps());
814
815 bool Simplified =
816 SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO);
817 if (Simplified) {
818 DCI.AddToWorklist(Op.getNode());
820 }
821 return Simplified;
822}
823
827 unsigned Depth,
828 bool AssumeSingleUse) const {
829 EVT VT = Op.getValueType();
830
831 // Since the number of lanes in a scalable vector is unknown at compile time,
832 // we track one bit which is implicitly broadcast to all lanes. This means
833 // that all lanes in a scalable vector are considered demanded.
834 APInt DemandedElts = VT.isFixedLengthVector()
836 : APInt(1, 1);
837 return SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO, Depth,
838 AssumeSingleUse);
839}
840
841// TODO: Under what circumstances can we create nodes? Constant folding?
843 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
844 SelectionDAG &DAG, unsigned Depth) const {
845 EVT VT = Op.getValueType();
846
847 // Limit search depth.
849 return SDValue();
850
851 // Ignore UNDEFs.
852 if (Op.isUndef())
853 return SDValue();
854
855 // Not demanding any bits/elts from Op.
856 if (DemandedBits == 0 || DemandedElts == 0)
857 return DAG.getUNDEF(VT);
858
859 bool IsLE = DAG.getDataLayout().isLittleEndian();
860 unsigned NumElts = DemandedElts.getBitWidth();
861 unsigned BitWidth = DemandedBits.getBitWidth();
862 KnownBits LHSKnown, RHSKnown;
863 switch (Op.getOpcode()) {
864 case ISD::Constant: {
865 const APInt &Value = Op->getAsAPIntVal();
866 if (!Value.isZero() && (Value & DemandedBits).isZero())
867 return DAG.getConstant(0, SDLoc(Op), VT);
868 break;
869 }
870 case ISD::BITCAST: {
871 if (VT.isScalableVector())
872 return SDValue();
873
874 SDValue Src = peekThroughBitcasts(Op.getOperand(0));
875 EVT SrcVT = Src.getValueType();
876 EVT DstVT = Op.getValueType();
877 if (SrcVT == DstVT)
878 return Src;
879
880 unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
881 unsigned NumDstEltBits = DstVT.getScalarSizeInBits();
882 if (NumSrcEltBits == NumDstEltBits)
884 Src, DemandedBits, DemandedElts, DAG, Depth + 1))
885 return DAG.getBitcast(DstVT, V);
886
887 if (SrcVT.isVector() && (NumDstEltBits % NumSrcEltBits) == 0) {
888 unsigned Scale = NumDstEltBits / NumSrcEltBits;
889 unsigned NumSrcElts = SrcVT.getVectorNumElements();
890 APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);
891 for (unsigned i = 0; i != Scale; ++i) {
892 unsigned EltOffset = IsLE ? i : (Scale - 1 - i);
893 unsigned BitOffset = EltOffset * NumSrcEltBits;
894 DemandedSrcBits |= DemandedBits.extractBits(NumSrcEltBits, BitOffset);
895 }
896 // Recursive calls below may turn not demanded elements into poison, so we
897 // need to demand all smaller source elements that maps to a demanded
898 // destination element.
899 APInt DemandedSrcElts = APIntOps::ScaleBitMask(DemandedElts, NumSrcElts);
900
902 Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))
903 return DAG.getBitcast(DstVT, V);
904 }
905
906 // TODO - bigendian once we have test coverage.
907 if (IsLE && (NumSrcEltBits % NumDstEltBits) == 0) {
908 unsigned Scale = NumSrcEltBits / NumDstEltBits;
909 unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
910 APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);
911 APInt DemandedSrcElts = APInt::getZero(NumSrcElts);
912 for (unsigned i = 0; i != NumElts; ++i)
913 if (DemandedElts[i]) {
914 unsigned Offset = (i % Scale) * NumDstEltBits;
915 DemandedSrcBits.insertBits(DemandedBits, Offset);
916 DemandedSrcElts.setBit(i / Scale);
917 }
918
920 Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))
921 return DAG.getBitcast(DstVT, V);
922 }
923
924 break;
925 }
927 if (!VT.isScalableVector() && !DemandedElts[0])
928 return DAG.getPOISON(VT);
929 break;
930 }
931 case ISD::AND: {
932 LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
933 RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
934
935 // If all of the demanded bits are known 1 on one side, return the other.
936 // These bits cannot contribute to the result of the 'and' in this
937 // context.
938 if (DemandedBits.isSubsetOf(LHSKnown.Zero | RHSKnown.One))
939 return Op.getOperand(0);
940 if (DemandedBits.isSubsetOf(RHSKnown.Zero | LHSKnown.One))
941 return Op.getOperand(1);
942 break;
943 }
944 case ISD::OR: {
945 LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
946 RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
947
948 // If all of the demanded bits are known zero on one side, return the
949 // other. These bits cannot contribute to the result of the 'or' in this
950 // context.
951 if (DemandedBits.isSubsetOf(LHSKnown.One | RHSKnown.Zero))
952 return Op.getOperand(0);
953 if (DemandedBits.isSubsetOf(RHSKnown.One | LHSKnown.Zero))
954 return Op.getOperand(1);
955 break;
956 }
957 case ISD::XOR: {
958 LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
959 RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
960
961 // If all of the demanded bits are known zero on one side, return the
962 // other.
963 if (DemandedBits.isSubsetOf(RHSKnown.Zero))
964 return Op.getOperand(0);
965 if (DemandedBits.isSubsetOf(LHSKnown.Zero))
966 return Op.getOperand(1);
967 break;
968 }
969 case ISD::ADD:
970 case ISD::MUL:
971 case ISD::SMIN:
972 case ISD::SMAX:
973 case ISD::UMIN:
974 case ISD::UMAX: {
975 if (DAG.isIdentityElement(Op.getOpcode(), Op->getFlags(), Op.getOperand(1),
976 DemandedElts, 1, Depth + 1))
977 return Op.getOperand(0);
978
979 if (DAG.isIdentityElement(Op.getOpcode(), Op->getFlags(), Op.getOperand(0),
980 DemandedElts, 0, Depth + 1))
981 return Op.getOperand(1);
982 break;
983 }
984 case ISD::SHL: {
985 // If we are only demanding sign bits then we can use the shift source
986 // directly.
987 if (std::optional<unsigned> MaxSA =
988 DAG.getValidMaximumShiftAmount(Op, DemandedElts, Depth + 1)) {
989 SDValue Op0 = Op.getOperand(0);
990 unsigned ShAmt = *MaxSA;
991 unsigned NumSignBits =
992 DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
993 unsigned UpperDemandedBits = BitWidth - DemandedBits.countr_zero();
994 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
995 return Op0;
996 }
997 break;
998 }
999 case ISD::SRL: {
1000 // If we are only demanding sign bits then we can use the shift source
1001 // directly.
1002 if (std::optional<unsigned> MaxSA =
1003 DAG.getValidMaximumShiftAmount(Op, DemandedElts, Depth + 1)) {
1004 SDValue Op0 = Op.getOperand(0);
1005 unsigned ShAmt = *MaxSA;
1006 // Must already be signbits in DemandedBits bounds, and can't demand any
1007 // shifted in zeroes.
1008 if (DemandedBits.countl_zero() >= ShAmt) {
1009 unsigned NumSignBits =
1010 DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
1011 if (DemandedBits.countr_zero() >= (BitWidth - NumSignBits))
1012 return Op0;
1013 }
1014 }
1015 break;
1016 }
1017 case ISD::SETCC: {
1018 SDValue Op0 = Op.getOperand(0);
1019 SDValue Op1 = Op.getOperand(1);
1020 ISD::CondCode CC = Op.getOperandCondCode(2);
1021 // If (1) we only need the sign-bit, (2) the setcc operands are the same
1022 // width as the setcc result, and (3) the result of a setcc conforms to 0 or
1023 // -1, we may be able to bypass the setcc.
1024 if (DemandedBits.isSignMask() &&
1028 // If we're testing X < 0, then this compare isn't needed - just use X!
1029 // FIXME: We're limiting to integer types here, but this should also work
1030 // if we don't care about FP signed-zero. The use of SETLT with FP means
1031 // that we don't care about NaNs.
1032 if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
1034 return Op0;
1035 }
1036 break;
1037 }
1039 // If none of the extended bits are demanded, eliminate the sextinreg.
1040 SDValue Op0 = Op.getOperand(0);
1041 EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
1042 unsigned ExBits = ExVT.getScalarSizeInBits();
1043 if (DemandedBits.getActiveBits() <= ExBits &&
1045 return Op0;
1046 // If the input is already sign extended, just drop the extension.
1047 unsigned NumSignBits = DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
1048 if (NumSignBits >= (BitWidth - ExBits + 1))
1049 return Op0;
1050 break;
1051 }
1055 if (VT.isScalableVector())
1056 return SDValue();
1057
1058 // If we only want the lowest element and none of extended bits, then we can
1059 // return the bitcasted source vector.
1060 SDValue Src = Op.getOperand(0);
1061 EVT SrcVT = Src.getValueType();
1062 EVT DstVT = Op.getValueType();
1063 if (IsLE && DemandedElts == 1 &&
1064 DstVT.getSizeInBits() == SrcVT.getSizeInBits() &&
1065 DemandedBits.getActiveBits() <= SrcVT.getScalarSizeInBits()) {
1066 return DAG.getBitcast(DstVT, Src);
1067 }
1068 break;
1069 }
1071 if (VT.isScalableVector())
1072 return SDValue();
1073
1074 // If we don't demand the inserted element, return the base vector.
1075 SDValue Vec = Op.getOperand(0);
1076 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
1077 EVT VecVT = Vec.getValueType();
1078 if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements()) &&
1079 !DemandedElts[CIdx->getZExtValue()])
1080 return Vec;
1081 break;
1082 }
1083 case ISD::INSERT_SUBVECTOR: {
1084 if (VT.isScalableVector())
1085 return SDValue();
1086
1087 SDValue Vec = Op.getOperand(0);
1088 SDValue Sub = Op.getOperand(1);
1089 uint64_t Idx = Op.getConstantOperandVal(2);
1090 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
1091 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
1092 // If we don't demand the inserted subvector, return the base vector.
1093 if (DemandedSubElts == 0)
1094 return Vec;
1095 break;
1096 }
1097 case ISD::VECTOR_SHUFFLE: {
1098 assert(!VT.isScalableVector());
1099 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
1100
1101 // If all the demanded elts are from one operand and are inline,
1102 // then we can use the operand directly.
1103 bool AllUndef = true, IdentityLHS = true, IdentityRHS = true;
1104 for (unsigned i = 0; i != NumElts; ++i) {
1105 int M = ShuffleMask[i];
1106 if (M < 0 || !DemandedElts[i])
1107 continue;
1108 AllUndef = false;
1109 IdentityLHS &= (M == (int)i);
1110 IdentityRHS &= ((M - NumElts) == i);
1111 }
1112
1113 if (AllUndef)
1114 return DAG.getUNDEF(Op.getValueType());
1115 if (IdentityLHS)
1116 return Op.getOperand(0);
1117 if (IdentityRHS)
1118 return Op.getOperand(1);
1119 break;
1120 }
1121 default:
1122 // TODO: Probably okay to remove after audit; here to reduce change size
1123 // in initial enablement patch for scalable vectors
1124 if (VT.isScalableVector())
1125 return SDValue();
1126
1127 if (Op.getOpcode() >= ISD::BUILTIN_OP_END)
1129 Op, DemandedBits, DemandedElts, DAG, Depth))
1130 return V;
1131 break;
1132 }
1133 return SDValue();
1134}
1135
1138 unsigned Depth) const {
1139 EVT VT = Op.getValueType();
1140 // Since the number of lanes in a scalable vector is unknown at compile time,
1141 // we track one bit which is implicitly broadcast to all lanes. This means
1142 // that all lanes in a scalable vector are considered demanded.
1143 APInt DemandedElts = VT.isFixedLengthVector()
1145 : APInt(1, 1);
1146 return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
1147 Depth);
1148}
1149
1151 SDValue Op, const APInt &DemandedElts, SelectionDAG &DAG,
1152 unsigned Depth) const {
1153 APInt DemandedBits = APInt::getAllOnes(Op.getScalarValueSizeInBits());
1154 return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
1155 Depth);
1156}
1157
1158// Find the smallest power-2 type that is at least 8 bits and can hold a value
1159// of type VT that has KnownBits known sign/zero bits. Returns std::nullopt if
1160// that is wider than VT.
1161static std::optional<EVT> getNarrowAVGType(EVT VT, unsigned KnownBits,
1162 LLVMContext &Ctx) {
1163 unsigned MinWidth =
1164 std::max<unsigned>(VT.getScalarSizeInBits() - KnownBits, 8);
1165 EVT NVT = EVT::getIntegerVT(Ctx, llvm::bit_ceil(MinWidth));
1167 return std::nullopt;
1168 if (VT.isVector())
1169 NVT = EVT::getVectorVT(Ctx, NVT, VT.getVectorElementCount());
1170 return NVT;
1171}
1172
1173// Attempt to form ext(avgfloor(A, B)) from shr(add(ext(A), ext(B)), 1).
1174// or to form ext(avgceil(A, B)) from shr(add(ext(A), ext(B), 1), 1).
1177 const TargetLowering &TLI,
1178 const APInt &DemandedBits,
1179 const APInt &DemandedElts, unsigned Depth) {
1180 assert((Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SRA) &&
1181 "SRL or SRA node is required here!");
1182 // Is the right shift using an immediate value of 1?
1183 ConstantSDNode *N1C = isConstOrConstSplat(Op.getOperand(1), DemandedElts);
1184 if (!N1C || !N1C->isOne())
1185 return SDValue();
1186
1187 // We are looking for an avgfloor
1188 // add(ext, ext)
1189 // or one of these as a avgceil
1190 // add(add(ext, ext), 1)
1191 // add(add(ext, 1), ext)
1192 // add(ext, add(ext, 1))
1193 SDValue Add = Op.getOperand(0);
1194 if (Add.getOpcode() != ISD::ADD)
1195 return SDValue();
1196
1197 SDValue ExtOpA = Add.getOperand(0);
1198 SDValue ExtOpB = Add.getOperand(1);
1199 SDValue Add2;
1200 auto MatchOperands = [&](SDValue Op1, SDValue Op2, SDValue Op3, SDValue A) {
1201 ConstantSDNode *ConstOp;
1202 if ((ConstOp = isConstOrConstSplat(Op2, DemandedElts)) &&
1203 ConstOp->isOne()) {
1204 ExtOpA = Op1;
1205 ExtOpB = Op3;
1206 Add2 = A;
1207 return true;
1208 }
1209 if ((ConstOp = isConstOrConstSplat(Op3, DemandedElts)) &&
1210 ConstOp->isOne()) {
1211 ExtOpA = Op1;
1212 ExtOpB = Op2;
1213 Add2 = A;
1214 return true;
1215 }
1216 return false;
1217 };
1218 bool IsCeil =
1219 (ExtOpA.getOpcode() == ISD::ADD &&
1220 MatchOperands(ExtOpA.getOperand(0), ExtOpA.getOperand(1), ExtOpB, ExtOpA)) ||
1221 (ExtOpB.getOpcode() == ISD::ADD &&
1222 MatchOperands(ExtOpB.getOperand(0), ExtOpB.getOperand(1), ExtOpA, ExtOpB));
1223
1224 // If the shift is signed (sra):
1225 // - Needs >= 2 sign bit for both operands.
1226 // - Needs >= 2 zero bits.
1227 // If the shift is unsigned (srl):
1228 // - Needs >= 1 zero bit for both operands.
1229 // - Needs 1 demanded bit zero and >= 2 sign bits.
1230 SelectionDAG &DAG = TLO.DAG;
1231 unsigned ShiftOpc = Op.getOpcode();
1232 bool IsSigned = false;
1233 unsigned KnownBits;
1234 unsigned NumSignedA = DAG.ComputeNumSignBits(ExtOpA, DemandedElts, Depth);
1235 unsigned NumSignedB = DAG.ComputeNumSignBits(ExtOpB, DemandedElts, Depth);
1236 unsigned NumSigned = std::min(NumSignedA, NumSignedB) - 1;
1237 unsigned NumZeroA =
1238 DAG.computeKnownBits(ExtOpA, DemandedElts, Depth).countMinLeadingZeros();
1239 unsigned NumZeroB =
1240 DAG.computeKnownBits(ExtOpB, DemandedElts, Depth).countMinLeadingZeros();
1241 unsigned NumZero = std::min(NumZeroA, NumZeroB);
1242
1243 switch (ShiftOpc) {
1244 default:
1245 llvm_unreachable("Unexpected ShiftOpc in combineShiftToAVG");
1246 case ISD::SRA: {
1247 if (NumZero >= 2 && NumSigned < NumZero) {
1248 IsSigned = false;
1249 KnownBits = NumZero;
1250 break;
1251 }
1252 if (NumSigned >= 1) {
1253 IsSigned = true;
1254 KnownBits = NumSigned;
1255 break;
1256 }
1257 return SDValue();
1258 }
1259 case ISD::SRL: {
1260 if (NumZero >= 1 && NumSigned < NumZero) {
1261 IsSigned = false;
1262 KnownBits = NumZero;
1263 break;
1264 }
1265 if (NumSigned >= 1 && DemandedBits.isSignBitClear()) {
1266 IsSigned = true;
1267 KnownBits = NumSigned;
1268 break;
1269 }
1270 return SDValue();
1271 }
1272 }
1273
1274 unsigned AVGOpc = IsCeil ? (IsSigned ? ISD::AVGCEILS : ISD::AVGCEILU)
1275 : (IsSigned ? ISD::AVGFLOORS : ISD::AVGFLOORU);
1276
1277 // Find the smallest power-2 type that is legal for this vector size and
1278 // operation, given the original type size and the number of known sign/zero
1279 // bits.
1280 EVT VT = Op.getValueType();
1281 std::optional<EVT> NarrowVT =
1283 if (!NarrowVT)
1284 return SDValue();
1285 EVT NVT = *NarrowVT;
1286 if (TLO.LegalTypes() && !TLI.isOperationLegal(AVGOpc, NVT)) {
1287 // If we could not transform, and (both) adds are nuw/nsw, we can use the
1288 // larger type size to do the transform.
1289 if (TLO.LegalOperations() && !TLI.isOperationLegal(AVGOpc, VT))
1290 return SDValue();
1291 if (DAG.willNotOverflowAdd(IsSigned, Add.getOperand(0),
1292 Add.getOperand(1)) &&
1293 (!Add2 || DAG.willNotOverflowAdd(IsSigned, Add2.getOperand(0),
1294 Add2.getOperand(1))))
1295 NVT = VT;
1296 else
1297 return SDValue();
1298 }
1299
1300 // Don't create a AVGFLOOR node with a scalar constant unless its legal as
1301 // this is likely to stop other folds (reassociation, value tracking etc.)
1302 if (!IsCeil && !TLI.isOperationLegal(AVGOpc, NVT) &&
1303 (isa<ConstantSDNode>(ExtOpA) || isa<ConstantSDNode>(ExtOpB)))
1304 return SDValue();
1305
1306 SDLoc DL(Op);
1307 SDValue ResultAVG =
1308 DAG.getNode(AVGOpc, DL, NVT, DAG.getExtOrTrunc(IsSigned, ExtOpA, DL, NVT),
1309 DAG.getExtOrTrunc(IsSigned, ExtOpB, DL, NVT));
1310 return DAG.getExtOrTrunc(IsSigned, ResultAVG, DL, VT);
1311}
1312
1313// Attempt to form ext(avgsubfloor(A, B)) from shr(sub(ext(A), ext(B)), 1).
1316 const TargetLowering &TLI,
1317 const APInt &DemandedBits,
1318 const APInt &DemandedElts, unsigned Depth) {
1319 assert((Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SRA) &&
1320 "SRL or SRA node is required here!");
1321 // Is the right shift using an immediate value of 1?
1322 ConstantSDNode *N1C = isConstOrConstSplat(Op.getOperand(1), DemandedElts);
1323 if (!N1C || !N1C->isOne())
1324 return SDValue();
1325
1326 SDValue Sub = Op.getOperand(0);
1327 if (Sub.getOpcode() != ISD::SUB)
1328 return SDValue();
1329
1330 // The result of the avgsub is sign extended to match the sra. For srl, the
1331 // top bit of the result is always zero so it must not be demanded.
1332 if (Op.getOpcode() == ISD::SRL && !DemandedBits.isSignBitClear())
1333 return SDValue();
1334
1335 SDValue A = Sub.getOperand(0);
1336 SDValue B = Sub.getOperand(1);
1337
1338 // If both operands have at least 2 sign bits or at least 1 leading zero, the
1339 // subtract can't overflow and the operands are extended from a narrower type.
1340 SelectionDAG &DAG = TLO.DAG;
1341 bool IsSigned;
1342 unsigned KnownBits;
1343 unsigned NumSigned =
1344 std::min(DAG.ComputeNumSignBits(A, DemandedElts, Depth),
1345 DAG.ComputeNumSignBits(B, DemandedElts, Depth)) -
1346 1;
1347 unsigned NumZero = std::min(
1348 DAG.computeKnownBits(A, DemandedElts, Depth).countMinLeadingZeros(),
1349 DAG.computeKnownBits(B, DemandedElts, Depth).countMinLeadingZeros());
1350 if (NumZero >= 1 && NumSigned < NumZero) {
1351 IsSigned = false;
1352 KnownBits = NumZero;
1353 } else if (NumSigned >= 1) {
1354 IsSigned = true;
1355 KnownBits = NumSigned;
1356 } else {
1357 return SDValue();
1358 }
1359
1360 unsigned AVGOpc = IsSigned ? ISD::AVGSUBFLOORS : ISD::AVGSUBFLOORU;
1361
1362 // Return true if the AVG node is legal or custom on the type VT will be
1363 // legalized to, provided that legalization doesn't change the scalar size.
1364 // That means vectors that are widened or split are accepted, but integers
1365 // that are promoted or expanded are not.
1366 LLVMContext &Ctx = *DAG.getContext();
1367 auto IsLegalAfterTypeLegalization = [&](EVT VT) {
1368 EVT LegalVT = TLI.getLegalTypeToTransformTo(Ctx, VT);
1369 return LegalVT.getScalarSizeInBits() == VT.getScalarSizeInBits() &&
1370 TLI.isOperationLegal(AVGOpc, LegalVT);
1371 };
1372
1373 // Use the narrow type if the node is legal or custom there. Otherwise fall
1374 // back to the original type, which is still correct since the subtract can't
1375 // overflow. Don't form the node if the target can't do it on either type, as
1376 // expanding it may be worse than the original subtract and shift.
1377 // FIXME: Enable for illegal types/operations like combineShiftToAVG.
1378 EVT VT = Op.getValueType();
1379 std::optional<EVT> NarrowVT = getNarrowAVGType(VT, KnownBits, Ctx);
1380 EVT NVT = VT;
1381 if (NarrowVT && IsLegalAfterTypeLegalization(*NarrowVT))
1382 NVT = *NarrowVT;
1383 else if (!IsLegalAfterTypeLegalization(VT))
1384 return SDValue();
1385
1386 // Don't create an AVGSUBFLOOR node with a scalar constant unless it's legal
1387 // as this is likely to stop other folds (reassociation, value tracking etc.)
1388 if (!TLI.isOperationLegal(AVGOpc, NVT) &&
1390 return SDValue();
1391
1392 // The difference of two unsigned values can be negative, so the result is
1393 // always sign extended.
1394 SDLoc DL(Op);
1395 SDValue ResultAVG =
1396 DAG.getNode(AVGOpc, DL, NVT, DAG.getExtOrTrunc(IsSigned, A, DL, NVT),
1397 DAG.getExtOrTrunc(IsSigned, B, DL, NVT));
1398 return DAG.getExtOrTrunc(/*IsSigned=*/true, ResultAVG, DL, VT);
1399}
1400
1401/// Look at Op. At this point, we know that only the OriginalDemandedBits of the
1402/// result of Op are ever used downstream. If we can use this information to
1403/// simplify Op, create a new simplified DAG node and return true, returning the
1404/// original and new nodes in Old and New. Otherwise, analyze the expression and
1405/// return a mask of Known bits for the expression (used to simplify the
1406/// caller). The Known bits may only be accurate for those bits in the
1407/// OriginalDemandedBits and OriginalDemandedElts.
1409 SDValue Op, const APInt &OriginalDemandedBits,
1410 const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
1411 unsigned Depth, bool AssumeSingleUse) const {
1412 unsigned BitWidth = OriginalDemandedBits.getBitWidth();
1413 assert(Op.getScalarValueSizeInBits() == BitWidth &&
1414 "Mask size mismatches value type size!");
1415
1416 // Don't know anything.
1418
1419 EVT VT = Op.getValueType();
1420 bool IsLE = TLO.DAG.getDataLayout().isLittleEndian();
1421 unsigned NumElts = OriginalDemandedElts.getBitWidth();
1422 assert((!VT.isFixedLengthVector() || NumElts == VT.getVectorNumElements()) &&
1423 "Unexpected vector size");
1424
1425 APInt DemandedBits = OriginalDemandedBits;
1426 APInt DemandedElts = OriginalDemandedElts;
1427 SDLoc dl(Op);
1428
1429 // Undef operand.
1430 if (Op.isUndef())
1431 return false;
1432
1433 // We can't simplify target constants.
1434 if (Op.getOpcode() == ISD::TargetConstant)
1435 return false;
1436
1437 if (Op.getOpcode() == ISD::Constant) {
1438 // We know all of the bits for a constant!
1439 Known = KnownBits::makeConstant(Op->getAsAPIntVal());
1440 return false;
1441 }
1442
1443 if (Op.getOpcode() == ISD::ConstantFP) {
1444 // We know all of the bits for a floating point constant!
1446 cast<ConstantFPSDNode>(Op)->getValueAPF().bitcastToAPInt());
1447 return false;
1448 }
1449
1450 // Other users may use these bits.
1451 bool HasMultiUse = false;
1452 if (!AssumeSingleUse && !Op.getNode()->hasOneUse()) {
1454 // Limit search depth.
1455 return false;
1456 }
1457 // Allow multiple uses, just set the DemandedBits/Elts to all bits.
1459 DemandedElts = APInt::getAllOnes(NumElts);
1460 HasMultiUse = true;
1461 } else if (OriginalDemandedBits == 0 || OriginalDemandedElts == 0) {
1462 // Not demanding any bits/elts from Op.
1463 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
1464 } else if (Depth >= SelectionDAG::MaxRecursionDepth) {
1465 // Limit search depth.
1466 return false;
1467 }
1468
1469 // Before folding to the LHS, recheck bits excluded from its earlier demand
1470 // using the full result demand. Mask identifies these excluded bits.
1471 // KnownMask selects the Zero/One member of the recomputed LHS known bits
1472 // that must cover Mask & DemandedBits.
1473 auto TryFoldToLHS = [&](const APInt &Mask,
1474 const APInt KnownBits::*KnownMask) {
1475 SDValue LHS = Op.getOperand(0);
1476 APInt MaskedBits = Mask & DemandedBits;
1477 if (MaskedBits.isZero())
1478 return TLO.CombineTo(Op, LHS);
1479
1480 KnownBits LHSKnown;
1481 if (SimplifyDemandedBits(LHS, DemandedBits, DemandedElts, LHSKnown, TLO,
1482 Depth + 1))
1483 return true;
1484
1485 return MaskedBits.isSubsetOf(LHSKnown.*KnownMask) && TLO.CombineTo(Op, LHS);
1486 };
1487
1488 // Try reducing the RHS demand using LHS known bits.
1489 // KnownMask selects the LHS Zero/One bits that make RHS bits irrelevant.
1490 auto TryReduceRHSDemand = [&](const KnownBits &LHSKnown,
1491 const APInt KnownBits::*KnownMask) {
1492 SDValue RHS = Op.getOperand(1);
1493 if (!RHS.hasOneUse() || isConstOrConstSplat(RHS, DemandedElts))
1494 return false;
1495
1496 // Reuse LHS facts if the RHS did not reduce its earlier demand. Otherwise,
1497 // query independent facts to avoid using bits outside that demand.
1498 SDValue LHS = Op.getOperand(0);
1499 // Keep only the selected mask, then compute DemandedBits & ~Mask in place.
1500 // This avoids copying the unused KnownBits mask and extra APInt
1501 // temporaries, which can allocate memory for masks wider than one word.
1502 APInt RHSDemandedBits =
1503 DemandedBits.intersects(Known.*KnownMask)
1504 ? TLO.DAG.computeKnownBits(LHS, DemandedElts, Depth + 1).*KnownMask
1505 : LHSKnown.*KnownMask;
1506 RHSDemandedBits.flipAllBits();
1507 RHSDemandedBits &= DemandedBits;
1508 if (RHSDemandedBits == DemandedBits)
1509 return false;
1510
1511 // A failed retry must not replace Known with reduced-demand facts.
1512 KnownBits RetryKnown;
1513 return SimplifyDemandedBits(RHS, RHSDemandedBits, DemandedElts, RetryKnown,
1514 TLO, Depth + 1);
1515 };
1516
1517 KnownBits Known2;
1518 switch (Op.getOpcode()) {
1519 case ISD::SCALAR_TO_VECTOR: {
1520 if (VT.isScalableVector())
1521 return false;
1522 if (!DemandedElts[0])
1523 return TLO.CombineTo(Op, TLO.DAG.getPOISON(VT));
1524
1525 KnownBits SrcKnown;
1526 SDValue Src = Op.getOperand(0);
1527 unsigned SrcBitWidth = Src.getScalarValueSizeInBits();
1528 APInt SrcDemandedBits = DemandedBits.zext(SrcBitWidth);
1529 if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcKnown, TLO, Depth + 1))
1530 return true;
1531
1532 // Upper elements are poison, so only get the knownbits if we just demand
1533 // the bottom element.
1534 if (DemandedElts == 1)
1535 Known = SrcKnown.anyextOrTrunc(BitWidth);
1536 break;
1537 }
1538 case ISD::BUILD_VECTOR:
1539 // Collect the known bits that are shared by every demanded element.
1540 // TODO: Call SimplifyDemandedBits for non-constant demanded elements.
1541 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
1542 return false; // Don't fall through, will infinitely loop.
1543 case ISD::SPLAT_VECTOR: {
1544 SDValue Scl = Op.getOperand(0);
1545 APInt DemandedSclBits = DemandedBits.zextOrTrunc(Scl.getValueSizeInBits());
1546 KnownBits KnownScl;
1547 if (SimplifyDemandedBits(Scl, DemandedSclBits, KnownScl, TLO, Depth + 1))
1548 return true;
1549
1550 // Implicitly truncate the bits to match the official semantics of
1551 // SPLAT_VECTOR.
1552 Known = KnownScl.trunc(BitWidth);
1553 break;
1554 }
1555 case ISD::FREEZE: {
1556 SDValue N0 = Op.getOperand(0);
1558 N0, DemandedElts, UndefPoisonKind::UndefOrPoison, Depth + 1))
1559 return TLO.CombineTo(Op, N0);
1560 break;
1561 }
1562 case ISD::LOAD: {
1563 auto *LD = cast<LoadSDNode>(Op);
1564 if (getTargetConstantFromLoad(LD)) {
1565 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
1566 return false; // Don't fall through, will infinitely loop.
1567 }
1568 if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
1569 // If this is a ZEXTLoad and we are looking at the loaded value.
1570 EVT MemVT = LD->getMemoryVT();
1571 unsigned MemBits = MemVT.getScalarSizeInBits();
1572 Known.Zero.setBitsFrom(MemBits);
1573 return false; // Don't fall through, will infinitely loop.
1574 }
1575 break;
1576 }
1578 if (VT.isScalableVector())
1579 return false;
1580 SDValue Vec = Op.getOperand(0);
1581 SDValue Scl = Op.getOperand(1);
1582 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
1583 EVT VecVT = Vec.getValueType();
1584
1585 // If index isn't constant, assume we need all vector elements AND the
1586 // inserted element.
1587 APInt DemandedVecElts(DemandedElts);
1588 if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements())) {
1589 unsigned Idx = CIdx->getZExtValue();
1590 DemandedVecElts.clearBit(Idx);
1591
1592 // Inserted element is not required.
1593 if (!DemandedElts[Idx])
1594 return TLO.CombineTo(Op, Vec);
1595 }
1596
1597 KnownBits KnownScl;
1598 unsigned NumSclBits = Scl.getScalarValueSizeInBits();
1599 APInt DemandedSclBits = DemandedBits.zextOrTrunc(NumSclBits);
1600 if (SimplifyDemandedBits(Scl, DemandedSclBits, KnownScl, TLO, Depth + 1))
1601 return true;
1602
1603 Known = KnownScl.anyextOrTrunc(BitWidth);
1604
1605 KnownBits KnownVec;
1606 if (SimplifyDemandedBits(Vec, DemandedBits, DemandedVecElts, KnownVec, TLO,
1607 Depth + 1))
1608 return true;
1609
1610 if (!!DemandedVecElts)
1611 Known = Known.intersectWith(KnownVec);
1612
1613 return false;
1614 }
1615 case ISD::INSERT_SUBVECTOR: {
1616 if (VT.isScalableVector())
1617 return false;
1618 // Demand any elements from the subvector and the remainder from the src its
1619 // inserted into.
1620 SDValue Src = Op.getOperand(0);
1621 SDValue Sub = Op.getOperand(1);
1622 uint64_t Idx = Op.getConstantOperandVal(2);
1623 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
1624 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
1625 APInt DemandedSrcElts = DemandedElts;
1626 DemandedSrcElts.clearBits(Idx, Idx + NumSubElts);
1627
1628 KnownBits KnownSub, KnownSrc;
1629 if (SimplifyDemandedBits(Sub, DemandedBits, DemandedSubElts, KnownSub, TLO,
1630 Depth + 1))
1631 return true;
1632 if (SimplifyDemandedBits(Src, DemandedBits, DemandedSrcElts, KnownSrc, TLO,
1633 Depth + 1))
1634 return true;
1635
1636 Known.setAllConflict();
1637 if (!!DemandedSubElts)
1638 Known = Known.intersectWith(KnownSub);
1639 if (!!DemandedSrcElts)
1640 Known = Known.intersectWith(KnownSrc);
1641
1642 // Attempt to avoid multi-use src if we don't need anything from it.
1643 if (!DemandedBits.isAllOnes() || !DemandedSubElts.isAllOnes() ||
1644 !DemandedSrcElts.isAllOnes()) {
1646 Sub, DemandedBits, DemandedSubElts, TLO.DAG, Depth + 1);
1648 Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);
1649 if (NewSub || NewSrc) {
1650 NewSub = NewSub ? NewSub : Sub;
1651 NewSrc = NewSrc ? NewSrc : Src;
1652 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc, NewSub,
1653 Op.getOperand(2));
1654 return TLO.CombineTo(Op, NewOp);
1655 }
1656 }
1657 break;
1658 }
1660 if (VT.isScalableVector())
1661 return false;
1662 // Offset the demanded elts by the subvector index.
1663 SDValue Src = Op.getOperand(0);
1664 if (Src.getValueType().isScalableVector())
1665 break;
1666 uint64_t Idx = Op.getConstantOperandVal(1);
1667 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
1668 APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx);
1669
1670 if (SimplifyDemandedBits(Src, DemandedBits, DemandedSrcElts, Known, TLO,
1671 Depth + 1))
1672 return true;
1673
1674 // Attempt to avoid multi-use src if we don't need anything from it.
1675 if (!DemandedBits.isAllOnes() || !DemandedSrcElts.isAllOnes()) {
1677 Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);
1678 if (DemandedSrc) {
1679 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc,
1680 Op.getOperand(1));
1681 return TLO.CombineTo(Op, NewOp);
1682 }
1683 }
1684 break;
1685 }
1686 case ISD::CONCAT_VECTORS: {
1687 if (VT.isScalableVector())
1688 return false;
1689 Known.setAllConflict();
1690 EVT SubVT = Op.getOperand(0).getValueType();
1691 unsigned NumSubVecs = Op.getNumOperands();
1692 unsigned NumSubElts = SubVT.getVectorNumElements();
1693 for (unsigned i = 0; i != NumSubVecs; ++i) {
1694 APInt DemandedSubElts =
1695 DemandedElts.extractBits(NumSubElts, i * NumSubElts);
1696 if (SimplifyDemandedBits(Op.getOperand(i), DemandedBits, DemandedSubElts,
1697 Known2, TLO, Depth + 1))
1698 return true;
1699 // Known bits are shared by every demanded subvector element.
1700 if (!!DemandedSubElts)
1701 Known = Known.intersectWith(Known2);
1702 }
1703 break;
1704 }
1705 case ISD::VECTOR_SHUFFLE: {
1706 assert(!VT.isScalableVector());
1707 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
1708
1709 // Collect demanded elements from shuffle operands..
1710 APInt DemandedLHS, DemandedRHS;
1711 if (!getShuffleDemandedElts(NumElts, ShuffleMask, DemandedElts, DemandedLHS,
1712 DemandedRHS))
1713 break;
1714
1715 if (!!DemandedLHS || !!DemandedRHS) {
1716 SDValue Op0 = Op.getOperand(0);
1717 SDValue Op1 = Op.getOperand(1);
1718
1719 Known.setAllConflict();
1720 if (!!DemandedLHS) {
1721 if (SimplifyDemandedBits(Op0, DemandedBits, DemandedLHS, Known2, TLO,
1722 Depth + 1))
1723 return true;
1724 Known = Known.intersectWith(Known2);
1725 }
1726 if (!!DemandedRHS) {
1727 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedRHS, Known2, TLO,
1728 Depth + 1))
1729 return true;
1730 Known = Known.intersectWith(Known2);
1731 }
1732
1733 // Attempt to avoid multi-use ops if we don't need anything from them.
1735 Op0, DemandedBits, DemandedLHS, TLO.DAG, Depth + 1);
1737 Op1, DemandedBits, DemandedRHS, TLO.DAG, Depth + 1);
1738 if (DemandedOp0 || DemandedOp1) {
1739 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1740 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1741 SDValue NewOp = TLO.DAG.getVectorShuffle(VT, dl, Op0, Op1, ShuffleMask);
1742 return TLO.CombineTo(Op, NewOp);
1743 }
1744 }
1745 break;
1746 }
1747 case ISD::AND: {
1748 SDValue Op0 = Op.getOperand(0);
1749 SDValue Op1 = Op.getOperand(1);
1750
1751 // If the RHS is a constant, check to see if the LHS would be zero without
1752 // using the bits from the RHS. Below, we use knowledge about the RHS to
1753 // simplify the LHS, here we're using information from the LHS to simplify
1754 // the RHS.
1755 if (ConstantSDNode *RHSC = isConstOrConstSplat(Op1, DemandedElts)) {
1756 // Do not increment Depth here; that can cause an infinite loop.
1757 KnownBits LHSKnown = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth);
1758 // If the LHS already has zeros where RHSC does, this 'and' is dead.
1759 if ((LHSKnown.Zero & DemandedBits) ==
1760 (~RHSC->getAPIntValue() & DemandedBits))
1761 return TLO.CombineTo(Op, Op0);
1762
1763 // If any of the set bits in the RHS are known zero on the LHS, shrink
1764 // the constant.
1765 if (ShrinkDemandedConstant(Op, ~LHSKnown.Zero & DemandedBits,
1766 DemandedElts, TLO))
1767 return true;
1768
1769 // Bitwise-not (xor X, -1) is a special case: we don't usually shrink its
1770 // constant, but if this 'and' is only clearing bits that were just set by
1771 // the xor, then this 'and' can be eliminated by shrinking the mask of
1772 // the xor. For example, for a 32-bit X:
1773 // and (xor (srl X, 31), -1), 1 --> xor (srl X, 31), 1
1774 if (isBitwiseNot(Op0) && Op0.hasOneUse() &&
1775 LHSKnown.One == ~RHSC->getAPIntValue()) {
1776 SDValue Xor = TLO.DAG.getNode(ISD::XOR, dl, VT, Op0.getOperand(0), Op1);
1777 return TLO.CombineTo(Op, Xor);
1778 }
1779 }
1780
1781 // (X +/- Y) & Y --> ~X & Y when Y is a power of 2 (or zero).
1782 SDValue X, Y;
1783 if (sd_match(Op,
1784 m_And(m_Value(Y),
1786 m_Sub(m_Value(X), m_Deferred(Y)))))) &&
1787 TLO.DAG.isKnownToBeAPowerOfTwo(Y, DemandedElts, /*OrZero=*/true)) {
1788 return TLO.CombineTo(
1789 Op, TLO.DAG.getNode(ISD::AND, dl, VT, TLO.DAG.getNOT(dl, X, VT), Y));
1790 }
1791
1792 // AND(INSERT_SUBVECTOR(C,X,I),M) -> INSERT_SUBVECTOR(AND(C,M),X,I)
1793 // iff 'C' is Undef/Constant and AND(X,M) == X (for DemandedBits).
1794 if (Op0.getOpcode() == ISD::INSERT_SUBVECTOR && !VT.isScalableVector() &&
1795 (Op0.getOperand(0).isUndef() ||
1797 Op0->hasOneUse()) {
1798 unsigned NumSubElts =
1800 unsigned SubIdx = Op0.getConstantOperandVal(2);
1801 APInt DemandedSub =
1802 APInt::getBitsSet(NumElts, SubIdx, SubIdx + NumSubElts);
1803 KnownBits KnownSubMask =
1804 TLO.DAG.computeKnownBits(Op1, DemandedSub & DemandedElts, Depth + 1);
1805 if (DemandedBits.isSubsetOf(KnownSubMask.One)) {
1806 SDValue NewAnd =
1807 TLO.DAG.getNode(ISD::AND, dl, VT, Op0.getOperand(0), Op1);
1808 SDValue NewInsert =
1809 TLO.DAG.getNode(ISD::INSERT_SUBVECTOR, dl, VT, NewAnd,
1810 Op0.getOperand(1), Op0.getOperand(2));
1811 return TLO.CombineTo(Op, NewInsert);
1812 }
1813 }
1814
1815 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1816 Depth + 1))
1817 return true;
1818 if (SimplifyDemandedBits(Op0, ~Known.Zero & DemandedBits, DemandedElts,
1819 Known2, TLO, Depth + 1))
1820 return true;
1821
1822 // If all of the demanded bits are known one on one side, return the other.
1823 // These bits cannot contribute to the result of the 'and'.
1824 if (DemandedBits.isSubsetOf(Known2.Zero | Known.One)) {
1825 if (TryFoldToLHS(Known.Zero, &KnownBits::Zero))
1826 return true;
1827 }
1828 if (DemandedBits.isSubsetOf(Known.Zero | Known2.One))
1829 return TLO.CombineTo(Op, Op1);
1830 // If all of the demanded bits in the inputs are known zeros, return zero.
1831 if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
1832 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, dl, VT));
1833 // If the RHS is a constant, see if we can simplify it.
1834 if (ShrinkDemandedConstant(Op, ~Known2.Zero & DemandedBits, DemandedElts,
1835 TLO))
1836 return true;
1837 // If the operation can be done in a smaller type, do so.
1839 return true;
1840
1841 // Attempt to avoid multi-use ops if we don't need anything from them.
1842 if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
1844 Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1846 Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1847 if (DemandedOp0 || DemandedOp1) {
1848 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1849 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1850 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1851 return TLO.CombineTo(Op, NewOp);
1852 }
1853 }
1854
1855 if (TryReduceRHSDemand(Known2, &KnownBits::Zero))
1856 return true;
1857
1858 Known &= Known2;
1859 break;
1860 }
1861 case ISD::OR: {
1862 SDValue Op0 = Op.getOperand(0);
1863 SDValue Op1 = Op.getOperand(1);
1864 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1865 Depth + 1)) {
1866 Op->dropFlags(SDNodeFlags::Disjoint);
1867 return true;
1868 }
1869
1870 if (SimplifyDemandedBits(Op0, ~Known.One & DemandedBits, DemandedElts,
1871 Known2, TLO, Depth + 1)) {
1872 Op->dropFlags(SDNodeFlags::Disjoint);
1873 return true;
1874 }
1875
1876 // If all of the demanded bits are known zero on one side, return the other.
1877 // These bits cannot contribute to the result of the 'or'.
1878 if (DemandedBits.isSubsetOf(Known2.One | Known.Zero)) {
1879 if (TryFoldToLHS(Known.One, &KnownBits::One)) {
1880 Op->dropFlags(SDNodeFlags::Disjoint);
1881 return true;
1882 }
1883 }
1884 if (DemandedBits.isSubsetOf(Known.One | Known2.Zero))
1885 return TLO.CombineTo(Op, Op1);
1886 // If the RHS is a constant, see if we can simplify it.
1887 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1888 return true;
1889 // If the operation can be done in a smaller type, do so.
1891 return true;
1892
1893 // Attempt to avoid multi-use ops if we don't need anything from them.
1894 if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
1896 Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1898 Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1899 if (DemandedOp0 || DemandedOp1) {
1900 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1901 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1902 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1903 return TLO.CombineTo(Op, NewOp);
1904 }
1905 }
1906
1907 // (or (and X, C1), (and (or X, Y), C2)) -> (or (and X, C1|C2), (and Y, C2))
1908 // TODO: Use SimplifyMultipleUseDemandedBits to peek through masks.
1909 SDValue X, Y, C1, C2;
1912 m_Value(C2)))))) {
1913 if (SDValue C12 =
1914 TLO.DAG.FoldConstantArithmetic(ISD::OR, dl, VT, {C1, C2})) {
1915 SDValue MaskX = TLO.DAG.getNode(ISD::AND, dl, VT, X, C12);
1916 SDValue MaskY = TLO.DAG.getNode(ISD::AND, dl, VT, Y, C2);
1917 return TLO.CombineTo(Op,
1918 TLO.DAG.getNode(ISD::OR, dl, VT, MaskX, MaskY));
1919 }
1920 }
1921
1922 if (TryReduceRHSDemand(Known2, &KnownBits::One)) {
1923 Op->dropFlags(SDNodeFlags::Disjoint);
1924 return true;
1925 }
1926
1927 Known |= Known2;
1928 break;
1929 }
1930 case ISD::XOR: {
1931 SDValue Op0 = Op.getOperand(0);
1932 SDValue Op1 = Op.getOperand(1);
1933
1934 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1935 Depth + 1))
1936 return true;
1937 if (SimplifyDemandedBits(Op0, DemandedBits, DemandedElts, Known2, TLO,
1938 Depth + 1))
1939 return true;
1940
1941 // If all of the demanded bits are known zero on one side, return the other.
1942 // These bits cannot contribute to the result of the 'xor'.
1943 if (DemandedBits.isSubsetOf(Known.Zero))
1944 return TLO.CombineTo(Op, Op0);
1945 if (DemandedBits.isSubsetOf(Known2.Zero))
1946 return TLO.CombineTo(Op, Op1);
1947 // If the operation can be done in a smaller type, do so.
1949 return true;
1950
1951 // If all of the unknown bits are known to be zero on one side or the other
1952 // turn this into an *inclusive* or.
1953 // e.g. (A & C1)^(B & C2) -> (A & C1)|(B & C2) iff C1&C2 == 0
1954 if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
1955 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::OR, dl, VT, Op0, Op1));
1956
1957 ConstantSDNode *C = isConstOrConstSplat(Op1, DemandedElts);
1958 if (C) {
1959 // If one side is a constant, and all of the set bits in the constant are
1960 // also known set on the other side, turn this into an AND, as we know
1961 // the bits will be cleared.
1962 // e.g. (X | C1) ^ C2 --> (X | C1) & ~C2 iff (C1&C2) == C2
1963 // NB: it is okay if more bits are known than are requested
1964 if (C->getAPIntValue() == Known2.One) {
1965 SDValue ANDC =
1966 TLO.DAG.getConstant(~C->getAPIntValue() & DemandedBits, dl, VT);
1967 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, dl, VT, Op0, ANDC));
1968 }
1969
1970 // If the RHS is a constant, see if we can change it. Don't alter a -1
1971 // constant because that's a 'not' op, and that is better for combining
1972 // and codegen.
1973 if (!C->isAllOnes() && DemandedBits.isSubsetOf(C->getAPIntValue())) {
1974 // We're flipping all demanded bits. Flip the undemanded bits too.
1975 SDValue New = TLO.DAG.getNOT(dl, Op0, VT);
1976 return TLO.CombineTo(Op, New);
1977 }
1978
1979 unsigned Op0Opcode = Op0.getOpcode();
1980 if ((Op0Opcode == ISD::SRL || Op0Opcode == ISD::SHL) && Op0.hasOneUse()) {
1981 if (ConstantSDNode *ShiftC =
1982 isConstOrConstSplat(Op0.getOperand(1), DemandedElts)) {
1983 // Don't crash on an oversized shift. We can not guarantee that a
1984 // bogus shift has been simplified to undef.
1985 if (ShiftC->getAPIntValue().ult(BitWidth)) {
1986 uint64_t ShiftAmt = ShiftC->getZExtValue();
1988 Ones = Op0Opcode == ISD::SHL ? Ones.shl(ShiftAmt)
1989 : Ones.lshr(ShiftAmt);
1990 if ((DemandedBits & C->getAPIntValue()) == (DemandedBits & Ones) &&
1992 // If the xor constant is a demanded mask, do a 'not' before the
1993 // shift:
1994 // xor (X << ShiftC), XorC --> (not X) << ShiftC
1995 // xor (X >> ShiftC), XorC --> (not X) >> ShiftC
1996 SDValue Not = TLO.DAG.getNOT(dl, Op0.getOperand(0), VT);
1997 return TLO.CombineTo(Op, TLO.DAG.getNode(Op0Opcode, dl, VT, Not,
1998 Op0.getOperand(1)));
1999 }
2000 }
2001 }
2002 }
2003 }
2004
2005 // If we can't turn this into a 'not', try to shrink the constant.
2006 if (!C || !C->isAllOnes())
2007 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
2008 return true;
2009
2010 // Attempt to avoid multi-use ops if we don't need anything from them.
2011 if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
2013 Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
2015 Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
2016 if (DemandedOp0 || DemandedOp1) {
2017 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
2018 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
2019 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
2020 return TLO.CombineTo(Op, NewOp);
2021 }
2022 }
2023
2024 Known ^= Known2;
2025 break;
2026 }
2027 case ISD::SELECT:
2028 if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, DemandedElts,
2029 Known, TLO, Depth + 1))
2030 return true;
2031 if (SimplifyDemandedBits(Op.getOperand(1), DemandedBits, DemandedElts,
2032 Known2, TLO, Depth + 1))
2033 return true;
2034
2035 // If the operands are constants, see if we can simplify them.
2036 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
2037 return true;
2038
2039 // Only known if known in both the LHS and RHS.
2040 Known = Known.intersectWith(Known2);
2041 break;
2042 case ISD::VSELECT:
2043 if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, DemandedElts,
2044 Known, TLO, Depth + 1))
2045 return true;
2046 if (SimplifyDemandedBits(Op.getOperand(1), DemandedBits, DemandedElts,
2047 Known2, TLO, Depth + 1))
2048 return true;
2049
2050 // Only known if known in both the LHS and RHS.
2051 Known = Known.intersectWith(Known2);
2052 break;
2053 case ISD::SELECT_CC:
2054 if (SimplifyDemandedBits(Op.getOperand(3), DemandedBits, DemandedElts,
2055 Known, TLO, Depth + 1))
2056 return true;
2057 if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, DemandedElts,
2058 Known2, TLO, Depth + 1))
2059 return true;
2060
2061 // If the operands are constants, see if we can simplify them.
2062 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
2063 return true;
2064
2065 // Only known if known in both the LHS and RHS.
2066 Known = Known.intersectWith(Known2);
2067 break;
2068 case ISD::SETCC: {
2069 SDValue Op0 = Op.getOperand(0);
2070 SDValue Op1 = Op.getOperand(1);
2071 ISD::CondCode CC = Op.getOperandCondCode(2);
2072 // If we're testing X < 0, X >= 0, X <= -1 or X > -1
2073 // (X is of integer type) then we only need the sign mask of the previous
2074 // result
2075 if (Op1.getValueType().isInteger() &&
2076 (((CC == ISD::SETLT || CC == ISD::SETGE) && isNullOrNullSplat(Op1)) ||
2077 ((CC == ISD::SETLE || CC == ISD::SETGT) &&
2078 isAllOnesOrAllOnesSplat(Op1)))) {
2079 KnownBits KnownOp0;
2082 DemandedElts, KnownOp0, TLO, Depth + 1))
2083 return true;
2084 // If (1) we only need the sign-bit, (2) the setcc operands are the same
2085 // width as the setcc result, and (3) the result of a setcc conforms to 0
2086 // or -1, we may be able to bypass the setcc.
2087 if (DemandedBits.isSignMask() &&
2091 // If we remove a >= 0 or > -1 (for integers), we need to introduce a
2092 // NOT Operation
2093 if (CC == ISD::SETGE || CC == ISD::SETGT) {
2094 SDLoc DL(Op);
2095 EVT VT = Op0.getValueType();
2096 SDValue NotOp0 = TLO.DAG.getNOT(DL, Op0, VT);
2097 return TLO.CombineTo(Op, NotOp0);
2098 }
2099 return TLO.CombineTo(Op, Op0);
2100 }
2101 }
2102 if (getBooleanContents(Op0.getValueType()) ==
2104 BitWidth > 1)
2105 Known.Zero.setBitsFrom(1);
2106 break;
2107 }
2108 case ISD::SHL: {
2109 SDValue Op0 = Op.getOperand(0);
2110 SDValue Op1 = Op.getOperand(1);
2111 EVT ShiftVT = Op1.getValueType();
2112
2113 if (std::optional<unsigned> KnownSA =
2114 TLO.DAG.getValidShiftAmount(Op, DemandedElts, Depth + 1)) {
2115 unsigned ShAmt = *KnownSA;
2116 if (ShAmt == 0)
2117 return TLO.CombineTo(Op, Op0);
2118
2119 // If this is ((X >>u C1) << ShAmt), see if we can simplify this into a
2120 // single shift. We can do this if the bottom bits (which are shifted
2121 // out) are never demanded.
2122 // TODO - support non-uniform vector amounts.
2123 if (Op0.getOpcode() == ISD::SRL) {
2124 if (!DemandedBits.intersects(APInt::getLowBitsSet(BitWidth, ShAmt))) {
2125 if (std::optional<unsigned> InnerSA =
2126 TLO.DAG.getValidShiftAmount(Op0, DemandedElts, Depth + 2)) {
2127 unsigned C1 = *InnerSA;
2128 unsigned Opc = ISD::SHL;
2129 int Diff = ShAmt - C1;
2130 if (Diff < 0) {
2131 Diff = -Diff;
2132 Opc = ISD::SRL;
2133 }
2134 SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);
2135 return TLO.CombineTo(
2136 Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));
2137 }
2138 }
2139 }
2140
2141 // Convert (shl (anyext x, c)) to (anyext (shl x, c)) if the high bits
2142 // are not demanded. This will likely allow the anyext to be folded away.
2143 // TODO - support non-uniform vector amounts.
2144 if (Op0.getOpcode() == ISD::ANY_EXTEND) {
2145 SDValue InnerOp = Op0.getOperand(0);
2146 EVT InnerVT = InnerOp.getValueType();
2147 unsigned InnerBits = InnerVT.getScalarSizeInBits();
2148 if (ShAmt < InnerBits && DemandedBits.getActiveBits() <= InnerBits &&
2149 isTypeDesirableForOp(ISD::SHL, InnerVT)) {
2150 SDValue NarrowShl = TLO.DAG.getNode(
2151 ISD::SHL, dl, InnerVT, InnerOp,
2152 TLO.DAG.getShiftAmountConstant(ShAmt, InnerVT, dl));
2153 return TLO.CombineTo(
2154 Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, NarrowShl));
2155 }
2156
2157 // Repeat the SHL optimization above in cases where an extension
2158 // intervenes: (shl (anyext (shr x, c1)), c2) to
2159 // (shl (anyext x), c2-c1). This requires that the bottom c1 bits
2160 // aren't demanded (as above) and that the shifted upper c1 bits of
2161 // x aren't demanded.
2162 // TODO - support non-uniform vector amounts.
2163 if (InnerOp.getOpcode() == ISD::SRL && Op0.hasOneUse() &&
2164 InnerOp.hasOneUse()) {
2165 if (std::optional<unsigned> SA2 = TLO.DAG.getValidShiftAmount(
2166 InnerOp, DemandedElts, Depth + 2)) {
2167 unsigned InnerShAmt = *SA2;
2168 if (InnerShAmt < ShAmt && InnerShAmt < InnerBits &&
2169 DemandedBits.getActiveBits() <=
2170 (InnerBits - InnerShAmt + ShAmt) &&
2171 DemandedBits.countr_zero() >= ShAmt) {
2172 SDValue NewSA =
2173 TLO.DAG.getConstant(ShAmt - InnerShAmt, dl, ShiftVT);
2174 SDValue NewExt = TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT,
2175 InnerOp.getOperand(0));
2176 return TLO.CombineTo(
2177 Op, TLO.DAG.getNode(ISD::SHL, dl, VT, NewExt, NewSA));
2178 }
2179 }
2180 }
2181 }
2182
2183 APInt InDemandedMask = DemandedBits.lshr(ShAmt);
2184 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
2185 Depth + 1)) {
2186 // Disable the nsw and nuw flags. We can no longer guarantee that we
2187 // won't wrap after simplification.
2188 Op->dropFlags(SDNodeFlags::NoWrap);
2189 return true;
2190 }
2191 Known <<= ShAmt;
2192 // low bits known zero.
2193 Known.Zero.setLowBits(ShAmt);
2194
2195 // Attempt to avoid multi-use ops if we don't need anything from them.
2196 if (!InDemandedMask.isAllOnes() || !DemandedElts.isAllOnes()) {
2198 Op0, InDemandedMask, DemandedElts, TLO.DAG, Depth + 1);
2199 if (DemandedOp0) {
2200 SDValue NewOp = TLO.DAG.getNode(ISD::SHL, dl, VT, DemandedOp0, Op1);
2201 return TLO.CombineTo(Op, NewOp);
2202 }
2203 }
2204
2205 // TODO: Can we merge this fold with the one below?
2206 // Try shrinking the operation as long as the shift amount will still be
2207 // in range.
2208 if (ShAmt < DemandedBits.getActiveBits() && !VT.isVector() &&
2209 Op.getNode()->hasOneUse()) {
2210 // Search for the smallest integer type with free casts to and from
2211 // Op's type. For expedience, just check power-of-2 integer types.
2212 unsigned DemandedSize = DemandedBits.getActiveBits();
2213 for (unsigned SmallVTBits = llvm::bit_ceil(DemandedSize);
2214 SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(SmallVTBits)) {
2215 EVT SmallVT = EVT::getIntegerVT(*TLO.DAG.getContext(), SmallVTBits);
2216 if (isNarrowingProfitable(Op.getNode(), VT, SmallVT) &&
2217 isTypeDesirableForOp(ISD::SHL, SmallVT) &&
2218 isTruncateFree(VT, SmallVT) && isZExtFree(SmallVT, VT) &&
2219 (!TLO.LegalOperations() || isOperationLegal(ISD::SHL, SmallVT))) {
2220 assert(DemandedSize <= SmallVTBits &&
2221 "Narrowed below demanded bits?");
2222 // We found a type with free casts.
2223 SDValue NarrowShl = TLO.DAG.getNode(
2224 ISD::SHL, dl, SmallVT,
2225 TLO.DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(0)),
2226 TLO.DAG.getShiftAmountConstant(ShAmt, SmallVT, dl));
2227 return TLO.CombineTo(
2228 Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, NarrowShl));
2229 }
2230 }
2231 }
2232
2233 // Narrow shift to lower half - similar to ShrinkDemandedOp.
2234 // (shl i64:x, K) -> (i64 zero_extend (shl (i32 (trunc i64:x)), K))
2235 // Only do this if we demand the upper half so the knownbits are correct.
2236 unsigned HalfWidth = BitWidth / 2;
2237 if ((BitWidth % 2) == 0 && !VT.isVector() && ShAmt < HalfWidth &&
2238 DemandedBits.countLeadingOnes() >= HalfWidth) {
2239 EVT HalfVT = EVT::getIntegerVT(*TLO.DAG.getContext(), HalfWidth);
2240 if (isNarrowingProfitable(Op.getNode(), VT, HalfVT) &&
2241 isTypeDesirableForOp(ISD::SHL, HalfVT) &&
2242 isTruncateFree(VT, HalfVT) && isZExtFree(HalfVT, VT) &&
2243 (!TLO.LegalOperations() || isOperationLegal(ISD::SHL, HalfVT))) {
2244 // If we're demanding the upper bits at all, we must ensure
2245 // that the upper bits of the shift result are known to be zero,
2246 // which is equivalent to the narrow shift being NUW.
2247 if (bool IsNUW = (Known.countMinLeadingZeros() >= HalfWidth)) {
2248 bool IsNSW = Known.countMinSignBits() > HalfWidth;
2249 SDNodeFlags Flags;
2250 Flags.setNoSignedWrap(IsNSW);
2251 Flags.setNoUnsignedWrap(IsNUW);
2252 SDValue NewOp = TLO.DAG.getNode(ISD::TRUNCATE, dl, HalfVT, Op0);
2253 SDValue NewShiftAmt =
2254 TLO.DAG.getShiftAmountConstant(ShAmt, HalfVT, dl);
2255 SDValue NewShift = TLO.DAG.getNode(ISD::SHL, dl, HalfVT, NewOp,
2256 NewShiftAmt, Flags);
2257 SDValue NewExt =
2258 TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, NewShift);
2259 return TLO.CombineTo(Op, NewExt);
2260 }
2261 }
2262 }
2263 } else {
2264 // This is a variable shift, so we can't shift the demand mask by a known
2265 // amount. But if we are not demanding high bits, then we are not
2266 // demanding those bits from the pre-shifted operand either.
2267 if (unsigned CTLZ = DemandedBits.countl_zero()) {
2268 APInt DemandedFromOp(APInt::getLowBitsSet(BitWidth, BitWidth - CTLZ));
2269 if (SimplifyDemandedBits(Op0, DemandedFromOp, DemandedElts, Known, TLO,
2270 Depth + 1)) {
2271 // Disable the nsw and nuw flags. We can no longer guarantee that we
2272 // won't wrap after simplification.
2273 Op->dropFlags(SDNodeFlags::NoWrap);
2274 return true;
2275 }
2276 Known.resetAll();
2277 }
2278 }
2279
2280 // If we are only demanding sign bits then we can use the shift source
2281 // directly.
2282 if (std::optional<unsigned> MaxSA =
2283 TLO.DAG.getValidMaximumShiftAmount(Op, DemandedElts, Depth + 1)) {
2284 unsigned ShAmt = *MaxSA;
2285 unsigned NumSignBits =
2286 TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
2287 unsigned UpperDemandedBits = BitWidth - DemandedBits.countr_zero();
2288 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
2289 return TLO.CombineTo(Op, Op0);
2290 }
2291 break;
2292 }
2293 case ISD::SRL: {
2294 SDValue Op0 = Op.getOperand(0);
2295 SDValue Op1 = Op.getOperand(1);
2296 EVT ShiftVT = Op1.getValueType();
2297
2298 if (std::optional<unsigned> KnownSA =
2299 TLO.DAG.getValidShiftAmount(Op, DemandedElts, Depth + 1)) {
2300 unsigned ShAmt = *KnownSA;
2301 if (ShAmt == 0)
2302 return TLO.CombineTo(Op, Op0);
2303
2304 // If this is ((X << C1) >>u ShAmt), see if we can simplify this into a
2305 // single shift. We can do this if the top bits (which are shifted out)
2306 // are never demanded.
2307 // TODO - support non-uniform vector amounts.
2308 if (Op0.getOpcode() == ISD::SHL) {
2309 if (!DemandedBits.intersects(APInt::getHighBitsSet(BitWidth, ShAmt))) {
2310 if (std::optional<unsigned> InnerSA =
2311 TLO.DAG.getValidShiftAmount(Op0, DemandedElts, Depth + 2)) {
2312 unsigned C1 = *InnerSA;
2313 unsigned Opc = ISD::SRL;
2314 int Diff = ShAmt - C1;
2315 if (Diff < 0) {
2316 Diff = -Diff;
2317 Opc = ISD::SHL;
2318 }
2319 SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);
2320 return TLO.CombineTo(
2321 Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));
2322 }
2323 }
2324 }
2325
2326 // If this is (srl (sra X, C1), ShAmt), see if we can combine this into a
2327 // single sra. We can do this if the top bits are never demanded.
2328 if (Op0.getOpcode() == ISD::SRA && Op0.hasOneUse()) {
2329 if (!DemandedBits.intersects(APInt::getHighBitsSet(BitWidth, ShAmt))) {
2330 if (std::optional<unsigned> InnerSA =
2331 TLO.DAG.getValidShiftAmount(Op0, DemandedElts, Depth + 2)) {
2332 unsigned C1 = *InnerSA;
2333 // Clamp the combined shift amount if it exceeds the bit width.
2334 unsigned Combined = std::min(C1 + ShAmt, BitWidth - 1);
2335 SDValue NewSA = TLO.DAG.getConstant(Combined, dl, ShiftVT);
2336 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRA, dl, VT,
2337 Op0.getOperand(0), NewSA));
2338 }
2339 }
2340 }
2341
2342 APInt InDemandedMask = (DemandedBits << ShAmt);
2343
2344 // If the shift is exact, then it does demand the low bits (and knows that
2345 // they are zero).
2346 if (Op->getFlags().hasExact())
2347 InDemandedMask.setLowBits(ShAmt);
2348
2349 // Narrow shift to lower half - similar to ShrinkDemandedOp.
2350 // (srl i64:x, K) -> (i64 zero_extend (srl (i32 (trunc i64:x)), K))
2351 if ((BitWidth % 2) == 0 && !VT.isVector()) {
2353 EVT HalfVT = EVT::getIntegerVT(*TLO.DAG.getContext(), BitWidth / 2);
2354 if (isNarrowingProfitable(Op.getNode(), VT, HalfVT) &&
2355 isTypeDesirableForOp(ISD::SRL, HalfVT) &&
2356 isTruncateFree(VT, HalfVT) && isZExtFree(HalfVT, VT) &&
2357 (!TLO.LegalOperations() || isOperationLegal(ISD::SRL, HalfVT)) &&
2358 ((InDemandedMask.countLeadingZeros() >= (BitWidth / 2)) ||
2359 TLO.DAG.MaskedValueIsZero(Op0, HiBits))) {
2360 SDValue NewOp = TLO.DAG.getNode(ISD::TRUNCATE, dl, HalfVT, Op0);
2361 SDValue NewShiftAmt =
2362 TLO.DAG.getShiftAmountConstant(ShAmt, HalfVT, dl);
2363 SDValue NewShift =
2364 TLO.DAG.getNode(ISD::SRL, dl, HalfVT, NewOp, NewShiftAmt,
2365 Op->getFlags() & SDNodeFlags::Exact);
2366 return TLO.CombineTo(
2367 Op, TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, NewShift));
2368 }
2369 }
2370
2371 // Compute the new bits that are at the top now.
2372 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
2373 Depth + 1))
2374 return true;
2375 Known >>= ShAmt;
2376 // High bits known zero.
2377 Known.Zero.setHighBits(ShAmt);
2378
2379 // Attempt to avoid multi-use ops if we don't need anything from them.
2380 if (!InDemandedMask.isAllOnes() || !DemandedElts.isAllOnes()) {
2382 Op0, InDemandedMask, DemandedElts, TLO.DAG, Depth + 1);
2383 if (DemandedOp0) {
2384 SDValue NewOp = TLO.DAG.getNode(ISD::SRL, dl, VT, DemandedOp0, Op1);
2385 return TLO.CombineTo(Op, NewOp);
2386 }
2387 }
2388 } else {
2389 // Use generic knownbits computation as it has support for non-uniform
2390 // shift amounts.
2391 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2392 }
2393
2394 // If we are only demanding sign bits then we can use the shift source
2395 // directly.
2396 if (std::optional<unsigned> MaxSA =
2397 TLO.DAG.getValidMaximumShiftAmount(Op, DemandedElts, Depth + 1)) {
2398 unsigned ShAmt = *MaxSA;
2399 // Must already be signbits in DemandedBits bounds, and can't demand any
2400 // shifted in zeroes.
2401 if (DemandedBits.countl_zero() >= ShAmt) {
2402 unsigned NumSignBits =
2403 TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
2404 if (DemandedBits.countr_zero() >= (BitWidth - NumSignBits))
2405 return TLO.CombineTo(Op, Op0);
2406 }
2407 }
2408
2409 // Try to match AVG patterns (after shift simplification).
2410 if (SDValue AVG = combineShiftToAVG(Op, TLO, *this, DemandedBits,
2411 DemandedElts, Depth + 1))
2412 return TLO.CombineTo(Op, AVG);
2413 if (SDValue AVG = combineShiftToAVGSUB(Op, TLO, *this, DemandedBits,
2414 DemandedElts, Depth + 1))
2415 return TLO.CombineTo(Op, AVG);
2416
2417 break;
2418 }
2419 case ISD::SRA: {
2420 SDValue Op0 = Op.getOperand(0);
2421 SDValue Op1 = Op.getOperand(1);
2422 EVT ShiftVT = Op1.getValueType();
2423
2424 // If we only want bits that already match the signbit then we don't need
2425 // to shift.
2426 unsigned NumHiDemandedBits = BitWidth - DemandedBits.countr_zero();
2427 if (TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1) >=
2428 NumHiDemandedBits)
2429 return TLO.CombineTo(Op, Op0);
2430
2431 // If this is an arithmetic shift right and only the low-bit is set, we can
2432 // always convert this into a logical shr, even if the shift amount is
2433 // variable. The low bit of the shift cannot be an input sign bit unless
2434 // the shift amount is >= the size of the datatype, which is undefined.
2435 if (DemandedBits.isOne())
2436 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1));
2437
2438 if (std::optional<unsigned> KnownSA =
2439 TLO.DAG.getValidShiftAmount(Op, DemandedElts, Depth + 1)) {
2440 unsigned ShAmt = *KnownSA;
2441 if (ShAmt == 0)
2442 return TLO.CombineTo(Op, Op0);
2443
2444 // fold (sra (shl x, c1), c1) -> sext_inreg for some c1 and target
2445 // supports sext_inreg.
2446 if (Op0.getOpcode() == ISD::SHL) {
2447 if (std::optional<unsigned> InnerSA =
2448 TLO.DAG.getValidShiftAmount(Op0, DemandedElts, Depth + 2)) {
2449 unsigned LowBits = BitWidth - ShAmt;
2450 EVT ExtVT = VT.changeElementType(
2451 *TLO.DAG.getContext(),
2452 EVT::getIntegerVT(*TLO.DAG.getContext(), LowBits));
2453
2454 if (*InnerSA == ShAmt) {
2455 if (!TLO.LegalOperations() ||
2457 return TLO.CombineTo(
2458 Op, TLO.DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, VT,
2459 Op0.getOperand(0),
2460 TLO.DAG.getValueType(ExtVT)));
2461
2462 // Even if we can't convert to sext_inreg, we might be able to
2463 // remove this shift pair if the input is already sign extended.
2464 unsigned NumSignBits =
2465 TLO.DAG.ComputeNumSignBits(Op0.getOperand(0), DemandedElts);
2466 if (NumSignBits > ShAmt)
2467 return TLO.CombineTo(Op, Op0.getOperand(0));
2468 }
2469 }
2470 }
2471
2472 APInt InDemandedMask = (DemandedBits << ShAmt);
2473
2474 // If the shift is exact, then it does demand the low bits (and knows that
2475 // they are zero).
2476 if (Op->getFlags().hasExact())
2477 InDemandedMask.setLowBits(ShAmt);
2478
2479 // If any of the demanded bits are produced by the sign extension, we also
2480 // demand the input sign bit.
2481 if (DemandedBits.countl_zero() < ShAmt)
2482 InDemandedMask.setSignBit();
2483
2484 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
2485 Depth + 1))
2486 return true;
2487 Known >>= ShAmt;
2488
2489 // If the input sign bit is known to be zero, or if none of the top bits
2490 // are demanded, turn this into an unsigned shift right.
2491 if (Known.Zero[BitWidth - ShAmt - 1] ||
2492 DemandedBits.countl_zero() >= ShAmt) {
2493 SDNodeFlags Flags;
2494 Flags.setExact(Op->getFlags().hasExact());
2495 return TLO.CombineTo(
2496 Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1, Flags));
2497 }
2498
2499 int Log2 = DemandedBits.exactLogBase2();
2500 if (Log2 >= 0) {
2501 // The bit must come from the sign.
2502 SDValue NewSA = TLO.DAG.getConstant(BitWidth - 1 - Log2, dl, ShiftVT);
2503 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, NewSA));
2504 }
2505
2506 if (Known.One[BitWidth - ShAmt - 1])
2507 // New bits are known one.
2508 Known.One.setHighBits(ShAmt);
2509
2510 // Attempt to avoid multi-use ops if we don't need anything from them.
2511 if (!InDemandedMask.isAllOnes() || !DemandedElts.isAllOnes()) {
2513 Op0, InDemandedMask, DemandedElts, TLO.DAG, Depth + 1);
2514 if (DemandedOp0) {
2515 SDValue NewOp = TLO.DAG.getNode(ISD::SRA, dl, VT, DemandedOp0, Op1);
2516 return TLO.CombineTo(Op, NewOp);
2517 }
2518 }
2519 }
2520
2521 // Try to match AVG patterns (after shift simplification).
2522 if (SDValue AVG = combineShiftToAVG(Op, TLO, *this, DemandedBits,
2523 DemandedElts, Depth + 1))
2524 return TLO.CombineTo(Op, AVG);
2525 if (SDValue AVG = combineShiftToAVGSUB(Op, TLO, *this, DemandedBits,
2526 DemandedElts, Depth + 1))
2527 return TLO.CombineTo(Op, AVG);
2528
2529 break;
2530 }
2531 case ISD::FSHL:
2532 case ISD::FSHR: {
2533 SDValue Op0 = Op.getOperand(0);
2534 SDValue Op1 = Op.getOperand(1);
2535 SDValue Op2 = Op.getOperand(2);
2536 bool IsFSHL = (Op.getOpcode() == ISD::FSHL);
2537
2538 if (ConstantSDNode *SA = isConstOrConstSplat(Op2, DemandedElts)) {
2539 unsigned Amt = SA->getAPIntValue().urem(BitWidth);
2540
2541 // For fshl, 0-shift returns the 1st arg.
2542 // For fshr, 0-shift returns the 2nd arg.
2543 if (Amt == 0) {
2544 if (SimplifyDemandedBits(IsFSHL ? Op0 : Op1, DemandedBits, DemandedElts,
2545 Known, TLO, Depth + 1))
2546 return true;
2547 break;
2548 }
2549
2550 // fshl: (Op0 << Amt) | (Op1 >> (BW - Amt))
2551 // fshr: (Op0 << (BW - Amt)) | (Op1 >> Amt)
2552 APInt Demanded0 = DemandedBits.lshr(IsFSHL ? Amt : (BitWidth - Amt));
2553 APInt Demanded1 = DemandedBits << (IsFSHL ? (BitWidth - Amt) : Amt);
2554 if (SimplifyDemandedBits(Op0, Demanded0, DemandedElts, Known2, TLO,
2555 Depth + 1))
2556 return true;
2557 if (SimplifyDemandedBits(Op1, Demanded1, DemandedElts, Known, TLO,
2558 Depth + 1))
2559 return true;
2560
2561 Known2 <<= (IsFSHL ? Amt : (BitWidth - Amt));
2562 Known >>= (IsFSHL ? (BitWidth - Amt) : Amt);
2563 Known = Known.unionWith(Known2);
2564
2565 // Attempt to avoid multi-use ops if we don't need anything from them.
2566 if (!Demanded0.isAllOnes() || !Demanded1.isAllOnes() ||
2567 !DemandedElts.isAllOnes()) {
2569 Op0, Demanded0, DemandedElts, TLO.DAG, Depth + 1);
2571 Op1, Demanded1, DemandedElts, TLO.DAG, Depth + 1);
2572 if (DemandedOp0 || DemandedOp1) {
2573 DemandedOp0 = DemandedOp0 ? DemandedOp0 : Op0;
2574 DemandedOp1 = DemandedOp1 ? DemandedOp1 : Op1;
2575 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedOp0,
2576 DemandedOp1, Op2);
2577 return TLO.CombineTo(Op, NewOp);
2578 }
2579 }
2580 }
2581
2582 if (isPowerOf2_32(BitWidth)) {
2583 // Fold FSHR(Op0,Op1,Op2) -> SRL(Op1,Op2)
2584 // iff we're guaranteed not to use Op0.
2585 // TODO: Add FSHL equivalent?
2586 if (!IsFSHL && !DemandedBits.isAllOnes() &&
2587 (!TLO.LegalOperations() || isOperationLegal(ISD::SRL, VT))) {
2588 KnownBits KnownAmt =
2589 TLO.DAG.computeKnownBits(Op2, DemandedElts, Depth + 1);
2590 unsigned MaxShiftAmt =
2591 KnownAmt.getMaxValue().getLimitedValue(BitWidth - 1);
2592 // Check we don't demand any shifted bits outside Op1.
2593 if (DemandedBits.countl_zero() >= MaxShiftAmt) {
2594 EVT AmtVT = Op2.getValueType();
2595 SDValue NewAmt =
2596 TLO.DAG.getNode(ISD::AND, dl, AmtVT, Op2,
2597 TLO.DAG.getConstant(BitWidth - 1, dl, AmtVT));
2598 SDValue NewOp = TLO.DAG.getNode(ISD::SRL, dl, VT, Op1, NewAmt);
2599 return TLO.CombineTo(Op, NewOp);
2600 }
2601 }
2602
2603 // For pow-2 bitwidths we only demand the bottom modulo amt bits.
2604 APInt DemandedAmtBits(Op2.getScalarValueSizeInBits(), BitWidth - 1);
2605 if (SimplifyDemandedBits(Op2, DemandedAmtBits, DemandedElts, Known2, TLO,
2606 Depth + 1))
2607 return true;
2608 }
2609 break;
2610 }
2611 case ISD::ROTL:
2612 case ISD::ROTR: {
2613 SDValue Op0 = Op.getOperand(0);
2614 SDValue Op1 = Op.getOperand(1);
2615 bool IsROTL = (Op.getOpcode() == ISD::ROTL);
2616
2617 // If we're rotating an 0/-1 value, then it stays an 0/-1 value.
2618 if (BitWidth == TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1))
2619 return TLO.CombineTo(Op, Op0);
2620
2621 if (ConstantSDNode *SA = isConstOrConstSplat(Op1, DemandedElts)) {
2622 unsigned Amt = SA->getAPIntValue().urem(BitWidth);
2623 unsigned RevAmt = BitWidth - Amt;
2624
2625 // rotl: (Op0 << Amt) | (Op0 >> (BW - Amt))
2626 // rotr: (Op0 << (BW - Amt)) | (Op0 >> Amt)
2627 APInt Demanded0 = DemandedBits.rotr(IsROTL ? Amt : RevAmt);
2628 if (SimplifyDemandedBits(Op0, Demanded0, DemandedElts, Known2, TLO,
2629 Depth + 1))
2630 return true;
2631
2632 // rot*(x, 0) --> x
2633 if (Amt == 0)
2634 return TLO.CombineTo(Op, Op0);
2635
2636 // See if we don't demand either half of the rotated bits.
2637 if ((!TLO.LegalOperations() || isOperationLegal(ISD::SHL, VT)) &&
2638 DemandedBits.countr_zero() >= (IsROTL ? Amt : RevAmt)) {
2639 Op1 = TLO.DAG.getConstant(IsROTL ? Amt : RevAmt, dl, Op1.getValueType());
2640 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, Op1));
2641 }
2642 if ((!TLO.LegalOperations() || isOperationLegal(ISD::SRL, VT)) &&
2643 DemandedBits.countl_zero() >= (IsROTL ? RevAmt : Amt)) {
2644 Op1 = TLO.DAG.getConstant(IsROTL ? RevAmt : Amt, dl, Op1.getValueType());
2645 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1));
2646 }
2647 }
2648
2649 // For pow-2 bitwidths we only demand the bottom modulo amt bits.
2650 if (isPowerOf2_32(BitWidth)) {
2651 APInt DemandedAmtBits(Op1.getScalarValueSizeInBits(), BitWidth - 1);
2652 if (SimplifyDemandedBits(Op1, DemandedAmtBits, DemandedElts, Known2, TLO,
2653 Depth + 1))
2654 return true;
2655 }
2656 break;
2657 }
2658 case ISD::SMIN:
2659 case ISD::SMAX:
2660 case ISD::UMIN:
2661 case ISD::UMAX: {
2662 unsigned Opc = Op.getOpcode();
2663 SDValue Op0 = Op.getOperand(0);
2664 SDValue Op1 = Op.getOperand(1);
2665
2666 // If we're only demanding signbits, then we can simplify to OR/AND node.
2667 unsigned BitOp =
2668 (Opc == ISD::SMIN || Opc == ISD::UMAX) ? ISD::OR : ISD::AND;
2669 unsigned NumSignBits =
2670 std::min(TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1),
2671 TLO.DAG.ComputeNumSignBits(Op1, DemandedElts, Depth + 1));
2672 unsigned NumDemandedUpperBits = BitWidth - DemandedBits.countr_zero();
2673 if (NumSignBits >= NumDemandedUpperBits)
2674 return TLO.CombineTo(Op, TLO.DAG.getNode(BitOp, SDLoc(Op), VT, Op0, Op1));
2675
2676 // Check if one arg is always less/greater than (or equal) to the other arg.
2677 KnownBits Known0 = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth + 1);
2678 KnownBits Known1 = TLO.DAG.computeKnownBits(Op1, DemandedElts, Depth + 1);
2679 switch (Opc) {
2680 case ISD::SMIN:
2681 if (std::optional<bool> IsSLE = KnownBits::sle(Known0, Known1))
2682 return TLO.CombineTo(Op, *IsSLE ? Op0 : Op1);
2683 if (std::optional<bool> IsSLT = KnownBits::slt(Known0, Known1))
2684 return TLO.CombineTo(Op, *IsSLT ? Op0 : Op1);
2685 Known = KnownBits::smin(Known0, Known1);
2686 break;
2687 case ISD::SMAX:
2688 if (std::optional<bool> IsSGE = KnownBits::sge(Known0, Known1))
2689 return TLO.CombineTo(Op, *IsSGE ? Op0 : Op1);
2690 if (std::optional<bool> IsSGT = KnownBits::sgt(Known0, Known1))
2691 return TLO.CombineTo(Op, *IsSGT ? Op0 : Op1);
2692 Known = KnownBits::smax(Known0, Known1);
2693 break;
2694 case ISD::UMIN:
2695 if (std::optional<bool> IsULE = KnownBits::ule(Known0, Known1))
2696 return TLO.CombineTo(Op, *IsULE ? Op0 : Op1);
2697 if (std::optional<bool> IsULT = KnownBits::ult(Known0, Known1))
2698 return TLO.CombineTo(Op, *IsULT ? Op0 : Op1);
2699 Known = KnownBits::umin(Known0, Known1);
2700 break;
2701 case ISD::UMAX:
2702 if (std::optional<bool> IsUGE = KnownBits::uge(Known0, Known1))
2703 return TLO.CombineTo(Op, *IsUGE ? Op0 : Op1);
2704 if (std::optional<bool> IsUGT = KnownBits::ugt(Known0, Known1))
2705 return TLO.CombineTo(Op, *IsUGT ? Op0 : Op1);
2706 Known = KnownBits::umax(Known0, Known1);
2707 break;
2708 }
2709 break;
2710 }
2711 case ISD::BITREVERSE: {
2712 SDValue Src = Op.getOperand(0);
2713 APInt DemandedSrcBits = DemandedBits.reverseBits();
2714 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
2715 Depth + 1))
2716 return true;
2717 Known = Known2.reverseBits();
2718 break;
2719 }
2720 case ISD::BSWAP: {
2721 SDValue Src = Op.getOperand(0);
2722
2723 // If the only bits demanded come from one byte of the bswap result,
2724 // just shift the input byte into position to eliminate the bswap.
2725 unsigned NLZ = DemandedBits.countl_zero();
2726 unsigned NTZ = DemandedBits.countr_zero();
2727
2728 // Round NTZ down to the next byte. If we have 11 trailing zeros, then
2729 // we need all the bits down to bit 8. Likewise, round NLZ. If we
2730 // have 14 leading zeros, round to 8.
2731 NLZ = alignDown(NLZ, 8);
2732 NTZ = alignDown(NTZ, 8);
2733 // If we need exactly one byte, we can do this transformation.
2734 if (BitWidth - NLZ - NTZ == 8) {
2735 // Replace this with either a left or right shift to get the byte into
2736 // the right place.
2737 unsigned ShiftOpcode = NLZ > NTZ ? ISD::SRL : ISD::SHL;
2738 if (!TLO.LegalOperations() || isOperationLegal(ShiftOpcode, VT)) {
2739 unsigned ShiftAmount = NLZ > NTZ ? NLZ - NTZ : NTZ - NLZ;
2740 SDValue ShAmt = TLO.DAG.getShiftAmountConstant(ShiftAmount, VT, dl);
2741 SDValue NewOp = TLO.DAG.getNode(ShiftOpcode, dl, VT, Src, ShAmt);
2742 return TLO.CombineTo(Op, NewOp);
2743 }
2744 }
2745
2746 APInt DemandedSrcBits = DemandedBits.byteSwap();
2747 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
2748 Depth + 1))
2749 return true;
2750 Known = Known2.byteSwap();
2751 break;
2752 }
2753 case ISD::CTPOP: {
2754 // If only 1 bit is demanded, replace with PARITY as long as we're before
2755 // op legalization.
2756 // FIXME: Limit to scalars for now.
2757 if (DemandedBits.isOne() && !TLO.LegalOps && !VT.isVector())
2758 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::PARITY, dl, VT,
2759 Op.getOperand(0)));
2760
2761 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2762 break;
2763 }
2764 case ISD::PDEP: {
2765 SDValue Op0 = Op.getOperand(0);
2766 SDValue Op1 = Op.getOperand(1);
2767
2768 unsigned DemandedBitsLZ = DemandedBits.countl_zero();
2769 APInt LoMask = APInt::getLowBitsSet(BitWidth, BitWidth - DemandedBitsLZ);
2770
2771 // If the demanded bits has leading zeroes, we don't demand those from the
2772 // mask.
2773 if (SimplifyDemandedBits(Op1, LoMask, Known, TLO, Depth + 1))
2774 return true;
2775
2776 // The number of possible 1s in the mask determines the number of LSBs of
2777 // operand 0 used. Undemanded bits from the mask don't matter so filter
2778 // them before counting.
2779 KnownBits Known2;
2780 uint64_t Count = (~Known.Zero & LoMask).popcount();
2781 APInt DemandedMask(APInt::getLowBitsSet(BitWidth, Count));
2782 if (SimplifyDemandedBits(Op0, DemandedMask, Known2, TLO, Depth + 1))
2783 return true;
2784
2785 // Zeroes are retained from the mask, but not ones.
2786 Known.One.clearAllBits();
2787 // The result will have at least as many trailing zeros as the non-mask
2788 // operand since bits can only map to the same or higher bit position.
2789 Known.Zero.setLowBits(Known2.countMinTrailingZeros());
2790 break;
2791 }
2793 SDValue Op0 = Op.getOperand(0);
2794 EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2795 unsigned ExVTBits = ExVT.getScalarSizeInBits();
2796
2797 // If we only care about the highest bit, don't bother shifting right.
2798 if (DemandedBits.isSignMask()) {
2799 unsigned MinSignedBits =
2800 TLO.DAG.ComputeMaxSignificantBits(Op0, DemandedElts, Depth + 1);
2801 bool AlreadySignExtended = ExVTBits >= MinSignedBits;
2802 // However if the input is already sign extended we expect the sign
2803 // extension to be dropped altogether later and do not simplify.
2804 if (!AlreadySignExtended) {
2805 // Compute the correct shift amount type, which must be getShiftAmountTy
2806 // for scalar types after legalization.
2807 SDValue ShiftAmt =
2808 TLO.DAG.getShiftAmountConstant(BitWidth - ExVTBits, VT, dl);
2809 return TLO.CombineTo(Op,
2810 TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, ShiftAmt));
2811 }
2812 }
2813
2814 // If none of the extended bits are demanded, eliminate the sextinreg.
2815 if (DemandedBits.getActiveBits() <= ExVTBits)
2816 return TLO.CombineTo(Op, Op0);
2817
2818 APInt InputDemandedBits = DemandedBits.getLoBits(ExVTBits);
2819
2820 // Since the sign extended bits are demanded, we know that the sign
2821 // bit is demanded.
2822 InputDemandedBits.setBit(ExVTBits - 1);
2823
2824 if (SimplifyDemandedBits(Op0, InputDemandedBits, DemandedElts, Known, TLO,
2825 Depth + 1))
2826 return true;
2827
2828 // If the sign bit of the input is known set or clear, then we know the
2829 // top bits of the result.
2830
2831 // If the input sign bit is known zero, convert this into a zero extension.
2832 if (Known.Zero[ExVTBits - 1])
2833 return TLO.CombineTo(Op, TLO.DAG.getZeroExtendInReg(Op0, dl, ExVT));
2834
2835 APInt Mask = APInt::getLowBitsSet(BitWidth, ExVTBits);
2836 if (Known.One[ExVTBits - 1]) { // Input sign bit known set
2837 Known.One.setBitsFrom(ExVTBits);
2838 Known.Zero &= Mask;
2839 } else { // Input sign bit unknown
2840 Known.Zero &= Mask;
2841 Known.One &= Mask;
2842 }
2843 break;
2844 }
2845 case ISD::BUILD_PAIR: {
2846 EVT HalfVT = Op.getOperand(0).getValueType();
2847 unsigned HalfBitWidth = HalfVT.getScalarSizeInBits();
2848
2849 APInt MaskLo = DemandedBits.getLoBits(HalfBitWidth).trunc(HalfBitWidth);
2850 APInt MaskHi = DemandedBits.getHiBits(HalfBitWidth).trunc(HalfBitWidth);
2851
2852 KnownBits KnownLo, KnownHi;
2853
2854 if (SimplifyDemandedBits(Op.getOperand(0), MaskLo, KnownLo, TLO, Depth + 1))
2855 return true;
2856
2857 if (SimplifyDemandedBits(Op.getOperand(1), MaskHi, KnownHi, TLO, Depth + 1))
2858 return true;
2859
2860 Known = KnownHi.concat(KnownLo);
2861 break;
2862 }
2864 if (VT.isScalableVector())
2865 return false;
2866 [[fallthrough]];
2867 case ISD::ZERO_EXTEND: {
2868 SDValue Src = Op.getOperand(0);
2869 EVT SrcVT = Src.getValueType();
2870 unsigned InBits = SrcVT.getScalarSizeInBits();
2871 unsigned InElts = SrcVT.isFixedLengthVector() ? SrcVT.getVectorNumElements() : 1;
2872 bool IsVecInReg = Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG;
2873
2874 // If none of the top bits are demanded, convert this into an any_extend.
2875 if (DemandedBits.getActiveBits() <= InBits) {
2876 // If we only need the non-extended bits of the bottom element
2877 // then we can just bitcast to the result.
2878 if (IsLE && IsVecInReg && DemandedElts == 1 &&
2879 VT.getSizeInBits() == SrcVT.getSizeInBits())
2880 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
2881
2882 unsigned Opc =
2884 if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
2885 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
2886 }
2887
2888 APInt InDemandedBits = DemandedBits.trunc(InBits);
2889 APInt InDemandedElts = DemandedElts.zext(InElts);
2890 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
2891 Depth + 1)) {
2892 Op->dropFlags(SDNodeFlags::NonNeg);
2893 return true;
2894 }
2895 assert(Known.getBitWidth() == InBits && "Src width has changed?");
2896 Known = Known.zext(BitWidth);
2897
2898 // Attempt to avoid multi-use ops if we don't need anything from them.
2900 Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
2901 return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
2902 break;
2903 }
2905 if (VT.isScalableVector())
2906 return false;
2907 [[fallthrough]];
2908 case ISD::SIGN_EXTEND: {
2909 SDValue Src = Op.getOperand(0);
2910 EVT SrcVT = Src.getValueType();
2911 unsigned InBits = SrcVT.getScalarSizeInBits();
2912 unsigned InElts = SrcVT.isFixedLengthVector() ? SrcVT.getVectorNumElements() : 1;
2913 bool IsVecInReg = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG;
2914
2915 APInt InDemandedElts = DemandedElts.zext(InElts);
2916 APInt InDemandedBits = DemandedBits.trunc(InBits);
2917
2918 // Since some of the sign extended bits are demanded, we know that the sign
2919 // bit is demanded.
2920 InDemandedBits.setBit(InBits - 1);
2921
2922 // If none of the top bits are demanded, convert this into an any_extend.
2923 if (DemandedBits.getActiveBits() <= InBits) {
2924 // If we only need the non-extended bits of the bottom element
2925 // then we can just bitcast to the result.
2926 if (IsLE && IsVecInReg && DemandedElts == 1 &&
2927 VT.getSizeInBits() == SrcVT.getSizeInBits())
2928 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
2929
2930 // Don't lose an all signbits 0/-1 splat on targets with 0/-1 booleans.
2932 TLO.DAG.ComputeNumSignBits(Src, InDemandedElts, Depth + 1) !=
2933 InBits) {
2934 unsigned Opc =
2936 if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
2937 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
2938 }
2939 }
2940
2941 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
2942 Depth + 1))
2943 return true;
2944 assert(Known.getBitWidth() == InBits && "Src width has changed?");
2945
2946 // If the sign bit is known one, the top bits match.
2947 Known = Known.sext(BitWidth);
2948
2949 // If the sign bit is known zero, convert this to a zero extend.
2950 if (Known.isNonNegative()) {
2951 unsigned Opc =
2953 if (!TLO.LegalOperations() || isOperationLegal(Opc, VT)) {
2954 SDNodeFlags Flags;
2955 if (!IsVecInReg)
2956 Flags |= SDNodeFlags::NonNeg;
2957 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src, Flags));
2958 }
2959 }
2960
2961 // Attempt to avoid multi-use ops if we don't need anything from them.
2963 Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
2964 return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
2965 break;
2966 }
2968 if (VT.isScalableVector())
2969 return false;
2970 [[fallthrough]];
2971 case ISD::ANY_EXTEND: {
2972 SDValue Src = Op.getOperand(0);
2973 EVT SrcVT = Src.getValueType();
2974 unsigned InBits = SrcVT.getScalarSizeInBits();
2975 unsigned InElts = SrcVT.isFixedLengthVector() ? SrcVT.getVectorNumElements() : 1;
2976 bool IsVecInReg = Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG;
2977
2978 // If we only need the bottom element then we can just bitcast.
2979 // TODO: Handle ANY_EXTEND?
2980 if (IsLE && IsVecInReg && DemandedElts == 1 &&
2981 VT.getSizeInBits() == SrcVT.getSizeInBits())
2982 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
2983
2984 APInt InDemandedBits = DemandedBits.trunc(InBits);
2985 APInt InDemandedElts = DemandedElts.zext(InElts);
2986 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
2987 Depth + 1))
2988 return true;
2989 assert(Known.getBitWidth() == InBits && "Src width has changed?");
2990 Known = Known.anyext(BitWidth);
2991
2992 // Attempt to avoid multi-use ops if we don't need anything from them.
2994 Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
2995 return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
2996 break;
2997 }
2998 case ISD::TRUNCATE: {
2999 SDValue Src = Op.getOperand(0);
3000
3001 // Simplify the input, using demanded bit information, and compute the known
3002 // zero/one bits live out.
3003 unsigned OperandBitWidth = Src.getScalarValueSizeInBits();
3004 APInt TruncMask = DemandedBits.zext(OperandBitWidth);
3005 if (SimplifyDemandedBits(Src, TruncMask, DemandedElts, Known, TLO,
3006 Depth + 1)) {
3007 // Disable the nsw and nuw flags. We can no longer guarantee that we
3008 // won't wrap after simplification.
3009 Op->dropFlags(SDNodeFlags::NoWrap);
3010 return true;
3011 }
3012 Known = Known.trunc(BitWidth);
3013
3014 // Attempt to avoid multi-use ops if we don't need anything from them.
3016 Src, TruncMask, DemandedElts, TLO.DAG, Depth + 1))
3017 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, NewSrc));
3018
3019 // If the input is only used by this truncate, see if we can shrink it based
3020 // on the known demanded bits.
3021 switch (Src.getOpcode()) {
3022 default:
3023 break;
3024 case ISD::SRL:
3025 // Shrink SRL by a constant if none of the high bits shifted in are
3026 // demanded.
3027 if (TLO.LegalTypes() && !isTypeDesirableForOp(ISD::SRL, VT))
3028 // Do not turn (vt1 truncate (vt2 srl)) into (vt1 srl) if vt1 is
3029 // undesirable.
3030 break;
3031
3032 if (Src.getNode()->hasOneUse()) {
3033 if (isTruncateFree(Src, VT) &&
3034 !isTruncateFree(Src.getValueType(), VT)) {
3035 // If truncate is only free at trunc(srl), do not turn it into
3036 // srl(trunc). The check is done by first check the truncate is free
3037 // at Src's opcode(srl), then check the truncate is not done by
3038 // referencing sub-register. In test, if both trunc(srl) and
3039 // srl(trunc)'s trunc are free, srl(trunc) performs better. If only
3040 // trunc(srl)'s trunc is free, trunc(srl) is better.
3041 break;
3042 }
3043
3044 std::optional<unsigned> ShAmtC =
3045 TLO.DAG.getValidShiftAmount(Src, DemandedElts, Depth + 2);
3046 if (!ShAmtC || *ShAmtC >= BitWidth)
3047 break;
3048 unsigned ShVal = *ShAmtC;
3049
3050 APInt HighBits =
3051 APInt::getHighBitsSet(OperandBitWidth, OperandBitWidth - BitWidth);
3052 HighBits.lshrInPlace(ShVal);
3053 HighBits = HighBits.trunc(BitWidth);
3054 if (!(HighBits & DemandedBits)) {
3055 // None of the shifted in bits are needed. Add a truncate of the
3056 // shift input, then shift it.
3057 SDValue NewShAmt = TLO.DAG.getShiftAmountConstant(ShVal, VT, dl);
3058 SDValue NewTrunc =
3059 TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, Src.getOperand(0));
3060 return TLO.CombineTo(
3061 Op, TLO.DAG.getNode(ISD::SRL, dl, VT, NewTrunc, NewShAmt));
3062 }
3063 }
3064 break;
3065 }
3066
3067 break;
3068 }
3069 case ISD::AssertZext: {
3070 // AssertZext demands all of the high bits, plus any of the low bits
3071 // demanded by its users.
3072 EVT ZVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
3074 if (SimplifyDemandedBits(Op.getOperand(0), ~InMask | DemandedBits, Known,
3075 TLO, Depth + 1))
3076 return true;
3077
3078 Known.Zero |= ~InMask;
3079 Known.One &= (~Known.Zero);
3080 break;
3081 }
3083 SDValue Src = Op.getOperand(0);
3084 SDValue Idx = Op.getOperand(1);
3085 ElementCount SrcEltCnt = Src.getValueType().getVectorElementCount();
3086 unsigned EltBitWidth = Src.getScalarValueSizeInBits();
3087
3088 if (SrcEltCnt.isScalable())
3089 return false;
3090
3091 // Demand the bits from every vector element without a constant index.
3092 unsigned NumSrcElts = SrcEltCnt.getFixedValue();
3093 APInt DemandedSrcElts = APInt::getAllOnes(NumSrcElts);
3094 if (auto *CIdx = dyn_cast<ConstantSDNode>(Idx))
3095 if (CIdx->getAPIntValue().ult(NumSrcElts))
3096 DemandedSrcElts = APInt::getOneBitSet(NumSrcElts, CIdx->getZExtValue());
3097
3098 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
3099 // anything about the extended bits.
3100 APInt DemandedSrcBits = DemandedBits;
3101 if (BitWidth > EltBitWidth)
3102 DemandedSrcBits = DemandedSrcBits.trunc(EltBitWidth);
3103
3104 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts, Known2, TLO,
3105 Depth + 1))
3106 return true;
3107
3108 // Attempt to avoid multi-use ops if we don't need anything from them.
3109 if (!DemandedSrcBits.isAllOnes() || !DemandedSrcElts.isAllOnes()) {
3110 if (SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
3111 Src, DemandedSrcBits, DemandedSrcElts, TLO.DAG, Depth + 1)) {
3112 SDValue NewOp =
3113 TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc, Idx);
3114 return TLO.CombineTo(Op, NewOp);
3115 }
3116 }
3117
3118 Known = Known2;
3119 if (BitWidth > EltBitWidth)
3120 Known = Known.anyext(BitWidth);
3121 break;
3122 }
3123 case ISD::BITCAST: {
3124 if (VT.isScalableVector())
3125 return false;
3126 SDValue Src = Op.getOperand(0);
3127 EVT SrcVT = Src.getValueType();
3128 unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
3129
3130 // If this is an FP->Int bitcast and if the sign bit is the only
3131 // thing demanded, turn this into a FGETSIGN.
3132 if (!TLO.LegalOperations() && !VT.isVector() && !SrcVT.isVector() &&
3133 DemandedBits == APInt::getSignMask(Op.getValueSizeInBits()) &&
3134 SrcVT.isFloatingPoint()) {
3136 // Make a FGETSIGN + SHL to move the sign bit into the appropriate
3137 // place. We expect the SHL to be eliminated by other optimizations.
3138 SDValue Sign = TLO.DAG.getNode(ISD::FGETSIGN, dl, VT, Src);
3139 unsigned ShVal = Op.getValueSizeInBits() - 1;
3140 SDValue ShAmt = TLO.DAG.getShiftAmountConstant(ShVal, VT, dl);
3141 return TLO.CombineTo(Op,
3142 TLO.DAG.getNode(ISD::SHL, dl, VT, Sign, ShAmt));
3143 }
3144 }
3145
3146 // Bitcast from a vector using SimplifyDemanded Bits/VectorElts.
3147 // Demand the elt/bit if any of the original elts/bits are demanded.
3148 if (SrcVT.isVector() && (BitWidth % NumSrcEltBits) == 0) {
3149 unsigned Scale = BitWidth / NumSrcEltBits;
3150 unsigned NumSrcElts = SrcVT.getVectorNumElements();
3151 APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);
3152 for (unsigned i = 0; i != Scale; ++i) {
3153 unsigned EltOffset = IsLE ? i : (Scale - 1 - i);
3154 unsigned BitOffset = EltOffset * NumSrcEltBits;
3155 DemandedSrcBits |= DemandedBits.extractBits(NumSrcEltBits, BitOffset);
3156 }
3157 // Recursive calls below may turn not demanded elements into poison, so we
3158 // need to demand all smaller source elements that maps to a demanded
3159 // destination element.
3160 APInt DemandedSrcElts = APIntOps::ScaleBitMask(DemandedElts, NumSrcElts);
3161
3162 APInt KnownSrcUndef, KnownSrcZero;
3163 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
3164 KnownSrcZero, TLO, Depth + 1))
3165 return true;
3166
3167 KnownBits KnownSrcBits;
3168 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
3169 KnownSrcBits, TLO, Depth + 1))
3170 return true;
3171 } else if (IsLE && (NumSrcEltBits % BitWidth) == 0) {
3172 // TODO - bigendian once we have test coverage.
3173 unsigned Scale = NumSrcEltBits / BitWidth;
3174 unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
3175 APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);
3176 APInt DemandedSrcElts = APInt::getZero(NumSrcElts);
3177 for (unsigned i = 0; i != NumElts; ++i)
3178 if (DemandedElts[i]) {
3179 unsigned Offset = (i % Scale) * BitWidth;
3180 DemandedSrcBits.insertBits(DemandedBits, Offset);
3181 DemandedSrcElts.setBit(i / Scale);
3182 }
3183
3184 if (SrcVT.isVector()) {
3185 APInt KnownSrcUndef, KnownSrcZero;
3186 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
3187 KnownSrcZero, TLO, Depth + 1))
3188 return true;
3189 }
3190
3191 KnownBits KnownSrcBits;
3192 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
3193 KnownSrcBits, TLO, Depth + 1))
3194 return true;
3195
3196 // Attempt to avoid multi-use ops if we don't need anything from them.
3197 if (!DemandedSrcBits.isAllOnes() || !DemandedSrcElts.isAllOnes()) {
3198 if (SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
3199 Src, DemandedSrcBits, DemandedSrcElts, TLO.DAG, Depth + 1)) {
3200 SDValue NewOp = TLO.DAG.getBitcast(VT, DemandedSrc);
3201 return TLO.CombineTo(Op, NewOp);
3202 }
3203 }
3204 }
3205
3206 // If this is a bitcast, let computeKnownBits handle it. Only do this on a
3207 // recursive call where Known may be useful to the caller.
3208 if (Depth > 0) {
3209 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
3210 return false;
3211 }
3212 break;
3213 }
3214 case ISD::MUL:
3215 if (DemandedBits.isPowerOf2()) {
3216 // The LSB of X*Y is set only if (X & 1) == 1 and (Y & 1) == 1.
3217 // If we demand exactly one bit N and we have "X * (C' << N)" where C' is
3218 // odd (has LSB set), then the left-shifted low bit of X is the answer.
3219 unsigned CTZ = DemandedBits.countr_zero();
3220 ConstantSDNode *C = isConstOrConstSplat(Op.getOperand(1), DemandedElts);
3221 if (C && C->getAPIntValue().countr_zero() == CTZ) {
3222 SDValue AmtC = TLO.DAG.getShiftAmountConstant(CTZ, VT, dl);
3223 SDValue Shl = TLO.DAG.getNode(ISD::SHL, dl, VT, Op.getOperand(0), AmtC);
3224 return TLO.CombineTo(Op, Shl);
3225 }
3226 }
3227 // For a squared value "X * X", the bottom 2 bits are 0 and X[0] because:
3228 // X * X is odd iff X is odd.
3229 // 'Quadratic Reciprocity': X * X -> 0 for bit[1]
3230 if (Op.getOperand(0) == Op.getOperand(1) && DemandedBits.ult(4)) {
3231 SDValue One = TLO.DAG.getConstant(1, dl, VT);
3232 SDValue And1 = TLO.DAG.getNode(ISD::AND, dl, VT, Op.getOperand(0), One);
3233 return TLO.CombineTo(Op, And1);
3234 }
3235 [[fallthrough]];
3236 case ISD::PTRADD:
3237 if (Op.getOperand(0).getValueType() != Op.getOperand(1).getValueType())
3238 break;
3239 // PTRADD behaves like ADD if pointers are represented as integers.
3240 [[fallthrough]];
3241 case ISD::ADD:
3242 case ISD::SUB: {
3243 // Add, Sub, and Mul don't demand any bits in positions beyond that
3244 // of the highest bit demanded of them.
3245 SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1);
3246 SDNodeFlags Flags = Op.getNode()->getFlags();
3247 unsigned DemandedBitsLZ = DemandedBits.countl_zero();
3248 APInt LoMask = APInt::getLowBitsSet(BitWidth, BitWidth - DemandedBitsLZ);
3249 KnownBits KnownOp0, KnownOp1;
3250 auto GetDemandedBitsLHSMask = [&](APInt Demanded,
3251 const KnownBits &KnownRHS) {
3252 if (Op.getOpcode() == ISD::MUL)
3253 Demanded.clearHighBits(KnownRHS.countMinTrailingZeros());
3254 return Demanded;
3255 };
3256 if (SimplifyDemandedBits(Op1, LoMask, DemandedElts, KnownOp1, TLO,
3257 Depth + 1) ||
3258 SimplifyDemandedBits(Op0, GetDemandedBitsLHSMask(LoMask, KnownOp1),
3259 DemandedElts, KnownOp0, TLO, Depth + 1) ||
3260 // See if the operation should be performed at a smaller bit width.
3262 // Disable the nsw and nuw flags. We can no longer guarantee that we
3263 // won't wrap after simplification.
3264 Op->dropFlags(SDNodeFlags::NoWrap);
3265 return true;
3266 }
3267
3268 // neg x with only low bit demanded is simply x.
3269 if (Op.getOpcode() == ISD::SUB && DemandedBits.isOne() &&
3270 isNullConstant(Op0))
3271 return TLO.CombineTo(Op, Op1);
3272
3273 // Attempt to avoid multi-use ops if we don't need anything from them.
3274 if (!LoMask.isAllOnes() || !DemandedElts.isAllOnes()) {
3276 Op0, LoMask, DemandedElts, TLO.DAG, Depth + 1);
3278 Op1, LoMask, DemandedElts, TLO.DAG, Depth + 1);
3279 if (DemandedOp0 || DemandedOp1) {
3280 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
3281 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
3282 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1,
3283 Flags & ~SDNodeFlags::NoWrap);
3284 return TLO.CombineTo(Op, NewOp);
3285 }
3286 }
3287
3288 // If we have a constant operand, we may be able to turn it into -1 if we
3289 // do not demand the high bits. This can make the constant smaller to
3290 // encode, allow more general folding, or match specialized instruction
3291 // patterns (eg, 'blsr' on x86). Don't bother changing 1 to -1 because that
3292 // is probably not useful (and could be detrimental).
3294 APInt HighMask = APInt::getHighBitsSet(BitWidth, DemandedBitsLZ);
3295 if (C && !C->isAllOnes() && !C->isOne() &&
3296 (C->getAPIntValue() | HighMask).isAllOnes()) {
3297 SDValue Neg1 = TLO.DAG.getAllOnesConstant(dl, VT);
3298 // Disable the nsw and nuw flags. We can no longer guarantee that we
3299 // won't wrap after simplification.
3300 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Neg1,
3301 Flags & ~SDNodeFlags::NoWrap);
3302 return TLO.CombineTo(Op, NewOp);
3303 }
3304
3305 // Match a multiply with a disguised negated-power-of-2 and convert to a
3306 // an equivalent shift-left amount.
3307 // Example: (X * MulC) + Op1 --> Op1 - (X << log2(-MulC))
3308 auto getShiftLeftAmt = [&HighMask](SDValue Mul) -> unsigned {
3309 if (Mul.getOpcode() != ISD::MUL || !Mul.hasOneUse())
3310 return 0;
3311
3312 // Don't touch opaque constants. Also, ignore zero and power-of-2
3313 // multiplies. Those will get folded later.
3314 ConstantSDNode *MulC = isConstOrConstSplat(Mul.getOperand(1));
3315 if (MulC && !MulC->isOpaque() && !MulC->isZero() &&
3316 !MulC->getAPIntValue().isPowerOf2()) {
3317 APInt UnmaskedC = MulC->getAPIntValue() | HighMask;
3318 if (UnmaskedC.isNegatedPowerOf2())
3319 return (-UnmaskedC).logBase2();
3320 }
3321 return 0;
3322 };
3323
3324 auto foldMul = [&](ISD::NodeType NT, SDValue X, SDValue Y,
3325 unsigned ShlAmt) {
3326 SDValue ShlAmtC = TLO.DAG.getShiftAmountConstant(ShlAmt, VT, dl);
3327 SDValue Shl = TLO.DAG.getNode(ISD::SHL, dl, VT, X, ShlAmtC);
3328 SDValue Res = TLO.DAG.getNode(NT, dl, VT, Y, Shl);
3329 return TLO.CombineTo(Op, Res);
3330 };
3331
3333 if (Op.getOpcode() == ISD::ADD) {
3334 // (X * MulC) + Op1 --> Op1 - (X << log2(-MulC))
3335 if (unsigned ShAmt = getShiftLeftAmt(Op0))
3336 return foldMul(ISD::SUB, Op0.getOperand(0), Op1, ShAmt);
3337 // Op0 + (X * MulC) --> Op0 - (X << log2(-MulC))
3338 if (unsigned ShAmt = getShiftLeftAmt(Op1))
3339 return foldMul(ISD::SUB, Op1.getOperand(0), Op0, ShAmt);
3340 }
3341 if (Op.getOpcode() == ISD::SUB) {
3342 // Op0 - (X * MulC) --> Op0 + (X << log2(-MulC))
3343 if (unsigned ShAmt = getShiftLeftAmt(Op1))
3344 return foldMul(ISD::ADD, Op1.getOperand(0), Op0, ShAmt);
3345 }
3346 }
3347
3348 if (Op.getOpcode() == ISD::MUL) {
3349 Known = KnownBits::mul(KnownOp0, KnownOp1);
3350 } else { // Op.getOpcode() is either ISD::ADD, ISD::PTRADD, or ISD::SUB.
3352 Op.getOpcode() != ISD::SUB, Flags.hasNoSignedWrap(),
3353 Flags.hasNoUnsignedWrap(), KnownOp0, KnownOp1);
3354 }
3355 break;
3356 }
3357 case ISD::FABS: {
3358 SDValue Op0 = Op.getOperand(0);
3359 APInt SignMask = APInt::getSignMask(BitWidth);
3360
3361 if (!DemandedBits.intersects(SignMask))
3362 return TLO.CombineTo(Op, Op0);
3363
3364 if (SimplifyDemandedBits(Op0, DemandedBits, DemandedElts, Known, TLO,
3365 Depth + 1))
3366 return true;
3367
3368 if (Known.isNonNegative())
3369 return TLO.CombineTo(Op, Op0);
3370 if (Known.isNegative())
3371 return TLO.CombineTo(
3372 Op, TLO.DAG.getNode(ISD::FNEG, dl, VT, Op0, Op->getFlags()));
3373
3374 Known.Zero |= SignMask;
3375 Known.One &= ~SignMask;
3376
3377 break;
3378 }
3379 case ISD::FCOPYSIGN: {
3380 SDValue Op0 = Op.getOperand(0);
3381 SDValue Op1 = Op.getOperand(1);
3382
3383 unsigned BitWidth0 = Op0.getScalarValueSizeInBits();
3384 unsigned BitWidth1 = Op1.getScalarValueSizeInBits();
3385 APInt SignMask0 = APInt::getSignMask(BitWidth0);
3386 APInt SignMask1 = APInt::getSignMask(BitWidth1);
3387
3388 if (!DemandedBits.intersects(SignMask0))
3389 return TLO.CombineTo(Op, Op0);
3390
3391 if (SimplifyDemandedBits(Op0, ~SignMask0 & DemandedBits, DemandedElts,
3392 Known, TLO, Depth + 1) ||
3393 SimplifyDemandedBits(Op1, SignMask1, DemandedElts, Known2, TLO,
3394 Depth + 1))
3395 return true;
3396
3397 if (Known2.isNonNegative())
3398 return TLO.CombineTo(
3399 Op, TLO.DAG.getNode(ISD::FABS, dl, VT, Op0, Op->getFlags()));
3400
3401 if (Known2.isNegative())
3402 return TLO.CombineTo(
3403 Op, TLO.DAG.getNode(ISD::FNEG, dl, VT,
3404 TLO.DAG.getNode(ISD::FABS, SDLoc(Op0), VT, Op0)));
3405
3406 Known.Zero &= ~SignMask0;
3407 Known.One &= ~SignMask0;
3408 break;
3409 }
3410 case ISD::FNEG: {
3411 SDValue Op0 = Op.getOperand(0);
3412 APInt SignMask = APInt::getSignMask(BitWidth);
3413
3414 if (!DemandedBits.intersects(SignMask))
3415 return TLO.CombineTo(Op, Op0);
3416
3417 if (SimplifyDemandedBits(Op0, DemandedBits, DemandedElts, Known, TLO,
3418 Depth + 1))
3419 return true;
3420
3421 if (!Known.isSignUnknown()) {
3422 Known.Zero ^= SignMask;
3423 Known.One ^= SignMask;
3424 }
3425
3426 break;
3427 }
3428 default:
3429 // We also ask the target about intrinsics (which could be specific to it).
3430 if (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3431 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN) {
3432 // TODO: Probably okay to remove after audit; here to reduce change size
3433 // in initial enablement patch for scalable vectors
3434 if (Op.getValueType().isScalableVector())
3435 break;
3437 Known, TLO, Depth))
3438 return true;
3439 break;
3440 }
3441
3442 // Just use computeKnownBits to compute output bits.
3443 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
3444 break;
3445 }
3446
3447 // If we know the value of all of the demanded bits, return this as a
3448 // constant.
3450 DemandedBits.isSubsetOf(Known.Zero | Known.One)) {
3451 // Avoid folding to a constant if any OpaqueConstant is involved.
3452 if (llvm::any_of(Op->ops(), [](SDValue V) {
3453 auto *C = dyn_cast<ConstantSDNode>(V);
3454 return C && C->isOpaque();
3455 }))
3456 return false;
3457 if (VT.isInteger())
3458 return TLO.CombineTo(Op, TLO.DAG.getConstant(Known.One, dl, VT));
3459 if (VT.isFloatingPoint())
3460 return TLO.CombineTo(
3462 dl, VT));
3463 }
3464
3465 // A multi use 'all demanded elts' simplify failed to find any knownbits.
3466 // Try again just for the original demanded elts.
3467 // Ensure we do this AFTER constant folding above.
3468 if (HasMultiUse && Known.isUnknown() && !OriginalDemandedElts.isAllOnes())
3469 Known = TLO.DAG.computeKnownBits(Op, OriginalDemandedElts, Depth);
3470
3471 return false;
3472}
3473
3475 const APInt &DemandedElts,
3476 DAGCombinerInfo &DCI) const {
3477 SelectionDAG &DAG = DCI.DAG;
3478 TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
3479 !DCI.isBeforeLegalizeOps());
3480
3481 APInt KnownUndef, KnownZero;
3482 bool Simplified =
3483 SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero, TLO);
3484 if (Simplified) {
3485 DCI.AddToWorklist(Op.getNode());
3486 DCI.CommitTargetLoweringOpt(TLO);
3487 }
3488
3489 return Simplified;
3490}
3491
3492/// Given a vector binary operation and known undefined elements for each input
3493/// operand, compute whether each element of the output is undefined.
3495 const APInt &UndefOp0,
3496 const APInt &UndefOp1) {
3497 EVT VT = BO.getValueType();
3499 "Vector binop only");
3500
3501 EVT EltVT = VT.getVectorElementType();
3502 unsigned NumElts = VT.isFixedLengthVector() ? VT.getVectorNumElements() : 1;
3503 assert(UndefOp0.getBitWidth() == NumElts &&
3504 UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis");
3505
3506 auto getUndefOrConstantElt = [&](SDValue V, unsigned Index,
3507 const APInt &UndefVals) {
3508 if (UndefVals[Index])
3509 return DAG.getUNDEF(EltVT);
3510
3511 if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
3512 // Try hard to make sure that the getNode() call is not creating temporary
3513 // nodes. Ignore opaque integers because they do not constant fold.
3514 SDValue Elt = BV->getOperand(Index);
3515 auto *C = dyn_cast<ConstantSDNode>(Elt);
3516 if (isa<ConstantFPSDNode>(Elt) || Elt.isUndef() || (C && !C->isOpaque()))
3517 return Elt;
3518 }
3519
3520 return SDValue();
3521 };
3522
3523 APInt KnownUndef = APInt::getZero(NumElts);
3524 for (unsigned i = 0; i != NumElts; ++i) {
3525 // If both inputs for this element are either constant or undef and match
3526 // the element type, compute the constant/undef result for this element of
3527 // the vector.
3528 // TODO: Ideally we would use FoldConstantArithmetic() here, but that does
3529 // not handle FP constants. The code within getNode() should be refactored
3530 // to avoid the danger of creating a bogus temporary node here.
3531 SDValue C0 = getUndefOrConstantElt(BO.getOperand(0), i, UndefOp0);
3532 SDValue C1 = getUndefOrConstantElt(BO.getOperand(1), i, UndefOp1);
3533 if (C0 && C1 && C0.getValueType() == EltVT && C1.getValueType() == EltVT)
3534 if (DAG.getNode(BO.getOpcode(), SDLoc(BO), EltVT, C0, C1).isUndef())
3535 KnownUndef.setBit(i);
3536 }
3537 return KnownUndef;
3538}
3539
3541 SDValue Op, const APInt &OriginalDemandedElts, APInt &KnownUndef,
3542 APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth,
3543 bool AssumeSingleUse) const {
3544 EVT VT = Op.getValueType();
3545 unsigned Opcode = Op.getOpcode();
3546 APInt DemandedElts = OriginalDemandedElts;
3547 unsigned NumElts = DemandedElts.getBitWidth();
3548 assert(VT.isVector() && "Expected vector op");
3549
3550 KnownUndef = KnownZero = APInt::getZero(NumElts);
3551
3553 return false;
3554
3555 // TODO: For now we assume we know nothing about scalable vectors.
3556 if (VT.isScalableVector())
3557 return false;
3558
3559 assert(VT.getVectorNumElements() == NumElts &&
3560 "Mask size mismatches value type element count!");
3561
3562 // Undef operand.
3563 if (Op.isUndef()) {
3564 KnownUndef.setAllBits();
3565 return false;
3566 }
3567
3568 // If Op has other users, assume that all elements are needed.
3569 if (!AssumeSingleUse && !Op.getNode()->hasOneUse())
3570 DemandedElts.setAllBits();
3571
3572 // Not demanding any elements from Op.
3573 if (DemandedElts == 0) {
3574 KnownUndef.setAllBits();
3575 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
3576 }
3577
3578 // Limit search depth.
3580 return false;
3581
3582 SDLoc DL(Op);
3583 unsigned EltSizeInBits = VT.getScalarSizeInBits();
3584 bool IsLE = TLO.DAG.getDataLayout().isLittleEndian();
3585
3586 auto TryShrinkBinOp = [&](SDValue Op0, SDValue Op1) {
3587 unsigned ShrunkSize = getPreferredShrunkVectorSizeInBits(Op, DemandedElts);
3588 if (!ShrunkSize)
3589 return false;
3590
3591 assert(ShrunkSize % EltSizeInBits == 0 &&
3592 "Shrunk size not a multiple of element size");
3593 assert(ShrunkSize < VT.getSizeInBits() &&
3594 "Shrunk size must be < original vector size");
3595 assert(ShrunkSize >= EltSizeInBits * DemandedElts.getActiveBits() &&
3596 "Shrunk size must be >= demanded size");
3597
3598 EVT ShrunkVT = VT.changeVectorElementCount(
3599 *TLO.DAG.getContext(),
3600 ElementCount::getFixed(ShrunkSize / EltSizeInBits));
3601 Op0 = TLO.DAG.getExtractSubvector(DL, ShrunkVT, Op0, 0);
3602 Op1 = TLO.DAG.getExtractSubvector(DL, ShrunkVT, Op1, 0);
3603 SDValue NewOp =
3604 TLO.DAG.getNode(Opcode, DL, ShrunkVT, Op0, Op1, Op->getFlags());
3605 return TLO.CombineTo(
3606 Op, TLO.DAG.getInsertSubvector(DL, TLO.DAG.getUNDEF(VT), NewOp, 0));
3607 };
3608
3609 // Helper for demanding the specified elements and all the bits of both binary
3610 // operands.
3611 auto SimplifyDemandedVectorEltsBinOp = [&](SDValue Op0, SDValue Op1) {
3612 SDValue NewOp0 = SimplifyMultipleUseDemandedVectorElts(Op0, DemandedElts,
3613 TLO.DAG, Depth + 1);
3614 SDValue NewOp1 = SimplifyMultipleUseDemandedVectorElts(Op1, DemandedElts,
3615 TLO.DAG, Depth + 1);
3616 if (NewOp0 || NewOp1) {
3617 SDValue NewOp =
3618 TLO.DAG.getNode(Opcode, SDLoc(Op), VT, NewOp0 ? NewOp0 : Op0,
3619 NewOp1 ? NewOp1 : Op1, Op->getFlags());
3620 return TLO.CombineTo(Op, NewOp);
3621 }
3622
3623 if (TryShrinkBinOp(Op0, Op1))
3624 return true;
3625
3626 return false;
3627 };
3628
3629 switch (Opcode) {
3630 case ISD::SCALAR_TO_VECTOR: {
3631 if (!DemandedElts[0])
3632 return TLO.CombineTo(Op, TLO.DAG.getPOISON(VT));
3633 // Upper elements are poison, not undef - don't mark them as KnownUndef.
3634 break;
3635 }
3636 case ISD::BITCAST: {
3637 SDValue Src = Op.getOperand(0);
3638 EVT SrcVT = Src.getValueType();
3639
3640 if (!SrcVT.isVector()) {
3641 // TODO - bigendian once we have test coverage.
3642 if (IsLE) {
3643 APInt DemandedSrcBits = APInt::getZero(SrcVT.getSizeInBits());
3644 unsigned EltSize = VT.getScalarSizeInBits();
3645 for (unsigned I = 0; I != NumElts; ++I) {
3646 if (DemandedElts[I]) {
3647 unsigned Offset = I * EltSize;
3648 DemandedSrcBits.setBits(Offset, Offset + EltSize);
3649 }
3650 }
3652 if (SimplifyDemandedBits(Src, DemandedSrcBits, Known, TLO, Depth + 1))
3653 return true;
3654 }
3655 break;
3656 }
3657
3658 // Fast handling of 'identity' bitcasts.
3659 unsigned NumSrcElts = SrcVT.getVectorNumElements();
3660 if (NumSrcElts == NumElts)
3661 return SimplifyDemandedVectorElts(Src, DemandedElts, KnownUndef,
3662 KnownZero, TLO, Depth + 1);
3663
3664 APInt SrcDemandedElts, SrcZero, SrcUndef;
3665
3666 // Bitcast from 'large element' src vector to 'small element' vector, we
3667 // must demand a source element if any DemandedElt maps to it.
3668 if ((NumElts % NumSrcElts) == 0) {
3669 unsigned Scale = NumElts / NumSrcElts;
3670 SrcDemandedElts = APIntOps::ScaleBitMask(DemandedElts, NumSrcElts);
3671 if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
3672 TLO, Depth + 1))
3673 return true;
3674
3675 // Try calling SimplifyDemandedBits, converting demanded elts to the bits
3676 // of the large element.
3677 // TODO - bigendian once we have test coverage.
3678 if (IsLE) {
3679 unsigned SrcEltSizeInBits = SrcVT.getScalarSizeInBits();
3680 APInt SrcDemandedBits = APInt::getZero(SrcEltSizeInBits);
3681 for (unsigned i = 0; i != NumElts; ++i)
3682 if (DemandedElts[i]) {
3683 unsigned Ofs = (i % Scale) * EltSizeInBits;
3684 SrcDemandedBits.setBits(Ofs, Ofs + EltSizeInBits);
3685 }
3686
3688 if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcDemandedElts, Known,
3689 TLO, Depth + 1))
3690 return true;
3691
3692 // The bitcast has split each wide element into a number of
3693 // narrow subelements. We have just computed the Known bits
3694 // for wide elements. See if element splitting results in
3695 // some subelements being zero. Only for demanded elements!
3696 for (unsigned SubElt = 0; SubElt != Scale; ++SubElt) {
3697 if (!Known.Zero.extractBits(EltSizeInBits, SubElt * EltSizeInBits)
3698 .isAllOnes())
3699 continue;
3700 for (unsigned SrcElt = 0; SrcElt != NumSrcElts; ++SrcElt) {
3701 unsigned Elt = Scale * SrcElt + SubElt;
3702 // A wholly-undef source lane is reported as undef below; don't also
3703 // flag it as zero, keeping the undef and zero sets disjoint.
3704 if (DemandedElts[Elt] && !SrcUndef[SrcElt])
3705 KnownZero.setBit(Elt);
3706 }
3707 }
3708 }
3709
3710 // If the src element is zero/undef then all the output elements will be -
3711 // only demanded elements are guaranteed to be correct.
3712 for (unsigned i = 0; i != NumSrcElts; ++i) {
3713 if (SrcDemandedElts[i]) {
3714 if (SrcZero[i])
3715 KnownZero.setBits(i * Scale, (i + 1) * Scale);
3716 if (SrcUndef[i])
3717 KnownUndef.setBits(i * Scale, (i + 1) * Scale);
3718 }
3719 }
3720 }
3721
3722 // Bitcast from 'small element' src vector to 'large element' vector, we
3723 // demand all smaller source elements covered by the larger demanded element
3724 // of this vector.
3725 if ((NumSrcElts % NumElts) == 0) {
3726 unsigned Scale = NumSrcElts / NumElts;
3727 SrcDemandedElts = APIntOps::ScaleBitMask(DemandedElts, NumSrcElts);
3728 if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
3729 TLO, Depth + 1))
3730 return true;
3731
3732 // If all the src elements covering an output element are zero/undef, then
3733 // the output element will be as well, assuming it was demanded.
3734 for (unsigned i = 0; i != NumElts; ++i) {
3735 if (DemandedElts[i]) {
3736 if (SrcZero.extractBits(Scale, i * Scale).isAllOnes())
3737 KnownZero.setBit(i);
3738 if (SrcUndef.extractBits(Scale, i * Scale).isAllOnes())
3739 KnownUndef.setBit(i);
3740 }
3741 }
3742 }
3743 break;
3744 }
3745 case ISD::FREEZE: {
3746 SDValue N0 = Op.getOperand(0);
3748 N0, DemandedElts, UndefPoisonKind::UndefOrPoison, Depth + 1))
3749 return TLO.CombineTo(Op, N0);
3750
3751 // TODO: Replace this with the general fold from DAGCombiner::visitFREEZE
3752 // freeze(op(x, ...)) -> op(freeze(x), ...).
3753 // Don't sink the freeze below SCALAR_TO_VECTOR when the scalar is a load
3754 // of a promoted (wider than the element) type: freeze(load) can never be
3755 // folded away (the loaded value may be poison in memory), and the extra
3756 // freeze node then blocks ISel patterns matching scalar_to_vector of a
3757 // load, e.g. the AArch64 scalar_to_vector(extload) -> ldr b/h forms.
3758 // freeze(scalar_to_vector(load)) is equivalent for the demanded element
3759 // zero, and ISel selects the freeze as a plain copy.
3760 if (N0.getOpcode() == ISD::SCALAR_TO_VECTOR && DemandedElts == 1) {
3761 SDValue Scalar = N0.getOperand(0);
3762 bool IsPromotedLoad = Scalar.getOpcode() == ISD::LOAD &&
3763 Scalar.getValueType() != VT.getVectorElementType();
3764 if (!IsPromotedLoad)
3765 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT,
3766 TLO.DAG.getFreeze(Scalar)));
3767 }
3768 break;
3769 }
3770 case ISD::BUILD_VECTOR: {
3771 // Check all elements and simplify any unused elements with UNDEF.
3772 if (!DemandedElts.isAllOnes()) {
3773 // Don't simplify BROADCASTS.
3774 if (llvm::any_of(Op->op_values(),
3775 [&](SDValue Elt) { return Op.getOperand(0) != Elt; })) {
3777 bool Updated = false;
3778 for (unsigned i = 0; i != NumElts; ++i) {
3779 if (!DemandedElts[i] && !Ops[i].isUndef()) {
3780 Ops[i] = TLO.DAG.getUNDEF(Ops[0].getValueType());
3781 KnownUndef.setBit(i);
3782 Updated = true;
3783 }
3784 }
3785 if (Updated)
3786 return TLO.CombineTo(Op, TLO.DAG.getBuildVector(VT, DL, Ops));
3787 }
3788 }
3789 for (unsigned i = 0; i != NumElts; ++i) {
3790 SDValue SrcOp = Op.getOperand(i);
3791 if (SrcOp.isUndef()) {
3792 KnownUndef.setBit(i);
3793 } else if (EltSizeInBits == SrcOp.getScalarValueSizeInBits() &&
3795 KnownZero.setBit(i);
3796 }
3797 }
3798 break;
3799 }
3800 case ISD::CONCAT_VECTORS: {
3801 EVT SubVT = Op.getOperand(0).getValueType();
3802 unsigned NumSubVecs = Op.getNumOperands();
3803 unsigned NumSubElts = SubVT.getVectorNumElements();
3804 for (unsigned i = 0; i != NumSubVecs; ++i) {
3805 SDValue SubOp = Op.getOperand(i);
3806 APInt SubElts = DemandedElts.extractBits(NumSubElts, i * NumSubElts);
3807 APInt SubUndef, SubZero;
3808 if (SimplifyDemandedVectorElts(SubOp, SubElts, SubUndef, SubZero, TLO,
3809 Depth + 1))
3810 return true;
3811 KnownUndef.insertBits(SubUndef, i * NumSubElts);
3812 KnownZero.insertBits(SubZero, i * NumSubElts);
3813 }
3814
3815 // Attempt to avoid multi-use ops if we don't need anything from them.
3816 if (!DemandedElts.isAllOnes()) {
3817 bool FoundNewSub = false;
3818 SmallVector<SDValue, 2> DemandedSubOps;
3819 for (unsigned i = 0; i != NumSubVecs; ++i) {
3820 SDValue SubOp = Op.getOperand(i);
3821 APInt SubElts = DemandedElts.extractBits(NumSubElts, i * NumSubElts);
3823 SubOp, SubElts, TLO.DAG, Depth + 1);
3824 DemandedSubOps.push_back(NewSubOp ? NewSubOp : SubOp);
3825 FoundNewSub = NewSubOp ? true : FoundNewSub;
3826 }
3827 if (FoundNewSub) {
3828 SDValue NewOp =
3829 TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, DemandedSubOps);
3830 return TLO.CombineTo(Op, NewOp);
3831 }
3832 }
3833 break;
3834 }
3835 case ISD::INSERT_SUBVECTOR: {
3836 // Demand any elements from the subvector and the remainder from the src it
3837 // is inserted into.
3838 SDValue Src = Op.getOperand(0);
3839 SDValue Sub = Op.getOperand(1);
3840 uint64_t Idx = Op.getConstantOperandVal(2);
3841 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
3842 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
3843 APInt DemandedSrcElts = DemandedElts;
3844 DemandedSrcElts.clearBits(Idx, Idx + NumSubElts);
3845
3846 // If none of the sub operand elements are demanded, bypass the insert.
3847 if (!DemandedSubElts)
3848 return TLO.CombineTo(Op, Src);
3849
3850 APInt SubUndef, SubZero;
3851 if (SimplifyDemandedVectorElts(Sub, DemandedSubElts, SubUndef, SubZero, TLO,
3852 Depth + 1))
3853 return true;
3854
3855 // If none of the src operand elements are demanded, replace it with undef.
3856 if (!DemandedSrcElts && !Src.isUndef())
3857 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
3858 TLO.DAG.getUNDEF(VT), Sub,
3859 Op.getOperand(2)));
3860
3861 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownUndef, KnownZero,
3862 TLO, Depth + 1))
3863 return true;
3864 KnownUndef.insertBits(SubUndef, Idx);
3865 KnownZero.insertBits(SubZero, Idx);
3866
3867 // Attempt to avoid multi-use ops if we don't need anything from them.
3868 if (!DemandedSrcElts.isAllOnes() || !DemandedSubElts.isAllOnes()) {
3870 Src, DemandedSrcElts, TLO.DAG, Depth + 1);
3872 Sub, DemandedSubElts, TLO.DAG, Depth + 1);
3873 if (NewSrc || NewSub) {
3874 NewSrc = NewSrc ? NewSrc : Src;
3875 NewSub = NewSub ? NewSub : Sub;
3876 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,
3877 NewSub, Op.getOperand(2));
3878 return TLO.CombineTo(Op, NewOp);
3879 }
3880 }
3881 break;
3882 }
3884 // Offset the demanded elts by the subvector index.
3885 SDValue Src = Op.getOperand(0);
3886 if (Src.getValueType().isScalableVector())
3887 break;
3888 uint64_t Idx = Op.getConstantOperandVal(1);
3889 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3890 APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx);
3891
3892 APInt SrcUndef, SrcZero;
3893 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
3894 Depth + 1))
3895 return true;
3896 KnownUndef = SrcUndef.extractBits(NumElts, Idx);
3897 KnownZero = SrcZero.extractBits(NumElts, Idx);
3898
3899 // Attempt to avoid multi-use ops if we don't need anything from them.
3900 SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(Src, DemandedSrcElts,
3901 TLO.DAG, Depth + 1);
3902 if (NewSrc) {
3903 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,
3904 Op.getOperand(1));
3905 return TLO.CombineTo(Op, NewOp);
3906 }
3907 break;
3908 }
3910 SDValue Vec = Op.getOperand(0);
3911 SDValue Scl = Op.getOperand(1);
3912 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
3913
3914 // For a legal, constant insertion index, if we don't need this insertion
3915 // then strip it, else remove it from the demanded elts.
3916 if (CIdx && CIdx->getAPIntValue().ult(NumElts)) {
3917 unsigned Idx = CIdx->getZExtValue();
3918 if (!DemandedElts[Idx])
3919 return TLO.CombineTo(Op, Vec);
3920
3921 APInt DemandedVecElts(DemandedElts);
3922 DemandedVecElts.clearBit(Idx);
3923 if (SimplifyDemandedVectorElts(Vec, DemandedVecElts, KnownUndef,
3924 KnownZero, TLO, Depth + 1))
3925 return true;
3926
3927 KnownUndef.setBitVal(Idx, Scl.isUndef());
3928
3929 KnownZero.setBitVal(Idx, isNullConstant(Scl) || isNullFPConstant(Scl));
3930 break;
3931 }
3932
3933 APInt VecUndef, VecZero;
3934 if (SimplifyDemandedVectorElts(Vec, DemandedElts, VecUndef, VecZero, TLO,
3935 Depth + 1))
3936 return true;
3937 // Without knowing the insertion index we can't set KnownUndef/KnownZero.
3938 break;
3939 }
3940 case ISD::VSELECT: {
3941 SDValue Sel = Op.getOperand(0);
3942 SDValue LHS = Op.getOperand(1);
3943 SDValue RHS = Op.getOperand(2);
3944
3945 // Try to transform the select condition based on the current demanded
3946 // elements.
3947 APInt UndefSel, ZeroSel;
3948 if (SimplifyDemandedVectorElts(Sel, DemandedElts, UndefSel, ZeroSel, TLO,
3949 Depth + 1))
3950 return true;
3951
3952 // See if we can simplify either vselect operand.
3953 APInt DemandedLHS(DemandedElts);
3954 APInt DemandedRHS(DemandedElts);
3955 APInt UndefLHS, ZeroLHS;
3956 APInt UndefRHS, ZeroRHS;
3957 if (SimplifyDemandedVectorElts(LHS, DemandedLHS, UndefLHS, ZeroLHS, TLO,
3958 Depth + 1))
3959 return true;
3960 if (SimplifyDemandedVectorElts(RHS, DemandedRHS, UndefRHS, ZeroRHS, TLO,
3961 Depth + 1))
3962 return true;
3963
3964 KnownUndef = UndefLHS & UndefRHS;
3965 KnownZero = ZeroLHS & ZeroRHS;
3966
3967 // If we know that the selected element is always zero, we don't need the
3968 // select value element.
3969 APInt DemandedSel = DemandedElts & ~KnownZero;
3970 if (DemandedSel != DemandedElts)
3971 if (SimplifyDemandedVectorElts(Sel, DemandedSel, UndefSel, ZeroSel, TLO,
3972 Depth + 1))
3973 return true;
3974
3975 break;
3976 }
3977 case ISD::VECTOR_SHUFFLE: {
3978 SDValue LHS = Op.getOperand(0);
3979 SDValue RHS = Op.getOperand(1);
3980 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
3981
3982 // Collect demanded elements from shuffle operands..
3983 APInt DemandedLHS(NumElts, 0);
3984 APInt DemandedRHS(NumElts, 0);
3985 for (unsigned i = 0; i != NumElts; ++i) {
3986 int M = ShuffleMask[i];
3987 if (M < 0 || !DemandedElts[i])
3988 continue;
3989 assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range");
3990 if (M < (int)NumElts)
3991 DemandedLHS.setBit(M);
3992 else
3993 DemandedRHS.setBit(M - NumElts);
3994 }
3995
3996 // If either side isn't demanded, replace it by UNDEF. We handle this
3997 // explicitly here to also simplify in case of multiple uses (on the
3998 // contrary to the SimplifyDemandedVectorElts calls below).
3999 bool FoldLHS = !DemandedLHS && !LHS.isUndef();
4000 bool FoldRHS = !DemandedRHS && !RHS.isUndef();
4001 if (FoldLHS || FoldRHS) {
4002 LHS = FoldLHS ? TLO.DAG.getUNDEF(LHS.getValueType()) : LHS;
4003 RHS = FoldRHS ? TLO.DAG.getUNDEF(RHS.getValueType()) : RHS;
4004 SDValue NewOp =
4005 TLO.DAG.getVectorShuffle(VT, SDLoc(Op), LHS, RHS, ShuffleMask);
4006 return TLO.CombineTo(Op, NewOp);
4007 }
4008
4009 // See if we can simplify either shuffle operand.
4010 APInt UndefLHS, ZeroLHS;
4011 APInt UndefRHS, ZeroRHS;
4012 if (SimplifyDemandedVectorElts(LHS, DemandedLHS, UndefLHS, ZeroLHS, TLO,
4013 Depth + 1))
4014 return true;
4015 if (SimplifyDemandedVectorElts(RHS, DemandedRHS, UndefRHS, ZeroRHS, TLO,
4016 Depth + 1))
4017 return true;
4018
4019 // Simplify mask using undef elements from LHS/RHS.
4020 bool Updated = false;
4021 bool IdentityLHS = true, IdentityRHS = true;
4022 SmallVector<int, 32> NewMask(ShuffleMask);
4023 for (unsigned i = 0; i != NumElts; ++i) {
4024 int &M = NewMask[i];
4025 if (M < 0)
4026 continue;
4027 if (!DemandedElts[i] || (M < (int)NumElts && UndefLHS[M]) ||
4028 (M >= (int)NumElts && UndefRHS[M - NumElts])) {
4029 Updated = true;
4030 M = -1;
4031 }
4032 IdentityLHS &= (M < 0) || (M == (int)i);
4033 IdentityRHS &= (M < 0) || ((M - NumElts) == i);
4034 }
4035
4036 // Update legal shuffle masks based on demanded elements if it won't reduce
4037 // to Identity which can cause premature removal of the shuffle mask.
4038 if (Updated && !IdentityLHS && !IdentityRHS && !TLO.LegalOps) {
4039 SDValue LegalShuffle =
4040 buildLegalVectorShuffle(VT, DL, LHS, RHS, NewMask, TLO.DAG);
4041 if (LegalShuffle)
4042 return TLO.CombineTo(Op, LegalShuffle);
4043 }
4044
4045 // Propagate undef/zero elements from LHS/RHS.
4046 for (unsigned i = 0; i != NumElts; ++i) {
4047 int M = ShuffleMask[i];
4048 if (M < 0) {
4049 KnownUndef.setBit(i);
4050 } else if (M < (int)NumElts) {
4051 if (UndefLHS[M])
4052 KnownUndef.setBit(i);
4053 if (ZeroLHS[M])
4054 KnownZero.setBit(i);
4055 } else {
4056 if (UndefRHS[M - NumElts])
4057 KnownUndef.setBit(i);
4058 if (ZeroRHS[M - NumElts])
4059 KnownZero.setBit(i);
4060 }
4061 }
4062 break;
4063 }
4067 APInt SrcUndef, SrcZero;
4068 SDValue Src = Op.getOperand(0);
4069 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
4070 APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts);
4071 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
4072 Depth + 1))
4073 return true;
4074 KnownZero = SrcZero.zextOrTrunc(NumElts);
4075 KnownUndef = SrcUndef.zextOrTrunc(NumElts);
4076
4077 if (IsLE && Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG &&
4078 Op.getValueSizeInBits() == Src.getValueSizeInBits() &&
4079 DemandedSrcElts == 1) {
4080 // aext - if we just need the bottom element then we can bitcast.
4081 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
4082 }
4083
4084 if (Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) {
4085 // zext(undef) upper bits are guaranteed to be zero.
4086 if (DemandedElts.isSubsetOf(KnownUndef))
4087 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
4088 KnownUndef.clearAllBits();
4089
4090 // zext - if we just need the bottom element then we can mask:
4091 // zext(and(x,c)) -> and(x,c') iff the zext is the only user of the and.
4092 if (IsLE && DemandedSrcElts == 1 && Src.getOpcode() == ISD::AND &&
4093 Op->isOnlyUserOf(Src.getNode()) &&
4094 Op.getValueSizeInBits() == Src.getValueSizeInBits()) {
4095 SDLoc DL(Op);
4096 EVT SrcVT = Src.getValueType();
4097 EVT SrcSVT = SrcVT.getScalarType();
4098
4099 // If we're after type legalization and SrcSVT is not legal, use the
4100 // promoted type for creating constants to avoid creating nodes with
4101 // illegal types.
4102 if (TLO.LegalTypes())
4103 SrcSVT = getLegalTypeToTransformTo(*TLO.DAG.getContext(), SrcSVT);
4104
4105 SmallVector<SDValue> MaskElts;
4106 MaskElts.push_back(TLO.DAG.getAllOnesConstant(DL, SrcSVT));
4107 MaskElts.append(NumSrcElts - 1, TLO.DAG.getConstant(0, DL, SrcSVT));
4108 SDValue Mask = TLO.DAG.getBuildVector(SrcVT, DL, MaskElts);
4109 if (SDValue Fold = TLO.DAG.FoldConstantArithmetic(
4110 ISD::AND, DL, SrcVT, {Src.getOperand(1), Mask})) {
4111 Fold = TLO.DAG.getNode(ISD::AND, DL, SrcVT, Src.getOperand(0), Fold);
4112 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Fold));
4113 }
4114 }
4115 }
4116 break;
4117 }
4118
4119 // TODO: There are more binop opcodes that could be handled here - MIN,
4120 // MAX, saturated math, etc.
4121 case ISD::ADD: {
4122 SDValue Op0 = Op.getOperand(0);
4123 SDValue Op1 = Op.getOperand(1);
4124 if (Op0 == Op1 && Op->isOnlyUserOf(Op0.getNode())) {
4125 APInt UndefLHS, ZeroLHS;
4126 if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
4127 Depth + 1, /*AssumeSingleUse*/ true))
4128 return true;
4129 }
4130 [[fallthrough]];
4131 }
4132 case ISD::AVGCEILS:
4133 case ISD::AVGCEILU:
4134 case ISD::AVGFLOORS:
4135 case ISD::AVGFLOORU:
4136 case ISD::AVGSUBFLOORS:
4137 case ISD::AVGSUBFLOORU:
4138 case ISD::OR:
4139 case ISD::XOR:
4140 case ISD::SUB:
4141 case ISD::FADD:
4142 case ISD::FSUB:
4143 case ISD::FMUL:
4144 case ISD::FDIV:
4145 case ISD::FREM:
4146 case ISD::PSEUDO_FMIN:
4147 case ISD::PSEUDO_FMAX: {
4148 SDValue Op0 = Op.getOperand(0);
4149 SDValue Op1 = Op.getOperand(1);
4150
4151 APInt UndefRHS, ZeroRHS;
4152 if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,
4153 Depth + 1))
4154 return true;
4155 APInt UndefLHS, ZeroLHS;
4156 if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
4157 Depth + 1))
4158 return true;
4159
4160 // 0.0 / 0.0 and frem(0.0, 0.0) are NaN, not zero.
4161 if (Opcode != ISD::FDIV && Opcode != ISD::FREM)
4162 KnownZero = ZeroLHS & ZeroRHS;
4163 KnownUndef = getKnownUndefForVectorBinop(Op, TLO.DAG, UndefLHS, UndefRHS);
4164
4165 // Attempt to avoid multi-use ops if we don't need anything from them.
4166 // TODO - use KnownUndef to relax the demandedelts?
4167 if (!DemandedElts.isAllOnes())
4168 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
4169 return true;
4170 break;
4171 }
4172 case ISD::SHL:
4173 case ISD::SRL:
4174 case ISD::SRA:
4175 case ISD::ROTL:
4176 case ISD::ROTR: {
4177 SDValue Op0 = Op.getOperand(0);
4178 SDValue Op1 = Op.getOperand(1);
4179
4180 APInt UndefRHS, ZeroRHS;
4181 if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,
4182 Depth + 1))
4183 return true;
4184 APInt UndefLHS, ZeroLHS;
4185 if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
4186 Depth + 1))
4187 return true;
4188
4189 KnownZero = ZeroLHS & ~UndefRHS;
4190 KnownUndef = UndefLHS & UndefRHS; // TODO: use getKnownUndefForVectorBinop?
4191
4192 // Attempt to avoid multi-use ops if we don't need anything from them.
4193 // TODO - use KnownUndef to relax the demandedelts?
4194 if (!DemandedElts.isAllOnes())
4195 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
4196 return true;
4197 break;
4198 }
4199 case ISD::MUL:
4200 case ISD::MULHU:
4201 case ISD::MULHS:
4202 case ISD::AND: {
4203 SDValue Op0 = Op.getOperand(0);
4204 SDValue Op1 = Op.getOperand(1);
4205
4206 APInt SrcUndef, SrcZero;
4207 if (SimplifyDemandedVectorElts(Op1, DemandedElts, SrcUndef, SrcZero, TLO,
4208 Depth + 1))
4209 return true;
4210 // FIXME: If we know that a demanded element was zero in Op1 we don't need
4211 // to demand it in Op0 - its guaranteed to be zero. There is however a
4212 // restriction, as we must not make any of the originally demanded elements
4213 // more poisonous. We could reduce amount of elements demanded, but then we
4214 // also need a to inform SimplifyDemandedVectorElts that some elements must
4215 // not be made more poisonous.
4216 if (SimplifyDemandedVectorElts(Op0, DemandedElts, KnownUndef, KnownZero,
4217 TLO, Depth + 1))
4218 return true;
4219
4220 KnownUndef &= DemandedElts;
4221 KnownZero &= DemandedElts;
4222
4223 // If every element pair has a zero/undef/poison then just fold to zero.
4224 // fold (and x, undef/poison) -> 0 / (and x, 0) -> 0
4225 // fold (mul x, undef/poison) -> 0 / (mul x, 0) -> 0
4226 if (DemandedElts.isSubsetOf(SrcZero | KnownZero | SrcUndef | KnownUndef))
4227 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
4228
4229 // If either side has a zero element, then the result element is zero, even
4230 // if the other is an UNDEF.
4231 // TODO: Extend getKnownUndefForVectorBinop to also deal with known zeros
4232 // and then handle 'and' nodes with the rest of the binop opcodes.
4233 KnownZero |= SrcZero;
4234 KnownUndef &= SrcUndef;
4235 KnownUndef &= ~KnownZero;
4236
4237 // Attempt to avoid multi-use ops if we don't need anything from them.
4238 if (!DemandedElts.isAllOnes())
4239 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
4240 return true;
4241 break;
4242 }
4243 case ISD::TRUNCATE:
4244 case ISD::SIGN_EXTEND:
4245 case ISD::ZERO_EXTEND:
4246 if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,
4247 KnownZero, TLO, Depth + 1))
4248 return true;
4249
4250 if (!DemandedElts.isAllOnes())
4252 Op.getOperand(0), DemandedElts, TLO.DAG, Depth + 1))
4253 return TLO.CombineTo(Op, TLO.DAG.getNode(Opcode, SDLoc(Op), VT, NewOp));
4254
4255 if (Op.getOpcode() == ISD::ZERO_EXTEND) {
4256 // zext(undef) upper bits are guaranteed to be zero.
4257 if (DemandedElts.isSubsetOf(KnownUndef))
4258 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
4259 KnownUndef.clearAllBits();
4260 }
4261 break;
4262 case ISD::SINT_TO_FP:
4263 case ISD::UINT_TO_FP:
4264 case ISD::FP_TO_SINT:
4265 case ISD::FP_TO_UINT:
4266 if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,
4267 KnownZero, TLO, Depth + 1))
4268 return true;
4269 // Don't fall through to generic undef -> undef handling.
4270 return false;
4271 default: {
4272 if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
4273 if (SimplifyDemandedVectorEltsForTargetNode(Op, DemandedElts, KnownUndef,
4274 KnownZero, TLO, Depth))
4275 return true;
4276 } else {
4278 APInt DemandedBits = APInt::getAllOnes(EltSizeInBits);
4279 if (SimplifyDemandedBits(Op, DemandedBits, OriginalDemandedElts, Known,
4280 TLO, Depth, AssumeSingleUse))
4281 return true;
4282 }
4283 break;
4284 }
4285 }
4286
4287 assert((KnownUndef & KnownZero) == 0 && "Elements flagged as undef AND zero");
4288
4289 // Constant fold all undef cases.
4290 // TODO: Handle zero cases as well.
4291 if (DemandedElts.isSubsetOf(KnownUndef))
4292 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
4293
4294 return false;
4295}
4296
4297/// Determine which of the bits specified in Mask are known to be either zero or
4298/// one and return them in the Known.
4301 const APInt &DemandedElts,
4302 const SelectionDAG &DAG,
4303 unsigned Depth) const {
4304 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4305 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4306 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4307 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4308 "Should use MaskedValueIsZero if you don't know whether Op"
4309 " is a target node!");
4310 Known.resetAll();
4311}
4312
4315 const APInt &DemandedElts, const MachineRegisterInfo &MRI,
4316 unsigned Depth) const {
4317 Known.resetAll();
4318}
4319
4322 const APInt &DemandedElts, const MachineRegisterInfo &MRI,
4323 unsigned Depth) const {
4324 Known.resetAll();
4325}
4326
4328 KnownBits &Known, const MachineFunction &, Align Alignment) const {
4329 // The low bits are known zero if the pointer is aligned.
4330 Known.Zero.setLowBits(Log2(Alignment));
4331}
4332
4334 SelectionDAG &DAG,
4335 const SDLoc &DL,
4336 Align Alignment) const {
4337 // Materialize leading-zero stack object pointer facts as AssertZext.
4338 // Alignment-derived low zero bits are not represented on the returned DAG
4339 // value here.
4340 EVT PtrVT = Ptr.getValueType();
4341
4342 unsigned RegSize = PtrVT.getScalarSizeInBits();
4345 Alignment);
4346
4347 unsigned NumZeroBits = Known.countMinLeadingZeros();
4348 if (!NumZeroBits)
4349 return Ptr;
4350
4351 EVT FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits);
4352 return DAG.getNode(ISD::AssertZext, DL, PtrVT, Ptr, DAG.getValueType(FromVT));
4353}
4354
4360
4361/// This method can be implemented by targets that want to expose additional
4362/// information about sign bits to the DAG Combiner.
4364 const APInt &,
4365 const SelectionDAG &,
4366 unsigned Depth) const {
4367 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4368 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4369 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4370 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4371 "Should use ComputeNumSignBits if you don't know whether Op"
4372 " is a target node!");
4373 return 1;
4374}
4375
4377 GISelValueTracking &Analysis, Register R, const APInt &DemandedElts,
4378 const MachineRegisterInfo &MRI, unsigned Depth) const {
4379 return 1;
4380}
4381
4383 SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,
4384 TargetLoweringOpt &TLO, unsigned Depth) const {
4385 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4386 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4387 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4388 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4389 "Should use SimplifyDemandedVectorElts if you don't know whether Op"
4390 " is a target node!");
4391 return false;
4392}
4393
4395 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
4396 KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const {
4397 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4398 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4399 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4400 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4401 "Should use SimplifyDemandedBits if you don't know whether Op"
4402 " is a target node!");
4403 computeKnownBitsForTargetNode(Op, Known, DemandedElts, TLO.DAG, Depth);
4404 return false;
4405}
4406
4408 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
4409 SelectionDAG &DAG, unsigned Depth) const {
4410 assert(
4411 (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4412 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4413 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4414 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4415 "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
4416 " is a target node!");
4417 return SDValue();
4418}
4419
4420SDValue
4423 SelectionDAG &DAG) const {
4424 bool LegalMask = isShuffleMaskLegal(Mask, VT);
4425 if (!LegalMask) {
4426 std::swap(N0, N1);
4428 LegalMask = isShuffleMaskLegal(Mask, VT);
4429 }
4430
4431 if (!LegalMask)
4432 return SDValue();
4433
4434 return DAG.getVectorShuffle(VT, DL, N0, N1, Mask);
4435}
4436
4438 return nullptr;
4439}
4440
4442 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
4443 UndefPoisonKind Kind, unsigned Depth) const {
4444 assert(
4445 (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4446 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4447 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4448 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4449 "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
4450 " is a target node!");
4451
4452 // If Op can't create undef/poison and none of its operands are undef/poison
4453 // then Op is never undef/poison.
4454 return !canCreateUndefOrPoisonForTargetNode(Op, DemandedElts, DAG, Kind,
4455 /*ConsiderFlags*/ true, Depth) &&
4456 all_of(Op->ops(), [&](SDValue V) {
4457 return DAG.isGuaranteedNotToBeUndefOrPoison(V, Kind, Depth + 1);
4458 });
4459}
4460
4462 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
4463 UndefPoisonKind Kind, bool ConsiderFlags, unsigned Depth) const {
4464 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4465 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4466 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4467 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4468 "Should use canCreateUndefOrPoison if you don't know whether Op"
4469 " is a target node!");
4470 // Be conservative and return true.
4471 return true;
4472}
4473
4476 const APInt &DemandedElts,
4477 const SelectionDAG &DAG,
4478 unsigned Depth) const {
4479 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4480 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4481 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4482 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4483 "Should use computeKnownFPClass if you don't know whether Op"
4484 " is a target node!");
4485}
4486
4488 const APInt &DemandedElts,
4489 const SelectionDAG &DAG,
4490 bool SNaN,
4491 unsigned Depth) const {
4492 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4493 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4494 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4495 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4496 "Should use isKnownNeverNaN if you don't know whether Op"
4497 " is a target node!");
4498 return false;
4499}
4500
4502 const APInt &DemandedElts,
4503 APInt &UndefElts,
4504 const SelectionDAG &DAG,
4505 unsigned Depth) const {
4506 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4507 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4508 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4509 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4510 "Should use isSplatValue if you don't know whether Op"
4511 " is a target node!");
4512 return false;
4513}
4514
4515// FIXME: Ideally, this would use ISD::isConstantSplatVector(), but that must
4516// work with truncating build vectors and vectors with elements of less than
4517// 8 bits.
4519 if (!N)
4520 return false;
4521
4522 unsigned EltWidth;
4523 APInt CVal;
4524 if (ConstantSDNode *CN = isConstOrConstSplat(N, /*AllowUndefs=*/false,
4525 /*AllowTruncation=*/true)) {
4526 CVal = CN->getAPIntValue();
4527 EltWidth = N.getValueType().getScalarSizeInBits();
4528 } else
4529 return false;
4530
4531 // If this is a truncating splat, truncate the splat value.
4532 // Otherwise, we may fail to match the expected values below.
4533 if (EltWidth < CVal.getBitWidth())
4534 CVal = CVal.trunc(EltWidth);
4535
4536 switch (getBooleanContents(N.getValueType())) {
4538 return CVal[0];
4540 return CVal.isOne();
4542 return CVal.isAllOnes();
4543 }
4544
4545 llvm_unreachable("Invalid boolean contents");
4546}
4547
4549 if (!N)
4550 return false;
4551
4553 if (!CN) {
4555 if (!BV)
4556 return false;
4557
4558 // Only interested in constant splats, we don't care about undef
4559 // elements in identifying boolean constants and getConstantSplatNode
4560 // returns NULL if all ops are undef;
4561 CN = BV->getConstantSplatNode();
4562 if (!CN)
4563 return false;
4564 }
4565
4566 if (getBooleanContents(N->getValueType(0)) == UndefinedBooleanContent)
4567 return !CN->getAPIntValue()[0];
4568
4569 return CN->isZero();
4570}
4571
4573 bool SExt) const {
4574 if (VT == MVT::i1)
4575 return N->isOne();
4576
4578 switch (Cnt) {
4580 // An extended value of 1 is always true, unless its original type is i1,
4581 // in which case it will be sign extended to -1.
4582 return (N->isOne() && !SExt) || (SExt && (N->getValueType(0) != MVT::i1));
4585 return N->isAllOnes() && SExt;
4586 }
4587 llvm_unreachable("Unexpected enumeration.");
4588}
4589
4590/// This helper function of SimplifySetCC tries to optimize the comparison when
4591/// either operand of the SetCC node is a bitwise-and instruction.
4592SDValue TargetLowering::foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1,
4593 ISD::CondCode Cond, const SDLoc &DL,
4594 DAGCombinerInfo &DCI) const {
4595 if (N1.getOpcode() == ISD::AND && N0.getOpcode() != ISD::AND)
4596 std::swap(N0, N1);
4597
4598 SelectionDAG &DAG = DCI.DAG;
4599 EVT OpVT = N0.getValueType();
4600 if (N0.getOpcode() != ISD::AND || !OpVT.isInteger() ||
4601 (Cond != ISD::SETEQ && Cond != ISD::SETNE))
4602 return SDValue();
4603
4604 // (X & Y) != 0 --> zextOrTrunc(X & Y)
4605 // iff everything but LSB is known zero:
4606 if (Cond == ISD::SETNE && isNullConstant(N1) &&
4609 unsigned NumEltBits = OpVT.getScalarSizeInBits();
4610 APInt UpperBits = APInt::getHighBitsSet(NumEltBits, NumEltBits - 1);
4611 if (DAG.MaskedValueIsZero(N0, UpperBits))
4612 return DAG.getBoolExtOrTrunc(N0, DL, VT, OpVT);
4613 }
4614
4615 // Try to eliminate a power-of-2 mask constant by converting to a signbit
4616 // test in a narrow type that we can truncate to with no cost. Examples:
4617 // (i32 X & 32768) == 0 --> (trunc X to i16) >= 0
4618 // (i32 X & 32768) != 0 --> (trunc X to i16) < 0
4619 // TODO: This conservatively checks for type legality on the source and
4620 // destination types. That may inhibit optimizations, but it also
4621 // allows setcc->shift transforms that may be more beneficial.
4622 auto *AndC = dyn_cast<ConstantSDNode>(N0.getOperand(1));
4623 if (AndC && isNullConstant(N1) && AndC->getAPIntValue().isPowerOf2() &&
4624 isTypeLegal(OpVT) && N0.hasOneUse()) {
4625 EVT NarrowVT = EVT::getIntegerVT(*DAG.getContext(),
4626 AndC->getAPIntValue().getActiveBits());
4627 if (isTruncateFree(OpVT, NarrowVT) && isTypeLegal(NarrowVT)) {
4628 SDValue Trunc = DAG.getZExtOrTrunc(N0.getOperand(0), DL, NarrowVT);
4629 SDValue Zero = DAG.getConstant(0, DL, NarrowVT);
4630 return DAG.getSetCC(DL, VT, Trunc, Zero,
4632 }
4633 }
4634
4635 // Match these patterns in any of their permutations:
4636 // (X & Y) == Y
4637 // (X & Y) != Y
4638 SDValue X, Y;
4639 if (N0.getOperand(0) == N1) {
4640 X = N0.getOperand(1);
4641 Y = N0.getOperand(0);
4642 } else if (N0.getOperand(1) == N1) {
4643 X = N0.getOperand(0);
4644 Y = N0.getOperand(1);
4645 } else {
4646 return SDValue();
4647 }
4648
4649 // TODO: We should invert (X & Y) eq/ne 0 -> (X & Y) ne/eq Y if
4650 // `isXAndYEqZeroPreferableToXAndYEqY` is false. This is a bit difficult as
4651 // its liable to create and infinite loop.
4652 SDValue Zero = DAG.getConstant(0, DL, OpVT);
4653 if (isXAndYEqZeroPreferableToXAndYEqY(Cond, OpVT) &&
4655 // Simplify X & Y == Y to X & Y != 0 if Y has exactly one bit set.
4656 // Note that where Y is variable and is known to have at most one bit set
4657 // (for example, if it is Z & 1) we cannot do this; the expressions are not
4658 // equivalent when Y == 0.
4659 assert(OpVT.isInteger());
4661 if (DCI.isBeforeLegalizeOps() ||
4663 return DAG.getSetCC(DL, VT, N0, Zero, Cond);
4664 } else if (N0.hasOneUse() && hasAndNotCompare(Y)) {
4665 // If the target supports an 'and-not' or 'and-complement' logic operation,
4666 // try to use that to make a comparison operation more efficient.
4667 // But don't do this transform if the mask is a single bit because there are
4668 // more efficient ways to deal with that case (for example, 'bt' on x86 or
4669 // 'rlwinm' on PPC).
4670
4671 // Bail out if the compare operand that we want to turn into a zero is
4672 // already a zero (otherwise, infinite loop).
4673 if (isNullConstant(Y))
4674 return SDValue();
4675
4676 // Transform this into: ~X & Y == 0.
4677 SDValue NotX = DAG.getNOT(SDLoc(X), X, OpVT);
4678 SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, NotX, Y);
4679 return DAG.getSetCC(DL, VT, NewAnd, Zero, Cond);
4680 }
4681
4682 return SDValue();
4683}
4684
4685/// This helper function of SimplifySetCC tries to optimize the comparison when
4686/// either operand of the SetCC node is a bitwise-or instruction.
4687/// For now, this just transforms (X | Y) ==/!= Y into X & ~Y ==/!= 0.
4688SDValue TargetLowering::foldSetCCWithOr(EVT VT, SDValue N0, SDValue N1,
4689 ISD::CondCode Cond, const SDLoc &DL,
4690 DAGCombinerInfo &DCI) const {
4691 if (N1.getOpcode() == ISD::OR && N0.getOpcode() != ISD::OR)
4692 std::swap(N0, N1);
4693
4694 SelectionDAG &DAG = DCI.DAG;
4695 EVT OpVT = N0.getValueType();
4696 if (!N0.hasOneUse() || !OpVT.isInteger() ||
4697 (Cond != ISD::SETEQ && Cond != ISD::SETNE))
4698 return SDValue();
4699
4700 // (X | Y) == Y
4701 // (X | Y) != Y
4702 SDValue X;
4703 if (sd_match(N0, m_Or(m_Value(X), m_Specific(N1))) && hasAndNotCompare(X)) {
4704 // If the target supports an 'and-not' or 'and-complement' logic operation,
4705 // try to use that to make a comparison operation more efficient.
4706
4707 // Bail out if the compare operand that we want to turn into a zero is
4708 // already a zero (otherwise, infinite loop).
4709 if (isNullConstant(N1))
4710 return SDValue();
4711
4712 // Transform this into: X & ~Y ==/!= 0.
4713 SDValue NotY = DAG.getNOT(SDLoc(N1), N1, OpVT);
4714 SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, X, NotY);
4715 return DAG.getSetCC(DL, VT, NewAnd, DAG.getConstant(0, DL, OpVT), Cond);
4716 }
4717
4718 return SDValue();
4719}
4720
4721/// There are multiple IR patterns that could be checking whether certain
4722/// truncation of a signed number would be lossy or not. The pattern which is
4723/// best at IR level, may not lower optimally. Thus, we want to unfold it.
4724/// We are looking for the following pattern: (KeptBits is a constant)
4725/// (add %x, (1 << (KeptBits-1))) srccond (1 << KeptBits)
4726/// KeptBits won't be bitwidth(x), that will be constant-folded to true/false.
4727/// KeptBits also can't be 1, that would have been folded to %x dstcond 0
4728/// We will unfold it into the natural trunc+sext pattern:
4729/// ((%x << C) a>> C) dstcond %x
4730/// Where C = bitwidth(x) - KeptBits and C u< bitwidth(x)
4731SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(
4732 EVT SCCVT, SDValue N0, SDValue N1, ISD::CondCode Cond, DAGCombinerInfo &DCI,
4733 const SDLoc &DL) const {
4734 // We must be comparing with a constant.
4735 ConstantSDNode *C1;
4736 if (!(C1 = dyn_cast<ConstantSDNode>(N1)))
4737 return SDValue();
4738
4739 // N0 should be: add %x, (1 << (KeptBits-1))
4740 if (N0->getOpcode() != ISD::ADD)
4741 return SDValue();
4742
4743 // And we must be 'add'ing a constant.
4744 ConstantSDNode *C01;
4745 if (!(C01 = dyn_cast<ConstantSDNode>(N0->getOperand(1))))
4746 return SDValue();
4747
4748 SDValue X = N0->getOperand(0);
4749 EVT XVT = X.getValueType();
4750
4751 // Validate constants ...
4752
4753 APInt I1 = C1->getAPIntValue();
4754
4755 ISD::CondCode NewCond;
4756 if (Cond == ISD::CondCode::SETULT) {
4757 NewCond = ISD::CondCode::SETEQ;
4758 } else if (Cond == ISD::CondCode::SETULE) {
4759 NewCond = ISD::CondCode::SETEQ;
4760 // But need to 'canonicalize' the constant.
4761 I1 += 1;
4762 } else if (Cond == ISD::CondCode::SETUGT) {
4763 NewCond = ISD::CondCode::SETNE;
4764 // But need to 'canonicalize' the constant.
4765 I1 += 1;
4766 } else if (Cond == ISD::CondCode::SETUGE) {
4767 NewCond = ISD::CondCode::SETNE;
4768 } else
4769 return SDValue();
4770
4771 APInt I01 = C01->getAPIntValue();
4772
4773 auto checkConstants = [&I1, &I01]() -> bool {
4774 // Both of them must be power-of-two, and the constant from setcc is bigger.
4775 return I1.ugt(I01) && I1.isPowerOf2() && I01.isPowerOf2();
4776 };
4777
4778 if (checkConstants()) {
4779 // Great, e.g. got icmp ult i16 (add i16 %x, 128), 256
4780 } else {
4781 // What if we invert constants? (and the target predicate)
4782 I1.negate();
4783 I01.negate();
4784 assert(XVT.isInteger());
4785 NewCond = getSetCCInverse(NewCond, XVT);
4786 if (!checkConstants())
4787 return SDValue();
4788 // Great, e.g. got icmp uge i16 (add i16 %x, -128), -256
4789 }
4790
4791 // They are power-of-two, so which bit is set?
4792 const unsigned KeptBits = I1.logBase2();
4793 const unsigned KeptBitsMinusOne = I01.logBase2();
4794
4795 // Magic!
4796 if (KeptBits != (KeptBitsMinusOne + 1))
4797 return SDValue();
4798 assert(KeptBits > 0 && KeptBits < XVT.getSizeInBits() && "unreachable");
4799
4800 // We don't want to do this in every single case.
4801 SelectionDAG &DAG = DCI.DAG;
4802 if (!shouldTransformSignedTruncationCheck(XVT, KeptBits))
4803 return SDValue();
4804
4805 // Unfold into: sext_inreg(%x) cond %x
4806 // Where 'cond' will be either 'eq' or 'ne'.
4807 SDValue SExtInReg = DAG.getNode(
4809 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), KeptBits)));
4810 return DAG.getSetCC(DL, SCCVT, SExtInReg, X, NewCond);
4811}
4812
4813// (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
4814SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(
4815 EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond,
4816 DAGCombinerInfo &DCI, const SDLoc &DL) const {
4818 "Should be a comparison with 0.");
4819 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4820 "Valid only for [in]equality comparisons.");
4821
4822 unsigned NewShiftOpcode;
4823 SDValue X, C, Y;
4824
4825 SelectionDAG &DAG = DCI.DAG;
4826
4827 // Look for '(C l>>/<< Y)'.
4828 auto Match = [&NewShiftOpcode, &X, &C, &Y, &DAG, this](SDValue V) {
4829 // The shift should be one-use.
4830 if (!V.hasOneUse())
4831 return false;
4832 unsigned OldShiftOpcode = V.getOpcode();
4833 switch (OldShiftOpcode) {
4834 case ISD::SHL:
4835 NewShiftOpcode = ISD::SRL;
4836 break;
4837 case ISD::SRL:
4838 NewShiftOpcode = ISD::SHL;
4839 break;
4840 default:
4841 return false; // must be a logical shift.
4842 }
4843 // We should be shifting a constant.
4844 // FIXME: best to use isConstantOrConstantVector().
4845 C = V.getOperand(0);
4846 ConstantSDNode *CC =
4847 isConstOrConstSplat(C, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
4848 if (!CC)
4849 return false;
4850 Y = V.getOperand(1);
4851
4852 ConstantSDNode *XC =
4853 isConstOrConstSplat(X, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
4855 X, XC, CC, Y, OldShiftOpcode, NewShiftOpcode, DAG);
4856 };
4857
4858 // LHS of comparison should be an one-use 'and'.
4859 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())
4860 return SDValue();
4861
4862 X = N0.getOperand(0);
4863 SDValue Mask = N0.getOperand(1);
4864
4865 // 'and' is commutative!
4866 if (!Match(Mask)) {
4867 std::swap(X, Mask);
4868 if (!Match(Mask))
4869 return SDValue();
4870 }
4871
4872 EVT VT = X.getValueType();
4873
4874 // Produce:
4875 // ((X 'OppositeShiftOpcode' Y) & C) Cond 0
4876 SDValue T0 = DAG.getNode(NewShiftOpcode, DL, VT, X, Y);
4877 SDValue T1 = DAG.getNode(ISD::AND, DL, VT, T0, C);
4878 SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, N1C, Cond);
4879 return T2;
4880}
4881
4882/// Try to fold an equality comparison with a {add/sub/xor} binary operation as
4883/// the 1st operand (N0). Callers are expected to swap the N0/N1 parameters to
4884/// handle the commuted versions of these patterns.
4885SDValue TargetLowering::foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1,
4886 ISD::CondCode Cond, const SDLoc &DL,
4887 DAGCombinerInfo &DCI) const {
4888 unsigned BOpcode = N0.getOpcode();
4889 assert((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) &&
4890 "Unexpected binop");
4891 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Unexpected condcode");
4892
4893 // (X + Y) == X --> Y == 0
4894 // (X - Y) == X --> Y == 0
4895 // (X ^ Y) == X --> Y == 0
4896 SelectionDAG &DAG = DCI.DAG;
4897 EVT OpVT = N0.getValueType();
4898 SDValue X = N0.getOperand(0);
4899 SDValue Y = N0.getOperand(1);
4900 if (X == N1)
4901 return DAG.getSetCC(DL, VT, Y, DAG.getConstant(0, DL, OpVT), Cond);
4902
4903 if (Y != N1)
4904 return SDValue();
4905
4906 // (X + Y) == Y --> X == 0
4907 // (X ^ Y) == Y --> X == 0
4908 if (BOpcode == ISD::ADD || BOpcode == ISD::XOR)
4909 return DAG.getSetCC(DL, VT, X, DAG.getConstant(0, DL, OpVT), Cond);
4910
4911 // The shift would not be valid if the operands are boolean (i1).
4912 if (!N0.hasOneUse() || OpVT.getScalarSizeInBits() == 1)
4913 return SDValue();
4914
4915 // (X - Y) == Y --> X == Y << 1
4916 SDValue One = DAG.getShiftAmountConstant(1, OpVT, DL);
4917 SDValue YShl1 = DAG.getNode(ISD::SHL, DL, N1.getValueType(), Y, One);
4918 if (!DCI.isCalledByLegalizer())
4919 DCI.AddToWorklist(YShl1.getNode());
4920 return DAG.getSetCC(DL, VT, X, YShl1, Cond);
4921}
4922
4924 SDValue N0, const APInt &C1,
4925 ISD::CondCode Cond, const SDLoc &dl,
4926 SelectionDAG &DAG) {
4927 // Look through truncs that don't change the value of a ctpop.
4928 // FIXME: Add vector support? Need to be careful with setcc result type below.
4929 SDValue CTPOP = N0;
4930 if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && !VT.isVector() &&
4932 CTPOP = N0.getOperand(0);
4933
4934 if (CTPOP.getOpcode() != ISD::CTPOP || !CTPOP.hasOneUse())
4935 return SDValue();
4936
4937 EVT CTVT = CTPOP.getValueType();
4938 SDValue CTOp = CTPOP.getOperand(0);
4939
4940 // Expand a power-of-2-or-zero comparison based on ctpop:
4941 // (ctpop x) u< 2 -> (x & x-1) == 0
4942 // (ctpop x) u> 1 -> (x & x-1) != 0
4943 if (Cond == ISD::SETULT || Cond == ISD::SETUGT) {
4944 // Keep the CTPOP if it is a cheap vector op.
4945 if (CTVT.isVector() && TLI.isCtpopFast(CTVT))
4946 return SDValue();
4947
4948 unsigned CostLimit = TLI.getCustomCtpopCost(CTVT, Cond);
4949 if (C1.ugt(CostLimit + (Cond == ISD::SETULT)))
4950 return SDValue();
4951 if (C1 == 0 && (Cond == ISD::SETULT))
4952 return SDValue(); // This is handled elsewhere.
4953
4954 unsigned Passes = C1.getLimitedValue() - (Cond == ISD::SETULT);
4955
4956 SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
4957 SDValue Result = CTOp;
4958 for (unsigned i = 0; i < Passes; i++) {
4959 SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, Result, NegOne);
4960 Result = DAG.getNode(ISD::AND, dl, CTVT, Result, Add);
4961 }
4963 return DAG.getSetCC(dl, VT, Result, DAG.getConstant(0, dl, CTVT), CC);
4964 }
4965
4966 // Expand a power-of-2 comparison based on ctpop
4967 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && C1 == 1) {
4968 // Keep the CTPOP if it is cheap.
4969 if (TLI.isCtpopFast(CTVT))
4970 return SDValue();
4971
4972 SDValue Zero = DAG.getConstant(0, dl, CTVT);
4973 SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
4974 assert(CTVT.isInteger());
4975 SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, CTOp, NegOne);
4976
4977 // Its not uncommon for known-never-zero X to exist in (ctpop X) eq/ne 1, so
4978 // check before emitting a potentially unnecessary op.
4979 if (DAG.isKnownNeverZero(CTOp)) {
4980 // (ctpop x) == 1 --> (x & x-1) == 0
4981 // (ctpop x) != 1 --> (x & x-1) != 0
4982 SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Add);
4983 SDValue RHS = DAG.getSetCC(dl, VT, And, Zero, Cond);
4984 return RHS;
4985 }
4986
4987 // (ctpop x) == 1 --> (x ^ x-1) > x-1
4988 // (ctpop x) != 1 --> (x ^ x-1) <= x-1
4989 SDValue Xor = DAG.getNode(ISD::XOR, dl, CTVT, CTOp, Add);
4991 return DAG.getSetCC(dl, VT, Xor, Add, CmpCond);
4992 }
4993
4994 return SDValue();
4995}
4996
4998 ISD::CondCode Cond, const SDLoc &dl,
4999 SelectionDAG &DAG) {
5000 if (Cond != ISD::SETEQ && Cond != ISD::SETNE)
5001 return SDValue();
5002
5003 auto *C1 = isConstOrConstSplat(N1, /* AllowUndefs */ true);
5004 if (!C1 || !(C1->isZero() || C1->isAllOnes()))
5005 return SDValue();
5006
5007 auto getRotateSource = [](SDValue X) {
5008 if (X.getOpcode() == ISD::ROTL || X.getOpcode() == ISD::ROTR)
5009 return X.getOperand(0);
5010 return SDValue();
5011 };
5012
5013 // Peek through a rotated value compared against 0 or -1:
5014 // (rot X, Y) == 0/-1 --> X == 0/-1
5015 // (rot X, Y) != 0/-1 --> X != 0/-1
5016 if (SDValue R = getRotateSource(N0))
5017 return DAG.getSetCC(dl, VT, R, N1, Cond);
5018
5019 // Peek through an 'or' of a rotated value compared against 0:
5020 // or (rot X, Y), Z ==/!= 0 --> (or X, Z) ==/!= 0
5021 // or Z, (rot X, Y) ==/!= 0 --> (or X, Z) ==/!= 0
5022 //
5023 // TODO: Add the 'and' with -1 sibling.
5024 // TODO: Recurse through a series of 'or' ops to find the rotate.
5025 EVT OpVT = N0.getValueType();
5026 if (N0.hasOneUse() && N0.getOpcode() == ISD::OR && C1->isZero()) {
5027 if (SDValue R = getRotateSource(N0.getOperand(0))) {
5028 SDValue NewOr = DAG.getNode(ISD::OR, dl, OpVT, R, N0.getOperand(1));
5029 return DAG.getSetCC(dl, VT, NewOr, N1, Cond);
5030 }
5031 if (SDValue R = getRotateSource(N0.getOperand(1))) {
5032 SDValue NewOr = DAG.getNode(ISD::OR, dl, OpVT, R, N0.getOperand(0));
5033 return DAG.getSetCC(dl, VT, NewOr, N1, Cond);
5034 }
5035 }
5036
5037 return SDValue();
5038}
5039
5041 ISD::CondCode Cond, const SDLoc &dl,
5042 SelectionDAG &DAG) {
5043 // If we are testing for all-bits-clear, we might be able to do that with
5044 // less shifting since bit-order does not matter.
5045 if (Cond != ISD::SETEQ && Cond != ISD::SETNE)
5046 return SDValue();
5047
5048 auto *C1 = isConstOrConstSplat(N1, /* AllowUndefs */ true);
5049 if (!C1 || !C1->isZero())
5050 return SDValue();
5051
5052 if (!N0.hasOneUse() ||
5053 (N0.getOpcode() != ISD::FSHL && N0.getOpcode() != ISD::FSHR))
5054 return SDValue();
5055
5056 unsigned BitWidth = N0.getScalarValueSizeInBits();
5057 auto *ShAmtC = isConstOrConstSplat(N0.getOperand(2));
5058 if (!ShAmtC)
5059 return SDValue();
5060
5061 uint64_t ShAmt = ShAmtC->getAPIntValue().urem(BitWidth);
5062 if (ShAmt == 0)
5063 return SDValue();
5064
5065 // Canonicalize fshr as fshl to reduce pattern-matching.
5066 if (N0.getOpcode() == ISD::FSHR)
5067 ShAmt = BitWidth - ShAmt;
5068
5069 // Match an 'or' with a specific operand 'Other' in either commuted variant.
5070 SDValue X, Y;
5071 auto matchOr = [&X, &Y](SDValue Or, SDValue Other) {
5072 if (Or.getOpcode() != ISD::OR || !Or.hasOneUse())
5073 return false;
5074 if (Or.getOperand(0) == Other) {
5075 X = Or.getOperand(0);
5076 Y = Or.getOperand(1);
5077 return true;
5078 }
5079 if (Or.getOperand(1) == Other) {
5080 X = Or.getOperand(1);
5081 Y = Or.getOperand(0);
5082 return true;
5083 }
5084 return false;
5085 };
5086
5087 EVT OpVT = N0.getValueType();
5088 EVT ShAmtVT = N0.getOperand(2).getValueType();
5089 SDValue F0 = N0.getOperand(0);
5090 SDValue F1 = N0.getOperand(1);
5091 if (matchOr(F0, F1)) {
5092 // fshl (or X, Y), X, C ==/!= 0 --> or (shl Y, C), X ==/!= 0
5093 SDValue NewShAmt = DAG.getConstant(ShAmt, dl, ShAmtVT);
5094 SDValue Shift = DAG.getNode(ISD::SHL, dl, OpVT, Y, NewShAmt);
5095 SDValue NewOr = DAG.getNode(ISD::OR, dl, OpVT, Shift, X);
5096 return DAG.getSetCC(dl, VT, NewOr, N1, Cond);
5097 }
5098 if (matchOr(F1, F0)) {
5099 // fshl X, (or X, Y), C ==/!= 0 --> or (srl Y, BW-C), X ==/!= 0
5100 SDValue NewShAmt = DAG.getConstant(BitWidth - ShAmt, dl, ShAmtVT);
5101 SDValue Shift = DAG.getNode(ISD::SRL, dl, OpVT, Y, NewShAmt);
5102 SDValue NewOr = DAG.getNode(ISD::OR, dl, OpVT, Shift, X);
5103 return DAG.getSetCC(dl, VT, NewOr, N1, Cond);
5104 }
5105
5106 return SDValue();
5107}
5108
5109/// Try to simplify a setcc built with the specified operands and cc. If it is
5110/// unable to simplify it, return a null SDValue.
5112 ISD::CondCode Cond, bool foldBooleans,
5113 DAGCombinerInfo &DCI,
5114 const SDLoc &dl) const {
5115 SelectionDAG &DAG = DCI.DAG;
5116 const DataLayout &Layout = DAG.getDataLayout();
5117 EVT OpVT = N0.getValueType();
5119
5120 // Constant fold or commute setcc.
5121 if (SDValue Fold = DAG.FoldSetCC(VT, N0, N1, Cond, dl))
5122 return Fold;
5123
5124 bool N0ConstOrSplat =
5125 isConstOrConstSplat(N0, /*AllowUndefs*/ false, /*AllowTruncate*/ true);
5126 bool N1ConstOrSplat =
5127 isConstOrConstSplat(N1, /*AllowUndefs*/ false, /*AllowTruncate*/ true);
5128
5129 // Canonicalize toward having the constant on the RHS.
5130 // TODO: Handle non-splat vector constants. All undef causes trouble.
5131 // FIXME: We can't yet fold constant scalable vector splats, so avoid an
5132 // infinite loop here when we encounter one.
5134 if (N0ConstOrSplat && !N1ConstOrSplat &&
5135 (DCI.isBeforeLegalizeOps() ||
5136 isCondCodeLegal(SwappedCC, N0.getSimpleValueType())))
5137 return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);
5138
5139 // If we have a subtract with the same 2 non-constant operands as this setcc
5140 // -- but in reverse order -- then try to commute the operands of this setcc
5141 // to match. A matching pair of setcc (cmp) and sub may be combined into 1
5142 // instruction on some targets.
5143 if (!N0ConstOrSplat && !N1ConstOrSplat &&
5144 (DCI.isBeforeLegalizeOps() ||
5145 isCondCodeLegal(SwappedCC, N0.getSimpleValueType())) &&
5146 DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N1, N0}) &&
5147 !DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N0, N1}))
5148 return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);
5149
5150 if (SDValue V = foldSetCCWithRotate(VT, N0, N1, Cond, dl, DAG))
5151 return V;
5152
5153 if (SDValue V = foldSetCCWithFunnelShift(VT, N0, N1, Cond, dl, DAG))
5154 return V;
5155
5156 if (auto *N1C = isConstOrConstSplat(N1)) {
5157 const APInt &C1 = N1C->getAPIntValue();
5158
5159 // Optimize some CTPOP cases.
5160 if (SDValue V = simplifySetCCWithCTPOP(*this, VT, N0, C1, Cond, dl, DAG))
5161 return V;
5162
5163 // For equality to 0 of a no-wrap multiply, decompose and test each op:
5164 // X * Y == 0 --> (X == 0) || (Y == 0)
5165 // X * Y != 0 --> (X != 0) && (Y != 0)
5166 // TODO: This bails out if minsize is set, but if the target doesn't have a
5167 // single instruction multiply for this type, it would likely be
5168 // smaller to decompose.
5169 if (C1.isZero() && (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5170 N0.getOpcode() == ISD::MUL && N0.hasOneUse() &&
5171 (N0->getFlags().hasNoUnsignedWrap() ||
5172 N0->getFlags().hasNoSignedWrap()) &&
5173 !Attr.hasFnAttr(Attribute::MinSize)) {
5174 SDValue IsXZero = DAG.getSetCC(dl, VT, N0.getOperand(0), N1, Cond);
5175 SDValue IsYZero = DAG.getSetCC(dl, VT, N0.getOperand(1), N1, Cond);
5176 unsigned LogicOp = Cond == ISD::SETEQ ? ISD::OR : ISD::AND;
5177 return DAG.getNode(LogicOp, dl, VT, IsXZero, IsYZero);
5178 }
5179
5180 // If the LHS is '(srl (ctlz x), 5)', the RHS is 0/1, and this is an
5181 // equality comparison, then we're just comparing whether X itself is
5182 // zero.
5183 if (N0.getOpcode() == ISD::SRL && (C1.isZero() || C1.isOne()) &&
5184 N0.getOperand(0).getOpcode() == ISD::CTLZ &&
5186 if (ConstantSDNode *ShAmt = isConstOrConstSplat(N0.getOperand(1))) {
5187 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5188 ShAmt->getAPIntValue() == Log2_32(N0.getScalarValueSizeInBits())) {
5189 if ((C1 == 0) == (Cond == ISD::SETEQ)) {
5190 // (srl (ctlz x), 5) == 0 -> X != 0
5191 // (srl (ctlz x), 5) != 1 -> X != 0
5192 Cond = ISD::SETNE;
5193 } else {
5194 // (srl (ctlz x), 5) != 0 -> X == 0
5195 // (srl (ctlz x), 5) == 1 -> X == 0
5196 Cond = ISD::SETEQ;
5197 }
5198 SDValue Zero = DAG.getConstant(0, dl, N0.getValueType());
5199 return DAG.getSetCC(dl, VT, N0.getOperand(0).getOperand(0), Zero,
5200 Cond);
5201 }
5202 }
5203 }
5204 }
5205
5206 // setcc X, 0, setlt --> X (when X is all sign bits)
5207 // setcc X, 0, setne --> X (when X is all sign bits)
5208 //
5209 // When we know that X has 0 or -1 in each element (or scalar), this
5210 // comparison will produce X. This is only true when boolean contents are
5211 // represented via 0s and -1s.
5212 if (VT == OpVT &&
5213 // Check that the result of setcc is 0 and -1.
5215 // Match only for checks X < 0 and X != 0
5216 (Cond == ISD::SETLT || Cond == ISD::SETNE) && isNullOrNullSplat(N1) &&
5217 // The identity holds iff we know all sign bits for all lanes.
5219 return N0;
5220
5221 // FIXME: Support vectors.
5222 if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
5223 const APInt &C1 = N1C->getAPIntValue();
5224
5225 // (zext x) == C --> x == (trunc C)
5226 // (sext x) == C --> x == (trunc C)
5227 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5228 DCI.isBeforeLegalize() && N0->hasOneUse()) {
5229 unsigned MinBits = N0.getValueSizeInBits();
5230 SDValue PreExt;
5231 bool Signed = false;
5232 if (N0->getOpcode() == ISD::ZERO_EXTEND) {
5233 // ZExt
5234 MinBits = N0->getOperand(0).getValueSizeInBits();
5235 PreExt = N0->getOperand(0);
5236 } else if (N0->getOpcode() == ISD::AND) {
5237 // DAGCombine turns costly ZExts into ANDs
5238 if (auto *C = dyn_cast<ConstantSDNode>(N0->getOperand(1)))
5239 if ((C->getAPIntValue()+1).isPowerOf2()) {
5240 MinBits = C->getAPIntValue().countr_one();
5241 PreExt = N0->getOperand(0);
5242 }
5243 } else if (N0->getOpcode() == ISD::SIGN_EXTEND) {
5244 // SExt
5245 MinBits = N0->getOperand(0).getValueSizeInBits();
5246 PreExt = N0->getOperand(0);
5247 Signed = true;
5248 } else if (auto *LN0 = dyn_cast<LoadSDNode>(N0)) {
5249 // ZEXTLOAD / SEXTLOAD
5250 if (LN0->getExtensionType() == ISD::ZEXTLOAD) {
5251 MinBits = LN0->getMemoryVT().getSizeInBits();
5252 PreExt = N0;
5253 } else if (LN0->getExtensionType() == ISD::SEXTLOAD) {
5254 Signed = true;
5255 MinBits = LN0->getMemoryVT().getSizeInBits();
5256 PreExt = N0;
5257 }
5258 }
5259
5260 // Figure out how many bits we need to preserve this constant.
5261 unsigned ReqdBits = Signed ? C1.getSignificantBits() : C1.getActiveBits();
5262
5263 // Make sure we're not losing bits from the constant.
5264 if (MinBits > 0 &&
5265 MinBits < C1.getBitWidth() &&
5266 MinBits >= ReqdBits) {
5267 EVT MinVT = EVT::getIntegerVT(*DAG.getContext(), MinBits);
5268 if (isTypeDesirableForOp(ISD::SETCC, MinVT)) {
5269 // Will get folded away.
5270 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MinVT, PreExt);
5271 if (MinBits == 1 && C1 == 1)
5272 // Invert the condition.
5273 return DAG.getSetCC(dl, VT, Trunc, DAG.getConstant(0, dl, MVT::i1),
5275 SDValue C = DAG.getConstant(C1.trunc(MinBits), dl, MinVT);
5276 return DAG.getSetCC(dl, VT, Trunc, C, Cond);
5277 }
5278
5279 // If truncating the setcc operands is not desirable, we can still
5280 // simplify the expression in some cases:
5281 // setcc ([sz]ext (setcc x, y, cc)), 0, setne) -> setcc (x, y, cc)
5282 // setcc ([sz]ext (setcc x, y, cc)), 0, seteq) -> setcc (x, y, inv(cc))
5283 // setcc (zext (setcc x, y, cc)), 1, setne) -> setcc (x, y, inv(cc))
5284 // setcc (zext (setcc x, y, cc)), 1, seteq) -> setcc (x, y, cc)
5285 // setcc (sext (setcc x, y, cc)), -1, setne) -> setcc (x, y, inv(cc))
5286 // setcc (sext (setcc x, y, cc)), -1, seteq) -> setcc (x, y, cc)
5287 SDValue TopSetCC = N0->getOperand(0);
5288 unsigned N0Opc = N0->getOpcode();
5289 bool SExt = (N0Opc == ISD::SIGN_EXTEND);
5290 if (TopSetCC.getValueType() == MVT::i1 && VT == MVT::i1 &&
5291 TopSetCC.getOpcode() == ISD::SETCC &&
5292 (N0Opc == ISD::ZERO_EXTEND || N0Opc == ISD::SIGN_EXTEND) &&
5293 (isConstFalseVal(N1) ||
5294 isExtendedTrueVal(N1C, N0->getValueType(0), SExt))) {
5295
5296 bool Inverse = (N1C->isZero() && Cond == ISD::SETEQ) ||
5297 (!N1C->isZero() && Cond == ISD::SETNE);
5298
5299 if (!Inverse)
5300 return TopSetCC;
5301
5302 ISD::CondCode InvCond =
5304 TopSetCC.getOperand(0).getValueType());
5305 return DAG.getSetCC(dl, VT, TopSetCC.getOperand(0),
5306 TopSetCC.getOperand(1), InvCond);
5307 }
5308 }
5309 }
5310
5311 // If the LHS is '(and load, const)', the RHS is 0, the test is for
5312 // equality or unsigned, and all 1 bits of the const are in the same
5313 // partial word, see if we can shorten the load.
5314 if (DCI.isBeforeLegalize() &&
5316 N0.getOpcode() == ISD::AND && C1 == 0 &&
5317 N0.getNode()->hasOneUse() &&
5318 isa<LoadSDNode>(N0.getOperand(0)) &&
5319 N0.getOperand(0).getNode()->hasOneUse() &&
5321 auto *Lod = cast<LoadSDNode>(N0.getOperand(0));
5322 APInt bestMask;
5323 unsigned bestWidth = 0, bestOffset = 0;
5324 if (Lod->isSimple() && Lod->isUnindexed() &&
5325 (Lod->getMemoryVT().isByteSized() ||
5326 isPaddedAtMostSignificantBitsWhenStored(Lod->getMemoryVT()))) {
5327 unsigned memWidth = Lod->getMemoryVT().getStoreSizeInBits();
5328 unsigned origWidth = N0.getValueSizeInBits();
5329 unsigned maskWidth = origWidth;
5330 // We can narrow (e.g.) 16-bit extending loads on 32-bit target to
5331 // 8 bits, but have to be careful...
5332 if (Lod->getExtensionType() != ISD::NON_EXTLOAD)
5333 origWidth = Lod->getMemoryVT().getSizeInBits();
5334 const APInt &Mask = N0.getConstantOperandAPInt(1);
5335 // Only consider power-of-2 widths (and at least one byte) as candiates
5336 // for the narrowed load.
5337 for (unsigned width = 8; width < origWidth; width *= 2) {
5338 EVT newVT = EVT::getIntegerVT(*DAG.getContext(), width);
5339 APInt newMask = APInt::getLowBitsSet(maskWidth, width);
5340 // Avoid accessing any padding here for now (we could use memWidth
5341 // instead of origWidth here otherwise).
5342 unsigned maxOffset = origWidth - width;
5343 for (unsigned offset = 0; offset <= maxOffset; offset += 8) {
5344 if (Mask.isSubsetOf(newMask)) {
5345 unsigned ptrOffset =
5346 Layout.isLittleEndian() ? offset : memWidth - width - offset;
5347 unsigned IsFast = 0;
5348 assert((ptrOffset % 8) == 0 && "Non-Bytealigned pointer offset");
5349 Align NewAlign = commonAlignment(Lod->getAlign(), ptrOffset / 8);
5351 ptrOffset / 8) &&
5353 *DAG.getContext(), Layout, newVT, Lod->getAddressSpace(),
5354 NewAlign, Lod->getMemOperand()->getFlags(), &IsFast) &&
5355 IsFast) {
5356 bestOffset = ptrOffset / 8;
5357 bestMask = Mask.lshr(offset);
5358 bestWidth = width;
5359 break;
5360 }
5361 }
5362 newMask <<= 8;
5363 }
5364 if (bestWidth)
5365 break;
5366 }
5367 }
5368 if (bestWidth) {
5369 EVT newVT = EVT::getIntegerVT(*DAG.getContext(), bestWidth);
5370 SDValue Ptr = Lod->getBasePtr();
5371 if (bestOffset != 0)
5372 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::getFixed(bestOffset));
5373 SDValue NewLoad =
5374 DAG.getLoad(newVT, dl, Lod->getChain(), Ptr,
5375 Lod->getPointerInfo().getWithOffset(bestOffset),
5376 Lod->getBaseAlign(), MachineMemOperand::MONone,
5377 Lod->getMMOMetadataForSubAccess());
5378 SDValue And =
5379 DAG.getNode(ISD::AND, dl, newVT, NewLoad,
5380 DAG.getConstant(bestMask.trunc(bestWidth), dl, newVT));
5381 return DAG.getSetCC(dl, VT, And, DAG.getConstant(0LL, dl, newVT), Cond);
5382 }
5383 }
5384
5385 // If the LHS is a ZERO_EXTEND, perform the comparison on the input.
5386 if (N0.getOpcode() == ISD::ZERO_EXTEND) {
5387 unsigned InSize = N0.getOperand(0).getValueSizeInBits();
5388
5389 // If the comparison constant has bits in the upper part, the
5390 // zero-extended value could never match.
5391 if (C1.intersects(APInt::getHighBitsSet(C1.getBitWidth(),
5392 C1.getBitWidth() - InSize))) {
5393 switch (Cond) {
5394 case ISD::SETUGT:
5395 case ISD::SETUGE:
5396 case ISD::SETEQ:
5397 return DAG.getConstant(0, dl, VT);
5398 case ISD::SETULT:
5399 case ISD::SETULE:
5400 case ISD::SETNE:
5401 return DAG.getConstant(1, dl, VT);
5402 case ISD::SETGT:
5403 case ISD::SETGE:
5404 // True if the sign bit of C1 is set.
5405 return DAG.getConstant(C1.isNegative(), dl, VT);
5406 case ISD::SETLT:
5407 case ISD::SETLE:
5408 // True if the sign bit of C1 isn't set.
5409 return DAG.getConstant(C1.isNonNegative(), dl, VT);
5410 default:
5411 break;
5412 }
5413 }
5414
5415 // Otherwise, we can perform the comparison with the low bits.
5416 switch (Cond) {
5417 case ISD::SETEQ:
5418 case ISD::SETNE:
5419 case ISD::SETUGT:
5420 case ISD::SETUGE:
5421 case ISD::SETULT:
5422 case ISD::SETULE: {
5423 EVT newVT = N0.getOperand(0).getValueType();
5424 // FIXME: Should use isNarrowingProfitable.
5425 if (DCI.isBeforeLegalizeOps() ||
5426 (isOperationLegal(ISD::SETCC, newVT) &&
5427 isCondCodeLegal(Cond, newVT.getSimpleVT()) &&
5429 EVT NewSetCCVT = getSetCCResultType(Layout, *DAG.getContext(), newVT);
5430 SDValue NewConst = DAG.getConstant(C1.trunc(InSize), dl, newVT);
5431
5432 SDValue NewSetCC = DAG.getSetCC(dl, NewSetCCVT, N0.getOperand(0),
5433 NewConst, Cond);
5434 return DAG.getBoolExtOrTrunc(NewSetCC, dl, VT, N0.getValueType());
5435 }
5436 break;
5437 }
5438 default:
5439 break; // todo, be more careful with signed comparisons
5440 }
5441 } else if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
5442 (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5444 OpVT)) {
5445 EVT ExtSrcTy = cast<VTSDNode>(N0.getOperand(1))->getVT();
5446 unsigned ExtSrcTyBits = ExtSrcTy.getSizeInBits();
5447 EVT ExtDstTy = N0.getValueType();
5448 unsigned ExtDstTyBits = ExtDstTy.getSizeInBits();
5449
5450 // If the constant doesn't fit into the number of bits for the source of
5451 // the sign extension, it is impossible for both sides to be equal.
5452 if (C1.getSignificantBits() > ExtSrcTyBits)
5453 return DAG.getBoolConstant(Cond == ISD::SETNE, dl, VT, OpVT);
5454
5455 assert(ExtDstTy == N0.getOperand(0).getValueType() &&
5456 ExtDstTy != ExtSrcTy && "Unexpected types!");
5457 APInt Imm = APInt::getLowBitsSet(ExtDstTyBits, ExtSrcTyBits);
5458 SDValue ZextOp = DAG.getNode(ISD::AND, dl, ExtDstTy, N0.getOperand(0),
5459 DAG.getConstant(Imm, dl, ExtDstTy));
5460 if (!DCI.isCalledByLegalizer())
5461 DCI.AddToWorklist(ZextOp.getNode());
5462 // Otherwise, make this a use of a zext.
5463 return DAG.getSetCC(dl, VT, ZextOp,
5464 DAG.getConstant(C1 & Imm, dl, ExtDstTy), Cond);
5465 } else if ((N1C->isZero() || N1C->isOne()) &&
5466 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
5467 // SETCC (X), [0|1], [EQ|NE] -> X if X is known 0/1. i1 types are
5468 // excluded as they are handled below whilst checking for foldBooleans.
5469 if ((N0.getOpcode() == ISD::SETCC || VT.getScalarType() != MVT::i1) &&
5470 isTypeLegal(VT) && VT.bitsLE(N0.getValueType()) &&
5471 (N0.getValueType() == MVT::i1 ||
5475 bool TrueWhenTrue = (Cond == ISD::SETEQ) ^ (!N1C->isOne());
5476 if (TrueWhenTrue)
5477 return DAG.getNode(ISD::TRUNCATE, dl, VT, N0);
5478 // Invert the condition.
5479 if (N0.getOpcode() == ISD::SETCC) {
5482 if (DCI.isBeforeLegalizeOps() ||
5484 return DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC);
5485 }
5486 }
5487
5488 if ((N0.getOpcode() == ISD::XOR ||
5489 (N0.getOpcode() == ISD::AND &&
5490 N0.getOperand(0).getOpcode() == ISD::XOR &&
5491 N0.getOperand(1) == N0.getOperand(0).getOperand(1))) &&
5492 isOneConstant(N0.getOperand(1))) {
5493 // If this is (X^1) == 0/1, swap the RHS and eliminate the xor. We
5494 // can only do this if the top bits are known zero.
5495 unsigned BitWidth = N0.getValueSizeInBits();
5496 if (DAG.MaskedValueIsZero(N0,
5498 BitWidth-1))) {
5499 // Okay, get the un-inverted input value.
5500 SDValue Val;
5501 if (N0.getOpcode() == ISD::XOR) {
5502 Val = N0.getOperand(0);
5503 } else {
5504 assert(N0.getOpcode() == ISD::AND &&
5505 N0.getOperand(0).getOpcode() == ISD::XOR);
5506 // ((X^1)&1)^1 -> X & 1
5507 Val = DAG.getNode(ISD::AND, dl, N0.getValueType(),
5508 N0.getOperand(0).getOperand(0),
5509 N0.getOperand(1));
5510 }
5511
5512 return DAG.getSetCC(dl, VT, Val, N1,
5514 }
5515 } else if (N1C->isOne()) {
5516 SDValue Op0 = N0;
5517 if (Op0.getOpcode() == ISD::TRUNCATE)
5518 Op0 = Op0.getOperand(0);
5519
5520 if ((Op0.getOpcode() == ISD::XOR) &&
5521 Op0.getOperand(0).getOpcode() == ISD::SETCC &&
5522 Op0.getOperand(1).getOpcode() == ISD::SETCC) {
5523 SDValue XorLHS = Op0.getOperand(0);
5524 SDValue XorRHS = Op0.getOperand(1);
5525 // Ensure that the input setccs return an i1 type or 0/1 value.
5526 if (Op0.getValueType() == MVT::i1 ||
5531 // (xor (setcc), (setcc)) == / != 1 -> (setcc) != / == (setcc)
5533 return DAG.getSetCC(dl, VT, XorLHS, XorRHS, Cond);
5534 }
5535 }
5536 if (Op0.getOpcode() == ISD::AND && isOneConstant(Op0.getOperand(1))) {
5537 // If this is (X&1) == / != 1, normalize it to (X&1) != / == 0.
5538 if (Op0.getValueType().bitsGT(VT))
5539 Op0 = DAG.getNode(ISD::AND, dl, VT,
5540 DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)),
5541 DAG.getConstant(1, dl, VT));
5542 else if (Op0.getValueType().bitsLT(VT))
5543 Op0 = DAG.getNode(ISD::AND, dl, VT,
5544 DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)),
5545 DAG.getConstant(1, dl, VT));
5546
5547 return DAG.getSetCC(dl, VT, Op0,
5548 DAG.getConstant(0, dl, Op0.getValueType()),
5550 }
5551 if (Op0.getOpcode() == ISD::AssertZext &&
5552 cast<VTSDNode>(Op0.getOperand(1))->getVT() == MVT::i1)
5553 return DAG.getSetCC(dl, VT, Op0,
5554 DAG.getConstant(0, dl, Op0.getValueType()),
5556 }
5557 }
5558
5559 // Given:
5560 // icmp eq/ne (urem %x, %y), 0
5561 // Iff %x has 0 or 1 bits set, and %y has at least 2 bits set, omit 'urem':
5562 // icmp eq/ne %x, 0
5563 if (N0.getOpcode() == ISD::UREM && N1C->isZero() &&
5564 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
5565 KnownBits XKnown = DAG.computeKnownBits(N0.getOperand(0));
5566 KnownBits YKnown = DAG.computeKnownBits(N0.getOperand(1));
5567 if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2)
5568 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1, Cond);
5569 }
5570
5571 // Fold set_cc seteq (ashr X, BW-1), -1 -> set_cc setlt X, 0
5572 // and set_cc setne (ashr X, BW-1), -1 -> set_cc setge X, 0
5573 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5575 N0.getConstantOperandAPInt(1) == OpVT.getScalarSizeInBits() - 1 &&
5576 N1C->isAllOnes()) {
5577 return DAG.getSetCC(dl, VT, N0.getOperand(0),
5578 DAG.getConstant(0, dl, OpVT),
5580 }
5581
5582 // fold (setcc (trunc x) c) -> (setcc x c)
5583 if (N0.getOpcode() == ISD::TRUNCATE &&
5585 (N0->getFlags().hasNoSignedWrap() &&
5588 EVT NewVT = N0.getOperand(0).getValueType();
5589 SDValue NewConst = DAG.getConstant(
5591 ? C1.sext(NewVT.getSizeInBits())
5592 : C1.zext(NewVT.getSizeInBits()),
5593 dl, NewVT);
5594 return DAG.getSetCC(dl, VT, N0.getOperand(0), NewConst, Cond);
5595 }
5596
5597 if (SDValue V =
5598 optimizeSetCCOfSignedTruncationCheck(VT, N0, N1, Cond, DCI, dl))
5599 return V;
5600 }
5601
5602 // These simplifications apply to splat vectors as well.
5603 // TODO: Handle more splat vector cases.
5604 if (auto *N1C = isConstOrConstSplat(N1)) {
5605 const APInt &C1 = N1C->getAPIntValue();
5606
5607 APInt MinVal, MaxVal;
5608 unsigned OperandBitSize = N1C->getValueType(0).getScalarSizeInBits();
5610 MinVal = APInt::getSignedMinValue(OperandBitSize);
5611 MaxVal = APInt::getSignedMaxValue(OperandBitSize);
5612 } else {
5613 MinVal = APInt::getMinValue(OperandBitSize);
5614 MaxVal = APInt::getMaxValue(OperandBitSize);
5615 }
5616
5617 // Canonicalize GE/LE comparisons to use GT/LT comparisons.
5618 if (Cond == ISD::SETGE || Cond == ISD::SETUGE) {
5619 // X >= MIN --> true
5620 if (C1 == MinVal)
5621 return DAG.getBoolConstant(true, dl, VT, OpVT);
5622
5623 if (!VT.isVector()) { // TODO: Support this for vectors.
5624 // X >= C0 --> X > (C0 - 1)
5625 APInt C = C1 - 1;
5627 if ((DCI.isBeforeLegalizeOps() ||
5628 isCondCodeLegal(NewCC, OpVT.getSimpleVT())) &&
5629 (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
5630 isLegalICmpImmediate(C.getSExtValue())))) {
5631 return DAG.getSetCC(dl, VT, N0,
5632 DAG.getConstant(C, dl, N1.getValueType()),
5633 NewCC);
5634 }
5635 }
5636 }
5637
5638 if (Cond == ISD::SETLE || Cond == ISD::SETULE) {
5639 // X <= MAX --> true
5640 if (C1 == MaxVal)
5641 return DAG.getBoolConstant(true, dl, VT, OpVT);
5642
5643 // X <= C0 --> X < (C0 + 1)
5644 if (!VT.isVector()) { // TODO: Support this for vectors.
5645 APInt C = C1 + 1;
5647 if ((DCI.isBeforeLegalizeOps() ||
5648 isCondCodeLegal(NewCC, OpVT.getSimpleVT())) &&
5649 (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
5650 isLegalICmpImmediate(C.getSExtValue())))) {
5651 return DAG.getSetCC(dl, VT, N0,
5652 DAG.getConstant(C, dl, N1.getValueType()),
5653 NewCC);
5654 }
5655 }
5656 }
5657
5658 if (Cond == ISD::SETLT || Cond == ISD::SETULT) {
5659 if (C1 == MinVal)
5660 return DAG.getBoolConstant(false, dl, VT, OpVT); // X < MIN --> false
5661
5662 // TODO: Support this for vectors after legalize ops.
5663 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
5664 // Canonicalize setlt X, Max --> setne X, Max
5665 if (C1 == MaxVal)
5666 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
5667
5668 // If we have setult X, 1, turn it into seteq X, 0
5669 if (C1 == MinVal+1)
5670 return DAG.getSetCC(dl, VT, N0,
5671 DAG.getConstant(MinVal, dl, N0.getValueType()),
5672 ISD::SETEQ);
5673 }
5674 }
5675
5676 if (Cond == ISD::SETGT || Cond == ISD::SETUGT) {
5677 if (C1 == MaxVal)
5678 return DAG.getBoolConstant(false, dl, VT, OpVT); // X > MAX --> false
5679
5680 // TODO: Support this for vectors after legalize ops.
5681 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
5682 // Canonicalize setgt X, Min --> setne X, Min
5683 if (C1 == MinVal)
5684 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
5685
5686 // If we have setugt X, Max-1, turn it into seteq X, Max
5687 if (C1 == MaxVal-1)
5688 return DAG.getSetCC(dl, VT, N0,
5689 DAG.getConstant(MaxVal, dl, N0.getValueType()),
5690 ISD::SETEQ);
5691 }
5692 }
5693
5694 if (Cond == ISD::SETEQ || Cond == ISD::SETNE) {
5695 // (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
5696 if (C1.isZero())
5697 if (SDValue CC = optimizeSetCCByHoistingAndByConstFromLogicalShift(
5698 VT, N0, N1, Cond, DCI, dl))
5699 return CC;
5700
5701 // For all/any comparisons, replace or(x,shl(y,bw/2)) with and/or(x,y).
5702 // For example, when high 32-bits of i64 X are known clear:
5703 // all bits clear: (X | (Y<<32)) == 0 --> (X | Y) == 0
5704 // all bits set: (X | (Y<<32)) == -1 --> (X & Y) == -1
5705 bool CmpZero = N1C->isZero();
5706 bool CmpNegOne = N1C->isAllOnes();
5707 if ((CmpZero || CmpNegOne) && N0.hasOneUse()) {
5708 // Match or(lo,shl(hi,bw/2)) pattern.
5709 auto IsConcat = [&](SDValue V, SDValue &Lo, SDValue &Hi) {
5710 unsigned EltBits = V.getScalarValueSizeInBits();
5711 if (V.getOpcode() != ISD::OR || (EltBits % 2) != 0)
5712 return false;
5713 SDValue LHS = V.getOperand(0);
5714 SDValue RHS = V.getOperand(1);
5715 APInt HiBits = APInt::getHighBitsSet(EltBits, EltBits / 2);
5716 // Unshifted element must have zero upperbits.
5717 if (RHS.getOpcode() == ISD::SHL &&
5718 isa<ConstantSDNode>(RHS.getOperand(1)) &&
5719 RHS.getConstantOperandAPInt(1) == (EltBits / 2) &&
5720 DAG.MaskedValueIsZero(LHS, HiBits)) {
5721 Lo = LHS;
5722 Hi = RHS.getOperand(0);
5723 return true;
5724 }
5725 if (LHS.getOpcode() == ISD::SHL &&
5726 isa<ConstantSDNode>(LHS.getOperand(1)) &&
5727 LHS.getConstantOperandAPInt(1) == (EltBits / 2) &&
5728 DAG.MaskedValueIsZero(RHS, HiBits)) {
5729 Lo = RHS;
5730 Hi = LHS.getOperand(0);
5731 return true;
5732 }
5733 return false;
5734 };
5735
5736 auto MergeConcat = [&](SDValue Lo, SDValue Hi) {
5737 unsigned EltBits = N0.getScalarValueSizeInBits();
5738 unsigned HalfBits = EltBits / 2;
5739 APInt HiBits = APInt::getHighBitsSet(EltBits, HalfBits);
5740 SDValue LoBits = DAG.getConstant(~HiBits, dl, OpVT);
5741 SDValue HiMask = DAG.getNode(ISD::AND, dl, OpVT, Hi, LoBits);
5742 SDValue NewN0 =
5743 DAG.getNode(CmpZero ? ISD::OR : ISD::AND, dl, OpVT, Lo, HiMask);
5744 SDValue NewN1 = CmpZero ? DAG.getConstant(0, dl, OpVT) : LoBits;
5745 return DAG.getSetCC(dl, VT, NewN0, NewN1, Cond);
5746 };
5747
5748 SDValue Lo, Hi;
5749 if (IsConcat(N0, Lo, Hi))
5750 return MergeConcat(Lo, Hi);
5751
5752 if (N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR) {
5753 SDValue Lo0, Lo1, Hi0, Hi1;
5754 if (IsConcat(N0.getOperand(0), Lo0, Hi0) &&
5755 IsConcat(N0.getOperand(1), Lo1, Hi1)) {
5756 return MergeConcat(DAG.getNode(N0.getOpcode(), dl, OpVT, Lo0, Lo1),
5757 DAG.getNode(N0.getOpcode(), dl, OpVT, Hi0, Hi1));
5758 }
5759 }
5760 }
5761 }
5762
5763 // If we have "setcc X, C0", check to see if we can shrink the immediate
5764 // by changing cc.
5765 // TODO: Support this for vectors after legalize ops.
5766 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
5767 // SETUGT X, SINTMAX -> SETLT X, 0
5768 // SETUGE X, SINTMIN -> SETLT X, 0
5769 if ((Cond == ISD::SETUGT && C1.isMaxSignedValue()) ||
5770 (Cond == ISD::SETUGE && C1.isMinSignedValue()))
5771 return DAG.getSetCC(dl, VT, N0,
5772 DAG.getConstant(0, dl, N1.getValueType()),
5773 ISD::SETLT);
5774
5775 // SETULT X, SINTMIN -> SETGT X, -1
5776 // SETULE X, SINTMAX -> SETGT X, -1
5777 if ((Cond == ISD::SETULT && C1.isMinSignedValue()) ||
5778 (Cond == ISD::SETULE && C1.isMaxSignedValue()))
5779 return DAG.getSetCC(dl, VT, N0,
5780 DAG.getAllOnesConstant(dl, N1.getValueType()),
5781 ISD::SETGT);
5782 }
5783 }
5784
5785 // Back to non-vector simplifications.
5786 // TODO: Can we do these for vector splats?
5787 if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
5788 const APInt &C1 = N1C->getAPIntValue();
5789 EVT ShValTy = N0.getValueType();
5790
5791 // Fold bit comparisons when we can. This will result in an
5792 // incorrect value when boolean false is negative one, unless
5793 // the bitsize is 1 in which case the false value is the same
5794 // in practice regardless of the representation.
5795 if ((VT.getSizeInBits() == 1 ||
5797 (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5798 (VT == ShValTy || (isTypeLegal(VT) && VT.bitsLE(ShValTy))) &&
5799 N0.getOpcode() == ISD::AND) {
5800 if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
5801 if (Cond == ISD::SETNE && C1 == 0) {// (X & 8) != 0 --> (X & 8) >> 3
5802 // Perform the xform if the AND RHS is a single bit.
5803 unsigned ShCt = AndRHS->getAPIntValue().logBase2();
5804 if (AndRHS->getAPIntValue().isPowerOf2() &&
5805 !shouldAvoidTransformToShift(ShValTy, ShCt)) {
5806 return DAG.getNode(
5807 ISD::TRUNCATE, dl, VT,
5808 DAG.getNode(ISD::SRL, dl, ShValTy, N0,
5809 DAG.getShiftAmountConstant(ShCt, ShValTy, dl)));
5810 }
5811 } else if (Cond == ISD::SETEQ && C1 == AndRHS->getAPIntValue()) {
5812 // (X & 8) == 8 --> (X & 8) >> 3
5813 // Perform the xform if C1 is a single bit.
5814 unsigned ShCt = C1.logBase2();
5815 if (C1.isPowerOf2() && !shouldAvoidTransformToShift(ShValTy, ShCt)) {
5816 return DAG.getNode(
5817 ISD::TRUNCATE, dl, VT,
5818 DAG.getNode(ISD::SRL, dl, ShValTy, N0,
5819 DAG.getShiftAmountConstant(ShCt, ShValTy, dl)));
5820 }
5821 }
5822 }
5823 }
5824
5825 if (C1.getSignificantBits() <= 64 &&
5826 !isLegalICmpImmediate(C1.getSExtValue())) {
5827 // (X & -256) == 256 -> (X >> 8) == 1
5828 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5829 N0.getOpcode() == ISD::AND && N0.hasOneUse()) {
5830 if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
5831 const APInt &AndRHSC = AndRHS->getAPIntValue();
5832 if (AndRHSC.isNegatedPowerOf2() && C1.isSubsetOf(AndRHSC)) {
5833 unsigned ShiftBits = AndRHSC.countr_zero();
5834 if (!shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
5835 // If using an unsigned shift doesn't yield a legal compare
5836 // immediate, try using sra instead.
5837 APInt NewC = C1.lshr(ShiftBits);
5838 if (NewC.getSignificantBits() <= 64 &&
5840 APInt SignedC = C1.ashr(ShiftBits);
5841 if (SignedC.getSignificantBits() <= 64 &&
5843 SDValue Shift = DAG.getNode(
5844 ISD::SRA, dl, ShValTy, N0.getOperand(0),
5845 DAG.getShiftAmountConstant(ShiftBits, ShValTy, dl));
5846 SDValue CmpRHS = DAG.getConstant(SignedC, dl, ShValTy);
5847 return DAG.getSetCC(dl, VT, Shift, CmpRHS, Cond);
5848 }
5849 }
5850 SDValue Shift = DAG.getNode(
5851 ISD::SRL, dl, ShValTy, N0.getOperand(0),
5852 DAG.getShiftAmountConstant(ShiftBits, ShValTy, dl));
5853 SDValue CmpRHS = DAG.getConstant(NewC, dl, ShValTy);
5854 return DAG.getSetCC(dl, VT, Shift, CmpRHS, Cond);
5855 }
5856 }
5857 }
5858 } else if (Cond == ISD::SETULT || Cond == ISD::SETUGE ||
5859 Cond == ISD::SETULE || Cond == ISD::SETUGT) {
5860 bool AdjOne = (Cond == ISD::SETULE || Cond == ISD::SETUGT);
5861 // X < 0x100000000 -> (X >> 32) < 1
5862 // X >= 0x100000000 -> (X >> 32) >= 1
5863 // X <= 0x0ffffffff -> (X >> 32) < 1
5864 // X > 0x0ffffffff -> (X >> 32) >= 1
5865 unsigned ShiftBits;
5866 APInt NewC = C1;
5867 ISD::CondCode NewCond = Cond;
5868 if (AdjOne) {
5869 ShiftBits = C1.countr_one();
5870 NewC = NewC + 1;
5871 NewCond = (Cond == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
5872 } else {
5873 ShiftBits = C1.countr_zero();
5874 }
5875 APInt RangeWidth = NewC;
5876 NewC.lshrInPlace(ShiftBits);
5877 if (ShiftBits && NewC.getSignificantBits() <= 64 &&
5879 !shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
5880 // If this is an offset range check, try to move the offset after the
5881 // shift to avoid preserving the pre-shift add with a mask.
5882 if (N0.getOpcode() == ISD::ADD && N0.hasOneUse()) {
5883 if (auto *AddC = isConstOrConstSplat(N0.getOperand(1))) {
5884 const APInt &AddVal = AddC->getAPIntValue();
5885 if (AddVal.countr_zero() >= ShiftBits) {
5886 APInt RangeLower = -AddVal;
5887 bool Overflow;
5888 (void)RangeLower.uadd_ov(RangeWidth, Overflow);
5889 if (!RangeWidth.isZero() && !Overflow) {
5890 SDValue Shift = DAG.getNode(
5891 ISD::SRL, dl, ShValTy, N0.getOperand(0),
5892 DAG.getShiftAmountConstant(ShiftBits, ShValTy, dl));
5893 APInt Offset = -RangeLower.lshr(ShiftBits);
5894 SDValue ShiftedAdd =
5895 DAG.getNode(ISD::ADD, dl, ShValTy, Shift,
5896 DAG.getConstant(Offset, dl, ShValTy));
5897 SDValue CmpRHS = DAG.getConstant(NewC, dl, ShValTy);
5898 return DAG.getSetCC(dl, VT, ShiftedAdd, CmpRHS, NewCond);
5899 }
5900 }
5901 }
5902 }
5903 SDValue Shift =
5904 DAG.getNode(ISD::SRL, dl, ShValTy, N0,
5905 DAG.getShiftAmountConstant(ShiftBits, ShValTy, dl));
5906 SDValue CmpRHS = DAG.getConstant(NewC, dl, ShValTy);
5907 return DAG.getSetCC(dl, VT, Shift, CmpRHS, NewCond);
5908 }
5909 }
5910 }
5911 }
5912
5914 auto *CFP = cast<ConstantFPSDNode>(N1);
5915 assert(!CFP->getValueAPF().isNaN() && "Unexpected NaN value");
5916
5917 // Otherwise, we know the RHS is not a NaN. Simplify the node to drop the
5918 // constant if knowing that the operand is non-nan is enough. We prefer to
5919 // have SETO(x,x) instead of SETO(x, 0.0) because this avoids having to
5920 // materialize 0.0.
5921 if (Cond == ISD::SETO || Cond == ISD::SETUO)
5922 return DAG.getSetCC(dl, VT, N0, N0, Cond);
5923
5924 // setcc (fneg x), C -> setcc swap(pred) x, -C
5925 if (N0.getOpcode() == ISD::FNEG) {
5927 if (DCI.isBeforeLegalizeOps() ||
5928 isCondCodeLegal(SwapCond, N0.getSimpleValueType())) {
5929 SDValue NegN1 = DAG.getNode(ISD::FNEG, dl, N0.getValueType(), N1);
5930 return DAG.getSetCC(dl, VT, N0.getOperand(0), NegN1, SwapCond);
5931 }
5932 }
5933
5934 // setueq/setoeq X, (fabs Inf) -> is_fpclass X, fcInf
5936 !isFPImmLegal(CFP->getValueAPF(), CFP->getValueType(0))) {
5937 bool IsFabs = N0.getOpcode() == ISD::FABS;
5938 SDValue Op = IsFabs ? N0.getOperand(0) : N0;
5939 if ((Cond == ISD::SETOEQ || Cond == ISD::SETUEQ) && CFP->isInfinity()) {
5940 FPClassTest Flag = CFP->isNegative() ? (IsFabs ? fcNone : fcNegInf)
5941 : (IsFabs ? fcInf : fcPosInf);
5942 if (Cond == ISD::SETUEQ)
5943 Flag |= fcNan;
5944 return DAG.getNode(ISD::IS_FPCLASS, dl, VT, Op,
5945 DAG.getTargetConstant(Flag, dl, MVT::i32));
5946 }
5947 }
5948
5949 // If the condition is not legal, see if we can find an equivalent one
5950 // which is legal.
5952 // If the comparison was an awkward floating-point == or != and one of
5953 // the comparison operands is infinity or negative infinity, convert the
5954 // condition to a less-awkward <= or >=.
5955 if (CFP->getValueAPF().isInfinity()) {
5956 bool IsNegInf = CFP->getValueAPF().isNegative();
5958 switch (Cond) {
5959 case ISD::SETOEQ: NewCond = IsNegInf ? ISD::SETOLE : ISD::SETOGE; break;
5960 case ISD::SETUEQ: NewCond = IsNegInf ? ISD::SETULE : ISD::SETUGE; break;
5961 case ISD::SETUNE: NewCond = IsNegInf ? ISD::SETUGT : ISD::SETULT; break;
5962 case ISD::SETONE: NewCond = IsNegInf ? ISD::SETOGT : ISD::SETOLT; break;
5963 default: break;
5964 }
5965 if (NewCond != ISD::SETCC_INVALID &&
5966 isCondCodeLegal(NewCond, N0.getSimpleValueType()))
5967 return DAG.getSetCC(dl, VT, N0, N1, NewCond);
5968 }
5969 }
5970 }
5971
5972 if (N0 == N1) {
5973 // The sext(setcc()) => setcc() optimization relies on the appropriate
5974 // constant being emitted.
5975 assert(!N0.getValueType().isInteger() &&
5976 "Integer types should be handled by FoldSetCC");
5977
5978 bool EqTrue = ISD::isTrueWhenEqual(Cond);
5979 unsigned UOF = ISD::getUnorderedFlavor(Cond);
5980 if (UOF == 2) // FP operators that are undefined on NaNs.
5981 return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
5982 if (UOF == unsigned(EqTrue))
5983 return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
5984 // Otherwise, we can't fold it. However, we can simplify it to SETUO/SETO
5985 // if it is not already.
5986 ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO;
5987 if (NewCond != Cond &&
5988 (DCI.isBeforeLegalizeOps() ||
5989 isCondCodeLegal(NewCond, N0.getSimpleValueType())))
5990 return DAG.getSetCC(dl, VT, N0, N1, NewCond);
5991 }
5992
5993 // ~X > ~Y --> Y > X
5994 // ~X < ~Y --> Y < X
5995 // ~X < C --> X > ~C
5996 // ~X > C --> X < ~C
5997 if ((isSignedIntSetCC(Cond) || isUnsignedIntSetCC(Cond)) &&
5998 N0.getValueType().isInteger()) {
5999 if (isBitwiseNot(N0)) {
6000 if (isBitwiseNot(N1))
6001 return DAG.getSetCC(dl, VT, N1.getOperand(0), N0.getOperand(0), Cond);
6002
6005 SDValue Not = DAG.getNOT(dl, N1, OpVT);
6006 return DAG.getSetCC(dl, VT, Not, N0.getOperand(0), Cond);
6007 }
6008 }
6009 }
6010
6011 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
6012 N0.getValueType().isInteger()) {
6013 if (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB ||
6014 N0.getOpcode() == ISD::XOR) {
6015 // Simplify (X+Y) == (X+Z) --> Y == Z
6016 if (N0.getOpcode() == N1.getOpcode()) {
6017 if (N0.getOperand(0) == N1.getOperand(0))
6018 return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(1), Cond);
6019 if (N0.getOperand(1) == N1.getOperand(1))
6020 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(0), Cond);
6021 if (isCommutativeBinOp(N0.getOpcode())) {
6022 // If X op Y == Y op X, try other combinations.
6023 if (N0.getOperand(0) == N1.getOperand(1))
6024 return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(0),
6025 Cond);
6026 if (N0.getOperand(1) == N1.getOperand(0))
6027 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(1),
6028 Cond);
6029 }
6030 }
6031
6032 // If RHS is a legal immediate value for a compare instruction, we need
6033 // to be careful about increasing register pressure needlessly.
6034 bool LegalRHSImm = false;
6035
6036 if (auto *RHSC = dyn_cast<ConstantSDNode>(N1)) {
6037 if (auto *LHSR = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
6038 // Turn (X+C1) == C2 --> X == C2-C1
6039 if (N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse())
6040 return DAG.getSetCC(
6041 dl, VT, N0.getOperand(0),
6042 DAG.getConstant(RHSC->getAPIntValue() - LHSR->getAPIntValue(),
6043 dl, N0.getValueType()),
6044 Cond);
6045
6046 // Turn (X^C1) == C2 --> X == C1^C2
6047 if (N0.getOpcode() == ISD::XOR && N0.getNode()->hasOneUse())
6048 return DAG.getSetCC(
6049 dl, VT, N0.getOperand(0),
6050 DAG.getConstant(LHSR->getAPIntValue() ^ RHSC->getAPIntValue(),
6051 dl, N0.getValueType()),
6052 Cond);
6053 }
6054
6055 // Turn (C1-X) == C2 --> X == C1-C2
6056 if (auto *SUBC = dyn_cast<ConstantSDNode>(N0.getOperand(0)))
6057 if (N0.getOpcode() == ISD::SUB && N0.getNode()->hasOneUse())
6058 return DAG.getSetCC(
6059 dl, VT, N0.getOperand(1),
6060 DAG.getConstant(SUBC->getAPIntValue() - RHSC->getAPIntValue(),
6061 dl, N0.getValueType()),
6062 Cond);
6063
6064 // Could RHSC fold directly into a compare?
6065 if (RHSC->getValueType(0).getSizeInBits() <= 64)
6066 LegalRHSImm = isLegalICmpImmediate(RHSC->getSExtValue());
6067 }
6068
6069 // (X+Y) == X --> Y == 0 and similar folds.
6070 // Don't do this if X is an immediate that can fold into a cmp
6071 // instruction and X+Y has other uses. It could be an induction variable
6072 // chain, and the transform would increase register pressure.
6073 if (!LegalRHSImm || N0.hasOneUse())
6074 if (SDValue V = foldSetCCWithBinOp(VT, N0, N1, Cond, dl, DCI))
6075 return V;
6076 }
6077
6078 if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB ||
6079 N1.getOpcode() == ISD::XOR)
6080 if (SDValue V = foldSetCCWithBinOp(VT, N1, N0, Cond, dl, DCI))
6081 return V;
6082
6083 if (SDValue V = foldSetCCWithAnd(VT, N0, N1, Cond, dl, DCI))
6084 return V;
6085
6086 if (SDValue V = foldSetCCWithOr(VT, N0, N1, Cond, dl, DCI))
6087 return V;
6088 }
6089
6090 // Fold remainder of division by a constant.
6091 if ((N0.getOpcode() == ISD::UREM || N0.getOpcode() == ISD::SREM) &&
6092 N0.hasOneUse() && (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
6093 // When division is cheap or optimizing for minimum size,
6094 // fall through to DIVREM creation by skipping this fold.
6095 if (!isIntDivCheap(VT, Attr) && !Attr.hasFnAttr(Attribute::MinSize)) {
6096 if (N0.getOpcode() == ISD::UREM) {
6097 if (SDValue Folded = buildUREMEqFold(VT, N0, N1, Cond, DCI, dl))
6098 return Folded;
6099 } else if (N0.getOpcode() == ISD::SREM) {
6100 if (SDValue Folded = buildSREMEqFold(VT, N0, N1, Cond, DCI, dl))
6101 return Folded;
6102 }
6103 }
6104 }
6105
6106 // Fold away ALL boolean setcc's.
6107 if (N0.getValueType().getScalarType() == MVT::i1 && foldBooleans) {
6108 SDValue Temp;
6109 switch (Cond) {
6110 default: llvm_unreachable("Unknown integer setcc!");
6111 case ISD::SETEQ: // X == Y -> ~(X^Y)
6112 Temp = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
6113 N0 = DAG.getNOT(dl, Temp, OpVT);
6114 if (!DCI.isCalledByLegalizer())
6115 DCI.AddToWorklist(Temp.getNode());
6116 break;
6117 case ISD::SETNE: // X != Y --> (X^Y)
6118 N0 = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
6119 break;
6120 case ISD::SETGT: // X >s Y --> X == 0 & Y == 1 --> ~X & Y
6121 case ISD::SETULT: // X <u Y --> X == 0 & Y == 1 --> ~X & Y
6122 Temp = DAG.getNOT(dl, N0, OpVT);
6123 N0 = DAG.getNode(ISD::AND, dl, OpVT, N1, Temp);
6124 if (!DCI.isCalledByLegalizer())
6125 DCI.AddToWorklist(Temp.getNode());
6126 break;
6127 case ISD::SETLT: // X <s Y --> X == 1 & Y == 0 --> ~Y & X
6128 case ISD::SETUGT: // X >u Y --> X == 1 & Y == 0 --> ~Y & X
6129 Temp = DAG.getNOT(dl, N1, OpVT);
6130 N0 = DAG.getNode(ISD::AND, dl, OpVT, N0, Temp);
6131 if (!DCI.isCalledByLegalizer())
6132 DCI.AddToWorklist(Temp.getNode());
6133 break;
6134 case ISD::SETULE: // X <=u Y --> X == 0 | Y == 1 --> ~X | Y
6135 case ISD::SETGE: // X >=s Y --> X == 0 | Y == 1 --> ~X | Y
6136 Temp = DAG.getNOT(dl, N0, OpVT);
6137 N0 = DAG.getNode(ISD::OR, dl, OpVT, N1, Temp);
6138 if (!DCI.isCalledByLegalizer())
6139 DCI.AddToWorklist(Temp.getNode());
6140 break;
6141 case ISD::SETUGE: // X >=u Y --> X == 1 | Y == 0 --> ~Y | X
6142 case ISD::SETLE: // X <=s Y --> X == 1 | Y == 0 --> ~Y | X
6143 Temp = DAG.getNOT(dl, N1, OpVT);
6144 N0 = DAG.getNode(ISD::OR, dl, OpVT, N0, Temp);
6145 break;
6146 }
6147 if (VT.getScalarType() != MVT::i1) {
6148 if (!DCI.isCalledByLegalizer())
6149 DCI.AddToWorklist(N0.getNode());
6150 // FIXME: If running after legalize, we probably can't do this.
6152 N0 = DAG.getNode(ExtendCode, dl, VT, N0);
6153 }
6154 return N0;
6155 }
6156
6157 // Fold (setcc (trunc x) (trunc y)) -> (setcc x y)
6158 if (N0.getOpcode() == ISD::TRUNCATE && N1.getOpcode() == ISD::TRUNCATE &&
6159 N0.getOperand(0).getValueType() == N1.getOperand(0).getValueType() &&
6161 N1->getFlags().hasNoUnsignedWrap()) ||
6163 N1->getFlags().hasNoSignedWrap())) &&
6165 if (VT.getScalarType() == MVT::i1)
6166 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(0), Cond);
6167 // For (legal) non vXi1 cases - ensure we adjust the cmp and result types.
6168 EVT OldCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
6169 N0.getValueType());
6170 if (VT == OldCCVT) {
6171 EVT NewCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
6172 N0.getOperand(0).getValueType());
6173 return DAG.getBoolExtOrTrunc(
6174 DAG.getSetCC(dl, NewCCVT, N0.getOperand(0), N1.getOperand(0), Cond),
6175 dl, VT, N0.getOperand(0).getValueType());
6176 }
6177 }
6178
6179 // Fold (setcc (sub nsw a, b), zero, s??) -> (setcc a, b, s??)
6180 // TODO: Remove that .isVector() check
6181 if (VT.isVector() && isZeroOrZeroSplat(N1) && N0.getOpcode() == ISD::SUB &&
6183 return DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), Cond);
6184 }
6185
6186 // Could not fold it.
6187 return SDValue();
6188}
6189
6190/// Returns true (and the GlobalValue and the offset) if the node is a
6191/// GlobalAddress + offset.
6193 int64_t &Offset) const {
6194
6195 SDNode *N = unwrapAddress(SDValue(WN, 0)).getNode();
6196
6197 if (auto *GASD = dyn_cast<GlobalAddressSDNode>(N)) {
6198 GA = GASD->getGlobal();
6199 Offset += GASD->getOffset();
6200 return true;
6201 }
6202
6203 if (N->isAnyAdd()) {
6204 SDValue N1 = N->getOperand(0);
6205 SDValue N2 = N->getOperand(1);
6206 if (isGAPlusOffset(N1.getNode(), GA, Offset)) {
6207 if (auto *V = dyn_cast<ConstantSDNode>(N2)) {
6208 Offset += V->getSExtValue();
6209 return true;
6210 }
6211 } else if (isGAPlusOffset(N2.getNode(), GA, Offset)) {
6212 if (auto *V = dyn_cast<ConstantSDNode>(N1)) {
6213 Offset += V->getSExtValue();
6214 return true;
6215 }
6216 }
6217 }
6218
6219 return false;
6220}
6221
6223 DAGCombinerInfo &DCI) const {
6224 // Default implementation: no optimization.
6225 return SDValue();
6226}
6227
6228//===----------------------------------------------------------------------===//
6229// Inline Assembler Implementation Methods
6230//===----------------------------------------------------------------------===//
6231
6234 unsigned S = Constraint.size();
6235
6236 if (S == 1) {
6237 switch (Constraint[0]) {
6238 default: break;
6239 case 'r':
6240 return C_RegisterClass;
6241 case 'm': // memory
6242 case 'o': // offsetable
6243 case 'V': // not offsetable
6244 return C_Memory;
6245 case 'p': // Address.
6246 return C_Address;
6247 case 'n': // Simple Integer
6248 case 'E': // Floating Point Constant
6249 case 'F': // Floating Point Constant
6250 return C_Immediate;
6251 case 'i': // Simple Integer or Relocatable Constant
6252 case 's': // Relocatable Constant
6253 case 'X': // Allow ANY value.
6254 case 'I': // Target registers.
6255 case 'J':
6256 case 'K':
6257 case 'L':
6258 case 'M':
6259 case 'N':
6260 case 'O':
6261 case 'P':
6262 case '<':
6263 case '>':
6264 return C_Other;
6265 }
6266 }
6267
6268 if (S > 1 && Constraint[0] == '{' && Constraint[S - 1] == '}') {
6269 if (S == 8 && Constraint.substr(1, 6) == "memory") // "{memory}"
6270 return C_Memory;
6271 return C_Register;
6272 }
6273 return C_Unknown;
6274}
6275
6276/// Try to replace an X constraint, which matches anything, with another that
6277/// has more specific requirements based on the type of the corresponding
6278/// operand.
6279const char *TargetLowering::LowerXConstraint(EVT ConstraintVT) const {
6280 if (ConstraintVT.isInteger())
6281 return "r";
6282 if (ConstraintVT.isFloatingPoint())
6283 return "f"; // works for many targets
6284 return nullptr;
6285}
6286
6288 SDValue &Chain, SDValue &Glue, const SDLoc &DL,
6289 const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const {
6290 return SDValue();
6291}
6292
6293/// Lower the specified operand into the Ops vector.
6294/// If it is invalid, don't add anything to Ops.
6296 StringRef Constraint,
6297 std::vector<SDValue> &Ops,
6298 SelectionDAG &DAG) const {
6299
6300 if (Constraint.size() > 1)
6301 return;
6302
6303 char ConstraintLetter = Constraint[0];
6304 switch (ConstraintLetter) {
6305 default: break;
6306 case 'X': // Allows any operand
6307 case 'i': // Simple Integer or Relocatable Constant
6308 case 'n': // Simple Integer
6309 case 's': { // Relocatable Constant
6310
6312 uint64_t Offset = 0;
6313
6314 // Match (GA) or (C) or (GA+C) or (GA-C) or ((GA+C)+C) or (((GA+C)+C)+C),
6315 // etc., since getelementpointer is variadic. We can't use
6316 // SelectionDAG::FoldSymbolOffset because it expects the GA to be accessible
6317 // while in this case the GA may be furthest from the root node which is
6318 // likely an ISD::ADD.
6319 while (true) {
6320 if ((C = dyn_cast<ConstantSDNode>(Op)) && ConstraintLetter != 's') {
6321 // gcc prints these as sign extended. Sign extend value to 64 bits
6322 // now; without this it would get ZExt'd later in
6323 // ScheduleDAGSDNodes::EmitNode, which is very generic.
6324 bool IsBool = C->getConstantIntValue()->getBitWidth() == 1;
6325 BooleanContent BCont = getBooleanContents(MVT::i64);
6326 ISD::NodeType ExtOpc =
6327 IsBool ? getExtendForContent(BCont) : ISD::SIGN_EXTEND;
6328 int64_t ExtVal =
6329 ExtOpc == ISD::ZERO_EXTEND ? C->getZExtValue() : C->getSExtValue();
6330 Ops.push_back(
6331 DAG.getTargetConstant(Offset + ExtVal, SDLoc(C), MVT::i64));
6332 return;
6333 }
6334 if (ConstraintLetter != 'n') {
6335 if (const auto *GA = dyn_cast<GlobalAddressSDNode>(Op)) {
6336 Ops.push_back(DAG.getTargetGlobalAddress(GA->getGlobal(), SDLoc(Op),
6337 GA->getValueType(0),
6338 Offset + GA->getOffset()));
6339 return;
6340 }
6341 if (const auto *BA = dyn_cast<BlockAddressSDNode>(Op)) {
6342 Ops.push_back(DAG.getTargetBlockAddress(
6343 BA->getBlockAddress(), BA->getValueType(0),
6344 Offset + BA->getOffset(), BA->getTargetFlags()));
6345 return;
6346 }
6348 Ops.push_back(Op);
6349 return;
6350 }
6351 }
6352 const unsigned OpCode = Op.getOpcode();
6353 if (OpCode == ISD::ADD || OpCode == ISD::SUB) {
6354 if ((C = dyn_cast<ConstantSDNode>(Op.getOperand(0))))
6355 Op = Op.getOperand(1);
6356 // Subtraction is not commutative.
6357 else if (OpCode == ISD::ADD &&
6358 (C = dyn_cast<ConstantSDNode>(Op.getOperand(1))))
6359 Op = Op.getOperand(0);
6360 else
6361 return;
6362 Offset += (OpCode == ISD::ADD ? 1 : -1) * C->getSExtValue();
6363 continue;
6364 }
6365 return;
6366 }
6367 break;
6368 }
6369 }
6370}
6371
6375
6376std::pair<unsigned, const TargetRegisterClass *>
6378 StringRef Constraint,
6379 MVT VT) const {
6380 if (!Constraint.starts_with("{"))
6381 return std::make_pair(0u, static_cast<TargetRegisterClass *>(nullptr));
6382 assert(*(Constraint.end() - 1) == '}' && "Not a brace enclosed constraint?");
6383
6384 // Remove the braces from around the name.
6385 StringRef RegName(Constraint.data() + 1, Constraint.size() - 2);
6386
6387 std::pair<unsigned, const TargetRegisterClass *> R =
6388 std::make_pair(0u, static_cast<const TargetRegisterClass *>(nullptr));
6389
6390 // Figure out which register class contains this reg.
6391 for (const TargetRegisterClass &RC : RI->regclasses()) {
6392 // If none of the value types for this register class are valid, we
6393 // can't use it. For example, 64-bit reg classes on 32-bit targets.
6394 if (!isLegalRC(*RI, RC))
6395 continue;
6396
6397 for (const MCPhysReg &PR : RC) {
6398 if (RegName.equals_insensitive(RI->getRegAsmName(PR))) {
6399 std::pair<unsigned, const TargetRegisterClass *> S =
6400 std::make_pair(PR, &RC);
6401
6402 // If this register class has the requested value type, return it,
6403 // otherwise keep searching and return the first class found
6404 // if no other is found which explicitly has the requested type.
6405 if (RI->isTypeLegalForClass(RC, VT))
6406 return S;
6407 if (!R.second)
6408 R = S;
6409 }
6410 }
6411 }
6412
6413 return R;
6414}
6415
6416//===----------------------------------------------------------------------===//
6417// Constraint Selection.
6418
6419/// Return true of this is an input operand that is a matching constraint like
6420/// "4".
6422 assert(!ConstraintCode.empty() && "No known constraint!");
6423 return isdigit(static_cast<unsigned char>(ConstraintCode[0]));
6424}
6425
6426/// If this is an input matching constraint, this method returns the output
6427/// operand it matches.
6429 assert(!ConstraintCode.empty() && "No known constraint!");
6430 return atoi(ConstraintCode.c_str());
6431}
6432
6433/// Split up the constraint string from the inline assembly value into the
6434/// specific constraints and their prefixes, and also tie in the associated
6435/// operand values.
6436/// If this returns an empty vector, and if the constraint string itself
6437/// isn't empty, there was an error parsing.
6440 const TargetRegisterInfo *TRI,
6441 const CallBase &Call) const {
6442 /// Information about all of the constraints.
6443 AsmOperandInfoVector ConstraintOperands;
6444 const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
6445 unsigned maCount = 0; // Largest number of multiple alternative constraints.
6446
6447 // Do a prepass over the constraints, canonicalizing them, and building up the
6448 // ConstraintOperands list.
6449 unsigned ArgNo = 0; // ArgNo - The argument of the CallInst.
6450 unsigned ResNo = 0; // ResNo - The result number of the next output.
6451 unsigned LabelNo = 0; // LabelNo - CallBr indirect dest number.
6452
6453 for (InlineAsm::ConstraintInfo &CI : IA->ParseConstraints()) {
6454 ConstraintOperands.emplace_back(std::move(CI));
6455 AsmOperandInfo &OpInfo = ConstraintOperands.back();
6456
6457 // Update multiple alternative constraint count.
6458 if (OpInfo.multipleAlternatives.size() > maCount)
6459 maCount = OpInfo.multipleAlternatives.size();
6460
6461 OpInfo.ConstraintVT = MVT::Other;
6462
6463 // Compute the value type for each operand.
6464 switch (OpInfo.Type) {
6465 case InlineAsm::isOutput: {
6466 // Indirect outputs just consume an argument.
6467 if (OpInfo.isIndirect) {
6468 OpInfo.CallOperandVal = Call.getArgOperand(ArgNo);
6469 break;
6470 }
6471
6472 // The return value of the call is this value. As such, there is no
6473 // corresponding argument.
6474 assert(!Call.getType()->isVoidTy() && "Bad inline asm!");
6475 EVT VT;
6476 if (auto *STy = dyn_cast<StructType>(Call.getType())) {
6477 VT = getAsmOperandValueType(DL, STy->getElementType(ResNo));
6478 } else {
6479 assert(ResNo == 0 && "Asm only has one result!");
6480 VT = getAsmOperandValueType(DL, Call.getType());
6481 }
6482 OpInfo.ConstraintVT = VT.isSimple() ? VT.getSimpleVT() : MVT::Other;
6483 ++ResNo;
6484 break;
6485 }
6486 case InlineAsm::isInput:
6487 OpInfo.CallOperandVal = Call.getArgOperand(ArgNo);
6488 break;
6489 case InlineAsm::isLabel:
6490 OpInfo.CallOperandVal = cast<CallBrInst>(&Call)->getIndirectDest(LabelNo);
6491 ++LabelNo;
6492 continue;
6494 // Nothing to do.
6495 break;
6496 }
6497
6498 if (OpInfo.CallOperandVal) {
6499 llvm::Type *OpTy = OpInfo.CallOperandVal->getType();
6500 if (OpInfo.isIndirect) {
6501 OpTy = Call.getParamElementType(ArgNo);
6502 assert(OpTy && "Indirect operand must have elementtype attribute");
6503 }
6504
6505 // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
6507 if (STy->getNumElements() == 1)
6508 OpTy = STy->getElementType(0);
6509
6510 // If OpTy is not a single value, it may be a struct/union that we
6511 // can tile with integers.
6512 if (!OpTy->isSingleValueType() && OpTy->isSized()) {
6513 unsigned BitSize = DL.getTypeSizeInBits(OpTy);
6514 switch (BitSize) {
6515 default: break;
6516 case 1:
6517 case 8:
6518 case 16:
6519 case 32:
6520 case 64:
6521 case 128:
6522 OpTy = IntegerType::get(OpTy->getContext(), BitSize);
6523 break;
6524 }
6525 }
6526
6527 EVT VT = getAsmOperandValueType(DL, OpTy, true);
6528 OpInfo.ConstraintVT = VT.isSimple() ? VT.getSimpleVT() : MVT::Other;
6529 ArgNo++;
6530 }
6531 }
6532
6533 // If we have multiple alternative constraints, select the best alternative.
6534 if (!ConstraintOperands.empty()) {
6535 if (maCount) {
6536 unsigned bestMAIndex = 0;
6537 int bestWeight = -1;
6538 // weight: -1 = invalid match, and 0 = so-so match to 5 = good match.
6539 int weight = -1;
6540 unsigned maIndex;
6541 // Compute the sums of the weights for each alternative, keeping track
6542 // of the best (highest weight) one so far.
6543 for (maIndex = 0; maIndex < maCount; ++maIndex) {
6544 int weightSum = 0;
6545 for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
6546 cIndex != eIndex; ++cIndex) {
6547 AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];
6548 if (OpInfo.Type == InlineAsm::isClobber)
6549 continue;
6550
6551 // If this is an output operand with a matching input operand,
6552 // look up the matching input. If their types mismatch, e.g. one
6553 // is an integer, the other is floating point, or their sizes are
6554 // different, flag it as an maCantMatch.
6555 if (OpInfo.hasMatchingInput()) {
6556 AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
6557 if (OpInfo.ConstraintVT != Input.ConstraintVT) {
6558 if ((OpInfo.ConstraintVT.isInteger() !=
6559 Input.ConstraintVT.isInteger()) ||
6560 (OpInfo.ConstraintVT.getSizeInBits() !=
6561 Input.ConstraintVT.getSizeInBits())) {
6562 weightSum = -1; // Can't match.
6563 break;
6564 }
6565 }
6566 }
6567 weight = getMultipleConstraintMatchWeight(OpInfo, maIndex);
6568 if (weight == -1) {
6569 weightSum = -1;
6570 break;
6571 }
6572 weightSum += weight;
6573 }
6574 // Update best.
6575 if (weightSum > bestWeight) {
6576 bestWeight = weightSum;
6577 bestMAIndex = maIndex;
6578 }
6579 }
6580
6581 // Now select chosen alternative in each constraint.
6582 for (AsmOperandInfo &cInfo : ConstraintOperands)
6583 if (cInfo.Type != InlineAsm::isClobber)
6584 cInfo.selectAlternative(bestMAIndex);
6585 }
6586 }
6587
6588 // Check and hook up tied operands, choose constraint code to use.
6589 for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
6590 cIndex != eIndex; ++cIndex) {
6591 AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];
6592
6593 // If this is an output operand with a matching input operand, look up the
6594 // matching input. If their types mismatch, e.g. one is an integer, the
6595 // other is floating point, or their sizes are different, flag it as an
6596 // error.
6597 if (OpInfo.hasMatchingInput()) {
6598 AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
6599
6600 if (OpInfo.ConstraintVT != Input.ConstraintVT) {
6601 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
6602 getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,
6603 OpInfo.ConstraintVT);
6604 std::pair<unsigned, const TargetRegisterClass *> InputRC =
6605 getRegForInlineAsmConstraint(TRI, Input.ConstraintCode,
6606 Input.ConstraintVT);
6607 const bool OutOpIsIntOrFP = OpInfo.ConstraintVT.isInteger() ||
6608 OpInfo.ConstraintVT.isFloatingPoint();
6609 const bool InOpIsIntOrFP = Input.ConstraintVT.isInteger() ||
6610 Input.ConstraintVT.isFloatingPoint();
6611 if ((OutOpIsIntOrFP != InOpIsIntOrFP) ||
6612 (MatchRC.second != InputRC.second)) {
6613 report_fatal_error("Unsupported asm: input constraint"
6614 " with a matching output constraint of"
6615 " incompatible type!");
6616 }
6617 }
6618 }
6619 }
6620
6621 return ConstraintOperands;
6622}
6623
6624/// Return a number indicating our preference for chosing a type of constraint
6625/// over another, for the purpose of sorting them. Immediates are almost always
6626/// preferrable (when they can be emitted). A higher return value means a
6627/// stronger preference for one constraint type relative to another.
6628/// FIXME: We should prefer registers over memory but doing so may lead to
6629/// unrecoverable register exhaustion later.
6630/// https://github.com/llvm/llvm-project/issues/20571
6632 switch (CT) {
6635 return 4;
6638 return 3;
6640 return 2;
6642 return 1;
6644 return 0;
6645 }
6646 llvm_unreachable("Invalid constraint type");
6647}
6648
6649/// Examine constraint type and operand type and determine a weight value.
6650/// This object must already have been set up with the operand type
6651/// and the current alternative constraint selected.
6654 AsmOperandInfo &info, int maIndex) const {
6656 if (maIndex >= (int)info.multipleAlternatives.size())
6657 rCodes = &info.Codes;
6658 else
6659 rCodes = &info.multipleAlternatives[maIndex].Codes;
6660 ConstraintWeight BestWeight = CW_Invalid;
6661
6662 // Loop over the options, keeping track of the most general one.
6663 for (const std::string &rCode : *rCodes) {
6664 ConstraintWeight weight =
6665 getSingleConstraintMatchWeight(info, rCode.c_str());
6666 if (weight > BestWeight)
6667 BestWeight = weight;
6668 }
6669
6670 return BestWeight;
6671}
6672
6673/// Examine constraint type and operand type and determine a weight value.
6674/// This object must already have been set up with the operand type
6675/// and the current alternative constraint selected.
6678 AsmOperandInfo &info, const char *constraint) const {
6680 Value *CallOperandVal = info.CallOperandVal;
6681 // If we don't have a value, we can't do a match,
6682 // but allow it at the lowest weight.
6683 if (!CallOperandVal)
6684 return CW_Default;
6685 // Look at the constraint type.
6686 switch (*constraint) {
6687 case 'i': // immediate integer.
6688 case 'n': // immediate integer with a known value.
6689 if (isa<ConstantInt>(CallOperandVal))
6690 weight = CW_Constant;
6691 break;
6692 case 's': // non-explicit intregal immediate.
6693 if (isa<GlobalValue>(CallOperandVal))
6694 weight = CW_Constant;
6695 break;
6696 case 'E': // immediate float if host format.
6697 case 'F': // immediate float.
6698 if (isa<ConstantFP>(CallOperandVal))
6699 weight = CW_Constant;
6700 break;
6701 case '<': // memory operand with autodecrement.
6702 case '>': // memory operand with autoincrement.
6703 case 'm': // memory operand.
6704 case 'o': // offsettable memory operand
6705 case 'V': // non-offsettable memory operand
6706 weight = CW_Memory;
6707 break;
6708 case 'r': // general register.
6709 case 'g': // general register, memory operand or immediate integer.
6710 // note: Clang converts "g" to "imr".
6711 if (CallOperandVal->getType()->isIntegerTy())
6712 weight = CW_Register;
6713 break;
6714 case 'X': // any operand.
6715 default:
6716 weight = CW_Default;
6717 break;
6718 }
6719 return weight;
6720}
6721
6722/// If there are multiple different constraints that we could pick for this
6723/// operand (e.g. "imr") try to pick the 'best' one.
6724/// This is somewhat tricky: constraints (TargetLowering::ConstraintType) fall
6725/// into seven classes:
6726/// Register -> one specific register
6727/// RegisterClass -> a group of regs
6728/// Memory -> memory
6729/// Address -> a symbolic memory reference
6730/// Immediate -> immediate values
6731/// Other -> magic values (such as "Flag Output Operands")
6732/// Unknown -> something we don't recognize yet and can't handle
6733/// Ideally, we would pick the most specific constraint possible: if we have
6734/// something that fits into a register, we would pick it. The problem here
6735/// is that if we have something that could either be in a register or in
6736/// memory that use of the register could cause selection of *other*
6737/// operands to fail: they might only succeed if we pick memory. Because of
6738/// this the heuristic we use is:
6739///
6740/// 1) If there is an 'other' constraint, and if the operand is valid for
6741/// that constraint, use it. This makes us take advantage of 'i'
6742/// constraints when available.
6743/// 2) Otherwise, pick the most general constraint present. This prefers
6744/// 'm' over 'r', for example.
6745///
6747 TargetLowering::AsmOperandInfo &OpInfo) const {
6748 ConstraintGroup Ret;
6749
6750 Ret.reserve(OpInfo.Codes.size());
6751 for (StringRef Code : OpInfo.Codes) {
6753
6754 // Indirect 'other' or 'immediate' constraints are not allowed.
6755 if (OpInfo.isIndirect && !(CType == TargetLowering::C_Memory ||
6756 CType == TargetLowering::C_Register ||
6758 continue;
6759
6760 // Things with matching constraints can only be registers, per gcc
6761 // documentation. This mainly affects "g" constraints.
6762 if (CType == TargetLowering::C_Memory && OpInfo.hasMatchingInput())
6763 continue;
6764
6765 Ret.emplace_back(Code, CType);
6766 }
6767
6769 return getConstraintPiority(a.second) > getConstraintPiority(b.second);
6770 });
6771
6772 return Ret;
6773}
6774
6775/// If we have an immediate, see if we can lower it. Return true if we can,
6776/// false otherwise.
6778 SDValue Op, SelectionDAG *DAG,
6779 const TargetLowering &TLI) {
6780
6781 assert((P.second == TargetLowering::C_Other ||
6782 P.second == TargetLowering::C_Immediate) &&
6783 "need immediate or other");
6784
6785 if (!Op.getNode())
6786 return false;
6787
6788 std::vector<SDValue> ResultOps;
6789 TLI.LowerAsmOperandForConstraint(Op, P.first, ResultOps, *DAG);
6790 return !ResultOps.empty();
6791}
6792
6793/// Determines the constraint code and constraint type to use for the specific
6794/// AsmOperandInfo, setting OpInfo.ConstraintCode and OpInfo.ConstraintType.
6796 SDValue Op,
6797 SelectionDAG *DAG) const {
6798 assert(!OpInfo.Codes.empty() && "Must have at least one constraint");
6799
6800 // Single-letter constraints ('r') are very common.
6801 if (OpInfo.Codes.size() == 1) {
6802 OpInfo.ConstraintCode = OpInfo.Codes[0];
6803 OpInfo.ConstraintType = getConstraintType(OpInfo.ConstraintCode);
6804 } else {
6806 if (G.empty())
6807 return;
6808
6809 unsigned BestIdx = 0;
6810 for (const unsigned E = G.size();
6811 BestIdx < E && (G[BestIdx].second == TargetLowering::C_Other ||
6812 G[BestIdx].second == TargetLowering::C_Immediate);
6813 ++BestIdx) {
6814 if (lowerImmediateIfPossible(G[BestIdx], Op, DAG, *this))
6815 break;
6816 // If we're out of constraints, just pick the first one.
6817 if (BestIdx + 1 == E) {
6818 BestIdx = 0;
6819 break;
6820 }
6821 }
6822
6823 OpInfo.ConstraintCode = G[BestIdx].first;
6824 OpInfo.ConstraintType = G[BestIdx].second;
6825 }
6826
6827 // 'X' matches anything.
6828 if (OpInfo.ConstraintCode == "X" && OpInfo.CallOperandVal) {
6829 // Constants are handled elsewhere. For Functions, the type here is the
6830 // type of the result, which is not what we want to look at; leave them
6831 // alone.
6832 Value *v = OpInfo.CallOperandVal;
6833 if (isa<ConstantInt>(v) || isa<Function>(v)) {
6834 return;
6835 }
6836
6837 if (isa<BasicBlock>(v) || isa<BlockAddress>(v)) {
6838 OpInfo.ConstraintCode = "i";
6839 return;
6840 }
6841
6842 // Otherwise, try to resolve it to something we know about by looking at
6843 // the actual operand type.
6844 if (const char *Repl = LowerXConstraint(OpInfo.ConstraintVT)) {
6845 OpInfo.ConstraintCode = Repl;
6846 OpInfo.ConstraintType = getConstraintType(OpInfo.ConstraintCode);
6847 }
6848 }
6849}
6850
6851/// Given an exact SDIV by a constant, create a multiplication
6852/// with the multiplicative inverse of the constant.
6853/// Ref: "Hacker's Delight" by Henry Warren, 2nd Edition, p. 242
6855 const SDLoc &dl, SelectionDAG &DAG,
6856 SmallVectorImpl<SDNode *> &Created) {
6857 SDValue Op0 = N->getOperand(0);
6858 SDValue Op1 = N->getOperand(1);
6859 EVT VT = N->getValueType(0);
6860 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
6861 EVT ShSVT = ShVT.getScalarType();
6862
6863 bool UseSRA = false;
6864 SmallVector<SDValue, 16> Shifts, Factors;
6865
6866 auto BuildSDIVPattern = [&](ConstantSDNode *C) {
6867 if (C->isZero())
6868 return false;
6869
6870 EVT CT = C->getValueType(0);
6871 APInt Divisor = C->getAPIntValue();
6872 unsigned Shift = Divisor.countr_zero();
6873 if (Shift) {
6874 Divisor.ashrInPlace(Shift);
6875 UseSRA = true;
6876 }
6877 APInt Factor = Divisor.multiplicativeInverse();
6878 Shifts.push_back(DAG.getConstant(Shift, dl, ShSVT));
6879 Factors.push_back(DAG.getConstant(Factor, dl, CT));
6880 return true;
6881 };
6882
6883 // Collect all magic values from the build vector.
6884 if (!ISD::matchUnaryPredicate(Op1, BuildSDIVPattern))
6885 return SDValue();
6886
6887 SDValue Shift, Factor;
6888 if (Op1.getOpcode() == ISD::BUILD_VECTOR) {
6889 Shift = DAG.getBuildVector(ShVT, dl, Shifts);
6890 Factor = DAG.getBuildVector(VT, dl, Factors);
6891 } else if (Op1.getOpcode() == ISD::SPLAT_VECTOR) {
6892 assert(Shifts.size() == 1 && Factors.size() == 1 &&
6893 "Expected matchUnaryPredicate to return one element for scalable "
6894 "vectors");
6895 Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]);
6896 Factor = DAG.getSplatVector(VT, dl, Factors[0]);
6897 } else {
6898 assert(isa<ConstantSDNode>(Op1) && "Expected a constant");
6899 Shift = Shifts[0];
6900 Factor = Factors[0];
6901 }
6902
6903 SDValue Res = Op0;
6904 if (UseSRA) {
6905 Res = DAG.getNode(ISD::SRA, dl, VT, Res, Shift, SDNodeFlags::Exact);
6906 Created.push_back(Res.getNode());
6907 }
6908
6909 return DAG.getNode(ISD::MUL, dl, VT, Res, Factor);
6910}
6911
6912/// Given an exact UDIV by a constant, create a multiplication
6913/// with the multiplicative inverse of the constant.
6914/// Ref: "Hacker's Delight" by Henry Warren, 2nd Edition, p. 242
6916 const SDLoc &dl, SelectionDAG &DAG,
6917 SmallVectorImpl<SDNode *> &Created) {
6918 EVT VT = N->getValueType(0);
6919 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
6920 EVT ShSVT = ShVT.getScalarType();
6921
6922 bool UseSRL = false;
6923 SmallVector<SDValue, 16> Shifts, Factors;
6924
6925 auto BuildUDIVPattern = [&](ConstantSDNode *C) {
6926 if (C->isZero())
6927 return false;
6928
6929 EVT CT = C->getValueType(0);
6930 APInt Divisor = C->getAPIntValue();
6931 unsigned Shift = Divisor.countr_zero();
6932 if (Shift) {
6933 Divisor.lshrInPlace(Shift);
6934 UseSRL = true;
6935 }
6936 // Calculate the multiplicative inverse modulo BW.
6937 APInt Factor = Divisor.multiplicativeInverse();
6938 Shifts.push_back(DAG.getConstant(Shift, dl, ShSVT));
6939 Factors.push_back(DAG.getConstant(Factor, dl, CT));
6940 return true;
6941 };
6942
6943 SDValue Op1 = N->getOperand(1);
6944
6945 // Collect all magic values from the build vector.
6946 if (!ISD::matchUnaryPredicate(Op1, BuildUDIVPattern))
6947 return SDValue();
6948
6949 SDValue Shift, Factor;
6950 if (Op1.getOpcode() == ISD::BUILD_VECTOR) {
6951 Shift = DAG.getBuildVector(ShVT, dl, Shifts);
6952 Factor = DAG.getBuildVector(VT, dl, Factors);
6953 } else if (Op1.getOpcode() == ISD::SPLAT_VECTOR) {
6954 assert(Shifts.size() == 1 && Factors.size() == 1 &&
6955 "Expected matchUnaryPredicate to return one element for scalable "
6956 "vectors");
6957 Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]);
6958 Factor = DAG.getSplatVector(VT, dl, Factors[0]);
6959 } else {
6960 assert(isa<ConstantSDNode>(Op1) && "Expected a constant");
6961 Shift = Shifts[0];
6962 Factor = Factors[0];
6963 }
6964
6965 SDValue Res = N->getOperand(0);
6966 if (UseSRL) {
6967 Res = DAG.getNode(ISD::SRL, dl, VT, Res, Shift, SDNodeFlags::Exact);
6968 Created.push_back(Res.getNode());
6969 }
6970
6971 return DAG.getNode(ISD::MUL, dl, VT, Res, Factor);
6972}
6973
6975 SelectionDAG &DAG,
6976 SmallVectorImpl<SDNode *> &Created) const {
6978 if (isIntDivCheap(N->getValueType(0), Attr))
6979 return SDValue(N, 0); // Lower SDIV as SDIV
6980 return SDValue();
6981}
6982
6983SDValue
6985 SelectionDAG &DAG,
6986 SmallVectorImpl<SDNode *> &Created) const {
6988 if (isIntDivCheap(N->getValueType(0), Attr))
6989 return SDValue(N, 0); // Lower SREM as SREM
6990 return SDValue();
6991}
6992
6993/// Build sdiv by power-of-2 with conditional move instructions
6994/// Ref: "Hacker's Delight" by Henry Warren 10-1
6995/// If conditional move/branch is preferred, we lower sdiv x, +/-2**k into:
6996/// bgez x, label
6997/// add x, x, 2**k-1
6998/// label:
6999/// sra res, x, k
7000/// neg res, res (when the divisor is negative)
7002 SDNode *N, const APInt &Divisor, SelectionDAG &DAG,
7003 SmallVectorImpl<SDNode *> &Created) const {
7004 unsigned Lg2 = Divisor.countr_zero();
7005 EVT VT = N->getValueType(0);
7006
7007 SDLoc DL(N);
7008 SDValue N0 = N->getOperand(0);
7009 SDValue Zero = DAG.getConstant(0, DL, VT);
7010 APInt Lg2Mask = APInt::getLowBitsSet(VT.getSizeInBits(), Lg2);
7011 SDValue Pow2MinusOne = DAG.getConstant(Lg2Mask, DL, VT);
7012
7013 // If N0 is negative, we need to add (Pow2 - 1) to it before shifting right.
7014 EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
7015 SDValue Cmp = DAG.getSetCC(DL, CCVT, N0, Zero, ISD::SETLT);
7016 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Pow2MinusOne);
7017 SDValue CMov = DAG.getNode(ISD::SELECT, DL, VT, Cmp, Add, N0);
7018
7019 Created.push_back(Cmp.getNode());
7020 Created.push_back(Add.getNode());
7021 Created.push_back(CMov.getNode());
7022
7023 // Divide by pow2.
7024 SDValue SRA = DAG.getNode(ISD::SRA, DL, VT, CMov,
7025 DAG.getShiftAmountConstant(Lg2, VT, DL));
7026
7027 // If we're dividing by a positive value, we're done. Otherwise, we must
7028 // negate the result.
7029 if (Divisor.isNonNegative())
7030 return SRA;
7031
7032 Created.push_back(SRA.getNode());
7033 return DAG.getNode(ISD::SUB, DL, VT, Zero, SRA);
7034}
7035
7036/// Given an ISD::SDIV node expressing a divide by constant,
7037/// return a DAG expression to select that will generate the same value by
7038/// multiplying by a magic number.
7039/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
7041 bool IsAfterLegalization,
7042 bool IsAfterLegalTypes,
7043 SmallVectorImpl<SDNode *> &Created) const {
7044 SDLoc dl(N);
7045
7046 // If the sdiv has an 'exact' bit we can use a simpler lowering.
7047 if (N->getFlags().hasExact())
7048 return BuildExactSDIV(*this, N, dl, DAG, Created);
7049
7050 EVT VT = N->getValueType(0);
7051 EVT SVT = VT.getScalarType();
7052 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
7053 EVT ShSVT = ShVT.getScalarType();
7054 unsigned EltBits = VT.getScalarSizeInBits();
7055 EVT MulVT;
7056
7057 // Check to see if we can do this.
7058 // FIXME: We should be more aggressive here.
7059 EVT QueryVT = VT;
7060 if (VT.isVector()) {
7061 // If the vector type will be legalized to a vector type with the same
7062 // element type, allow the transform before type legalization if MULHS or
7063 // SMUL_LOHI are supported.
7064 QueryVT = getLegalTypeToTransformTo(*DAG.getContext(), VT);
7065 if (!QueryVT.isVector() ||
7067 return SDValue();
7068 } else if (!isTypeLegal(VT)) {
7069 // Limit this to simple scalars for now.
7070 if (!VT.isSimple())
7071 return SDValue();
7072
7073 // If this type will be promoted to a large enough type with a legal
7074 // multiply operation, we can go ahead and do this transform.
7076 return SDValue();
7077
7078 MulVT = getTypeToTransformTo(*DAG.getContext(), VT);
7079 if (MulVT.getSizeInBits() < (2 * EltBits) ||
7080 !isOperationLegal(ISD::MUL, MulVT))
7081 return SDValue();
7082 }
7083
7084 bool HasMULHS =
7085 isOperationLegalOrCustom(ISD::MULHS, QueryVT, IsAfterLegalization);
7086 bool HasSMUL_LOHI =
7087 isOperationLegalOrCustom(ISD::SMUL_LOHI, QueryVT, IsAfterLegalization);
7088
7089 if (isTypeLegal(VT) && !HasMULHS && !HasSMUL_LOHI && MulVT == EVT()) {
7090 // If type twice as wide legal, widen and use a mul plus a shift.
7091 EVT WideVT = VT.widenIntegerElementType(*DAG.getContext());
7092 // Some targets like AMDGPU try to go from SDIV to SDIVREM which is then
7093 // custom lowered. This is very expensive so avoid it at all costs for
7094 // constant divisors.
7095 if ((!IsAfterLegalTypes && isOperationExpand(ISD::SDIV, VT) &&
7098 MulVT = WideVT;
7099 }
7100
7101 if (!HasMULHS && !HasSMUL_LOHI && MulVT == EVT())
7102 return SDValue();
7103
7104 // If we're after type legalization and SVT is not legal, use the
7105 // promoted type for creating constants to avoid creating nodes with
7106 // illegal types.
7107 if (IsAfterLegalTypes && VT.isVector()) {
7108 SVT = getTypeToTransformTo(*DAG.getContext(), SVT);
7109 if (SVT.bitsLT(VT.getScalarType()))
7110 return SDValue();
7111 ShSVT = getTypeToTransformTo(*DAG.getContext(), ShSVT);
7112 if (ShSVT.bitsLT(ShVT.getScalarType()))
7113 return SDValue();
7114 }
7115 const unsigned SVTBits = SVT.getSizeInBits();
7116
7117 SmallVector<SDValue, 16> MagicFactors, Factors, Shifts, ShiftMasks;
7118
7119 auto BuildSDIVPattern = [&](ConstantSDNode *C) {
7120 if (C->isZero())
7121 return false;
7122 // Truncate the divisor to the target scalar type in case it was promoted
7123 // during type legalization.
7124 APInt Divisor = C->getAPIntValue().trunc(EltBits);
7126 int NumeratorFactor = 0;
7127 int ShiftMask = -1;
7128
7129 if (Divisor.isOne() || Divisor.isAllOnes()) {
7130 // If d is +1/-1, we just multiply the numerator by +1/-1.
7131 NumeratorFactor = Divisor.getSExtValue();
7132 magics.Magic = 0;
7133 magics.ShiftAmount = 0;
7134 ShiftMask = 0;
7135 } else if (Divisor.isStrictlyPositive() && magics.Magic.isNegative()) {
7136 // If d > 0 and m < 0, add the numerator.
7137 NumeratorFactor = 1;
7138 } else if (Divisor.isNegative() && magics.Magic.isStrictlyPositive()) {
7139 // If d < 0 and m > 0, subtract the numerator.
7140 NumeratorFactor = -1;
7141 }
7142
7143 MagicFactors.push_back(
7144 DAG.getConstant(magics.Magic.zext(SVTBits), dl, SVT));
7145 Factors.push_back(DAG.getSignedConstant(NumeratorFactor, dl, SVT));
7146 Shifts.push_back(DAG.getConstant(magics.ShiftAmount, dl, ShSVT));
7147 ShiftMasks.push_back(DAG.getSignedConstant(ShiftMask, dl, SVT));
7148 return true;
7149 };
7150
7151 SDValue N0 = N->getOperand(0);
7152 SDValue N1 = N->getOperand(1);
7153
7154 // Collect the shifts / magic values from each element.
7155 if (!ISD::matchUnaryPredicate(N1, BuildSDIVPattern, /*AllowUndefs=*/false,
7156 /*AllowTruncation=*/true))
7157 return SDValue();
7158
7159 SDValue MagicFactor, Factor, Shift, ShiftMask;
7160 if (N1.getOpcode() == ISD::BUILD_VECTOR) {
7161 MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);
7162 Factor = DAG.getBuildVector(VT, dl, Factors);
7163 Shift = DAG.getBuildVector(ShVT, dl, Shifts);
7164 ShiftMask = DAG.getBuildVector(VT, dl, ShiftMasks);
7165 } else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {
7166 assert(MagicFactors.size() == 1 && Factors.size() == 1 &&
7167 Shifts.size() == 1 && ShiftMasks.size() == 1 &&
7168 "Expected matchUnaryPredicate to return one element for scalable "
7169 "vectors");
7170 MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]);
7171 Factor = DAG.getSplatVector(VT, dl, Factors[0]);
7172 Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]);
7173 ShiftMask = DAG.getSplatVector(VT, dl, ShiftMasks[0]);
7174 } else {
7175 assert(isa<ConstantSDNode>(N1) && "Expected a constant");
7176 MagicFactor = MagicFactors[0];
7177 Factor = Factors[0];
7178 Shift = Shifts[0];
7179 ShiftMask = ShiftMasks[0];
7180 }
7181
7182 // Multiply the numerator (operand 0) by the magic value.
7183 auto GetMULHS = [&](SDValue X, SDValue Y) {
7184 if (HasMULHS)
7185 return DAG.getNode(ISD::MULHS, dl, VT, X, Y);
7186 if (HasSMUL_LOHI) {
7187 SDValue LoHi =
7188 DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);
7189 return LoHi.getValue(1);
7190 }
7191
7192 X = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, X);
7193 Y = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, Y);
7194 Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y);
7195 Y = DAG.getNode(ISD::SRL, dl, MulVT, Y,
7196 DAG.getShiftAmountConstant(EltBits, MulVT, dl));
7197 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);
7198 };
7199
7200 SDValue Q = GetMULHS(N0, MagicFactor);
7201 if (!Q)
7202 return SDValue();
7203
7204 Created.push_back(Q.getNode());
7205
7206 // (Optionally) Add/subtract the numerator using Factor.
7207 Factor = DAG.getNode(ISD::MUL, dl, VT, N0, Factor);
7208 Created.push_back(Factor.getNode());
7209 Q = DAG.getNode(ISD::ADD, dl, VT, Q, Factor);
7210 Created.push_back(Q.getNode());
7211
7212 // Shift right algebraic by shift value.
7213 Q = DAG.getNode(ISD::SRA, dl, VT, Q, Shift);
7214 Created.push_back(Q.getNode());
7215
7216 // Extract the sign bit, mask it and add it to the quotient.
7217 SDValue SignShift = DAG.getConstant(EltBits - 1, dl, ShVT);
7218 SDValue T = DAG.getNode(ISD::SRL, dl, VT, Q, SignShift);
7219 Created.push_back(T.getNode());
7220 T = DAG.getNode(ISD::AND, dl, VT, T, ShiftMask);
7221 Created.push_back(T.getNode());
7222 return DAG.getNode(ISD::ADD, dl, VT, Q, T);
7223}
7224
7225/// Given an ISD::UDIV node expressing a divide by constant,
7226/// return a DAG expression to select that will generate the same value by
7227/// multiplying by a magic number.
7228/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
7230 bool IsAfterLegalization,
7231 bool IsAfterLegalTypes,
7232 SmallVectorImpl<SDNode *> &Created) const {
7233 SDLoc dl(N);
7234
7235 // If the udiv has an 'exact' bit we can use a simpler lowering.
7236 if (N->getFlags().hasExact())
7237 return BuildExactUDIV(*this, N, dl, DAG, Created);
7238
7239 EVT VT = N->getValueType(0);
7240 EVT SVT = VT.getScalarType();
7241 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
7242 EVT ShSVT = ShVT.getScalarType();
7243 unsigned EltBits = VT.getScalarSizeInBits();
7244 EVT MulVT;
7245
7246 // Check to see if we can do this.
7247 // FIXME: We should be more aggressive here.
7248 EVT QueryVT = VT;
7249 if (VT.isVector()) {
7250 // If the vector type will be legalized to a vector type with the same
7251 // element type, allow the transform before type legalization if MULHU or
7252 // UMUL_LOHI are supported.
7253 QueryVT = getLegalTypeToTransformTo(*DAG.getContext(), VT);
7254 if (!QueryVT.isVector() ||
7256 return SDValue();
7257 } else if (!isTypeLegal(VT)) {
7258 // Limit this to simple scalars for now.
7259 if (!VT.isSimple())
7260 return SDValue();
7261
7262 // If this type will be promoted to a large enough type with a legal
7263 // multiply operation, we can go ahead and do this transform.
7265 return SDValue();
7266
7267 MulVT = getTypeToTransformTo(*DAG.getContext(), VT);
7268 if (MulVT.getSizeInBits() < (2 * EltBits) ||
7269 !isOperationLegal(ISD::MUL, MulVT))
7270 return SDValue();
7271 }
7272
7273 bool HasMULHU =
7274 isOperationLegalOrCustom(ISD::MULHU, QueryVT, IsAfterLegalization);
7275 bool HasUMUL_LOHI =
7276 isOperationLegalOrCustom(ISD::UMUL_LOHI, QueryVT, IsAfterLegalization);
7277
7278 if (isTypeLegal(VT) && !HasMULHU && !HasUMUL_LOHI && MulVT == EVT()) {
7279 // If type twice as wide legal, widen and use a mul plus a shift.
7280 EVT WideVT = VT.widenIntegerElementType(*DAG.getContext());
7281 // Some targets like AMDGPU try to go from UDIV to UDIVREM which is then
7282 // custom lowered. This is very expensive so avoid it at all costs for
7283 // constant divisors.
7284 if ((!IsAfterLegalTypes && isOperationExpand(ISD::UDIV, VT) &&
7287 MulVT = WideVT;
7288 }
7289
7290 if (!HasMULHU && !HasUMUL_LOHI && MulVT == EVT())
7291 return SDValue();
7292
7293 SDValue N0 = N->getOperand(0);
7294 SDValue N1 = N->getOperand(1);
7295
7296 // Try to use leading zeros of the dividend to reduce the multiplier and
7297 // avoid expensive fixups.
7298 unsigned KnownLeadingZeros = DAG.computeKnownBits(N0).countMinLeadingZeros();
7299
7300 // If we're after type legalization and SVT is not legal, use the
7301 // promoted type for creating constants to avoid creating nodes with
7302 // illegal types.
7303 if (IsAfterLegalTypes && VT.isVector()) {
7304 SVT = getTypeToTransformTo(*DAG.getContext(), SVT);
7305 if (SVT.bitsLT(VT.getScalarType()))
7306 return SDValue();
7307 ShSVT = getTypeToTransformTo(*DAG.getContext(), ShSVT);
7308 if (ShSVT.bitsLT(ShVT.getScalarType()))
7309 return SDValue();
7310 }
7311 const unsigned SVTBits = SVT.getSizeInBits();
7312
7313 // Allow i32 to be widened to i64 for uncooperative divisors if i64 MULHU or
7314 // UMUL_LOHI is supported.
7315 const EVT WideSVT = MVT::i64;
7316 const bool HasWideMULHU =
7317 VT == MVT::i32 &&
7318 isOperationLegalOrCustom(ISD::MULHU, WideSVT, IsAfterLegalization);
7319 const bool HasWideUMUL_LOHI =
7320 VT == MVT::i32 &&
7321 isOperationLegalOrCustom(ISD::UMUL_LOHI, WideSVT, IsAfterLegalization);
7322 const bool AllowWiden = (HasWideMULHU || HasWideUMUL_LOHI);
7323
7324 // For even divisors with a 33-bit magic number, the widened high-multiply
7325 // path is only worthwhile over the even-divisor rewrite on targets that
7326 // zero-extend i32 to i64 for free (e.g. x86-64 and AArch64). Elsewhere (e.g.
7327 // RISC-V) keep the even-divisor rewrite, which avoids the explicit extension.
7328 const bool AllowEvenToWiden = AllowWiden && isZExtFree(VT, WideSVT);
7329
7330 bool UseNPQ = false, UsePreShift = false, UsePostShift = false;
7331 bool UseWiden = false;
7332 SmallVector<SDValue, 16> PreShifts, PostShifts, MagicFactors, NPQFactors;
7333
7334 auto BuildUDIVPattern = [&](ConstantSDNode *C) {
7335 if (C->isZero())
7336 return false;
7337 // Truncate the divisor to the target scalar type in case it was promoted
7338 // during type legalization.
7339 APInt Divisor = C->getAPIntValue().trunc(EltBits);
7340
7341 SDValue PreShift, MagicFactor, NPQFactor, PostShift;
7342
7343 // Magic algorithm doesn't work for division by 1. We need to emit a select
7344 // at the end.
7345 if (Divisor.isOne()) {
7346 PreShift = PostShift = DAG.getUNDEF(ShSVT);
7347 MagicFactor = NPQFactor = DAG.getUNDEF(SVT);
7348 } else {
7351 Divisor, std::min(KnownLeadingZeros, Divisor.countl_zero()),
7352 /*AllowEvenDivisorOptimization=*/!AllowEvenToWiden,
7353 /*AllowWidenOptimization=*/AllowWiden);
7354
7355 if (magics.Widen) {
7356 UseWiden = true;
7357 MagicFactor = DAG.getConstant(magics.Magic, dl, WideSVT);
7358 } else {
7359 MagicFactor = DAG.getConstant(magics.Magic.zext(SVTBits), dl, SVT);
7360 }
7361
7362 assert(magics.PreShift < Divisor.getBitWidth() &&
7363 "We shouldn't generate an undefined shift!");
7364 assert(magics.PostShift < Divisor.getBitWidth() &&
7365 "We shouldn't generate an undefined shift!");
7366 assert((!magics.IsAdd || magics.PreShift == 0) &&
7367 "Unexpected pre-shift");
7368 PreShift = DAG.getConstant(magics.PreShift, dl, ShSVT);
7369 PostShift = DAG.getConstant(magics.PostShift, dl, ShSVT);
7370 NPQFactor = DAG.getConstant(
7371 magics.IsAdd ? APInt::getOneBitSet(SVTBits, EltBits - 1)
7372 : APInt::getZero(SVTBits),
7373 dl, SVT);
7374 UseNPQ |= magics.IsAdd;
7375 UsePreShift |= magics.PreShift != 0;
7376 UsePostShift |= magics.PostShift != 0;
7377 }
7378
7379 PreShifts.push_back(PreShift);
7380 MagicFactors.push_back(MagicFactor);
7381 NPQFactors.push_back(NPQFactor);
7382 PostShifts.push_back(PostShift);
7383 return true;
7384 };
7385
7386 // Collect the shifts/magic values from each element.
7387 if (!ISD::matchUnaryPredicate(N1, BuildUDIVPattern, /*AllowUndefs=*/false,
7388 /*AllowTruncation=*/true))
7389 return SDValue();
7390
7391 SDValue PreShift, PostShift, MagicFactor, NPQFactor;
7392 if (N1.getOpcode() == ISD::BUILD_VECTOR) {
7393 PreShift = DAG.getBuildVector(ShVT, dl, PreShifts);
7394 MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);
7395 NPQFactor = DAG.getBuildVector(VT, dl, NPQFactors);
7396 PostShift = DAG.getBuildVector(ShVT, dl, PostShifts);
7397 } else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {
7398 assert(PreShifts.size() == 1 && MagicFactors.size() == 1 &&
7399 NPQFactors.size() == 1 && PostShifts.size() == 1 &&
7400 "Expected matchUnaryPredicate to return one for scalable vectors");
7401 PreShift = DAG.getSplatVector(ShVT, dl, PreShifts[0]);
7402 MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]);
7403 NPQFactor = DAG.getSplatVector(VT, dl, NPQFactors[0]);
7404 PostShift = DAG.getSplatVector(ShVT, dl, PostShifts[0]);
7405 } else {
7406 assert(isa<ConstantSDNode>(N1) && "Expected a constant");
7407 PreShift = PreShifts[0];
7408 MagicFactor = MagicFactors[0];
7409 PostShift = PostShifts[0];
7410 }
7411
7412 if (UseWiden) {
7413 // Compute: (WideSVT(x) * MagicFactor) >> WideSVTBits.
7414 SDValue WideN0 = DAG.getNode(ISD::ZERO_EXTEND, dl, WideSVT, N0);
7415
7416 // Perform WideSVTxWideSVT -> 2*WideSVT multiplication and extract high
7417 // WideSVT bits
7418 SDValue High;
7419 if (HasWideMULHU) {
7420 High = DAG.getNode(ISD::MULHU, dl, WideSVT, WideN0, MagicFactor);
7421 } else {
7422 assert(HasWideUMUL_LOHI);
7423 SDValue LoHi =
7424 DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(WideSVT, WideSVT),
7425 WideN0, MagicFactor);
7426 High = LoHi.getValue(1);
7427 }
7428
7429 Created.push_back(High.getNode());
7430 return DAG.getNode(ISD::TRUNCATE, dl, VT, High);
7431 }
7432
7433 SDValue Q = N0;
7434 if (UsePreShift) {
7435 Q = DAG.getNode(ISD::SRL, dl, VT, Q, PreShift);
7436 Created.push_back(Q.getNode());
7437 }
7438
7439 auto GetMULHU = [&](SDValue X, SDValue Y) {
7440 if (HasMULHU)
7441 return DAG.getNode(ISD::MULHU, dl, VT, X, Y);
7442 if (HasUMUL_LOHI) {
7443 SDValue LoHi =
7444 DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);
7445 return LoHi.getValue(1);
7446 }
7447
7448 X = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, X);
7449 Y = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, Y);
7450 Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y);
7451 Y = DAG.getNode(ISD::SRL, dl, MulVT, Y,
7452 DAG.getShiftAmountConstant(EltBits, MulVT, dl));
7453 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);
7454 };
7455
7456 // Multiply the numerator (operand 0) by the magic value.
7457 Q = GetMULHU(Q, MagicFactor);
7458 if (!Q)
7459 return SDValue();
7460
7461 Created.push_back(Q.getNode());
7462
7463 if (UseNPQ) {
7464 SDValue NPQ = DAG.getNode(ISD::SUB, dl, VT, N0, Q);
7465 Created.push_back(NPQ.getNode());
7466
7467 // For vectors we might have a mix of non-NPQ/NPQ paths, so use
7468 // MULHU to act as a SRL-by-1 for NPQ, else multiply by zero.
7469 if (VT.isVector())
7470 NPQ = GetMULHU(NPQ, NPQFactor);
7471 else
7472 NPQ = DAG.getNode(ISD::SRL, dl, VT, NPQ, DAG.getConstant(1, dl, ShVT));
7473
7474 Created.push_back(NPQ.getNode());
7475
7476 Q = DAG.getNode(ISD::ADD, dl, VT, NPQ, Q);
7477 Created.push_back(Q.getNode());
7478 }
7479
7480 if (UsePostShift) {
7481 Q = DAG.getNode(ISD::SRL, dl, VT, Q, PostShift);
7482 Created.push_back(Q.getNode());
7483 }
7484
7485 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
7486
7487 SDValue One = DAG.getConstant(1, dl, VT);
7488 SDValue IsOne = DAG.getSetCC(dl, SetCCVT, N1, One, ISD::SETEQ);
7489 return DAG.getSelect(dl, VT, IsOne, N0, Q);
7490}
7491
7492/// If all values in Values that *don't* match the predicate are same 'splat'
7493/// value, then replace all values with that splat value.
7494/// Else, if AlternativeReplacement was provided, then replace all values that
7495/// do match predicate with AlternativeReplacement value.
7496static void
7498 std::function<bool(SDValue)> Predicate,
7499 SDValue AlternativeReplacement = SDValue()) {
7500 SDValue Replacement;
7501 // Is there a value for which the Predicate does *NOT* match? What is it?
7502 auto SplatValue = llvm::find_if_not(Values, Predicate);
7503 if (SplatValue != Values.end()) {
7504 // Does Values consist only of SplatValue's and values matching Predicate?
7505 if (llvm::all_of(Values, [Predicate, SplatValue](SDValue Value) {
7506 return Value == *SplatValue || Predicate(Value);
7507 })) // Then we shall replace values matching predicate with SplatValue.
7508 Replacement = *SplatValue;
7509 }
7510 if (!Replacement) {
7511 // Oops, we did not find the "baseline" splat value.
7512 if (!AlternativeReplacement)
7513 return; // Nothing to do.
7514 // Let's replace with provided value then.
7515 Replacement = AlternativeReplacement;
7516 }
7517 std::replace_if(Values.begin(), Values.end(), Predicate, Replacement);
7518}
7519
7520/// Given an ISD::UREM used only by an ISD::SETEQ or ISD::SETNE
7521/// where the divisor and comparison target are constants,
7522/// return a DAG expression that will generate the same comparison result
7523/// using only multiplications, additions and shifts/rotations.
7524/// Ref: "Hacker's Delight" 10-17.
7525SDValue TargetLowering::buildUREMEqFold(EVT SETCCVT, SDValue REMNode,
7526 SDValue CompTargetNode,
7528 DAGCombinerInfo &DCI,
7529 const SDLoc &DL) const {
7531 if (SDValue Folded = prepareUREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
7532 DCI, DL, Built)) {
7533 for (SDNode *N : Built)
7534 DCI.AddToWorklist(N);
7535 return Folded;
7536 }
7537
7538 return SDValue();
7539}
7540
7541SDValue
7542TargetLowering::prepareUREMEqFold(EVT SETCCVT, SDValue REMNode,
7543 SDValue CompTargetNode, ISD::CondCode Cond,
7544 DAGCombinerInfo &DCI, const SDLoc &DL,
7545 SmallVectorImpl<SDNode *> &Created) const {
7546 // fold (seteq/ne (urem N, D), C) ->
7547 // (setule/ugt (rotr (mul (sub N, C), P), K), Q)
7548 // - D must be constant, with D = D0 * 2^K where D0 is odd
7549 // - P is the multiplicative inverse of D0 modulo 2^W
7550 // - Q = floor(((2^W) - 1) / D)
7551 // where W is the width of the common type of N and D.
7552 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
7553 "Only applicable for (in)equality comparisons.");
7554
7555 SelectionDAG &DAG = DCI.DAG;
7556
7557 EVT VT = REMNode.getValueType();
7558 EVT SVT = VT.getScalarType();
7559 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
7560 EVT ShSVT = ShVT.getScalarType();
7561
7562 // If MUL is unavailable, we cannot proceed in any case.
7563 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT))
7564 return SDValue();
7565
7566 bool ComparingWithAllZeros = true;
7567 bool AllComparisonsWithNonZerosAreTautological = true;
7568 bool HadTautologicalLanes = false;
7569 bool AllLanesAreTautological = true;
7570 bool HadEvenDivisor = false;
7571 bool AllDivisorsArePowerOfTwo = true;
7572 bool HadTautologicalInvertedLanes = false;
7573 SmallVector<SDValue, 16> PAmts, KAmts, QAmts;
7574
7575 auto BuildUREMPattern = [&](ConstantSDNode *CDiv, ConstantSDNode *CCmp) {
7576 // Division by 0 is UB. Leave it to be constant-folded elsewhere.
7577 if (CDiv->isZero())
7578 return false;
7579
7580 const APInt &D = CDiv->getAPIntValue();
7581 const APInt &Cmp = CCmp->getAPIntValue();
7582
7583 ComparingWithAllZeros &= Cmp.isZero();
7584
7585 // x u% C1` is *always* less than C1. So given `x u% C1 == C2`,
7586 // if C2 is not less than C1, the comparison is always false.
7587 // But we will only be able to produce the comparison that will give the
7588 // opposive tautological answer. So this lane would need to be fixed up.
7589 bool TautologicalInvertedLane = D.ule(Cmp);
7590 HadTautologicalInvertedLanes |= TautologicalInvertedLane;
7591
7592 // If all lanes are tautological (either all divisors are ones, or divisor
7593 // is not greater than the constant we are comparing with),
7594 // we will prefer to avoid the fold.
7595 bool TautologicalLane = D.isOne() || TautologicalInvertedLane;
7596 HadTautologicalLanes |= TautologicalLane;
7597 AllLanesAreTautological &= TautologicalLane;
7598
7599 // If we are comparing with non-zero, we need'll need to subtract said
7600 // comparison value from the LHS. But there is no point in doing that if
7601 // every lane where we are comparing with non-zero is tautological..
7602 if (!Cmp.isZero())
7603 AllComparisonsWithNonZerosAreTautological &= TautologicalLane;
7604
7605 // Decompose D into D0 * 2^K
7606 unsigned K = D.countr_zero();
7607 assert((!D.isOne() || (K == 0)) && "For divisor '1' we won't rotate.");
7608 APInt D0 = D.lshr(K);
7609
7610 // D is even if it has trailing zeros.
7611 HadEvenDivisor |= (K != 0);
7612 // D is a power-of-two if D0 is one.
7613 // If all divisors are power-of-two, we will prefer to avoid the fold.
7614 AllDivisorsArePowerOfTwo &= D0.isOne();
7615
7616 // P = inv(D0, 2^W)
7617 // 2^W requires W + 1 bits, so we have to extend and then truncate.
7618 unsigned W = D.getBitWidth();
7619 APInt P = D0.multiplicativeInverse();
7620 assert((D0 * P).isOne() && "Multiplicative inverse basic check failed.");
7621
7622 // Q = floor((2^W - 1) u/ D)
7623 // R = ((2^W - 1) u% D)
7624 APInt Q, R;
7626
7627 // If we are comparing with zero, then that comparison constant is okay,
7628 // else it may need to be one less than that.
7629 if (Cmp.ugt(R))
7630 Q -= 1;
7631
7633 "We are expecting that K is always less than all-ones for ShSVT");
7634
7635 // If the lane is tautological the result can be constant-folded.
7636 if (TautologicalLane) {
7637 // Set P and K amount to a bogus values so we can try to splat them.
7638 P = 0;
7639 KAmts.push_back(DAG.getAllOnesConstant(DL, ShSVT));
7640 // And ensure that comparison constant is tautological,
7641 // it will always compare true/false.
7642 Q.setAllBits();
7643 } else {
7644 KAmts.push_back(DAG.getConstant(K, DL, ShSVT));
7645 }
7646
7647 PAmts.push_back(DAG.getConstant(P, DL, SVT));
7648 QAmts.push_back(DAG.getConstant(Q, DL, SVT));
7649 return true;
7650 };
7651
7652 SDValue N = REMNode.getOperand(0);
7653 SDValue D = REMNode.getOperand(1);
7654
7655 // Collect the values from each element.
7656 if (!ISD::matchBinaryPredicate(D, CompTargetNode, BuildUREMPattern))
7657 return SDValue();
7658
7659 // If all lanes are tautological, the result can be constant-folded.
7660 if (AllLanesAreTautological)
7661 return SDValue();
7662
7663 // If this is a urem by a powers-of-two, avoid the fold since it can be
7664 // best implemented as a bit test.
7665 if (AllDivisorsArePowerOfTwo)
7666 return SDValue();
7667
7668 SDValue PVal, KVal, QVal;
7669 if (D.getOpcode() == ISD::BUILD_VECTOR) {
7670 if (HadTautologicalLanes) {
7671 // Try to turn PAmts into a splat, since we don't care about the values
7672 // that are currently '0'. If we can't, just keep '0'`s.
7674 // Try to turn KAmts into a splat, since we don't care about the values
7675 // that are currently '-1'. If we can't, change them to '0'`s.
7677 DAG.getConstant(0, DL, ShSVT));
7678 }
7679
7680 PVal = DAG.getBuildVector(VT, DL, PAmts);
7681 KVal = DAG.getBuildVector(ShVT, DL, KAmts);
7682 QVal = DAG.getBuildVector(VT, DL, QAmts);
7683 } else if (D.getOpcode() == ISD::SPLAT_VECTOR) {
7684 assert(PAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 &&
7685 "Expected matchBinaryPredicate to return one element for "
7686 "SPLAT_VECTORs");
7687 PVal = DAG.getSplatVector(VT, DL, PAmts[0]);
7688 KVal = DAG.getSplatVector(ShVT, DL, KAmts[0]);
7689 QVal = DAG.getSplatVector(VT, DL, QAmts[0]);
7690 } else {
7691 PVal = PAmts[0];
7692 KVal = KAmts[0];
7693 QVal = QAmts[0];
7694 }
7695
7696 if (!ComparingWithAllZeros && !AllComparisonsWithNonZerosAreTautological) {
7697 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::SUB, VT))
7698 return SDValue(); // FIXME: Could/should use `ISD::ADD`?
7699 assert(CompTargetNode.getValueType() == N.getValueType() &&
7700 "Expecting that the types on LHS and RHS of comparisons match.");
7701 N = DAG.getNode(ISD::SUB, DL, VT, N, CompTargetNode);
7702 }
7703
7704 // (mul N, P)
7705 SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal);
7706 Created.push_back(Op0.getNode());
7707
7708 // Rotate right only if any divisor was even. We avoid rotates for all-odd
7709 // divisors as a performance improvement, since rotating by 0 is a no-op.
7710 if (HadEvenDivisor) {
7711 // We need ROTR to do this.
7712 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ROTR, VT))
7713 return SDValue();
7714 // UREM: (rotr (mul N, P), K)
7715 Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal);
7716 Created.push_back(Op0.getNode());
7717 }
7718
7719 // UREM: (setule/setugt (rotr (mul N, P), K), Q)
7720 SDValue NewCC =
7721 DAG.getSetCC(DL, SETCCVT, Op0, QVal,
7723 if (!HadTautologicalInvertedLanes)
7724 return NewCC;
7725
7726 // If any lanes previously compared always-false, the NewCC will give
7727 // always-true result for them, so we need to fixup those lanes.
7728 // Or the other way around for inequality predicate.
7729 assert(VT.isVector() && "Can/should only get here for vectors.");
7730 Created.push_back(NewCC.getNode());
7731
7732 // x u% C1` is *always* less than C1. So given `x u% C1 == C2`,
7733 // if C2 is not less than C1, the comparison is always false.
7734 // But we have produced the comparison that will give the
7735 // opposive tautological answer. So these lanes would need to be fixed up.
7736 SDValue TautologicalInvertedChannels =
7737 DAG.getSetCC(DL, SETCCVT, D, CompTargetNode, ISD::SETULE);
7738 Created.push_back(TautologicalInvertedChannels.getNode());
7739
7740 // NOTE: we avoid letting illegal types through even if we're before legalize
7741 // ops – legalization has a hard time producing good code for this.
7742 if (isOperationLegalOrCustom(ISD::VSELECT, SETCCVT)) {
7743 // If we have a vector select, let's replace the comparison results in the
7744 // affected lanes with the correct tautological result.
7745 SDValue Replacement = DAG.getBoolConstant(Cond == ISD::SETEQ ? false : true,
7746 DL, SETCCVT, SETCCVT);
7747 return DAG.getNode(ISD::VSELECT, DL, SETCCVT, TautologicalInvertedChannels,
7748 Replacement, NewCC);
7749 }
7750
7751 // Else, we can just invert the comparison result in the appropriate lanes.
7752 //
7753 // NOTE: see the note above VSELECT above.
7754 if (isOperationLegalOrCustom(ISD::XOR, SETCCVT))
7755 return DAG.getNode(ISD::XOR, DL, SETCCVT, NewCC,
7756 TautologicalInvertedChannels);
7757
7758 return SDValue(); // Don't know how to lower.
7759}
7760
7761/// Given an ISD::SREM used only by an ISD::SETEQ or ISD::SETNE
7762/// where the divisor is constant and the comparison target is zero,
7763/// return a DAG expression that will generate the same comparison result
7764/// using only multiplications, additions and shifts/rotations.
7765/// Ref: "Hacker's Delight" 10-17.
7766SDValue TargetLowering::buildSREMEqFold(EVT SETCCVT, SDValue REMNode,
7767 SDValue CompTargetNode,
7769 DAGCombinerInfo &DCI,
7770 const SDLoc &DL) const {
7772 if (SDValue Folded = prepareSREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
7773 DCI, DL, Built)) {
7774 assert(Built.size() <= 7 && "Max size prediction failed.");
7775 for (SDNode *N : Built)
7776 DCI.AddToWorklist(N);
7777 return Folded;
7778 }
7779
7780 return SDValue();
7781}
7782
7783SDValue
7784TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode,
7785 SDValue CompTargetNode, ISD::CondCode Cond,
7786 DAGCombinerInfo &DCI, const SDLoc &DL,
7787 SmallVectorImpl<SDNode *> &Created) const {
7788 // Derived from Hacker's Delight, 2nd Edition, by Hank Warren. Section 10-17.
7789 // Fold:
7790 // (seteq/ne (srem N, D), 0)
7791 // To:
7792 // (setule/ugt (rotr (add (mul N, P), A), K), Q)
7793 //
7794 // - D must be constant, with D = D0 * 2^K where D0 is odd
7795 // - P is the multiplicative inverse of D0 modulo 2^W
7796 // - A = bitwiseand(floor((2^(W - 1) - 1) / D0), (-(2^k)))
7797 // - Q = floor((2 * A) / (2^K))
7798 // where W is the width of the common type of N and D.
7799 //
7800 // When D is a power of two (and thus D0 is 1), the normal
7801 // formula for A and Q don't apply, because the derivation
7802 // depends on D not dividing 2^(W-1), and thus theorem ZRS
7803 // does not apply. This specifically fails when N = INT_MIN.
7804 //
7805 // Instead, for power-of-two D, we use:
7806 // - A = 0
7807 // | -> No offset needed. We're effectively treating it the same as urem.
7808 // - Q = 2^(W-K) - 1
7809 // |-> Test that the top K bits are zero after rotation
7810 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
7811 "Only applicable for (in)equality comparisons.");
7812
7813 SelectionDAG &DAG = DCI.DAG;
7814
7815 EVT VT = REMNode.getValueType();
7816 EVT SVT = VT.getScalarType();
7817 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
7818 EVT ShSVT = ShVT.getScalarType();
7819
7820 // If we are after ops legalization, and MUL is unavailable, we can not
7821 // proceed.
7822 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT))
7823 return SDValue();
7824
7825 // TODO: Could support comparing with non-zero too.
7826 ConstantSDNode *CompTarget = isConstOrConstSplat(CompTargetNode);
7827 if (!CompTarget || !CompTarget->isZero())
7828 return SDValue();
7829
7830 bool HadOneDivisor = false;
7831 bool AllDivisorsAreOnes = true;
7832 bool HadEvenDivisor = false;
7833 bool AllDivisorsArePowerOfTwo = true;
7834 SmallVector<SDValue, 16> PAmts, AAmts, KAmts, QAmts;
7835
7836 auto BuildSREMPattern = [&](ConstantSDNode *C) {
7837 // Division by 0 is UB. Leave it to be constant-folded elsewhere.
7838 if (C->isZero())
7839 return false;
7840
7841 // FIXME: we don't fold `rem %X, -C` to `rem %X, C` in DAGCombine.
7842
7843 // WARNING: this fold is only valid for positive divisors!
7844 // `rem %X, -C` is equivalent to `rem %X, C`
7845 APInt D = C->getAPIntValue().abs();
7846
7847 // If all divisors are ones, we will prefer to avoid the fold.
7848 HadOneDivisor |= D.isOne();
7849 AllDivisorsAreOnes &= D.isOne();
7850
7851 // Decompose D into D0 * 2^K
7852 unsigned K = D.countr_zero();
7853 assert((!D.isOne() || (K == 0)) && "For divisor '1' we won't rotate.");
7854 APInt D0 = D.lshr(K);
7855
7856 // D is even if it has trailing zeros.
7857 HadEvenDivisor |= (K != 0);
7858
7859 // D is a power-of-two if D0 is one. This includes INT_MIN.
7860 // If all divisors are power-of-two, we will prefer to avoid the fold.
7861 AllDivisorsArePowerOfTwo &= D0.isOne();
7862
7863 // P = inv(D0, 2^W)
7864 // 2^W requires W + 1 bits, so we have to extend and then truncate.
7865 unsigned W = D.getBitWidth();
7866 APInt P = D0.multiplicativeInverse();
7867 assert((D0 * P).isOne() && "Multiplicative inverse basic check failed.");
7868
7869 // A = floor((2^(W - 1) - 1) / D0) & -2^K
7870 APInt A = APInt::getSignedMaxValue(W).udiv(D0);
7871 A.clearLowBits(K);
7872
7873 // Q = floor((2 * A) / (2^K))
7874 APInt Q = (2 * A).udiv(APInt::getOneBitSet(W, K));
7875
7877 "We are expecting that A is always less than all-ones for SVT");
7879 "We are expecting that K is always less than all-ones for ShSVT");
7880
7881 // If D was a power of two, apply the alternate constant derivation.
7882 if (D0.isOne()) {
7883 // A = 0
7884 A = APInt(W, 0);
7885 // - Q = 2^(W-K) - 1
7886 Q = APInt::getLowBitsSet(W, W - K);
7887 }
7888
7889 // If the divisor is 1 the result can be constant-folded.
7890 if (D.isOne()) {
7891 // Set P, A and K to a bogus values so we can try to splat them.
7892 P = 0;
7893 A.setAllBits();
7894 KAmts.push_back(DAG.getAllOnesConstant(DL, ShSVT));
7895
7896 // x ?% 1 == 0 <--> true <--> x u<= -1
7897 Q.setAllBits();
7898 } else {
7899 KAmts.push_back(DAG.getConstant(K, DL, ShSVT));
7900 }
7901
7902 PAmts.push_back(DAG.getConstant(P, DL, SVT));
7903 AAmts.push_back(DAG.getConstant(A, DL, SVT));
7904 QAmts.push_back(DAG.getConstant(Q, DL, SVT));
7905 return true;
7906 };
7907
7908 SDValue N = REMNode.getOperand(0);
7909 SDValue D = REMNode.getOperand(1);
7910
7911 // Collect the values from each element.
7912 if (!ISD::matchUnaryPredicate(D, BuildSREMPattern))
7913 return SDValue();
7914
7915 // If this is a srem by a one, avoid the fold since it can be constant-folded.
7916 if (AllDivisorsAreOnes)
7917 return SDValue();
7918
7919 // If this is a srem by a powers-of-two (including INT_MIN), avoid the fold
7920 // since it can be best implemented as a bit test.
7921 if (AllDivisorsArePowerOfTwo)
7922 return SDValue();
7923
7924 SDValue PVal, AVal, KVal, QVal;
7925 if (D.getOpcode() == ISD::BUILD_VECTOR) {
7926 if (HadOneDivisor) {
7927 // Try to turn PAmts into a splat, since we don't care about the values
7928 // that are currently '0'. If we can't, just keep '0'`s.
7930 // Try to turn AAmts into a splat, since we don't care about the
7931 // values that are currently '-1'. If we can't, change them to '0'`s.
7933 DAG.getConstant(0, DL, SVT));
7934 // Try to turn KAmts into a splat, since we don't care about the values
7935 // that are currently '-1'. If we can't, change them to '0'`s.
7937 DAG.getConstant(0, DL, ShSVT));
7938 }
7939
7940 PVal = DAG.getBuildVector(VT, DL, PAmts);
7941 AVal = DAG.getBuildVector(VT, DL, AAmts);
7942 KVal = DAG.getBuildVector(ShVT, DL, KAmts);
7943 QVal = DAG.getBuildVector(VT, DL, QAmts);
7944 } else if (D.getOpcode() == ISD::SPLAT_VECTOR) {
7945 assert(PAmts.size() == 1 && AAmts.size() == 1 && KAmts.size() == 1 &&
7946 QAmts.size() == 1 &&
7947 "Expected matchUnaryPredicate to return one element for scalable "
7948 "vectors");
7949 PVal = DAG.getSplatVector(VT, DL, PAmts[0]);
7950 AVal = DAG.getSplatVector(VT, DL, AAmts[0]);
7951 KVal = DAG.getSplatVector(ShVT, DL, KAmts[0]);
7952 QVal = DAG.getSplatVector(VT, DL, QAmts[0]);
7953 } else {
7954 assert(isa<ConstantSDNode>(D) && "Expected a constant");
7955 PVal = PAmts[0];
7956 AVal = AAmts[0];
7957 KVal = KAmts[0];
7958 QVal = QAmts[0];
7959 }
7960
7961 // (mul N, P)
7962 SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal);
7963 Created.push_back(Op0.getNode());
7964
7965 // We need ADD to do this.
7966 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ADD, VT))
7967 return SDValue();
7968
7969 // (add (mul N, P), A)
7970 Op0 = DAG.getNode(ISD::ADD, DL, VT, Op0, AVal);
7971 Created.push_back(Op0.getNode());
7972
7973 // Rotate right only if any divisor was even. We avoid rotates for all-odd
7974 // divisors as a performance improvement, since rotating by 0 is a no-op.
7975 if (HadEvenDivisor) {
7976 // We need ROTR to do this.
7977 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ROTR, VT))
7978 return SDValue();
7979 // SREM: (rotr (add (mul N, P), A), K)
7980 Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal);
7981 Created.push_back(Op0.getNode());
7982 }
7983
7984 // SREM: (setule/setugt (rotr (add (mul N, P), A), K), Q)
7985 return DAG.getSetCC(DL, SETCCVT, Op0, QVal,
7987}
7988
7990 const DenormalMode &Mode,
7991 SDNodeFlags Flags) const {
7992 SDLoc DL(Op);
7993 EVT VT = Op.getValueType();
7994 EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
7995 SDValue FPZero = DAG.getConstantFP(0.0, DL, VT);
7996
7997 // This is specifically a check for the handling of denormal inputs, not the
7998 // result.
7999 if (Mode.Input == DenormalMode::PreserveSign ||
8000 Mode.Input == DenormalMode::PositiveZero) {
8001 // Test = X == 0.0
8002 return DAG.getSetCC(DL, CCVT, Op, FPZero, ISD::SETEQ, /*Chain=*/{},
8003 /*Signaling=*/false, Flags);
8004 }
8005
8006 // Testing it with denormal inputs to avoid wrong estimate.
8007 //
8008 // Test = fabs(X) < SmallestNormal
8009 const fltSemantics &FltSem = VT.getFltSemantics();
8010 APFloat SmallestNorm = APFloat::getSmallestNormalized(FltSem);
8011 SDValue NormC = DAG.getConstantFP(SmallestNorm, DL, VT);
8012 SDValue Fabs = DAG.getNode(ISD::FABS, DL, VT, Op, Flags);
8013 return DAG.getSetCC(DL, CCVT, Fabs, NormC, ISD::SETLT, /*Chain=*/{},
8014 /*Signaling=*/false, Flags);
8015}
8016
8018 bool LegalOps, bool OptForSize,
8020 unsigned Depth) const {
8021 // fneg is removable even if it has multiple uses.
8022 if (Op.getOpcode() == ISD::FNEG) {
8024 return Op.getOperand(0);
8025 }
8026
8027 // Don't recurse exponentially.
8029 return SDValue();
8030
8031 // Pre-increment recursion depth for use in recursive calls.
8032 ++Depth;
8033 const SDNodeFlags Flags = Op->getFlags();
8034 EVT VT = Op.getValueType();
8035 unsigned Opcode = Op.getOpcode();
8036
8037 // Don't allow anything with multiple uses unless we know it is free.
8038 if (!Op.hasOneUse() && Opcode != ISD::ConstantFP) {
8039 bool IsFreeExtend = Opcode == ISD::FP_EXTEND &&
8040 isFPExtFree(VT, Op.getOperand(0).getValueType());
8041 if (!IsFreeExtend)
8042 return SDValue();
8043 }
8044
8045 auto RemoveDeadNode = [&](SDValue N) {
8046 if (N && N.getNode()->use_empty())
8047 DAG.RemoveDeadNode(N.getNode());
8048 };
8049
8050 SDLoc DL(Op);
8051
8052 // Because getNegatedExpression can delete nodes we need a handle to keep
8053 // temporary nodes alive in case the recursion manages to create an identical
8054 // node.
8055 std::list<HandleSDNode> Handles;
8056
8057 switch (Opcode) {
8058 case ISD::ConstantFP: {
8059 // Don't invert constant FP values after legalization unless the target says
8060 // the negated constant is legal.
8061 bool IsOpLegal =
8063 isFPImmLegal(neg(cast<ConstantFPSDNode>(Op)->getValueAPF()), VT,
8064 OptForSize);
8065
8066 if (LegalOps && !IsOpLegal)
8067 break;
8068
8069 APFloat V = cast<ConstantFPSDNode>(Op)->getValueAPF();
8070 V.changeSign();
8071 SDValue CFP = DAG.getConstantFP(V, DL, VT);
8072
8073 // If we already have the use of the negated floating constant, it is free
8074 // to negate it even it has multiple uses.
8075 if (!Op.hasOneUse() && CFP.use_empty())
8076 break;
8078 return CFP;
8079 }
8080 case ISD::SPLAT_VECTOR: {
8081 // fold splat_vector(fneg(X)) -> splat_vector(-X)
8082 SDValue X = Op.getOperand(0);
8084 break;
8085
8086 SDValue NegX = getCheaperNegatedExpression(X, DAG, LegalOps, OptForSize);
8087 if (!NegX)
8088 break;
8090 return DAG.getNode(ISD::SPLAT_VECTOR, DL, VT, NegX);
8091 }
8092 case ISD::BUILD_VECTOR: {
8093 // Only permit BUILD_VECTOR of constants.
8094 if (llvm::any_of(Op->op_values(), [&](SDValue N) {
8095 return !N.isUndef() && !isa<ConstantFPSDNode>(N);
8096 }))
8097 break;
8098
8099 bool IsOpLegal =
8102 llvm::all_of(Op->op_values(), [&](SDValue N) {
8103 return N.isUndef() ||
8104 isFPImmLegal(neg(cast<ConstantFPSDNode>(N)->getValueAPF()), VT,
8105 OptForSize);
8106 });
8107
8108 if (LegalOps && !IsOpLegal)
8109 break;
8110
8112 for (SDValue C : Op->op_values()) {
8113 if (C.isUndef()) {
8114 Ops.push_back(C);
8115 continue;
8116 }
8117 APFloat V = cast<ConstantFPSDNode>(C)->getValueAPF();
8118 V.changeSign();
8119 Ops.push_back(DAG.getConstantFP(V, DL, C.getValueType()));
8120 }
8122 return DAG.getBuildVector(VT, DL, Ops);
8123 }
8124 case ISD::FADD: {
8125 if (!Flags.hasNoSignedZeros())
8126 break;
8127
8128 // After operation legalization, it might not be legal to create new FSUBs.
8129 if (LegalOps && !isOperationLegalOrCustom(ISD::FSUB, VT))
8130 break;
8131 SDValue X = Op.getOperand(0), Y = Op.getOperand(1);
8132
8133 // fold (fneg (fadd X, Y)) -> (fsub (fneg X), Y)
8135 SDValue NegX =
8136 getNegatedExpression(X, DAG, LegalOps, OptForSize, CostX, Depth);
8137 // Prevent this node from being deleted by the next call.
8138 if (NegX)
8139 Handles.emplace_back(NegX);
8140
8141 // fold (fneg (fadd X, Y)) -> (fsub (fneg Y), X)
8143 SDValue NegY =
8144 getNegatedExpression(Y, DAG, LegalOps, OptForSize, CostY, Depth);
8145
8146 // We're done with the handles.
8147 Handles.clear();
8148
8149 // Negate the X if its cost is less or equal than Y.
8150 if (NegX && (CostX <= CostY)) {
8151 Cost = CostX;
8152 SDValue N = DAG.getNode(ISD::FSUB, DL, VT, NegX, Y, Flags);
8153 if (NegY != N)
8154 RemoveDeadNode(NegY);
8155 return N;
8156 }
8157
8158 // Negate the Y if it is not expensive.
8159 if (NegY) {
8160 Cost = CostY;
8161 SDValue N = DAG.getNode(ISD::FSUB, DL, VT, NegY, X, Flags);
8162 if (NegX != N)
8163 RemoveDeadNode(NegX);
8164 return N;
8165 }
8166 break;
8167 }
8168 case ISD::FSUB: {
8169 // We can't turn -(A-B) into B-A when we honor signed zeros.
8170 if (!Flags.hasNoSignedZeros())
8171 break;
8172
8173 SDValue X = Op.getOperand(0), Y = Op.getOperand(1);
8174 // fold (fneg (fsub 0, Y)) -> Y
8175 if (ConstantFPSDNode *C = isConstOrConstSplatFP(X, /*AllowUndefs*/ true))
8176 if (C->isZero()) {
8178 return Y;
8179 }
8180
8181 // fold (fneg (fsub X, Y)) -> (fsub Y, X)
8183 return DAG.getNode(ISD::FSUB, DL, VT, Y, X, Flags);
8184 }
8185 case ISD::FMUL:
8186 case ISD::FDIV: {
8187 SDValue X = Op.getOperand(0), Y = Op.getOperand(1);
8188
8189 // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y)
8191 SDValue NegX =
8192 getNegatedExpression(X, DAG, LegalOps, OptForSize, CostX, Depth);
8193 // Prevent this node from being deleted by the next call.
8194 if (NegX)
8195 Handles.emplace_back(NegX);
8196
8197 // fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y))
8199 SDValue NegY =
8200 getNegatedExpression(Y, DAG, LegalOps, OptForSize, CostY, Depth);
8201
8202 // We're done with the handles.
8203 Handles.clear();
8204
8205 // Negate the X if its cost is less or equal than Y.
8206 if (NegX && (CostX <= CostY)) {
8207 Cost = CostX;
8208 SDValue N = DAG.getNode(Opcode, DL, VT, NegX, Y, Flags);
8209 if (NegY != N)
8210 RemoveDeadNode(NegY);
8211 return N;
8212 }
8213
8214 // Ignore X * 2.0 because that is expected to be canonicalized to X + X.
8215 if (auto *C = isConstOrConstSplatFP(Op.getOperand(1)))
8216 if (C->isExactlyValue(2.0) && Op.getOpcode() == ISD::FMUL)
8217 break;
8218
8219 // Negate the Y if it is not expensive.
8220 if (NegY) {
8221 Cost = CostY;
8222 SDValue N = DAG.getNode(Opcode, DL, VT, X, NegY, Flags);
8223 if (NegX != N)
8224 RemoveDeadNode(NegX);
8225 return N;
8226 }
8227 break;
8228 }
8229 case ISD::FMA:
8230 case ISD::FMULADD:
8231 case ISD::FMAD: {
8232 if (!Flags.hasNoSignedZeros())
8233 break;
8234
8235 SDValue X = Op.getOperand(0), Y = Op.getOperand(1), Z = Op.getOperand(2);
8237 SDValue NegZ =
8238 getNegatedExpression(Z, DAG, LegalOps, OptForSize, CostZ, Depth);
8239 // Give up if fail to negate the Z.
8240 if (!NegZ)
8241 break;
8242
8243 // Prevent this node from being deleted by the next two calls.
8244 Handles.emplace_back(NegZ);
8245
8246 // fold (fneg (fma X, Y, Z)) -> (fma (fneg X), Y, (fneg Z))
8248 SDValue NegX =
8249 getNegatedExpression(X, DAG, LegalOps, OptForSize, CostX, Depth);
8250 // Prevent this node from being deleted by the next call.
8251 if (NegX)
8252 Handles.emplace_back(NegX);
8253
8254 // fold (fneg (fma X, Y, Z)) -> (fma X, (fneg Y), (fneg Z))
8256 SDValue NegY =
8257 getNegatedExpression(Y, DAG, LegalOps, OptForSize, CostY, Depth);
8258
8259 // We're done with the handles.
8260 Handles.clear();
8261
8262 // Negate the X if its cost is less or equal than Y.
8263 if (NegX && (CostX <= CostY)) {
8264 Cost = std::min(CostX, CostZ);
8265 SDValue N = DAG.getNode(Opcode, DL, VT, NegX, Y, NegZ, Flags);
8266 if (NegY != N)
8267 RemoveDeadNode(NegY);
8268 return N;
8269 }
8270
8271 // Negate the Y if it is not expensive.
8272 if (NegY) {
8273 Cost = std::min(CostY, CostZ);
8274 SDValue N = DAG.getNode(Opcode, DL, VT, X, NegY, NegZ, Flags);
8275 if (NegX != N)
8276 RemoveDeadNode(NegX);
8277 return N;
8278 }
8279 break;
8280 }
8281
8282 case ISD::FP_EXTEND:
8283 case ISD::FSIN:
8284 if (SDValue NegV = getNegatedExpression(Op.getOperand(0), DAG, LegalOps,
8285 OptForSize, Cost, Depth))
8286 return DAG.getNode(Opcode, DL, VT, NegV);
8287 break;
8288 case ISD::FP_ROUND:
8289 if (SDValue NegV = getNegatedExpression(Op.getOperand(0), DAG, LegalOps,
8290 OptForSize, Cost, Depth))
8291 return DAG.getNode(ISD::FP_ROUND, DL, VT, NegV, Op.getOperand(1));
8292 break;
8293 case ISD::SELECT:
8294 case ISD::VSELECT: {
8295 // fold (fneg (select C, LHS, RHS)) -> (select C, (fneg LHS), (fneg RHS))
8296 // iff at least one cost is cheaper and the other is neutral/cheaper
8297 SDValue LHS = Op.getOperand(1);
8299 SDValue NegLHS =
8300 getNegatedExpression(LHS, DAG, LegalOps, OptForSize, CostLHS, Depth);
8301 if (!NegLHS || CostLHS > NegatibleCost::Neutral) {
8302 RemoveDeadNode(NegLHS);
8303 break;
8304 }
8305
8306 // Prevent this node from being deleted by the next call.
8307 Handles.emplace_back(NegLHS);
8308
8309 SDValue RHS = Op.getOperand(2);
8311 SDValue NegRHS =
8312 getNegatedExpression(RHS, DAG, LegalOps, OptForSize, CostRHS, Depth);
8313
8314 // We're done with the handles.
8315 Handles.clear();
8316
8317 if (!NegRHS || CostRHS > NegatibleCost::Neutral ||
8318 (CostLHS != NegatibleCost::Cheaper &&
8319 CostRHS != NegatibleCost::Cheaper)) {
8320 RemoveDeadNode(NegLHS);
8321 RemoveDeadNode(NegRHS);
8322 break;
8323 }
8324
8325 Cost = std::min(CostLHS, CostRHS);
8326 return DAG.getSelect(DL, VT, Op.getOperand(0), NegLHS, NegRHS);
8327 }
8328 }
8329
8330 return SDValue();
8331}
8332
8333//===----------------------------------------------------------------------===//
8334// Legalization Utilities
8335//===----------------------------------------------------------------------===//
8336
8337bool TargetLowering::expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl,
8338 SDValue LHS, SDValue RHS,
8340 EVT HiLoVT, SelectionDAG &DAG,
8341 MulExpansionKind Kind, SDValue LL,
8342 SDValue LH, SDValue RL, SDValue RH) const {
8343 assert(Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI ||
8344 Opcode == ISD::SMUL_LOHI);
8345
8346 bool HasMULHS = (Kind == MulExpansionKind::Always) ||
8348 bool HasMULHU = (Kind == MulExpansionKind::Always) ||
8350 bool HasSMUL_LOHI = (Kind == MulExpansionKind::Always) ||
8352 bool HasUMUL_LOHI = (Kind == MulExpansionKind::Always) ||
8354
8355 if (!HasMULHU && !HasMULHS && !HasUMUL_LOHI && !HasSMUL_LOHI)
8356 return false;
8357
8358 unsigned OuterBitSize = VT.getScalarSizeInBits();
8359 unsigned InnerBitSize = HiLoVT.getScalarSizeInBits();
8360
8361 // LL, LH, RL, and RH must be either all NULL or all set to a value.
8362 assert((LL.getNode() && LH.getNode() && RL.getNode() && RH.getNode()) ||
8363 (!LL.getNode() && !LH.getNode() && !RL.getNode() && !RH.getNode()));
8364
8365 auto MakeMUL_LOHI = [&](SDValue L, SDValue R, SDValue &Lo, SDValue &Hi,
8366 bool Signed) -> bool {
8367 if ((Signed && HasSMUL_LOHI) || (!Signed && HasUMUL_LOHI)) {
8368 SDVTList VTs = DAG.getVTList(HiLoVT, HiLoVT);
8369 Lo = DAG.getNode(Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI, dl, VTs, L, R);
8370 Hi = Lo.getValue(1);
8371 return true;
8372 }
8373 if ((Signed && HasMULHS) || (!Signed && HasMULHU)) {
8374 Lo = DAG.getNode(ISD::MUL, dl, HiLoVT, L, R);
8375 Hi = DAG.getNode(Signed ? ISD::MULHS : ISD::MULHU, dl, HiLoVT, L, R);
8376 return true;
8377 }
8378 return false;
8379 };
8380
8381 SDValue Lo, Hi;
8382
8383 if (!LL.getNode() && !RL.getNode() &&
8385 LL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LHS);
8386 RL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RHS);
8387 }
8388
8389 if (!LL.getNode())
8390 return false;
8391
8392 APInt HighMask = APInt::getHighBitsSet(OuterBitSize, InnerBitSize);
8393 if (DAG.MaskedValueIsZero(LHS, HighMask) &&
8394 DAG.MaskedValueIsZero(RHS, HighMask)) {
8395 // The inputs are both zero-extended.
8396 if (MakeMUL_LOHI(LL, RL, Lo, Hi, false)) {
8397 Result.push_back(Lo);
8398 Result.push_back(Hi);
8399 if (Opcode != ISD::MUL) {
8400 SDValue Zero = DAG.getConstant(0, dl, HiLoVT);
8401 Result.push_back(Zero);
8402 Result.push_back(Zero);
8403 }
8404 return true;
8405 }
8406 }
8407
8408 if (!VT.isVector() && Opcode == ISD::MUL &&
8409 DAG.ComputeMaxSignificantBits(LHS) <= InnerBitSize &&
8410 DAG.ComputeMaxSignificantBits(RHS) <= InnerBitSize) {
8411 // The input values are both sign-extended.
8412 // TODO non-MUL case?
8413 if (MakeMUL_LOHI(LL, RL, Lo, Hi, true)) {
8414 Result.push_back(Lo);
8415 Result.push_back(Hi);
8416 return true;
8417 }
8418 }
8419
8420 unsigned ShiftAmount = OuterBitSize - InnerBitSize;
8421 SDValue Shift = DAG.getShiftAmountConstant(ShiftAmount, VT, dl);
8422
8423 if (!LH.getNode() && !RH.getNode() &&
8426 LH = DAG.getNode(ISD::SRL, dl, VT, LHS, Shift);
8427 LH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LH);
8428 RH = DAG.getNode(ISD::SRL, dl, VT, RHS, Shift);
8429 RH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RH);
8430 }
8431
8432 if (!LH.getNode())
8433 return false;
8434
8435 if (!MakeMUL_LOHI(LL, RL, Lo, Hi, false))
8436 return false;
8437
8438 Result.push_back(Lo);
8439
8440 if (Opcode == ISD::MUL) {
8441 RH = DAG.getNode(ISD::MUL, dl, HiLoVT, LL, RH);
8442 LH = DAG.getNode(ISD::MUL, dl, HiLoVT, LH, RL);
8443 Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, RH);
8444 Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, LH);
8445 Result.push_back(Hi);
8446 return true;
8447 }
8448
8449 // Compute the full width result.
8450 auto Merge = [&](SDValue Lo, SDValue Hi) -> SDValue {
8451 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
8452 Hi = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi);
8453 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
8454 return DAG.getNode(ISD::OR, dl, VT, Lo, Hi);
8455 };
8456
8457 SDValue Next = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi);
8458 if (!MakeMUL_LOHI(LL, RH, Lo, Hi, false))
8459 return false;
8460
8461 // This is effectively the add part of a multiply-add of half-sized operands,
8462 // so it cannot overflow.
8463 Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi));
8464
8465 if (!MakeMUL_LOHI(LH, RL, Lo, Hi, false))
8466 return false;
8467
8468 SDValue Zero = DAG.getConstant(0, dl, HiLoVT);
8469 EVT BoolType = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
8470
8471 bool UseGlue = (isOperationLegalOrCustom(ISD::ADDC, VT) &&
8473 if (UseGlue)
8474 Next = DAG.getNode(ISD::ADDC, dl, DAG.getVTList(VT, MVT::Glue), Next,
8475 Merge(Lo, Hi));
8476 else
8477 Next = DAG.getNode(ISD::UADDO_CARRY, dl, DAG.getVTList(VT, BoolType), Next,
8478 Merge(Lo, Hi), DAG.getConstant(0, dl, BoolType));
8479
8480 SDValue Carry = Next.getValue(1);
8481 Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
8482 Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift);
8483
8484 if (!MakeMUL_LOHI(LH, RH, Lo, Hi, Opcode == ISD::SMUL_LOHI))
8485 return false;
8486
8487 if (UseGlue)
8488 Hi = DAG.getNode(ISD::ADDE, dl, DAG.getVTList(HiLoVT, MVT::Glue), Hi, Zero,
8489 Carry);
8490 else
8491 Hi = DAG.getNode(ISD::UADDO_CARRY, dl, DAG.getVTList(HiLoVT, BoolType), Hi,
8492 Zero, Carry);
8493
8494 Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi));
8495
8496 if (Opcode == ISD::SMUL_LOHI) {
8497 SDValue NextSub = DAG.getNode(ISD::SUB, dl, VT, Next,
8498 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RL));
8499 Next = DAG.getSelectCC(dl, LH, Zero, NextSub, Next, ISD::SETLT);
8500
8501 NextSub = DAG.getNode(ISD::SUB, dl, VT, Next,
8502 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LL));
8503 Next = DAG.getSelectCC(dl, RH, Zero, NextSub, Next, ISD::SETLT);
8504 }
8505
8506 Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
8507 Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift);
8508 Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
8509 return true;
8510}
8511
8513 SelectionDAG &DAG, MulExpansionKind Kind,
8514 SDValue LL, SDValue LH, SDValue RL,
8515 SDValue RH) const {
8517 bool Ok = expandMUL_LOHI(N->getOpcode(), N->getValueType(0), SDLoc(N),
8518 N->getOperand(0), N->getOperand(1), Result, HiLoVT,
8519 DAG, Kind, LL, LH, RL, RH);
8520 if (Ok) {
8521 assert(Result.size() == 2);
8522 Lo = Result[0];
8523 Hi = Result[1];
8524 }
8525 return Ok;
8526}
8527
8528// Optimize unsigned division or remainder by constants for types twice as large
8529// as a legal VT.
8530//
8531// If (1 << (BitWidth / 2)) % Constant == 1, then the remainder
8532// can be computed
8533// as:
8534// Sum = __builtin_uadd_overflow(Lo, High, &Sum);
8535// Remainder = Sum % Constant;
8536//
8537// If (1 << (BitWidth / 2)) % Constant != 1, we can search for a smaller value
8538// W such that W != (BitWidth / 2) and (1 << W) % Constant == 1. We can break
8539// High:Low into 3 chunks of W bits and compute remainder as
8540// Sum = Chunk0 + Chunk1 + Chunk2;
8541// Remainder = Sum % Constant;
8542//
8543// This is based on "Remainder by Summing Digits" from Hacker's Delight.
8544//
8545// For division, we can compute the remainder using the algorithm described
8546// above, subtract it from the dividend to get an exact multiple of Constant.
8547// Then multiply that exact multiply by the multiplicative inverse modulo
8548// (1 << (BitWidth / 2)) to get the quotient.
8549
8550// If Constant is even, we can shift right the dividend and the divisor by the
8551// number of trailing zeros in Constant before applying the remainder algorithm.
8552// If we're after the quotient, we can subtract this value from the shifted
8553// dividend and multiply by the multiplicative inverse of the shifted divisor.
8554// If we want the remainder, we shift the value left by the number of trailing
8555// zeros and add the bits that were shifted out of the dividend.
8556bool TargetLowering::expandUDIVREMByConstantViaUREMDecomposition(
8557 SDNode *N, APInt Divisor, SmallVectorImpl<SDValue> &Result, EVT HiLoVT,
8558 SelectionDAG &DAG, SDValue LL, SDValue LH) const {
8559 unsigned Opcode = N->getOpcode();
8560 EVT VT = N->getValueType(0);
8561
8562 unsigned BitWidth = Divisor.getBitWidth();
8563 unsigned HBitWidth = BitWidth / 2;
8565 HiLoVT.getScalarSizeInBits() == HBitWidth && "Unexpected VTs");
8566
8567 // If the divisor is even, shift it until it becomes odd.
8568 unsigned TrailingZeros = 0;
8569 if (!Divisor[0]) {
8570 TrailingZeros = Divisor.countr_zero();
8571 Divisor.lshrInPlace(TrailingZeros);
8572 }
8573
8574 // After removing trailing zeros, the divisor needs to be less than
8575 // (1 << HBitWidth).
8576 APInt HalfMaxPlus1 = APInt::getOneBitSet(BitWidth, HBitWidth);
8577 if (Divisor.uge(HalfMaxPlus1))
8578 return false;
8579
8580 // Look for the largest chunk width W such that (1 << W) % Divisor == 1 or
8581 // (1 << W) % Divisor == -1.
8582 unsigned BestChunkWidth = 0, AltChunkWidth = 0;
8583 for (unsigned I = HBitWidth, E = HBitWidth / 2; I > E; --I) {
8584 // Skip HBitWidth-1, it doesn't have enough bits for carries.
8585 if (I == HBitWidth - 1)
8586 continue;
8587
8588 APInt Mod = APInt::getOneBitSet(Divisor.getBitWidth(), I).urem(Divisor);
8589
8590 if (Mod.isOne()) {
8591 BestChunkWidth = I;
8592 break;
8593 }
8594
8595 // We have an alternate strategy for Remainder == Divisor - 1.
8596 // FIXME: Support HBitWidth.
8597 if (I != HBitWidth && Mod == Divisor - 1)
8598 AltChunkWidth = I;
8599 }
8600
8601 bool Alternate = false;
8602 if (!BestChunkWidth) {
8603 if (!AltChunkWidth)
8604 return false;
8605 Alternate = true;
8606 BestChunkWidth = AltChunkWidth;
8607 }
8608
8609 SDLoc dl(N);
8610
8611 assert(!LL == !LH && "Expected both input halves or no input halves!");
8612 if (!LL)
8613 std::tie(LL, LH) = DAG.SplitScalar(N->getOperand(0), dl, HiLoVT, HiLoVT);
8614
8615 bool HasFSHR = isOperationLegal(ISD::FSHR, HiLoVT);
8616
8617 auto GetFSHR = [&](SDValue Lo, SDValue Hi, unsigned ShiftAmt) {
8618 assert(ShiftAmt > 0 && ShiftAmt < HBitWidth);
8619 if (HasFSHR)
8620 return DAG.getNode(ISD::FSHR, dl, HiLoVT, Hi, Lo,
8621 DAG.getShiftAmountConstant(ShiftAmt, HiLoVT, dl));
8622 return DAG.getNode(
8623 ISD::OR, dl, HiLoVT,
8624 DAG.getNode(ISD::SRL, dl, HiLoVT, Lo,
8625 DAG.getShiftAmountConstant(ShiftAmt, HiLoVT, dl)),
8626 DAG.getNode(
8627 ISD::SHL, dl, HiLoVT, Hi,
8628 DAG.getShiftAmountConstant(HBitWidth - ShiftAmt, HiLoVT, dl)));
8629 };
8630
8631 // Helper to perform a right shift on a 128-bit value split into two halves.
8632 // Handles shifts >= HBitWidth by moving Hi to Lo and shifting Hi.
8633 auto ShiftRight = [&](SDValue &Lo, SDValue &Hi, unsigned ShiftAmt) {
8634 if (ShiftAmt == 0)
8635 return;
8636 if (ShiftAmt < HBitWidth) {
8637 Lo = GetFSHR(Lo, Hi, ShiftAmt);
8638 Hi = DAG.getNode(ISD::SRL, dl, HiLoVT, Hi,
8639 DAG.getShiftAmountConstant(ShiftAmt, HiLoVT, dl));
8640 } else if (ShiftAmt == HBitWidth) {
8641 Lo = Hi;
8642 Hi = DAG.getConstant(0, dl, HiLoVT);
8643 } else {
8644 Lo = DAG.getNode(
8645 ISD::SRL, dl, HiLoVT, Hi,
8646 DAG.getShiftAmountConstant(ShiftAmt - HBitWidth, HiLoVT, dl));
8647 Hi = DAG.getConstant(0, dl, HiLoVT);
8648 }
8649 };
8650
8651 // Shift the input by the number of TrailingZeros in the divisor. The
8652 // shifted out bits will be added to the remainder later.
8653 SDValue PartialRemL, PartialRemH;
8654 if (TrailingZeros && Opcode != ISD::UDIV) {
8655 // Save the shifted off bits if we need the remainder.
8656 if (TrailingZeros < HBitWidth) {
8657 APInt Mask = APInt::getLowBitsSet(HBitWidth, TrailingZeros);
8658 PartialRemL = DAG.getNode(ISD::AND, dl, HiLoVT, LL,
8659 DAG.getConstant(Mask, dl, HiLoVT));
8660 } else if (TrailingZeros == HBitWidth) {
8661 // All of LL is part of the remainder.
8662 PartialRemL = LL;
8663 } else {
8664 // TrailingZeros > HBitWidth: LL and part of LH are the remainder.
8665 PartialRemL = LL;
8666 APInt Mask = APInt::getLowBitsSet(HBitWidth, TrailingZeros - HBitWidth);
8667 PartialRemH = DAG.getNode(ISD::AND, dl, HiLoVT, LH,
8668 DAG.getConstant(Mask, dl, HiLoVT));
8669 }
8670 }
8671
8672 SDValue Sum;
8673 // If BestChunkWidth is HBitWidth add low and high half. If there is a carry
8674 // out, add that to the final sum.
8675 if (BestChunkWidth == HBitWidth) {
8676 assert(!Alternate);
8677 // Shift LH:LL right if there were trailing zeros in the divisor.
8678 ShiftRight(LL, LH, TrailingZeros);
8679
8680 // Use uaddo_carry if we can, otherwise use a compare to detect overflow.
8681 EVT SetCCType =
8682 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), HiLoVT);
8684 SDVTList VTList = DAG.getVTList(HiLoVT, SetCCType);
8685 Sum = DAG.getNode(ISD::UADDO, dl, VTList, LL, LH);
8686 Sum = DAG.getNode(ISD::UADDO_CARRY, dl, VTList, Sum,
8687 DAG.getConstant(0, dl, HiLoVT), Sum.getValue(1));
8688 } else {
8689 Sum = DAG.getNode(ISD::ADD, dl, HiLoVT, LL, LH);
8690 SDValue Carry = DAG.getSetCC(dl, SetCCType, Sum, LL, ISD::SETULT);
8691 // If the boolean for the target is 0 or 1, we can add the setcc result
8692 // directly.
8693 if (getBooleanContents(HiLoVT) ==
8695 Carry = DAG.getZExtOrTrunc(Carry, dl, HiLoVT);
8696 else
8697 Carry = DAG.getSelect(dl, HiLoVT, Carry, DAG.getConstant(1, dl, HiLoVT),
8698 DAG.getConstant(0, dl, HiLoVT));
8699 Sum = DAG.getNode(ISD::ADD, dl, HiLoVT, Sum, Carry);
8700 }
8701 } else {
8702 // Otherwise split into multple chunks and add them together. We chose
8703 // BestChunkWidth so that the sum will not overflow.
8704 SDValue Mask = DAG.getConstant(
8705 APInt::getLowBitsSet(HBitWidth, BestChunkWidth), dl, HiLoVT);
8706
8707 for (unsigned I = 0; I < BitWidth - TrailingZeros; I += BestChunkWidth) {
8708 // If there were trailing zeros in the divisor, increase the shift amount.
8709 unsigned Shift = I + TrailingZeros;
8710 SDValue Chunk;
8711 if (Shift == 0)
8712 Chunk = LL;
8713 else if (Shift >= HBitWidth)
8714 Chunk = DAG.getNode(
8715 ISD::SRL, dl, HiLoVT, LH,
8716 DAG.getShiftAmountConstant(Shift - HBitWidth, HiLoVT, dl));
8717 else
8718 Chunk = GetFSHR(LL, LH, Shift);
8719 // If we're on the last chunk, we don't need an AND.
8720 if (I + BestChunkWidth < BitWidth - TrailingZeros)
8721 Chunk = DAG.getNode(ISD::AND, dl, HiLoVT, Chunk, Mask);
8722 if (!Sum) {
8723 Sum = Chunk;
8724 } else {
8725 // For Alternate, we need to subtract odd chunks.
8726 unsigned ChunkNum = I / BestChunkWidth;
8727 unsigned Opc = (Alternate && (ChunkNum % 2) != 0) ? ISD::SUB : ISD::ADD;
8728 Sum = DAG.getNode(Opc, dl, HiLoVT, Sum, Chunk);
8729 }
8730 }
8731
8732 // For Alternate, the sum may be negative, but we need a positive sum. We
8733 // can increase it by a multiple of the divisor to make it positive. For 3
8734 // chunks the largest negative value is -(2^BestChunkWidth - 1). For 4
8735 // chunks, it's 2*-(2^BestChunkWidth - 1). We know that 2^BestChunkWidth + 1
8736 // is a multiple of the divisor. Add that 1 or 2 times to make the sum
8737 // positive.
8738 if (Alternate) {
8739 unsigned NumChunks = divideCeil(BitWidth - TrailingZeros, BestChunkWidth);
8740 assert(NumChunks <= 4);
8741
8742 APInt Adjust = APInt::getOneBitSet(HBitWidth, BestChunkWidth);
8743 Adjust.setBit(0);
8744 // If there are 4 chunks, we need to adjust twice.
8745 if (NumChunks == 4)
8746 Adjust <<= 1;
8747 Sum = DAG.getNode(ISD::ADD, dl, HiLoVT, Sum,
8748 DAG.getConstant(Adjust, dl, HiLoVT));
8749 }
8750 }
8751
8752 // Perform a HiLoVT urem on the Sum using truncated divisor.
8753 SDValue RemL =
8754 DAG.getNode(ISD::UREM, dl, HiLoVT, Sum,
8755 DAG.getConstant(Divisor.trunc(HBitWidth), dl, HiLoVT));
8756 SDValue RemH = DAG.getConstant(0, dl, HiLoVT);
8757
8758 if (Opcode != ISD::UREM) {
8759 // If we didn't shift LH/LR earlier, do it now.
8760 if (BestChunkWidth != HBitWidth)
8761 ShiftRight(LL, LH, TrailingZeros);
8762
8763 // Subtract the remainder from the shifted dividend.
8764 SDValue Dividend = DAG.getNode(ISD::BUILD_PAIR, dl, VT, LL, LH);
8765 SDValue Rem = DAG.getNode(ISD::BUILD_PAIR, dl, VT, RemL, RemH);
8766
8767 Dividend = DAG.getNode(ISD::SUB, dl, VT, Dividend, Rem);
8768
8769 // Multiply by the multiplicative inverse of the divisor modulo
8770 // (1 << BitWidth).
8771 APInt MulFactor = Divisor.multiplicativeInverse();
8772
8773 SDValue Quotient = DAG.getNode(ISD::MUL, dl, VT, Dividend,
8774 DAG.getConstant(MulFactor, dl, VT));
8775
8776 // Split the quotient into low and high parts.
8777 SDValue QuotL, QuotH;
8778 std::tie(QuotL, QuotH) = DAG.SplitScalar(Quotient, dl, HiLoVT, HiLoVT);
8779 Result.push_back(QuotL);
8780 Result.push_back(QuotH);
8781 }
8782
8783 if (Opcode != ISD::UDIV) {
8784 // If we shifted the input, shift the remainder left and add the bits we
8785 // shifted off the input.
8786 if (TrailingZeros) {
8787 if (TrailingZeros < HBitWidth) {
8788 // Shift RemH:RemL left by TrailingZeros.
8789 // RemH gets the high bits shifted out of RemL.
8790 RemH = DAG.getNode(
8791 ISD::SRL, dl, HiLoVT, RemL,
8792 DAG.getShiftAmountConstant(HBitWidth - TrailingZeros, HiLoVT, dl));
8793 RemL =
8794 DAG.getNode(ISD::SHL, dl, HiLoVT, RemL,
8795 DAG.getShiftAmountConstant(TrailingZeros, HiLoVT, dl));
8796 // OR in the partial remainder.
8797 RemL = DAG.getNode(ISD::OR, dl, HiLoVT, RemL, PartialRemL,
8799 } else if (TrailingZeros == HBitWidth) {
8800 // Shift left by exactly HBitWidth: RemH becomes RemL, RemL becomes
8801 // PartialRemL.
8802 RemH = RemL;
8803 RemL = PartialRemL;
8804 } else {
8805 // Shift left by more than HBitWidth.
8806 RemH = DAG.getNode(
8807 ISD::SHL, dl, HiLoVT, RemL,
8808 DAG.getShiftAmountConstant(TrailingZeros - HBitWidth, HiLoVT, dl));
8809 RemH = DAG.getNode(ISD::OR, dl, HiLoVT, RemH, PartialRemH,
8811 RemL = PartialRemL;
8812 }
8813 }
8814 Result.push_back(RemL);
8815 Result.push_back(RemH);
8816 }
8817
8818 return true;
8819}
8820
8821bool TargetLowering::expandUDIVREMByConstantViaUMulHiMagic(
8822 SDNode *N, const APInt &Divisor, SmallVectorImpl<SDValue> &Result,
8823 EVT HiLoVT, SelectionDAG &DAG, SDValue LL, SDValue LH) const {
8824
8825 SDValue N0 = N->getOperand(0);
8826 EVT VT = N0->getValueType(0);
8827 SDLoc DL{N};
8828
8829 assert(!Divisor.isOne() && "Magic algorithm does not work for division by 1");
8830
8831 // This helper creates a MUL_LOHI of the pair (LL, LH) by a constant.
8832 auto MakeMUL_LOHIByConst = [&](unsigned Opc, SDValue LL, SDValue LH,
8833 const APInt &Const,
8834 SmallVectorImpl<SDValue> &Result) {
8835 SDValue LHS = DAG.getNode(ISD::BUILD_PAIR, DL, VT, LL, LH);
8836 SDValue RHS = DAG.getConstant(Const, DL, VT);
8837 auto [RL, RH] = DAG.SplitScalar(RHS, DL, HiLoVT, HiLoVT);
8838 return expandMUL_LOHI(Opc, VT, DL, LHS, RHS, Result, HiLoVT, DAG,
8840 LL, LH, RL, RH);
8841 };
8842
8843 // This helper creates an ADD/SUB of the pairs (LL, LH) and (RL, RH).
8844 auto MakeAddSubLong = [&](unsigned Opc, SDValue LL, SDValue LH, SDValue RL,
8845 SDValue RH) {
8846 SDValue AddSubNode =
8848 DAG.getVTList(HiLoVT, MVT::i1), LL, RL);
8849 SDValue OutL = AddSubNode.getValue(0);
8850 SDValue Overflow = AddSubNode.getValue(1);
8851 SDValue AddSubWithOverflow =
8853 DAG.getVTList(HiLoVT, MVT::i1), LH, RH, Overflow);
8854 SDValue OutH = AddSubWithOverflow.getValue(0);
8855 return std::make_pair(OutL, OutH);
8856 };
8857
8858 // This helper creates a SRL of the pair (LL, LH) by Shift.
8859 auto MakeSRLLong = [&](SDValue LL, SDValue LH, unsigned Shift) {
8860 unsigned HBitWidth = HiLoVT.getScalarSizeInBits();
8861 if (Shift < HBitWidth) {
8862 SDValue ShAmt = DAG.getShiftAmountConstant(Shift, HiLoVT, DL);
8863 SDValue ResL = DAG.getNode(ISD::FSHR, DL, HiLoVT, LH, LL, ShAmt);
8864 SDValue ResH = DAG.getNode(ISD::SRL, DL, HiLoVT, LH, ShAmt);
8865 return std::make_pair(ResL, ResH);
8866 }
8867 SDValue Zero = DAG.getConstant(0, DL, HiLoVT);
8868 if (Shift == HBitWidth)
8869 return std::make_pair(LH, Zero);
8870 assert(Shift - HBitWidth < HBitWidth &&
8871 "We shouldn't generate an undefined shift");
8872 SDValue ShAmt = DAG.getShiftAmountConstant(Shift - HBitWidth, HiLoVT, DL);
8873 return std::make_pair(DAG.getNode(ISD::SRL, DL, HiLoVT, LH, ShAmt), Zero);
8874 };
8875
8876 // Knowledge of leading zeros may help to reduce the multiplier.
8877 unsigned KnownLeadingZeros = DAG.computeKnownBits(N0).countMinLeadingZeros();
8878
8879 UnsignedDivisionByConstantInfo Magics = UnsignedDivisionByConstantInfo::get(
8880 Divisor, std::min(KnownLeadingZeros, Divisor.countl_zero()));
8881
8882 assert(!LL == !LH && "Expected both input halves or no input halves!");
8883 if (!LL)
8884 std::tie(LL, LH) = DAG.SplitScalar(N0, DL, HiLoVT, HiLoVT);
8885 SDValue QL = LL;
8886 SDValue QH = LH;
8887 if (Magics.PreShift != 0)
8888 std::tie(QL, QH) = MakeSRLLong(QL, QH, Magics.PreShift);
8889
8890 SmallVector<SDValue, 4> UMulResult;
8891 if (!MakeMUL_LOHIByConst(ISD::UMUL_LOHI, QL, QH, Magics.Magic, UMulResult))
8892 return false;
8893
8894 QL = UMulResult[2];
8895 QH = UMulResult[3];
8896
8897 if (Magics.IsAdd) {
8898 auto [NPQL, NPQH] = MakeAddSubLong(ISD::SUB, LL, LH, QL, QH);
8899 std::tie(NPQL, NPQH) = MakeSRLLong(NPQL, NPQH, 1);
8900 std::tie(QL, QH) = MakeAddSubLong(ISD::ADD, NPQL, NPQH, QL, QH);
8901 }
8902
8903 if (Magics.PostShift != 0)
8904 std::tie(QL, QH) = MakeSRLLong(QL, QH, Magics.PostShift);
8905
8906 unsigned Opcode = N->getOpcode();
8907 if (Opcode != ISD::UREM) {
8908 Result.push_back(QL);
8909 Result.push_back(QH);
8910 }
8911
8912 if (Opcode != ISD::UDIV) {
8913 SmallVector<SDValue, 2> MulResult;
8914 if (!MakeMUL_LOHIByConst(ISD::MUL, QL, QH, Divisor, MulResult))
8915 return false;
8916
8917 assert(MulResult.size() == 2);
8918
8919 auto [RemL, RemH] =
8920 MakeAddSubLong(ISD::SUB, LL, LH, MulResult[0], MulResult[1]);
8921
8922 Result.push_back(RemL);
8923 Result.push_back(RemH);
8924 }
8925
8926 return true;
8927}
8928
8931 EVT HiLoVT, SelectionDAG &DAG,
8932 SDValue LL, SDValue LH) const {
8933 unsigned Opcode = N->getOpcode();
8934
8935 // TODO: Support signed division/remainder.
8936 if (Opcode == ISD::SREM || Opcode == ISD::SDIV || Opcode == ISD::SDIVREM)
8937 return false;
8938 assert(
8939 (Opcode == ISD::UREM || Opcode == ISD::UDIV || Opcode == ISD::UDIVREM) &&
8940 "Unexpected opcode");
8941
8942 auto *CN = dyn_cast<ConstantSDNode>(N->getOperand(1));
8943 if (!CN)
8944 return false;
8945
8946 APInt Divisor = CN->getAPIntValue();
8947
8948 // The generated half-width UREM is normally optimized using high multiply.
8949 // If the wide UREM libcall is unavailable, a legal or custom half-width
8950 // UDIVREM can lower it instead.
8951 bool CanDecomposeUREMWithoutMulHi =
8952 Opcode == ISD::UREM &&
8953 DAG.getLibcalls().getLibcallImpl(RTLIB::getUREM(N->getValueType(0))) ==
8954 RTLIB::Unsupported &&
8956 if (!CanDecomposeUREMWithoutMulHi &&
8959 return false;
8960
8961 // Prefer the smaller libcall when one is available.
8962 if (DAG.shouldOptForSize() && !CanDecomposeUREMWithoutMulHi)
8963 return false;
8964
8965 // Early out for 0 or 1 divisors.
8966 if (Divisor.ule(1))
8967 return false;
8968
8969 if (expandUDIVREMByConstantViaUREMDecomposition(N, Divisor, Result, HiLoVT,
8970 DAG, LL, LH))
8971 return true;
8972
8973 if (expandUDIVREMByConstantViaUMulHiMagic(N, Divisor, Result, HiLoVT, DAG, LL,
8974 LH))
8975 return true;
8976
8977 return false;
8978}
8979
8980// Check that (every element of) Z is undef or not an exact multiple of BW.
8981static bool isNonZeroModBitWidthOrUndef(SDValue Z, unsigned BW) {
8983 Z,
8984 [=](ConstantSDNode *C) { return !C || C->getAPIntValue().urem(BW) != 0; },
8985 /*AllowUndefs=*/true, /*AllowTruncation=*/true);
8986}
8987
8989 SelectionDAG &DAG) const {
8990 EVT VT = Node->getValueType(0);
8991
8992 if (VT.isVector() && (!isOperationLegalOrCustom(ISD::SHL, VT) ||
8996 return SDValue();
8997
8998 SDValue X = Node->getOperand(0);
8999 SDValue Y = Node->getOperand(1);
9000 SDValue Z = Node->getOperand(2);
9001
9002 unsigned BW = VT.getScalarSizeInBits();
9003 bool IsFSHL = Node->getOpcode() == ISD::FSHL;
9004 SDLoc DL(SDValue(Node, 0));
9005
9006 EVT ShVT = Z.getValueType();
9007
9008 // If a funnel shift in the other direction is more supported, use it.
9009 unsigned RevOpcode = IsFSHL ? ISD::FSHR : ISD::FSHL;
9010 if (!isOperationLegalOrCustom(Node->getOpcode(), VT) &&
9011 isOperationLegalOrCustom(RevOpcode, VT) && isPowerOf2_32(BW)) {
9012 if (isNonZeroModBitWidthOrUndef(Z, BW)) {
9013 // fshl X, Y, Z -> fshr X, Y, -Z
9014 // fshr X, Y, Z -> fshl X, Y, -Z
9015 Z = DAG.getNegative(Z, DL, ShVT);
9016 } else {
9017 // fshl X, Y, Z -> fshr (srl X, 1), (fshr X, Y, 1), ~Z
9018 // fshr X, Y, Z -> fshl (fshl X, Y, 1), (shl Y, 1), ~Z
9019 SDValue One = DAG.getConstant(1, DL, ShVT);
9020 if (IsFSHL) {
9021 Y = DAG.getNode(RevOpcode, DL, VT, X, Y, One);
9022 X = DAG.getNode(ISD::SRL, DL, VT, X, One);
9023 } else {
9024 X = DAG.getNode(RevOpcode, DL, VT, X, Y, One);
9025 Y = DAG.getNode(ISD::SHL, DL, VT, Y, One);
9026 }
9027 Z = DAG.getNOT(DL, Z, ShVT);
9028 }
9029 return DAG.getNode(RevOpcode, DL, VT, X, Y, Z);
9030 }
9031
9032 SDValue ShX, ShY;
9033 SDValue ShAmt, InvShAmt;
9034 if (isNonZeroModBitWidthOrUndef(Z, BW)) {
9035 // fshl: X << C | Y >> (BW - C)
9036 // fshr: X << (BW - C) | Y >> C
9037 // where C = Z % BW is not zero
9038 SDValue BitWidthC = DAG.getConstant(BW, DL, ShVT);
9039 ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Z, BitWidthC);
9040 InvShAmt = DAG.getNode(ISD::SUB, DL, ShVT, BitWidthC, ShAmt);
9041 ShX = DAG.getNode(ISD::SHL, DL, VT, X, IsFSHL ? ShAmt : InvShAmt);
9042 ShY = DAG.getNode(ISD::SRL, DL, VT, Y, IsFSHL ? InvShAmt : ShAmt);
9043 } else {
9044 // fshl: X << (Z % BW) | Y >> 1 >> (BW - 1 - (Z % BW))
9045 // fshr: X << 1 << (BW - 1 - (Z % BW)) | Y >> (Z % BW)
9046 SDValue Mask = DAG.getConstant(BW - 1, DL, ShVT);
9047 if (isPowerOf2_32(BW)) {
9048 // Z % BW -> Z & (BW - 1)
9049 ShAmt = DAG.getNode(ISD::AND, DL, ShVT, Z, Mask);
9050 // (BW - 1) - (Z % BW) -> ~Z & (BW - 1)
9051 InvShAmt = DAG.getNode(ISD::AND, DL, ShVT, DAG.getNOT(DL, Z, ShVT), Mask);
9052 } else {
9053 SDValue BitWidthC = DAG.getConstant(BW, DL, ShVT);
9054 ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Z, BitWidthC);
9055 InvShAmt = DAG.getNode(ISD::SUB, DL, ShVT, Mask, ShAmt);
9056 }
9057
9058 SDValue One = DAG.getConstant(1, DL, ShVT);
9059 if (IsFSHL) {
9060 ShX = DAG.getNode(ISD::SHL, DL, VT, X, ShAmt);
9061 SDValue ShY1 = DAG.getNode(ISD::SRL, DL, VT, Y, One);
9062 ShY = DAG.getNode(ISD::SRL, DL, VT, ShY1, InvShAmt);
9063 } else {
9064 SDValue ShX1 = DAG.getNode(ISD::SHL, DL, VT, X, One);
9065 ShX = DAG.getNode(ISD::SHL, DL, VT, ShX1, InvShAmt);
9066 ShY = DAG.getNode(ISD::SRL, DL, VT, Y, ShAmt);
9067 }
9068 }
9069 return DAG.getNode(ISD::OR, DL, VT, ShX, ShY);
9070}
9071
9072// TODO: Merge with expandFunnelShift.
9074 SelectionDAG &DAG) const {
9075 EVT VT = Node->getValueType(0);
9076 unsigned EltSizeInBits = VT.getScalarSizeInBits();
9077 bool IsLeft = Node->getOpcode() == ISD::ROTL;
9078 SDValue Op0 = Node->getOperand(0);
9079 SDValue Op1 = Node->getOperand(1);
9080 SDLoc DL(SDValue(Node, 0));
9081
9082 EVT ShVT = Op1.getValueType();
9083 SDValue Zero = DAG.getConstant(0, DL, ShVT);
9084
9085 // If a rotate in the other direction is more supported, use it.
9086 unsigned RevRot = IsLeft ? ISD::ROTR : ISD::ROTL;
9087 if (!isOperationLegalOrCustom(Node->getOpcode(), VT) &&
9088 isOperationLegalOrCustom(RevRot, VT) && isPowerOf2_32(EltSizeInBits)) {
9089 SDValue Sub = DAG.getNode(ISD::SUB, DL, ShVT, Zero, Op1);
9090 return DAG.getNode(RevRot, DL, VT, Op0, Sub);
9091 }
9092
9093 if (!AllowVectorOps && VT.isVector() &&
9099 return SDValue();
9100
9101 unsigned ShOpc = IsLeft ? ISD::SHL : ISD::SRL;
9102 unsigned HsOpc = IsLeft ? ISD::SRL : ISD::SHL;
9103 SDValue BitWidthMinusOneC = DAG.getConstant(EltSizeInBits - 1, DL, ShVT);
9104 SDValue ShVal;
9105 SDValue HsVal;
9106 if (isPowerOf2_32(EltSizeInBits)) {
9107 // (rotl x, c) -> x << (c & (w - 1)) | x >> (-c & (w - 1))
9108 // (rotr x, c) -> x >> (c & (w - 1)) | x << (-c & (w - 1))
9109 SDValue NegOp1 = DAG.getNode(ISD::SUB, DL, ShVT, Zero, Op1);
9110 SDValue ShAmt = DAG.getNode(ISD::AND, DL, ShVT, Op1, BitWidthMinusOneC);
9111 ShVal = DAG.getNode(ShOpc, DL, VT, Op0, ShAmt);
9112 SDValue HsAmt = DAG.getNode(ISD::AND, DL, ShVT, NegOp1, BitWidthMinusOneC);
9113 HsVal = DAG.getNode(HsOpc, DL, VT, Op0, HsAmt);
9114 } else {
9115 // (rotl x, c) -> x << (c % w) | x >> 1 >> (w - 1 - (c % w))
9116 // (rotr x, c) -> x >> (c % w) | x << 1 << (w - 1 - (c % w))
9117 SDValue BitWidthC = DAG.getConstant(EltSizeInBits, DL, ShVT);
9118 SDValue ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Op1, BitWidthC);
9119 ShVal = DAG.getNode(ShOpc, DL, VT, Op0, ShAmt);
9120 SDValue HsAmt = DAG.getNode(ISD::SUB, DL, ShVT, BitWidthMinusOneC, ShAmt);
9121 SDValue One = DAG.getConstant(1, DL, ShVT);
9122 HsVal =
9123 DAG.getNode(HsOpc, DL, VT, DAG.getNode(HsOpc, DL, VT, Op0, One), HsAmt);
9124 }
9125 return DAG.getNode(ISD::OR, DL, VT, ShVal, HsVal);
9126}
9127
9128/// Check if CLMUL on VT can eventually reach a type with legal CLMUL through
9129/// a chain of halving decompositions (halving element width) and/or vector
9130/// widening (doubling element count). This guides expansion strategy selection:
9131/// if true, the halving/widening path produces better code than bit-by-bit.
9132///
9133/// HalveDepth tracks halving steps only (each creates ~4x more operations).
9134/// Widening steps are cheap (O(1) pad/extract) and don't count.
9135/// Limiting halvings to 2 prevents exponential blowup:
9136/// 1 halving: ~4 sub-CLMULs (good, e.g. v8i16 -> v8i8)
9137/// 2 halvings: ~16 sub-CLMULs (acceptable, e.g. v4i32 -> v4i16 -> v8i8)
9138/// 3 halvings: ~64 sub-CLMULs (worse than bit-by-bit expansion)
9140 EVT VT, unsigned HalveDepth = 0,
9141 unsigned TotalDepth = 0) {
9142 if (HalveDepth > 2 || TotalDepth > 8 || !VT.isFixedLengthVector())
9143 return false;
9144 if (TLI.isOperationLegalOrCustom(ISD::CLMUL, VT))
9145 return true;
9146 if (!TLI.isTypeLegal(VT))
9147 return false;
9148
9149 unsigned BW = VT.getScalarSizeInBits();
9150
9151 // Halve: halve element width, same element count.
9152 // This is the expensive step -- each halving creates ~4x more operations.
9153 if (BW % 2 == 0) {
9154 EVT HalfEltVT = EVT::getIntegerVT(Ctx, BW / 2);
9155 EVT HalfVT = VT.changeVectorElementType(Ctx, HalfEltVT);
9156 if (TLI.isTypeLegal(HalfVT) &&
9157 canNarrowCLMULToLegal(TLI, Ctx, HalfVT, HalveDepth + 1, TotalDepth + 1))
9158 return true;
9159 }
9160
9161 // Widen: double element count (fixed-width vectors only).
9162 // This is cheap -- just INSERT_SUBVECTOR + EXTRACT_SUBVECTOR.
9163 EVT WideVT = VT.getDoubleNumVectorElementsVT(Ctx);
9164 if (TLI.isTypeLegal(WideVT) &&
9165 canNarrowCLMULToLegal(TLI, Ctx, WideVT, HalveDepth, TotalDepth + 1))
9166 return true;
9167
9168 return false;
9169}
9170
9172 SDLoc DL(Node);
9173 EVT VT = Node->getValueType(0);
9174 SDValue X = Node->getOperand(0);
9175 SDValue Y = Node->getOperand(1);
9176 unsigned BW = VT.getScalarSizeInBits();
9177 unsigned Opcode = Node->getOpcode();
9178 LLVMContext &Ctx = *DAG.getContext();
9179
9180 switch (Opcode) {
9181 case ISD::CLMUL: {
9182 // For vector types, try decomposition strategies that leverage legal
9183 // CLMUL on narrower or wider element types, avoiding the expensive
9184 // bit-by-bit expansion.
9185 if (VT.isVector()) {
9186 // Strategy 1: Halving decomposition to half-element-width CLMUL.
9187 // Applies ExpandIntRes_CLMUL's identity element-wise:
9188 // CLMUL(X, Y) = (Hi << HalfBW) | Lo
9189 // where:
9190 // Lo = CLMUL(XLo, YLo)
9191 // Hi = CLMULH(XLo, YLo) ^ CLMUL(XLo, YHi) ^ CLMUL(XHi, YLo)
9192 unsigned HalfBW = BW / 2;
9193 if (BW % 2 == 0) {
9194 EVT HalfEltVT = EVT::getIntegerVT(Ctx, HalfBW);
9195 EVT HalfVT =
9196 EVT::getVectorVT(Ctx, HalfEltVT, VT.getVectorElementCount());
9197 if (isTypeLegal(HalfVT) && canNarrowCLMULToLegal(*this, Ctx, HalfVT,
9198 /*HalveDepth=*/1)) {
9199 SDValue ShAmt = DAG.getShiftAmountConstant(HalfBW, VT, DL);
9200
9201 // Extract low and high halves of each element.
9202 SDValue XLo = DAG.getNode(ISD::TRUNCATE, DL, HalfVT, X);
9203 SDValue XHi = DAG.getNode(ISD::TRUNCATE, DL, HalfVT,
9204 DAG.getNode(ISD::SRL, DL, VT, X, ShAmt));
9205 SDValue YLo = DAG.getNode(ISD::TRUNCATE, DL, HalfVT, Y);
9206 SDValue YHi = DAG.getNode(ISD::TRUNCATE, DL, HalfVT,
9207 DAG.getNode(ISD::SRL, DL, VT, Y, ShAmt));
9208
9209 // Lo = CLMUL(XLo, YLo)
9210 SDValue Lo = DAG.getNode(ISD::CLMUL, DL, HalfVT, XLo, YLo);
9211
9212 // Hi = CLMULH(XLo, YLo) ^ CLMUL(XLo, YHi) ^ CLMUL(XHi, YLo)
9213 SDValue LoH = DAG.getNode(ISD::CLMULH, DL, HalfVT, XLo, YLo);
9214 SDValue Cross1 = DAG.getNode(ISD::CLMUL, DL, HalfVT, XLo, YHi);
9215 SDValue Cross2 = DAG.getNode(ISD::CLMUL, DL, HalfVT, XHi, YLo);
9216 SDValue Cross = DAG.getNode(ISD::XOR, DL, HalfVT, Cross1, Cross2);
9217 SDValue Hi = DAG.getNode(ISD::XOR, DL, HalfVT, LoH, Cross);
9218
9219 // Reassemble: Result = ZExt(Lo) | (AnyExt(Hi) << HalfBW)
9220 SDValue LoExt = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Lo);
9221 SDValue HiExt = DAG.getNode(ISD::ANY_EXTEND, DL, VT, Hi);
9222 SDValue HiShifted = DAG.getNode(ISD::SHL, DL, VT, HiExt, ShAmt);
9223 return DAG.getNode(ISD::OR, DL, VT, LoExt, HiShifted);
9224 }
9225 }
9226
9227 // Strategy 2: Promote to double-element-width CLMUL.
9228 // CLMUL(X, Y) = Trunc(CLMUL(AnyExt(X), AnyExt(Y)))
9229 {
9230 EVT ExtVT = VT.widenIntegerElementType(Ctx);
9231 if (isTypeLegal(ExtVT) && isOperationLegalOrCustom(ISD::CLMUL, ExtVT)) {
9232 // If CLMUL on ExtVT is Custom (not Legal), the target may
9233 // scalarize it, costing O(NumElements) scalar ops. The bit-by-bit
9234 // fallback costs O(BW) vectorized iterations. Only widen when
9235 // element count is small enough that scalarization is cheaper.
9236 unsigned NumElts = VT.getVectorMinNumElements();
9237 if (isOperationLegal(ISD::CLMUL, ExtVT) || NumElts < BW) {
9238 SDValue XExt = DAG.getNode(ISD::ANY_EXTEND, DL, ExtVT, X);
9239 SDValue YExt = DAG.getNode(ISD::ANY_EXTEND, DL, ExtVT, Y);
9240 SDValue Mul = DAG.getNode(ISD::CLMUL, DL, ExtVT, XExt, YExt);
9241 return DAG.getNode(ISD::TRUNCATE, DL, VT, Mul);
9242 }
9243 }
9244 }
9245
9246 // Strategy 3: Widen element count (pad with undef, do CLMUL on wider
9247 // vector, extract lower result). CLMUL is element-wise, so upper
9248 // (undef) lanes don't affect the lower results.
9249 // e.g. v4i16 => pad to v8i16 => halve to v8i8 PMUL => extract v4i16.
9250 if (auto EC = VT.getVectorElementCount(); EC.isFixed()) {
9251 EVT WideVT = EVT::getVectorVT(Ctx, VT.getVectorElementType(), EC * 2);
9252 if (isTypeLegal(WideVT) && canNarrowCLMULToLegal(*this, Ctx, WideVT)) {
9253 SDValue Undef = DAG.getUNDEF(WideVT);
9254 SDValue XWide = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WideVT, Undef,
9255 X, DAG.getVectorIdxConstant(0, DL));
9256 SDValue YWide = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WideVT, Undef,
9257 Y, DAG.getVectorIdxConstant(0, DL));
9258 SDValue WideRes = DAG.getNode(ISD::CLMUL, DL, WideVT, XWide, YWide);
9259 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, WideRes,
9260 DAG.getVectorIdxConstant(0, DL));
9261 }
9262 }
9263 }
9264
9265 // Special case: clmul(X, Y) where Y is a known constant (splat) that forms
9266 // a contiguous block of trailing ones whose length N is a power of two
9267 // (e.g. i8 0xFF, i8 0x0F, ...) or equal to the operand width. In this
9268 // special case, clmul(X, Y) is equivalent to a "parallel prefix XOR" or
9269 // "bitwise parity" operation on X.
9270 //
9271 // Note: This special currently dose NOT apply when the mask is neither a
9272 // power of two nor equal to the operand width because the loop inside
9273 // behaves as if the mask was bit-ceiled, and "undoing" the XOR with parts
9274 // of that CLMUL is a recursive problem (e.g. CLMUL with a 20-bit mask
9275 // requires correction XOR with CLMUL with 12-bit mask).
9276 if (auto *C = isConstOrConstSplat(Y, /*AllowUndefs=*/true)) {
9277 const APInt &YVal = C->getAPIntValue();
9278 unsigned N = YVal.countr_one();
9279 if (YVal.isAllOnes() || (YVal.isMask() && isPowerOf2_32(N))) {
9280 SDValue R = X;
9281 for (unsigned I = 1; I < N; I <<= 1) {
9282 SDValue ShAmt = DAG.getShiftAmountConstant(I, VT, DL);
9283 SDValue Shifted = DAG.getNode(ISD::SHL, DL, VT, R, ShAmt);
9284 R = DAG.getNode(ISD::XOR, DL, VT, R, Shifted);
9285 }
9286 return R;
9287 }
9288 }
9289
9290 // NOTE: If you change this expansion, please update the cost model
9291 // calculation in BasicTTIImpl::getTypeBasedIntrinsicInstrCost for
9292 // Intrinsic::clmul.
9293
9294 // Strategy 4: multiplication with holes.
9295 //
9296 // Uses "holes" (sequences of zeroes) to avoid carry spilling. When carries
9297 // do occur, they wind up in a "hole" and are subsequently masked out of the
9298 // result.
9299 //
9300 // https://www.bearssl.org/constanttime.html#ghash-for-gcm describes this
9301 // approach.
9302
9303 // Stride S handles operands up to S·2^S bits using S² multiplies.
9304 //
9305 // * BW <= 8 uses S = 2 (holes of 1 bit)
9306 // * BW <= 24 uses S = 3 (holes of 2 bits)
9307 // * BW <= 64 uses S = 4 (holes of 3 bits)
9308 // * BW <= 160 uses S = 5 (holes of 4 bits)
9309 // * BW <= 384 uses S = 6 (holes of 5 bits)
9310 //
9311 // We distribute the BW bits over S phases:
9312 //
9313 // phase 0 keeps bits: 0, S, 2S, ...
9314 // phase 1 keeps bits: 1, S + 1, 2S + 1, ...
9315 // ...
9316 //
9317 // Each phase has up to n = ceil(BW / S) bits set, and the holes are S-1
9318 // bits wide.
9319 //
9320 // Take BW = 4, S = 2, n = 2. The worst case is a fully populated phase (all
9321 // non-hole bits are set to 1) multiplied by itself, 0b0101 * 0b0101. Each
9322 // set bit of one operand shifts a copy of the other, and we add the copies:
9323 //
9324 // col: 4 3 2 1 0
9325 // 0b0101 << 0: 0 0 1 0 1
9326 // 0b0101 << 2: 1 0 1 0 0
9327 // ----------------- +
9328 // count: 1 0 2 0 1
9329 //
9330 // Counting the number of one-bits in each column gives a triangle: the
9331 // counts climb 1, 2, ..., n and back down (here 1, 2, 1 across the data
9332 // columns). So a column holds at most n one-bits, and that maximum n is
9333 // reached in only one column: the peak. Every other column holds at most n
9334 // - 1 one-bits.
9335 //
9336 // A stack of one-bits in a column turns into carries: column 2 above really
9337 // stores the value 1 + 1 = 2 = n. A column spans S bits, its kept bit
9338 // plus S-1 hole bits, and the count is written from the kept bit upward,
9339 // so any count <= 2^S - 1 stays within the column and never interferes with
9340 // the next data bit S positions up. Every non-peak column holds at most n -
9341 // 1, so they all fit as soon as n - 1 <= 2^S - 1.
9342 //
9343 // That leaves only the peak column. Because both operands set all data
9344 // bits, the triangle peaks at the top of the word at the highest data bit
9345 // still inside BW. Here the count reaches exactly n = 2^S and overflows.
9346 // But its carry lands at bit n*S >= BW, off the top, where it (and the
9347 // whole descending half of the triangle) is truncated.
9348 //
9349 // Hence the holes suffice exactly when n = ceil(BW / S) <= 2^S, i.e. BW <=
9350 // S*2^S.
9351 //
9352 // Here we find the smallest S that satisfies this inequality.
9353 unsigned S = 1;
9354 while (S < 32 && divideCeil(BW, S) > (1u << S))
9355 ++S;
9356
9357 // The "multiplication with holes" expansion emits S*S MULs, 3*S ANDs,
9358 // S*(S-1) XORs and S-1 ORs.
9359 unsigned HolesCost = S * S + 3 * S + S * (S - 1) + (S - 1);
9360
9361 // Estimate the cost of the naive algorithm.
9362 KnownBits KnownY = DAG.computeKnownBits(Y);
9363 unsigned NaiveCost = 0;
9364 for (unsigned I = 0; I < BW; ++I) {
9365 // The iteration folds away entirely and is free.
9366 if (KnownY.Zero[I])
9367 continue;
9368
9369 // On targets with a fast bit test instruction more instructions are used
9370 // to not need a (potentially expensive) multiplication. See also below.
9371 if (hasBitTest(Y, DAG.getShiftAmountConstant(I, VT, DL))) {
9372 // AND + SETCC + SHL + SELECT + XOR.
9373 NaiveCost += 5;
9374 } else {
9375 // AND + MUL + XOR.
9376 NaiveCost += 3;
9377 }
9378 }
9379
9380 // Only use multiplication with holes when it is cheaper, else use the naive
9381 // fallback below.
9382 if (HolesCost < NaiveCost &&
9384
9385 // Set a bit every S positions, e.g. for S = 4 this is equivalent to
9386 // 0b...00010001...0001.
9387 APInt MaskVal = APInt::getSplat(BW, APInt(S, 1));
9388
9389 // Create versions of X and Y that keep only the I-th bit of each S-bit
9390 // slice.
9391 SmallVector<SDValue, 4> M(S), Xp(S), Yp(S);
9392 for (unsigned I = 0; I < S; ++I) {
9393 M[I] = DAG.getConstant(MaskVal.shl(I), DL, VT);
9394 Xp[I] = DAG.getNode(ISD::AND, DL, VT, X, M[I]);
9395 Yp[I] = DAG.getNode(ISD::AND, DL, VT, Y, M[I]);
9396 }
9397
9398 // Codegens these expressions (S*S multiplications), e.g. for S=4:
9399 //
9400 // z0 = (x0 * y0) ^ (x1 * y3) ^ (x2 * y2) ^ (x3 * y1);
9401 // z1 = (x0 * y1) ^ (x1 * y0) ^ (x2 * y3) ^ (x3 * y2);
9402 // z2 = (x0 * y2) ^ (x1 * y1) ^ (x2 * y0) ^ (x3 * y3);
9403 // z3 = (x0 * y3) ^ (x1 * y2) ^ (x2 * y1) ^ (x3 * y0);
9404 SDValue Res = DAG.getConstant(0, DL, VT);
9405 for (unsigned I = 0; I < S; ++I) {
9406 SDValue Zi = DAG.getConstant(0, DL, VT);
9407 for (unsigned J = 0; J < S; ++J) {
9408 unsigned K = (I + S - J) % S;
9409 SDValue P = DAG.getNode(ISD::MUL, DL, VT, Xp[J], Yp[K]);
9410 Zi = DAG.getNode(ISD::XOR, DL, VT, Zi, P);
9411 }
9412
9413 // Keep only the bits belonging to this iteration, and bitwise or it all
9414 // together.
9415 Zi = DAG.getNode(ISD::AND, DL, VT, Zi, M[I]);
9416 Res = DAG.getNode(ISD::OR, DL, VT, Res, Zi, SDNodeFlags::Disjoint);
9417 }
9418 return Res;
9419 }
9420
9421 // Strategy 5: the naive fallback.
9422 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), Ctx, VT);
9423
9424 SDValue Res = DAG.getConstant(0, DL, VT);
9425 for (unsigned I = 0; I < BW; ++I) {
9426 SDValue ShiftAmt = DAG.getShiftAmountConstant(I, VT, DL);
9427 SDValue Mask = DAG.getConstant(APInt::getOneBitSet(BW, I), DL, VT);
9428 SDValue YMasked = DAG.getNode(ISD::AND, DL, VT, Y, Mask);
9429
9430 // For targets with a fast bit test instruction (e.g., x86 BT) or without
9431 // multiply, use a shift-based expansion to avoid expensive MUL
9432 // instructions.
9433 SDValue Part;
9434 if (!hasBitTest(Y, ShiftAmt) &&
9437 Part = DAG.getNode(ISD::MUL, DL, VT, X, YMasked);
9438 } else {
9439 // Canonical bit test: (Y & (1 << I)) != 0
9440 SDValue Zero = DAG.getConstant(0, DL, VT);
9441 SDValue Cond = DAG.getSetCC(DL, SetCCVT, YMasked, Zero, ISD::SETEQ);
9442 SDValue XShifted = DAG.getNode(ISD::SHL, DL, VT, X, ShiftAmt);
9443 Part = DAG.getSelect(DL, VT, Cond, Zero, XShifted);
9444 }
9445 Res = DAG.getNode(ISD::XOR, DL, VT, Res, Part);
9446 }
9447 return Res;
9448 }
9449 case ISD::CLMULR:
9450 // If we have CLMUL/CLMULH, merge the shifted results to form CLMULR.
9453 SDValue Lo = DAG.getNode(ISD::CLMUL, DL, VT, X, Y);
9454 SDValue Hi = DAG.getNode(ISD::CLMULH, DL, VT, X, Y);
9455 Lo = DAG.getNode(ISD::SRL, DL, VT, Lo,
9456 DAG.getShiftAmountConstant(BW - 1, VT, DL));
9457 Hi = DAG.getNode(ISD::SHL, DL, VT, Hi,
9458 DAG.getShiftAmountConstant(1, VT, DL));
9459 return DAG.getNode(ISD::OR, DL, VT, Lo, Hi);
9460 }
9461 [[fallthrough]];
9462 case ISD::CLMULH: {
9463 EVT ExtVT = VT.widenIntegerElementType(Ctx);
9464 // Use bitreverse-based lowering (CLMULR/H = rev(CLMUL(rev,rev)) >> S)
9465 // when any of these hold:
9466 // (a) ZERO_EXTEND to ExtVT or SRL on ExtVT isn't legal.
9467 // (b) CLMUL is legal on VT but not on ExtVT (e.g. v8i8 on AArch64).
9468 // (c) CLMUL on ExtVT isn't legal, but CLMUL on VT can be efficiently
9469 // expanded via halving/widening to reach legal CLMUL. The bitreverse
9470 // path creates CLMUL(VT) which will be expanded efficiently. The
9471 // promote path would create CLMUL(ExtVT) => halving => CLMULH(VT),
9472 // causing a cycle.
9473 // Note: when CLMUL is legal on ExtVT, the zext => CLMUL(ExtVT) => shift
9474 // => trunc path is preferred over the bitreverse path, as it avoids the
9475 // cost of 3 bitreverse operations.
9480 canNarrowCLMULToLegal(*this, Ctx, VT)))) {
9481 SDValue XRev = DAG.getNode(ISD::BITREVERSE, DL, VT, X);
9482 SDValue YRev = DAG.getNode(ISD::BITREVERSE, DL, VT, Y);
9483 SDValue ClMul = DAG.getNode(ISD::CLMUL, DL, VT, XRev, YRev);
9484 SDValue Res = DAG.getNode(ISD::BITREVERSE, DL, VT, ClMul);
9485 if (Opcode == ISD::CLMULH)
9486 Res = DAG.getNode(ISD::SRL, DL, VT, Res,
9487 DAG.getShiftAmountConstant(1, VT, DL));
9488 return Res;
9489 }
9490 SDValue XExt = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVT, X);
9491 SDValue YExt = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVT, Y);
9492 SDValue ClMul = DAG.getNode(ISD::CLMUL, DL, ExtVT, XExt, YExt);
9493 unsigned ShAmt = Opcode == ISD::CLMULR ? BW - 1 : BW;
9494 SDValue HiBits = DAG.getNode(ISD::SRL, DL, ExtVT, ClMul,
9495 DAG.getShiftAmountConstant(ShAmt, ExtVT, DL));
9496 return DAG.getNode(ISD::TRUNCATE, DL, VT, HiBits);
9497 }
9498 }
9499 llvm_unreachable("Expected CLMUL, CLMULR, or CLMULH");
9500}
9501
9504 SDLoc dl(Node);
9505 SDValue Val = Node->getOperand(0);
9506 EVT VT = Val.getValueType();
9507 SDNodeFlags Flags = Node->getFlags();
9508
9509 SDValue IntPart = DAG.getNode(ISD::FTRUNC, dl, VT, Val, Flags);
9510 SDValue FracPart = DAG.getNode(ISD::FSUB, dl, VT, Val, IntPart, Flags);
9511
9512 SDValue FracToUse = FracPart;
9513 if (!Flags.hasNoInfs()) {
9514 SDValue Abs = DAG.getNode(ISD::FABS, dl, VT, Val, Flags);
9515 SDValue Inf =
9517 EVT SetCCVT =
9518 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
9519 SDValue IsInf = DAG.getSetCC(dl, SetCCVT, Abs, Inf, ISD::SETOEQ);
9520 SDValue Zero = DAG.getConstantFP(0.0, dl, VT);
9521 FracToUse = DAG.getSelect(dl, VT, IsInf, Zero, FracPart);
9522 }
9523
9524 SDValue ResultFrac =
9525 DAG.getNode(ISD::FCOPYSIGN, dl, VT, FracToUse, Val, Flags);
9526 Results.push_back(ResultFrac);
9527 Results.push_back(IntPart);
9528 return true;
9529}
9530
9532 SDLoc DL(Node);
9533 EVT VT = Node->getValueType(0);
9534 SDValue Val = Node->getOperand(0);
9535 SDValue Msk = Node->getOperand(1);
9536 unsigned BW = VT.getScalarSizeInBits();
9537
9538 // Just scalarize if scalar PEXT is legal
9540 return DAG.UnrollVectorOp(Node);
9541
9542 // Hacker's Delight §7-4: Compress, or Generalized Extract
9543 SDValue X = DAG.getNode(ISD::AND, DL, VT, Val, Msk);
9544 SDValue M = Msk;
9545 SDValue One = DAG.getShiftAmountConstant(1, VT, DL);
9546 SDValue Mk = DAG.getNode(ISD::SHL, DL, VT, DAG.getNOT(DL, M, VT), One);
9547
9548 // Repeatedly compute which bits would shift to the right by an odd amount,
9549 // shift all such bits in parallel using a mask, and double the shift amount.
9550 for (unsigned I = 1; I < BW; I *= 2) {
9551 // This expands the "parallel prefix" operation to clmul(Mk, ~0).
9552 SDValue Mp =
9553 DAG.getNode(ISD::CLMUL, DL, VT, Mk, DAG.getAllOnesConstant(DL, VT));
9554 SDValue Mv = DAG.getNode(ISD::AND, DL, VT, Mp, M);
9555 SDValue ShiftI = DAG.getShiftAmountConstant(I, VT, DL);
9556 SDValue MvS = DAG.getNode(ISD::SRL, DL, VT, Mv, ShiftI);
9557 M = DAG.getNode(ISD::OR, DL, VT, DAG.getNode(ISD::XOR, DL, VT, M, Mv), MvS,
9559 SDValue T = DAG.getNode(ISD::AND, DL, VT, X, Mv);
9560 SDValue TS = DAG.getNode(ISD::SRL, DL, VT, T, ShiftI);
9561 X = DAG.getNode(ISD::OR, DL, VT, DAG.getNode(ISD::XOR, DL, VT, X, T), TS,
9563 if (I * 2 < BW)
9564 Mk = DAG.getNode(ISD::AND, DL, VT, Mk, DAG.getNOT(DL, Mp, VT));
9565 }
9566
9567 return X;
9568}
9569
9571 SDLoc DL(Node);
9572 EVT VT = Node->getValueType(0);
9573 SDValue Val = Node->getOperand(0);
9574 SDValue Msk = Node->getOperand(1);
9575 unsigned BW = VT.getScalarSizeInBits();
9576
9577 // Just scalarize if scalar PDEP is legal
9579 return DAG.UnrollVectorOp(Node);
9580
9581 // Hacker's Delight §7-5: Expand, or Generalized Insert.
9582 unsigned LogBW = Log2_32_Ceil(BW);
9583 SmallVector<SDValue, 8> MvArray(LogBW);
9584 SDValue One = DAG.getShiftAmountConstant(1, VT, DL);
9585 SDValue Mc = Msk;
9586 SDValue Mk = DAG.getNode(ISD::SHL, DL, VT, DAG.getNOT(DL, Msk, VT), One);
9587
9588 // First pass: compute move masks for each power of two that a bit moves by.
9589 for (unsigned S = 0; S < LogBW; ++S) {
9590 unsigned ShiftS = 1u << S;
9591 // This expands the "parallel prefix" operation to clmul(Mk, ~0).
9592 SDValue Mp =
9593 DAG.getNode(ISD::CLMUL, DL, VT, Mk, DAG.getAllOnesConstant(DL, VT));
9594 SDValue Mv = DAG.getNode(ISD::AND, DL, VT, Mp, Mc);
9595 MvArray[S] = Mv;
9596 if (S + 1 < LogBW) {
9597 SDValue McXorMv = DAG.getNode(ISD::XOR, DL, VT, Mc, Mv);
9598 SDValue MvShifted = DAG.getNode(
9599 ISD::SRL, DL, VT, Mv, DAG.getShiftAmountConstant(ShiftS, VT, DL));
9600 Mc = DAG.getNode(ISD::OR, DL, VT, McXorMv, MvShifted,
9602 Mk = DAG.getNode(ISD::AND, DL, VT, Mk, DAG.getNOT(DL, Mp, VT));
9603 }
9604 }
9605
9606 // Second pass: move bits by 32, 16, 8, 4, 2, 1, using masks, in parallel.
9607 // Each pass handles half the shift amount of the previous pass.
9608 SDValue X = Val;
9609 for (int S = (int)LogBW - 1; S >= 0; --S) {
9610 SDValue ShiftSv = DAG.getShiftAmountConstant(1ull << S, VT, DL);
9611 SDValue T = DAG.getNode(ISD::SHL, DL, VT, X, ShiftSv);
9612 SDValue UnshiftedBits =
9613 DAG.getNode(ISD::AND, DL, VT, X, DAG.getNOT(DL, MvArray[S], VT));
9614 SDValue ShiftedBits = DAG.getNode(ISD::AND, DL, VT, T, MvArray[S]);
9615 X = DAG.getNode(ISD::OR, DL, VT, UnshiftedBits, ShiftedBits,
9617 }
9618
9619 return DAG.getNode(ISD::AND, DL, VT, X, Msk);
9620}
9621
9623 SelectionDAG &DAG) const {
9624 assert(Node->getNumOperands() == 3 && "Not a double-shift!");
9625 EVT VT = Node->getValueType(0);
9626 unsigned VTBits = VT.getScalarSizeInBits();
9627 assert(isPowerOf2_32(VTBits) && "Power-of-two integer type expected");
9628
9629 bool IsSHL = Node->getOpcode() == ISD::SHL_PARTS;
9630 bool IsSRA = Node->getOpcode() == ISD::SRA_PARTS;
9631 SDValue ShOpLo = Node->getOperand(0);
9632 SDValue ShOpHi = Node->getOperand(1);
9633 SDValue ShAmt = Node->getOperand(2);
9634 EVT ShAmtVT = ShAmt.getValueType();
9635 EVT ShAmtCCVT =
9636 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), ShAmtVT);
9637 SDLoc dl(Node);
9638
9639 // ISD::FSHL and ISD::FSHR have defined overflow behavior but ISD::SHL and
9640 // ISD::SRA/L nodes haven't. Insert an AND to be safe, it's usually optimized
9641 // away during isel.
9642 SDValue SafeShAmt = DAG.getNode(ISD::AND, dl, ShAmtVT, ShAmt,
9643 DAG.getConstant(VTBits - 1, dl, ShAmtVT));
9644 SDValue Tmp1 = IsSRA ? DAG.getNode(ISD::SRA, dl, VT, ShOpHi,
9645 DAG.getConstant(VTBits - 1, dl, ShAmtVT))
9646 : DAG.getConstant(0, dl, VT);
9647
9648 SDValue Tmp2, Tmp3;
9649 if (IsSHL) {
9650 Tmp2 = DAG.getNode(ISD::FSHL, dl, VT, ShOpHi, ShOpLo, ShAmt);
9651 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, SafeShAmt);
9652 } else {
9653 Tmp2 = DAG.getNode(ISD::FSHR, dl, VT, ShOpHi, ShOpLo, ShAmt);
9654 Tmp3 = DAG.getNode(IsSRA ? ISD::SRA : ISD::SRL, dl, VT, ShOpHi, SafeShAmt);
9655 }
9656
9657 // If the shift amount is larger or equal than the width of a part we don't
9658 // use the result from the FSHL/FSHR. Insert a test and select the appropriate
9659 // values for large shift amounts.
9660 SDValue AndNode = DAG.getNode(ISD::AND, dl, ShAmtVT, ShAmt,
9661 DAG.getConstant(VTBits, dl, ShAmtVT));
9662 SDValue Cond = DAG.getSetCC(dl, ShAmtCCVT, AndNode,
9663 DAG.getConstant(0, dl, ShAmtVT), ISD::SETNE);
9664
9665 if (IsSHL) {
9666 Hi = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp3, Tmp2);
9667 Lo = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp1, Tmp3);
9668 } else {
9669 Lo = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp3, Tmp2);
9670 Hi = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp1, Tmp3);
9671 }
9672}
9673
9675 SelectionDAG &DAG) const {
9676 // This implements llvm.canonicalize.f* by multiplication with 1.0, as
9677 // suggested in
9678 // https://llvm.org/docs/LangRef.html#llvm-canonicalize-intrinsic.
9679 // It uses strict_fp operations even outside a strict_fp context in order
9680 // to guarantee that the canonicalization is not optimized away by later
9681 // passes. The result chain introduced by that is intentionally ignored
9682 // since no ordering requirement is intended here.
9683 EVT VT = Node->getValueType(0);
9684 SDLoc DL(Node);
9685 SDNodeFlags Flags = Node->getFlags();
9686 Flags.setNoFPExcept(true);
9687 SDValue One = DAG.getConstantFP(1.0, DL, VT);
9688 SDValue Mul =
9689 DAG.getNode(ISD::STRICT_FMUL, DL, {VT, MVT::Other},
9690 {DAG.getEntryNode(), Node->getOperand(0), One}, Flags);
9691 return Mul;
9692}
9693
9695 SelectionDAG &DAG) const {
9696 // Expand conversion from a native IEEE float type to an arbitrary FP format
9697 // returning the result as an integer using bit manipulation.
9698 EVT ResVT = Node->getValueType(0);
9699 SDLoc dl(Node);
9700
9701 SDValue FloatVal = Node->getOperand(0);
9702 const uint64_t SemEnum = Node->getConstantOperandVal(1);
9703 const auto Sem = static_cast<APFloatBase::Semantics>(SemEnum);
9704 const auto RoundMode =
9705 static_cast<RoundingMode>(Node->getConstantOperandVal(2));
9706 const bool Saturate = Node->getConstantOperandVal(3) != 0;
9707
9708 // Supported destination formats.
9709 switch (Sem) {
9716 break;
9717 default:
9718 DAG.getContext()->emitError("CONVERT_TO_ARBITRARY_FP: not implemented "
9719 "destination format (semantics enum " +
9720 Twine(SemEnum) + ")");
9721 return SDValue();
9722 }
9723
9724 // Supported rounding modes.
9725 switch (RoundMode) {
9731 break;
9732 default:
9733 DAG.getContext()->emitError(
9734 "CONVERT_TO_ARBITRARY_FP: unsupported rounding mode (enum " +
9735 Twine(static_cast<int>(RoundMode)) + ")");
9736 return SDValue();
9737 }
9738
9739 // Destination format parameters.
9740 const fltSemantics &DstSem = APFloatBase::EnumToSemantics(Sem);
9741 const unsigned DstBits = APFloat::getSizeInBits(DstSem);
9742 const unsigned DstPrecision = APFloat::semanticsPrecision(DstSem);
9743 const unsigned DstMant = DstPrecision - 1;
9744 // Unsigned formats spend no bit on the sign.
9745 const bool DstHasSign = APFloat::semanticsHasSignedRepr(DstSem);
9746 const unsigned DstExpBits = DstBits - (DstHasSign ? 1 : 0) - DstMant;
9747 const int DstBias = 1 - APFloat::semanticsMinExponent(DstSem);
9748 const unsigned DstExpMax = (1U << DstExpBits) - 1;
9749 const uint64_t DstMantMask = (DstMant > 0) ? ((1ULL << DstMant) - 1) : 0;
9750 const fltNonfiniteBehavior DstNFBehavior = DstSem.nonFiniteBehavior;
9751 const fltNanEncoding DstNanEnc = DstSem.nanEncoding;
9752
9753 // Compute the maximum normal exponent for the destination format.
9754 const unsigned DstExpMaxNormal =
9755 DstNFBehavior == fltNonfiniteBehavior::IEEE754 ? DstExpMax - 1
9756 : DstExpMax;
9757
9758 // For NanOnly formats the max exponent field for finite values
9759 // is DstExpMax, but the encoding with exp = DstExpMax and
9760 // mant = all-ones is NaN. So DstExpMaxNormal = DstExpMax, but max
9761 // mantissa at that exponent is DstMantMask - 1 (if NanEnc == AllOnes) to
9762 // avoid the NaN encoding.
9763 uint64_t DstMaxMantAtMaxExp = DstMantMask;
9764 if (DstNFBehavior == fltNonfiniteBehavior::NanOnly &&
9765 DstNanEnc == fltNanEncoding::AllOnes)
9766 DstMaxMantAtMaxExp = DstMantMask - 1;
9767
9768 // Source format parameters.
9769 EVT SrcVT = FloatVal.getValueType();
9770 const fltSemantics &SrcSem = SrcVT.getScalarType().getFltSemantics();
9771 const unsigned SrcBits = APFloat::getSizeInBits(SrcSem);
9772 const unsigned SrcPrecision = APFloat::semanticsPrecision(SrcSem);
9773 const unsigned SrcMant = SrcPrecision - 1;
9774 const uint64_t SrcMantMask = (1ULL << SrcMant) - 1;
9775
9776 // Work in the source integer type. Match the destination shape so the
9777 // expansion stays vector when ResVT is a vector.
9778 EVT IntScalarVT = EVT::getIntegerVT(*DAG.getContext(), SrcBits);
9779 EVT IntVT = ResVT.changeElementType(*DAG.getContext(), IntScalarVT);
9780 EVT SetCCVT =
9781 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), IntVT);
9782 EVT FPSetCCVT =
9783 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), SrcVT);
9784
9785 SDValue Zero = DAG.getConstant(0, dl, IntVT);
9786 SDValue One = DAG.getConstant(1, dl, IntVT);
9787
9788 // Bitcast source float to integer to extract the sign bit.
9789 SDValue Src = DAG.getNode(ISD::BITCAST, dl, IntVT, FloatVal);
9790 SDValue SignBit =
9791 DAG.getNode(ISD::SRL, dl, IntVT, Src,
9792 DAG.getShiftAmountConstant(SrcBits - 1, IntVT, dl));
9793
9794 // Classify the input.
9795 SDValue FPZero = DAG.getConstantFP(0.0, dl, SrcVT);
9796 SDValue FPInf = DAG.getConstantFP(APFloat::getInf(SrcSem), dl, SrcVT);
9797 SDValue AbsVal = DAG.getNode(ISD::FABS, dl, SrcVT, FloatVal);
9798 SDValue IsNaN = DAG.getSetCC(dl, FPSetCCVT, FloatVal, FPZero, ISD::SETUO);
9799 SDValue IsInf = DAG.getSetCC(dl, FPSetCCVT, AbsVal, FPInf, ISD::SETOEQ);
9800 SDValue IsZero = DAG.getSetCC(dl, FPSetCCVT, FloatVal, FPZero, ISD::SETOEQ);
9801
9802 // Split into a normalized fraction and unbiased exponent. FFREXP normalizes
9803 // source denormals automatically. The result is unspecified for Inf/NaN, but
9804 // those inputs are detected above and override the final result.
9805 EVT FrexpExpScalarVT =
9807 EVT FrexpExpVT = SrcVT.changeElementType(*DAG.getContext(), FrexpExpScalarVT);
9808 SDValue Frexp =
9809 DAG.getNode(ISD::FFREXP, dl, DAG.getVTList(SrcVT, FrexpExpVT), FloatVal);
9810 SDValue FrexpFrac = Frexp.getValue(0);
9811 SDValue FrexpExp = Frexp.getValue(1);
9812
9813 SDValue FrexpFracInt = DAG.getNode(ISD::BITCAST, dl, IntVT, FrexpFrac);
9814 SDValue EffSrcMant = DAG.getNode(ISD::AND, dl, IntVT, FrexpFracInt,
9815 DAG.getConstant(SrcMantMask, dl, IntVT));
9816
9817 SDValue FrexpExpExt = DAG.getSExtOrTrunc(FrexpExp, dl, IntVT);
9818 SDValue NewExp = DAG.getNode(ISD::ADD, dl, IntVT, FrexpExpExt,
9819 DAG.getConstant(DstBias - 1, dl, IntVT));
9820
9821 // Compute rounding increment given the round bit, sticky bits, and LSB
9822 // of the truncated mantissa.
9823 auto ComputeRoundUp = [&](SDValue RoundBit, SDValue StickyBits,
9824 SDValue LSB) -> SDValue {
9825 switch (RoundMode) {
9827 // Round up if round_bit && (sticky || lsb)
9828 SDValue StickyOrLSB = DAG.getNode(ISD::OR, dl, IntVT, StickyBits, LSB);
9829 return DAG.getNode(ISD::AND, dl, IntVT, RoundBit, StickyOrLSB);
9830 }
9832 return Zero;
9834 // Round up if positive and any truncated bits are set.
9835 SDValue AnyTruncBits =
9836 DAG.getNode(ISD::OR, dl, IntVT, RoundBit, StickyBits);
9837 SDValue HasTruncBits =
9838 DAG.getSetCC(dl, SetCCVT, AnyTruncBits, Zero, ISD::SETNE);
9839 SDValue IsPositive = DAG.getSetCC(dl, SetCCVT, SignBit, Zero, ISD::SETEQ);
9840 SDValue DoRound =
9841 DAG.getNode(ISD::AND, dl, SetCCVT, HasTruncBits, IsPositive);
9842 return DAG.getNode(ISD::ZERO_EXTEND, dl, IntVT, DoRound);
9843 }
9845 // Round up if negative and any truncated bits are set (to -Inf).
9846 SDValue AnyTruncBits =
9847 DAG.getNode(ISD::OR, dl, IntVT, RoundBit, StickyBits);
9848 SDValue HasTruncBits =
9849 DAG.getSetCC(dl, SetCCVT, AnyTruncBits, Zero, ISD::SETNE);
9850 SDValue IsNegative = DAG.getSetCC(dl, SetCCVT, SignBit, Zero, ISD::SETNE);
9851 SDValue DoRound =
9852 DAG.getNode(ISD::AND, dl, SetCCVT, HasTruncBits, IsNegative);
9853 return DAG.getNode(ISD::ZERO_EXTEND, dl, IntVT, DoRound);
9854 }
9856 return RoundBit;
9857 default:
9858 llvm_unreachable("unsupported rounding mode");
9859 }
9860 };
9861
9862 // Round mantissa from SrcMant bits to DstMant bits.
9863 SDValue TruncMant;
9864 SDValue RoundUp;
9865 if (SrcMant > DstMant) {
9866 const unsigned Shift = SrcMant - DstMant;
9867 SDValue ShiftConst = DAG.getShiftAmountConstant(Shift, IntVT, dl);
9868 TruncMant = DAG.getNode(ISD::SRL, dl, IntVT, EffSrcMant, ShiftConst);
9869
9870 // Check bit at position Shift - 1 aka the round bit.
9871 SDValue RoundBit;
9872 if (Shift >= 1) {
9873 SDValue RoundBitShift = DAG.getShiftAmountConstant(Shift - 1, IntVT, dl);
9874 SDValue ShiftedMant =
9875 DAG.getNode(ISD::SRL, dl, IntVT, EffSrcMant, RoundBitShift);
9876 RoundBit = DAG.getNode(ISD::AND, dl, IntVT, ShiftedMant, One);
9877 } else {
9878 RoundBit = Zero;
9879 }
9880
9881 // OR of all bits below the round bit to get sticky bits.
9882 SDValue StickyBits;
9883 if (Shift >= 2) {
9884 uint64_t StickyMask = maskTrailingOnes<uint64_t>(Shift - 1);
9885 StickyBits = DAG.getNode(ISD::AND, dl, IntVT, EffSrcMant,
9886 DAG.getConstant(StickyMask, dl, IntVT));
9887 StickyBits = DAG.getSetCC(dl, SetCCVT, StickyBits, Zero, ISD::SETNE);
9888 StickyBits = DAG.getNode(ISD::ZERO_EXTEND, dl, IntVT, StickyBits);
9889 } else {
9890 StickyBits = Zero;
9891 }
9892
9893 // LSB of truncated mantissa.
9894 SDValue LSB = DAG.getNode(ISD::AND, dl, IntVT, TruncMant, One);
9895
9896 RoundUp = ComputeRoundUp(RoundBit, StickyBits, LSB);
9897 } else {
9898 // If DstMant >= SrcMant, then no rounding needed, just shift left.
9899 SDValue MantShift =
9900 DAG.getShiftAmountConstant(DstMant - SrcMant, IntVT, dl);
9901 TruncMant = DAG.getNode(ISD::SHL, dl, IntVT, EffSrcMant, MantShift);
9902 RoundUp = Zero;
9903 }
9904
9905 // Apply rounding.
9906 SDValue RoundedMant = DAG.getNode(ISD::ADD, dl, IntVT, TruncMant, RoundUp);
9907
9908 // Handle mantissa overflow from rounding.
9909 // If rounded_mant > DstMantMask, carry into exponent.
9910 SDValue MantOverflow =
9911 DAG.getSetCC(dl, SetCCVT, RoundedMant,
9912 DAG.getConstant(DstMantMask, dl, IntVT), ISD::SETGT);
9913 // On overflow: mant = 0, exp += 1.
9914 SDValue AdjMant = DAG.getSelect(dl, IntVT, MantOverflow, Zero, RoundedMant);
9915 SDValue AdjExp =
9916 DAG.getNode(ISD::ADD, dl, IntVT, NewExp,
9917 DAG.getNode(ISD::ZERO_EXTEND, dl, IntVT, MantOverflow));
9918
9919 // Precompute sign shifted to MSB of destination. Unsigned formats have no
9920 // sign bit to merge in.
9921 SDValue SignShifted =
9922 DstHasSign
9923 ? DAG.getNode(ISD::SHL, dl, IntVT, SignBit,
9924 DAG.getShiftAmountConstant(DstBits - 1, IntVT, dl))
9925 : Zero;
9926
9927 // Destination denormal conversion (when new_exp <= 0).
9928 // Shift the mantissa right by 1 - new_exp additional bits and set the
9929 // exponent field to 0.
9930 SDValue ExpIsNeg = DAG.getSetCC(dl, SetCCVT, AdjExp,
9931 DAG.getConstant(1, dl, IntVT), ISD::SETLT);
9932
9933 SDValue DenormResult;
9934 {
9935 // denorm_shift = 1 - NewExp.
9936 SDValue DenormShift = DAG.getNode(ISD::SUB, dl, IntVT, One, NewExp);
9937
9938 // full_src_mant = (1 << SrcMant) | EffSrcMant.
9939 SDValue ImplicitOne =
9940 DAG.getNode(ISD::SHL, dl, IntVT, One,
9941 DAG.getShiftAmountConstant(SrcMant, IntVT, dl));
9942 SDValue FullSrcMant =
9943 DAG.getNode(ISD::OR, dl, IntVT, EffSrcMant, ImplicitOne);
9944
9945 // Total right shift = DenormShift + (SrcMant - DstMant).
9946 int64_t MantDelta = static_cast<int64_t>(SrcMant) - DstMant;
9947 SDValue TotalShift =
9948 DAG.getNode(ISD::ADD, dl, IntVT, DenormShift,
9949 DAG.getSignedConstant(MantDelta, dl, IntVT));
9950
9951 // Clamp total shift to avoid UB, then truncate denorm mantissa.
9952 EVT ShiftVT = getShiftAmountTy(IntVT, DAG.getDataLayout());
9953 SDValue MaxShift = DAG.getConstant(SrcBits - 1, dl, IntVT);
9954 SDValue ClampedShift =
9955 DAG.getNode(ISD::UMIN, dl, IntVT, TotalShift, MaxShift);
9956 SDValue DenormTruncMant =
9957 DAG.getNode(ISD::SRL, dl, IntVT, FullSrcMant,
9958 DAG.getZExtOrTrunc(ClampedShift, dl, ShiftVT));
9959
9960 // Rounding for denorm path.
9961 SDValue DenormRoundUp;
9962 {
9963 // Round bit is at position TotalShift - 1 of FullSrcMant.
9964 // Clamp to at least 1 so the subtraction doesn't underflow and create
9965 // shift nodes with invalid shift amounts.
9966 SDValue SafeShift = DAG.getNode(ISD::UMAX, dl, IntVT, ClampedShift, One);
9967 SDValue RoundBitPos = DAG.getNode(ISD::SUB, dl, IntVT, SafeShift, One);
9968 SDValue RoundBitPosAmt = DAG.getZExtOrTrunc(RoundBitPos, dl, ShiftVT);
9969 SDValue DenormRoundBit = DAG.getNode(
9970 ISD::AND, dl, IntVT,
9971 DAG.getNode(ISD::SRL, dl, IntVT, FullSrcMant, RoundBitPosAmt), One);
9972
9973 // Sticky: all bits below round bit.
9974 // sticky_mask = (1 << RoundBitPos) - 1
9975 SDValue StickyMask = DAG.getNode(
9976 ISD::SUB, dl, IntVT,
9977 DAG.getNode(ISD::SHL, dl, IntVT, One, RoundBitPosAmt), One);
9978 SDValue DenormStickyBits =
9979 DAG.getNode(ISD::AND, dl, IntVT, FullSrcMant, StickyMask);
9980 SDValue HasSticky = DAG.getNode(
9981 ISD::ZERO_EXTEND, dl, IntVT,
9982 DAG.getSetCC(dl, SetCCVT, DenormStickyBits, Zero, ISD::SETNE));
9983
9984 SDValue DenormLSB =
9985 DAG.getNode(ISD::AND, dl, IntVT, DenormTruncMant, One);
9986
9987 DenormRoundUp = ComputeRoundUp(DenormRoundBit, HasSticky, DenormLSB);
9988
9989 // Only apply rounding if TotalShift >= 1 (i.e., there are bits to round).
9990 SDValue ShiftGEOne =
9991 DAG.getSetCC(dl, SetCCVT, ClampedShift, One, ISD::SETUGE);
9992 DenormRoundUp = DAG.getSelect(dl, IntVT, ShiftGEOne, DenormRoundUp, Zero);
9993 }
9994
9995 SDValue DenormRoundedMant =
9996 DAG.getNode(ISD::ADD, dl, IntVT, DenormTruncMant, DenormRoundUp);
9997
9998 // If rounding caused overflow into the normal range, then we get the
9999 // smallest normal number.
10000 SDValue DenormMantOF =
10001 DAG.getSetCC(dl, SetCCVT, DenormRoundedMant,
10002 DAG.getConstant(DstMantMask, dl, IntVT), ISD::SETGT);
10003 SDValue DenormFinalMant =
10004 DAG.getSelect(dl, IntVT, DenormMantOF, Zero, DenormRoundedMant);
10005 SDValue DenormFinalExp = DAG.getSelect(dl, IntVT, DenormMantOF, One, Zero);
10006
10007 // Assemble: sign | (exp << DstMant) | mant
10008 SDValue DenormExpShifted =
10009 DAG.getNode(ISD::SHL, dl, IntVT, DenormFinalExp,
10010 DAG.getShiftAmountConstant(DstMant, IntVT, dl));
10011 DenormResult = DAG.getNode(
10012 ISD::OR, dl, IntVT,
10013 DAG.getNode(ISD::OR, dl, IntVT, SignShifted, DenormExpShifted),
10014 DenormFinalMant);
10015 }
10016
10017 // Exponent overflow detection.
10018 SDValue ExpOF =
10019 DAG.getSetCC(dl, SetCCVT, AdjExp,
10020 DAG.getConstant(DstExpMaxNormal, dl, IntVT), ISD::SETGT);
10021
10022 // Also check if AdjExp == DstExpMaxNormal and mantissa overflow into
10023 // a value that exceeds the max allowed mantissa at that exponent.
10024 SDValue ExpAtMax =
10025 DAG.getSetCC(dl, SetCCVT, AdjExp,
10026 DAG.getConstant(DstExpMaxNormal, dl, IntVT), ISD::SETEQ);
10027 SDValue MantExceedsMax =
10028 DAG.getSetCC(dl, SetCCVT, AdjMant,
10029 DAG.getConstant(DstMaxMantAtMaxExp, dl, IntVT), ISD::SETGT);
10030 SDValue ExpMantOF =
10031 DAG.getNode(ISD::AND, dl, SetCCVT, ExpAtMax, MantExceedsMax);
10032 SDValue IsOverflow = DAG.getNode(ISD::OR, dl, SetCCVT, ExpOF, ExpMantOF);
10033
10034 // Build overflow result.
10036
10037 if (Saturate) {
10038 // Clamp to max finite value:
10039 // sign | (DstExpMaxNormal << DstMant) | DstMaxMantAtMaxExp
10040 uint64_t MaxFinite =
10041 ((uint64_t)DstExpMaxNormal << DstMant) | DstMaxMantAtMaxExp;
10042 OverflowResult = DAG.getNode(ISD::OR, dl, IntVT, SignShifted,
10043 DAG.getConstant(MaxFinite, dl, IntVT));
10044 } else if (DstNFBehavior == fltNonfiniteBehavior::IEEE754) {
10045 // Produce infinity.
10046 uint64_t InfBits = (uint64_t)DstExpMax << DstMant;
10047 OverflowResult = DAG.getNode(ISD::OR, dl, IntVT, SignShifted,
10048 DAG.getConstant(InfBits, dl, IntVT));
10049 } else {
10050 // Emit poison if no Inf in format and not saturating.
10051 OverflowResult = DAG.getPOISON(IntVT);
10052 }
10053
10054 // Assemble normal result: sign | (AdjExp << DstMant) | AdjMant
10055 SDValue NormExpShifted =
10056 DAG.getNode(ISD::SHL, dl, IntVT, AdjExp,
10057 DAG.getShiftAmountConstant(DstMant, IntVT, dl));
10058 SDValue NormResult = DAG.getNode(
10059 ISD::OR, dl, IntVT,
10060 DAG.getNode(ISD::OR, dl, IntVT, SignShifted, NormExpShifted), AdjMant);
10061
10062 // Build special-value results.
10063 SDValue NaNResult;
10064 if (DstNFBehavior == fltNonfiniteBehavior::IEEE754) {
10065 // Produce canonical NaN.
10066 const uint64_t QNaNBit = (DstMant > 0) ? (1ULL << (DstMant - 1)) : 0;
10067 NaNResult =
10068 DAG.getConstant(((uint64_t)DstExpMax << DstMant) | QNaNBit, dl, IntVT);
10069 } else if (DstNFBehavior == fltNonfiniteBehavior::NanOnly &&
10070 DstNanEnc == fltNanEncoding::AllOnes) {
10071 // E4M3FN-style: NaN is exp=all-ones, mant=all-ones.
10072 NaNResult = DAG.getConstant(((uint64_t)DstExpMax << DstMant) | DstMantMask,
10073 dl, IntVT);
10074 } else {
10075 // NaN -> poison for finite only values.
10076 NaNResult = DAG.getPOISON(IntVT);
10077 }
10078
10079 // Inf handling.
10080 SDValue InfResult;
10081 if (DstNFBehavior == fltNonfiniteBehavior::IEEE754) {
10082 // Produce signed infinity.
10083 uint64_t InfBits = (uint64_t)DstExpMax << DstMant;
10084 InfResult = DAG.getNode(ISD::OR, dl, IntVT, SignShifted,
10085 DAG.getConstant(InfBits, dl, IntVT));
10086 } else if (Saturate) {
10087 // Inf saturates to max finite.
10088 uint64_t MaxFinite =
10089 ((uint64_t)DstExpMaxNormal << DstMant) | DstMaxMantAtMaxExp;
10090 InfResult = DAG.getNode(ISD::OR, dl, IntVT, SignShifted,
10091 DAG.getConstant(MaxFinite, dl, IntVT));
10092 } else {
10093 // No Inf and not saturating -> poison.
10094 InfResult = DAG.getPOISON(IntVT);
10095 }
10096
10097 SDValue ZeroResult = SignShifted;
10098
10099 // Final selection in an order: NaN takes priority, then Inf, then Zero.
10100 SDValue FiniteResult =
10101 DAG.getSelect(dl, IntVT, ExpIsNeg, DenormResult, NormResult);
10102 FiniteResult =
10103 DAG.getSelect(dl, IntVT, IsOverflow, OverflowResult, FiniteResult);
10104
10105 SDValue Result = FiniteResult;
10106 Result = DAG.getSelect(dl, IntVT, IsZero, ZeroResult, Result);
10107 Result = DAG.getSelect(dl, IntVT, IsInf, InfResult, Result);
10108
10109 // Negative values are unrepresentable in an unsigned format: clamp to zero
10110 // when saturating, poison otherwise so no select is needed. -0.0 is handled
10111 // by IsZero above. Run before the NaN case so a negative NaN still yields
10112 // NaN.
10113 if (!DstHasSign && Saturate) {
10114 SDValue IsNegative =
10115 DAG.getSetCC(dl, FPSetCCVT, FloatVal, FPZero, ISD::SETOLT);
10116 Result = DAG.getSelect(dl, IntVT, IsNegative, Zero, Result);
10117 }
10118
10119 Result = DAG.getSelect(dl, IntVT, IsNaN, NaNResult, Result);
10120
10121 // Truncate to destination integer type.
10122 return DAG.getZExtOrTrunc(Result, dl, ResVT);
10123}
10124
10125SDValue
10127 SelectionDAG &DAG) const {
10128 SDLoc dl(Node);
10129 EVT DstVT = Node->getValueType(0);
10130 EVT DstScalarVT = DstVT.getScalarType();
10131
10132 SDValue IntVal = Node->getOperand(0);
10133 const uint64_t SemEnum = Node->getConstantOperandVal(1);
10134 const auto Sem = static_cast<APFloatBase::Semantics>(SemEnum);
10135
10136 // Supported source formats.
10137 switch (Sem) {
10144 break;
10145 default:
10146 DAG.getContext()->emitError("CONVERT_FROM_ARBITRARY_FP: not implemented "
10147 "source format (semantics enum " +
10148 Twine(SemEnum) + ")");
10149 return SDValue();
10150 }
10151
10152 const fltSemantics &SrcSem = APFloatBase::EnumToSemantics(Sem);
10153 const unsigned SrcBits = APFloat::getSizeInBits(SrcSem);
10154 const unsigned SrcPrecision = APFloat::semanticsPrecision(SrcSem);
10155 const unsigned SrcMant = SrcPrecision - 1;
10156 // Unsigned formats spend no bit on the sign.
10157 const bool SrcHasSign = APFloat::semanticsHasSignedRepr(SrcSem);
10158 const unsigned SrcExp = SrcBits - (SrcHasSign ? 1 : 0) - SrcMant;
10159 const int SrcBias = 1 - APFloat::semanticsMinExponent(SrcSem);
10160 const fltNonfiniteBehavior NFBehavior = SrcSem.nonFiniteBehavior;
10161
10162 // Destination format parameters.
10163 const fltSemantics &DstSem = DstScalarVT.getFltSemantics();
10164 const unsigned DstBits = APFloat::getSizeInBits(DstSem);
10165 const unsigned DstMant = APFloat::semanticsPrecision(DstSem) - 1;
10166 const unsigned DstExpBits = DstBits - DstMant - 1;
10167 const int DstMinExp = APFloat::semanticsMinExponent(DstSem);
10168 const int DstBias = 1 - DstMinExp;
10169 const uint64_t DstExpAllOnes = (1ULL << DstExpBits) - 1;
10170
10171 // Work in an integer type matching the destination float width.
10172 EVT IntScalarVT = EVT::getIntegerVT(*DAG.getContext(), DstBits);
10173 EVT IntVT = IntScalarVT;
10174 if (DstVT.isVector()) {
10175 IntVT = EVT::getVectorVT(*DAG.getContext(), IntScalarVT,
10176 DstVT.getVectorElementCount());
10177 } else if (!isTypeLegal(IntScalarVT)) {
10178 // Avoid generating illegal type as there is no other places that'll
10179 // legalize it. Vector types don't have this problem because they
10180 // are subject to LegalizeVectorOps and another type legalization phase
10181 // will follow.
10182 if (getTypeAction(*DAG.getContext(), IntScalarVT) != TypePromoteInteger) {
10183 // We only know how to handle situations where the legal type is wider.
10184 DAG.getContext()->emitError(
10185 "CONVERT_FROM_ARBITRARY_FP: the requested integer value type for its "
10186 "legalization is not supported");
10187 return SDValue();
10188 }
10189 IntVT = getTypeToTransformTo(*DAG.getContext(), IntScalarVT);
10190 }
10191
10192 SDValue Src = DAG.getZExtOrTrunc(IntVal, dl, IntVT);
10193
10194 EVT SetCCVT =
10195 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), IntVT);
10196
10197 SDValue Zero = DAG.getConstant(0, dl, IntVT);
10198 SDValue One = DAG.getConstant(1, dl, IntVT);
10199
10200 // Extract bit fields.
10201 const uint64_t MantMask = (SrcMant > 0) ? ((1ULL << SrcMant) - 1) : 0;
10202 const uint64_t ExpMask = (1ULL << SrcExp) - 1;
10203
10204 SDValue MantField = DAG.getNode(ISD::AND, dl, IntVT, Src,
10205 DAG.getConstant(MantMask, dl, IntVT));
10206
10207 SDValue ExpField =
10208 DAG.getNode(ISD::AND, dl, IntVT,
10209 DAG.getNode(ISD::SRL, dl, IntVT, Src,
10210 DAG.getShiftAmountConstant(SrcMant, IntVT, dl)),
10211 DAG.getConstant(ExpMask, dl, IntVT));
10212
10213 // An unsigned source has no sign bit; bit SrcBits - 1 is part of the
10214 // exponent.
10215 SDValue SignShifted =
10216 SrcHasSign
10217 ? DAG.getNode(
10218 ISD::SHL, dl, IntVT,
10219 DAG.getNode(ISD::SRL, dl, IntVT, Src,
10220 DAG.getShiftAmountConstant(SrcBits - 1, IntVT, dl)),
10221 DAG.getShiftAmountConstant(DstBits - 1, IntVT, dl))
10222 : Zero;
10223
10224 // Classify the input.
10225 SDValue ExpAllOnes = DAG.getConstant(ExpMask, dl, IntVT);
10226 SDValue IsExpAllOnes =
10227 DAG.getSetCC(dl, SetCCVT, ExpField, ExpAllOnes, ISD::SETEQ);
10228 SDValue IsExpZero = DAG.getSetCC(dl, SetCCVT, ExpField, Zero, ISD::SETEQ);
10229 SDValue IsMantZero = DAG.getSetCC(dl, SetCCVT, MantField, Zero, ISD::SETEQ);
10230 SDValue IsMantNonZero =
10231 DAG.getSetCC(dl, SetCCVT, MantField, Zero, ISD::SETNE);
10232
10233 SDValue IsNaN;
10234 if (NFBehavior == fltNonfiniteBehavior::FiniteOnly) {
10235 IsNaN = DAG.getBoolConstant(false, dl, SetCCVT, IntVT);
10236 } else if (NFBehavior == fltNonfiniteBehavior::IEEE754) {
10237 IsNaN = DAG.getNode(ISD::AND, dl, SetCCVT, IsExpAllOnes, IsMantNonZero);
10238 } else {
10240 SDValue MantAllOnes = DAG.getConstant(MantMask, dl, IntVT);
10241 SDValue IsMantAllOnes =
10242 DAG.getSetCC(dl, SetCCVT, MantField, MantAllOnes, ISD::SETEQ);
10243 IsNaN = DAG.getNode(ISD::AND, dl, SetCCVT, IsExpAllOnes, IsMantAllOnes);
10244 }
10245
10246 SDValue IsInf;
10247 if (NFBehavior == fltNonfiniteBehavior::IEEE754)
10248 IsInf = DAG.getNode(ISD::AND, dl, SetCCVT, IsExpAllOnes, IsMantZero);
10249 else
10250 IsInf = DAG.getBoolConstant(false, dl, SetCCVT, IntVT);
10251
10252 SDValue IsZero = DAG.getNode(ISD::AND, dl, SetCCVT, IsExpZero, IsMantZero);
10253 SDValue IsDenorm =
10254 DAG.getNode(ISD::AND, dl, SetCCVT, IsExpZero, IsMantNonZero);
10255
10256 // Normal value conversion.
10257 const int BiasAdjust = DstBias - SrcBias;
10258 SDValue NormDstExp = DAG.getNode(
10259 ISD::ADD, dl, IntVT, ExpField,
10260 DAG.getConstant(APInt(IntVT.getScalarSizeInBits(), BiasAdjust, true), dl,
10261 IntVT));
10262
10263 SDValue NormDstMant;
10264 if (DstMant > SrcMant) {
10265 SDValue NormDstMantShift =
10266 DAG.getShiftAmountConstant(DstMant - SrcMant, IntVT, dl);
10267 NormDstMant = DAG.getNode(ISD::SHL, dl, IntVT, MantField, NormDstMantShift);
10268 } else {
10269 NormDstMant = MantField;
10270 }
10271
10272 SDValue DstMantShift = DAG.getShiftAmountConstant(DstMant, IntVT, dl);
10273 SDValue NormExpShifted =
10274 DAG.getNode(ISD::SHL, dl, IntVT, NormDstExp, DstMantShift);
10275 SDValue NormResult =
10276 DAG.getNode(ISD::OR, dl, IntVT,
10277 DAG.getNode(ISD::OR, dl, IntVT, SignShifted, NormExpShifted),
10278 NormDstMant);
10279
10280 // With identical exponent biases, denormal values remain denormal and the
10281 // normal conversion's mantissa shift is sufficient.
10282 SDValue DenormResult = NormResult;
10283 if (BiasAdjust != 0) {
10284 const unsigned IntVTBits = IntVT.getScalarSizeInBits();
10285 SDValue LeadingZeros =
10286 DAG.getNode(ISD::CTLZ_ZERO_POISON, dl, IntVT, MantField);
10287
10288 const int DenormExpConst =
10289 (int)IntVTBits + DstBias - SrcBias - (int)SrcMant;
10290 SDValue DenormDstExp = DAG.getNode(
10291 ISD::SUB, dl, IntVT,
10292 DAG.getConstant(APInt(IntVTBits, DenormExpConst, true), dl, IntVT),
10293 LeadingZeros);
10294
10295 SDValue MantMSB =
10296 DAG.getNode(ISD::SUB, dl, IntVT,
10297 DAG.getConstant(IntVTBits - 1, dl, IntVT), LeadingZeros);
10298
10299 SDValue LeadingOne = DAG.getNode(ISD::SHL, dl, IntVT, One, MantMSB);
10300 SDValue Frac = DAG.getNode(ISD::XOR, dl, IntVT, MantField, LeadingOne);
10301
10302 const unsigned ShiftSub = IntVTBits - 1 - DstMant;
10303 SDValue ShiftAmount = DAG.getNode(ISD::SUB, dl, IntVT, LeadingZeros,
10304 DAG.getConstant(ShiftSub, dl, IntVT));
10305
10306 SDValue DenormDstMant = DAG.getNode(ISD::SHL, dl, IntVT, Frac, ShiftAmount);
10307
10308 SDValue DenormExpShifted =
10309 DAG.getNode(ISD::SHL, dl, IntVT, DenormDstExp, DstMantShift);
10310 DenormResult = DAG.getNode(
10311 ISD::OR, dl, IntVT,
10312 DAG.getNode(ISD::OR, dl, IntVT, SignShifted, DenormExpShifted),
10313 DenormDstMant);
10314 }
10315
10316 SDValue FiniteResult =
10317 DAG.getSelect(dl, IntVT, IsDenorm, DenormResult, NormResult);
10318
10319 const uint64_t QNaNBit = (DstMant > 0) ? (1ULL << (DstMant - 1)) : 0;
10320 SDValue NaNResult =
10321 DAG.getConstant((DstExpAllOnes << DstMant) | QNaNBit, dl, IntVT);
10322
10323 SDValue InfResult =
10324 DAG.getNode(ISD::OR, dl, IntVT, SignShifted,
10325 DAG.getConstant(DstExpAllOnes << DstMant, dl, IntVT));
10326
10327 // A source format may have a larger finite exponent range despite having
10328 // fewer bits, as with Float8E5M3FNU converted to half. Its overflowing finite
10329 // values become infinity. The NaN selection below still takes precedence.
10330 if (APFloat::semanticsMaxExponent(SrcSem) >
10332 SDValue IsOverflow =
10333 DAG.getSetCC(dl, SetCCVT, NormDstExp,
10334 DAG.getConstant(DstExpAllOnes, dl, IntVT), ISD::SETUGE);
10335 FiniteResult =
10336 DAG.getSelect(dl, IntVT, IsOverflow, InfResult, FiniteResult);
10337 }
10338
10339 SDValue ZeroResult = SignShifted;
10340
10341 SDValue Result = FiniteResult;
10342 Result = DAG.getSelect(dl, IntVT, IsZero, ZeroResult, Result);
10343 Result = DAG.getSelect(dl, IntVT, IsInf, InfResult, Result);
10344 Result = DAG.getSelect(dl, IntVT, IsNaN, NaNResult, Result);
10345
10346 if (!DstVT.bitsEq(IntVT)) {
10347 // Store to stack before loading it back.
10348 assert(!IntVT.isVector() && IntVT.bitsGT(DstVT));
10349 // IntScalarVT is the original type that has the same width as DstVT.
10350 Align Alignment = DAG.getReducedAlign(IntScalarVT, /*UseABI=*/false);
10351 SDValue StackPtr =
10352 DAG.CreateStackTemporary(IntScalarVT.getStoreSize(), Alignment);
10353 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
10355 MachinePointerInfo PtrInfo =
10356 MachinePointerInfo::getFixedStack(MF, FrameIndex);
10357 SDValue Store = DAG.getTruncStore(DAG.getEntryNode(), dl, Result, StackPtr,
10358 PtrInfo, IntScalarVT, Alignment);
10359
10360 SDValue Load = DAG.getLoad(DstVT, dl, Store, StackPtr, PtrInfo, Alignment);
10361 return DAG.getMergeValues({Load, Load.getValue(1)}, dl);
10362 }
10363
10364 return DAG.getNode(ISD::BITCAST, dl, DstVT, Result);
10365}
10366
10368 SelectionDAG &DAG) const {
10369 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
10370 SDValue Src = Node->getOperand(OpNo);
10371 EVT SrcVT = Src.getValueType();
10372 EVT DstVT = Node->getValueType(0);
10373 SDLoc dl(SDValue(Node, 0));
10374
10375 // FIXME: Only f32 to i64 conversions are supported.
10376 if (SrcVT != MVT::f32 || DstVT != MVT::i64)
10377 return false;
10378
10379 if (Node->isStrictFPOpcode())
10380 // When a NaN is converted to an integer a trap is allowed. We can't
10381 // use this expansion here because it would eliminate that trap. Other
10382 // traps are also allowed and cannot be eliminated. See
10383 // IEEE 754-2008 sec 5.8.
10384 return false;
10385
10386 // Expand f32 -> i64 conversion
10387 // This algorithm comes from compiler-rt's implementation of fixsfdi:
10388 // https://github.com/llvm/llvm-project/blob/main/compiler-rt/lib/builtins/fixsfdi.c
10389 unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
10390 EVT IntVT = SrcVT.changeTypeToInteger();
10391 EVT IntShVT = getShiftAmountTy(IntVT, DAG.getDataLayout());
10392
10393 SDValue ExponentMask = DAG.getConstant(0x7F800000, dl, IntVT);
10394 SDValue ExponentLoBit = DAG.getConstant(23, dl, IntVT);
10395 SDValue Bias = DAG.getConstant(127, dl, IntVT);
10396 SDValue SignMask = DAG.getConstant(APInt::getSignMask(SrcEltBits), dl, IntVT);
10397 SDValue SignLowBit = DAG.getConstant(SrcEltBits - 1, dl, IntVT);
10398 SDValue MantissaMask = DAG.getConstant(0x007FFFFF, dl, IntVT);
10399
10400 SDValue Bits = DAG.getNode(ISD::BITCAST, dl, IntVT, Src);
10401
10402 SDValue ExponentBits = DAG.getNode(
10403 ISD::SRL, dl, IntVT, DAG.getNode(ISD::AND, dl, IntVT, Bits, ExponentMask),
10404 DAG.getZExtOrTrunc(ExponentLoBit, dl, IntShVT));
10405 SDValue Exponent = DAG.getNode(ISD::SUB, dl, IntVT, ExponentBits, Bias);
10406
10407 SDValue Sign = DAG.getNode(ISD::SRA, dl, IntVT,
10408 DAG.getNode(ISD::AND, dl, IntVT, Bits, SignMask),
10409 DAG.getZExtOrTrunc(SignLowBit, dl, IntShVT));
10410 Sign = DAG.getSExtOrTrunc(Sign, dl, DstVT);
10411
10412 SDValue R = DAG.getNode(ISD::OR, dl, IntVT,
10413 DAG.getNode(ISD::AND, dl, IntVT, Bits, MantissaMask),
10414 DAG.getConstant(0x00800000, dl, IntVT));
10415
10416 R = DAG.getZExtOrTrunc(R, dl, DstVT);
10417
10418 R = DAG.getSelectCC(
10419 dl, Exponent, ExponentLoBit,
10420 DAG.getNode(ISD::SHL, dl, DstVT, R,
10421 DAG.getZExtOrTrunc(
10422 DAG.getNode(ISD::SUB, dl, IntVT, Exponent, ExponentLoBit),
10423 dl, IntShVT)),
10424 DAG.getNode(ISD::SRL, dl, DstVT, R,
10425 DAG.getZExtOrTrunc(
10426 DAG.getNode(ISD::SUB, dl, IntVT, ExponentLoBit, Exponent),
10427 dl, IntShVT)),
10428 ISD::SETGT);
10429
10430 SDValue Ret = DAG.getNode(ISD::SUB, dl, DstVT,
10431 DAG.getNode(ISD::XOR, dl, DstVT, R, Sign), Sign);
10432
10433 Result = DAG.getSelectCC(dl, Exponent, DAG.getConstant(0, dl, IntVT),
10434 DAG.getConstant(0, dl, DstVT), Ret, ISD::SETLT);
10435 return true;
10436}
10437
10439 SDValue &Chain,
10440 SelectionDAG &DAG) const {
10441 SDLoc dl(SDValue(Node, 0));
10442 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
10443 SDValue Src = Node->getOperand(OpNo);
10444
10445 EVT SrcVT = Src.getValueType();
10446 EVT DstVT = Node->getValueType(0);
10447 EVT SetCCVT =
10448 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), SrcVT);
10449 EVT DstSetCCVT =
10450 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), DstVT);
10451
10452 // Only expand vector types if we have the appropriate vector bit operations.
10453 unsigned SIntOpcode = Node->isStrictFPOpcode() ? ISD::STRICT_FP_TO_SINT :
10455 if (DstVT.isVector() && (!isOperationLegalOrCustom(SIntOpcode, DstVT) ||
10457 return false;
10458
10459 // If the maximum float value is smaller then the signed integer range,
10460 // the destination signmask can't be represented by the float, so we can
10461 // just use FP_TO_SINT directly.
10462 const fltSemantics &APFSem = SrcVT.getFltSemantics();
10463 APFloat APF(APFSem, APInt::getZero(SrcVT.getScalarSizeInBits()));
10464 APInt SignMask = APInt::getSignMask(DstVT.getScalarSizeInBits());
10466 APF.convertFromAPInt(SignMask, false, APFloat::rmNearestTiesToEven)) {
10467 if (Node->isStrictFPOpcode()) {
10468 Result = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, { DstVT, MVT::Other },
10469 { Node->getOperand(0), Src });
10470 Chain = Result.getValue(1);
10471 } else
10472 Result = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Src);
10473 return true;
10474 }
10475
10476 // Don't expand it if there isn't cheap fsub instruction.
10478 Node->isStrictFPOpcode() ? ISD::STRICT_FSUB : ISD::FSUB, SrcVT))
10479 return false;
10480
10481 SDValue Cst = DAG.getConstantFP(APF, dl, SrcVT);
10482 SDValue Sel;
10483
10484 if (Node->isStrictFPOpcode()) {
10485 Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT,
10486 Node->getOperand(0), /*IsSignaling*/ true);
10487 Chain = Sel.getValue(1);
10488 } else {
10489 Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT);
10490 }
10491
10492 bool Strict = Node->isStrictFPOpcode() ||
10493 shouldUseStrictFP_TO_INT(SrcVT, DstVT, /*IsSigned*/ false);
10494
10495 if (Strict) {
10496 // Expand based on maximum range of FP_TO_SINT, if the value exceeds the
10497 // signmask then offset (the result of which should be fully representable).
10498 // Sel = Src < 0x8000000000000000
10499 // FltOfs = select Sel, 0, 0x8000000000000000
10500 // IntOfs = select Sel, 0, 0x8000000000000000
10501 // Result = fp_to_sint(Src - FltOfs) ^ IntOfs
10502
10503 // TODO: Should any fast-math-flags be set for the FSUB?
10504 SDValue FltOfs = DAG.getSelect(dl, SrcVT, Sel,
10505 DAG.getConstantFP(0.0, dl, SrcVT), Cst);
10506 Sel = DAG.getBoolExtOrTrunc(Sel, dl, DstSetCCVT, DstVT);
10507 SDValue IntOfs = DAG.getSelect(dl, DstVT, Sel,
10508 DAG.getConstant(0, dl, DstVT),
10509 DAG.getConstant(SignMask, dl, DstVT));
10510 SDValue SInt;
10511 if (Node->isStrictFPOpcode()) {
10512 SDValue Val = DAG.getNode(ISD::STRICT_FSUB, dl, { SrcVT, MVT::Other },
10513 { Chain, Src, FltOfs });
10514 SInt = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, { DstVT, MVT::Other },
10515 { Val.getValue(1), Val });
10516 Chain = SInt.getValue(1);
10517 } else {
10518 SDValue Val = DAG.getNode(ISD::FSUB, dl, SrcVT, Src, FltOfs);
10519 SInt = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Val);
10520 }
10521 Result = DAG.getNode(ISD::XOR, dl, DstVT, SInt, IntOfs);
10522 } else {
10523 // Expand based on maximum range of FP_TO_SINT:
10524 // True = fp_to_sint(Src)
10525 // False = 0x8000000000000000 + fp_to_sint(Src - 0x8000000000000000)
10526 // Result = select (Src < 0x8000000000000000), True, False
10527
10528 SDValue True = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Src);
10529 // TODO: Should any fast-math-flags be set for the FSUB?
10530 SDValue False = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT,
10531 DAG.getNode(ISD::FSUB, dl, SrcVT, Src, Cst));
10532 False = DAG.getNode(ISD::XOR, dl, DstVT, False,
10533 DAG.getConstant(SignMask, dl, DstVT));
10534 Sel = DAG.getBoolExtOrTrunc(Sel, dl, DstSetCCVT, DstVT);
10535 Result = DAG.getSelect(dl, DstVT, Sel, True, False);
10536 }
10537 return true;
10538}
10539
10541 SDValue &Chain, SelectionDAG &DAG) const {
10542 // This transform is not correct for converting 0 when rounding mode is set
10543 // to round toward negative infinity which will produce -0.0. So disable
10544 // under strictfp.
10545 if (Node->isStrictFPOpcode())
10546 return false;
10547
10548 SDValue Src = Node->getOperand(0);
10549 EVT SrcVT = Src.getValueType();
10550 EVT DstVT = Node->getValueType(0);
10551
10552 // If the input is known to be non-negative and SINT_TO_FP is legal then use
10553 // it.
10554 if (Node->getFlags().hasNonNeg() &&
10556 Result =
10557 DAG.getNode(ISD::SINT_TO_FP, SDLoc(Node), DstVT, Node->getOperand(0));
10558 return true;
10559 }
10560
10561 if (SrcVT.getScalarType() != MVT::i64 || DstVT.getScalarType() != MVT::f64)
10562 return false;
10563
10564 // Only expand vector types if we have the appropriate vector bit
10565 // operations.
10566 if (SrcVT.isVector() && (!isOperationLegalOrCustom(ISD::SRL, SrcVT) ||
10571 return false;
10572
10573 SDLoc dl(SDValue(Node, 0));
10574
10575 // Implementation of unsigned i64 to f64 following the algorithm in
10576 // __floatundidf in compiler_rt. This implementation performs rounding
10577 // correctly in all rounding modes with the exception of converting 0
10578 // when rounding toward negative infinity. In that case the fsub will
10579 // produce -0.0. This will be added to +0.0 and produce -0.0 which is
10580 // incorrect.
10581 SDValue TwoP52 = DAG.getConstant(UINT64_C(0x4330000000000000), dl, SrcVT);
10582 SDValue TwoP84PlusTwoP52 = DAG.getConstantFP(
10583 llvm::bit_cast<double>(UINT64_C(0x4530000000100000)), dl, DstVT);
10584 SDValue TwoP84 = DAG.getConstant(UINT64_C(0x4530000000000000), dl, SrcVT);
10585 SDValue LoMask = DAG.getConstant(UINT64_C(0x00000000FFFFFFFF), dl, SrcVT);
10586 SDValue HiShift = DAG.getShiftAmountConstant(32, SrcVT, dl);
10587
10588 SDValue Lo = DAG.getNode(ISD::AND, dl, SrcVT, Src, LoMask);
10589 SDValue Hi = DAG.getNode(ISD::SRL, dl, SrcVT, Src, HiShift);
10590 SDValue LoOr = DAG.getNode(ISD::OR, dl, SrcVT, Lo, TwoP52);
10591 SDValue HiOr = DAG.getNode(ISD::OR, dl, SrcVT, Hi, TwoP84);
10592 SDValue LoFlt = DAG.getBitcast(DstVT, LoOr);
10593 SDValue HiFlt = DAG.getBitcast(DstVT, HiOr);
10594 SDValue HiSub = DAG.getNode(ISD::FSUB, dl, DstVT, HiFlt, TwoP84PlusTwoP52);
10595 Result = DAG.getNode(ISD::FADD, dl, DstVT, LoFlt, HiSub);
10596 return true;
10597}
10598
10599SDValue
10601 SelectionDAG &DAG) const {
10602 unsigned Opcode = Node->getOpcode();
10603 assert((Opcode == ISD::FMINNUM || Opcode == ISD::FMAXNUM ||
10604 Opcode == ISD::STRICT_FMINNUM || Opcode == ISD::STRICT_FMAXNUM) &&
10605 "Wrong opcode");
10606
10607 if (Node->getFlags().hasNoNaNs()) {
10608 ISD::CondCode Pred = Opcode == ISD::FMINNUM ? ISD::SETLT : ISD::SETGT;
10609 EVT VT = Node->getValueType(0);
10610 if ((!isCondCodeLegal(Pred, VT.getSimpleVT()) ||
10612 VT.isVector())
10613 return SDValue();
10614 SDValue Op1 = Node->getOperand(0);
10615 SDValue Op2 = Node->getOperand(1);
10616 return DAG.getSelectCC(SDLoc(Node), Op1, Op2, Op1, Op2, Pred,
10617 Node->getFlags());
10618 }
10619
10620 return SDValue();
10621}
10622
10624 SelectionDAG &DAG) const {
10625 if (SDValue Expanded = expandVectorNaryOpBySplitting(Node, DAG))
10626 return Expanded;
10627
10628 EVT VT = Node->getValueType(0);
10629 if (VT.isScalableVector())
10631 "Expanding fminnum/fmaxnum for scalable vectors is undefined.");
10632
10633 SDLoc dl(Node);
10634 unsigned NewOp =
10636
10637 if (isOperationLegalOrCustom(NewOp, VT)) {
10638 SDValue Quiet0 = Node->getOperand(0);
10639 SDValue Quiet1 = Node->getOperand(1);
10640
10641 if (!Node->getFlags().hasNoNaNs()) {
10642 // Insert canonicalizes if it's possible we need to quiet to get correct
10643 // sNaN behavior.
10644 if (!DAG.isKnownNeverSNaN(Quiet0)) {
10645 Quiet0 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet0,
10646 Node->getFlags());
10647 }
10648 if (!DAG.isKnownNeverSNaN(Quiet1)) {
10649 Quiet1 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet1,
10650 Node->getFlags());
10651 }
10652 }
10653
10654 return DAG.getNode(NewOp, dl, VT, Quiet0, Quiet1, Node->getFlags());
10655 }
10656
10657 // If the target has FMINIMUM/FMAXIMUM but not FMINNUM/FMAXNUM use that
10658 // instead if there are no NaNs.
10659 if (Node->getFlags().hasNoNaNs() ||
10660 (DAG.isKnownNeverNaN(Node->getOperand(0)) &&
10661 DAG.isKnownNeverNaN(Node->getOperand(1)))) {
10662 unsigned IEEE2018Op =
10663 Node->getOpcode() == ISD::FMINNUM ? ISD::FMINIMUM : ISD::FMAXIMUM;
10664 if (isOperationLegalOrCustom(IEEE2018Op, VT))
10665 return DAG.getNode(IEEE2018Op, dl, VT, Node->getOperand(0),
10666 Node->getOperand(1), Node->getFlags());
10667 }
10668
10670 return SelCC;
10671
10672 return SDValue();
10673}
10674
10676 const TargetLowering &TLI,
10677 const SDLoc &DL, SDValue Val,
10678 FPClassTest FPClass) {
10679 EVT VT = Val.getValueType();
10680 EVT CCVT = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
10681 EVT IntVT = VT.changeTypeToInteger();
10682 EVT FloatVT = VT.changeElementType(*DAG.getContext(), MVT::f32);
10683 SDValue TestZero = DAG.getTargetConstant(FPClass, DL, MVT::i32);
10684 if (!TLI.isTypeLegal(IntVT) &&
10685 !TLI.isOperationLegalOrCustom(ISD::IS_FPCLASS, VT))
10686 Val = DAG.getNode(ISD::FP_ROUND, DL, FloatVT, Val,
10687 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true));
10688 return DAG.getNode(ISD::IS_FPCLASS, DL, CCVT, Val, TestZero);
10689}
10690
10692 SelectionDAG &DAG) const {
10693 if (SDValue Expanded = expandVectorNaryOpBySplitting(N, DAG))
10694 return Expanded;
10695
10696 SDLoc DL(N);
10697 SDValue LHS = N->getOperand(0);
10698 SDValue RHS = N->getOperand(1);
10699 unsigned Opc = N->getOpcode();
10700 EVT VT = N->getValueType(0);
10701 EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
10702 bool IsMax = Opc == ISD::FMAXIMUM;
10703 SDNodeFlags Flags = N->getFlags();
10704
10705 // First, implement comparison not propagating NaN. If no native fmin or fmax
10706 // available, use plain select with setcc instead.
10708 unsigned CompOpcIeee = IsMax ? ISD::FMAXNUM_IEEE : ISD::FMINNUM_IEEE;
10709 unsigned CompOpc = IsMax ? ISD::FMAXNUM : ISD::FMINNUM;
10710
10711 // FIXME: We should probably define fminnum/fmaxnum variants with correct
10712 // signed zero behavior.
10713 bool MinMaxMustRespectOrderedZero = false;
10714
10715 if (isOperationLegalOrCustom(CompOpcIeee, VT)) {
10716 MinMax = DAG.getNode(CompOpcIeee, DL, VT, LHS, RHS, Flags);
10717 MinMaxMustRespectOrderedZero = true;
10718 } else if (isOperationLegalOrCustom(CompOpc, VT)) {
10719 MinMax = DAG.getNode(CompOpc, DL, VT, LHS, RHS, Flags);
10720 } else {
10722 return DAG.UnrollVectorOp(N);
10723
10724 // NaN (if exists) will be propagated later, so orderness doesn't matter.
10725 SDValue Compare =
10726 DAG.getSetCC(DL, CCVT, LHS, RHS, IsMax ? ISD::SETOGT : ISD::SETOLT);
10727 MinMax = DAG.getSelect(DL, VT, Compare, LHS, RHS, Flags);
10728 }
10729
10730 // Propagate any NaN of both operands
10731 if (!N->getFlags().hasNoNaNs() &&
10732 (!DAG.isKnownNeverNaN(RHS) || !DAG.isKnownNeverNaN(LHS))) {
10733 ConstantFP *FPNaN = ConstantFP::get(*DAG.getContext(),
10735 MinMax = DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, LHS, RHS, ISD::SETUO),
10736 DAG.getConstantFP(*FPNaN, DL, VT), MinMax, Flags);
10737 }
10738
10739 // fminimum/fmaximum requires -0.0 less than +0.0
10740 if (!MinMaxMustRespectOrderedZero && !N->getFlags().hasNoSignedZeros() &&
10741 !DAG.isKnownNeverLogicalZero(RHS) && !DAG.isKnownNeverLogicalZero(LHS)) {
10742 SDValue IsEqual = DAG.getSetCC(DL, CCVT, LHS, RHS, ISD::SETOEQ);
10744 DAG, *this, DL, LHS, IsMax ? fcPosZero : fcNegZero);
10745 SDValue RetZero = DAG.getSelect(DL, VT, IsSpecificZero, LHS, RHS, Flags);
10746 MinMax = DAG.getSelect(DL, VT, IsEqual, RetZero, MinMax, Flags);
10747 }
10748
10749 return MinMax;
10750}
10751
10753 SelectionDAG &DAG) const {
10754 SDLoc DL(Node);
10755 SDValue LHS = Node->getOperand(0);
10756 SDValue RHS = Node->getOperand(1);
10757 unsigned Opc = Node->getOpcode();
10758 EVT VT = Node->getValueType(0);
10759 EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
10760 bool IsMax = Opc == ISD::FMAXIMUMNUM;
10761 SDNodeFlags Flags = Node->getFlags();
10762
10763 unsigned NewOp =
10765
10766 if (isOperationLegalOrCustom(NewOp, VT)) {
10767 if (!Flags.hasNoNaNs()) {
10768 // Insert canonicalizes if it's possible we need to quiet to get correct
10769 // sNaN behavior.
10770 if (!DAG.isKnownNeverSNaN(LHS)) {
10771 LHS = DAG.getNode(ISD::FCANONICALIZE, DL, VT, LHS, Flags);
10772 }
10773 if (!DAG.isKnownNeverSNaN(RHS)) {
10774 RHS = DAG.getNode(ISD::FCANONICALIZE, DL, VT, RHS, Flags);
10775 }
10776 }
10777
10778 return DAG.getNode(NewOp, DL, VT, LHS, RHS, Flags);
10779 }
10780
10781 // We can use FMINIMUM/FMAXIMUM if there is no NaN, since it has
10782 // same behaviors for all of other cases: +0.0 vs -0.0 included.
10783 if (Flags.hasNoNaNs() ||
10784 (DAG.isKnownNeverNaN(LHS) && DAG.isKnownNeverNaN(RHS))) {
10785 unsigned IEEE2019Op =
10787 if (isOperationLegalOrCustom(IEEE2019Op, VT))
10788 return DAG.getNode(IEEE2019Op, DL, VT, LHS, RHS, Flags);
10789 }
10790
10791 // FMINNUM/FMAXMUM returns qNaN if either operand is sNaN, and it may return
10792 // either one for +0.0 vs -0.0.
10793 if ((Flags.hasNoNaNs() ||
10794 (DAG.isKnownNeverSNaN(LHS) && DAG.isKnownNeverSNaN(RHS))) &&
10795 (Flags.hasNoSignedZeros() || DAG.isKnownNeverLogicalZero(LHS) ||
10796 DAG.isKnownNeverLogicalZero(RHS))) {
10797 unsigned IEEE2008Op = Opc == ISD::FMINIMUMNUM ? ISD::FMINNUM : ISD::FMAXNUM;
10798 if (isOperationLegalOrCustom(IEEE2008Op, VT))
10799 return DAG.getNode(IEEE2008Op, DL, VT, LHS, RHS, Flags);
10800 }
10801
10802 if (VT.isVector() &&
10805 return DAG.UnrollVectorOp(Node);
10806
10807 // If only one operand is NaN, override it with another operand.
10808 if (!Flags.hasNoNaNs() && !DAG.isKnownNeverNaN(LHS)) {
10809 LHS = DAG.getSelectCC(DL, LHS, LHS, RHS, LHS, ISD::SETUO);
10810 }
10811 if (!Flags.hasNoNaNs() && !DAG.isKnownNeverNaN(RHS)) {
10812 RHS = DAG.getSelectCC(DL, RHS, RHS, LHS, RHS, ISD::SETUO);
10813 }
10814
10815 // Always prefer RHS if equal.
10816 SDValue MinMax =
10817 DAG.getSelectCC(DL, LHS, RHS, LHS, RHS, IsMax ? ISD::SETGT : ISD::SETLT);
10818
10819 // TODO: We need quiet sNaN if strictfp.
10820
10821 // Fixup signed zero behavior.
10822 if (Flags.hasNoSignedZeros() || DAG.isKnownNeverLogicalZero(LHS) ||
10823 DAG.isKnownNeverLogicalZero(RHS)) {
10824 return MinMax;
10825 }
10826 SDValue IsZero = DAG.getSetCC(DL, CCVT, MinMax,
10827 DAG.getConstantFP(0.0, DL, VT), ISD::SETEQ);
10829 DAG, *this, DL, LHS, IsMax ? fcPosZero : fcNegZero);
10830 // It's OK to select from LHS and MinMax, with only one ISD::IS_FPCLASS, as
10831 // we preferred RHS when generate MinMax, if the operands are equal.
10832 SDValue RetZero = DAG.getSelect(DL, VT, IsSpecificZero, LHS, MinMax, Flags);
10833 return DAG.getSelect(DL, VT, IsZero, RetZero, MinMax, Flags);
10834}
10835
10836/// Returns a true value if if this FPClassTest can be performed with an ordered
10837/// fcmp to 0, and a false value if it's an unordered fcmp to 0. Returns
10838/// std::nullopt if it cannot be performed as a compare with 0.
10839static std::optional<bool> isFCmpEqualZero(FPClassTest Test,
10840 const fltSemantics &Semantics,
10841 const MachineFunction &MF) {
10842 FPClassTest OrderedMask = Test & ~fcNan;
10843 FPClassTest NanTest = Test & fcNan;
10844 bool IsOrdered = NanTest == fcNone;
10845 bool IsUnordered = NanTest == fcNan;
10846
10847 // Skip cases that are testing for only a qnan or snan.
10848 if (!IsOrdered && !IsUnordered)
10849 return std::nullopt;
10850
10851 if (OrderedMask == fcZero &&
10852 MF.getDenormalMode(Semantics).Input == DenormalMode::IEEE)
10853 return IsOrdered;
10854 if (OrderedMask == (fcZero | fcSubnormal) &&
10855 MF.getDenormalMode(Semantics).inputsAreZero())
10856 return IsOrdered;
10857 return std::nullopt;
10858}
10859
10861 const FPClassTest OrigTestMask,
10862 SDNodeFlags Flags, const SDLoc &DL,
10863 SelectionDAG &DAG) const {
10864 EVT OperandVT = Op.getValueType();
10865 assert(OperandVT.isFloatingPoint());
10866 FPClassTest Test = OrigTestMask;
10867
10868 // Degenerated cases.
10869 if (Test == fcNone)
10870 return DAG.getBoolConstant(false, DL, ResultVT, OperandVT);
10871 if (Test == fcAllFlags)
10872 return DAG.getBoolConstant(true, DL, ResultVT, OperandVT);
10873
10874 // PPC double double is a pair of doubles, of which the higher part determines
10875 // the value class.
10876 if (OperandVT == MVT::ppcf128) {
10877 Op = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::f64, Op,
10878 DAG.getConstant(1, DL, MVT::i32));
10879 OperandVT = MVT::f64;
10880 }
10881
10882 // Floating-point type properties.
10883 EVT ScalarFloatVT = OperandVT.getScalarType();
10884 const Type *FloatTy = ScalarFloatVT.getTypeForEVT(*DAG.getContext());
10885 const llvm::fltSemantics &Semantics = FloatTy->getFltSemantics();
10886 bool IsF80 = (ScalarFloatVT == MVT::f80);
10887
10888 // Some checks can be implemented using float comparisons, if floating point
10889 // exceptions are ignored.
10890 if (Flags.hasNoFPExcept() &&
10892 FPClassTest FPTestMask = Test;
10893 bool IsInvertedFP = false;
10894
10895 if (FPClassTest InvertedFPCheck =
10896 invertFPClassTestIfSimpler(FPTestMask, true)) {
10897 FPTestMask = InvertedFPCheck;
10898 IsInvertedFP = true;
10899 }
10900
10901 ISD::CondCode OrderedCmpOpcode = IsInvertedFP ? ISD::SETUNE : ISD::SETOEQ;
10902 ISD::CondCode UnorderedCmpOpcode = IsInvertedFP ? ISD::SETONE : ISD::SETUEQ;
10903
10904 // See if we can fold an | fcNan into an unordered compare.
10905 FPClassTest OrderedFPTestMask = FPTestMask & ~fcNan;
10906
10907 // Can't fold the ordered check if we're only testing for snan or qnan
10908 // individually.
10909 if ((FPTestMask & fcNan) != fcNan)
10910 OrderedFPTestMask = FPTestMask;
10911
10912 const bool IsOrdered = FPTestMask == OrderedFPTestMask;
10913
10914 if (std::optional<bool> IsCmp0 =
10915 isFCmpEqualZero(FPTestMask, Semantics, DAG.getMachineFunction());
10916 IsCmp0 && (isCondCodeLegalOrCustom(
10917 *IsCmp0 ? OrderedCmpOpcode : UnorderedCmpOpcode,
10918 OperandVT.getScalarType().getSimpleVT()))) {
10919
10920 // If denormals could be implicitly treated as 0, this is not equivalent
10921 // to a compare with 0 since it will also be true for denormals.
10922 return DAG.getSetCC(DL, ResultVT, Op,
10923 DAG.getConstantFP(0.0, DL, OperandVT),
10924 *IsCmp0 ? OrderedCmpOpcode : UnorderedCmpOpcode);
10925 }
10926
10927 if (FPTestMask == fcNan &&
10929 OperandVT.getScalarType().getSimpleVT()))
10930 return DAG.getSetCC(DL, ResultVT, Op, Op,
10931 IsInvertedFP ? ISD::SETO : ISD::SETUO);
10932
10933 bool IsOrderedInf = FPTestMask == fcInf;
10934 if ((FPTestMask == fcInf || FPTestMask == (fcInf | fcNan)) &&
10935 isCondCodeLegalOrCustom(IsOrderedInf ? OrderedCmpOpcode
10936 : UnorderedCmpOpcode,
10937 OperandVT.getScalarType().getSimpleVT()) &&
10940 (OperandVT.isVector() &&
10942 // isinf(x) --> fabs(x) == inf
10943 SDValue Abs = DAG.getNode(ISD::FABS, DL, OperandVT, Op);
10944 SDValue Inf =
10945 DAG.getConstantFP(APFloat::getInf(Semantics), DL, OperandVT);
10946 return DAG.getSetCC(DL, ResultVT, Abs, Inf,
10947 IsOrderedInf ? OrderedCmpOpcode : UnorderedCmpOpcode);
10948 }
10949
10950 if ((OrderedFPTestMask == fcPosInf || OrderedFPTestMask == fcNegInf) &&
10951 isCondCodeLegalOrCustom(IsOrdered ? OrderedCmpOpcode
10952 : UnorderedCmpOpcode,
10953 OperandVT.getSimpleVT())) {
10954 // isposinf(x) --> x == inf
10955 // isneginf(x) --> x == -inf
10956 // isposinf(x) || nan --> x u== inf
10957 // isneginf(x) || nan --> x u== -inf
10958
10959 SDValue Inf = DAG.getConstantFP(
10960 APFloat::getInf(Semantics, OrderedFPTestMask == fcNegInf), DL,
10961 OperandVT);
10962 return DAG.getSetCC(DL, ResultVT, Op, Inf,
10963 IsOrdered ? OrderedCmpOpcode : UnorderedCmpOpcode);
10964 }
10965
10966 if (OrderedFPTestMask == (fcSubnormal | fcZero) && !IsOrdered) {
10967 // TODO: Could handle ordered case, but it produces worse code for
10968 // x86. Maybe handle ordered if fabs is free?
10969
10970 ISD::CondCode OrderedOp = IsInvertedFP ? ISD::SETUGE : ISD::SETOLT;
10971 ISD::CondCode UnorderedOp = IsInvertedFP ? ISD::SETOGE : ISD::SETULT;
10972
10973 if (isCondCodeLegalOrCustom(IsOrdered ? OrderedOp : UnorderedOp,
10974 OperandVT.getScalarType().getSimpleVT())) {
10975 // (issubnormal(x) || iszero(x)) --> fabs(x) < smallest_normal
10976
10977 // TODO: Maybe only makes sense if fabs is free. Integer test of
10978 // exponent bits seems better for x86.
10979 SDValue Abs = DAG.getNode(ISD::FABS, DL, OperandVT, Op);
10980 SDValue SmallestNormal = DAG.getConstantFP(
10981 APFloat::getSmallestNormalized(Semantics), DL, OperandVT);
10982 return DAG.getSetCC(DL, ResultVT, Abs, SmallestNormal,
10983 IsOrdered ? OrderedOp : UnorderedOp);
10984 }
10985 }
10986
10987 if (FPTestMask == fcNormal) {
10988 // TODO: Handle unordered
10989 ISD::CondCode IsFiniteOp = IsInvertedFP ? ISD::SETUGE : ISD::SETOLT;
10990 ISD::CondCode IsNormalOp = IsInvertedFP ? ISD::SETOLT : ISD::SETUGE;
10991
10992 if (isCondCodeLegalOrCustom(IsFiniteOp,
10993 OperandVT.getScalarType().getSimpleVT()) &&
10994 isCondCodeLegalOrCustom(IsNormalOp,
10995 OperandVT.getScalarType().getSimpleVT()) &&
10996 isFAbsFree(OperandVT)) {
10997 // isnormal(x) --> fabs(x) < infinity && !(fabs(x) < smallest_normal)
10998 SDValue Inf =
10999 DAG.getConstantFP(APFloat::getInf(Semantics), DL, OperandVT);
11000 SDValue SmallestNormal = DAG.getConstantFP(
11001 APFloat::getSmallestNormalized(Semantics), DL, OperandVT);
11002
11003 SDValue Abs = DAG.getNode(ISD::FABS, DL, OperandVT, Op);
11004 SDValue IsFinite = DAG.getSetCC(DL, ResultVT, Abs, Inf, IsFiniteOp);
11005 SDValue IsNormal =
11006 DAG.getSetCC(DL, ResultVT, Abs, SmallestNormal, IsNormalOp);
11007 unsigned LogicOp = IsInvertedFP ? ISD::OR : ISD::AND;
11008 return DAG.getNode(LogicOp, DL, ResultVT, IsFinite, IsNormal);
11009 }
11010 }
11011 }
11012
11013 // Some checks may be represented as inversion of simpler check, for example
11014 // "inf|normal|subnormal|zero" => !"nan".
11015 bool IsInverted = false;
11016
11017 if (FPClassTest InvertedCheck = invertFPClassTestIfSimpler(Test, false)) {
11018 Test = InvertedCheck;
11019 IsInverted = true;
11020 }
11021
11022 // In the general case use integer operations.
11023 unsigned BitSize = OperandVT.getScalarSizeInBits();
11024 EVT IntVT = OperandVT.changeElementType(
11025 *DAG.getContext(), EVT::getIntegerVT(*DAG.getContext(), BitSize));
11026 SDValue OpAsInt = DAG.getBitcast(IntVT, Op);
11027
11028 // Various masks.
11029 APInt SignBit = APInt::getSignMask(BitSize);
11030 APInt ValueMask = APInt::getSignedMaxValue(BitSize); // All bits but sign.
11031 APInt Inf = APFloat::getInf(Semantics).bitcastToAPInt(); // Exp and int bit.
11032 const unsigned ExplicitIntBitInF80 = 63;
11033 APInt ExpMask = Inf;
11034 if (IsF80)
11035 ExpMask.clearBit(ExplicitIntBitInF80);
11036 APInt AllOneMantissa = APFloat::getLargest(Semantics).bitcastToAPInt() & ~Inf;
11037 APInt QNaNBitMask =
11038 APInt::getOneBitSet(BitSize, AllOneMantissa.getActiveBits() - 1);
11039 APInt InversionMask = APInt::getAllOnes(ResultVT.getScalarSizeInBits());
11040
11041 SDValue ValueMaskV = DAG.getConstant(ValueMask, DL, IntVT);
11042 SDValue SignBitV = DAG.getConstant(SignBit, DL, IntVT);
11043 SDValue ExpMaskV = DAG.getConstant(ExpMask, DL, IntVT);
11044 SDValue ZeroV = DAG.getConstant(0, DL, IntVT);
11045 SDValue InfV = DAG.getConstant(Inf, DL, IntVT);
11046 SDValue ResultInversionMask = DAG.getConstant(InversionMask, DL, ResultVT);
11047
11048 SDValue Res;
11049 const auto appendResult = [&](SDValue PartialRes) {
11050 if (PartialRes) {
11051 if (Res)
11052 Res = DAG.getNode(ISD::OR, DL, ResultVT, Res, PartialRes);
11053 else
11054 Res = PartialRes;
11055 }
11056 };
11057
11058 SDValue IntBitIsSetV; // Explicit integer bit in f80 mantissa is set.
11059 const auto getIntBitIsSet = [&]() -> SDValue {
11060 if (!IntBitIsSetV) {
11061 APInt IntBitMask(BitSize, 0);
11062 IntBitMask.setBit(ExplicitIntBitInF80);
11063 SDValue IntBitMaskV = DAG.getConstant(IntBitMask, DL, IntVT);
11064 SDValue IntBitV = DAG.getNode(ISD::AND, DL, IntVT, OpAsInt, IntBitMaskV);
11065 IntBitIsSetV = DAG.getSetCC(DL, ResultVT, IntBitV, ZeroV, ISD::SETNE);
11066 }
11067 return IntBitIsSetV;
11068 };
11069
11070 // Split the value into sign bit and absolute value.
11071 SDValue AbsV = DAG.getNode(ISD::AND, DL, IntVT, OpAsInt, ValueMaskV);
11072 SDValue SignV = DAG.getSetCC(DL, ResultVT, OpAsInt,
11073 DAG.getConstant(0, DL, IntVT), ISD::SETLT);
11074
11075 // Tests that involve more than one class should be processed first.
11076 SDValue PartialRes;
11077
11078 if (IsF80)
11079 ; // Detect finite numbers of f80 by checking individual classes because
11080 // they have different settings of the explicit integer bit.
11081 else if ((Test & fcFinite) == fcFinite) {
11082 // finite(V) ==> (a << 1) < (inf << 1)
11083 //
11084 // See https://github.com/llvm/llvm-project/issues/169270, this is slightly
11085 // shorter than the `finite(V) ==> abs(V) < exp_mask` formula used before.
11086
11088 "finite check requires IEEE-like FP");
11089
11090 SDValue One = DAG.getShiftAmountConstant(1, IntVT, DL);
11091 SDValue TwiceOp = DAG.getNode(ISD::SHL, DL, IntVT, OpAsInt, One);
11092 SDValue TwiceInf = DAG.getNode(ISD::SHL, DL, IntVT, ExpMaskV, One);
11093
11094 PartialRes = DAG.getSetCC(DL, ResultVT, TwiceOp, TwiceInf, ISD::SETULT);
11095 Test &= ~fcFinite;
11096 } else if ((Test & fcFinite) == fcPosFinite) {
11097 // finite(V) && V > 0 ==> V < exp_mask
11098 PartialRes = DAG.getSetCC(DL, ResultVT, OpAsInt, ExpMaskV, ISD::SETULT);
11099 Test &= ~fcPosFinite;
11100 } else if ((Test & fcFinite) == fcNegFinite) {
11101 // finite(V) && V < 0 ==> abs(V) < exp_mask && signbit == 1
11102 PartialRes = DAG.getSetCC(DL, ResultVT, AbsV, ExpMaskV, ISD::SETLT);
11103 PartialRes = DAG.getNode(ISD::AND, DL, ResultVT, PartialRes, SignV);
11104 Test &= ~fcNegFinite;
11105 }
11106 appendResult(PartialRes);
11107
11108 if (FPClassTest PartialCheck = Test & (fcZero | fcSubnormal)) {
11109 // fcZero | fcSubnormal => test all exponent bits are 0
11110 // TODO: Handle sign bit specific cases
11111 if (PartialCheck == (fcZero | fcSubnormal)) {
11112 SDValue ExpBits = DAG.getNode(ISD::AND, DL, IntVT, OpAsInt, ExpMaskV);
11113 SDValue ExpIsZero =
11114 DAG.getSetCC(DL, ResultVT, ExpBits, ZeroV, ISD::SETEQ);
11115 appendResult(ExpIsZero);
11116 Test &= ~PartialCheck & fcAllFlags;
11117 }
11118 }
11119
11120 // Check for individual classes.
11121
11122 if (unsigned PartialCheck = Test & fcZero) {
11123 if (PartialCheck == fcPosZero)
11124 PartialRes = DAG.getSetCC(DL, ResultVT, OpAsInt, ZeroV, ISD::SETEQ);
11125 else if (PartialCheck == fcZero)
11126 PartialRes = DAG.getSetCC(DL, ResultVT, AbsV, ZeroV, ISD::SETEQ);
11127 else // ISD::fcNegZero
11128 PartialRes = DAG.getSetCC(DL, ResultVT, OpAsInt, SignBitV, ISD::SETEQ);
11129 appendResult(PartialRes);
11130 }
11131
11132 if (unsigned PartialCheck = Test & fcSubnormal) {
11133 // issubnormal(V) ==> unsigned(abs(V) - 1) < (all mantissa bits set)
11134 // issubnormal(V) && V>0 ==> unsigned(V - 1) < (all mantissa bits set)
11135 SDValue V = (PartialCheck == fcPosSubnormal) ? OpAsInt : AbsV;
11136 SDValue MantissaV = DAG.getConstant(AllOneMantissa, DL, IntVT);
11137 SDValue VMinusOneV =
11138 DAG.getNode(ISD::SUB, DL, IntVT, V, DAG.getConstant(1, DL, IntVT));
11139 PartialRes = DAG.getSetCC(DL, ResultVT, VMinusOneV, MantissaV, ISD::SETULT);
11140 if (PartialCheck == fcNegSubnormal)
11141 PartialRes = DAG.getNode(ISD::AND, DL, ResultVT, PartialRes, SignV);
11142 appendResult(PartialRes);
11143 }
11144
11145 if (unsigned PartialCheck = Test & fcInf) {
11146 if (PartialCheck == fcPosInf)
11147 PartialRes = DAG.getSetCC(DL, ResultVT, OpAsInt, InfV, ISD::SETEQ);
11148 else if (PartialCheck == fcInf)
11149 PartialRes = DAG.getSetCC(DL, ResultVT, AbsV, InfV, ISD::SETEQ);
11150 else { // ISD::fcNegInf
11151 APInt NegInf = APFloat::getInf(Semantics, true).bitcastToAPInt();
11152 SDValue NegInfV = DAG.getConstant(NegInf, DL, IntVT);
11153 PartialRes = DAG.getSetCC(DL, ResultVT, OpAsInt, NegInfV, ISD::SETEQ);
11154 }
11155 appendResult(PartialRes);
11156 }
11157
11158 if (unsigned PartialCheck = Test & fcNan) {
11159 APInt InfWithQnanBit = Inf | QNaNBitMask;
11160 SDValue InfWithQnanBitV = DAG.getConstant(InfWithQnanBit, DL, IntVT);
11161 if (PartialCheck == fcNan) {
11162 // isnan(V) ==> abs(V) > int(inf)
11163 PartialRes = DAG.getSetCC(DL, ResultVT, AbsV, InfV, ISD::SETGT);
11164 if (IsF80) {
11165 // Recognize unsupported values as NaNs for compatibility with glibc.
11166 // In them (exp(V)==0) == int_bit.
11167 SDValue ExpBits = DAG.getNode(ISD::AND, DL, IntVT, AbsV, ExpMaskV);
11168 SDValue ExpIsZero =
11169 DAG.getSetCC(DL, ResultVT, ExpBits, ZeroV, ISD::SETEQ);
11170 SDValue IsPseudo =
11171 DAG.getSetCC(DL, ResultVT, getIntBitIsSet(), ExpIsZero, ISD::SETEQ);
11172 PartialRes = DAG.getNode(ISD::OR, DL, ResultVT, PartialRes, IsPseudo);
11173 }
11174 } else if (PartialCheck == fcQNan) {
11175 // isquiet(V) ==> abs(V) >= (unsigned(Inf) | quiet_bit)
11176 PartialRes =
11177 DAG.getSetCC(DL, ResultVT, AbsV, InfWithQnanBitV, ISD::SETGE);
11178 } else { // ISD::fcSNan
11179 // issignaling(V) ==> abs(V) > unsigned(Inf) &&
11180 // abs(V) < (unsigned(Inf) | quiet_bit)
11181 SDValue IsNan = DAG.getSetCC(DL, ResultVT, AbsV, InfV, ISD::SETGT);
11182 SDValue IsNotQnan =
11183 DAG.getSetCC(DL, ResultVT, AbsV, InfWithQnanBitV, ISD::SETLT);
11184 PartialRes = DAG.getNode(ISD::AND, DL, ResultVT, IsNan, IsNotQnan);
11185 }
11186 appendResult(PartialRes);
11187 }
11188
11189 if (unsigned PartialCheck = Test & fcNormal) {
11190 // isnormal(V) ==> (0 < exp < max_exp) ==> (unsigned(exp-1) < (max_exp-1))
11191 APInt ExpLSB = ExpMask & ~(ExpMask.shl(1));
11192 SDValue ExpLSBV = DAG.getConstant(ExpLSB, DL, IntVT);
11193 SDValue ExpMinus1 = DAG.getNode(ISD::SUB, DL, IntVT, AbsV, ExpLSBV);
11194 APInt ExpLimit = ExpMask - ExpLSB;
11195 SDValue ExpLimitV = DAG.getConstant(ExpLimit, DL, IntVT);
11196 PartialRes = DAG.getSetCC(DL, ResultVT, ExpMinus1, ExpLimitV, ISD::SETULT);
11197 if (PartialCheck == fcNegNormal)
11198 PartialRes = DAG.getNode(ISD::AND, DL, ResultVT, PartialRes, SignV);
11199 else if (PartialCheck == fcPosNormal) {
11200 SDValue PosSignV =
11201 DAG.getNode(ISD::XOR, DL, ResultVT, SignV, ResultInversionMask);
11202 PartialRes = DAG.getNode(ISD::AND, DL, ResultVT, PartialRes, PosSignV);
11203 }
11204 if (IsF80)
11205 PartialRes =
11206 DAG.getNode(ISD::AND, DL, ResultVT, PartialRes, getIntBitIsSet());
11207 appendResult(PartialRes);
11208 }
11209
11210 if (!Res)
11211 return DAG.getConstant(IsInverted, DL, ResultVT);
11212 if (IsInverted)
11213 Res = DAG.getNode(ISD::XOR, DL, ResultVT, Res, ResultInversionMask);
11214 return Res;
11215}
11216
11217// Only expand vector types if we have the appropriate vector bit operations.
11219 assert(VT.isVector() && "Expected vector type");
11220 unsigned Len = VT.getScalarSizeInBits();
11221 return TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
11222 TLI.isOperationLegalOrCustom(ISD::SUB, VT) &&
11223 TLI.isOperationLegalOrCustom(ISD::SRL, VT) &&
11224 (Len == 8 || TLI.isOperationLegalOrCustom(ISD::MUL, VT)) &&
11225 TLI.isOperationLegalOrCustomOrPromote(ISD::AND, VT);
11226}
11227
11229 SDLoc dl(Node);
11230 EVT VT = Node->getValueType(0);
11231 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
11232 SDValue Op = Node->getOperand(0);
11233 unsigned Len = VT.getScalarSizeInBits();
11234
11235 // Compute effective bit width from known bits, allowing us to shift the
11236 // active bits down if necessary to fit into smaller specialized expansions.
11238 unsigned LZ = Known.countMinLeadingZeros();
11239 unsigned TZ = Known.countMinTrailingZeros();
11240 unsigned ShiftedActiveBits = Known.getBitWidth() - (LZ + TZ);
11241
11242 // Round up to 8-bit boundary for byte-oriented SWAR algorithm
11243 unsigned EffectiveLen = Len;
11244 if (ShiftedActiveBits > 0 && ShiftedActiveBits < Len)
11245 EffectiveLen = std::min(alignTo(ShiftedActiveBits, 8), Len);
11246
11247 assert(VT.isInteger() && "CTPOP not implemented for this type.");
11248
11249 // TODO: Add support for irregular type lengths.
11250 if (!(Len <= 128 && Len % 8 == 0))
11251 return SDValue();
11252
11253 // Only expand vector types if we have the appropriate vector bit operations.
11254 if (VT.isVector() && !canExpandVectorCTPOP(*this, VT))
11255 return SDValue();
11256
11257 // If the active bits are not at the low end, shift them down
11258 if (EffectiveLen < Len && TZ > 0) {
11259 Op = DAG.getNode(ISD::SRL, dl, VT, Op,
11260 DAG.getShiftAmountConstant(TZ, VT, dl));
11261 }
11262
11263 // This is the "best" algorithm from
11264 // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
11265 SDValue Mask55 =
11266 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)), dl, VT);
11267 SDValue Mask33 =
11268 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)), dl, VT);
11269 SDValue Mask0F =
11270 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)), dl, VT);
11271
11272 // v = v - ((v >> 1) & 0x55555555...)
11273 Op = DAG.getNode(ISD::SUB, dl, VT, Op,
11274 DAG.getNode(ISD::AND, dl, VT,
11275 DAG.getNode(ISD::SRL, dl, VT, Op,
11276 DAG.getConstant(1, dl, ShVT)),
11277 Mask55));
11278 // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...)
11279 Op = DAG.getNode(ISD::ADD, dl, VT, DAG.getNode(ISD::AND, dl, VT, Op, Mask33),
11280 DAG.getNode(ISD::AND, dl, VT,
11281 DAG.getNode(ISD::SRL, dl, VT, Op,
11282 DAG.getConstant(2, dl, ShVT)),
11283 Mask33));
11284 // v = (v + (v >> 4)) & 0x0F0F0F0F...
11285 Op = DAG.getNode(ISD::AND, dl, VT,
11286 DAG.getNode(ISD::ADD, dl, VT, Op,
11287 DAG.getNode(ISD::SRL, dl, VT, Op,
11288 DAG.getConstant(4, dl, ShVT))),
11289 Mask0F);
11290
11291 if (EffectiveLen <= 8)
11292 return Op;
11293
11294 // Avoid the multiply if we only have 2 bytes to add.
11295 // TODO: Only doing this for scalars because vectors weren't as obviously
11296 // improved.
11297 if (EffectiveLen == 16 && !VT.isVector()) {
11298 // v = (v + (v >> 8)) & 0x00FF;
11299 return DAG.getNode(ISD::AND, dl, VT,
11300 DAG.getNode(ISD::ADD, dl, VT, Op,
11301 DAG.getNode(ISD::SRL, dl, VT, Op,
11302 DAG.getConstant(8, dl, ShVT))),
11303 DAG.getConstant(0xFF, dl, VT));
11304 }
11305
11306 // v = (v * 0x01010101...) >> (Len - 8)
11307 SDValue V;
11310 SDValue Mask01 =
11311 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)), dl, VT);
11312 V = DAG.getNode(ISD::MUL, dl, VT, Op, Mask01);
11313 } else {
11314 V = Op;
11315 for (unsigned Shift = 8; Shift < EffectiveLen; Shift *= 2) {
11316 SDValue ShiftC = DAG.getShiftAmountConstant(Shift, VT, dl);
11317 V = DAG.getNode(ISD::ADD, dl, VT, V,
11318 DAG.getNode(ISD::SHL, dl, VT, V, ShiftC));
11319 }
11320 }
11321 return DAG.getNode(ISD::SRL, dl, VT, V, DAG.getConstant(Len - 8, dl, ShVT));
11322}
11323
11325 SDLoc dl(Node);
11326 EVT VT = Node->getValueType(0);
11327 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
11328 SDValue Op = Node->getOperand(0);
11329 unsigned NumBitsPerElt = VT.getScalarSizeInBits();
11330
11331 // If the non-ZERO_POISON version is supported we can use that instead.
11332 if (Node->getOpcode() == ISD::CTLZ_ZERO_POISON &&
11334 return DAG.getNode(ISD::CTLZ, dl, VT, Op);
11335
11336 // If the ZERO_POISON version is supported use that and handle the zero case.
11338 EVT SetCCVT =
11339 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
11340 SDValue CTLZ = DAG.getNode(ISD::CTLZ_ZERO_POISON, dl, VT, Op);
11341 SDValue Zero = DAG.getConstant(0, dl, VT);
11342 SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
11343 return DAG.getSelect(dl, VT, SrcIsZero,
11344 DAG.getConstant(NumBitsPerElt, dl, VT), CTLZ);
11345 }
11346
11347 // Only expand vector types if we have the appropriate vector bit operations.
11348 // This includes the operations needed to expand CTPOP if it isn't supported.
11349 if (VT.isVector() && (!isPowerOf2_32(NumBitsPerElt) ||
11351 !canExpandVectorCTPOP(*this, VT)) ||
11354 return SDValue();
11355
11356 // for now, we do this:
11357 // x = x | (x >> 1);
11358 // x = x | (x >> 2);
11359 // ...
11360 // x = x | (x >>16);
11361 // x = x | (x >>32); // for 64-bit input
11362 // return popcount(~x);
11363 //
11364 // Ref: "Hacker's Delight" by Henry Warren
11365 for (unsigned i = 0; (1U << i) < NumBitsPerElt; ++i) {
11366 SDValue Tmp = DAG.getConstant(1ULL << i, dl, ShVT);
11367 Op = DAG.getNode(ISD::OR, dl, VT, Op,
11368 DAG.getNode(ISD::SRL, dl, VT, Op, Tmp));
11369 }
11370 Op = DAG.getNOT(dl, Op, VT);
11371 return DAG.getNode(ISD::CTPOP, dl, VT, Op);
11372}
11373
11375 SDLoc dl(Node);
11376 EVT VT = Node->getValueType(0);
11377 SDValue Op = DAG.getFreeze(Node->getOperand(0));
11378 unsigned NumBitsPerElt = VT.getScalarSizeInBits();
11379
11380 // CTLS(x) = CTLZ(OR(SHL(XOR(x, SRA(x, BW-1)), 1), 1))
11381 // This transforms the sign bits into leading zeros that can be counted.
11382 SDValue ShiftAmt = DAG.getShiftAmountConstant(NumBitsPerElt - 1, VT, dl);
11383 SDValue SignBit = DAG.getNode(ISD::SRA, dl, VT, Op, ShiftAmt);
11384 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, Op, SignBit);
11385 SDValue Shl =
11386 DAG.getNode(ISD::SHL, dl, VT, Xor, DAG.getShiftAmountConstant(1, VT, dl));
11387 SDValue Or = DAG.getNode(ISD::OR, dl, VT, Shl, DAG.getConstant(1, dl, VT));
11388 return DAG.getNode(ISD::CTLZ_ZERO_POISON, dl, VT, Or);
11389}
11390
11392 const SDLoc &DL, EVT VT, SDValue Op,
11393 unsigned BitWidth) const {
11394 if (BitWidth != 32 && BitWidth != 64)
11395 return SDValue();
11396
11397 const DataLayout &TD = DAG.getDataLayout();
11399 return SDValue();
11400
11401 APInt DeBruijn = BitWidth == 32 ? APInt(32, 0x077CB531U)
11402 : APInt(64, 0x0218A392CD3D5DBFULL);
11403 MachinePointerInfo PtrInfo =
11405 unsigned ShiftAmt = BitWidth - Log2_32(BitWidth);
11406 SDValue Neg = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Op);
11407 SDValue Lookup = DAG.getNode(
11408 ISD::SRL, DL, VT,
11409 DAG.getNode(ISD::MUL, DL, VT, DAG.getNode(ISD::AND, DL, VT, Op, Neg),
11410 DAG.getConstant(DeBruijn, DL, VT)),
11411 DAG.getShiftAmountConstant(ShiftAmt, VT, DL));
11413
11415 for (unsigned i = 0; i < BitWidth; i++) {
11416 APInt Shl = DeBruijn.shl(i);
11417 APInt Lshr = Shl.lshr(ShiftAmt);
11418 Table[Lshr.getZExtValue()] = i;
11419 }
11420
11421 // Create a ConstantArray in Constant Pool
11422 auto *CA = ConstantDataArray::get(*DAG.getContext(), Table);
11423 SDValue CPIdx = DAG.getConstantPool(CA, getPointerTy(TD),
11424 TD.getPrefTypeAlign(CA->getType()));
11425 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, DL, VT, DAG.getEntryNode(),
11426 DAG.getMemBasePlusOffset(CPIdx, Lookup, DL),
11427 PtrInfo, MVT::i8);
11428 if (Node->getOpcode() == ISD::CTTZ_ZERO_POISON)
11429 return ExtLoad;
11430
11431 EVT SetCCVT =
11432 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
11433 SDValue Zero = DAG.getConstant(0, DL, VT);
11434 SDValue SrcIsZero = DAG.getSetCC(DL, SetCCVT, Op, Zero, ISD::SETEQ);
11435 return DAG.getSelect(DL, VT, SrcIsZero,
11436 DAG.getConstant(BitWidth, DL, VT), ExtLoad);
11437}
11438
11440 SDLoc dl(Node);
11441 EVT VT = Node->getValueType(0);
11442 SDValue Op = Node->getOperand(0);
11443 unsigned NumBitsPerElt = VT.getScalarSizeInBits();
11444
11445 // If the non-ZERO_POISON version is supported we can use that instead.
11446 if (Node->getOpcode() == ISD::CTTZ_ZERO_POISON &&
11448 return DAG.getNode(ISD::CTTZ, dl, VT, Op);
11449
11450 // If the ZERO_POISON version is supported use that and handle the zero case.
11452 EVT SetCCVT =
11453 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
11454 SDValue CTTZ = DAG.getNode(ISD::CTTZ_ZERO_POISON, dl, VT, Op);
11455 SDValue Zero = DAG.getConstant(0, dl, VT);
11456 SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
11457 return DAG.getSelect(dl, VT, SrcIsZero,
11458 DAG.getConstant(NumBitsPerElt, dl, VT), CTTZ);
11459 }
11460
11461 // Only expand vector types if we have the appropriate vector bit operations.
11462 // This includes the operations needed to expand CTPOP if it isn't supported.
11463 if (VT.isVector() && (!isPowerOf2_32(NumBitsPerElt) ||
11466 !canExpandVectorCTPOP(*this, VT)) ||
11470 return SDValue();
11471
11472 // Emit Table Lookup if ISD::CTPOP used in the fallback path below is going
11473 // to be expanded or converted to a libcall.
11476 if (SDValue V = CTTZTableLookup(Node, DAG, dl, VT, Op, NumBitsPerElt))
11477 return V;
11478
11479 bool UseCTLZ =
11481
11482 // When only ctlz is available and the operand is nonzero we can use:
11483 // { return nlz(x & -x) ^ 31; }
11484 // which is more efficient than:
11485 // { return 32 - nlz(~x & (x - 1)); }.
11486 if (UseCTLZ && Node->getOpcode() == ISD::CTTZ_ZERO_POISON) {
11487 SDValue LowestBit =
11488 DAG.getNode(ISD::AND, dl, VT, Op, DAG.getNegative(Op, dl, VT));
11489 return DAG.getNode(ISD::XOR, dl, VT,
11490 DAG.getNode(ISD::CTLZ_ZERO_POISON, dl, VT, LowestBit),
11491 DAG.getConstant(NumBitsPerElt - 1, dl, VT));
11492 }
11493
11494 // If ctpop is available, we use:
11495 // { return popcount(~x & (x-1)); }
11496 // If the target has ctlz but not ctpop, we use:
11497 // { return 32 - nlz(~x & (x-1)); }
11498 // Ref: "Hacker's Delight" by Henry Warren
11499 SDValue Tmp = DAG.getNode(
11500 ISD::AND, dl, VT, DAG.getNOT(dl, Op, VT),
11501 DAG.getNode(ISD::SUB, dl, VT, Op, DAG.getConstant(1, dl, VT)));
11502
11503 if (UseCTLZ)
11504 return DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(NumBitsPerElt, dl, VT),
11505 DAG.getNode(ISD::CTLZ, dl, VT, Tmp));
11506
11507 return DAG.getNode(ISD::CTPOP, dl, VT, Tmp);
11508}
11509
11511 SelectionDAG &DAG) const {
11512 // %cond = to_bool_vec %source
11513 // %splat = splat /*val=*/VL
11514 // %tz = step_vector
11515 // %v = select %cond, /*true=*/tz, /*false=*/%splat
11516 // %r = vp.reduce.umin %v
11517 SDLoc DL(N);
11518 SDValue Source = N->getOperand(0);
11519 SDValue Mask = N->getOperand(1);
11520 SDValue EVL = N->getOperand(2);
11521 EVT SrcVT = Source.getValueType();
11522 EVT ResVT = N->getValueType(0);
11523 EVT ResVecVT =
11524 EVT::getVectorVT(*DAG.getContext(), ResVT, SrcVT.getVectorElementCount());
11525
11526 // Convert to boolean vector.
11527 if (SrcVT.getScalarType() != MVT::i1) {
11528 SDValue AllZero = DAG.getConstant(0, DL, SrcVT);
11529 SrcVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
11530 SrcVT.getVectorElementCount());
11531 Source = DAG.getSetCC(DL, SrcVT, Source, AllZero, ISD::SETNE);
11532 }
11533
11534 SDValue ExtEVL = DAG.getZExtOrTrunc(EVL, DL, ResVT);
11535 SDValue Splat = DAG.getSplat(ResVecVT, DL, ExtEVL);
11536 SDValue StepVec = DAG.getStepVector(DL, ResVecVT);
11537 SDValue Select = DAG.getSelect(DL, ResVecVT, Source, StepVec, Splat);
11538 return DAG.getNode(ISD::VP_REDUCE_UMIN, DL, ResVT, ExtEVL, Select, Mask, EVL);
11539}
11540
11541/// Returns a type-legalized version of \p Mask as the first item in the
11542/// pair. The second item contains a type-legalized step vector that's
11543/// guaranteed to fit the number of elements in \p Mask.
11544/// If the stepvector would require splitting, returns an empty SDValue
11545/// as the second item to signal that the operation should be split instead.
11546static std::pair<SDValue, SDValue>
11548 SelectionDAG &DAG) {
11549 EVT MaskVT = Mask.getValueType();
11550 EVT BoolVT = MaskVT.getScalarType();
11551
11552 // Find a suitable type for a stepvector.
11553 // If zero is poison, we can assume the upper limit of the result is VF-1.
11554 ConstantRange VScaleRange(1, /*isFullSet=*/true); // Fixed length default.
11555 if (MaskVT.isScalableVector())
11556 VScaleRange = getVScaleRange(&DAG.getMachineFunction().getFunction(), 64);
11558 uint64_t EltWidth = TLI.getBitWidthForCttzElements(
11559 EVT(TLI.getVectorIdxTy(DAG.getDataLayout())),
11560 MaskVT.getVectorElementCount(), ZeroIsPoison, &VScaleRange);
11561 // If the step vector element type is smaller than the mask element type,
11562 // use the mask type directly to avoid widening issues.
11563 EltWidth = std::max(EltWidth, BoolVT.getFixedSizeInBits());
11564 EVT StepVT = MVT::getIntegerVT(EltWidth);
11565 EVT StepVecVT = MaskVT.changeVectorElementType(*DAG.getContext(), StepVT);
11566
11567 // If promotion or widening is required to make the type legal, do it here.
11568 // Promotion of integers within LegalizeVectorOps is looking for types of
11569 // the same size but with a smaller number of larger elements, not the usual
11570 // larger size with the same number of larger elements.
11572 TLI.getTypeAction(*DAG.getContext(), StepVecVT);
11573 SDValue StepVec;
11574 if (TypeAction == TargetLowering::TypePromoteInteger) {
11575 StepVecVT = TLI.getTypeToTransformTo(*DAG.getContext(), StepVecVT);
11576 StepVec = DAG.getStepVector(DL, StepVecVT);
11577 } else if (TypeAction == TargetLowering::TypeWidenVector) {
11578 // For widening, the element count changes. Create a step vector with only
11579 // the original elements valid and zeros for padding. Also widen the mask.
11580 EVT WideVecVT = TLI.getTypeToTransformTo(*DAG.getContext(), StepVecVT);
11581 unsigned WideNumElts = WideVecVT.getVectorNumElements();
11582
11583 // Build widened step vector: <0, 1, ..., OrigNumElts-1, poison, poison, ..>
11584 SDValue OrigStepVec = DAG.getStepVector(DL, StepVecVT);
11585 SDValue UndefStep = DAG.getPOISON(WideVecVT);
11586 StepVec = DAG.getInsertSubvector(DL, UndefStep, OrigStepVec, 0);
11587
11588 // Widen mask: pad with zeros.
11589 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), BoolVT, WideNumElts);
11590 SDValue ZeroMask = DAG.getConstant(0, DL, WideMaskVT);
11591 Mask = DAG.getInsertSubvector(DL, ZeroMask, Mask, 0);
11592 } else if (TypeAction == TargetLowering::TypeSplitVector) {
11593 // The stepvector type would require splitting. Signal to the caller
11594 // that the operation should be split instead of expanded.
11595 return {Mask, SDValue()};
11596 } else {
11597 StepVec = DAG.getStepVector(DL, StepVecVT);
11598 }
11599
11600 return {Mask, StepVec};
11601}
11602
11604 SelectionDAG &DAG) const {
11605 SDLoc DL(N);
11606 auto [Mask, StepVec] = getLegalMaskAndStepVector(
11607 N->getOperand(0), /*ZeroIsPoison=*/true, DL, DAG);
11608
11609 // If StepVec is empty, the stepvector would require splitting.
11610 // Split the operation instead and let it be recursively legalized.
11611 if (!StepVec) {
11612 EVT MaskVT = N->getOperand(0).getValueType();
11613 EVT ResVT = N->getValueType(0);
11614
11615 // Split the mask
11616 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(MaskVT);
11617 auto [MaskLo, MaskHi] = DAG.SplitVector(N->getOperand(0), DL);
11618
11619 // Create split VECTOR_FIND_LAST_ACTIVE operations
11620 SDValue LoResult =
11621 DAG.getNode(ISD::VECTOR_FIND_LAST_ACTIVE, DL, ResVT, MaskLo);
11622 SDValue HiResult =
11623 DAG.getNode(ISD::VECTOR_FIND_LAST_ACTIVE, DL, ResVT, MaskHi);
11624
11625 // Check if any lane is active in the high mask.
11626 SDValue AnyHiActive = DAG.getNode(ISD::VECREDUCE_OR, DL, MVT::i1, MaskHi);
11628 AnyHiActive, DL,
11629 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i1),
11630 MVT::i1);
11631
11632 // Adjust HiResult by adding the number of elements in Lo
11633 SDValue LoNumElts =
11634 DAG.getElementCount(DL, ResVT, LoVT.getVectorElementCount());
11635 SDValue AdjustedHiResult =
11636 DAG.getNode(ISD::ADD, DL, ResVT, HiResult, LoNumElts);
11637
11638 // Return: AnyHiActive ? AdjustedHiResult : LoResult;
11639 return DAG.getNode(ISD::SELECT, DL, ResVT, Cond, AdjustedHiResult,
11640 LoResult);
11641 }
11642
11643 EVT StepVecVT = StepVec.getValueType();
11644 EVT StepVT = StepVec.getValueType().getVectorElementType();
11645
11646 // Zero out lanes with inactive elements, then find the highest remaining
11647 // value from the stepvector.
11648 SDValue Zeroes = DAG.getConstant(0, DL, StepVecVT);
11649 SDValue ActiveElts = DAG.getSelect(DL, StepVecVT, Mask, StepVec, Zeroes);
11650 SDValue HighestIdx = DAG.getNode(ISD::VECREDUCE_UMAX, DL, StepVT, ActiveElts);
11651 return DAG.getZExtOrTrunc(HighestIdx, DL, N->getValueType(0));
11652}
11653
11655 SelectionDAG &DAG) const {
11656 SDLoc DL(N);
11657
11658 SDValue Start = N->getOperand(0);
11659 SDValue End = N->getOperand(1);
11660 EVT VT = N->getValueType(0);
11661 EVT OpVT = Start.getValueType();
11662
11663 auto [Mask, StepVector] = getLegalMaskAndStepVector(
11664 DAG.getPOISON(VT), /*ZeroIsPoison=*/false, DL, DAG);
11665
11666 // No legal step vector: split the mask in half and recombine the results.
11667 // Lo = get_active_lane_mask Start, End
11668 // Hi = get_active_lane_mask (uaddsat Start, LoNumElts), End
11669 if (!StepVector) {
11670 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT);
11671 SDValue Lo = DAG.getNode(ISD::GET_ACTIVE_LANE_MASK, DL, LoVT, Start, End);
11672 SDValue LoNumElts =
11673 DAG.getElementCount(DL, OpVT, LoVT.getVectorElementCount());
11674 SDValue HiStart = DAG.getNode(ISD::UADDSAT, DL, OpVT, Start, LoNumElts);
11675 SDValue Hi = DAG.getNode(ISD::GET_ACTIVE_LANE_MASK, DL, HiVT, HiStart, End);
11676 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
11677 }
11678
11679 // Rebase and saturate the termination value.
11680 SDValue Max =
11681 DAG.getConstant(maxUIntN(std::min(StepVector.getScalarValueSizeInBits(),
11682 OpVT.getScalarSizeInBits())),
11683 DL, OpVT);
11684 End = DAG.getNode(ISD::USUBSAT, DL, OpVT, End, Start);
11685 End = DAG.getNode(ISD::UMIN, DL, OpVT, End, Max);
11686 End = DAG.getZExtOrTrunc(End, DL, StepVector.getValueType().getScalarType());
11687
11688 // cmp <0, 1, 2, 3...>, End
11689 SDValue EndV = DAG.getSplat(StepVector.getValueType(), DL, End);
11690 EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
11691 StepVector.getValueType());
11692 SDValue Cmp = DAG.getSetCC(DL, CCVT, StepVector, EndV, ISD::SETULT);
11693 SDValue Res = DAG.getBoolExtOrTrunc(Cmp, DL, Mask.getValueType(),
11694 StepVector.getValueType());
11695 if (Mask.getValueType() != VT)
11696 Res = DAG.getExtractSubvector(DL, VT, Res, 0);
11697 return Res;
11698}
11699
11701 SelectionDAG &DAG) const {
11702 SDLoc DL(N);
11703 EVT VT = N->getValueType(0);
11704 SDValue SourceValue = N->getOperand(0);
11705 SDValue SinkValue = N->getOperand(1);
11706 SDValue EltSizeInBytes = N->getOperand(2);
11707
11708 // Note: The lane offset is scalable if the mask is scalable.
11709 ElementCount LaneOffsetEC =
11710 ElementCount::get(N->getConstantOperandVal(3), VT.isScalableVT());
11711
11712 EVT AddrVT = SourceValue->getValueType(0);
11713 bool IsReadAfterWrite = N->getOpcode() == ISD::LOOP_DEPENDENCE_RAW_MASK;
11714
11715 EVT CmpVT =
11716 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), AddrVT);
11717
11718 // Unsigned compare: Source >= Sink.
11719 SDValue SourceAheadOfOrEqualToSink =
11720 DAG.getSetCC(DL, CmpVT, SourceValue, SinkValue, ISD::SETUGE);
11721
11722 // Take the difference between the pointers and divided by the element size,
11723 // to see how many lanes separate them.
11724 SDValue Diff = DAG.getNode(ISD::SUB, DL, AddrVT, SinkValue, SourceValue);
11725
11726 // RAW_MASK: Diff = Source >= Sink ? (Source - Sink) : (Sink - Source)
11727 if (IsReadAfterWrite)
11728 Diff = DAG.getSelect(DL, AddrVT, SourceAheadOfOrEqualToSink,
11729 DAG.getNegative(Diff, DL, AddrVT), Diff);
11730
11731 Diff = DAG.getNode(ISD::SDIV, DL, AddrVT, Diff, EltSizeInBytes);
11732
11733 // The pointers do not alias if:
11734 // - Source >= Sink (WAR_MASK)
11735 // - Source == Sink (RAW_MASK)
11736 SDValue NoAlias = SourceAheadOfOrEqualToSink;
11737 if (IsReadAfterWrite)
11738 NoAlias = DAG.getSetCC(DL, CmpVT, SourceValue, SinkValue, ISD::SETEQ);
11739
11740 // The pointers do not alias if:
11741 // Lane + LaneOffset < Diff (WAR/RAW_MASK)
11742 SDValue LaneOffset = DAG.getElementCount(DL, AddrVT, LaneOffsetEC);
11743 SDValue MaskN = DAG.getSelect(
11744 DL, AddrVT, NoAlias,
11746 AddrVT),
11747 Diff);
11748
11749 return DAG.getNode(ISD::GET_ACTIVE_LANE_MASK, DL, VT, LaneOffset, MaskN);
11750}
11751
11753 bool IsNegative) const {
11754 SDLoc dl(N);
11755 EVT VT = N->getValueType(0);
11756 SDValue Op = N->getOperand(0);
11757
11758 // If expanding ABS_MIN_POISON, fall back to ABS if the target supports it.
11759 if (N->getOpcode() == ISD::ABS_MIN_POISON &&
11761 SDValue AbsVal = DAG.getNode(ISD::ABS, dl, VT, Op);
11762 if (IsNegative)
11763 return DAG.getNegative(AbsVal, dl, VT);
11764 return AbsVal;
11765 }
11766
11767 // abs(x) -> smax(x,sub(0,x))
11768 if (!IsNegative && isOperationLegal(ISD::SUB, VT) &&
11770 SDValue Zero = DAG.getConstant(0, dl, VT);
11771 Op = DAG.getFreeze(Op);
11772 return DAG.getNode(ISD::SMAX, dl, VT, Op,
11773 DAG.getNode(ISD::SUB, dl, VT, Zero, Op));
11774 }
11775
11776 // abs(x) -> umin(x,sub(0,x))
11777 if (!IsNegative && isOperationLegal(ISD::SUB, VT) &&
11779 SDValue Zero = DAG.getConstant(0, dl, VT);
11780 Op = DAG.getFreeze(Op);
11781 return DAG.getNode(ISD::UMIN, dl, VT, Op,
11782 DAG.getNode(ISD::SUB, dl, VT, Zero, Op));
11783 }
11784
11785 // 0 - abs(x) -> smin(x, sub(0,x))
11786 if (IsNegative && isOperationLegal(ISD::SUB, VT) &&
11788 SDValue Zero = DAG.getConstant(0, dl, VT);
11789 Op = DAG.getFreeze(Op);
11790 return DAG.getNode(ISD::SMIN, dl, VT, Op,
11791 DAG.getNode(ISD::SUB, dl, VT, Zero, Op));
11792 }
11793
11794 // Only expand vector types if we have the appropriate vector operations.
11795 if (VT.isVector() &&
11797 (!IsNegative && !isOperationLegalOrCustom(ISD::ADD, VT)) ||
11798 (IsNegative && !isOperationLegalOrCustom(ISD::SUB, VT)) ||
11800 return SDValue();
11801
11802 Op = DAG.getFreeze(Op);
11803 SDValue Shift = DAG.getNode(
11804 ISD::SRA, dl, VT, Op,
11805 DAG.getShiftAmountConstant(VT.getScalarSizeInBits() - 1, VT, dl));
11806 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, Op, Shift);
11807
11808 // abs(x) -> Y = sra (X, size(X)-1); sub (xor (X, Y), Y)
11809 if (!IsNegative)
11810 return DAG.getNode(ISD::SUB, dl, VT, Xor, Shift);
11811
11812 // 0 - abs(x) -> Y = sra (X, size(X)-1); sub (Y, xor (X, Y))
11813 return DAG.getNode(ISD::SUB, dl, VT, Shift, Xor);
11814}
11815
11817 SDLoc dl(N);
11818 EVT VT = N->getValueType(0);
11819 SDValue LHS = N->getOperand(0);
11820 SDValue RHS = N->getOperand(1);
11821 bool IsSigned = N->getOpcode() == ISD::ABDS;
11822
11823 // abds(lhs, rhs) -> sub(smax(lhs,rhs), smin(lhs,rhs))
11824 // abdu(lhs, rhs) -> sub(umax(lhs,rhs), umin(lhs,rhs))
11825 unsigned MaxOpc = IsSigned ? ISD::SMAX : ISD::UMAX;
11826 unsigned MinOpc = IsSigned ? ISD::SMIN : ISD::UMIN;
11827 if (isOperationLegal(MaxOpc, VT) && isOperationLegal(MinOpc, VT)) {
11828 LHS = DAG.getFreeze(LHS);
11829 RHS = DAG.getFreeze(RHS);
11830 SDValue Max = DAG.getNode(MaxOpc, dl, VT, LHS, RHS);
11831 SDValue Min = DAG.getNode(MinOpc, dl, VT, LHS, RHS);
11832 return DAG.getNode(ISD::SUB, dl, VT, Max, Min);
11833 }
11834
11835 // abdu(lhs, rhs) -> or(usubsat(lhs,rhs), usubsat(rhs,lhs))
11836 if (!IsSigned && isOperationLegal(ISD::USUBSAT, VT)) {
11837 LHS = DAG.getFreeze(LHS);
11838 RHS = DAG.getFreeze(RHS);
11839 return DAG.getNode(ISD::OR, dl, VT,
11840 DAG.getNode(ISD::USUBSAT, dl, VT, LHS, RHS),
11841 DAG.getNode(ISD::USUBSAT, dl, VT, RHS, LHS));
11842 }
11843
11844 // abds: if the sub is nsw -> abs(sub)
11845 // abdu with non-negative operands -> abs(sub)
11846 // The abdu case where the sub is nuw is handled in DAGCombine.
11847 if (IsSigned) {
11848 if (DAG.willNotOverflowSub(/*IsSigned=*/true, LHS, RHS))
11849 return DAG.getNode(ISD::ABS, dl, VT,
11850 DAG.getNode(ISD::SUB, dl, VT, LHS, RHS));
11851
11852 if (DAG.willNotOverflowSub(/*IsSigned=*/true, RHS, LHS))
11853 return DAG.getNode(ISD::ABS, dl, VT,
11854 DAG.getNode(ISD::SUB, dl, VT, RHS, LHS));
11855 } else if (DAG.SignBitIsZero(LHS) && DAG.SignBitIsZero(RHS)) {
11856 return DAG.getNode(ISD::ABS, dl, VT,
11857 DAG.getNode(ISD::SUB, dl, VT, LHS, RHS));
11858 }
11859
11860 EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
11862 LHS = DAG.getFreeze(LHS);
11863 RHS = DAG.getFreeze(RHS);
11864 SDValue Cmp = DAG.getSetCC(dl, CCVT, LHS, RHS, CC);
11865
11866 // Branchless expansion iff cmp result is allbits:
11867 // abds(lhs, rhs) -> sub(sgt(lhs, rhs), xor(sgt(lhs, rhs), sub(lhs, rhs)))
11868 // abdu(lhs, rhs) -> sub(ugt(lhs, rhs), xor(ugt(lhs, rhs), sub(lhs, rhs)))
11869 if (CCVT == VT && getBooleanContents(VT) == ZeroOrNegativeOneBooleanContent) {
11870 SDValue Diff = DAG.getNode(ISD::SUB, dl, VT, LHS, RHS);
11871 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, Diff, Cmp);
11872 return DAG.getNode(ISD::SUB, dl, VT, Cmp, Xor);
11873 }
11874
11875 // Similar to the branchless expansion, if we don't prefer selects, use the
11876 // (sign-extended) usubo overflow flag if the (scalar) type is illegal as this
11877 // is more likely to legalize cleanly: abdu(lhs, rhs) -> sub(xor(sub(lhs,
11878 // rhs), uof(lhs, rhs)), uof(lhs, rhs))
11879 if (!IsSigned && VT.isScalarInteger() && !isTypeLegal(VT) &&
11881 SDValue USubO =
11882 DAG.getNode(ISD::USUBO, dl, DAG.getVTList(VT, MVT::i1), {LHS, RHS});
11883 SDValue Cmp = DAG.getNode(ISD::SIGN_EXTEND, dl, VT, USubO.getValue(1));
11884 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, USubO.getValue(0), Cmp);
11885 return DAG.getNode(ISD::SUB, dl, VT, Xor, Cmp);
11886 }
11887
11888 // FIXME: Should really try to split the vector in case it's legal on a
11889 // subvector.
11891 return DAG.UnrollVectorOp(N);
11892
11893 // abds(lhs, rhs) -> select(sgt(lhs,rhs), sub(lhs,rhs), sub(rhs,lhs))
11894 // abdu(lhs, rhs) -> select(ugt(lhs,rhs), sub(lhs,rhs), sub(rhs,lhs))
11895 return DAG.getSelect(dl, VT, Cmp, DAG.getNode(ISD::SUB, dl, VT, LHS, RHS),
11896 DAG.getNode(ISD::SUB, dl, VT, RHS, LHS));
11897}
11898
11900 SDLoc dl(N);
11901 EVT VT = N->getValueType(0);
11902 SDValue LHS = N->getOperand(0);
11903 SDValue RHS = N->getOperand(1);
11904
11905 unsigned Opc = N->getOpcode();
11906 bool IsFloor = Opc == ISD::AVGFLOORS || Opc == ISD::AVGFLOORU;
11907 bool IsSigned = Opc == ISD::AVGCEILS || Opc == ISD::AVGFLOORS;
11908 unsigned SumOpc = IsFloor ? ISD::ADD : ISD::SUB;
11909 unsigned SignOpc = IsFloor ? ISD::AND : ISD::OR;
11910 unsigned ShiftOpc = IsSigned ? ISD::SRA : ISD::SRL;
11911 unsigned ExtOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
11913 Opc == ISD::AVGFLOORU || Opc == ISD::AVGCEILU) &&
11914 "Unknown AVG node");
11915
11916 // If the operands are already extended, we can add+shift.
11917 bool IsExt =
11918 (IsSigned && DAG.ComputeNumSignBits(LHS) >= 2 &&
11919 DAG.ComputeNumSignBits(RHS) >= 2) ||
11920 (!IsSigned && DAG.computeKnownBits(LHS).countMinLeadingZeros() >= 1 &&
11921 DAG.computeKnownBits(RHS).countMinLeadingZeros() >= 1);
11922 if (IsExt) {
11923 SDValue Sum = DAG.getNode(ISD::ADD, dl, VT, LHS, RHS);
11924 if (!IsFloor)
11925 Sum = DAG.getNode(ISD::ADD, dl, VT, Sum, DAG.getConstant(1, dl, VT));
11926 return DAG.getNode(ShiftOpc, dl, VT, Sum,
11927 DAG.getShiftAmountConstant(1, VT, dl));
11928 }
11929
11930 // For scalars, see if we can efficiently extend/truncate to use add+shift.
11931 if (VT.isScalarInteger()) {
11932 EVT ExtVT = VT.widenIntegerElementType(*DAG.getContext());
11933 if (isTypeLegal(ExtVT) && isTruncateFree(ExtVT, VT)) {
11934 LHS = DAG.getNode(ExtOpc, dl, ExtVT, LHS);
11935 RHS = DAG.getNode(ExtOpc, dl, ExtVT, RHS);
11936 SDValue Avg = DAG.getNode(ISD::ADD, dl, ExtVT, LHS, RHS);
11937 if (!IsFloor)
11938 Avg = DAG.getNode(ISD::ADD, dl, ExtVT, Avg,
11939 DAG.getConstant(1, dl, ExtVT));
11940 // Just use SRL as we will be truncating away the extended sign bits.
11941 Avg = DAG.getNode(ISD::SRL, dl, ExtVT, Avg,
11942 DAG.getShiftAmountConstant(1, ExtVT, dl));
11943 return DAG.getNode(ISD::TRUNCATE, dl, VT, Avg);
11944 }
11945 }
11946
11947 // avgflooru(lhs, rhs) -> or(lshr(add(lhs, rhs),1),shl(overflow, typesize-1))
11948 if (Opc == ISD::AVGFLOORU && VT.isScalarInteger() && !isTypeLegal(VT) &&
11951 SDValue UAddWithOverflow =
11952 DAG.getNode(ISD::UADDO, dl, DAG.getVTList(VT, MVT::i1), {RHS, LHS});
11953
11954 SDValue Sum = UAddWithOverflow.getValue(0);
11955 SDValue Overflow = UAddWithOverflow.getValue(1);
11956
11957 // Right shift the sum by 1
11958 SDValue LShrVal = DAG.getNode(ISD::SRL, dl, VT, Sum,
11959 DAG.getShiftAmountConstant(1, VT, dl));
11960
11961 SDValue ZeroExtOverflow = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Overflow);
11962 SDValue OverflowShl = DAG.getNode(
11963 ISD::SHL, dl, VT, ZeroExtOverflow,
11964 DAG.getShiftAmountConstant(VT.getScalarSizeInBits() - 1, VT, dl));
11965
11966 return DAG.getNode(ISD::OR, dl, VT, LShrVal, OverflowShl);
11967 }
11968
11969 // avgceils(lhs, rhs) -> sub(or(lhs,rhs),ashr(xor(lhs,rhs),1))
11970 // avgceilu(lhs, rhs) -> sub(or(lhs,rhs),lshr(xor(lhs,rhs),1))
11971 // avgfloors(lhs, rhs) -> add(and(lhs,rhs),ashr(xor(lhs,rhs),1))
11972 // avgflooru(lhs, rhs) -> add(and(lhs,rhs),lshr(xor(lhs,rhs),1))
11973 LHS = DAG.getFreeze(LHS);
11974 RHS = DAG.getFreeze(RHS);
11975 SDValue Sign = DAG.getNode(SignOpc, dl, VT, LHS, RHS);
11976 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS);
11977 SDValue Shift =
11978 DAG.getNode(ShiftOpc, dl, VT, Xor, DAG.getShiftAmountConstant(1, VT, dl));
11979 return DAG.getNode(SumOpc, dl, VT, Sign, Shift);
11980}
11981
11983 SDLoc dl(N);
11984 EVT VT = N->getValueType(0);
11985 SDValue Op = N->getOperand(0);
11986
11987 if (!VT.isSimple())
11988 return SDValue();
11989
11990 EVT SHVT = getShiftAmountTy(VT, DAG.getDataLayout());
11991 SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
11992 switch (VT.getSimpleVT().getScalarType().SimpleTy) {
11993 default:
11994 return SDValue();
11995 case MVT::i16:
11996 // Use a rotate by 8. This can be further expanded if necessary.
11997 return DAG.getNode(ISD::ROTL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
11998 case MVT::i32:
11999 // This is meant for ARM specifically, which has ROTR but no ROTL.
12000 // t = x ^ rotr(x, 16)
12001 // t = bic(t, 0x00ff0000)
12002 // t = lshr(t, 8)
12003 // x = t ^ rotr(x, 8)
12005 SDValue Rotr16 =
12006 DAG.getNode(ISD::ROTR, dl, VT, Op, DAG.getConstant(16, dl, SHVT));
12007 SDValue Tmp = DAG.getNode(ISD::XOR, dl, VT, Op, Rotr16);
12008 Tmp = DAG.getNode(ISD::AND, dl, VT, Tmp,
12009 DAG.getConstant(0xFF00FFFF, dl, VT));
12010 Tmp = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(8, dl, SHVT));
12011 SDValue Rotr8 =
12012 DAG.getNode(ISD::ROTR, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
12013 return DAG.getNode(ISD::XOR, dl, VT, Tmp, Rotr8);
12014 }
12015 Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
12016 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Op,
12017 DAG.getConstant(0xFF00, dl, VT));
12018 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(8, dl, SHVT));
12019 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
12020 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT));
12021 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
12022 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
12023 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
12024 return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
12025 case MVT::i64:
12026 Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
12027 Tmp7 = DAG.getNode(ISD::AND, dl, VT, Op,
12028 DAG.getConstant(255ULL<<8, dl, VT));
12029 Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Tmp7, DAG.getConstant(40, dl, SHVT));
12030 Tmp6 = DAG.getNode(ISD::AND, dl, VT, Op,
12031 DAG.getConstant(255ULL<<16, dl, VT));
12032 Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Tmp6, DAG.getConstant(24, dl, SHVT));
12033 Tmp5 = DAG.getNode(ISD::AND, dl, VT, Op,
12034 DAG.getConstant(255ULL<<24, dl, VT));
12035 Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Tmp5, DAG.getConstant(8, dl, SHVT));
12036 Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
12037 Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4,
12038 DAG.getConstant(255ULL<<24, dl, VT));
12039 Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
12040 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
12041 DAG.getConstant(255ULL<<16, dl, VT));
12042 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
12043 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2,
12044 DAG.getConstant(255ULL<<8, dl, VT));
12045 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
12046 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7);
12047 Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5);
12048 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
12049 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
12050 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6);
12051 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
12052 return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4);
12053 }
12054}
12055
12057 SDLoc dl(N);
12058 EVT VT = N->getValueType(0);
12059 SDValue Op = N->getOperand(0);
12060 EVT SHVT = getShiftAmountTy(VT, DAG.getDataLayout());
12061 unsigned Sz = VT.getScalarSizeInBits();
12062
12063 SDValue Tmp, Tmp2, Tmp3;
12064
12065 // If we can, perform BSWAP first and then the mask+swap the i4, then i2
12066 // and finally the i1 pairs.
12067 // TODO: We can easily support i4/i2 legal types if any target ever does.
12068 if (Sz >= 8 && isPowerOf2_32(Sz)) {
12069 // Create the masks - repeating the pattern every byte.
12070 APInt Mask4 = APInt::getSplat(Sz, APInt(8, 0x0F));
12071 APInt Mask2 = APInt::getSplat(Sz, APInt(8, 0x33));
12072 APInt Mask1 = APInt::getSplat(Sz, APInt(8, 0x55));
12073
12074 // BSWAP if the type is wider than a single byte.
12075 Tmp = (Sz > 8 ? DAG.getNode(ISD::BSWAP, dl, VT, Op) : Op);
12076
12077 // swap i4: ((V >> 4) & 0x0F) | ((V & 0x0F) << 4)
12078 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(4, dl, SHVT));
12079 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Mask4, dl, VT));
12080 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(Mask4, dl, VT));
12081 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, SHVT));
12082 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
12083
12084 // swap i2: ((V >> 2) & 0x33) | ((V & 0x33) << 2)
12085 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(2, dl, SHVT));
12086 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Mask2, dl, VT));
12087 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(Mask2, dl, VT));
12088 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, SHVT));
12089 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
12090
12091 // swap i1: ((V >> 1) & 0x55) | ((V & 0x55) << 1)
12092 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(1, dl, SHVT));
12093 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Mask1, dl, VT));
12094 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(Mask1, dl, VT));
12095 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, SHVT));
12096 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
12097 return Tmp;
12098 }
12099
12100 Tmp = DAG.getConstant(0, dl, VT);
12101 for (unsigned I = 0, J = Sz-1; I < Sz; ++I, --J) {
12102 if (I < J)
12103 Tmp2 =
12104 DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(J - I, dl, SHVT));
12105 else
12106 Tmp2 =
12107 DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(I - J, dl, SHVT));
12108
12109 APInt Shift = APInt::getOneBitSet(Sz, J);
12110 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Shift, dl, VT));
12111 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp, Tmp2);
12112 }
12113
12114 return Tmp;
12115}
12116
12117std::pair<SDValue, SDValue>
12119 SelectionDAG &DAG) const {
12120 SDLoc SL(LD);
12121 SDValue Chain = LD->getChain();
12122 SDValue BasePTR = LD->getBasePtr();
12123 EVT SrcVT = LD->getMemoryVT();
12124 EVT DstVT = LD->getValueType(0);
12125 ISD::LoadExtType ExtType = LD->getExtensionType();
12126
12127 if (SrcVT.isScalableVector())
12128 report_fatal_error("Cannot scalarize scalable vector loads");
12129
12130 unsigned NumElem = SrcVT.getVectorNumElements();
12131
12132 EVT SrcEltVT = SrcVT.getScalarType();
12133 EVT DstEltVT = DstVT.getScalarType();
12134
12135 // A vector must always be stored in memory as-is, i.e. without any padding
12136 // between the elements, since various code depend on it, e.g. in the
12137 // handling of a bitcast of a vector type to int, which may be done with a
12138 // vector store followed by an integer load. A vector that does not have
12139 // elements that are byte-sized must therefore be stored as an integer
12140 // built out of the extracted vector elements.
12141 if (!SrcEltVT.isByteSized()) {
12142 unsigned NumLoadBits = SrcVT.getStoreSizeInBits();
12143 EVT LoadVT = EVT::getIntegerVT(*DAG.getContext(), NumLoadBits);
12144
12145 unsigned NumSrcBits = SrcVT.getSizeInBits();
12146 EVT SrcIntVT = EVT::getIntegerVT(*DAG.getContext(), NumSrcBits);
12147
12148 unsigned SrcEltBits = SrcEltVT.getSizeInBits();
12149 SDValue SrcEltBitMask = DAG.getConstant(
12150 APInt::getLowBitsSet(NumLoadBits, SrcEltBits), SL, LoadVT);
12151
12152 // Load the whole vector and avoid masking off the top bits as it makes
12153 // the codegen worse.
12154 SDValue Load = DAG.getExtLoad(
12155 ISD::EXTLOAD, SL, LoadVT, Chain, BasePTR, LD->getPointerInfo(),
12156 SrcIntVT, LD->getBaseAlign(), LD->getMemOperand()->getFlags(),
12157 LD->getMMOMetadataForSubAccess());
12158
12160 for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
12161 unsigned ShiftIntoIdx =
12162 (DAG.getDataLayout().isBigEndian() ? (NumElem - 1) - Idx : Idx);
12163 SDValue ShiftAmount = DAG.getShiftAmountConstant(
12164 ShiftIntoIdx * SrcEltVT.getSizeInBits(), LoadVT, SL);
12165 SDValue ShiftedElt = DAG.getNode(ISD::SRL, SL, LoadVT, Load, ShiftAmount);
12166 SDValue Elt =
12167 DAG.getNode(ISD::AND, SL, LoadVT, ShiftedElt, SrcEltBitMask);
12168 SDValue Scalar = DAG.getNode(ISD::TRUNCATE, SL, SrcEltVT, Elt);
12169
12170 if (ExtType != ISD::NON_EXTLOAD) {
12171 unsigned ExtendOp = ISD::getExtForLoadExtType(false, ExtType);
12172 Scalar = DAG.getNode(ExtendOp, SL, DstEltVT, Scalar);
12173 }
12174
12175 Vals.push_back(Scalar);
12176 }
12177
12178 SDValue Value = DAG.getBuildVector(DstVT, SL, Vals);
12179 return std::make_pair(Value, Load.getValue(1));
12180 }
12181
12182 unsigned Stride = SrcEltVT.getSizeInBits() / 8;
12183 assert(SrcEltVT.isByteSized());
12184
12186 SmallVector<SDValue, 8> LoadChains;
12187
12188 for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
12189 SDValue ScalarLoad = DAG.getExtLoad(
12190 ExtType, SL, DstEltVT, Chain, BasePTR,
12191 LD->getPointerInfo().getWithOffset(Idx * Stride), SrcEltVT,
12192 LD->getBaseAlign(), LD->getMemOperand()->getFlags(),
12193 LD->getMMOMetadataForSubAccess());
12194
12195 BasePTR = DAG.getObjectPtrOffset(SL, BasePTR, TypeSize::getFixed(Stride));
12196
12197 Vals.push_back(ScalarLoad.getValue(0));
12198 LoadChains.push_back(ScalarLoad.getValue(1));
12199 }
12200
12201 SDValue NewChain = DAG.getNode(ISD::TokenFactor, SL, MVT::Other, LoadChains);
12202 SDValue Value = DAG.getBuildVector(DstVT, SL, Vals);
12203
12204 return std::make_pair(Value, NewChain);
12205}
12206
12208 SelectionDAG &DAG) const {
12209 SDLoc SL(ST);
12210
12211 SDValue Chain = ST->getChain();
12212 SDValue BasePtr = ST->getBasePtr();
12213 SDValue Value = ST->getValue();
12214 EVT StVT = ST->getMemoryVT();
12215
12216 if (StVT.isScalableVector())
12217 report_fatal_error("Cannot scalarize scalable vector stores");
12218
12219 // The type of the data we want to save
12220 EVT RegVT = Value.getValueType();
12221 EVT RegSclVT = RegVT.getScalarType();
12222
12223 // The type of data as saved in memory.
12224 EVT MemSclVT = StVT.getScalarType();
12225
12226 unsigned NumElem = StVT.getVectorNumElements();
12227
12228 // A vector must always be stored in memory as-is, i.e. without any padding
12229 // between the elements, since various code depend on it, e.g. in the
12230 // handling of a bitcast of a vector type to int, which may be done with a
12231 // vector store followed by an integer load. A vector that does not have
12232 // elements that are byte-sized must therefore be stored as an integer
12233 // built out of the extracted vector elements.
12234 if (!MemSclVT.isByteSized()) {
12235 unsigned NumBits = StVT.getSizeInBits();
12236 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);
12237
12238 SDValue CurrVal = DAG.getConstant(0, SL, IntVT);
12239
12240 for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
12241 SDValue Elt = DAG.getExtractVectorElt(SL, RegSclVT, Value, Idx);
12242 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, MemSclVT, Elt);
12243 SDValue ExtElt = DAG.getNode(ISD::ZERO_EXTEND, SL, IntVT, Trunc);
12244 unsigned ShiftIntoIdx =
12245 (DAG.getDataLayout().isBigEndian() ? (NumElem - 1) - Idx : Idx);
12246 SDValue ShiftAmount =
12247 DAG.getConstant(ShiftIntoIdx * MemSclVT.getSizeInBits(), SL, IntVT);
12248 SDValue ShiftedElt =
12249 DAG.getNode(ISD::SHL, SL, IntVT, ExtElt, ShiftAmount);
12250 CurrVal = DAG.getNode(ISD::OR, SL, IntVT, CurrVal, ShiftedElt);
12251 }
12252
12253 return DAG.getStore(Chain, SL, CurrVal, BasePtr, ST->getPointerInfo(),
12254 ST->getBaseAlign(), ST->getMemOperand()->getFlags(),
12255 ST->getMMOMetadataForSubAccess());
12256 }
12257
12258 // Store Stride in bytes
12259 unsigned Stride = MemSclVT.getSizeInBits() / 8;
12260 assert(Stride && "Zero stride!");
12261 // Extract each of the elements from the original vector and save them into
12262 // memory individually.
12264 for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
12265 SDValue Elt = DAG.getExtractVectorElt(SL, RegSclVT, Value, Idx);
12266
12267 SDValue Ptr =
12268 DAG.getObjectPtrOffset(SL, BasePtr, TypeSize::getFixed(Idx * Stride));
12269
12270 // This scalar TruncStore may be illegal, but we legalize it later.
12272 Chain, SL, Elt, Ptr, ST->getPointerInfo().getWithOffset(Idx * Stride),
12273 MemSclVT, ST->getBaseAlign(), ST->getMemOperand()->getFlags(),
12274 ST->getMMOMetadataForSubAccess());
12275
12276 Stores.push_back(Store);
12277 }
12278
12279 return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, Stores);
12280}
12281
12282std::pair<SDValue, SDValue>
12284 assert(LD->getAddressingMode() == ISD::UNINDEXED &&
12285 "unaligned indexed loads not implemented!");
12286 SDValue Chain = LD->getChain();
12287 SDValue Ptr = LD->getBasePtr();
12288 EVT VT = LD->getValueType(0);
12289 EVT LoadedVT = LD->getMemoryVT();
12290 SDLoc dl(LD);
12291 auto &MF = DAG.getMachineFunction();
12292
12293 if (VT.isFloatingPoint() || VT.isVector()) {
12294 EVT intVT = EVT::getIntegerVT(*DAG.getContext(), LoadedVT.getSizeInBits());
12295 if (isTypeLegal(intVT) && isTypeLegal(LoadedVT)) {
12296 if (!isOperationLegalOrCustom(ISD::LOAD, intVT) &&
12297 LoadedVT.isVector()) {
12298 // Scalarize the load and let the individual components be handled.
12299 return scalarizeVectorLoad(LD, DAG);
12300 }
12301
12302 // Expand to a (misaligned) integer load of the same size,
12303 // then bitconvert to floating point or vector.
12304 SDValue newLoad = DAG.getLoad(intVT, dl, Chain, Ptr,
12305 LD->getMemOperand());
12306 SDValue Result = DAG.getNode(ISD::BITCAST, dl, LoadedVT, newLoad);
12307 if (LoadedVT != VT)
12308 Result = DAG.getNode(VT.isFloatingPoint() ? ISD::FP_EXTEND :
12309 ISD::ANY_EXTEND, dl, VT, Result);
12310
12311 return std::make_pair(Result, newLoad.getValue(1));
12312 }
12313
12314 // Copy the value to a (aligned) stack slot using (unaligned) integer
12315 // loads and stores, then do a (aligned) load from the stack slot.
12316 MVT RegVT = getRegisterType(*DAG.getContext(), intVT);
12317 unsigned LoadedBytes = LoadedVT.getStoreSize();
12318 unsigned RegBytes = RegVT.getSizeInBits() / 8;
12319 unsigned NumRegs = (LoadedBytes + RegBytes - 1) / RegBytes;
12320
12321 // Make sure the stack slot is also aligned for the register type.
12322 SDValue StackBase = DAG.CreateStackTemporary(LoadedVT, RegVT);
12323 auto FrameIndex = cast<FrameIndexSDNode>(StackBase.getNode())->getIndex();
12325 SDValue StackPtr = StackBase;
12326 unsigned Offset = 0;
12327
12328 EVT PtrVT = Ptr.getValueType();
12329 EVT StackPtrVT = StackPtr.getValueType();
12330
12331 SDValue PtrIncrement = DAG.getConstant(RegBytes, dl, PtrVT);
12332 SDValue StackPtrIncrement = DAG.getConstant(RegBytes, dl, StackPtrVT);
12333
12334 // Do all but one copies using the full register width.
12335 for (unsigned i = 1; i < NumRegs; i++) {
12336 // Load one integer register's worth from the original location.
12337 SDValue Load = DAG.getLoad(
12338 RegVT, dl, Chain, Ptr, LD->getPointerInfo().getWithOffset(Offset),
12339 LD->getBaseAlign(), LD->getMemOperand()->getFlags(),
12340 LD->getMMOMetadataForSubAccess());
12341 // Follow the load with a store to the stack slot. Remember the store.
12342 Stores.push_back(DAG.getStore(
12343 Load.getValue(1), dl, Load, StackPtr,
12344 MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset)));
12345 // Increment the pointers.
12346 Offset += RegBytes;
12347
12348 Ptr = DAG.getObjectPtrOffset(dl, Ptr, PtrIncrement);
12349 StackPtr = DAG.getObjectPtrOffset(dl, StackPtr, StackPtrIncrement);
12350 }
12351
12352 // The last copy may be partial. Do an extending load.
12353 EVT MemVT = EVT::getIntegerVT(*DAG.getContext(),
12354 8 * (LoadedBytes - Offset));
12355 SDValue Load = DAG.getExtLoad(
12356 ISD::EXTLOAD, dl, RegVT, Chain, Ptr,
12357 LD->getPointerInfo().getWithOffset(Offset), MemVT, LD->getBaseAlign(),
12358 LD->getMemOperand()->getFlags(), LD->getMMOMetadataForSubAccess());
12359 // Follow the load with a store to the stack slot. Remember the store.
12360 // On big-endian machines this requires a truncating store to ensure
12361 // that the bits end up in the right place.
12362 Stores.push_back(DAG.getTruncStore(
12363 Load.getValue(1), dl, Load, StackPtr,
12364 MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset), MemVT));
12365
12366 // The order of the stores doesn't matter - say it with a TokenFactor.
12367 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
12368
12369 // Finally, perform the original load only redirected to the stack slot.
12370 Load = DAG.getExtLoad(LD->getExtensionType(), dl, VT, TF, StackBase,
12371 MachinePointerInfo::getFixedStack(MF, FrameIndex, 0),
12372 LoadedVT);
12373
12374 // Callers expect a MERGE_VALUES node.
12375 return std::make_pair(Load, TF);
12376 }
12377
12378 assert(LoadedVT.isInteger() && !LoadedVT.isVector() &&
12379 "Unaligned load of unsupported type.");
12380
12381 // Compute the new VT that is half the size of the old one. This is an
12382 // integer MVT.
12383 unsigned NumBits = LoadedVT.getSizeInBits();
12384 EVT NewLoadedVT;
12385 NewLoadedVT = EVT::getIntegerVT(*DAG.getContext(), NumBits/2);
12386 NumBits >>= 1;
12387
12388 Align Alignment = LD->getBaseAlign();
12389 unsigned IncrementSize = NumBits / 8;
12390 ISD::LoadExtType HiExtType = LD->getExtensionType();
12391
12392 // If the original load is NON_EXTLOAD, the hi part load must be ZEXTLOAD.
12393 if (HiExtType == ISD::NON_EXTLOAD)
12394 HiExtType = ISD::ZEXTLOAD;
12395
12396 // Load the value in two parts
12397 SDValue Lo, Hi;
12398 if (DAG.getDataLayout().isLittleEndian()) {
12399 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, LD->getPointerInfo(),
12400 NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
12401 LD->getMMOMetadataForSubAccess());
12402
12403 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::getFixed(IncrementSize));
12404 Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr,
12405 LD->getPointerInfo().getWithOffset(IncrementSize),
12406 NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
12407 LD->getMMOMetadataForSubAccess());
12408 } else {
12409 Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr, LD->getPointerInfo(),
12410 NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
12411 LD->getMMOMetadataForSubAccess());
12412
12413 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::getFixed(IncrementSize));
12414 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr,
12415 LD->getPointerInfo().getWithOffset(IncrementSize),
12416 NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
12417 LD->getMMOMetadataForSubAccess());
12418 }
12419
12420 // aggregate the two parts
12421 SDValue ShiftAmount = DAG.getShiftAmountConstant(NumBits, VT, dl);
12422 SDValue Result = DAG.getNode(ISD::SHL, dl, VT, Hi, ShiftAmount);
12423 Result = DAG.getNode(ISD::OR, dl, VT, Result, Lo);
12424
12425 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
12426 Hi.getValue(1));
12427
12428 return std::make_pair(Result, TF);
12429}
12430
12432 SelectionDAG &DAG) const {
12433 assert(ST->getAddressingMode() == ISD::UNINDEXED &&
12434 "unaligned indexed stores not implemented!");
12435 SDValue Chain = ST->getChain();
12436 SDValue Ptr = ST->getBasePtr();
12437 SDValue Val = ST->getValue();
12438 EVT VT = Val.getValueType();
12439 Align Alignment = ST->getBaseAlign();
12440 auto &MF = DAG.getMachineFunction();
12441 EVT StoreMemVT = ST->getMemoryVT();
12442
12443 SDLoc dl(ST);
12444 if (StoreMemVT.isFloatingPoint() || StoreMemVT.isVector()) {
12445 EVT intVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
12446 if (isTypeLegal(intVT)) {
12447 if (!isOperationLegalOrCustom(ISD::STORE, intVT) &&
12448 StoreMemVT.isVector()) {
12449 // Scalarize the store and let the individual components be handled.
12450 SDValue Result = scalarizeVectorStore(ST, DAG);
12451 return Result;
12452 }
12453 // Expand to a bitconvert of the value to the integer type of the
12454 // same size, then a (misaligned) int store.
12455 // FIXME: Does not handle truncating floating point stores!
12456 SDValue Result = DAG.getNode(ISD::BITCAST, dl, intVT, Val);
12457 Result = DAG.getStore(Chain, dl, Result, Ptr, ST->getPointerInfo(),
12458 Alignment, ST->getMemOperand()->getFlags(),
12459 ST->getMMOMetadataForSubAccess());
12460 return Result;
12461 }
12462 // Do a (aligned) store to a stack slot, then copy from the stack slot
12463 // to the final destination using (unaligned) integer loads and stores.
12464 MVT RegVT = getRegisterType(
12465 *DAG.getContext(),
12466 EVT::getIntegerVT(*DAG.getContext(), StoreMemVT.getSizeInBits()));
12467 EVT PtrVT = Ptr.getValueType();
12468 unsigned StoredBytes = StoreMemVT.getStoreSize();
12469 unsigned RegBytes = RegVT.getSizeInBits() / 8;
12470 unsigned NumRegs = (StoredBytes + RegBytes - 1) / RegBytes;
12471
12472 // Make sure the stack slot is also aligned for the register type.
12473 SDValue StackPtr = DAG.CreateStackTemporary(StoreMemVT, RegVT);
12474 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
12475
12476 // Perform the original store, only redirected to the stack slot.
12478 Chain, dl, Val, StackPtr,
12479 MachinePointerInfo::getFixedStack(MF, FrameIndex, 0), StoreMemVT);
12480
12481 EVT StackPtrVT = StackPtr.getValueType();
12482
12483 SDValue PtrIncrement = DAG.getConstant(RegBytes, dl, PtrVT);
12484 SDValue StackPtrIncrement = DAG.getConstant(RegBytes, dl, StackPtrVT);
12486 unsigned Offset = 0;
12487
12488 // Do all but one copies using the full register width.
12489 for (unsigned i = 1; i < NumRegs; i++) {
12490 // Load one integer register's worth from the stack slot.
12491 SDValue Load = DAG.getLoad(
12492 RegVT, dl, Store, StackPtr,
12493 MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset));
12494 // Store it to the final location. Remember the store.
12495 Stores.push_back(DAG.getStore(
12496 Load.getValue(1), dl, Load, Ptr,
12497 ST->getPointerInfo().getWithOffset(Offset), ST->getBaseAlign(),
12498 ST->getMemOperand()->getFlags(), ST->getMMOMetadataForSubAccess()));
12499 // Increment the pointers.
12500 Offset += RegBytes;
12501 StackPtr = DAG.getObjectPtrOffset(dl, StackPtr, StackPtrIncrement);
12502 Ptr = DAG.getObjectPtrOffset(dl, Ptr, PtrIncrement);
12503 }
12504
12505 // The last store may be partial. Do a truncating store. On big-endian
12506 // machines this requires an extending load from the stack slot to ensure
12507 // that the bits are in the right place.
12508 EVT LoadMemVT =
12509 EVT::getIntegerVT(*DAG.getContext(), 8 * (StoredBytes - Offset));
12510
12511 // Load from the stack slot.
12512 SDValue Load = DAG.getExtLoad(
12513 ISD::EXTLOAD, dl, RegVT, Store, StackPtr,
12514 MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset), LoadMemVT);
12515
12516 Stores.push_back(
12517 DAG.getTruncStore(Load.getValue(1), dl, Load, Ptr,
12518 ST->getPointerInfo().getWithOffset(Offset), LoadMemVT,
12519 ST->getBaseAlign(), ST->getMemOperand()->getFlags(),
12520 ST->getMMOMetadataForSubAccess()));
12521 // The order of the stores doesn't matter - say it with a TokenFactor.
12522 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
12523 return Result;
12524 }
12525
12526 assert(StoreMemVT.isInteger() && !StoreMemVT.isVector() &&
12527 "Unaligned store of unknown type.");
12528 // Get the half-size VT
12529 EVT NewStoredVT = StoreMemVT.getHalfSizedIntegerVT(*DAG.getContext());
12530 unsigned NumBits = NewStoredVT.getFixedSizeInBits();
12531 unsigned IncrementSize = NumBits / 8;
12532
12533 // Divide the stored value in two parts.
12534 SDValue ShiftAmount =
12535 DAG.getShiftAmountConstant(NumBits, Val.getValueType(), dl);
12536 SDValue Lo = Val;
12537 // If Val is a constant, replace the upper bits with 0. The SRL will constant
12538 // fold and not use the upper bits. A smaller constant may be easier to
12539 // materialize.
12540 if (auto *C = dyn_cast<ConstantSDNode>(Lo); C && !C->isOpaque())
12541 Lo = DAG.getNode(
12542 ISD::AND, dl, VT, Lo,
12543 DAG.getConstant(APInt::getLowBitsSet(VT.getSizeInBits(), NumBits), dl,
12544 VT));
12545 SDValue Hi = DAG.getNode(ISD::SRL, dl, VT, Val, ShiftAmount);
12546
12547 // Store the two parts
12548 SDValue Store1, Store2;
12549 Store1 = DAG.getTruncStore(
12550 Chain, dl, DAG.getDataLayout().isLittleEndian() ? Lo : Hi, Ptr,
12551 ST->getPointerInfo(), NewStoredVT, Alignment,
12552 ST->getMemOperand()->getFlags(), ST->getMMOMetadataForSubAccess());
12553
12554 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::getFixed(IncrementSize));
12555 Store2 = DAG.getTruncStore(
12556 Chain, dl, DAG.getDataLayout().isLittleEndian() ? Hi : Lo, Ptr,
12557 ST->getPointerInfo().getWithOffset(IncrementSize), NewStoredVT, Alignment,
12558 ST->getMemOperand()->getFlags(), ST->getMMOMetadataForSubAccess());
12559
12560 SDValue Result =
12561 DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2);
12562 return Result;
12563}
12564
12565SDValue
12567 const SDLoc &DL, EVT DataVT,
12568 SelectionDAG &DAG,
12569 bool IsCompressedMemory) const {
12571 EVT AddrVT = Addr.getValueType();
12572 EVT MaskVT = Mask.getValueType();
12573 assert(DataVT.getVectorElementCount() == MaskVT.getVectorElementCount() &&
12574 "Incompatible types of Data and Mask");
12575 if (IsCompressedMemory) {
12576 // Incrementing the pointer according to number of '1's in the mask.
12577 if (DataVT.isScalableVector()) {
12578 EVT MaskExtVT = MaskVT.changeElementType(*DAG.getContext(), MVT::i32);
12579 SDValue MaskExt = DAG.getNode(ISD::ZERO_EXTEND, DL, MaskExtVT, Mask);
12580 Increment = DAG.getNode(ISD::VECREDUCE_ADD, DL, MVT::i32, MaskExt);
12581 } else {
12582 EVT MaskIntVT =
12583 EVT::getIntegerVT(*DAG.getContext(), MaskVT.getSizeInBits());
12584 SDValue MaskInIntReg = DAG.getBitcast(MaskIntVT, Mask);
12585 if (MaskIntVT.getSizeInBits() < 32) {
12586 MaskInIntReg =
12587 DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, MaskInIntReg);
12588 MaskIntVT = MVT::i32;
12589 }
12590 Increment = DAG.getNode(ISD::CTPOP, DL, MaskIntVT, MaskInIntReg);
12591 }
12592 // Scale is an element size in bytes.
12593 SDValue Scale = DAG.getConstant(DataVT.getScalarSizeInBits() / 8, DL,
12594 AddrVT);
12595 Increment = DAG.getZExtOrTrunc(Increment, DL, AddrVT);
12596 Increment = DAG.getNode(ISD::MUL, DL, AddrVT, Increment, Scale);
12597 } else
12598 Increment = DAG.getTypeSize(DL, AddrVT, DataVT.getStoreSize());
12599
12600 return DAG.getNode(ISD::ADD, DL, AddrVT, Addr, Increment);
12601}
12602
12604 EVT VecVT, const SDLoc &DL,
12605 ElementCount SubEC) {
12606 assert(!(SubEC.isScalable() && VecVT.isFixedLengthVector()) &&
12607 "Cannot index a scalable vector within a fixed-width vector");
12608 ElementCount NElts = VecVT.getVectorElementCount();
12609 EVT IdxVT = Idx.getValueType();
12610
12611 // Attempt to prove all accessed lanes sit within VecVT.
12612 ElementCount IdxAsEC;
12613 if (sd_match(Idx, m_ElementCount(IdxAsEC))) {
12614 if (SubEC.isScalar() && ElementCount::isKnownLT(IdxAsEC, NElts))
12615 return Idx;
12616 // Calculate end index as TypeSize to remove possibility of overflow.
12617 if (IdxAsEC.isScalable() == SubEC.isScalable() &&
12618 TypeSize::isKnownLE(TypeSize(IdxAsEC) + TypeSize(SubEC),
12619 TypeSize(NElts)))
12620 return Idx;
12621 }
12622
12623 // Can we mask Idx rather than clamp it.
12624 if (SubEC.isScalar() && NElts.isPowerOf2()) {
12625 SDValue EC = DAG.getElementCount(DL, IdxVT, NElts);
12626 SDValue Mask =
12627 DAG.getNode(ISD::SUB, DL, IdxVT, EC, DAG.getConstant(1, DL, IdxVT));
12628 return DAG.getNode(ISD::AND, DL, IdxVT, Idx, Mask);
12629 }
12630
12631 unsigned SubOpcode =
12633 SDValue MaxSafeIndex =
12634 DAG.getNode(SubOpcode, DL, IdxVT, DAG.getElementCount(DL, IdxVT, NElts),
12635 DAG.getElementCount(DL, IdxVT, SubEC));
12636 return DAG.getNode(ISD::UMIN, DL, IdxVT, Idx, MaxSafeIndex);
12637}
12638
12639SDValue
12641 EVT VecVT, SDValue Index,
12642 const SDNodeFlags PtrArithFlags) const {
12644 DAG, VecPtr, VecVT,
12646 Index, PtrArithFlags);
12647}
12648
12649SDValue
12651 EVT VecVT, EVT SubVecVT, SDValue Index,
12652 const SDNodeFlags PtrArithFlags) const {
12653 SDLoc dl(Index);
12654 // Make sure the index type is big enough to compute in.
12655 Index = DAG.getZExtOrTrunc(Index, dl, VecPtr.getValueType());
12656
12657 EVT EltVT = VecVT.getVectorElementType();
12658
12659 // Calculate the element offset and add it to the pointer.
12660 unsigned EltSize = EltVT.getFixedSizeInBits() / 8; // FIXME: should be ABI size.
12661 assert(EltSize * 8 == EltVT.getFixedSizeInBits() &&
12662 "Converting bits to bytes lost precision");
12663 assert(SubVecVT.getVectorElementType() == EltVT &&
12664 "Sub-vector must be a vector with matching element type");
12665 assert((SubVecVT.getVectorElementCount().isScalar() ||
12666 isa<ConstantSDNode>(Index)) &&
12667 "Subvector Index must be constant!");
12668
12669 // An out-of-range index only makes the vector operation return poison, but
12670 // a load/store through the pointer computed below would be immediate UB, so
12671 // freeze the index before clamping it into range.
12672 if (!DAG.isGuaranteedNotToBePoison(Index))
12673 Index = DAG.getFreeze(Index);
12674
12675 EVT IdxVT = Index.getValueType();
12676 if (SubVecVT.isScalableVector())
12677 Index =
12678 DAG.getVScale(dl, IdxVT, cast<ConstantSDNode>(Index)->getAPIntValue());
12679
12680 Index = clampDynamicVectorIndex(DAG, Index, VecVT, dl,
12681 SubVecVT.getVectorElementCount());
12682 Index = DAG.getNode(ISD::MUL, dl, IdxVT, Index,
12683 DAG.getConstant(EltSize, dl, IdxVT));
12684 return DAG.getMemBasePlusOffset(VecPtr, Index, dl, PtrArithFlags);
12685}
12686
12687//===----------------------------------------------------------------------===//
12688// Implementation of Emulated TLS Model
12689//===----------------------------------------------------------------------===//
12690
12692 SelectionDAG &DAG) const {
12693 // Access to address of TLS varialbe xyz is lowered to a function call:
12694 // __emutls_get_address( address of global variable named "__emutls_v.xyz" )
12695 EVT PtrVT = getPointerTy(DAG.getDataLayout());
12696 PointerType *VoidPtrType = PointerType::get(*DAG.getContext(), 0);
12697 SDLoc dl(GA);
12698
12699 ArgListTy Args;
12700 const GlobalValue *GV =
12702 SmallString<32> NameString("__emutls_v.");
12703 NameString += GV->getName();
12704 StringRef EmuTlsVarName(NameString);
12705 const GlobalVariable *EmuTlsVar =
12706 GV->getParent()->getNamedGlobal(EmuTlsVarName);
12707 assert(EmuTlsVar && "Cannot find EmuTlsVar ");
12708 Args.emplace_back(DAG.getGlobalAddress(EmuTlsVar, dl, PtrVT), VoidPtrType);
12709
12710 SDValue EmuTlsGetAddr = DAG.getExternalSymbol("__emutls_get_address", PtrVT);
12711
12713 CLI.setDebugLoc(dl).setChain(DAG.getEntryNode());
12714 CLI.setLibCallee(CallingConv::C, VoidPtrType, EmuTlsGetAddr, std::move(Args));
12715 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
12716
12717 // TLSADDR will be codegen'ed as call. Inform MFI that function has calls.
12718 // At last for X86 targets, maybe good for other targets too?
12720 MFI.setAdjustsStack(true); // Is this only for X86 target?
12721 MFI.setHasCalls(true);
12722
12723 assert((GA->getOffset() == 0) &&
12724 "Emulated TLS must have zero offset in GlobalAddressSDNode");
12725 return CallResult.first;
12726}
12727
12729 SelectionDAG &DAG) const {
12730 assert((Op->getOpcode() == ISD::SETCC) && "Input has to be a SETCC node.");
12731 if (!isCtlzFast())
12732 return SDValue();
12733 ISD::CondCode CC = Op.getOperandCondCode(2);
12734 SDLoc dl(Op);
12735 if (isNullConstant(Op.getOperand(1)) && CC == ISD::SETEQ) {
12736 EVT VT = Op.getOperand(0).getValueType();
12737 SDValue Zext = Op.getOperand(0);
12738 if (VT.bitsLT(MVT::i32)) {
12739 VT = MVT::i32;
12740 Zext = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Op.getOperand(0));
12741 }
12742 unsigned Log2b = Log2_32(VT.getSizeInBits());
12743 SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Zext);
12744 SDValue Scc = DAG.getNode(ISD::SRL, dl, VT, Clz,
12745 DAG.getConstant(Log2b, dl, MVT::i32));
12746 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Scc);
12747 }
12748 return SDValue();
12749}
12750
12752 SDValue Op0 = Node->getOperand(0);
12753 SDValue Op1 = Node->getOperand(1);
12754 EVT VT = Op0.getValueType();
12755 EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
12756 unsigned Opcode = Node->getOpcode();
12757 SDLoc DL(Node);
12758
12759 // If both sign bits are zero, flip UMIN/UMAX <-> SMIN/SMAX if legal.
12760 unsigned AltOpcode = ISD::getOppositeSignednessMinMaxOpcode(Opcode);
12761 if (isOperationLegal(AltOpcode, VT) && DAG.SignBitIsZero(Op0) &&
12762 DAG.SignBitIsZero(Op1))
12763 return DAG.getNode(AltOpcode, DL, VT, Op0, Op1);
12764
12765 // umax(x,1) --> sub(x,cmpeq(x,0)) iff cmp result is allbits
12766 if (Opcode == ISD::UMAX && llvm::isOneOrOneSplat(Op1, true) && BoolVT == VT &&
12768 Op0 = DAG.getFreeze(Op0);
12769 SDValue Zero = DAG.getConstant(0, DL, VT);
12770 return DAG.getNode(ISD::SUB, DL, VT, Op0,
12771 DAG.getSetCC(DL, VT, Op0, Zero, ISD::SETEQ));
12772 }
12773
12774 // umin(x,y) -> sub(x,usubsat(x,y))
12775 // TODO: Missing freeze(Op0)?
12776 if (Opcode == ISD::UMIN && isOperationLegal(ISD::SUB, VT) &&
12778 return DAG.getNode(ISD::SUB, DL, VT, Op0,
12779 DAG.getNode(ISD::USUBSAT, DL, VT, Op0, Op1));
12780 }
12781
12782 // umax(x,y) -> add(x,usubsat(y,x))
12783 // TODO: Missing freeze(Op0)?
12784 if (Opcode == ISD::UMAX && isOperationLegal(ISD::ADD, VT) &&
12786 return DAG.getNode(ISD::ADD, DL, VT, Op0,
12787 DAG.getNode(ISD::USUBSAT, DL, VT, Op1, Op0));
12788 }
12789
12790 // FIXME: Should really try to split the vector in case it's legal on a
12791 // subvector.
12793 return DAG.UnrollVectorOp(Node);
12794
12795 // Attempt to find an existing SETCC node that we can reuse.
12796 // TODO: Do we need a generic doesSETCCNodeExist?
12797 // TODO: Missing freeze(Op0)/freeze(Op1)?
12798 auto buildMinMax = [&](ISD::CondCode PrefCC, ISD::CondCode AltCC,
12799 ISD::CondCode PrefCommuteCC,
12800 ISD::CondCode AltCommuteCC) {
12801 SDVTList BoolVTList = DAG.getVTList(BoolVT);
12802 for (ISD::CondCode CC : {PrefCC, AltCC}) {
12803 if (DAG.doesNodeExist(ISD::SETCC, BoolVTList,
12804 {Op0, Op1, DAG.getCondCode(CC)})) {
12805 SDValue Cond = DAG.getSetCC(DL, BoolVT, Op0, Op1, CC);
12806 return DAG.getSelect(DL, VT, Cond, Op0, Op1);
12807 }
12808 }
12809 for (ISD::CondCode CC : {PrefCommuteCC, AltCommuteCC}) {
12810 if (DAG.doesNodeExist(ISD::SETCC, BoolVTList,
12811 {Op0, Op1, DAG.getCondCode(CC)})) {
12812 SDValue Cond = DAG.getSetCC(DL, BoolVT, Op0, Op1, CC);
12813 return DAG.getSelect(DL, VT, Cond, Op1, Op0);
12814 }
12815 }
12816 SDValue Cond = DAG.getSetCC(DL, BoolVT, Op0, Op1, PrefCC);
12817 return DAG.getSelect(DL, VT, Cond, Op0, Op1);
12818 };
12819
12820 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
12821 // -> Y = (A < B) ? B : A
12822 // -> Y = (A >= B) ? A : B
12823 // -> Y = (A <= B) ? B : A
12824 switch (Opcode) {
12825 case ISD::SMAX:
12826 return buildMinMax(ISD::SETGT, ISD::SETGE, ISD::SETLT, ISD::SETLE);
12827 case ISD::SMIN:
12828 return buildMinMax(ISD::SETLT, ISD::SETLE, ISD::SETGT, ISD::SETGE);
12829 case ISD::UMAX:
12830 return buildMinMax(ISD::SETUGT, ISD::SETUGE, ISD::SETULT, ISD::SETULE);
12831 case ISD::UMIN:
12832 return buildMinMax(ISD::SETULT, ISD::SETULE, ISD::SETUGT, ISD::SETUGE);
12833 }
12834
12835 llvm_unreachable("How did we get here?");
12836}
12837
12839 unsigned Opcode = Node->getOpcode();
12840 SDValue LHS = Node->getOperand(0);
12841 SDValue RHS = Node->getOperand(1);
12842 EVT VT = LHS.getValueType();
12843 SDLoc dl(Node);
12844
12845 assert(VT == RHS.getValueType() && "Expected operands to be the same type");
12846 assert(VT.isInteger() && "Expected operands to be integers");
12847
12848 // usub.sat(a, b) -> umax(a, b) - b
12849 if (Opcode == ISD::USUBSAT && isOperationLegal(ISD::UMAX, VT)) {
12850 SDValue Max = DAG.getNode(ISD::UMAX, dl, VT, LHS, RHS);
12851 return DAG.getNode(ISD::SUB, dl, VT, Max, RHS);
12852 }
12853
12854 // usub.sat(a, 1) -> sub(a, zext(a != 0))
12855 // Prefer this on targets without legal/cost-effective overflow-carry nodes.
12856 if (Opcode == ISD::USUBSAT && isOneOrOneSplat(RHS) &&
12858 LHS = DAG.getFreeze(LHS);
12859 SDValue Zero = DAG.getConstant(0, dl, VT);
12860 EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
12861 SDValue IsNonZero = DAG.getSetCC(dl, BoolVT, LHS, Zero, ISD::SETNE);
12862 SDValue Subtrahend = DAG.getBoolExtOrTrunc(IsNonZero, dl, VT, BoolVT);
12863 Subtrahend =
12864 DAG.getNode(ISD::AND, dl, VT, Subtrahend, DAG.getConstant(1, dl, VT));
12865 return DAG.getNode(ISD::SUB, dl, VT, LHS, Subtrahend);
12866 }
12867
12868 // uadd.sat(a, b) -> umin(a, ~b) + b
12869 if (Opcode == ISD::UADDSAT && isOperationLegal(ISD::UMIN, VT)) {
12870 SDValue InvRHS = DAG.getNOT(dl, RHS, VT);
12871 SDValue Min = DAG.getNode(ISD::UMIN, dl, VT, LHS, InvRHS);
12872 return DAG.getNode(ISD::ADD, dl, VT, Min, RHS);
12873 }
12874
12875 unsigned OverflowOp;
12876 switch (Opcode) {
12877 case ISD::SADDSAT:
12878 OverflowOp = ISD::SADDO;
12879 break;
12880 case ISD::UADDSAT:
12881 OverflowOp = ISD::UADDO;
12882 break;
12883 case ISD::SSUBSAT:
12884 OverflowOp = ISD::SSUBO;
12885 break;
12886 case ISD::USUBSAT:
12887 OverflowOp = ISD::USUBO;
12888 break;
12889 default:
12890 llvm_unreachable("Expected method to receive signed or unsigned saturation "
12891 "addition or subtraction node.");
12892 }
12893
12894 // FIXME: Should really try to split the vector in case it's legal on a
12895 // subvector.
12897 return DAG.UnrollVectorOp(Node);
12898
12899 unsigned BitWidth = LHS.getScalarValueSizeInBits();
12900 EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
12901 SDValue Result = DAG.getNode(OverflowOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
12902 SDValue SumDiff = Result.getValue(0);
12903 SDValue Overflow = Result.getValue(1);
12904 SDValue Zero = DAG.getConstant(0, dl, VT);
12905 SDValue AllOnes = DAG.getAllOnesConstant(dl, VT);
12906
12907 if (Opcode == ISD::UADDSAT) {
12909 // (LHS + RHS) | OverflowMask
12910 SDValue OverflowMask = DAG.getSExtOrTrunc(Overflow, dl, VT);
12911 return DAG.getNode(ISD::OR, dl, VT, SumDiff, OverflowMask);
12912 }
12913 // Overflow ? 0xffff.... : (LHS + RHS)
12914 return DAG.getSelect(dl, VT, Overflow, AllOnes, SumDiff);
12915 }
12916
12917 if (Opcode == ISD::USUBSAT) {
12919 // (LHS - RHS) & ~OverflowMask
12920 SDValue OverflowMask = DAG.getSExtOrTrunc(Overflow, dl, VT);
12921 SDValue Not = DAG.getNOT(dl, OverflowMask, VT);
12922 return DAG.getNode(ISD::AND, dl, VT, SumDiff, Not);
12923 }
12924 // Overflow ? 0 : (LHS - RHS)
12925 return DAG.getSelect(dl, VT, Overflow, Zero, SumDiff);
12926 }
12927
12928 assert((Opcode == ISD::SADDSAT || Opcode == ISD::SSUBSAT) &&
12929 "Expected signed saturating add/sub opcode");
12930
12931 const APInt MinVal = APInt::getSignedMinValue(BitWidth);
12932 const APInt MaxVal = APInt::getSignedMaxValue(BitWidth);
12933
12934 KnownBits KnownLHS = DAG.computeKnownBits(LHS);
12935 KnownBits KnownRHS = DAG.computeKnownBits(RHS);
12936
12937 // If either of the operand signs are known, then they are guaranteed to
12938 // only saturate in one direction. If non-negative they will saturate
12939 // towards SIGNED_MAX, if negative they will saturate towards SIGNED_MIN.
12940 //
12941 // In the case of ISD::SSUBSAT, 'x - y' is equivalent to 'x + (-y)', so the
12942 // sign of 'y' has to be flipped.
12943
12944 bool LHSIsNonNegative = KnownLHS.isNonNegative();
12945 bool RHSIsNonNegative =
12946 Opcode == ISD::SADDSAT ? KnownRHS.isNonNegative() : KnownRHS.isNegative();
12947 if (LHSIsNonNegative || RHSIsNonNegative) {
12948 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
12949 return DAG.getSelect(dl, VT, Overflow, SatMax, SumDiff);
12950 }
12951
12952 bool LHSIsNegative = KnownLHS.isNegative();
12953 bool RHSIsNegative =
12954 Opcode == ISD::SADDSAT ? KnownRHS.isNegative() : KnownRHS.isNonNegative();
12955 if (LHSIsNegative || RHSIsNegative) {
12956 SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
12957 return DAG.getSelect(dl, VT, Overflow, SatMin, SumDiff);
12958 }
12959
12960 // Overflow ? (SumDiff >> BW) ^ MinVal : SumDiff
12961 SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
12962 SDValue Shift = DAG.getNode(ISD::SRA, dl, VT, SumDiff,
12963 DAG.getConstant(BitWidth - 1, dl, VT));
12964 Result = DAG.getNode(ISD::XOR, dl, VT, Shift, SatMin);
12965 return DAG.getSelect(dl, VT, Overflow, Result, SumDiff);
12966}
12967
12969 unsigned Opcode = Node->getOpcode();
12970 SDValue LHS = Node->getOperand(0);
12971 SDValue RHS = Node->getOperand(1);
12972 EVT VT = LHS.getValueType();
12973 EVT ResVT = Node->getValueType(0);
12974 EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
12975 SDLoc dl(Node);
12976
12977 auto LTPredicate = (Opcode == ISD::UCMP ? ISD::SETULT : ISD::SETLT);
12978 auto GTPredicate = (Opcode == ISD::UCMP ? ISD::SETUGT : ISD::SETGT);
12979 SDValue IsLT = DAG.getSetCC(dl, BoolVT, LHS, RHS, LTPredicate);
12980 SDValue IsGT = DAG.getSetCC(dl, BoolVT, LHS, RHS, GTPredicate);
12981
12982 // We can't perform arithmetic on i1 values. Extending them would
12983 // probably result in worse codegen, so let's just use two selects instead.
12984 // Some targets are also just better off using selects rather than subtraction
12985 // because one of the conditions can be merged with one of the selects.
12986 // And finally, if we don't know the contents of high bits of a boolean value
12987 // we can't perform any arithmetic either.
12989 BoolVT.getScalarSizeInBits() == 1 ||
12991 SDValue SelectZeroOrOne =
12992 DAG.getSelect(dl, ResVT, IsGT, DAG.getConstant(1, dl, ResVT),
12993 DAG.getConstant(0, dl, ResVT));
12994 return DAG.getSelect(dl, ResVT, IsLT, DAG.getAllOnesConstant(dl, ResVT),
12995 SelectZeroOrOne);
12996 }
12997
12999 std::swap(IsGT, IsLT);
13000 return DAG.getSExtOrTrunc(DAG.getNode(ISD::SUB, dl, BoolVT, IsGT, IsLT), dl,
13001 ResVT);
13002}
13003
13005 unsigned Opcode = Node->getOpcode();
13006 bool IsSigned = Opcode == ISD::SSHLSAT;
13007 SDValue LHS = Node->getOperand(0);
13008 SDValue RHS = Node->getOperand(1);
13009 EVT VT = LHS.getValueType();
13010 SDLoc dl(Node);
13011
13012 assert((Node->getOpcode() == ISD::SSHLSAT ||
13013 Node->getOpcode() == ISD::USHLSAT) &&
13014 "Expected a SHLSAT opcode");
13015 assert(VT.isInteger() && "Expected operands to be integers");
13016
13018 return DAG.UnrollVectorOp(Node);
13019
13020 // If LHS != (LHS << RHS) >> RHS, we have overflow and must saturate.
13021
13022 unsigned BW = VT.getScalarSizeInBits();
13023 EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
13024 SDValue Result = DAG.getNode(ISD::SHL, dl, VT, LHS, RHS);
13025 SDValue Orig =
13026 DAG.getNode(IsSigned ? ISD::SRA : ISD::SRL, dl, VT, Result, RHS);
13027
13028 SDValue SatVal;
13029 if (IsSigned) {
13030 SDValue SatMin = DAG.getConstant(APInt::getSignedMinValue(BW), dl, VT);
13031 SDValue SatMax = DAG.getConstant(APInt::getSignedMaxValue(BW), dl, VT);
13032 SDValue Cond =
13033 DAG.getSetCC(dl, BoolVT, LHS, DAG.getConstant(0, dl, VT), ISD::SETLT);
13034 SatVal = DAG.getSelect(dl, VT, Cond, SatMin, SatMax);
13035 } else {
13036 SatVal = DAG.getConstant(APInt::getMaxValue(BW), dl, VT);
13037 }
13038 SDValue Cond = DAG.getSetCC(dl, BoolVT, LHS, Orig, ISD::SETNE);
13039 return DAG.getSelect(dl, VT, Cond, SatVal, Result);
13040}
13041
13043 bool Signed, SDValue &Lo, SDValue &Hi,
13044 SDValue LHS, SDValue RHS,
13045 SDValue HiLHS, SDValue HiRHS) const {
13046 EVT VT = LHS.getValueType();
13047 assert(RHS.getValueType() == VT && "Mismatching operand types");
13048
13049 assert((HiLHS && HiRHS) || (!HiLHS && !HiRHS));
13050 assert((!Signed || !HiLHS) &&
13051 "Signed flag should only be set when HiLHS and RiRHS are null");
13052
13053 // We'll expand the multiplication by brute force because we have no other
13054 // options. This is a trivially-generalized version of the code from
13055 // Hacker's Delight (itself derived from Knuth's Algorithm M from section
13056 // 4.3.1). If Signed is set, we can use arithmetic right shifts to propagate
13057 // sign bits while calculating the Hi half.
13058 unsigned Bits = VT.getScalarSizeInBits();
13059 unsigned HalfBits = Bits / 2;
13060 SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl, VT);
13061 SDValue LL = DAG.getNode(ISD::AND, dl, VT, LHS, Mask);
13062 SDValue RL = DAG.getNode(ISD::AND, dl, VT, RHS, Mask);
13063
13064 SDValue T = DAG.getNode(ISD::MUL, dl, VT, LL, RL);
13065 SDValue TL = DAG.getNode(ISD::AND, dl, VT, T, Mask);
13066
13067 SDValue Shift = DAG.getShiftAmountConstant(HalfBits, VT, dl);
13068 // This is always an unsigned shift.
13069 SDValue TH = DAG.getNode(ISD::SRL, dl, VT, T, Shift);
13070
13071 unsigned ShiftOpc = Signed ? ISD::SRA : ISD::SRL;
13072 SDValue LH = DAG.getNode(ShiftOpc, dl, VT, LHS, Shift);
13073 SDValue RH = DAG.getNode(ShiftOpc, dl, VT, RHS, Shift);
13074
13075 SDValue U =
13076 DAG.getNode(ISD::ADD, dl, VT, DAG.getNode(ISD::MUL, dl, VT, LH, RL), TH);
13077 SDValue UL = DAG.getNode(ISD::AND, dl, VT, U, Mask);
13078 SDValue UH = DAG.getNode(ShiftOpc, dl, VT, U, Shift);
13079
13080 SDValue V =
13081 DAG.getNode(ISD::ADD, dl, VT, DAG.getNode(ISD::MUL, dl, VT, LL, RH), UL);
13082 SDValue VH = DAG.getNode(ShiftOpc, dl, VT, V, Shift);
13083
13084 Lo = DAG.getNode(ISD::ADD, dl, VT, TL,
13085 DAG.getNode(ISD::SHL, dl, VT, V, Shift));
13086
13087 Hi = DAG.getNode(ISD::ADD, dl, VT, DAG.getNode(ISD::MUL, dl, VT, LH, RH),
13088 DAG.getNode(ISD::ADD, dl, VT, UH, VH));
13089
13090 // If HiLHS and HiRHS are set, multiply them by the opposite low part and add
13091 // the products to Hi.
13092 if (HiLHS) {
13093 SDValue RHLL = DAG.getNode(ISD::MUL, dl, VT, HiRHS, LHS);
13094 SDValue RLLH = DAG.getNode(ISD::MUL, dl, VT, RHS, HiLHS);
13095 Hi = DAG.getNode(ISD::ADD, dl, VT, Hi,
13096 DAG.getNode(ISD::ADD, dl, VT, RHLL, RLLH));
13097 }
13098}
13099
13101 bool Signed, const SDValue LHS,
13102 const SDValue RHS, SDValue &Lo,
13103 SDValue &Hi) const {
13104 EVT VT = LHS.getValueType();
13105 assert(RHS.getValueType() == VT && "Mismatching operand types");
13106 EVT WideVT = VT.widenIntegerElementType(*DAG.getContext());
13107 // We can fall back to a libcall with an illegal type for the MUL if we
13108 // have a libcall big enough.
13109 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
13110 if (WideVT == MVT::i16)
13111 LC = RTLIB::MUL_I16;
13112 else if (WideVT == MVT::i32)
13113 LC = RTLIB::MUL_I32;
13114 else if (WideVT == MVT::i64)
13115 LC = RTLIB::MUL_I64;
13116 else if (WideVT == MVT::i128)
13117 LC = RTLIB::MUL_I128;
13118
13119 RTLIB::LibcallImpl LibcallImpl = DAG.getLibcalls().getLibcallImpl(LC);
13120 if (LibcallImpl == RTLIB::Unsupported) {
13121 forceExpandMultiply(DAG, dl, Signed, Lo, Hi, LHS, RHS);
13122 return;
13123 }
13124
13125 // The libcall takes and returns WideVT values as packed VT halves. This only
13126 // works if VT is legal and WideVT isn't. Otherwise, emit a wide multiply and
13127 // let it legalise normally, possibly into a wider libcall.
13128 if (isTypeLegal(WideVT) || !isTypeLegal(VT)) {
13129 SDValue Mul = DAG.getNode(ISD::MUL, dl, WideVT,
13130 DAG.getExtOrTrunc(Signed, LHS, dl, WideVT),
13131 DAG.getExtOrTrunc(Signed, RHS, dl, WideVT));
13132 SDValue Shift =
13133 DAG.getShiftAmountConstant(VT.getScalarSizeInBits(), WideVT, dl);
13134 Lo = DAG.getNode(ISD::TRUNCATE, dl, VT, Mul);
13135 Hi = DAG.getNode(ISD::TRUNCATE, dl, VT,
13136 DAG.getNode(ISD::SRL, dl, WideVT, Mul, Shift));
13137 return;
13138 }
13139
13140 SDValue HiLHS, HiRHS;
13141 if (Signed) {
13142 // The high part is obtained by SRA'ing all but one of the bits of low
13143 // part.
13144 unsigned LoSize = VT.getFixedSizeInBits();
13145 SDValue Shift = DAG.getShiftAmountConstant(LoSize - 1, VT, dl);
13146 HiLHS = DAG.getNode(ISD::SRA, dl, VT, LHS, Shift);
13147 HiRHS = DAG.getNode(ISD::SRA, dl, VT, RHS, Shift);
13148 } else {
13149 HiLHS = DAG.getConstant(0, dl, VT);
13150 HiRHS = DAG.getConstant(0, dl, VT);
13151 }
13152
13153 // Attempt a libcall.
13154 SDValue Ret;
13156 CallOptions.setIsSigned(Signed);
13157 CallOptions.setIsPostTypeLegalization(true);
13159 // Halves of WideVT are packed into registers in different order
13160 // depending on platform endianness. This is usually handled by
13161 // the C calling convention, but we can't defer to it in
13162 // the legalizer.
13163 SDValue Args[] = {LHS, HiLHS, RHS, HiRHS};
13164 Ret = makeLibCall(DAG, LC, WideVT, Args, CallOptions, dl).first;
13165 } else {
13166 SDValue Args[] = {HiLHS, LHS, HiRHS, RHS};
13167 Ret = makeLibCall(DAG, LC, WideVT, Args, CallOptions, dl).first;
13168 }
13170 "Ret value is a collection of constituent nodes holding result.");
13171 if (DAG.getDataLayout().isLittleEndian()) {
13172 // Same as above.
13173 Lo = Ret.getOperand(0);
13174 Hi = Ret.getOperand(1);
13175 } else {
13176 Lo = Ret.getOperand(1);
13177 Hi = Ret.getOperand(0);
13178 }
13179}
13180
13181SDValue
13183 assert((Node->getOpcode() == ISD::SMULFIX ||
13184 Node->getOpcode() == ISD::UMULFIX ||
13185 Node->getOpcode() == ISD::SMULFIXSAT ||
13186 Node->getOpcode() == ISD::UMULFIXSAT) &&
13187 "Expected a fixed point multiplication opcode");
13188
13189 SDLoc dl(Node);
13190 SDValue LHS = Node->getOperand(0);
13191 SDValue RHS = Node->getOperand(1);
13192 EVT VT = LHS.getValueType();
13193 unsigned Scale = Node->getConstantOperandVal(2);
13194 bool Saturating = (Node->getOpcode() == ISD::SMULFIXSAT ||
13195 Node->getOpcode() == ISD::UMULFIXSAT);
13196 bool Signed = (Node->getOpcode() == ISD::SMULFIX ||
13197 Node->getOpcode() == ISD::SMULFIXSAT);
13198 EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
13199 unsigned VTSize = VT.getScalarSizeInBits();
13200
13201 if (!Scale) {
13202 // [us]mul.fix(a, b, 0) -> mul(a, b)
13203 if (!Saturating) {
13205 return DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
13206 } else if (Signed && isOperationLegalOrCustom(ISD::SMULO, VT)) {
13207 SDValue Result =
13208 DAG.getNode(ISD::SMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
13209 SDValue Product = Result.getValue(0);
13210 SDValue Overflow = Result.getValue(1);
13211 SDValue Zero = DAG.getConstant(0, dl, VT);
13212
13213 APInt MinVal = APInt::getSignedMinValue(VTSize);
13214 APInt MaxVal = APInt::getSignedMaxValue(VTSize);
13215 SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
13216 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
13217 // Xor the inputs, if resulting sign bit is 0 the product will be
13218 // positive, else negative.
13219 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS);
13220 SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Xor, Zero, ISD::SETLT);
13221 Result = DAG.getSelect(dl, VT, ProdNeg, SatMin, SatMax);
13222 return DAG.getSelect(dl, VT, Overflow, Result, Product);
13223 } else if (!Signed && isOperationLegalOrCustom(ISD::UMULO, VT)) {
13224 SDValue Result =
13225 DAG.getNode(ISD::UMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
13226 SDValue Product = Result.getValue(0);
13227 SDValue Overflow = Result.getValue(1);
13228
13229 APInt MaxVal = APInt::getMaxValue(VTSize);
13230 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
13231 return DAG.getSelect(dl, VT, Overflow, SatMax, Product);
13232 }
13233 }
13234
13235 assert(((Signed && Scale < VTSize) || (!Signed && Scale <= VTSize)) &&
13236 "Expected scale to be less than the number of bits if signed or at "
13237 "most the number of bits if unsigned.");
13238 assert(LHS.getValueType() == RHS.getValueType() &&
13239 "Expected both operands to be the same type");
13240
13241 // Select the saturated value when Cond0 <CC> Cond1, keeping it vectorized:
13242 // SELECT_CC is scalarized for vector types, so build SETCC + VSELECT there.
13243 auto getSaturatingSelect = [&](SDValue Cond0, SDValue Cond1, SDValue Sat,
13244 SDValue Val, ISD::CondCode CC) {
13245 if (VT.isVector())
13246 return DAG.getSelect(dl, VT, DAG.getSetCC(dl, BoolVT, Cond0, Cond1, CC),
13247 Sat, Val);
13248 return DAG.getSelectCC(dl, Cond0, Cond1, Sat, Val, CC);
13249 };
13250
13251 // Get the upper and lower bits of the result.
13252 SDValue Lo, Hi;
13253 unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
13254 unsigned HiOp = Signed ? ISD::MULHS : ISD::MULHU;
13255 EVT WideVT = VT.widenIntegerElementType(*DAG.getContext());
13256 if (isOperationLegalOrCustom(LoHiOp, VT)) {
13257 SDValue Result = DAG.getNode(LoHiOp, dl, DAG.getVTList(VT, VT), LHS, RHS);
13258 Lo = Result.getValue(0);
13259 Hi = Result.getValue(1);
13260 } else if (isOperationLegalOrCustom(HiOp, VT)) {
13261 Lo = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
13262 Hi = DAG.getNode(HiOp, dl, VT, LHS, RHS);
13263 } else if (isOperationLegalOrCustom(ISD::MUL, WideVT)) {
13264 // Try for a multiplication using a wider type.
13265 unsigned Ext = Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
13266 SDValue LHSExt = DAG.getNode(Ext, dl, WideVT, LHS);
13267 SDValue RHSExt = DAG.getNode(Ext, dl, WideVT, RHS);
13268 SDValue Res = DAG.getNode(ISD::MUL, dl, WideVT, LHSExt, RHSExt);
13269 Lo = DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
13270 SDValue Shifted =
13271 DAG.getNode(ISD::SRA, dl, WideVT, Res,
13272 DAG.getShiftAmountConstant(VTSize, WideVT, dl));
13273 Hi = DAG.getNode(ISD::TRUNCATE, dl, VT, Shifted);
13274 } else if (VT.isVector()) {
13275 return SDValue();
13276 } else {
13277 forceExpandWideMUL(DAG, dl, Signed, LHS, RHS, Lo, Hi);
13278 }
13279
13280 if (Scale == VTSize)
13281 // Result is just the top half since we'd be shifting by the width of the
13282 // operand. Overflow impossible so this works for both UMULFIX and
13283 // UMULFIXSAT.
13284 return Hi;
13285
13286 // The result will need to be shifted right by the scale since both operands
13287 // are scaled. The result is given to us in 2 halves, so we only want part of
13288 // both in the result.
13289 SDValue Result = DAG.getNode(ISD::FSHR, dl, VT, Hi, Lo,
13290 DAG.getShiftAmountConstant(Scale, VT, dl));
13291 if (!Saturating)
13292 return Result;
13293
13294 if (!Signed) {
13295 // Unsigned overflow happened if the upper (VTSize - Scale) bits (of the
13296 // widened multiplication) aren't all zeroes.
13297
13298 // Saturate to max if ((Hi >> Scale) != 0),
13299 // which is the same as if (Hi > ((1 << Scale) - 1))
13300 APInt MaxVal = APInt::getMaxValue(VTSize);
13301 SDValue LowMask =
13302 DAG.getConstant(APInt::getLowBitsSet(VTSize, Scale), dl, VT);
13303 return getSaturatingSelect(Hi, LowMask, DAG.getConstant(MaxVal, dl, VT),
13304 Result, ISD::SETUGT);
13305 }
13306
13307 // Signed overflow happened if the upper (VTSize - Scale + 1) bits (of the
13308 // widened multiplication) aren't all ones or all zeroes.
13309
13310 SDValue SatMin = DAG.getConstant(APInt::getSignedMinValue(VTSize), dl, VT);
13311 SDValue SatMax = DAG.getConstant(APInt::getSignedMaxValue(VTSize), dl, VT);
13312
13313 if (Scale == 0) {
13314 SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, Lo,
13315 DAG.getShiftAmountConstant(VTSize - 1, VT, dl));
13316 SDValue Overflow = DAG.getSetCC(dl, BoolVT, Hi, Sign, ISD::SETNE);
13317 // Saturated to SatMin if wide product is negative, and SatMax if wide
13318 // product is positive ...
13319 SDValue Zero = DAG.getConstant(0, dl, VT);
13320 SDValue ResultIfOverflow =
13321 getSaturatingSelect(Hi, Zero, SatMin, SatMax, ISD::SETLT);
13322 // ... but only if we overflowed.
13323 return DAG.getSelect(dl, VT, Overflow, ResultIfOverflow, Result);
13324 }
13325
13326 // We handled Scale==0 above so all the bits to examine is in Hi.
13327
13328 // Saturate to max if ((Hi >> (Scale - 1)) > 0),
13329 // which is the same as if (Hi > (1 << (Scale - 1)) - 1)
13330 SDValue LowMask =
13331 DAG.getConstant(APInt::getLowBitsSet(VTSize, Scale - 1), dl, VT);
13332 // Saturate to min if (Hi >> (Scale - 1)) < -1),
13333 // which is the same as if (HI < (-1 << (Scale - 1))
13334 SDValue HighMask = DAG.getConstant(
13335 APInt::getHighBitsSet(VTSize, VTSize - Scale + 1), dl, VT);
13336 Result = getSaturatingSelect(Hi, LowMask, SatMax, Result, ISD::SETGT);
13337 Result = getSaturatingSelect(Hi, HighMask, SatMin, Result, ISD::SETLT);
13338 return Result;
13339}
13340
13341SDValue
13343 SDValue LHS, SDValue RHS,
13344 unsigned Scale, SelectionDAG &DAG) const {
13345 assert((Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT ||
13346 Opcode == ISD::UDIVFIX || Opcode == ISD::UDIVFIXSAT) &&
13347 "Expected a fixed point division opcode");
13348
13349 EVT VT = LHS.getValueType();
13350 bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;
13351 bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;
13352 EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
13353
13354 // If there is enough room in the type to upscale the LHS or downscale the
13355 // RHS before the division, we can perform it in this type without having to
13356 // resize. For signed operations, the LHS headroom is the number of
13357 // redundant sign bits, and for unsigned ones it is the number of zeroes.
13358 // The headroom for the RHS is the number of trailing zeroes.
13359 unsigned LHSLead = Signed ? DAG.ComputeNumSignBits(LHS) - 1
13361 unsigned RHSTrail = DAG.computeKnownBits(RHS).countMinTrailingZeros();
13362
13363 // For signed saturating operations, we need to be able to detect true integer
13364 // division overflow; that is, when you have MIN / -EPS. However, this
13365 // is undefined behavior and if we emit divisions that could take such
13366 // values it may cause undesired behavior (arithmetic exceptions on x86, for
13367 // example).
13368 // Avoid this by requiring an extra bit so that we never get this case.
13369 // FIXME: This is a bit unfortunate as it means that for an 8-bit 7-scale
13370 // signed saturating division, we need to emit a whopping 32-bit division.
13371 if (LHSLead + RHSTrail < Scale + (unsigned)(Saturating && Signed))
13372 return SDValue();
13373
13374 unsigned LHSShift = std::min(LHSLead, Scale);
13375 unsigned RHSShift = Scale - LHSShift;
13376
13377 // At this point, we know that if we shift the LHS up by LHSShift and the
13378 // RHS down by RHSShift, we can emit a regular division with a final scaling
13379 // factor of Scale.
13380
13381 if (LHSShift)
13382 LHS = DAG.getNode(ISD::SHL, dl, VT, LHS,
13383 DAG.getShiftAmountConstant(LHSShift, VT, dl));
13384 if (RHSShift)
13385 RHS = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, VT, RHS,
13386 DAG.getShiftAmountConstant(RHSShift, VT, dl));
13387
13388 SDValue Quot;
13389 if (Signed) {
13390 // For signed operations, if the resulting quotient is negative and the
13391 // remainder is nonzero, subtract 1 from the quotient to round towards
13392 // negative infinity.
13393 SDValue Rem;
13394 // FIXME: Ideally we would always produce an SDIVREM here, but if the
13395 // type isn't legal, SDIVREM cannot be expanded. There is no reason why
13396 // we couldn't just form a libcall, but the type legalizer doesn't do it.
13397 if (isTypeLegal(VT) &&
13399 Quot = DAG.getNode(ISD::SDIVREM, dl,
13400 DAG.getVTList(VT, VT),
13401 LHS, RHS);
13402 Rem = Quot.getValue(1);
13403 Quot = Quot.getValue(0);
13404 } else {
13405 Quot = DAG.getNode(ISD::SDIV, dl, VT,
13406 LHS, RHS);
13407 Rem = DAG.getNode(ISD::SREM, dl, VT,
13408 LHS, RHS);
13409 }
13410 SDValue Zero = DAG.getConstant(0, dl, VT);
13411 SDValue RemNonZero = DAG.getSetCC(dl, BoolVT, Rem, Zero, ISD::SETNE);
13412 SDValue LHSNeg = DAG.getSetCC(dl, BoolVT, LHS, Zero, ISD::SETLT);
13413 SDValue RHSNeg = DAG.getSetCC(dl, BoolVT, RHS, Zero, ISD::SETLT);
13414 SDValue QuotNeg = DAG.getNode(ISD::XOR, dl, BoolVT, LHSNeg, RHSNeg);
13415 SDValue Sub1 = DAG.getNode(ISD::SUB, dl, VT, Quot,
13416 DAG.getConstant(1, dl, VT));
13417 Quot = DAG.getSelect(dl, VT,
13418 DAG.getNode(ISD::AND, dl, BoolVT, RemNonZero, QuotNeg),
13419 Sub1, Quot);
13420 } else
13421 Quot = DAG.getNode(ISD::UDIV, dl, VT,
13422 LHS, RHS);
13423
13424 return Quot;
13425}
13426
13428 SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const {
13429 SDLoc dl(Node);
13430 SDValue LHS = Node->getOperand(0);
13431 SDValue RHS = Node->getOperand(1);
13432 bool IsAdd = Node->getOpcode() == ISD::UADDO;
13433
13434 // If UADDO_CARRY/SUBO_CARRY is legal, use that instead.
13435 unsigned OpcCarry = IsAdd ? ISD::UADDO_CARRY : ISD::USUBO_CARRY;
13436 if (isOperationLegalOrCustom(OpcCarry, Node->getValueType(0))) {
13437 SDValue CarryIn = DAG.getConstant(0, dl, Node->getValueType(1));
13438 SDValue NodeCarry = DAG.getNode(OpcCarry, dl, Node->getVTList(),
13439 { LHS, RHS, CarryIn });
13440 Result = SDValue(NodeCarry.getNode(), 0);
13441 Overflow = SDValue(NodeCarry.getNode(), 1);
13442 return;
13443 }
13444
13445 Result = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl,
13446 LHS.getValueType(), LHS, RHS);
13447
13448 EVT ResultType = Node->getValueType(1);
13449 EVT SetCCType = getSetCCResultType(
13450 DAG.getDataLayout(), *DAG.getContext(), Node->getValueType(0));
13451 SDValue SetCC;
13452 if (IsAdd && isOneConstant(RHS)) {
13453 // Special case: uaddo X, 1 overflowed if X+1 is 0. This potential reduces
13454 // the live range of X. We assume comparing with 0 is cheap.
13455 // The general case (X + C) < C is not necessarily beneficial. Although we
13456 // reduce the live range of X, we may introduce the materialization of
13457 // constant C.
13458 SetCC =
13459 DAG.getSetCC(dl, SetCCType, Result,
13460 DAG.getConstant(0, dl, Node->getValueType(0)), ISD::SETEQ);
13461 } else if (IsAdd && isAllOnesConstant(RHS)) {
13462 // Special case: uaddo X, -1 overflows if X != 0.
13463 SetCC =
13464 DAG.getSetCC(dl, SetCCType, LHS,
13465 DAG.getConstant(0, dl, Node->getValueType(0)), ISD::SETNE);
13466 } else {
13468 SetCC = DAG.getSetCC(dl, SetCCType, Result, LHS, CC);
13469 }
13470 Overflow = DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType);
13471}
13472
13474 SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const {
13475 SDLoc dl(Node);
13476 SDValue LHS = Node->getOperand(0);
13477 SDValue RHS = Node->getOperand(1);
13478 bool IsAdd = Node->getOpcode() == ISD::SADDO;
13479
13480 Result = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl,
13481 LHS.getValueType(), LHS, RHS);
13482
13483 EVT ResultType = Node->getValueType(1);
13484 EVT OType = getSetCCResultType(
13485 DAG.getDataLayout(), *DAG.getContext(), Node->getValueType(0));
13486
13487 // If SADDSAT/SSUBSAT is legal, compare results to detect overflow.
13488 unsigned OpcSat = IsAdd ? ISD::SADDSAT : ISD::SSUBSAT;
13489 if (isOperationLegal(OpcSat, LHS.getValueType())) {
13490 SDValue Sat = DAG.getNode(OpcSat, dl, LHS.getValueType(), LHS, RHS);
13491 SDValue SetCC = DAG.getSetCC(dl, OType, Result, Sat, ISD::SETNE);
13492 Overflow = DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType);
13493 return;
13494 }
13495
13496 SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
13497
13498 if (IsAdd) {
13499 // For an addition, the result should be less than one of the operands (LHS)
13500 // if and only if the other operand (RHS) is negative, otherwise there will
13501 // be overflow.
13502 SDValue ResultLowerThanLHS =
13503 DAG.getSetCC(dl, OType, Result, LHS, ISD::SETLT);
13504 SDValue RHSNegative = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETLT);
13505 Overflow = DAG.getBoolExtOrTrunc(
13506 DAG.getNode(ISD::XOR, dl, OType, RHSNegative, ResultLowerThanLHS), dl,
13507 ResultType, ResultType);
13508 } else {
13509 // For subtraction, overflow occurs when the signed comparison of operands
13510 // doesn't match the sign of the result.
13511 SDValue LHSLessThanRHS = DAG.getSetCC(dl, OType, LHS, RHS, ISD::SETLT);
13512 SDValue ResultNegative = DAG.getSetCC(dl, OType, Result, Zero, ISD::SETLT);
13513 Overflow = DAG.getBoolExtOrTrunc(
13514 DAG.getNode(ISD::XOR, dl, OType, LHSLessThanRHS, ResultNegative), dl,
13515 ResultType, ResultType);
13516 }
13517}
13518
13520 SDValue &Overflow, SelectionDAG &DAG) const {
13521 SDLoc dl(Node);
13522 EVT VT = Node->getValueType(0);
13523 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
13524 SDValue LHS = Node->getOperand(0);
13525 SDValue RHS = Node->getOperand(1);
13526 bool isSigned = Node->getOpcode() == ISD::SMULO;
13527
13528 // For power-of-two multiplications we can use a simpler shift expansion.
13529 if (ConstantSDNode *RHSC = isConstOrConstSplat(RHS)) {
13530 const APInt &C = RHSC->getAPIntValue();
13531 // mulo(X, 1 << S) -> { X << S, (X << S) >> S != X }
13532 if (C.isPowerOf2()) {
13533 // smulo(x, signed_min) is same as umulo(x, signed_min).
13534 bool UseArithShift = isSigned && !C.isMinSignedValue();
13535 SDValue ShiftAmt = DAG.getShiftAmountConstant(C.logBase2(), VT, dl);
13536 Result = DAG.getNode(ISD::SHL, dl, VT, LHS, ShiftAmt);
13537 Overflow = DAG.getSetCC(dl, SetCCVT,
13538 DAG.getNode(UseArithShift ? ISD::SRA : ISD::SRL,
13539 dl, VT, Result, ShiftAmt),
13540 LHS, ISD::SETNE);
13541 return true;
13542 }
13543 }
13544
13545 SDValue BottomHalf;
13546 SDValue TopHalf;
13547 EVT WideVT = VT.widenIntegerElementType(*DAG.getContext());
13548
13549 static const unsigned Ops[2][3] =
13552 if (isOperationLegalOrCustom(Ops[isSigned][0], VT)) {
13553 BottomHalf = DAG.getNode(Ops[isSigned][0], dl, DAG.getVTList(VT, VT), LHS,
13554 RHS);
13555 TopHalf = BottomHalf.getValue(1);
13556 } else if (isOperationLegalOrCustom(Ops[isSigned][1], VT)) {
13557 BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
13558 TopHalf = DAG.getNode(Ops[isSigned][1], dl, VT, LHS, RHS);
13559 } else if (isTypeLegal(WideVT)) {
13560 LHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, LHS);
13561 RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS);
13562 SDValue Mul = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS);
13563 BottomHalf = DAG.getNode(ISD::TRUNCATE, dl, VT, Mul);
13564 SDValue ShiftAmt =
13565 DAG.getShiftAmountConstant(VT.getScalarSizeInBits(), WideVT, dl);
13566 TopHalf = DAG.getNode(ISD::TRUNCATE, dl, VT,
13567 DAG.getNode(ISD::SRL, dl, WideVT, Mul, ShiftAmt));
13568 } else {
13569 if (VT.isVector())
13570 return false;
13571
13572 forceExpandWideMUL(DAG, dl, isSigned, LHS, RHS, BottomHalf, TopHalf);
13573 }
13574
13575 Result = BottomHalf;
13576 if (isSigned) {
13577 SDValue ShiftAmt = DAG.getShiftAmountConstant(
13578 VT.getScalarSizeInBits() - 1, BottomHalf.getValueType(), dl);
13579 SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, ShiftAmt);
13580 Overflow = DAG.getSetCC(dl, SetCCVT, TopHalf, Sign, ISD::SETNE);
13581 } else {
13582 Overflow = DAG.getSetCC(dl, SetCCVT, TopHalf,
13583 DAG.getConstant(0, dl, VT), ISD::SETNE);
13584 }
13585
13586 // Truncate the result if SetCC returns a larger type than needed.
13587 EVT RType = Node->getValueType(1);
13588 if (RType.bitsLT(Overflow.getValueType()))
13589 Overflow = DAG.getNode(ISD::TRUNCATE, dl, RType, Overflow);
13590
13591 assert(RType.getSizeInBits() == Overflow.getValueSizeInBits() &&
13592 "Unexpected result type for S/UMULO legalization");
13593 return true;
13594}
13595
13597 SDLoc dl(Node);
13598 EVT VT = Node->getValueType(0);
13599 SDValue LHS = Node->getOperand(0);
13600 SDValue RHS = Node->getOperand(1);
13601 bool IsSigned = Node->getOpcode() == ISD::MULHS;
13602
13603 // Use MUL_LOHI if legal/custom for the original type.
13604 unsigned LoHiOp = IsSigned ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
13605 if (isOperationLegalOrCustom(LoHiOp, VT))
13606 return DAG.getNode(LoHiOp, dl, DAG.getVTList(VT, VT), LHS, RHS).getValue(1);
13607
13608 // Use a wide multiply if available.
13609 EVT WideVT = VT.widenIntegerElementType(*DAG.getContext());
13610 if (isOperationLegalOrCustom(ISD::MUL, WideVT)) {
13611 unsigned BW = VT.getScalarSizeInBits();
13612 LHS = DAG.getExtOrTrunc(IsSigned, LHS, dl, WideVT);
13613 RHS = DAG.getExtOrTrunc(IsSigned, RHS, dl, WideVT);
13614 return DAG.getNode(ISD::TRUNCATE, dl, VT,
13615 DAG.getNode(ISD::SRL, dl, WideVT,
13616 DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS),
13617 DAG.getShiftAmountConstant(BW, WideVT, dl)));
13618 }
13619
13620 // Let fixed-length vectors be scalarised by the caller.
13621 // Expand everything else with a wide multiply.
13622 if (!VT.isFixedLengthVector()) {
13623 SDValue Lo, Hi;
13624 forceExpandWideMUL(DAG, dl, IsSigned, LHS, RHS, Lo, Hi);
13625 return Hi;
13626 }
13627
13628 return SDValue();
13629}
13630
13632 SDLoc dl(Node);
13633 ISD::NodeType BaseOpcode = ISD::getVecReduceBaseOpcode(Node->getOpcode());
13634 SDValue Op = Node->getOperand(0);
13635 SDNodeFlags Flags = Node->getFlags();
13636 EVT VT = Op.getValueType();
13637
13638 // Try to use a shuffle reduction for power of two vectors.
13639 if (VT.isPow2VectorType()) {
13640 // See if the reduction opcode is safe to use with widened types.
13641 bool WidenSrc = false;
13642 switch (Node->getOpcode()) {
13645 case ISD::VECREDUCE_ADD:
13646 case ISD::VECREDUCE_MUL:
13647 case ISD::VECREDUCE_AND:
13648 case ISD::VECREDUCE_OR:
13649 case ISD::VECREDUCE_XOR:
13654 WidenSrc = VT.isFixedLengthVector();
13655 break;
13656 }
13657
13659 EVT HalfVT = VT.getHalfNumVectorElementsVT(*DAG.getContext());
13660 if (!isOperationLegalOrCustom(BaseOpcode, HalfVT)) {
13661 if (WidenSrc && Op.getOpcode() != ISD::BUILD_VECTOR) {
13662 // Attempt to widen the source vectors to a legal op.
13663 EVT WideVT = getTypeToTransformTo(*DAG.getContext(), HalfVT);
13664 if (WideVT.isVector() &&
13665 WideVT.getScalarType() == HalfVT.getScalarType() &&
13666 WideVT.getVectorNumElements() >= HalfVT.getVectorNumElements() &&
13667 isOperationLegalOrCustom(BaseOpcode, WideVT)) {
13668 SDValue Lo, Hi;
13669 std::tie(Lo, Hi) = DAG.SplitVector(Op, dl);
13670 Lo = DAG.getInsertSubvector(dl, DAG.getPOISON(WideVT), Lo, 0);
13671 Hi = DAG.getInsertSubvector(dl, DAG.getPOISON(WideVT), Hi, 0);
13672 Op = DAG.getNode(BaseOpcode, dl, WideVT, Lo, Hi, Flags);
13673 Op = DAG.getExtractSubvector(dl, HalfVT, Op, 0);
13674 VT = HalfVT;
13675 continue;
13676 }
13677 }
13678 break;
13679 }
13680
13681 SDValue Lo, Hi;
13682 std::tie(Lo, Hi) = DAG.SplitVector(Op, dl);
13683 Op = DAG.getNode(BaseOpcode, dl, HalfVT, Lo, Hi, Flags);
13684 VT = HalfVT;
13685
13686 // Stop if splitting is enough to make the reduction legal.
13687 if (isOperationLegalOrCustom(Node->getOpcode(), HalfVT))
13688 return DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0), Op,
13689 Flags);
13690 }
13691 }
13692
13693 if (VT.isScalableVector())
13695 "Expanding reductions for scalable vectors is undefined.");
13696
13697 EVT EltVT = VT.getVectorElementType();
13698 unsigned NumElts = VT.getVectorNumElements();
13699
13701 DAG.ExtractVectorElements(Op, Ops, 0, NumElts);
13702
13703 SDValue Res = Ops[0];
13704 for (unsigned i = 1; i < NumElts; i++)
13705 Res = DAG.getNode(BaseOpcode, dl, EltVT, Res, Ops[i], Flags);
13706
13707 // Result type may be wider than element type.
13708 if (EltVT != Node->getValueType(0))
13709 Res = DAG.getNode(ISD::ANY_EXTEND, dl, Node->getValueType(0), Res);
13710 return Res;
13711}
13712
13714 SDLoc dl(Node);
13715 SDValue AccOp = Node->getOperand(0);
13716 SDValue VecOp = Node->getOperand(1);
13717 SDNodeFlags Flags = Node->getFlags();
13718
13719 EVT VT = VecOp.getValueType();
13720 EVT EltVT = VT.getVectorElementType();
13721
13722 if (VT.isScalableVector())
13724 "Expanding reductions for scalable vectors is undefined.");
13725
13726 unsigned NumElts = VT.getVectorNumElements();
13727
13729 DAG.ExtractVectorElements(VecOp, Ops, 0, NumElts);
13730
13731 unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(Node->getOpcode());
13732
13733 SDValue Res = AccOp;
13734 for (unsigned i = 0; i < NumElts; i++)
13735 Res = DAG.getNode(BaseOpcode, dl, EltVT, Res, Ops[i], Flags);
13736
13737 return Res;
13738}
13739
13741 SelectionDAG &DAG) const {
13742 EVT VT = Node->getValueType(0);
13743 SDLoc dl(Node);
13744 bool isSigned = Node->getOpcode() == ISD::SREM;
13745 unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV;
13746 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
13747 SDValue Dividend = Node->getOperand(0);
13748 SDValue Divisor = Node->getOperand(1);
13749 if (isOperationLegalOrCustom(DivRemOpc, VT)) {
13750 SDVTList VTs = DAG.getVTList(VT, VT);
13751 Result = DAG.getNode(DivRemOpc, dl, VTs, Dividend, Divisor).getValue(1);
13752 return true;
13753 }
13754 if (isOperationLegalOrCustom(DivOpc, VT)) {
13755 // X % Y -> X-X/Y*Y
13756 SDValue Divide = DAG.getNode(DivOpc, dl, VT, Dividend, Divisor);
13757 SDValue Mul = DAG.getNode(ISD::MUL, dl, VT, Divide, Divisor);
13758 Result = DAG.getNode(ISD::SUB, dl, VT, Dividend, Mul);
13759 return true;
13760 }
13761 return false;
13762}
13763
13765 SelectionDAG &DAG) const {
13766 bool IsSigned = Node->getOpcode() == ISD::FP_TO_SINT_SAT;
13767 SDLoc dl(SDValue(Node, 0));
13768 SDValue Src = Node->getOperand(0);
13769
13770 // DstVT is the result type, while SatVT is the size to which we saturate
13771 EVT SrcVT = Src.getValueType();
13772 EVT DstVT = Node->getValueType(0);
13773
13774 EVT SatVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
13775 unsigned SatWidth = SatVT.getScalarSizeInBits();
13776 unsigned DstWidth = DstVT.getScalarSizeInBits();
13777 assert(SatWidth <= DstWidth &&
13778 "Expected saturation width smaller than result width");
13779
13780 // Determine minimum and maximum integer values and their corresponding
13781 // floating-point values.
13782 APInt MinInt, MaxInt;
13783 if (IsSigned) {
13784 MinInt = APInt::getSignedMinValue(SatWidth).sext(DstWidth);
13785 MaxInt = APInt::getSignedMaxValue(SatWidth).sext(DstWidth);
13786 } else {
13787 MinInt = APInt::getMinValue(SatWidth).zext(DstWidth);
13788 MaxInt = APInt::getMaxValue(SatWidth).zext(DstWidth);
13789 }
13790
13791 // We cannot risk emitting FP_TO_XINT nodes with a source VT of [b]f16, as
13792 // libcall emission cannot handle this. Large result types will fail.
13793 if (SrcVT == MVT::f16 || SrcVT == MVT::bf16) {
13794 Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, Src);
13795 SrcVT = Src.getValueType();
13796 }
13797
13798 const fltSemantics &Sem = SrcVT.getFltSemantics();
13799 APFloat MinFloat(Sem);
13800 APFloat MaxFloat(Sem);
13801
13802 APFloat::opStatus MinStatus =
13803 MinFloat.convertFromAPInt(MinInt, IsSigned, APFloat::rmTowardZero);
13804 APFloat::opStatus MaxStatus =
13805 MaxFloat.convertFromAPInt(MaxInt, IsSigned, APFloat::rmTowardZero);
13806 bool AreExactFloatBounds = !(MinStatus & APFloat::opStatus::opInexact) &&
13807 !(MaxStatus & APFloat::opStatus::opInexact);
13808
13809 SDValue MinFloatNode = DAG.getConstantFP(MinFloat, dl, SrcVT);
13810 SDValue MaxFloatNode = DAG.getConstantFP(MaxFloat, dl, SrcVT);
13811
13812 // If the integer bounds are exactly representable as floats and min/max are
13813 // legal, emit a min+max+fptoi sequence. Otherwise we have to use a sequence
13814 // of comparisons and selects.
13815 auto EmitMinMax = [&](unsigned MinOpcode, unsigned MaxOpcode,
13816 bool MayPropagateNaN) {
13817 bool MinMaxLegal = isOperationLegalOrCustom(MinOpcode, SrcVT) &&
13818 isOperationLegalOrCustom(MaxOpcode, SrcVT);
13819 if (!MinMaxLegal)
13820 return SDValue();
13821
13822 SDValue Clamped = Src;
13823
13824 // Clamp Src by MinFloat from below. If !MayPropagateNaN and Src is NaN
13825 // then the result is MinFloat.
13826 Clamped = DAG.getNode(MaxOpcode, dl, SrcVT, Clamped, MinFloatNode);
13827 // Clamp by MaxFloat from above. If !MayPropagateNaN then NaN cannot occur.
13828 Clamped = DAG.getNode(MinOpcode, dl, SrcVT, Clamped, MaxFloatNode);
13829 // Convert clamped value to integer.
13830 SDValue FpToInt = DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT,
13831 dl, DstVT, Clamped);
13832
13833 // If !MayPropagateNan and the conversion is unsigned case we're done,
13834 // because we mapped NaN to MinFloat, which will cast to zero.
13835 if (!MayPropagateNaN && !IsSigned)
13836 return FpToInt;
13837
13838 // Otherwise, select 0 if Src is NaN.
13839 SDValue ZeroInt = DAG.getConstant(0, dl, DstVT);
13840 EVT SetCCVT =
13841 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), SrcVT);
13842 SDValue IsNan = DAG.getSetCC(dl, SetCCVT, Src, Src, ISD::CondCode::SETUO);
13843 return DAG.getSelect(dl, DstVT, IsNan, ZeroInt, FpToInt);
13844 };
13845 if (AreExactFloatBounds) {
13846 if (SDValue Res = EmitMinMax(ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM,
13847 /*MayPropagateNaN=*/false))
13848 return Res;
13849 // These may propagate NaN for sNaN operands.
13850 if (SDValue Res =
13851 EmitMinMax(ISD::FMINNUM, ISD::FMAXNUM, /*MayPropagateNaN=*/true))
13852 return Res;
13853 // These always propagate NaN.
13854 if (SDValue Res =
13855 EmitMinMax(ISD::FMINIMUM, ISD::FMAXIMUM, /*MayPropagateNaN=*/true))
13856 return Res;
13857 }
13858
13859 SDValue MinIntNode = DAG.getConstant(MinInt, dl, DstVT);
13860 SDValue MaxIntNode = DAG.getConstant(MaxInt, dl, DstVT);
13861
13862 // Result of direct conversion. The assumption here is that the operation is
13863 // non-trapping and it's fine to apply it to an out-of-range value if we
13864 // select it away later.
13865 SDValue FpToInt =
13866 DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, dl, DstVT, Src);
13867
13868 SDValue Select = FpToInt;
13869
13870 EVT SetCCVT =
13871 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), SrcVT);
13872
13873 // If Src ULT MinFloat, select MinInt. In particular, this also selects
13874 // MinInt if Src is NaN.
13875 SDValue ULT = DAG.getSetCC(dl, SetCCVT, Src, MinFloatNode, ISD::SETULT);
13876 Select = DAG.getSelect(dl, DstVT, ULT, MinIntNode, Select);
13877 // If Src OGT MaxFloat, select MaxInt.
13878 SDValue OGT = DAG.getSetCC(dl, SetCCVT, Src, MaxFloatNode, ISD::SETOGT);
13879 Select = DAG.getSelect(dl, DstVT, OGT, MaxIntNode, Select);
13880
13881 // In the unsigned case we are done, because we mapped NaN to MinInt, which
13882 // is already zero.
13883 if (!IsSigned)
13884 return Select;
13885
13886 // Otherwise, select 0 if Src is NaN.
13887 SDValue ZeroInt = DAG.getConstant(0, dl, DstVT);
13888 SDValue IsNan = DAG.getSetCC(dl, SetCCVT, Src, Src, ISD::CondCode::SETUO);
13889 return DAG.getSelect(dl, DstVT, IsNan, ZeroInt, Select);
13890}
13891
13893 const SDLoc &dl,
13894 SelectionDAG &DAG) const {
13895 EVT OperandVT = Op.getValueType();
13896 if (OperandVT.getScalarType() == ResultVT.getScalarType())
13897 return Op;
13898 EVT ResultIntVT = ResultVT.changeTypeToInteger();
13899 // We are rounding binary64/binary128 -> binary32 -> bfloat16. This
13900 // can induce double-rounding which may alter the results. We can
13901 // correct for this using a trick explained in: Boldo, Sylvie, and
13902 // Guillaume Melquiond. "When double rounding is odd." 17th IMACS
13903 // World Congress. 2005.
13904 SDValue Narrow = DAG.getFPExtendOrRound(Op, dl, ResultVT);
13905 SDValue NarrowAsWide = DAG.getFPExtendOrRound(Narrow, dl, OperandVT);
13906
13907 // We can keep the narrow value as-is if narrowing was exact (no
13908 // rounding error), the wide value was NaN (the narrow value is also
13909 // NaN and should be preserved) or if we rounded to the odd value.
13910 SDValue NarrowBits = DAG.getNode(ISD::BITCAST, dl, ResultIntVT, Narrow);
13911 SDValue One = DAG.getConstant(1, dl, ResultIntVT);
13912 SDValue NegativeOne = DAG.getAllOnesConstant(dl, ResultIntVT);
13913 SDValue And = DAG.getNode(ISD::AND, dl, ResultIntVT, NarrowBits, One);
13914 EVT ResultIntVTCCVT = getSetCCResultType(
13915 DAG.getDataLayout(), *DAG.getContext(), And.getValueType());
13916 SDValue Zero = DAG.getConstant(0, dl, ResultIntVT);
13917 // The result is already odd so we don't need to do anything.
13918 SDValue AlreadyOdd = DAG.getSetCC(dl, ResultIntVTCCVT, And, Zero, ISD::SETNE);
13919
13920 EVT WideSetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
13921 Op.getValueType());
13922 // We keep results which are exact, odd or NaN.
13923 SDValue KeepNarrow =
13924 DAG.getSetCC(dl, WideSetCCVT, Op, NarrowAsWide, ISD::SETUEQ);
13925 KeepNarrow = DAG.getNode(ISD::OR, dl, WideSetCCVT, KeepNarrow, AlreadyOdd);
13926 // We morally performed a round-down if AbsNarrow is smaller than
13927 // AbsWide.
13928 SDValue AbsWide = DAG.getNode(ISD::FABS, dl, OperandVT, Op);
13929 SDValue AbsNarrowAsWide = DAG.getNode(ISD::FABS, dl, OperandVT, NarrowAsWide);
13930 SDValue NarrowIsRd =
13931 DAG.getSetCC(dl, WideSetCCVT, AbsWide, AbsNarrowAsWide, ISD::SETOGT);
13932 // If the narrow value is odd or exact, pick it.
13933 // Otherwise, narrow is even and corresponds to either the rounded-up
13934 // or rounded-down value. If narrow is the rounded-down value, we want
13935 // the rounded-up value as it will be odd.
13936 SDValue Adjust = DAG.getSelect(dl, ResultIntVT, NarrowIsRd, One, NegativeOne);
13937 SDValue Adjusted = DAG.getNode(ISD::ADD, dl, ResultIntVT, NarrowBits, Adjust);
13938 Op = DAG.getSelect(dl, ResultIntVT, KeepNarrow, NarrowBits, Adjusted);
13939 return DAG.getNode(ISD::BITCAST, dl, ResultVT, Op);
13940}
13941
13943 assert(Node->getOpcode() == ISD::FP_ROUND && "Unexpected opcode!");
13944 SDValue Op = Node->getOperand(0);
13945 EVT VT = Node->getValueType(0);
13946 SDLoc dl(Node);
13947 if (VT.getScalarType() == MVT::bf16) {
13948 if (Node->getConstantOperandVal(1) == 1) {
13949 return DAG.getNode(ISD::FP_TO_BF16, dl, VT, Node->getOperand(0));
13950 }
13951 EVT OperandVT = Op.getValueType();
13952
13953 // We are rounding binary64/binary128 -> binary32 -> bfloat16. This
13954 // can induce double-rounding which may alter the results. We can
13955 // correct for this using a trick explained in: Boldo, Sylvie, and
13956 // Guillaume Melquiond. "When double rounding is odd." 17th IMACS
13957 // World Congress. 2005.
13958 EVT F32 = VT.changeElementType(*DAG.getContext(), MVT::f32);
13959 EVT I32 = F32.changeTypeToInteger();
13960 Op = expandRoundInexactToOdd(F32, Op, dl, DAG);
13961 Op = DAG.getNode(ISD::BITCAST, dl, I32, Op);
13962
13963 // Factor in the contribution of the low 16 bits.
13964 SDValue One = DAG.getConstant(1, dl, I32);
13965 SDValue Lsb = DAG.getNode(ISD::SRL, dl, I32, Op,
13966 DAG.getShiftAmountConstant(16, I32, dl));
13967 Lsb = DAG.getNode(ISD::AND, dl, I32, Lsb, One);
13968 SDValue RoundingBias =
13969 DAG.getNode(ISD::ADD, dl, I32, Lsb, DAG.getConstant(0x7fff, dl, I32));
13970 SDValue Add = DAG.getNode(ISD::ADD, dl, I32, Op, RoundingBias);
13971
13972 // Don't round if we had a NaN, we don't want to turn 0x7fffffff into
13973 // 0x80000000.
13974 if (!Node->getFlags().hasNoNaNs() &&
13975 !DAG.isKnownNeverNaN(Node->getOperand(0))) {
13976 // Conversions should set NaN's quiet bit. This also prevents NaNs from
13977 // turning into infinities.
13978 SDValue NaN =
13979 DAG.getNode(ISD::OR, dl, I32, Op, DAG.getConstant(0x400000, dl, I32));
13980
13981 SDValue IsNaN = DAG.getSetCC(
13982 dl,
13983 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), OperandVT),
13984 Node->getOperand(0), Node->getOperand(0), ISD::SETUO);
13985 Add = DAG.getSelect(dl, I32, IsNaN, NaN, Add);
13986 }
13987
13988 // Now that we have rounded, shift the bits into position.
13989 Op = DAG.getNode(ISD::SRL, dl, I32, Add,
13990 DAG.getShiftAmountConstant(16, I32, dl));
13991 EVT I16 = I32.changeElementType(*DAG.getContext(), MVT::i16);
13992 Op = DAG.getNode(ISD::TRUNCATE, dl, I16, Op);
13993 return DAG.getNode(ISD::BITCAST, dl, VT, Op);
13994 }
13995 return SDValue();
13996}
13997
14000 SelectionDAG &DAG) const {
14001 assert(Node->getOpcode() == ISD::VECTOR_INTERLEAVE && "Unexpected opcode");
14002 EVT VT = Node->getValueType(0);
14003 unsigned Factor = Node->getNumOperands();
14004 SDLoc DL(Node);
14005
14006 if (Factor > 2 && Factor % 2 == 0) {
14007 SmallVector<EVT> HalfVTs(Factor / 2, VT);
14008 SmallVector<SDValue, 8> LOps, ROps;
14009 // Interleave so we have 2 factors per result:
14010 // a0a1 b0b1 c0c1 d0d1 -> [a0c0 b0d0] [a1c1 b1d1]
14011 for (unsigned I = 0; I < Factor / 2; I++) {
14012 SDValue Interleave =
14013 DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, {VT, VT},
14014 {Node->getOperand(I), Node->getOperand(I + Factor / 2)});
14015 LOps.push_back(Interleave.getValue(0));
14016 ROps.push_back(Interleave.getValue(1));
14017 }
14018 // Interleave at Factor/2:
14019 // [a0c0 b0d0] [a1c1 b1d1] -> a0b0 c0d0 a1b1 c1d1
14020 SDValue L = DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, HalfVTs, LOps);
14021 SDValue R = DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, HalfVTs, ROps);
14022 for (unsigned I = 0; I < Factor / 2; I++)
14023 Results.push_back(L.getValue(I));
14024 for (unsigned I = 0; I < Factor / 2; I++)
14025 Results.push_back(R.getValue(I));
14026 return;
14027 }
14028
14029 if (!VT.isFixedLengthVector())
14030 return;
14031
14032 unsigned NumElts = VT.getVectorNumElements();
14033 EVT EltVT = VT.getVectorElementType();
14034
14035 // A factor of two needs only same-width, two-input shuffles.
14036 if (Factor == 2) {
14037 SmallVector<int, 8> Mask = createInterleaveMask(NumElts, Factor);
14038 for (unsigned I = 0; I != Factor; ++I)
14039 Results.push_back(DAG.getVectorShuffle(
14040 VT, DL, Node->getOperand(0), Node->getOperand(1),
14041 ArrayRef<int>(Mask).slice(I * NumElts, NumElts)));
14042 return;
14043 }
14044
14045 SmallVector<SDValue> InterleavedElts;
14046 InterleavedElts.reserve(Factor * NumElts);
14047 for (unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx)
14048 for (unsigned OperandIdx = 0; OperandIdx != Factor; ++OperandIdx)
14049 InterleavedElts.push_back(DAG.getExtractVectorElt(
14050 DL, EltVT, Node->getOperand(OperandIdx), EltIdx));
14051
14052 for (unsigned ResultIdx = 0; ResultIdx != Factor; ++ResultIdx)
14053 Results.push_back(DAG.getBuildVector(
14054 VT, DL, ArrayRef(InterleavedElts).slice(ResultIdx * NumElts, NumElts)));
14055}
14056
14059 SelectionDAG &DAG) const {
14060 assert(Node->getOpcode() == ISD::VECTOR_DEINTERLEAVE && "Unexpected opcode");
14061 EVT VT = Node->getValueType(0);
14062 unsigned Factor = Node->getNumOperands();
14063 SDLoc DL(Node);
14064
14065 if (Factor > 2 && Factor % 2 == 0) {
14067 SmallVector<EVT> HalfVTs(Factor / 2, VT);
14068 // Deinterleave at Factor/2 so each result contains two factors interleaved:
14069 // a0b0 c0d0 a1b1 c1d1 -> [a0c0 b0d0] [a1c1 b1d1]
14070 SDValue L = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL, HalfVTs,
14071 ArrayRef(Ops).take_front(Factor / 2));
14072 SDValue R = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL, HalfVTs,
14073 ArrayRef(Ops).take_back(Factor / 2));
14074 Results.resize(Factor);
14075 // Deinterleave the 2 factors out:
14076 // [a0c0 a1c1] [b0d0 b1d1] -> a0a1 b0b1 c0c1 d0d1
14077 for (unsigned I = 0; I < Factor / 2; ++I) {
14079 {L.getValue(I), R.getValue(I)});
14080 Results[I] = Deinterleave.getValue(0);
14081 Results[I + Factor / 2] = Deinterleave.getValue(1);
14082 }
14083 return;
14084 }
14085
14086 if (!VT.isFixedLengthVector())
14087 return;
14088
14089 unsigned NumElts = VT.getVectorNumElements();
14090 EVT EltVT = VT.getVectorElementType();
14091
14092 // A factor of two needs only same-width, two-input shuffles.
14093 if (Factor == 2) {
14094 for (unsigned I = 0; I != Factor; ++I)
14095 Results.push_back(
14096 DAG.getVectorShuffle(VT, DL, Node->getOperand(0), Node->getOperand(1),
14097 createStrideMask(I, Factor, NumElts)));
14098 return;
14099 }
14100
14101 SmallVector<SDValue> ConcatElts;
14102 ConcatElts.reserve(Factor * NumElts);
14103 for (unsigned OperandIdx = 0; OperandIdx != Factor; ++OperandIdx)
14104 for (unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx)
14105 ConcatElts.push_back(DAG.getExtractVectorElt(
14106 DL, EltVT, Node->getOperand(OperandIdx), EltIdx));
14107
14108 SmallVector<SDValue> DeinterleavedElts;
14109 DeinterleavedElts.reserve(Factor * NumElts);
14110 for (unsigned ResultIdx = 0; ResultIdx != Factor; ++ResultIdx) {
14111 for (unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx)
14112 DeinterleavedElts.push_back(ConcatElts[ResultIdx + EltIdx * Factor]);
14113 Results.push_back(DAG.getBuildVector(
14114 VT, DL,
14115 ArrayRef(DeinterleavedElts).slice(ResultIdx * NumElts, NumElts)));
14116 }
14117}
14118
14120 SelectionDAG &DAG) const {
14121 assert((Node->getOpcode() == ISD::VECTOR_SPLICE_LEFT ||
14122 Node->getOpcode() == ISD::VECTOR_SPLICE_RIGHT) &&
14123 "Unexpected opcode!");
14124 assert((Node->getValueType(0).isScalableVector() ||
14125 !isa<ConstantSDNode>(Node->getOperand(2))) &&
14126 "Fixed length vector types with constant offsets expected to use "
14127 "SHUFFLE_VECTOR!");
14128
14129 EVT VT = Node->getValueType(0);
14130 SDValue V1 = Node->getOperand(0);
14131 SDValue V2 = Node->getOperand(1);
14132 SDValue Offset = Node->getOperand(2);
14133 SDLoc DL(Node);
14134
14135 // Expand through memory thusly:
14136 // Alloca CONCAT_VECTORS_TYPES(V1, V2) Ptr
14137 // Store V1, Ptr
14138 // Store V2, Ptr + sizeof(V1)
14139 // if (VECTOR_SPLICE_LEFT)
14140 // Ptr = Ptr + (Offset * sizeof(VT.Elt))
14141 // else
14142 // Ptr = Ptr + sizeof(V1) - (Offset * size(VT.Elt))
14143 // Res = Load Ptr
14144
14145 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
14146
14148 VT.getVectorElementCount() * 2);
14149 SDValue StackPtr = DAG.CreateStackTemporary(MemVT.getStoreSize(), Alignment);
14150 EVT PtrVT = StackPtr.getValueType();
14151 auto &MF = DAG.getMachineFunction();
14152 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
14153 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
14154
14155 // Store the lo part of CONCAT_VECTORS(V1, V2)
14156 SDValue StoreV1 =
14157 DAG.getStore(DAG.getEntryNode(), DL, V1, StackPtr, PtrInfo, Alignment);
14158 // Store the hi part of CONCAT_VECTORS(V1, V2)
14159 SDValue VTBytes = DAG.getTypeSize(DL, PtrVT, VT.getStoreSize());
14160 SDValue StackPtr2 = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, VTBytes);
14161 SDValue StoreV2 =
14162 DAG.getStore(StoreV1, DL, V2, StackPtr2, PtrInfo, Alignment);
14163
14164 // NOTE: TrailingBytes must be clamped so as not to read outside of V1:V2.
14165 SDValue EltByteSize =
14166 DAG.getTypeSize(DL, PtrVT, VT.getVectorElementType().getStoreSize());
14167 Offset = DAG.getZExtOrTrunc(Offset, DL, PtrVT);
14168 SDValue TrailingBytes = DAG.getNode(ISD::MUL, DL, PtrVT, Offset, EltByteSize);
14169
14170 TrailingBytes = DAG.getNode(ISD::UMIN, DL, PtrVT, TrailingBytes, VTBytes);
14171
14172 if (Node->getOpcode() == ISD::VECTOR_SPLICE_LEFT)
14173 StackPtr = DAG.getMemBasePlusOffset(StackPtr, TrailingBytes, DL);
14174 else
14175 StackPtr = DAG.getNode(ISD::SUB, DL, PtrVT, StackPtr2, TrailingBytes);
14176
14177 // Load the spliced result
14178 return DAG.getLoad(VT, DL, StoreV2, StackPtr,
14180}
14181
14183 SelectionDAG &DAG) const {
14184 SDLoc DL(Node);
14185 SDValue Vec = Node->getOperand(0);
14186 SDValue Mask = Node->getOperand(1);
14187 SDValue Passthru = Node->getOperand(2);
14188
14189 EVT VecVT = Vec.getValueType();
14190 EVT ScalarVT = VecVT.getScalarType();
14191 EVT MaskVT = Mask.getValueType();
14192 EVT MaskScalarVT = MaskVT.getScalarType();
14193
14194 // Needs to be handled by targets that have scalable vector types.
14195 if (VecVT.isScalableVector())
14196 report_fatal_error("Cannot expand masked_compress for scalable vectors.");
14197
14198 Align Alignment = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
14199 SDValue StackPtr = DAG.CreateStackTemporary(VecVT.getStoreSize(), Alignment);
14200 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
14201 MachinePointerInfo PtrInfo =
14203
14204 MVT PositionVT = getVectorIdxTy(DAG.getDataLayout());
14205 SDValue Chain = DAG.getEntryNode();
14206 SDValue OutPos = DAG.getConstant(0, DL, PositionVT);
14207
14208 bool HasPassthru = !Passthru.isUndef();
14209
14210 // If we have a passthru vector, store it on the stack, overwrite the matching
14211 // positions and then re-write the last element that was potentially
14212 // overwritten even though mask[i] = false.
14213 if (HasPassthru)
14214 Chain = DAG.getStore(Chain, DL, Passthru, StackPtr, PtrInfo, Alignment);
14215
14216 SDValue LastWriteVal;
14217 APInt PassthruSplatVal;
14218 bool IsSplatPassthru = ISD::isConstantSplatVector(Passthru, PassthruSplatVal);
14219
14220 if (IsSplatPassthru) {
14221 // As we do not know which position we wrote to last, we cannot simply
14222 // access that index from the passthru vector. So we first check if passthru
14223 // is a splat vector, to use any element ...
14224 LastWriteVal = DAG.getConstant(PassthruSplatVal, DL, ScalarVT);
14225 } else if (HasPassthru) {
14226 // ... if it is not a splat vector, we need to get the passthru value at
14227 // position = popcount(mask) and re-load it from the stack before it is
14228 // overwritten in the loop below.
14229 EVT PopcountVT = ScalarVT.changeTypeToInteger();
14230 SDValue Popcount = DAG.getNode(
14232 MaskVT.changeVectorElementType(*DAG.getContext(), MVT::i1), Mask);
14233 Popcount = DAG.getNode(
14235 MaskVT.changeVectorElementType(*DAG.getContext(), PopcountVT),
14236 Popcount);
14237 Popcount = DAG.getNode(ISD::VECREDUCE_ADD, DL, PopcountVT, Popcount);
14238 SDValue LastElmtPtr =
14239 getVectorElementPointer(DAG, StackPtr, VecVT, Popcount);
14240 LastWriteVal = DAG.getLoad(
14241 ScalarVT, DL, Chain, LastElmtPtr,
14243 Chain = LastWriteVal.getValue(1);
14244 }
14245
14246 unsigned NumElms = VecVT.getVectorNumElements();
14247 for (unsigned I = 0; I < NumElms; I++) {
14248 SDValue ValI = DAG.getExtractVectorElt(DL, ScalarVT, Vec, I);
14249 SDValue OutPtr = getVectorElementPointer(DAG, StackPtr, VecVT, OutPos);
14250 Chain = DAG.getStore(
14251 Chain, DL, ValI, OutPtr,
14253
14254 // Get the mask value and add it to the current output position. This
14255 // either increments by 1 if MaskI is true or adds 0 otherwise.
14256 // Freeze in case we have poison/undef mask entries.
14257 SDValue MaskI = DAG.getExtractVectorElt(DL, MaskScalarVT, Mask, I);
14258 MaskI = DAG.getFreeze(MaskI);
14259 MaskI = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, MaskI);
14260 MaskI = DAG.getNode(ISD::ZERO_EXTEND, DL, PositionVT, MaskI);
14261 OutPos = DAG.getNode(ISD::ADD, DL, PositionVT, OutPos, MaskI);
14262
14263 if (HasPassthru && I == NumElms - 1) {
14264 SDValue EndOfVector =
14265 DAG.getConstant(VecVT.getVectorNumElements() - 1, DL, PositionVT);
14266 SDValue AllLanesSelected =
14267 DAG.getSetCC(DL, MVT::i1, OutPos, EndOfVector, ISD::CondCode::SETUGT);
14268 OutPos = DAG.getNode(ISD::UMIN, DL, PositionVT, OutPos, EndOfVector);
14269 OutPtr = getVectorElementPointer(DAG, StackPtr, VecVT, OutPos);
14270
14271 // Re-write the last ValI if all lanes were selected. Otherwise,
14272 // overwrite the last write it with the passthru value.
14273 LastWriteVal = DAG.getSelect(DL, ScalarVT, AllLanesSelected, ValI,
14274 LastWriteVal, SDNodeFlags::Unpredictable);
14275 Chain = DAG.getStore(
14276 Chain, DL, LastWriteVal, OutPtr,
14278 }
14279 }
14280
14281 return DAG.getLoad(VecVT, DL, Chain, StackPtr, PtrInfo, Alignment);
14282}
14283
14285 SelectionDAG &DAG) const {
14286 SDLoc DL(Node);
14287 SDValue V = Node->getOperand(0);
14288 SDValue Mask = Node->getOperand(1);
14289
14290 EVT VecVT = Node->getValueType(0);
14291 EVT ScalarVT = VecVT.getScalarType();
14292
14293 // Needs to be handled by targets that have scalable vector types.
14294 if (VecVT.isScalableVector())
14296 "Cannot expand vector_shuffle_var for scalable vectors.");
14297
14298 if (VecVT.getVectorNumElements() == 2) {
14299 // Select between V and V with its lanes swapped.
14300 EVT IdxVT = VecVT.changeVectorElementTypeToInteger();
14301 SDValue Idx = DAG.getZExtOrTrunc(Mask, DL, IdxVT);
14302 EVT CCVT =
14303 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), IdxVT);
14304 SDValue Keep =
14305 DAG.getSetCC(DL, CCVT, Idx, DAG.getStepVector(DL, IdxVT), ISD::SETEQ);
14306 SDValue Swapped = DAG.getVectorShuffle(VecVT, DL, V, V, {1, 0});
14307 return DAG.getNode(ISD::VSELECT, DL, VecVT, Keep, V, Swapped);
14308 }
14309
14310 // Res[i] = V[Mask[i]]. Out-of-range extracts are undefined, as required.
14311 MVT IdxVT = getVectorIdxTy(DAG.getDataLayout());
14313 DAG.ExtractVectorElements(Mask, Elts);
14314 for (SDValue &Elt : Elts) {
14315 SDValue Idx = DAG.getZExtOrTrunc(Elt, DL, IdxVT);
14316 Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ScalarVT, V, Idx);
14317 }
14318 return DAG.getBuildVector(VecVT, DL, Elts);
14319}
14320
14322 SDLoc DL(Node);
14323 EVT VT = Node->getValueType(0);
14324 SDValue Op = Node->getOperand(0);
14325 ElementCount EC = Op.getValueType().getVectorElementCount();
14326
14327 bool ZeroIsPoison = Node->getOpcode() == ISD::CTTZ_ELTS_ZERO_POISON;
14328 auto [Mask, StepVec] = getLegalMaskAndStepVector(Op, ZeroIsPoison, DL, DAG);
14329
14330 // No legal step vector: split mask in half and recombine results.
14331 // LoNumElts uses the non-poison CTTZ_ELTS so its result is well-defined
14332 // (== LoNumElts when no active lane), allowing the SETNE comparison.
14333 // Result: (ResLo != LoNumElts) ? ResLo : (LoNumElts + ResHi)
14334 if (!StepVec) {
14335 EVT ResVT = Node->getValueType(0);
14336 auto [MaskLo, MaskHi] = DAG.SplitVector(Op, DL);
14337 SDValue LoNumElts = DAG.getElementCount(
14338 DL, ResVT, MaskLo.getValueType().getVectorElementCount());
14339 SDValue ResLo = DAG.getNode(ISD::CTTZ_ELTS, DL, ResVT, MaskLo);
14340 SDValue ResHi = DAG.getNode(Node->getOpcode(), DL, ResVT, MaskHi);
14341 SDValue ResLoNotNumElts = DAG.getSetCC(
14342 DL, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), ResVT),
14343 ResLo, LoNumElts, ISD::SETNE);
14344 // Per LangRef, ResVT must be wide enough to hold the total element count,
14345 // so the sum cannot wrap as an unsigned add. NSW is not guaranteed since
14346 // the count is only required to fit unsigned.
14347 SDValue Sum = DAG.getNode(ISD::ADD, DL, ResVT, LoNumElts, ResHi,
14349 return DAG.getSelect(DL, ResVT, ResLoNotNumElts, ResLo, Sum);
14350 }
14351
14352 EVT StepVecVT = StepVec.getValueType();
14353 EVT StepVT = StepVecVT.getVectorElementType();
14354
14355 // Promote the scalar result type early to avoid redundant zexts.
14357 StepVT = getTypeToTransformTo(*DAG.getContext(), StepVT);
14358
14359 SDValue VL = DAG.getElementCount(DL, StepVT, EC);
14360 SDValue SplatVL = DAG.getSplat(StepVecVT, DL, VL);
14361 StepVec = DAG.getNode(ISD::SUB, DL, StepVecVT, SplatVL, StepVec);
14362 SDValue Zeroes = DAG.getConstant(0, DL, StepVecVT);
14363 SDValue Select = DAG.getSelect(DL, StepVecVT, Mask, StepVec, Zeroes);
14365 StepVecVT.getVectorElementType(), Select);
14366 SDValue Sub = DAG.getNode(ISD::SUB, DL, StepVT, VL,
14367 DAG.getZExtOrTrunc(Max, DL, StepVT));
14368
14369 return DAG.getZExtOrTrunc(Sub, DL, VT);
14370}
14371
14373 SDLoc DL(N);
14374 SDValue Source = N->getOperand(0);
14375 SDValue Needle = N->getOperand(1);
14376 SDValue Mask = N->getOperand(2);
14377 EVT SourceVT = Source.getValueType();
14378 EVT NeedleVT = Needle.getValueType();
14379 EVT ResVT = N->getValueType(0);
14380 EVT CmpVT =
14381 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), SourceVT);
14382
14383 assert(NeedleVT.isFixedLengthVector() && "Needle must be a fixed vector");
14384
14385 SDValue Ret = DAG.getConstant(0, DL, CmpVT);
14386 EVT NeedleEltVT = NeedleVT.getVectorElementType();
14387 for (unsigned I = 0, E = NeedleVT.getVectorNumElements(); I != E; ++I) {
14388 SDValue Splat;
14389 if (NeedleVT == SourceVT) {
14390 // Prefer a shuffle over scalar extracts + splat for fixed vectors.
14391 Splat = DAG.getVectorShuffle(
14392 SourceVT, DL, Needle, DAG.getUNDEF(SourceVT),
14394 } else {
14395 SDValue NeedleElt = DAG.getExtractVectorElt(DL, NeedleEltVT, Needle, I);
14396 Splat = DAG.getNode(ISD::SPLAT_VECTOR, DL, SourceVT, NeedleElt);
14397 }
14398 SDValue Cmp = DAG.getSetCC(DL, CmpVT, Source, Splat, ISD::SETEQ);
14399 Ret = DAG.getNode(ISD::OR, DL, CmpVT, Ret, Cmp);
14400 }
14401
14402 EVT UseVT = ResVT;
14403 // If the result is immediately truncated, only extend to that type (to avoid
14404 // unnecessary sign/zero extends).
14405 if (N->hasOneUse() && N->user_begin()->getOpcode() == ISD::TRUNCATE)
14406 UseVT = N->user_begin()->getValueType(0);
14407
14408 Mask = DAG.getBoolExtOrTrunc(Mask, DL, UseVT, Mask.getValueType());
14409 Ret = DAG.getBoolExtOrTrunc(Ret, DL, UseVT, Ret.getValueType());
14410
14411 Ret = DAG.getNode(ISD::AND, DL, UseVT, Ret, Mask);
14412 if (UseVT != ResVT)
14413 Ret = DAG.getNode(ISD::ANY_EXTEND, DL, ResVT, Ret);
14414 return Ret;
14415}
14416
14418 SelectionDAG &DAG) const {
14419 SDLoc DL(N);
14420 SDValue Acc = N->getOperand(0);
14421 SDValue MulLHS = N->getOperand(1);
14422 SDValue MulRHS = N->getOperand(2);
14423 EVT AccVT = Acc.getValueType();
14424 EVT MulOpVT = MulLHS.getValueType();
14425
14426 EVT ExtMulOpVT =
14428 MulOpVT.getVectorElementCount());
14429
14430 unsigned ExtOpcLHS, ExtOpcRHS;
14431 switch (N->getOpcode()) {
14432 default:
14433 llvm_unreachable("Unexpected opcode");
14435 ExtOpcLHS = ExtOpcRHS = ISD::ZERO_EXTEND;
14436 break;
14438 ExtOpcLHS = ExtOpcRHS = ISD::SIGN_EXTEND;
14439 break;
14441 ExtOpcLHS = ISD::SIGN_EXTEND;
14442 ExtOpcRHS = ISD::ZERO_EXTEND;
14443 break;
14445 ExtOpcLHS = ExtOpcRHS = ISD::FP_EXTEND;
14446 break;
14447 }
14448
14449 // A wide partial reduction is built from a ladder of narrower ones, a rung
14450 // at a time, each halving the element count and doubling the width.
14451 unsigned Opc = N->getOpcode();
14452 ElementCount MulEC = MulOpVT.getVectorElementCount();
14453 ElementCount AccEC = AccVT.getVectorElementCount();
14454 unsigned CountRatio =
14455 MulEC.hasKnownScalarFactor(AccEC) ? MulEC.getKnownScalarFactor(AccEC) : 0;
14456 unsigned WidthRatio =
14457 AccVT.getScalarSizeInBits() / MulOpVT.getScalarSizeInBits();
14458 if (Opc != ISD::PARTIAL_REDUCE_FMLA && CountRatio > 2 && WidthRatio >= 2) {
14459 LLVMContext &Ctx = *DAG.getContext();
14460 EVT ProdVT = MulOpVT.widenIntegerVectorElementType(Ctx);
14461
14462 // A pure reduction peels one rung and re-enters.
14463 if (llvm::isOneOrOneSplat(MulRHS)) {
14464 EVT RungVT = ProdVT.getHalfNumVectorElementsVT(Ctx);
14465 return DAG.getNode(Opc, DL, AccVT, Acc,
14466 DAG.getNode(Opc, DL, RungVT,
14467 DAG.getConstant(0, DL, RungVT), MulLHS,
14468 MulRHS),
14469 DAG.getConstant(1, DL, RungVT));
14470 }
14471
14472 // A multiply widens the products by one rung, which legalizes back into a
14473 // widening multiply per half, and the ladder re-enters as a plain sum.
14474 SDValue Prod = DAG.getNode(ISD::MUL, DL, ProdVT,
14475 DAG.getNode(ExtOpcLHS, DL, ProdVT, MulLHS),
14476 DAG.getNode(ExtOpcRHS, DL, ProdVT, MulRHS));
14477 auto [Lo, Hi] = DAG.SplitVector(Prod, DL);
14478 SDValue One = DAG.getConstant(1, DL, Lo.getValueType());
14479
14480 // The halves meet at the narrowest rung, so the accumulator is added once.
14481 EVT MidVT = Lo.getValueType()
14482 .widenIntegerVectorElementType(Ctx)
14483 .getHalfNumVectorElementsVT(Ctx);
14485 return DAG.getNode(Opc, DL, AccVT,
14486 DAG.getNode(Opc, DL, AccVT, Acc, Lo, One), Hi, One);
14487 SDValue Mid =
14488 DAG.getNode(Opc, DL, MidVT, DAG.getConstant(0, DL, MidVT), Lo, One);
14489 Mid = DAG.getNode(Opc, DL, MidVT, Mid, Hi, One);
14490 return DAG.getNode(Opc, DL, AccVT, Acc, Mid, DAG.getConstant(1, DL, MidVT));
14491 }
14492
14493 if (ExtMulOpVT != MulOpVT) {
14494 MulLHS = DAG.getNode(ExtOpcLHS, DL, ExtMulOpVT, MulLHS);
14495 MulRHS = DAG.getNode(ExtOpcRHS, DL, ExtMulOpVT, MulRHS);
14496 }
14497 SDValue Input = MulLHS;
14498 // Maintain the FP flags (if any).
14499 SDNodeFlags Flags = N->getFlags();
14500 if (N->getOpcode() == ISD::PARTIAL_REDUCE_FMLA) {
14501 if (!llvm::isOneOrOneSplatFP(MulRHS))
14502 Input = DAG.getNode(ISD::FMUL, DL, ExtMulOpVT, MulLHS, MulRHS, Flags);
14503 } else if (!llvm::isOneOrOneSplat(MulRHS)) {
14504 Input = DAG.getNode(ISD::MUL, DL, ExtMulOpVT, MulLHS, MulRHS);
14505 }
14506
14507 unsigned Stride = AccVT.getVectorMinNumElements();
14508 unsigned ScaleFactor = MulOpVT.getVectorMinNumElements() / Stride;
14509
14510 // Collect all of the subvectors
14511 std::deque<SDValue> Subvectors = {Acc};
14512 for (unsigned I = 0; I < ScaleFactor; I++)
14513 Subvectors.push_back(DAG.getExtractSubvector(DL, AccVT, Input, I * Stride));
14514
14515 unsigned FlatNode =
14516 N->getOpcode() == ISD::PARTIAL_REDUCE_FMLA ? ISD::FADD : ISD::ADD;
14517
14518 // Flatten the subvector tree
14519 while (Subvectors.size() > 1) {
14520 Subvectors.push_back(DAG.getNode(FlatNode, DL, AccVT,
14521 {Subvectors[0], Subvectors[1]}, Flags));
14522 Subvectors.pop_front();
14523 Subvectors.pop_front();
14524 }
14525
14526 assert(Subvectors.size() == 1 &&
14527 "There should only be one subvector after tree flattening");
14528
14529 return Subvectors[0];
14530}
14531
14532/// Given a store node \p StoreNode, return true if it is safe to fold that node
14533/// into \p FPNode, which expands to a library call with output pointers.
14535 SDNode *FPNode,
14536 unsigned MaxSteps) {
14538 SmallVector<const SDNode *, 8> DeferredNodes;
14540
14541 // Skip FPNode use by StoreNode (that's the use we want to fold into FPNode).
14542 for (SDValue Op : StoreNode->ops())
14543 if (Op.getNode() != FPNode)
14544 Worklist.push_back(Op.getNode());
14545
14546 while (!Worklist.empty()) {
14547 const SDNode *Node = Worklist.pop_back_val();
14548 auto [_, Inserted] = Visited.insert(Node);
14549 if (!Inserted)
14550 continue;
14551
14552 if (MaxSteps > 0 && Visited.size() >= MaxSteps)
14553 return false;
14554
14555 // Reached the FPNode (would result in a cycle).
14556 // OR Reached CALLSEQ_START (would result in nested call sequences).
14557 if (Node == FPNode || Node->getOpcode() == ISD::CALLSEQ_START)
14558 return false;
14559
14560 if (Node->getOpcode() == ISD::CALLSEQ_END) {
14561 // Defer looking into call sequences (so we can check we're outside one).
14562 // We still need to look through these for the predecessor check.
14563 DeferredNodes.push_back(Node);
14564 continue;
14565 }
14566
14567 for (SDValue Op : Node->ops())
14568 Worklist.push_back(Op.getNode());
14569 }
14570
14571 // True if we're outside a call sequence and don't have the FPNode as a
14572 // predecessor. No cycles or nested call sequences possible.
14573 return !SDNode::hasPredecessorHelper(FPNode, Visited, DeferredNodes,
14574 MaxSteps);
14575}
14576
14578 SelectionDAG &DAG, RTLIB::Libcall LC, SDNode *Node,
14580 std::optional<unsigned> CallRetResNo) const {
14581 if (LC == RTLIB::UNKNOWN_LIBCALL)
14582 return false;
14583
14584 RTLIB::LibcallImpl LibcallImpl =
14586 if (LibcallImpl == RTLIB::Unsupported)
14587 return false;
14588
14589 LLVMContext &Ctx = *DAG.getContext();
14590 EVT VT = Node->getValueType(0);
14591 unsigned NumResults = Node->getNumValues();
14592 unsigned MaxSteps = DAG.getHasPredecessorMaxSteps();
14593
14594 // Find users of the node that store the results (and share input chains). The
14595 // destination pointers can be used instead of creating stack allocations.
14596 SDValue StoresInChain;
14597 SmallVector<StoreSDNode *, 2> ResultStores(NumResults);
14598 for (SDNode *User : Node->users()) {
14600 continue;
14601 auto *ST = cast<StoreSDNode>(User);
14602 SDValue StoreValue = ST->getValue();
14603 unsigned ResNo = StoreValue.getResNo();
14604 // Ensure the store corresponds to an output pointer.
14605 if (CallRetResNo == ResNo)
14606 continue;
14607 // Ensure the store to the default address space and not atomic or volatile.
14608 if (!ST->isSimple() || ST->getAddressSpace() != 0)
14609 continue;
14610 // Ensure all store chains are the same (so they don't alias).
14611 if (StoresInChain && ST->getChain() != StoresInChain)
14612 continue;
14613 // Ensure the store is properly aligned.
14614 Type *StoreType = StoreValue.getValueType().getTypeForEVT(Ctx);
14615 if (ST->getAlign() <
14616 DAG.getDataLayout().getABITypeAlign(StoreType->getScalarType()))
14617 continue;
14618 // Avoid:
14619 // 1. Creating cyclic dependencies.
14620 // 2. Expanding the node to a call within a call sequence.
14621 if (!canFoldStoreIntoLibCallOutputPointers(ST, Node, MaxSteps))
14622 continue;
14623 ResultStores[ResNo] = ST;
14624 StoresInChain = ST->getChain();
14625 }
14626
14627 ArgListTy Args;
14628
14629 // Pass the arguments.
14630 for (const SDValue &Op : Node->op_values()) {
14631 EVT ArgVT = Op.getValueType();
14632 Type *ArgTy = ArgVT.getTypeForEVT(Ctx);
14633 Args.emplace_back(Op, ArgTy);
14634 }
14635
14636 // Pass the output pointers.
14637 SmallVector<SDValue, 2> ResultPtrs(NumResults);
14639 for (auto [ResNo, ST] : llvm::enumerate(ResultStores)) {
14640 if (ResNo == CallRetResNo)
14641 continue;
14642 EVT ResVT = Node->getValueType(ResNo);
14643 SDValue ResultPtr = ST ? ST->getBasePtr() : DAG.CreateStackTemporary(ResVT);
14644 ResultPtrs[ResNo] = ResultPtr;
14645 Args.emplace_back(ResultPtr, PointerTy);
14646 }
14647
14648 SDLoc DL(Node);
14649
14651 // Pass the vector mask (if required).
14652 EVT MaskVT = getSetCCResultType(DAG.getDataLayout(), Ctx, VT);
14653 SDValue Mask = DAG.getBoolConstant(true, DL, MaskVT, VT);
14654 Args.emplace_back(Mask, MaskVT.getTypeForEVT(Ctx));
14655 }
14656
14657 Type *RetType = CallRetResNo.has_value()
14658 ? Node->getValueType(*CallRetResNo).getTypeForEVT(Ctx)
14659 : Type::getVoidTy(Ctx);
14660 SDValue InChain = StoresInChain ? StoresInChain : DAG.getEntryNode();
14661 SDValue Callee =
14662 DAG.getExternalSymbol(LibcallImpl, getPointerTy(DAG.getDataLayout()));
14664 CLI.setDebugLoc(DL).setChain(InChain).setLibCallee(
14665 DAG.getLibcalls().getLibcallImplCallingConv(LibcallImpl), RetType, Callee,
14666 std::move(Args));
14667
14668 auto [Call, CallChain] = LowerCallTo(CLI);
14669
14670 for (auto [ResNo, ResultPtr] : llvm::enumerate(ResultPtrs)) {
14671 if (ResNo == CallRetResNo) {
14672 Results.push_back(Call);
14673 continue;
14674 }
14675 MachinePointerInfo PtrInfo;
14676 SDValue LoadResult = DAG.getLoad(Node->getValueType(ResNo), DL, CallChain,
14677 ResultPtr, PtrInfo);
14678 SDValue OutChain = LoadResult.getValue(1);
14679
14680 if (StoreSDNode *ST = ResultStores[ResNo]) {
14681 // Replace store with the library call.
14682 DAG.ReplaceAllUsesOfValueWith(SDValue(ST, 0), OutChain);
14683 PtrInfo = ST->getPointerInfo();
14684 } else {
14686 DAG.getMachineFunction(),
14687 cast<FrameIndexSDNode>(ResultPtr)->getIndex());
14688 }
14689
14690 Results.push_back(LoadResult);
14691 }
14692
14693 return true;
14694}
14695
14697 SDValue &LHS, SDValue &RHS,
14698 SDValue &CC, bool &NeedInvert,
14699 const SDLoc &dl, SDValue &Chain,
14700 bool IsSignaling) const {
14701 MVT OpVT = LHS.getSimpleValueType();
14702 ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
14703 NeedInvert = false;
14704 switch (getCondCodeAction(CCCode, OpVT)) {
14705 default:
14706 llvm_unreachable("Unknown condition code action!");
14708 // Nothing to do.
14709 break;
14712 if (isCondCodeLegalOrCustom(InvCC, OpVT)) {
14713 std::swap(LHS, RHS);
14714 CC = DAG.getCondCode(InvCC);
14715 return true;
14716 }
14717 // Swapping operands didn't work. Try inverting the condition.
14718 bool NeedSwap = false;
14719 InvCC = getSetCCInverse(CCCode, OpVT);
14720 if (!isCondCodeLegalOrCustom(InvCC, OpVT)) {
14721 // If inverting the condition is not enough, try swapping operands
14722 // on top of it.
14723 InvCC = ISD::getSetCCSwappedOperands(InvCC);
14724 NeedSwap = true;
14725 }
14726 if (isCondCodeLegalOrCustom(InvCC, OpVT)) {
14727 CC = DAG.getCondCode(InvCC);
14728 NeedInvert = true;
14729 if (NeedSwap)
14730 std::swap(LHS, RHS);
14731 return true;
14732 }
14733
14734 // Special case: expand i1 comparisons using logical operations.
14735 if (OpVT == MVT::i1) {
14736 SDValue Ret;
14737 switch (CCCode) {
14738 default:
14739 llvm_unreachable("Unknown integer setcc!");
14740 case ISD::SETEQ: // X == Y --> ~(X ^ Y)
14741 Ret = DAG.getNOT(dl, DAG.getNode(ISD::XOR, dl, MVT::i1, LHS, RHS),
14742 MVT::i1);
14743 break;
14744 case ISD::SETNE: // X != Y --> (X ^ Y)
14745 Ret = DAG.getNode(ISD::XOR, dl, MVT::i1, LHS, RHS);
14746 break;
14747 case ISD::SETGT: // X >s Y --> X == 0 & Y == 1 --> ~X & Y
14748 case ISD::SETULT: // X <u Y --> X == 0 & Y == 1 --> ~X & Y
14749 Ret = DAG.getNode(ISD::AND, dl, MVT::i1, RHS,
14750 DAG.getNOT(dl, LHS, MVT::i1));
14751 break;
14752 case ISD::SETLT: // X <s Y --> X == 1 & Y == 0 --> ~Y & X
14753 case ISD::SETUGT: // X >u Y --> X == 1 & Y == 0 --> ~Y & X
14754 Ret = DAG.getNode(ISD::AND, dl, MVT::i1, LHS,
14755 DAG.getNOT(dl, RHS, MVT::i1));
14756 break;
14757 case ISD::SETULE: // X <=u Y --> X == 0 | Y == 1 --> ~X | Y
14758 case ISD::SETGE: // X >=s Y --> X == 0 | Y == 1 --> ~X | Y
14759 Ret = DAG.getNode(ISD::OR, dl, MVT::i1, RHS,
14760 DAG.getNOT(dl, LHS, MVT::i1));
14761 break;
14762 case ISD::SETUGE: // X >=u Y --> X == 1 | Y == 0 --> ~Y | X
14763 case ISD::SETLE: // X <=s Y --> X == 1 | Y == 0 --> ~Y | X
14764 Ret = DAG.getNode(ISD::OR, dl, MVT::i1, LHS,
14765 DAG.getNOT(dl, RHS, MVT::i1));
14766 break;
14767 }
14768
14769 LHS = DAG.getZExtOrTrunc(Ret, dl, VT);
14770 RHS = SDValue();
14771 CC = SDValue();
14772 return true;
14773 }
14774
14776 unsigned Opc = 0;
14777 switch (CCCode) {
14778 default:
14779 llvm_unreachable("Don't know how to expand this condition!");
14780 case ISD::SETUO:
14781 if (isCondCodeLegal(ISD::SETUNE, OpVT)) {
14782 CC1 = ISD::SETUNE;
14783 CC2 = ISD::SETUNE;
14784 Opc = ISD::OR;
14785 break;
14786 }
14788 "If SETUE is expanded, SETOEQ or SETUNE must be legal!");
14789 NeedInvert = true;
14790 [[fallthrough]];
14791 case ISD::SETO:
14793 "If SETO is expanded, SETOEQ must be legal!");
14794 CC1 = ISD::SETOEQ;
14795 CC2 = ISD::SETOEQ;
14796 Opc = ISD::AND;
14797 break;
14798 case ISD::SETONE:
14799 case ISD::SETUEQ:
14800 // If the SETUO or SETO CC isn't legal, we might be able to use
14801 // SETOGT || SETOLT, inverting the result for SETUEQ. We only need one
14802 // of SETOGT/SETOLT to be legal, the other can be emulated by swapping
14803 // the operands.
14804 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
14805 if (!isCondCodeLegal(CC2, OpVT) && (isCondCodeLegal(ISD::SETOGT, OpVT) ||
14806 isCondCodeLegal(ISD::SETOLT, OpVT))) {
14807 CC1 = ISD::SETOGT;
14808 CC2 = ISD::SETOLT;
14809 Opc = ISD::OR;
14810 NeedInvert = ((unsigned)CCCode & 0x8U);
14811 break;
14812 }
14813 [[fallthrough]];
14814 case ISD::SETOEQ:
14815 case ISD::SETOGT:
14816 case ISD::SETOGE:
14817 case ISD::SETOLT:
14818 case ISD::SETOLE:
14819 case ISD::SETUNE:
14820 case ISD::SETUGT:
14821 case ISD::SETUGE:
14822 case ISD::SETULT:
14823 case ISD::SETULE:
14824 // If we are floating point, assign and break, otherwise fall through.
14825 if (!OpVT.isInteger()) {
14826 // We can use the 4th bit to tell if we are the unordered
14827 // or ordered version of the opcode.
14828 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
14829 Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND;
14830 CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10);
14831 break;
14832 }
14833 // Fallthrough if we are unsigned integer.
14834 [[fallthrough]];
14835 case ISD::SETLE:
14836 case ISD::SETGT:
14837 case ISD::SETGE:
14838 case ISD::SETLT:
14839 case ISD::SETNE:
14840 case ISD::SETEQ:
14841 // If all combinations of inverting the condition and swapping operands
14842 // didn't work then we have no means to expand the condition.
14843 llvm_unreachable("Don't know how to expand this condition!");
14844 }
14845
14846 SDValue SetCC1, SetCC2;
14847 if (CCCode != ISD::SETO && CCCode != ISD::SETUO) {
14848 // If we aren't the ordered or unorder operation,
14849 // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS).
14850 SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1, Chain, IsSignaling);
14851 SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2, Chain, IsSignaling);
14852 } else {
14853 // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS)
14854 SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1, Chain, IsSignaling);
14855 SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2, Chain, IsSignaling);
14856 }
14857 if (Chain)
14858 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, SetCC1.getValue(1),
14859 SetCC2.getValue(1));
14860 LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2);
14861 RHS = SDValue();
14862 CC = SDValue();
14863 return true;
14864 }
14865 }
14866 return false;
14867}
14868
14870 SelectionDAG &DAG) const {
14871 EVT VT = Node->getValueType(0);
14872 // Despite its documentation, GetSplitDestVTs will assert if VT cannot be
14873 // split into two equal parts.
14874 if (!VT.isVector() || !VT.getVectorElementCount().isKnownMultipleOf(2))
14875 return SDValue();
14876
14877 // Restrict expansion to cases where both parts can be concatenated.
14878 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT);
14879 if (LoVT != HiVT || !isTypeLegal(LoVT))
14880 return SDValue();
14881
14882 SDLoc DL(Node);
14883 unsigned Opcode = Node->getOpcode();
14884
14885 // Don't expand if the result is likely to be unrolled anyway.
14886 if (!isOperationLegalOrCustomOrPromote(Opcode, LoVT))
14887 return SDValue();
14888
14889 SmallVector<SDValue, 4> LoOps, HiOps;
14890 for (const SDValue &V : Node->op_values()) {
14891 if (!V.getValueType().isVector()) {
14892 // Scalar operands pass through to both halves unchanged.
14893 LoOps.push_back(V);
14894 HiOps.push_back(V);
14895 continue;
14896 }
14897 auto [Lo, Hi] = DAG.SplitVector(V, DL, LoVT, HiVT);
14898 LoOps.push_back(Lo);
14899 HiOps.push_back(Hi);
14900 }
14901
14902 bool IsStrict = Node->isStrictFPOpcode();
14903 SDVTList LoVTs =
14904 IsStrict ? DAG.getVTList(LoVT, MVT::Other) : DAG.getVTList(LoVT);
14905 SDVTList HiVTs =
14906 IsStrict ? DAG.getVTList(HiVT, MVT::Other) : DAG.getVTList(HiVT);
14907 SDValue SplitOpLo = DAG.getNode(Opcode, DL, LoVTs, LoOps, Node->getFlags());
14908 SDValue SplitOpHi = DAG.getNode(Opcode, DL, HiVTs, HiOps, Node->getFlags());
14909 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, SplitOpLo, SplitOpHi);
14910
14911 if (!IsStrict)
14912 return Res;
14913
14914 SDValue Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
14915 SplitOpLo.getValue(1), SplitOpHi.getValue(1));
14916 return DAG.getMergeValues({Res, Chain}, DL);
14917}
14918
14920 const SDLoc &DL,
14921 EVT InVecVT, SDValue EltNo,
14922 LoadSDNode *OriginalLoad,
14923 SelectionDAG &DAG) const {
14924 assert(OriginalLoad->isSimple());
14925
14926 EVT VecEltVT = InVecVT.getVectorElementType();
14927
14928 // If the vector element type is not a multiple of a byte then we are unable
14929 // to correctly compute an address to load only the extracted element as a
14930 // scalar.
14931 if (!VecEltVT.isByteSized())
14932 return SDValue();
14933
14934 ISD::LoadExtType ExtTy =
14935 ResultVT.bitsGT(VecEltVT) ? ISD::EXTLOAD : ISD::NON_EXTLOAD;
14936 if (!isOperationLegalOrCustom(ISD::LOAD, VecEltVT))
14937 return SDValue();
14938
14939 std::optional<unsigned> ByteOffset;
14940 Align Alignment = OriginalLoad->getAlign();
14942 if (auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo)) {
14943 int Elt = ConstEltNo->getZExtValue();
14944 ByteOffset = VecEltVT.getSizeInBits() * Elt / 8;
14945 MPI = OriginalLoad->getPointerInfo().getWithOffset(*ByteOffset);
14946 Alignment = commonAlignment(Alignment, *ByteOffset);
14947 } else {
14948 // Discard the pointer info except the address space because the memory
14949 // operand can't represent this new access since the offset is variable.
14950 MPI = MachinePointerInfo(OriginalLoad->getPointerInfo().getAddrSpace());
14951 Alignment = commonAlignment(Alignment, VecEltVT.getSizeInBits() / 8);
14952 }
14953
14954 if (!shouldReduceLoadWidth(OriginalLoad, ExtTy, VecEltVT, ByteOffset))
14955 return SDValue();
14956
14957 unsigned IsFast = 0;
14958 if (!allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VecEltVT,
14959 OriginalLoad->getAddressSpace(), Alignment,
14960 OriginalLoad->getMemOperand()->getFlags(), &IsFast) ||
14961 !IsFast)
14962 return SDValue();
14963
14964 // The original DAG loaded the entire vector from memory, so arithmetic
14965 // within it must be inbounds.
14967 DAG, OriginalLoad->getBasePtr(), InVecVT, EltNo);
14968
14969 // We are replacing a vector load with a scalar load. The new load must have
14970 // identical memory op ordering to the original.
14971 SDValue Load;
14972 if (ResultVT.bitsGT(VecEltVT)) {
14973 // If the result type of vextract is wider than the load, then issue an
14974 // extending load instead.
14975 ISD::LoadExtType ExtType =
14976 isLoadLegal(ResultVT, VecEltVT, Alignment,
14977 OriginalLoad->getAddressSpace(), ISD::ZEXTLOAD, false)
14979 : ISD::EXTLOAD;
14980 Load = DAG.getExtLoad(ExtType, DL, ResultVT, OriginalLoad->getChain(),
14981 NewPtr, MPI, VecEltVT, Alignment,
14982 OriginalLoad->getMemOperand()->getFlags(),
14983 OriginalLoad->getMMOMetadataForSubAccess());
14984 DAG.makeEquivalentMemoryOrdering(OriginalLoad, Load);
14985 } else {
14986 // The result type is narrower or the same width as the vector element
14987 Load = DAG.getLoad(VecEltVT, DL, OriginalLoad->getChain(), NewPtr, MPI,
14988 Alignment, OriginalLoad->getMemOperand()->getFlags(),
14989 OriginalLoad->getMMOMetadataForSubAccess());
14990 DAG.makeEquivalentMemoryOrdering(OriginalLoad, Load);
14991 if (ResultVT.bitsLT(VecEltVT))
14992 Load = DAG.getNode(ISD::TRUNCATE, DL, ResultVT, Load);
14993 else
14994 Load = DAG.getBitcast(ResultVT, Load);
14995 }
14996
14997 return Load;
14998}
14999
15000// Set type id for call site info and metadata 'call_target'.
15001// We are filtering for:
15002// a) The call-graph-section use case that wants to know about indirect
15003// calls, or
15004// b) We want to annotate indirect calls.
15006 const CallBase *CB, MachineFunction &MF,
15007 MachineFunction::CallSiteInfo &CSInfo) const {
15008 if (CB && CB->isIndirectCall() &&
15011 CSInfo = MachineFunction::CallSiteInfo(*CB);
15012}
return SDValue()
const TargetLowering & TLI
unsigned RegSize
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
uint64_t uint64_t
uint64_t Imm
constexpr LLT F32
AMDGPU Register Bank Select
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis Results
#define X(NUM, ENUM, NAME)
Definition ELF.h:665
block Block Frequency Analysis
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static std::optional< bool > isBigEndian(const SmallDenseMap< int64_t, int64_t, 8 > &MemOffset2Idx, int64_t LowestIdx)
Given a map from byte offsets in memory to indices in a load/store, determine if that map corresponds...
static bool isSigned(unsigned Opcode)
#define _
static bool ShrinkDemandedConstant(Instruction *I, unsigned OpNo, const APInt &Demanded)
Check to see if the specified operand of the specified instruction is a constant integer.
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
#define RegName(no)
lazy value info
static bool isNonZeroModBitWidthOrUndef(const MachineRegisterInfo &MRI, Register Reg, unsigned BW)
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition Lint.cpp:550
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
#define G(x, y, z)
Definition MD5.cpp:55
static bool isUndef(const MachineInstr &MI)
Register const TargetRegisterInfo * TRI
#define T
#define T1
uint64_t High
#define P(N)
Function const char * Passes
if(auto Err=PB.parsePassPipeline(MPM, Passes)) return wrap(std MPM run * Mod
R600 Clause Merge
MachineBasicBlock * Folded
SmallVector< MachineOperand, 4 > Cond
int64_t C1
RISCVCC::CondCode CC
MachineRegisterInfo & MRI
int64_t C0
Contains matchers for matching SelectionDAG nodes and values.
This file contains some templates that are useful if you are working with the STL at all.
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static std::pair< SDValue, SDValue > getLegalMaskAndStepVector(SDValue Mask, bool ZeroIsPoison, SDLoc DL, SelectionDAG &DAG)
Returns a type-legalized version of Mask as the first item in the pair.
static SDValue foldSetCCWithFunnelShift(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, const SDLoc &dl, SelectionDAG &DAG)
static bool lowerImmediateIfPossible(TargetLowering::ConstraintPair &P, SDValue Op, SelectionDAG *DAG, const TargetLowering &TLI)
If we have an immediate, see if we can lower it.
#define FP_CMP_LIBCALL(BASE)
static APInt getKnownUndefForVectorBinop(SDValue BO, SelectionDAG &DAG, const APInt &UndefOp0, const APInt &UndefOp1)
Given a vector binary operation and known undefined elements for each input operand,...
static SDValue BuildExactUDIV(const TargetLowering &TLI, SDNode *N, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl< SDNode * > &Created)
Given an exact UDIV by a constant, create a multiplication with the multiplicative inverse of the con...
static void setArgListEntryAttributes(TargetLoweringBase::ArgListEntry &Entry, const SourceT &Src, unsigned ArgIdx)
Set CallLoweringInfo attribute flags based on a call instruction and called function attributes.
static bool canFoldStoreIntoLibCallOutputPointers(StoreSDNode *StoreNode, SDNode *FPNode, unsigned MaxSteps)
Given a store node StoreNode, return true if it is safe to fold that node into FPNode,...
static std::pair< RTLIB::Libcall, ISD::CondCode > selectFPCmpLibcall(const LibcallLoweringInfo &Libcalls, RTLIB::Libcall BoolLC, RTLIB::Libcall TriStateLC, RTLIB::Libcall GenericLC, ISD::CondCode TriStateCC)
Select the libcall and the condition code to test its result against 0 for an ordered floating-point ...
static SDValue isSpecificZeroAfterMaybeRounding(SelectionDAG &DAG, const TargetLowering &TLI, const SDLoc &DL, SDValue Val, FPClassTest FPClass)
static bool canNarrowCLMULToLegal(const TargetLowering &TLI, LLVMContext &Ctx, EVT VT, unsigned HalveDepth=0, unsigned TotalDepth=0)
Check if CLMUL on VT can eventually reach a type with legal CLMUL through a chain of halving decompos...
static SDValue combineShiftToAVGSUB(SDValue Op, TargetLowering::TargetLoweringOpt &TLO, const TargetLowering &TLI, const APInt &DemandedBits, const APInt &DemandedElts, unsigned Depth)
static unsigned getConstraintPiority(TargetLowering::ConstraintType CT)
Return a number indicating our preference for chosing a type of constraint over another,...
static std::optional< EVT > getNarrowAVGType(EVT VT, unsigned KnownBits, LLVMContext &Ctx)
static std::optional< bool > isFCmpEqualZero(FPClassTest Test, const fltSemantics &Semantics, const MachineFunction &MF)
Returns a true value if if this FPClassTest can be performed with an ordered fcmp to 0,...
static SDValue clampDynamicVectorIndex(SelectionDAG &DAG, SDValue Idx, EVT VecVT, const SDLoc &DL, ElementCount SubEC)
static void turnVectorIntoSplatVector(MutableArrayRef< SDValue > Values, std::function< bool(SDValue)> Predicate, SDValue AlternativeReplacement=SDValue())
If all values in Values that don't match the predicate are same 'splat' value, then replace all value...
static MaybeAlign getParamAlign(const CallBase &Call, unsigned ArgIdx)
static MaybeAlign getParamStackAlign(const CallBase &Call, unsigned ArgIdx)
static bool canExpandVectorCTPOP(const TargetLowering &TLI, EVT VT)
static SDValue foldSetCCWithRotate(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, const SDLoc &dl, SelectionDAG &DAG)
static SDValue BuildExactSDIV(const TargetLowering &TLI, SDNode *N, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl< SDNode * > &Created)
Given an exact SDIV by a constant, create a multiplication with the multiplicative inverse of the con...
static SDValue simplifySetCCWithCTPOP(const TargetLowering &TLI, EVT VT, SDValue N0, const APInt &C1, ISD::CondCode Cond, const SDLoc &dl, SelectionDAG &DAG)
static SDValue combineShiftToAVG(SDValue Op, TargetLowering::TargetLoweringOpt &TLO, const TargetLowering &TLI, const APInt &DemandedBits, const APInt &DemandedElts, unsigned Depth)
static bool paramHasAttr(const CallBase &Call, unsigned ArgIdx, Attribute::AttrKind Kind)
This file describes how to lower LLVM code to machine code.
static int Lookup(ArrayRef< TableEntry > Table, unsigned Opcode)
static SDValue scalarizeVectorStore(StoreSDNode *Store, MVT StoreVT, SelectionDAG &DAG)
Scalarize a vector store, bitcasting to TargetVT to determine the scalar type.
Value * RHS
Value * LHS
The Input class is used to parse a yaml document into in-memory structs and vectors.
static LLVM_ABI const llvm::fltSemantics & EnumToSemantics(Semantics S)
Definition APFloat.cpp:136
static constexpr roundingMode rmTowardZero
Definition APFloat.h:365
static LLVM_ABI ExponentType semanticsMinExponent(const fltSemantics &)
Definition APFloat.cpp:343
static LLVM_ABI bool semanticsHasSignedRepr(const fltSemantics &)
Definition APFloat.cpp:364
static LLVM_ABI unsigned getSizeInBits(const fltSemantics &Sem)
Returns the size of the floating point number (in bits) in the given semantics.
Definition APFloat.cpp:399
static constexpr roundingMode rmNearestTiesToEven
Definition APFloat.h:361
static LLVM_ABI ExponentType semanticsMaxExponent(const fltSemantics &)
Definition APFloat.cpp:339
static LLVM_ABI unsigned int semanticsPrecision(const fltSemantics &)
Definition APFloat.cpp:335
static LLVM_ABI bool isIEEELikeFP(const fltSemantics &)
Definition APFloat.cpp:376
opStatus
IEEE-754R 7: Default exception handling.
Definition APFloat.h:377
opStatus convertFromAPInt(const APInt &Input, bool IsSigned, roundingMode RM)
Definition APFloat.h:1451
static APFloat getSmallestNormalized(const fltSemantics &Sem, bool Negative=false)
Returns the smallest (by magnitude) normalized finite number in the given semantics.
Definition APFloat.h:1262
APInt bitcastToAPInt() const
Definition APFloat.h:1475
static APFloat getLargest(const fltSemantics &Sem, bool Negative=false)
Returns the largest finite number in the given semantics.
Definition APFloat.h:1242
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
Definition APFloat.h:1202
void changeSign()
Definition APFloat.h:1401
static APFloat getNaN(const fltSemantics &Sem, bool Negative=false, uint64_t payload=0)
Factory for NaN values.
Definition APFloat.h:1213
Class for arbitrary precision integers.
Definition APInt.h:78
LLVM_ABI APInt udiv(const APInt &RHS) const
Unsigned division operation.
Definition APInt.cpp:1602
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition APInt.h:230
static LLVM_ABI void udivrem(const APInt &LHS, const APInt &RHS, APInt &Quotient, APInt &Remainder)
Dual division/remainder interface.
Definition APInt.cpp:1796
void clearBit(unsigned BitPosition)
Set a given bit to 0.
Definition APInt.h:1426
bool isNegatedPowerOf2() const
Check if this APInt's negated value is a power of two greater than zero.
Definition APInt.h:445
LLVM_ABI APInt zext(unsigned width) const
Zero extend to a new width.
Definition APInt.cpp:1057
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
Definition APInt.h:225
uint64_t getZExtValue() const
Get zero extended value.
Definition APInt.h:1560
LLVM_ABI APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
Definition APInt.cpp:1078
unsigned getActiveBits() const
Compute the number of active bits in the value.
Definition APInt.h:1532
LLVM_ABI APInt trunc(unsigned width) const
Truncate to new width.
Definition APInt.cpp:970
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
Definition APInt.h:202
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
Definition APInt.h:1350
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
Definition APInt.h:367
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
Definition APInt.h:1186
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
Definition APInt.h:254
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
Definition APInt.h:376
LLVM_ABI APInt urem(const APInt &RHS) const
Unsigned remainder operation.
Definition APInt.cpp:1695
void setSignBit()
Set the sign bit to 1.
Definition APInt.h:1360
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition APInt.h:1508
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
Definition APInt.h:205
static APInt getMinValue(unsigned numBits)
Gets minimum unsigned value of APInt for a specific bit width.
Definition APInt.h:212
bool isNegative() const
Determine sign of this APInt.
Definition APInt.h:325
void clearAllBits()
Set every bit to 0.
Definition APInt.h:1416
void ashrInPlace(unsigned ShiftAmt)
Arithmetic right-shift this APInt by ShiftAmt in place.
Definition APInt.h:836
LLVM_ABI APInt uadd_ov(const APInt &RHS, bool &Overflow) const
Definition APInt.cpp:1973
void negate()
Negate this APInt in place.
Definition APInt.h:1488
unsigned countr_zero() const
Count the number of trailing zero bits.
Definition APInt.h:1659
unsigned countl_zero() const
The APInt version of std::countl_zero.
Definition APInt.h:1618
static LLVM_ABI APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
Definition APInt.cpp:648
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
Definition APInt.h:215
unsigned getSignificantBits() const
Get the minimum bit size for this signed APInt.
Definition APInt.h:1551
unsigned countLeadingZeros() const
Definition APInt.h:1626
bool isStrictlyPositive() const
Determine if this APInt Value is positive.
Definition APInt.h:352
void flipAllBits()
Toggle every bit to its opposite value.
Definition APInt.h:1472
LLVM_ABI void insertBits(const APInt &SubBits, unsigned bitPosition)
Insert the bits from a smaller APInt starting at bitPosition.
Definition APInt.cpp:393
void clearLowBits(unsigned loBits)
Set bottom loBits bits to 0.
Definition APInt.h:1455
unsigned logBase2() const
Definition APInt.h:1781
uint64_t getLimitedValue(uint64_t Limit=UINT64_MAX) const
If this value is smaller than the specified limit, return it, otherwise return the limit value.
Definition APInt.h:471
void setAllBits()
Set every bit to 1.
Definition APInt.h:1339
LLVM_ABI APInt multiplicativeInverse() const
Definition APInt.cpp:1303
bool isMask(unsigned numBits) const
Definition APInt.h:484
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
Definition APInt.h:330
bool ule(const APInt &RHS) const
Unsigned less or equal comparison.
Definition APInt.h:1154
LLVM_ABI APInt sext(unsigned width) const
Sign extend to a new width.
Definition APInt.cpp:1030
void setBits(unsigned loBit, unsigned hiBit)
Set the bits from loBit (inclusive) to hiBit (exclusive) to 1.
Definition APInt.h:1387
APInt shl(unsigned shiftAmt) const
Left-shift function.
Definition APInt.h:875
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Definition APInt.h:1261
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
Definition APInt.h:436
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
Definition APInt.h:302
void clearBits(unsigned LoBit, unsigned HiBit)
Clear the bits from LoBit (inclusive) to HiBit (exclusive) to 0.
Definition APInt.h:1437
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
Definition APInt.h:292
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
Definition APInt.h:196
void setLowBits(unsigned loBits)
Set the bottom loBits bits.
Definition APInt.h:1408
LLVM_ABI APInt extractBits(unsigned numBits, unsigned bitPosition) const
Return an APInt with the extracted bits [bitPosition,bitPosition+numBits).
Definition APInt.cpp:478
bool isOne() const
Determine if this is a value of 1.
Definition APInt.h:385
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
Definition APInt.h:282
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
Definition APInt.h:235
void clearHighBits(unsigned hiBits)
Set top hiBits bits to 0.
Definition APInt.h:1462
int64_t getSExtValue() const
Get sign extended value.
Definition APInt.h:1582
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
Definition APInt.h:860
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
Definition APInt.h:853
unsigned countr_one() const
Count the number of trailing one bits.
Definition APInt.h:1676
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
Definition APInt.h:1225
void setBitVal(unsigned BitPosition, bool BitValue)
Set a given bit to a given value.
Definition APInt.h:1363
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
size_t size() const
Get the array size.
Definition ArrayRef.h:141
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
Definition ArrayRef.h:185
AttrKind
This enumeration lists the attributes that can be associated with parameters, function results,...
Definition Attributes.h:126
@ None
No attributes have been set.
Definition Attributes.h:128
A "pseudo-class" with methods for operating on BUILD_VECTORs.
LLVM_ABI ConstantSDNode * getConstantSplatNode(const APInt &DemandedElts, BitVector *UndefElements=nullptr) const
Returns the demanded splatted constant or null if this is not a constant splat.
CCValAssign - Represent assignment of one arg/retval to a location.
Register getLocReg() const
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
LLVM_ABI bool isIndirectCall() const
Return true if the callsite is an indirect call.
This class represents a function call, abstracting a target machine's calling convention.
static Constant * get(LLVMContext &Context, ArrayRef< ElementTy > Elts)
get() constructor - Return a constant with array type with an element count and element type matching...
Definition Constants.h:878
ConstantFP - Floating Point Values [float, double].
Definition Constants.h:420
This class represents a range of values.
const APInt & getAPIntValue() const
This is an important base class in LLVM.
Definition Constant.h:44
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
bool isLittleEndian() const
Layout endianness...
Definition DataLayout.h:220
LLVM_ABI Align getABITypeAlign(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
LLVM_ABI Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
static constexpr ElementCount getFixed(ScalarTy MinVal)
Definition TypeSize.h:306
static constexpr ElementCount get(ScalarTy MinVal, bool Scalable)
Definition TypeSize.h:312
constexpr bool isScalar() const
Exactly one element.
Definition TypeSize.h:317
constexpr bool isPowerOf2() const
Definition TypeSize.h:324
Class to represent function types.
AttributeList getAttributes() const
Return the attribute list for this Function.
Definition Function.h:329
const GlobalValue * getGlobal() const
Module * getParent()
Get the module that this global value is contained inside of...
std::vector< std::string > ConstraintCodeVector
Definition InlineAsm.h:104
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition Type.cpp:333
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:76
LLVM_ABI void emitError(const Instruction *I, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
Tracks which library functions to use for a particular subtarget or function.
CallingConv::ID getLibcallImplCallingConv(RTLIB::LibcallImpl Call) const
Get the CallingConv that should be used for the specified libcall.
RTLIB::LibcallImpl getLibcallImpl(RTLIB::Libcall Call) const
Return the lowering's selection of implementation call for Call.
RTLIB::LibcallImpl getVectorLibcallImplOrMasked(RTLIB::Libcall Call) const
Return the implementation for Call, or for its masked variant if Call has no implementation.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
Context object for machine code objects.
Definition MCContext.h:83
Base class for the full range of assembler expressions which are needed for parsing.
Definition MCExpr.h:34
iterator_range< regclass_iterator > regclasses() const
Wrapper class representing physical registers. Should be passed by value.
Definition MCRegister.h:41
static const MCSymbolRefExpr * create(const MCSymbol *Symbol, MCContext &Ctx, SMLoc Loc=SMLoc())
Definition MCExpr.h:213
Machine Value Type.
SimpleValueType SimpleTy
bool isInteger() const
Return true if this is an integer or a vector integer type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static MVT getIntegerVT(unsigned BitWidth)
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
DenormalMode getDenormalMode(const fltSemantics &FPType) const
Returns the denormal handling type for the default rounding mode of the function.
MCSymbol * getJTISymbol(unsigned JTI, MCContext &Ctx, bool isLinkerPrivate=false) const
getJTISymbol - Return the MCSymbol for the specified non-empty jump table.
Function & getFunction()
Return the LLVM function that this machine code represents.
const TargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
@ EK_LabelDifference32
EK_LabelDifference32 - Each entry is the address of the block minus the address of the jump table.
@ EK_BlockAddress
EK_BlockAddress - Each entry is a plain address of block, e.g.: .word LBB123.
Flags getFlags() const
Return the raw flags of the source value,.
static bool clobbersPhysReg(const uint32_t *RegMask, MCRegister PhysReg)
clobbersPhysReg - Returns true if this RegMask clobbers PhysReg.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
unsigned getAddressSpace() const
Return the address space for the associated pointer.
Align getAlign() const
bool isSimple() const
Returns true if the memory operation is neither atomic or volatile.
MachineMemOperand * getMemOperand() const
Return the unique MachineMemOperand object describing the memory reference performed by operation.
MMOMetadata getMMOMetadataForSubAccess() const
Returns metadata that can be copied unchanged to an access covering all or part of this access's byte...
const MachinePointerInfo & getPointerInfo() const
const SDValue & getChain() const
const GlobalVariable * getNamedGlobal(StringRef Name) const
Return the global variable in the module with the specified name, of arbitrary type.
Definition Module.h:534
Represent a mutable reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:294
Class to represent pointers.
static PointerType * getUnqual(LLVMContext &C)
This constructs an opaque pointer to an object in the default address space (address space zero).
static LLVM_ABI PointerType * get(LLVMContext &C, unsigned AddressSpace)
This constructs an opaque pointer to an object in a numbered address space.
Definition Type.cpp:882
Wrapper class representing virtual and physical registers.
Definition Register.h:20
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
SDNodeFlags getFlags() const
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
const SDValue & getOperand(unsigned Num) const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
bool isUndef() const
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node, in exactly one operand.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
ISD::CondCode getOperandCondCode(unsigned i) const
bool use_empty() const
Return true if there are no nodes using value ResNo of Node.
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
uint64_t getConstantOperandVal(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
LLVM_ABI SDValue getElementCount(const SDLoc &DL, EVT VT, ElementCount EC)
bool willNotOverflowAdd(bool IsSigned, SDValue N0, SDValue N1) const
Determine if the result of the addition of 2 nodes can never overflow.
LLVM_ABI Align getReducedAlign(EVT VT, bool UseABI)
In most cases this function returns the ABI alignment for a given type, except for illegal vector typ...
LLVM_ABI bool isKnownNeverLogicalZero(SDValue Op, const APInt &DemandedElts, unsigned Depth=0) const
Test whether the given floating point SDValue (or all elements of it, if it is a vector) is known to ...
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
SDValue getExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT, unsigned Opcode)
Convert Op, which must be of integer type, to the integer type VT, by either any/sign/zero-extending ...
SDValue getExtractVectorElt(const SDLoc &DL, EVT VT, SDValue Vec, unsigned Idx)
Extract element at Idx from Vec.
LLVM_ABI unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
LLVM_ABI unsigned getHasPredecessorMaxSteps() const
LLVM_ABI SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond, const SDLoc &dl, SDNodeFlags Flags={})
Constant fold a setcc to true or false.
bool isKnownNeverSNaN(SDValue Op, const APInt &DemandedElts, unsigned Depth=0) const
LLVM_ABI SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
LLVM_ABI SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
LLVM_ABI SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
LLVM_ABI SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
LLVM_ABI void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
LLVM_ABI SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
LLVM_ABI SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offs=0, bool isT=false, unsigned TargetFlags=0)
LLVM_ABI SDValue makeEquivalentMemoryOrdering(SDValue OldChain, SDValue NewMemOpChain)
If an existing load has uses of its chain, create a token factor node with that chain and the new mem...
LLVM_ABI bool isConstantIntBuildVectorOrConstantInt(SDValue N, bool AllowOpaques=true) const
Test whether the given value is a constant int or similar node.
LLVM_ABI SDValue getJumpTableDebugInfo(int JTI, SDValue Chain, const SDLoc &DL)
LLVM_ABI std::optional< unsigned > getValidMaximumShiftAmount(SDValue V, const APInt &DemandedElts, unsigned Depth=0) const
If a SHL/SRA/SRL node V has shift amounts that are all less than the element bit-width of the shift n...
LLVM_ABI SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
LLVM_ABI SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
LLVM_ABI SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getExtractSubvector(const SDLoc &DL, EVT VT, SDValue Vec, unsigned Idx)
Return the VT typed sub-vector of Vec at Idx.
SDValue getInsertSubvector(const SDLoc &DL, SDValue Vec, SDValue SubVec, unsigned Idx)
Insert SubVec at the Idx element of Vec.
LLVM_ABI SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false, SDNodeFlags Flags={})
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
bool willNotOverflowSub(bool IsSigned, SDValue N0, SDValue N1) const
Determine if the result of the sub of 2 nodes can never overflow.
LLVM_ABI bool shouldOptForSize() const
LLVM_ABI SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
const TargetLowering & getTargetLoweringInfo() const
static constexpr unsigned MaxRecursionDepth
bool isGuaranteedNotToBePoison(SDValue Op, unsigned Depth=0) const
Return true if this function can prove that Op is never poison.
LLVM_ABI std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
LLVM_ABI SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
LLVM_ABI SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
LLVM_ABI SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
LLVM_ABI std::optional< unsigned > getValidShiftAmount(SDValue V, const APInt &DemandedElts, unsigned Depth=0) const
If a SHL/SRA/SRL node V has a uniform shift amount that is less than the element bit-width of the shi...
LLVM_ABI SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
LLVM_ABI SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Helper function to build ISD::STORE nodes.
LLVM_ABI bool doesNodeExist(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops)
Check if a node exists without modifying its flags.
LLVM_ABI SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
LLVM_ABI SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
LLVM_ABI SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, bool isTargetGA=false, unsigned TargetFlags=0)
LLVM_ABI SDValue getTypeSize(const SDLoc &DL, EVT VT, TypeSize TS)
LLVM_ABI SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
LLVM_ABI std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provided VTs and return the low/high part.
LLVM_ABI SDValue getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op)
LLVM_ABI bool SignBitIsZero(SDValue Op, unsigned Depth=0) const
Return true if the sign bit of Op is known to be zero.
LLVM_ABI void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
LLVM_ABI SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
LLVM_ABI SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Loads are not normal binary operators: their result type is not determined by their operands,...
LLVM_ABI bool isIdentityElement(unsigned Opc, SDNodeFlags Flags, SDValue V, unsigned OperandNo, unsigned Depth=0) const
Returns true if V is an identity element of Opc with Flags.
LLVM_ABI bool isGuaranteedNotToBeUndefOrPoison(SDValue Op, UndefPoisonKind Kind=UndefPoisonKind::UndefOrPoison, unsigned Depth=0) const
Return true if this function can prove that Op is never poison and, Kind can be used to track poison ...
LLVM_ABI bool isKnownNeverZero(SDValue Op, unsigned Depth=0) const
Test whether the given SDValue is known to contain non-zero value(s).
LLVM_ABI SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, SDNodeFlags Flags=SDNodeFlags())
LLVM_ABI SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
LLVM_ABI SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
const LibcallLoweringInfo & getLibcalls() const
LLVM_ABI SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI SDValue getValueType(EVT)
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
LLVM_ABI SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
LLVM_ABI bool isKnownNeverNaN(SDValue Op, const APInt &DemandedElts, bool SNaN=false, unsigned Depth=0) const
Test whether the given SDValue (or all elements of it, if it is a vector) is known to never be NaN in...
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
LLVM_ABI unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
LLVM_ABI SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT)
Create a true or false constant of type VT using the target's BooleanContent for type OpVT.
SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, unsigned TargetFlags=0)
LLVM_ABI SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getPOISON(EVT VT)
Return a POISON node. POISON does not have a useful SDLoc.
LLVM_ABI KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
LLVM_ABI SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
LLVM_ABI SDValue getCondCode(ISD::CondCode Cond)
LLVM_ABI bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
LLVM_ABI bool isKnownToBeAPowerOfTwo(SDValue Val, bool OrZero=false, unsigned Depth=0) const
Test if the given value is known to have exactly one bit set.
LLVM_ABI SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getSplat(EVT VT, const SDLoc &DL, SDValue Op)
Returns a node representing a splat of one value into all lanes of the provided vector type.
LLVM_ABI std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
LLVM_ABI SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
static void commuteMask(MutableArrayRef< int > Mask)
Change values in a shuffle permute mask assuming the two vector operands have swapped position.
size_type size() const
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
Definition SmallString.h:28
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition StringRef.h:597
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
Definition StringRef.h:258
constexpr size_t size() const
Get the string size.
Definition StringRef.h:144
constexpr const char * data() const
Get a pointer to the start of the string (which may not be null terminated).
Definition StringRef.h:138
iterator end() const
Definition StringRef.h:116
Class to represent struct types.
LLVM_ABI void setExtAttribute(Attribute::AttrKind Attr)
LLVM_ABI void setAttributes(const CallBase *Call, unsigned ArgIdx)
bool isOperationExpand(unsigned Op, EVT VT) const
Return true if the specified operation is illegal on this target or unlikely to be made legal with cu...
virtual IntegerType * getCmpLibcallReturnType(LLVMContext &Ctx) const
Return the integer type returned by comparison libcalls.
virtual bool isShuffleMaskLegal(ArrayRef< int >, EVT) const
Targets can use this to indicate that they only support some VECTOR_SHUFFLE operations,...
virtual bool shouldRemoveRedundantExtend(SDValue Op) const
Return true (the default) if it is profitable to remove a sext_inreg(x) where the sext is redundant,...
virtual bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT, std::optional< unsigned > ByteOffset=std::nullopt) const
Return true if it is profitable to reduce a load to a smaller type.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
virtual bool preferSelectsOverBooleanArithmetic(EVT VT) const
Should we prefer selects to doing arithmetic on boolean types.
virtual bool isLegalICmpImmediate(int64_t) const
Return true if the specified immediate is legal icmp immediate, that is the target has icmp instructi...
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
virtual bool isSafeMemOpType(MVT) const
Returns true if it's safe to use load / store of the specified type to expand memcpy / memset inline.
const TargetMachine & getTargetMachine() const
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
bool isPaddedAtMostSignificantBitsWhenStored(EVT VT) const
Indicates if any padding is guaranteed to go at the most significant bits when storing the type to me...
LegalizeTypeAction
This enum indicates whether a types are legal for a target, and if not, what action should be used to...
bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize=false) const
Returns true if the specified FP immediate can be materialized without a load from the constant pool.
virtual bool hasBitTest(SDValue X, SDValue Y) const
Return true if the target has a bit-test instruction: (X & (1 << Y)) ==/!= 0 This knowledge can be us...
MVT getRegisterType(LLVMContext &Context, EVT VT) const
Return the type of registers that this ValueType will eventually require.
EVT getLegalTypeToTransformTo(LLVMContext &Context, EVT VT) const
Perform getTypeToTransformTo repeatedly until a legal type is obtained.
LegalizeAction getCondCodeAction(ISD::CondCode CC, MVT VT) const
Return how the condition code should be treated: either it is legal, needs to be expanded to some oth...
virtual bool isCommutativeBinOp(unsigned Opcode) const
Returns true if the opcode is a commutative binary operation.
virtual bool shouldTransformSignedTruncationCheck(EVT XVT, unsigned KeptBits) const
Should we tranform the IR-optimal check for whether given truncation down into KeptBits would be trun...
bool isLegalRC(const TargetRegisterInfo &TRI, const TargetRegisterClass &RC) const
Return true if the value types that can be represented by the specified register class are all legal.
virtual bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *=nullptr) const
Determine if the target supports unaligned memory accesses.
bool isOperationCustom(unsigned Op, EVT VT) const
Return true if the operation uses custom lowering, regardless of whether the type is legal or not.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const
Returns the type for the shift amount of a shift opcode.
virtual bool shouldExtendTypeInLibCall(EVT Type) const
Returns true if arguments should be extended in lib calls.
virtual bool isTruncateFree(Type *FromTy, Type *ToTy) const
Return true if it's free to truncate a value of type FromTy to type ToTy.
virtual bool shouldAvoidTransformToShift(EVT VT, unsigned Amount) const
Return true if creating a shift of the type by the given amount is not profitable.
virtual bool isFPExtFree(EVT DestVT, EVT SrcVT) const
Return true if an fpext operation is free (for instance, because single-precision floating-point numb...
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
bool isCondCodeLegal(ISD::CondCode CC, MVT VT) const
Return true if the specified condition code is legal for a comparison of the specified types on this ...
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
TargetLoweringBase(const TargetMachine &TM, const TargetSubtargetInfo &STI)
NOTE: The TargetMachine owns TLOF.
virtual bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y, unsigned OldShiftOpcode, unsigned NewShiftOpcode, SelectionDAG &DAG) const
Given the pattern (X & (C l>>/<< Y)) ==/!= 0 return true if it should be transformed into: ((X <</l>>...
BooleanContent
Enum that describes how the target represents true/false values.
virtual bool isIntDivCheap(EVT VT, AttributeList Attr) const
Return true if integer divide is usually cheaper than a sequence of several shifts,...
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
Return true if the target supports a memory access of this type for the given address space and align...
virtual bool hasAndNotCompare(SDValue Y) const
Return true if the target should transform: (X & Y) == Y ---> (~X & Y) == 0 (X & Y) !...
virtual bool isNarrowingProfitable(SDNode *N, EVT SrcVT, EVT DestVT) const
Return true if it's profitable to narrow operations of type SrcVT to DestVT.
virtual bool isBinOp(unsigned Opcode) const
Return true if the node is a math/logic binary operator.
virtual bool isCtlzFast() const
Return true if ctlz instruction is fast.
virtual bool shouldUseStrictFP_TO_INT(EVT FpVT, EVT IntVT, bool IsSigned) const
Return true if it is more correct/profitable to use strict FP_TO_INT conversion operations - canonica...
NegatibleCost
Enum that specifies when a float negation is beneficial.
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
virtual bool shouldSignExtendTypeInLibCall(Type *Ty, bool IsSigned) const
Returns true if arguments should be sign-extended in lib calls.
std::vector< ArgListEntry > ArgListTy
virtual EVT getOptimalMemOpType(LLVMContext &Context, const MemOp &Op, const AttributeList &) const
Returns the target specific optimal type for load and store operations as a result of memset,...
virtual EVT getAsmOperandValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
bool isCondCodeLegalOrCustom(ISD::CondCode CC, MVT VT) const
Return true if the specified condition code is legal or custom for a comparison of the specified type...
bool isLoadLegal(EVT ValVT, EVT MemVT, Align Alignment, unsigned AddrSpace, unsigned ExtType, bool Atomic) const
Return true if the specified load with extension is legal on this target.
virtual bool isFAbsFree(EVT VT) const
Return true if an fabs operation is free to the point where it is never worthwhile to replace it with...
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
MulExpansionKind
Enum that specifies when a multiplication should be expanded.
static ISD::NodeType getExtendForContent(BooleanContent Content)
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US][ADD|SUB]SAT.
SDValue buildSDIVPow2WithCMov(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl< SDNode * > &Created) const
Build sdiv by power-of-2 with conditional move instructions Ref: "Hacker's Delight" by Henry Warren 1...
virtual ConstraintWeight getMultipleConstraintMatchWeight(AsmOperandInfo &info, int maIndex) const
Examine constraint type and operand type and determine a weight value.
void expandVectorDeinterleave(SDNode *Node, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const
Expand a VECTOR_DEINTERLEAVE without introducing wider vector types.
bool expandMultipleResultFPLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, SDNode *Node, SmallVectorImpl< SDValue > &Results, std::optional< unsigned > CallRetResNo={}) const
Expands a node with multiple results to an FP or vector libcall.
bool expandMULO(SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]MULO.
bool expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL into two nodes.
SmallVector< ConstraintPair > ConstraintGroup
virtual const MCExpr * getPICJumpTableRelocBaseExpr(const MachineFunction *MF, unsigned JTI, MCContext &Ctx) const
This returns the relocation base for the given PIC jumptable, the same as getPICJumpTableRelocBase,...
virtual Align computeKnownAlignForTargetInstr(GISelValueTracking &Analysis, Register R, const MachineRegisterInfo &MRI, unsigned Depth=0) const
Determine the known alignment for the pointer value R.
bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedEltMask, APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Vector Op.
virtual bool isUsedByReturnOnly(SDNode *, SDValue &) const
Return true if result of the specified node is used by a return node only.
bool LegalizeSetCCCondCode(SelectionDAG &DAG, EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC, bool &NeedInvert, const SDLoc &dl, SDValue &Chain, bool IsSignaling=false) const
Legalize a SETCC with given LHS and RHS and condition code CC on the current target.
SDValue scalarizeVectorStore(StoreSDNode *ST, SelectionDAG &DAG) const
virtual unsigned getPreferredShrunkVectorSizeInBits(SDValue Op, const APInt &DemandedElts) const
If only low elements of a vector are demanded, shrink the operation to the returned size in bits by c...
virtual unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const
This method can be implemented by targets that want to expose additional information about sign bits ...
SDValue lowerCmpEqZeroToCtlzSrl(SDValue Op, SelectionDAG &DAG) const
void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &DL, const SDValue OldLHS, const SDValue OldRHS) const
Soften the operands of a comparison.
void forceExpandWideMUL(SelectionDAG &DAG, const SDLoc &dl, bool Signed, const SDValue LHS, const SDValue RHS, SDValue &Lo, SDValue &Hi) const
Calculate full product of LHS and RHS either via a libcall or through brute force expansion of the mu...
SDValue expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_SEQ_* into an explicit ordered calculation.
SDValue expandFCANONICALIZE(SDNode *Node, SelectionDAG &DAG) const
Expand FCANONICALIZE to FMUL with 1.
SDValue expandCTLZ(SDNode *N, SelectionDAG &DAG) const
Expand CTLZ/CTLZ_ZERO_POISON nodes.
SDValue expandBITREVERSE(SDNode *N, SelectionDAG &DAG) const
Expand BITREVERSE nodes.
SDValue expandCTTZ(SDNode *N, SelectionDAG &DAG) const
Expand CTTZ/CTTZ_ZERO_POISON nodes.
virtual SDValue expandIndirectJTBranch(const SDLoc &dl, SDValue Value, SDValue Addr, int JTI, SelectionDAG &DAG) const
Expands target specific indirect branch for the case of JumpTable expansion.
SDValue expandABD(SDNode *N, SelectionDAG &DAG) const
Expand ABDS/ABDU nodes.
virtual bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const
std::vector< AsmOperandInfo > AsmOperandInfoVector
SDValue expandGetActiveLaneMask(SDNode *N, SelectionDAG &DAG) const
Expand GET_ACTIVE_LANE_MASK nodes.
SDValue expandCLMUL(SDNode *N, SelectionDAG &DAG) const
Expand carryless multiply.
SDValue expandShlSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]SHLSAT.
SDValue expandIS_FPCLASS(EVT ResultVT, SDValue Op, FPClassTest Test, SDNodeFlags Flags, const SDLoc &DL, SelectionDAG &DAG) const
Expand check for floating point class.
virtual bool isTargetCanonicalConstantNode(SDValue Op) const
Returns true if the given Opc is considered a canonical constant for the target, which should not be ...
SDValue expandVECTOR_SHUFFLE_VAR(SDNode *Node, SelectionDAG &DAG) const
Expand a VECTOR_SHUFFLE_VAR into variable-index element extracts.
SDValue expandFP_TO_INT_SAT(SDNode *N, SelectionDAG &DAG) const
Expand FP_TO_[US]INT_SAT into FP_TO_[US]INT and selects or min/max.
SDValue expandCttzElts(SDNode *Node, SelectionDAG &DAG) const
Expand a CTTZ_ELTS or CTTZ_ELTS_ZERO_POISON by calculating (VL - i) for each active lane (i),...
SDValue getCheaperNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOps, bool OptForSize, unsigned Depth=0) const
This is the helper function to return the newly negated expression only when the cost is cheaper.
virtual unsigned computeNumSignBitsForTargetInstr(GISelValueTracking &Analysis, Register R, const APInt &DemandedElts, const MachineRegisterInfo &MRI, unsigned Depth=0) const
This method can be implemented by targets that want to expose additional information about sign bits ...
SDValue SimplifyMultipleUseDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, SelectionDAG &DAG, unsigned Depth=0) const
More limited version of SimplifyDemandedBits that can be used to "lookthrough" ops that don't contrib...
SDValue expandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG) const
Expands an unaligned store to 2 half-size stores for integer values, and possibly more for vectors.
SDValue SimplifyMultipleUseDemandedVectorElts(SDValue Op, const APInt &DemandedElts, SelectionDAG &DAG, unsigned Depth=0) const
Helper wrapper around SimplifyMultipleUseDemandedBits, demanding all bits from only some vector eleme...
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
virtual bool findOptimalMemOpLowering(LLVMContext &Context, std::vector< EVT > &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS, unsigned SrcAS, const AttributeList &FuncAttributes, EVT *LargestVT=nullptr) const
Determines the optimal series of memory ops to replace the memset / memcpy.
virtual SDValue unwrapAddress(SDValue N) const
void expandSADDSUBO(SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::S(ADD|SUB)O.
SDValue expandABS(SDNode *N, SelectionDAG &DAG, bool IsNegative=false) const
Expand ABS nodes.
SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_* into an explicit calculation.
bool ShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const
Check to see if the specified operand of the specified instruction is a constant integer.
virtual bool isGuaranteedNotToBeUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, UndefPoisonKind Kind, unsigned Depth) const
Return true if this function can prove that Op is never poison and, Kind can be used to track poison ...
SDValue expandMULH(SDNode *Node, SelectionDAG &DAG) const
SDValue expandVPCTTZElements(SDNode *N, SelectionDAG &DAG) const
Expand VP_CTTZ_ELTS/VP_CTTZ_ELTS_ZERO_POISON nodes.
SDValue BuildSDIV(SDNode *N, SelectionDAG &DAG, bool IsAfterLegalization, bool IsAfterLegalTypes, SmallVectorImpl< SDNode * > &Created) const
Given an ISD::SDIV node expressing a divide by constant, return a DAG expression to select that will ...
virtual const char * getTargetNodeName(unsigned Opcode) const
This method returns the name of a target specific DAG node.
bool expandFP_TO_UINT(SDNode *N, SDValue &Result, SDValue &Chain, SelectionDAG &DAG) const
Expand float to UINT conversion.
bool parametersInCSRMatch(const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask, const SmallVectorImpl< CCValAssign > &ArgLocs, const SmallVectorImpl< SDValue > &OutVals) const
Check whether parameters to a call that are passed in callee saved registers are the same as from the...
virtual bool SimplifyDemandedVectorEltsForTargetNode(SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth=0) const
Attempt to simplify any target nodes based on the demanded vector elements, returning true on success...
bool expandREM(SDNode *Node, SDValue &Result, SelectionDAG &DAG) const
Expand an SREM or UREM using SDIV/UDIV or SDIVREM/UDIVREM, if legal.
std::pair< SDValue, SDValue > expandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG) const
Expands an unaligned load to 2 half-size loads for an integer, and possibly more for vectors.
SDValue expandFMINIMUMNUM_FMAXIMUMNUM(SDNode *N, SelectionDAG &DAG) const
Expand fminimumnum/fmaximumnum into multiple comparison with selects.
void expandVectorInterleave(SDNode *Node, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const
Expand a VECTOR_INTERLEAVE without introducing wider vector types.
void forceExpandMultiply(SelectionDAG &DAG, const SDLoc &dl, bool Signed, SDValue &Lo, SDValue &Hi, SDValue LHS, SDValue RHS, SDValue HiLHS=SDValue(), SDValue HiRHS=SDValue()) const
Calculate the product twice the width of LHS and RHS.
virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, SelectionDAG &DAG) const
Lower TLS global address SDNode for target independent emulated TLS model.
virtual bool isTypeDesirableForOp(unsigned, EVT VT) const
Return true if the target has native support for the specified value type and it is 'desirable' to us...
SDValue expandVectorSplice(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::VECTOR_SPLICE.
SDValue getVectorSubVecPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT, EVT SubVecVT, SDValue Index, const SDNodeFlags PtrArithFlags=SDNodeFlags()) const
Get a pointer to a sub-vector of type SubVecVT at index Idx located in memory for a vector of type Ve...
SDValue expandLoopDependenceMask(SDNode *N, SelectionDAG &DAG) const
Expand LOOP_DEPENDENCE_MASK nodes.
virtual const char * LowerXConstraint(EVT ConstraintVT) const
Try to replace an X constraint, which matches anything, with another that has more specific requireme...
SDValue expandCTPOP(SDNode *N, SelectionDAG &DAG) const
Expand CTPOP nodes.
virtual void computeKnownBitsForTargetInstr(GISelValueTracking &Analysis, Register R, KnownBits &Known, const APInt &DemandedElts, const MachineRegisterInfo &MRI, unsigned Depth=0) const
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
SDValue BuildUDIV(SDNode *N, SelectionDAG &DAG, bool IsAfterLegalization, bool IsAfterLegalTypes, SmallVectorImpl< SDNode * > &Created) const
Given an ISD::UDIV node expressing a divide by constant, return a DAG expression to select that will ...
SDValue expandVectorNaryOpBySplitting(SDNode *Node, SelectionDAG &DAG) const
~TargetLowering() override
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
SDValue expandBSWAP(SDNode *N, SelectionDAG &DAG) const
Expand BSWAP nodes.
SDValue expandFMINIMUM_FMAXIMUM(SDNode *N, SelectionDAG &DAG) const
Expand fminimum/fmaximum into multiple comparison with selects.
SDValue CTTZTableLookup(SDNode *N, SelectionDAG &DAG, const SDLoc &DL, EVT VT, SDValue Op, unsigned NumBitsPerElt) const
Expand CTTZ via Table Lookup.
bool expandDIVREMByConstant(SDNode *N, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, SDValue LL=SDValue(), SDValue LH=SDValue()) const
Attempt to expand an n-bit div/rem/divrem by constant using an n/2-bit algorithm.
virtual void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
bool isPositionIndependent() const
std::pair< StringRef, TargetLowering::ConstraintType > ConstraintPair
virtual SDValue getNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOps, bool OptForSize, NegatibleCost &Cost, unsigned Depth=0) const
Return the newly negated expression if the cost is not expensive and set the cost in Cost to indicate...
virtual ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &info, const char *constraint) const
Examine constraint string and operand type and determine a weight value.
ConstraintGroup getConstraintPreferences(AsmOperandInfo &OpInfo) const
Given an OpInfo with list of constraints codes as strings, return a sorted Vector of pairs of constra...
bool expandFP_TO_SINT(SDNode *N, SDValue &Result, SelectionDAG &DAG) const
Expand float(f32) to SINT(i64) conversion.
virtual SDValue SimplifyMultipleUseDemandedBitsForTargetNode(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, SelectionDAG &DAG, unsigned Depth) const
More limited version of SimplifyDemandedBits that can be used to "lookthrough" ops that don't contrib...
virtual SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Glue, const SDLoc &DL, const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const
SDValue buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0, SDValue N1, MutableArrayRef< int > Mask, SelectionDAG &DAG) const
Tries to build a legal vector shuffle using the provided parameters or equivalent variations.
static ArgListTy getArgListForFunctionType(FunctionType *FuncTy, const AttributeList &FuncAttrs, ArrayRef< SDValue > Ops)
Build a call argument list for FuncTy, taking the argument node values from Ops and the parameter typ...
virtual void computeKnownBitsForStackObjectPointer(KnownBits &Known, const MachineFunction &MF, Align Alignment) const
Determine known bits of a pointer to a known valid stack object.
virtual SDValue getPICJumpTableRelocBase(SDValue Table, SelectionDAG &DAG) const
Returns relocation base for the given PIC jumptable.
std::pair< SDValue, SDValue > scalarizeVectorLoad(LoadSDNode *LD, SelectionDAG &DAG) const
Turn load of vector type into a load of the individual elements.
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
virtual bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0) const
Attempt to simplify any target nodes based on the demanded bits/elts, returning true on success.
virtual bool isDesirableToCommuteXorWithShift(const SDNode *N) const
Return true if it is profitable to combine an XOR of a logical shift to create a logical shift of NOT...
TargetLowering(const TargetLowering &)=delete
virtual bool shouldSimplifyDemandedVectorElts(SDValue Op, const TargetLoweringOpt &TLO) const
Return true if the target supports simplifying demanded vector elements by converting them to undefs.
bool isConstFalseVal(SDValue N) const
Return if the N is a constant or constant vector equal to the false value from getBooleanContents().
SDValue IncrementMemoryAddress(SDValue Addr, SDValue Mask, const SDLoc &DL, EVT DataVT, SelectionDAG &DAG, bool IsCompressedMemory) const
Increments memory address Addr according to the type of the value DataVT that should be stored.
SDValue expandVectorMatch(SDNode *N, SelectionDAG &DAG) const
Expand VECTOR_MATCH nodes.
bool isInTailCallPosition(SelectionDAG &DAG, SDNode *Node, SDValue &Chain) const
Check whether a given call node is in tail position within its function.
SDValue expandCONVERT_TO_ARBITRARY_FP(SDNode *Node, SelectionDAG &DAG) const
Expand CONVERT_TO_ARBITRARY_FP using bit manipulation.
virtual AsmOperandInfoVector ParseConstraints(const DataLayout &DL, const TargetRegisterInfo *TRI, const CallBase &Call) const
Split up the constraint string from the inline assembly value into the specific constraints and their...
virtual bool isSplatValueForTargetNode(SDValue Op, const APInt &DemandedElts, APInt &UndefElts, const SelectionDAG &DAG, unsigned Depth=0) const
Return true if vector Op has the same value across all DemandedElts, indicating any elements which ma...
SDValue expandRoundInexactToOdd(EVT ResultVT, SDValue Op, const SDLoc &DL, SelectionDAG &DAG) const
Truncate Op to ResultVT.
virtual bool shouldSplitFunctionArgumentsAsLittleEndian(const DataLayout &DL) const
For most targets, an LLVM type must be broken down into multiple smaller types.
SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, bool foldBooleans, DAGCombinerInfo &DCI, const SDLoc &dl) const
Try to simplify a setcc built with the specified operands and cc.
SDValue expandFunnelShift(SDNode *N, SelectionDAG &DAG) const
Expand funnel shift.
virtual bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const
Return true if folding a constant offset with the given GlobalAddress is legal.
bool isExtendedTrueVal(const ConstantSDNode *N, EVT VT, bool SExt) const
Return if N is a True value when extended to VT.
bool ShrinkDemandedOp(SDValue Op, unsigned BitWidth, const APInt &DemandedBits, TargetLoweringOpt &TLO) const
Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.
bool isConstTrueVal(SDValue N) const
Return if the N is a constant or constant vector equal to the true value from getBooleanContents().
SDValue expandFixedPointDiv(unsigned Opcode, const SDLoc &dl, SDValue LHS, SDValue RHS, unsigned Scale, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]DIVFIX[SAT].
SDValue expandPEXT(SDNode *N, SelectionDAG &DAG) const
Expand parallel bit extract (compress).
virtual void ComputeConstraintToUse(AsmOperandInfo &OpInfo, SDValue Op, SelectionDAG *DAG=nullptr) const
Determines the constraint code and constraint type to use for the specific AsmOperandInfo,...
virtual void CollectTargetIntrinsicOperands(const CallInst &I, SmallVectorImpl< SDValue > &Ops, SelectionDAG &DAG) const
virtual bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, UndefPoisonKind Kind, bool ConsiderFlags, unsigned Depth) const
Return true if Op can create undef or poison from non-undef & non-poison operands.
bool expandMODF(SDNode *N, SelectionDAG &DAG, SmallVectorImpl< SDValue > &Results) const
Expand MODF node into fractional and integer parts.
SDValue expandVECTOR_COMPRESS(SDNode *Node, SelectionDAG &DAG) const
Expand a vector VECTOR_COMPRESS into a sequence of extract element, store temporarily,...
virtual const Constant * getTargetConstantFromLoad(LoadSDNode *LD) const
This method returns the constant pool value that will be loaded by LD.
SDValue expandFP_ROUND(SDNode *Node, SelectionDAG &DAG) const
Expand round(fp) to fp conversion.
SDValue createSelectForFMINNUM_FMAXNUM(SDNode *Node, SelectionDAG &DAG) const
Try to convert the fminnum/fmaxnum to a compare/select sequence.
SDValue expandCONVERT_FROM_ARBITRARY_FP(SDNode *Node, SelectionDAG &DAG) const
Expand CONVERT_FROM_ARBITRARY_FP using bit manipulation.
SDValue expandROT(SDNode *N, bool AllowVectorOps, SelectionDAG &DAG) const
Expand rotations.
SDValue annotateStackObjectPointer(SDValue Ptr, SelectionDAG &DAG, const SDLoc &DL, Align Alignment) const
Annotate a stack object pointer with known-bits assertions.
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
virtual SDValue getSqrtInputTest(SDValue Operand, SelectionDAG &DAG, const DenormalMode &Mode, SDNodeFlags Flags={}) const
Return a target-dependent comparison result if the input operand is suitable for use with a square ro...
SDValue getVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT, SDValue Index, const SDNodeFlags PtrArithFlags=SDNodeFlags()) const
Get a pointer to vector element Idx located in memory for a vector of type VecVT starting at a base a...
SDValue expandFMINNUM_FMAXNUM(SDNode *N, SelectionDAG &DAG) const
Expand fminnum/fmaxnum into fminnum_ieee/fmaxnum_ieee with quieted inputs.
virtual bool isGAPlusOffset(SDNode *N, const GlobalValue *&GA, int64_t &Offset) const
Returns true (and the GlobalValue and the offset) if the node is a GlobalAddress + offset.
virtual void computeKnownFPClassForTargetNode(const SDValue Op, KnownFPClass &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const
Determine floating-point class information for a target node.
virtual unsigned getJumpTableEncoding() const
Return the entry encoding for a jump table in the current function.
virtual void computeKnownFPClassForTargetInstr(GISelValueTracking &Analysis, Register R, KnownFPClass &Known, const APInt &DemandedElts, const MachineRegisterInfo &MRI, unsigned Depth=0) const
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::LibcallImpl LibcallImpl, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
SDValue expandCMP(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]CMP.
void expandShiftParts(SDNode *N, SDValue &Lo, SDValue &Hi, SelectionDAG &DAG) const
Expand shift-by-parts.
virtual bool isKnownNeverNaNForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool SNaN=false, unsigned Depth=0) const
If SNaN is false,.
virtual SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const
This method will be invoked for all target nodes and for any target-independent nodes that the target...
SDValue expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[U|S]MULFIX[SAT].
SDValue getInboundsVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT, SDValue Index) const
Get a pointer to vector element Idx located in memory for a vector of type VecVT starting at a base a...
SDValue expandIntMINMAX(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US][MIN|MAX].
SDValue expandVectorFindLastActive(SDNode *N, SelectionDAG &DAG) const
Expand VECTOR_FIND_LAST_ACTIVE nodes.
SDValue expandPartialReduceMLA(SDNode *Node, SelectionDAG &DAG) const
Expands PARTIAL_REDUCE_S/UMLA nodes to a series of simpler operations, consisting of zext/sext,...
void expandUADDSUBO(SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::U(ADD|SUB)O.
SDValue expandPDEP(SDNode *N, SelectionDAG &DAG) const
Expand parallel bit deposit (expand).
virtual SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl< SDNode * > &Created) const
Targets may override this function to provide custom SDIV lowering for power-of-2 denominators.
SDValue scalarizeExtractedVectorLoad(EVT ResultVT, const SDLoc &DL, EVT InVecVT, SDValue EltNo, LoadSDNode *OriginalLoad, SelectionDAG &DAG) const
Replace an extraction of a load with a narrowed load.
virtual SDValue BuildSREMPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl< SDNode * > &Created) const
Targets may override this function to provide custom SREM lowering for power-of-2 denominators.
bool expandUINT_TO_FP(SDNode *N, SDValue &Result, SDValue &Chain, SelectionDAG &DAG) const
Expand UINT(i64) to double(f64) conversion.
bool expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl, SDValue LHS, SDValue RHS, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL or [US]MUL_LOHI of n-bit values into two or four nodes, respectively,...
SDValue expandAVG(SDNode *N, SelectionDAG &DAG) const
Expand vector/scalar AVGCEILS/AVGCEILU/AVGFLOORS/AVGFLOORU nodes.
SDValue expandCTLS(SDNode *N, SelectionDAG &DAG) const
Expand CTLS (count leading sign bits) nodes.
void setTypeIdForCallsiteInfo(const CallBase *CB, MachineFunction &MF, MachineFunction::CallSiteInfo &CSInfo) const
Primary interface to the complete machine description for the target machine.
bool isPositionIndependent() const
const Triple & getTargetTriple() const
TargetOptions Options
unsigned EmitCallSiteInfo
The flag enables call site info production.
unsigned EmitCallGraphSection
Emit section containing call graph metadata.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual StringRef getRegAsmName(MCRegister Reg) const
Return the assembly name for Reg.
bool isTypeLegalForClass(const TargetRegisterClass &RC, MVT T) const
Return true if the given TargetRegisterClass has the ValueType T.
TargetSubtargetInfo - Generic base class for all target subtargets.
bool isOSBinFormatCOFF() const
Tests whether the OS uses the COFF binary format.
Definition Triple.h:872
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition TypeSize.h:352
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
Definition Type.cpp:294
static LLVM_ABI Type * getVoidTy(LLVMContext &C)
Definition Type.cpp:267
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:370
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:252
LLVM_ABI const fltSemantics & getFltSemantics() const
Definition Type.cpp:96
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
LLVM_ABI const Value * stripPointerCastsAndAliases() const
Strip off pointer casts, all-zero GEPs, address space casts, and aliases.
Definition Value.cpp:711
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:314
constexpr bool isKnownMultipleOf(ScalarTy RHS) const
This function tells the caller whether the element count is known at compile time to be a multiple of...
Definition TypeSize.h:181
constexpr bool hasKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns true if there exists a value X where RHS*X will result in a value whose quantity matches our ...
Definition TypeSize.h:266
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:201
static constexpr bool isKnownLE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition TypeSize.h:231
constexpr ScalarTy getKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns a value X where RHS*X will result in a value whose quantity matches our own.
Definition TypeSize.h:274
static constexpr bool isKnownLT(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition TypeSize.h:217
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition TypeSize.h:169
CallInst * Call
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
LLVM_ABI APInt ScaleBitMask(const APInt &A, unsigned NewBitWidth, bool MatchAllBits=false)
Splat/Merge neighboring bits to widen/narrow the bitmask represented by.
Definition APInt.cpp:3052
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ C
The default llvm calling convention, compatible with C.
Definition CallingConv.h:34
LLVM_ABI bool isConstantSplatVector(SDValue V, APInt &SplatValue)
Node predicates.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition ISDOpcodes.h:43
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition ISDOpcodes.h:851
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition ISDOpcodes.h:263
@ PTRADD
PTRADD represents pointer arithmetic semantics, for targets that opt in using shouldPreservePtrArith(...
@ PARTIAL_REDUCE_SMLA
PARTIAL_REDUCE_[U|S]MLA(Accumulator, Input1, Input2) The partial reduction nodes sign or zero extend ...
@ LOOP_DEPENDENCE_RAW_MASK
@ FGETSIGN
INT = FGETSIGN(FP) - Return the sign bit of the specified floating point value as an integer 0/1 valu...
Definition ISDOpcodes.h:543
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition ISDOpcodes.h:277
@ AVGSUBFLOORS
AVGSUBFLOORS/AVGSUBFLOORU - Averaging subtract - Subtract two integers using an integer of type i[N+1...
Definition ISDOpcodes.h:739
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition ISDOpcodes.h:605
@ BSWAP
Byte Swap and Counting operators.
Definition ISDOpcodes.h:811
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
Definition ISDOpcodes.h:397
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
Definition ISDOpcodes.h:296
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
Definition ISDOpcodes.h:527
@ ADD
Simple integer binary arithmetic operators.
Definition ISDOpcodes.h:266
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition ISDOpcodes.h:403
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition ISDOpcodes.h:885
@ CTTZ_ELTS
Returns the number of number of trailing (least significant) zero elements in a vector.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition ISDOpcodes.h:523
@ VECTOR_FIND_LAST_ACTIVE
Finds the index of the last active mask element Operands: Mask.
@ PSEUDO_FMIN
PSEUDO_FMIN is strictly equivalent to op0 olt op1 ?
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition ISDOpcodes.h:222
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition ISDOpcodes.h:912
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition ISDOpcodes.h:589
@ FADD
Simple binary floating point operators.
Definition ISDOpcodes.h:420
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition ISDOpcodes.h:771
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition ISDOpcodes.h:942
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition ISDOpcodes.h:282
@ FMULADD
FMULADD - Performs a * b + c, with, or without, intermediate rounding.
Definition ISDOpcodes.h:533
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition ISDOpcodes.h:256
@ CLMUL
Carry-less multiplication operations.
Definition ISDOpcodes.h:802
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
Definition ISDOpcodes.h:410
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ CTLZ_ZERO_POISON
Definition ISDOpcodes.h:820
@ PARTIAL_REDUCE_UMLA
@ SIGN_EXTEND
Conversion operators.
Definition ISDOpcodes.h:876
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
Definition ISDOpcodes.h:731
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
Definition ISDOpcodes.h:681
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ PARTIAL_REDUCE_FMLA
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ SSUBO
Same for subtraction.
Definition ISDOpcodes.h:355
@ BRIND
BRIND - Indirect branch.
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2, ...) - Returns N vectors from N input vectors, where N is the factor to...
Definition ISDOpcodes.h:640
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition ISDOpcodes.h:546
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
Definition ISDOpcodes.h:553
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition ISDOpcodes.h:377
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition ISDOpcodes.h:828
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
Definition ISDOpcodes.h:249
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition ISDOpcodes.h:688
@ GET_ACTIVE_LANE_MASK
GET_ACTIVE_LANE_MASK - this corrosponds to the llvm.get.active.lane.mask intrinsic.
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition ISDOpcodes.h:232
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition ISDOpcodes.h:351
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition ISDOpcodes.h:720
@ SHL
Shift and rotation operations.
Definition ISDOpcodes.h:793
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition ISDOpcodes.h:659
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition ISDOpcodes.h:619
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Same as FMINNUM/FMAXNUM, except that a signaling NaN operand deterministi...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition ISDOpcodes.h:581
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition ISDOpcodes.h:882
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition ISDOpcodes.h:843
@ FMINNUM
FMINNUM/FMAXNUM - Same semantics as the llvm.minnum/maxnum intrinsics.
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
Definition ISDOpcodes.h:389
@ SMULO
Same for multiplication.
Definition ISDOpcodes.h:359
@ VECTOR_SPLICE_LEFT
VECTOR_SPLICE_LEFT(VEC1, VEC2, OFFSET) - Shifts CONCAT_VECTORS(VEC1, VEC2) left by OFFSET elements an...
Definition ISDOpcodes.h:669
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition ISDOpcodes.h:931
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition ISDOpcodes.h:920
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition ISDOpcodes.h:751
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition ISDOpcodes.h:416
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition ISDOpcodes.h:837
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
Definition ISDOpcodes.h:331
@ PEXT
Parallel bit extract (compress) and parallel bit deposit (expand).
Definition ISDOpcodes.h:807
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition ISDOpcodes.h:482
@ FMINIMUM
FMINIMUM/FMAXIMUM - Same semantics as the llvm.minimum/maximum intrinsics.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition ISDOpcodes.h:958
@ TargetConstant
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification,...
Definition ISDOpcodes.h:181
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition ISDOpcodes.h:763
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition ISDOpcodes.h:207
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
Definition ISDOpcodes.h:726
@ VECTOR_SPLICE_RIGHT
VECTOR_SPLICE_RIGHT(VEC1, VEC2, OFFSET) - Shifts CONCAT_VECTORS(VEC1,VEC2) right by OFFSET elements a...
Definition ISDOpcodes.h:673
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
Definition ISDOpcodes.h:306
@ FREEZE
FREEZE - FREEZE(VAL) returns an arbitrary value if VAL is UNDEF (or is evaluated to UNDEF),...
Definition ISDOpcodes.h:243
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition ISDOpcodes.h:570
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition ISDOpcodes.h:55
@ CTTZ_ZERO_POISON
Bit counting operators with a poisoned result for zero inputs.
Definition ISDOpcodes.h:819
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition ISDOpcodes.h:991
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition ISDOpcodes.h:953
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition ISDOpcodes.h:977
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition ISDOpcodes.h:888
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
Definition ISDOpcodes.h:865
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition ISDOpcodes.h:539
@ PARTIAL_REDUCE_SUMLA
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition ISDOpcodes.h:368
@ CALLSEQ_START
CALLSEQ_START/CALLSEQ_END - These operators mark the beginning and end of a call sequence,...
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2, ...) - Returns N vectors from N input vectors, where N is the factor ...
Definition ISDOpcodes.h:629
@ CTTZ_ELTS_ZERO_POISON
@ FMINIMUMNUM
FMINIMUMNUM/FMAXIMUMNUM - Same semantics as the llvm.minimumnum/maximumnum intrinsics.
@ ABDS
ABDS/ABDU - Absolute difference - Return the absolute difference between two numbers interpreted as s...
Definition ISDOpcodes.h:746
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition ISDOpcodes.h:215
@ ABS_MIN_POISON
ABS with a poison result for INT_MIN.
Definition ISDOpcodes.h:775
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition ISDOpcodes.h:561
LLVM_ABI NodeType getOppositeSignednessMinMaxOpcode(unsigned MinMaxOpc)
Given a MinMaxOpc of ISD::(U|S)MIN or ISD::(U|S)MAX, returns the corresponding opcode with the opposi...
LLVM_ABI NodeType getExtForLoadExtType(bool IsFP, LoadExtType)
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isZEXTLoad(const SDNode *N)
Returns true if the specified node is a ZEXTLOAD.
LLVM_ABI CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
bool isTrueWhenEqual(CondCode Cond)
Return true if the specified condition returns true if the two operands to the condition are equal.
unsigned getUnorderedFlavor(CondCode Cond)
This function returns 0 if the condition is always false if an operand is a NaN, 1 if the condition i...
LLVM_ABI bool matchBinaryPredicate(SDValue LHS, SDValue RHS, const APInt &DemandedElts, std::function< bool(ConstantSDNode *, ConstantSDNode *)> Match, bool AllowUndefs=false, bool AllowTypeMismatch=false)
Attempt to match a binary predicate against a pair of scalar/splat constants or every element of a pa...
LLVM_ABI CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
LLVM_ABI StringRef getCondCodeName(CondCode Operation)
Return the name of the given condition code, e.g. "setoeq".
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
LLVM_ABI bool isBuildVectorAllZeros(SDValue V)
Return true if the specified value is a BUILD_VECTOR where all of the elements are 0 or undef.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LLVM_ABI bool isBuildVectorOfConstantSDNodes(SDValue V)
Return true if the specified value is a BUILD_VECTOR node of all ConstantSDNode or undef.
LLVM_ABI NodeType getVecReduceBaseOpcode(unsigned VecReduceOpcode)
Get underlying scalar opcode for VECREDUCE opcode.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isUnsignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs an unsigned comparison when used with intege...
bool matchUnaryPredicate(SDValue Op, const APInt &DemandedElts, std::function< bool(ConstantSDNode *)> Match, bool AllowUndefs=false, bool AllowTruncation=false)
Hook for matching ConstantSDNode predicate.
BinaryOp_match< LHS, RHS, Instruction::And > m_And(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
match_deferred< Value > m_Deferred(Value *const &V)
Like m_Specific(), but works if the specific value to match is determined as part of the same match()...
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
auto m_Value()
Match an arbitrary value and ignore it.
BinaryOp_match< LHS, RHS, Instruction::Or > m_Or(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::Sub > m_Sub(const LHS &L, const RHS &R)
LLVM_ABI Libcall getUREM(EVT VT)
Or< Preds... > m_AnyOf(const Preds &...preds)
TypeSize_match< ElementCount > m_ElementCount(ElementCount &V)
Match any integer constant or constant multiple of VSCALE that can be represented using ElementCount.
bool sd_match(SDValue N, Pattern &&P)
NUses_match< 1, Value_match > m_OneUse()
This is an optimization pass for GlobalISel generic memory operations.
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition MathExtras.h:339
@ Offset
Definition DWP.cpp:577
void stable_sort(R &&Range)
Definition STLExtras.h:2132
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
constexpr uint64_t maxUIntN(uint64_t N)
Gets the maximum value for a N-bit unsigned integer.
Definition MathExtras.h:208
InstructionCost Cost
LLVM_ABI bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
RelativeUniformCounterPtr Values
Definition InstrProf.h:91
LLVM_ABI bool isAllOnesOrAllOnesSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant -1 integer or a splatted vector of a constant -1 integer (with...
Definition Utils.cpp:1717
@ Known
Known to have no common set bits.
@ Undef
Value of the register doesn't matter.
LLVM_ABI SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
Definition STLExtras.h:2570
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
LLVM_ABI FPClassTest invertFPClassTestIfSimpler(FPClassTest Test, bool UseFCmp)
Evaluates if the specified FP class test is better performed as the inverse (i.e.
@ Load
The value being inserted comes from a load (InsertElement only).
@ Store
The extracted value is stored (ExtractElement only).
LLVM_ABI bool isOneOrOneSplatFP(SDValue V, bool AllowUndefs=false)
Return true if the value is a constant floating-point value, or a splatted vector of a constant float...
constexpr T alignDown(U Value, V Align, W Skew=0)
Returns the largest unsigned integer less than or equal to Value and is Skew mod Align.
Definition MathExtras.h:546
void * PointerTy
LLVM_ABI bool isNullOrNullSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
Definition Utils.cpp:1699
T bit_ceil(T Value)
Returns the smallest integral power of two no smaller than Value if Value is nonzero.
Definition bit.h:362
LLVM_ABI void reportFatalInternalError(Error Err)
Report a fatal error that indicates a bug in LLVM.
Definition Error.cpp:173
constexpr int popcount(T Value) noexcept
Count the number of set bits in a value.
Definition bit.h:156
LLVM_ABI ConstantFPSDNode * isConstOrConstSplatFP(SDValue N, bool AllowUndefs=false)
Returns the SDNode if it is a constant splat BuildVector or constant float.
constexpr bool has_single_bit(T Value) noexcept
Definition bit.h:149
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1762
LLVM_ABI bool getShuffleDemandedElts(int SrcWidth, ArrayRef< int > Mask, const APInt &DemandedElts, APInt &DemandedLHS, APInt &DemandedRHS, bool AllowUndefElts=false)
Transform a shuffle mask's output demanded element mask into demanded element masks for the 2 operand...
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition MathExtras.h:326
LLVM_ABI bool isBitwiseNot(SDValue V, bool AllowUndefs=false)
Returns true if V is a bitwise not operation.
LLVM_ABI llvm::SmallVector< int, 16 > createStrideMask(unsigned Start, unsigned Stride, unsigned VF)
Create a stride shuffle mask.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
auto find_if_not(R &&Range, UnaryPredicate P)
Definition STLExtras.h:1793
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
constexpr uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition Alignment.h:149
LLVM_ABI ConstantRange getVScaleRange(const Function *F, unsigned BitWidth)
Determine the possible constant range of vscale with the given bit width, based on the vscale_range f...
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
LLVM_ABI bool isOneOrOneSplat(SDValue V, bool AllowUndefs=false)
Return true if the value is a constant 1 integer or a splatted vector of a constant 1 integer (with n...
@ Mod
The access may modify the value stored in memory.
Definition ModRef.h:34
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
Definition MathExtras.h:389
@ Other
Any other memory.
Definition ModRef.h:68
To bit_cast(const From &from) noexcept
Definition bit.h:90
LLVM_ABI llvm::SmallVector< int, 16 > createInterleaveMask(unsigned VF, unsigned NumVecs)
Create an interleave shuffle mask.
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
@ Sub
Subtraction of integers.
@ Add
Sum of integers.
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
Definition MCRegister.h:21
@ Fast
Assign the register banks as fast as possible (default).
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Count
Definition InstrProf.h:145
fltNonfiniteBehavior
Definition APFloat.h:977
DWARFExpression::Operation Op
RoundingMode
Rounding mode.
@ TowardZero
roundTowardZero.
@ NearestTiesToEven
roundTiesToEven.
@ TowardPositive
roundTowardPositive.
@ NearestTiesToAway
roundTiesToAway.
@ TowardNegative
roundTowardNegative.
ArrayRef(const T &OneElt) -> ArrayRef< T >
LLVM_ABI ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
constexpr unsigned BitWidth
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
LLVM_ABI bool isZeroOrZeroSplat(SDValue N, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
LLVM_ABI bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
UndefPoisonKind
Enumeration to track whether we are interested in Undef, Poison, or both.
Definition UndefPoison.h:20
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition Alignment.h:206
LLVM_ABI bool isNullFPConstant(SDValue V)
Returns true if V is an FP constant with a value of positive zero.
APFloat neg(APFloat X)
Returns the negated value of the argument.
Definition APFloat.h:1742
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Next
Definition InstrProf.h:147
unsigned Log2(Align A)
Returns the log2 of the alignment.
Definition Alignment.h:202
constexpr T maskTrailingOnes(unsigned N)
Create a bitmask with the N right-most bits set to 1, and all other bits set to 0.
Definition MathExtras.h:78
@ Increment
Incrementally increasing token ID.
Definition AllocToken.h:26
@ End
Run the machine verifier once, at the end of the pipeline.
Definition CodeGen.h:320
fltNanEncoding
Definition APFloat.h:1001
@ Keep
No function return thunk.
Definition CodeGen.h:370
LLVM_ABI bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
constexpr uint64_t NextPowerOf2(uint64_t A)
Returns the next power of two (in 64-bits) that is strictly greater than A.
Definition MathExtras.h:368
MCRegisterClass TargetRegisterClass
Definition FastISel.h:58
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
Definition Error.cpp:177
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:881
#define N
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
Represent subnormal handling kind for floating point instruction inputs and outputs.
DenormalModeKind Input
Denormal treatment kind for floating point instruction inputs in the default floating-point environme...
@ PreserveSign
The sign of a flushed-to-zero number is preserved in the sign of 0.
@ PositiveZero
Denormals are flushed to positive zero.
@ IEEE
IEEE-754 denormal numbers preserved.
constexpr bool inputsAreZero() const
Return true if input denormals must be implicitly treated as 0.
Extended Value Type.
Definition ValueTypes.h:35
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
Definition ValueTypes.h:90
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition ValueTypes.h:433
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition ValueTypes.h:160
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition ValueTypes.h:70
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
Definition ValueTypes.h:144
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
Definition ValueTypes.h:322
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition ValueTypes.h:338
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition ValueTypes.h:170
ElementCount getVectorElementCount() const
Definition ValueTypes.h:388
EVT getDoubleNumVectorElementsVT(LLVMContext &Context) const
Definition ValueTypes.h:509
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition ValueTypes.h:411
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition ValueTypes.h:281
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
Definition ValueTypes.h:397
uint64_t getScalarSizeInBits() const
Definition ValueTypes.h:423
EVT getHalfSizedIntegerVT(LLVMContext &Context) const
Finds the smallest simple value type that is greater than or equal to half the width of this EVT.
Definition ValueTypes.h:468
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
Definition ValueTypes.h:516
static LLVM_ABI EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
TypeSize getStoreSizeInBits() const
Return the number of bits overwritten by a store of the specified value type.
Definition ValueTypes.h:450
EVT changeVectorElementType(LLVMContext &Context, EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
Definition ValueTypes.h:98
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition ValueTypes.h:354
std::string getEVTString() const
This function returns value type as a string, e.g. "i32".
Definition ValueTypes.h:539
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition ValueTypes.h:61
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
Definition ValueTypes.h:419
EVT widenIntegerVectorElementType(LLVMContext &Context) const
Return a VT for an integer vector type with the size of the elements doubled.
Definition ValueTypes.h:490
EVT changeVectorElementCount(LLVMContext &Context, ElementCount EC) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element coun...
Definition ValueTypes.h:109
bool isScalableVT() const
Return true if the type is a scalable type.
Definition ValueTypes.h:225
bool isFixedLengthVector() const
Definition ValueTypes.h:214
bool isVector() const
Return true if this is a vector value type.
Definition ValueTypes.h:191
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition ValueTypes.h:361
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Definition ValueTypes.h:294
LLVM_ABI Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
EVT widenIntegerElementType(LLVMContext &Context) const
Return a VT for an integer element type with doubled bit width.
Definition ValueTypes.h:482
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
Definition ValueTypes.h:202
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition ValueTypes.h:366
EVT changeElementType(LLVMContext &Context, EVT EltVT) const
Return a VT for a type whose attributes match ourselves with the exception of the element type that i...
Definition ValueTypes.h:136
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
Definition ValueTypes.h:180
LLVM_ABI const fltSemantics & getFltSemantics() const
Returns an APFloat semantics tag appropriate for the value type.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition ValueTypes.h:374
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition ValueTypes.h:346
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
Definition ValueTypes.h:499
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition ValueTypes.h:175
static KnownBits makeConstant(const APInt &C)
Create known bits from a known constant.
Definition KnownBits.h:315
KnownBits anyextOrTrunc(unsigned BitWidth) const
Return known bits for an "any" extension or truncation of the value we're tracking.
Definition KnownBits.h:190
static LLVM_ABI KnownBits smax(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for smax(LHS, RHS).
bool isNonNegative() const
Returns true if this value is known to be non-negative.
Definition KnownBits.h:106
unsigned countMinTrailingZeros() const
Returns the minimum number of trailing zero bits.
Definition KnownBits.h:256
KnownBits trunc(unsigned BitWidth) const
Return known bits for a truncation of the value we're tracking.
Definition KnownBits.h:165
KnownBits byteSwap() const
Definition KnownBits.h:559
static LLVM_ABI std::optional< bool > sge(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_SGE result.
unsigned countMaxPopulation() const
Returns the maximum number of bits that could be one.
Definition KnownBits.h:303
KnownBits reverseBits() const
Definition KnownBits.h:563
KnownBits concat(const KnownBits &Lo) const
Concatenate the bits from Lo onto the bottom of *this.
Definition KnownBits.h:247
static LLVM_ABI KnownBits umax(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for umax(LHS, RHS).
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
Definition KnownBits.h:262
APInt getMaxValue() const
Return the maximal unsigned value possible given these KnownBits.
Definition KnownBits.h:146
static LLVM_ABI KnownBits smin(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for smin(LHS, RHS).
static LLVM_ABI std::optional< bool > ugt(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_UGT result.
static LLVM_ABI std::optional< bool > slt(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_SLT result.
static LLVM_ABI KnownBits computeForAddSub(bool Add, bool NSW, bool NUW, const KnownBits &LHS, const KnownBits &RHS)
Compute known bits resulting from adding LHS and RHS.
Definition KnownBits.cpp:61
static LLVM_ABI std::optional< bool > ult(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_ULT result.
static LLVM_ABI std::optional< bool > ule(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_ULE result.
bool isNegative() const
Returns true if this value is known to be negative.
Definition KnownBits.h:103
static LLVM_ABI KnownBits mul(const KnownBits &LHS, const KnownBits &RHS, bool NoUndefSelfMultiply=false)
Compute known bits resulting from multiplying LHS and RHS.
static LLVM_ABI std::optional< bool > sle(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_SLE result.
static LLVM_ABI std::optional< bool > sgt(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_SGT result.
unsigned countMinPopulation() const
Returns the number of bits known to be one.
Definition KnownBits.h:300
static LLVM_ABI std::optional< bool > uge(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_UGE result.
static LLVM_ABI KnownBits umin(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for umin(LHS, RHS).
Matching combinators.
This class contains a discriminated union of information about pointers in memory operands,...
LLVM_ABI unsigned getAddrSpace() const
Return the LLVM IR address space number that this pointer points into.
static LLVM_ABI MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
MachinePointerInfo getWithOffset(int64_t O) const
static LLVM_ABI MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition Alignment.h:111
static bool hasVectorMaskArgument(RTLIB::LibcallImpl Impl)
Returns true if the function has a vector mask argument, which is assumed to be the last argument.
These are IR-level optimization flags that may be propagated to SDNodes.
bool hasNoUnsignedWrap() const
bool hasNoSignedWrap() const
void setNoSignedWrap(bool b)
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
Magic data for optimising signed division by a constant.
static LLVM_ABI SignedDivisionByConstantInfo get(const APInt &D)
Calculate the magic numbers required to implement a signed integer division by a constant as a sequen...
This contains information for each constraint that we are lowering.
std::string ConstraintCode
This contains the actual string for the code, like "m".
LLVM_ABI unsigned getMatchedOperand() const
If this is an input matching constraint, this method returns the output operand it matches.
LLVM_ABI bool isMatchingInputConstraint() const
Return true of this is an input operand that is a matching constraint like "4".
This structure contains all information that is necessary for lowering calls.
CallLoweringInfo & setIsPostTypeLegalization(bool Value=true)
CallLoweringInfo & setLibCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList)
CallLoweringInfo & setDiscardResult(bool Value=true)
CallLoweringInfo & setZExtResult(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setSExtResult(bool Value=true)
CallLoweringInfo & setNoReturn(bool Value=true)
CallLoweringInfo & setChain(SDValue InChain)
LLVM_ABI void AddToWorklist(SDNode *N)
LLVM_ABI void CommitTargetLoweringOpt(const TargetLoweringOpt &TLO)
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setIsPostTypeLegalization(bool Value=true)
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT)
MakeLibCallOptions & setIsSigned(bool Value=true)
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
Magic data for optimising unsigned division by a constant.
static LLVM_ABI UnsignedDivisionByConstantInfo get(const APInt &D, unsigned LeadingZeros=0, bool AllowEvenDivisorOptimization=true, bool AllowWidenOptimization=false)
Calculate the magic numbers required to implement an unsigned integer division by a constant as a seq...
fltNonfiniteBehavior nonFiniteBehavior
Definition APFloat.h:1039
fltNanEncoding nanEncoding
Definition APFloat.h:1041