Bug Summary

File:llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
Warning:line 698, column 32
The result of the '%' expression is undefined

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name TargetLowering.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/build-llvm -resource-dir /usr/lib/llvm-14/lib/clang/14.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG -I include -I /build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/include -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-14/lib/clang/14.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-command-line-argument -Wno-unknown-warning-option -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/build-llvm -ferror-limit 19 -fvisibility-inlines-hidden -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2021-10-17-004846-21170-1 -x c++ /build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

1//===-- TargetLowering.cpp - Implement the TargetLowering class -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the TargetLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/CodeGen/TargetLowering.h"
14#include "llvm/ADT/STLExtras.h"
15#include "llvm/CodeGen/CallingConvLower.h"
16#include "llvm/CodeGen/MachineFrameInfo.h"
17#include "llvm/CodeGen/MachineFunction.h"
18#include "llvm/CodeGen/MachineJumpTableInfo.h"
19#include "llvm/CodeGen/MachineRegisterInfo.h"
20#include "llvm/CodeGen/SelectionDAG.h"
21#include "llvm/CodeGen/TargetRegisterInfo.h"
22#include "llvm/CodeGen/TargetSubtargetInfo.h"
23#include "llvm/IR/DataLayout.h"
24#include "llvm/IR/DerivedTypes.h"
25#include "llvm/IR/GlobalVariable.h"
26#include "llvm/IR/LLVMContext.h"
27#include "llvm/MC/MCAsmInfo.h"
28#include "llvm/MC/MCExpr.h"
29#include "llvm/Support/DivisionByConstantInfo.h"
30#include "llvm/Support/ErrorHandling.h"
31#include "llvm/Support/KnownBits.h"
32#include "llvm/Support/MathExtras.h"
33#include "llvm/Target/TargetLoweringObjectFile.h"
34#include "llvm/Target/TargetMachine.h"
35#include <cctype>
36using namespace llvm;
37
38/// NOTE: The TargetMachine owns TLOF.
39TargetLowering::TargetLowering(const TargetMachine &tm)
40 : TargetLoweringBase(tm) {}
41
42const char *TargetLowering::getTargetNodeName(unsigned Opcode) const {
43 return nullptr;
44}
45
46bool TargetLowering::isPositionIndependent() const {
47 return getTargetMachine().isPositionIndependent();
48}
49
50/// Check whether a given call node is in tail position within its function. If
51/// so, it sets Chain to the input chain of the tail call.
52bool TargetLowering::isInTailCallPosition(SelectionDAG &DAG, SDNode *Node,
53 SDValue &Chain) const {
54 const Function &F = DAG.getMachineFunction().getFunction();
55
56 // First, check if tail calls have been disabled in this function.
57 if (F.getFnAttribute("disable-tail-calls").getValueAsBool())
58 return false;
59
60 // Conservatively require the attributes of the call to match those of
61 // the return. Ignore following attributes because they don't affect the
62 // call sequence.
63 AttrBuilder CallerAttrs(F.getAttributes(), AttributeList::ReturnIndex);
64 for (const auto &Attr : {Attribute::Alignment, Attribute::Dereferenceable,
65 Attribute::DereferenceableOrNull, Attribute::NoAlias,
66 Attribute::NonNull})
67 CallerAttrs.removeAttribute(Attr);
68
69 if (CallerAttrs.hasAttributes())
70 return false;
71
72 // It's not safe to eliminate the sign / zero extension of the return value.
73 if (CallerAttrs.contains(Attribute::ZExt) ||
74 CallerAttrs.contains(Attribute::SExt))
75 return false;
76
77 // Check if the only use is a function return node.
78 return isUsedByReturnOnly(Node, Chain);
79}
80
81bool TargetLowering::parametersInCSRMatch(const MachineRegisterInfo &MRI,
82 const uint32_t *CallerPreservedMask,
83 const SmallVectorImpl<CCValAssign> &ArgLocs,
84 const SmallVectorImpl<SDValue> &OutVals) const {
85 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
86 const CCValAssign &ArgLoc = ArgLocs[I];
87 if (!ArgLoc.isRegLoc())
88 continue;
89 MCRegister Reg = ArgLoc.getLocReg();
90 // Only look at callee saved registers.
91 if (MachineOperand::clobbersPhysReg(CallerPreservedMask, Reg))
92 continue;
93 // Check that we pass the value used for the caller.
94 // (We look for a CopyFromReg reading a virtual register that is used
95 // for the function live-in value of register Reg)
96 SDValue Value = OutVals[I];
97 if (Value->getOpcode() != ISD::CopyFromReg)
98 return false;
99 Register ArgReg = cast<RegisterSDNode>(Value->getOperand(1))->getReg();
100 if (MRI.getLiveInPhysReg(ArgReg) != Reg)
101 return false;
102 }
103 return true;
104}
105
106/// Set CallLoweringInfo attribute flags based on a call instruction
107/// and called function attributes.
108void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call,
109 unsigned ArgIdx) {
110 IsSExt = Call->paramHasAttr(ArgIdx, Attribute::SExt);
111 IsZExt = Call->paramHasAttr(ArgIdx, Attribute::ZExt);
112 IsInReg = Call->paramHasAttr(ArgIdx, Attribute::InReg);
113 IsSRet = Call->paramHasAttr(ArgIdx, Attribute::StructRet);
114 IsNest = Call->paramHasAttr(ArgIdx, Attribute::Nest);
115 IsByVal = Call->paramHasAttr(ArgIdx, Attribute::ByVal);
116 IsPreallocated = Call->paramHasAttr(ArgIdx, Attribute::Preallocated);
117 IsInAlloca = Call->paramHasAttr(ArgIdx, Attribute::InAlloca);
118 IsReturned = Call->paramHasAttr(ArgIdx, Attribute::Returned);
119 IsSwiftSelf = Call->paramHasAttr(ArgIdx, Attribute::SwiftSelf);
120 IsSwiftAsync = Call->paramHasAttr(ArgIdx, Attribute::SwiftAsync);
121 IsSwiftError = Call->paramHasAttr(ArgIdx, Attribute::SwiftError);
122 Alignment = Call->getParamStackAlign(ArgIdx);
123 IndirectType = nullptr;
124 assert(IsByVal + IsPreallocated + IsInAlloca <= 1 &&(static_cast <bool> (IsByVal + IsPreallocated + IsInAlloca
<= 1 && "multiple ABI attributes?") ? void (0) : __assert_fail
("IsByVal + IsPreallocated + IsInAlloca <= 1 && \"multiple ABI attributes?\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 125, __extension__ __PRETTY_FUNCTION__))
125 "multiple ABI attributes?")(static_cast <bool> (IsByVal + IsPreallocated + IsInAlloca
<= 1 && "multiple ABI attributes?") ? void (0) : __assert_fail
("IsByVal + IsPreallocated + IsInAlloca <= 1 && \"multiple ABI attributes?\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 125, __extension__ __PRETTY_FUNCTION__))
;
126 if (IsByVal) {
127 IndirectType = Call->getParamByValType(ArgIdx);
128 if (!Alignment)
129 Alignment = Call->getParamAlign(ArgIdx);
130 }
131 if (IsPreallocated)
132 IndirectType = Call->getParamPreallocatedType(ArgIdx);
133 if (IsInAlloca)
134 IndirectType = Call->getParamInAllocaType(ArgIdx);
135}
136
137/// Generate a libcall taking the given operands as arguments and returning a
138/// result of type RetVT.
139std::pair<SDValue, SDValue>
140TargetLowering::makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT,
141 ArrayRef<SDValue> Ops,
142 MakeLibCallOptions CallOptions,
143 const SDLoc &dl,
144 SDValue InChain) const {
145 if (!InChain)
146 InChain = DAG.getEntryNode();
147
148 TargetLowering::ArgListTy Args;
149 Args.reserve(Ops.size());
150
151 TargetLowering::ArgListEntry Entry;
152 for (unsigned i = 0; i < Ops.size(); ++i) {
153 SDValue NewOp = Ops[i];
154 Entry.Node = NewOp;
155 Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
156 Entry.IsSExt = shouldSignExtendTypeInLibCall(NewOp.getValueType(),
157 CallOptions.IsSExt);
158 Entry.IsZExt = !Entry.IsSExt;
159
160 if (CallOptions.IsSoften &&
161 !shouldExtendTypeInLibCall(CallOptions.OpsVTBeforeSoften[i])) {
162 Entry.IsSExt = Entry.IsZExt = false;
163 }
164 Args.push_back(Entry);
165 }
166
167 if (LC == RTLIB::UNKNOWN_LIBCALL)
168 report_fatal_error("Unsupported library call operation!");
169 SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
170 getPointerTy(DAG.getDataLayout()));
171
172 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
173 TargetLowering::CallLoweringInfo CLI(DAG);
174 bool signExtend = shouldSignExtendTypeInLibCall(RetVT, CallOptions.IsSExt);
175 bool zeroExtend = !signExtend;
176
177 if (CallOptions.IsSoften &&
178 !shouldExtendTypeInLibCall(CallOptions.RetVTBeforeSoften)) {
179 signExtend = zeroExtend = false;
180 }
181
182 CLI.setDebugLoc(dl)
183 .setChain(InChain)
184 .setLibCallee(getLibcallCallingConv(LC), RetTy, Callee, std::move(Args))
185 .setNoReturn(CallOptions.DoesNotReturn)
186 .setDiscardResult(!CallOptions.IsReturnValueUsed)
187 .setIsPostTypeLegalization(CallOptions.IsPostTypeLegalization)
188 .setSExtResult(signExtend)
189 .setZExtResult(zeroExtend);
190 return LowerCallTo(CLI);
191}
192
193bool TargetLowering::findOptimalMemOpLowering(
194 std::vector<EVT> &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS,
195 unsigned SrcAS, const AttributeList &FuncAttributes) const {
196 if (Op.isMemcpyWithFixedDstAlign() && Op.getSrcAlign() < Op.getDstAlign())
197 return false;
198
199 EVT VT = getOptimalMemOpType(Op, FuncAttributes);
200
201 if (VT == MVT::Other) {
202 // Use the largest integer type whose alignment constraints are satisfied.
203 // We only need to check DstAlign here as SrcAlign is always greater or
204 // equal to DstAlign (or zero).
205 VT = MVT::i64;
206 if (Op.isFixedDstAlign())
207 while (Op.getDstAlign() < (VT.getSizeInBits() / 8) &&
208 !allowsMisalignedMemoryAccesses(VT, DstAS, Op.getDstAlign()))
209 VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1);
210 assert(VT.isInteger())(static_cast <bool> (VT.isInteger()) ? void (0) : __assert_fail
("VT.isInteger()", "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 210, __extension__ __PRETTY_FUNCTION__))
;
211
212 // Find the largest legal integer type.
213 MVT LVT = MVT::i64;
214 while (!isTypeLegal(LVT))
215 LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
216 assert(LVT.isInteger())(static_cast <bool> (LVT.isInteger()) ? void (0) : __assert_fail
("LVT.isInteger()", "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 216, __extension__ __PRETTY_FUNCTION__))
;
217
218 // If the type we've chosen is larger than the largest legal integer type
219 // then use that instead.
220 if (VT.bitsGT(LVT))
221 VT = LVT;
222 }
223
224 unsigned NumMemOps = 0;
225 uint64_t Size = Op.size();
226 while (Size) {
227 unsigned VTSize = VT.getSizeInBits() / 8;
228 while (VTSize > Size) {
229 // For now, only use non-vector load / store's for the left-over pieces.
230 EVT NewVT = VT;
231 unsigned NewVTSize;
232
233 bool Found = false;
234 if (VT.isVector() || VT.isFloatingPoint()) {
235 NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;
236 if (isOperationLegalOrCustom(ISD::STORE, NewVT) &&
237 isSafeMemOpType(NewVT.getSimpleVT()))
238 Found = true;
239 else if (NewVT == MVT::i64 &&
240 isOperationLegalOrCustom(ISD::STORE, MVT::f64) &&
241 isSafeMemOpType(MVT::f64)) {
242 // i64 is usually not legal on 32-bit targets, but f64 may be.
243 NewVT = MVT::f64;
244 Found = true;
245 }
246 }
247
248 if (!Found) {
249 do {
250 NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
251 if (NewVT == MVT::i8)
252 break;
253 } while (!isSafeMemOpType(NewVT.getSimpleVT()));
254 }
255 NewVTSize = NewVT.getSizeInBits() / 8;
256
257 // If the new VT cannot cover all of the remaining bits, then consider
258 // issuing a (or a pair of) unaligned and overlapping load / store.
259 bool Fast;
260 if (NumMemOps && Op.allowOverlap() && NewVTSize < Size &&
261 allowsMisalignedMemoryAccesses(
262 VT, DstAS, Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1),
263 MachineMemOperand::MONone, &Fast) &&
264 Fast)
265 VTSize = Size;
266 else {
267 VT = NewVT;
268 VTSize = NewVTSize;
269 }
270 }
271
272 if (++NumMemOps > Limit)
273 return false;
274
275 MemOps.push_back(VT);
276 Size -= VTSize;
277 }
278
279 return true;
280}
281
282/// Soften the operands of a comparison. This code is shared among BR_CC,
283/// SELECT_CC, and SETCC handlers.
284void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,
285 SDValue &NewLHS, SDValue &NewRHS,
286 ISD::CondCode &CCCode,
287 const SDLoc &dl, const SDValue OldLHS,
288 const SDValue OldRHS) const {
289 SDValue Chain;
290 return softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, dl, OldLHS,
291 OldRHS, Chain);
292}
293
294void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,
295 SDValue &NewLHS, SDValue &NewRHS,
296 ISD::CondCode &CCCode,
297 const SDLoc &dl, const SDValue OldLHS,
298 const SDValue OldRHS,
299 SDValue &Chain,
300 bool IsSignaling) const {
301 // FIXME: Currently we cannot really respect all IEEE predicates due to libgcc
302 // not supporting it. We can update this code when libgcc provides such
303 // functions.
304
305 assert((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128)(static_cast <bool> ((VT == MVT::f32 || VT == MVT::f64 ||
VT == MVT::f128 || VT == MVT::ppcf128) && "Unsupported setcc type!"
) ? void (0) : __assert_fail ("(VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128) && \"Unsupported setcc type!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 306, __extension__ __PRETTY_FUNCTION__))
306 && "Unsupported setcc type!")(static_cast <bool> ((VT == MVT::f32 || VT == MVT::f64 ||
VT == MVT::f128 || VT == MVT::ppcf128) && "Unsupported setcc type!"
) ? void (0) : __assert_fail ("(VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128) && \"Unsupported setcc type!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 306, __extension__ __PRETTY_FUNCTION__))
;
307
308 // Expand into one or more soft-fp libcall(s).
309 RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL;
310 bool ShouldInvertCC = false;
311 switch (CCCode) {
312 case ISD::SETEQ:
313 case ISD::SETOEQ:
314 LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
315 (VT == MVT::f64) ? RTLIB::OEQ_F64 :
316 (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
317 break;
318 case ISD::SETNE:
319 case ISD::SETUNE:
320 LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 :
321 (VT == MVT::f64) ? RTLIB::UNE_F64 :
322 (VT == MVT::f128) ? RTLIB::UNE_F128 : RTLIB::UNE_PPCF128;
323 break;
324 case ISD::SETGE:
325 case ISD::SETOGE:
326 LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
327 (VT == MVT::f64) ? RTLIB::OGE_F64 :
328 (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
329 break;
330 case ISD::SETLT:
331 case ISD::SETOLT:
332 LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
333 (VT == MVT::f64) ? RTLIB::OLT_F64 :
334 (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
335 break;
336 case ISD::SETLE:
337 case ISD::SETOLE:
338 LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
339 (VT == MVT::f64) ? RTLIB::OLE_F64 :
340 (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
341 break;
342 case ISD::SETGT:
343 case ISD::SETOGT:
344 LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
345 (VT == MVT::f64) ? RTLIB::OGT_F64 :
346 (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
347 break;
348 case ISD::SETO:
349 ShouldInvertCC = true;
350 LLVM_FALLTHROUGH[[gnu::fallthrough]];
351 case ISD::SETUO:
352 LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
353 (VT == MVT::f64) ? RTLIB::UO_F64 :
354 (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
355 break;
356 case ISD::SETONE:
357 // SETONE = O && UNE
358 ShouldInvertCC = true;
359 LLVM_FALLTHROUGH[[gnu::fallthrough]];
360 case ISD::SETUEQ:
361 LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
362 (VT == MVT::f64) ? RTLIB::UO_F64 :
363 (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
364 LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
365 (VT == MVT::f64) ? RTLIB::OEQ_F64 :
366 (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
367 break;
368 default:
369 // Invert CC for unordered comparisons
370 ShouldInvertCC = true;
371 switch (CCCode) {
372 case ISD::SETULT:
373 LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
374 (VT == MVT::f64) ? RTLIB::OGE_F64 :
375 (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
376 break;
377 case ISD::SETULE:
378 LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
379 (VT == MVT::f64) ? RTLIB::OGT_F64 :
380 (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
381 break;
382 case ISD::SETUGT:
383 LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
384 (VT == MVT::f64) ? RTLIB::OLE_F64 :
385 (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
386 break;
387 case ISD::SETUGE:
388 LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
389 (VT == MVT::f64) ? RTLIB::OLT_F64 :
390 (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
391 break;
392 default: llvm_unreachable("Do not know how to soften this setcc!")::llvm::llvm_unreachable_internal("Do not know how to soften this setcc!"
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 392)
;
393 }
394 }
395
396 // Use the target specific return value for comparions lib calls.
397 EVT RetVT = getCmpLibcallReturnType();
398 SDValue Ops[2] = {NewLHS, NewRHS};
399 TargetLowering::MakeLibCallOptions CallOptions;
400 EVT OpsVT[2] = { OldLHS.getValueType(),
401 OldRHS.getValueType() };
402 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT, true);
403 auto Call = makeLibCall(DAG, LC1, RetVT, Ops, CallOptions, dl, Chain);
404 NewLHS = Call.first;
405 NewRHS = DAG.getConstant(0, dl, RetVT);
406
407 CCCode = getCmpLibcallCC(LC1);
408 if (ShouldInvertCC) {
409 assert(RetVT.isInteger())(static_cast <bool> (RetVT.isInteger()) ? void (0) : __assert_fail
("RetVT.isInteger()", "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 409, __extension__ __PRETTY_FUNCTION__))
;
410 CCCode = getSetCCInverse(CCCode, RetVT);
411 }
412
413 if (LC2 == RTLIB::UNKNOWN_LIBCALL) {
414 // Update Chain.
415 Chain = Call.second;
416 } else {
417 EVT SetCCVT =
418 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), RetVT);
419 SDValue Tmp = DAG.getSetCC(dl, SetCCVT, NewLHS, NewRHS, CCCode);
420 auto Call2 = makeLibCall(DAG, LC2, RetVT, Ops, CallOptions, dl, Chain);
421 CCCode = getCmpLibcallCC(LC2);
422 if (ShouldInvertCC)
423 CCCode = getSetCCInverse(CCCode, RetVT);
424 NewLHS = DAG.getSetCC(dl, SetCCVT, Call2.first, NewRHS, CCCode);
425 if (Chain)
426 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Call.second,
427 Call2.second);
428 NewLHS = DAG.getNode(ShouldInvertCC ? ISD::AND : ISD::OR, dl,
429 Tmp.getValueType(), Tmp, NewLHS);
430 NewRHS = SDValue();
431 }
432}
433
434/// Return the entry encoding for a jump table in the current function. The
435/// returned value is a member of the MachineJumpTableInfo::JTEntryKind enum.
436unsigned TargetLowering::getJumpTableEncoding() const {
437 // In non-pic modes, just use the address of a block.
438 if (!isPositionIndependent())
439 return MachineJumpTableInfo::EK_BlockAddress;
440
441 // In PIC mode, if the target supports a GPRel32 directive, use it.
442 if (getTargetMachine().getMCAsmInfo()->getGPRel32Directive() != nullptr)
443 return MachineJumpTableInfo::EK_GPRel32BlockAddress;
444
445 // Otherwise, use a label difference.
446 return MachineJumpTableInfo::EK_LabelDifference32;
447}
448
449SDValue TargetLowering::getPICJumpTableRelocBase(SDValue Table,
450 SelectionDAG &DAG) const {
451 // If our PIC model is GP relative, use the global offset table as the base.
452 unsigned JTEncoding = getJumpTableEncoding();
453
454 if ((JTEncoding == MachineJumpTableInfo::EK_GPRel64BlockAddress) ||
455 (JTEncoding == MachineJumpTableInfo::EK_GPRel32BlockAddress))
456 return DAG.getGLOBAL_OFFSET_TABLE(getPointerTy(DAG.getDataLayout()));
457
458 return Table;
459}
460
461/// This returns the relocation base for the given PIC jumptable, the same as
462/// getPICJumpTableRelocBase, but as an MCExpr.
463const MCExpr *
464TargetLowering::getPICJumpTableRelocBaseExpr(const MachineFunction *MF,
465 unsigned JTI,MCContext &Ctx) const{
466 // The normal PIC reloc base is the label at the start of the jump table.
467 return MCSymbolRefExpr::create(MF->getJTISymbol(JTI, Ctx), Ctx);
468}
469
470bool
471TargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
472 const TargetMachine &TM = getTargetMachine();
473 const GlobalValue *GV = GA->getGlobal();
474
475 // If the address is not even local to this DSO we will have to load it from
476 // a got and then add the offset.
477 if (!TM.shouldAssumeDSOLocal(*GV->getParent(), GV))
478 return false;
479
480 // If the code is position independent we will have to add a base register.
481 if (isPositionIndependent())
482 return false;
483
484 // Otherwise we can do it.
485 return true;
486}
487
488//===----------------------------------------------------------------------===//
489// Optimization Methods
490//===----------------------------------------------------------------------===//
491
492/// If the specified instruction has a constant integer operand and there are
493/// bits set in that constant that are not demanded, then clear those bits and
494/// return true.
495bool TargetLowering::ShrinkDemandedConstant(SDValue Op,
496 const APInt &DemandedBits,
497 const APInt &DemandedElts,
498 TargetLoweringOpt &TLO) const {
499 SDLoc DL(Op);
500 unsigned Opcode = Op.getOpcode();
501
502 // Do target-specific constant optimization.
503 if (targetShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
504 return TLO.New.getNode();
505
506 // FIXME: ISD::SELECT, ISD::SELECT_CC
507 switch (Opcode) {
508 default:
509 break;
510 case ISD::XOR:
511 case ISD::AND:
512 case ISD::OR: {
513 auto *Op1C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
514 if (!Op1C || Op1C->isOpaque())
515 return false;
516
517 // If this is a 'not' op, don't touch it because that's a canonical form.
518 const APInt &C = Op1C->getAPIntValue();
519 if (Opcode == ISD::XOR && DemandedBits.isSubsetOf(C))
520 return false;
521
522 if (!C.isSubsetOf(DemandedBits)) {
523 EVT VT = Op.getValueType();
524 SDValue NewC = TLO.DAG.getConstant(DemandedBits & C, DL, VT);
525 SDValue NewOp = TLO.DAG.getNode(Opcode, DL, VT, Op.getOperand(0), NewC);
526 return TLO.CombineTo(Op, NewOp);
527 }
528
529 break;
530 }
531 }
532
533 return false;
534}
535
536bool TargetLowering::ShrinkDemandedConstant(SDValue Op,
537 const APInt &DemandedBits,
538 TargetLoweringOpt &TLO) const {
539 EVT VT = Op.getValueType();
540 APInt DemandedElts = VT.isVector()
541 ? APInt::getAllOnes(VT.getVectorNumElements())
542 : APInt(1, 1);
543 return ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO);
544}
545
546/// Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.
547/// This uses isZExtFree and ZERO_EXTEND for the widening cast, but it could be
548/// generalized for targets with other types of implicit widening casts.
549bool TargetLowering::ShrinkDemandedOp(SDValue Op, unsigned BitWidth,
550 const APInt &Demanded,
551 TargetLoweringOpt &TLO) const {
552 assert(Op.getNumOperands() == 2 &&(static_cast <bool> (Op.getNumOperands() == 2 &&
"ShrinkDemandedOp only supports binary operators!") ? void (
0) : __assert_fail ("Op.getNumOperands() == 2 && \"ShrinkDemandedOp only supports binary operators!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 553, __extension__ __PRETTY_FUNCTION__))
553 "ShrinkDemandedOp only supports binary operators!")(static_cast <bool> (Op.getNumOperands() == 2 &&
"ShrinkDemandedOp only supports binary operators!") ? void (
0) : __assert_fail ("Op.getNumOperands() == 2 && \"ShrinkDemandedOp only supports binary operators!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 553, __extension__ __PRETTY_FUNCTION__))
;
554 assert(Op.getNode()->getNumValues() == 1 &&(static_cast <bool> (Op.getNode()->getNumValues() ==
1 && "ShrinkDemandedOp only supports nodes with one result!"
) ? void (0) : __assert_fail ("Op.getNode()->getNumValues() == 1 && \"ShrinkDemandedOp only supports nodes with one result!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 555, __extension__ __PRETTY_FUNCTION__))
555 "ShrinkDemandedOp only supports nodes with one result!")(static_cast <bool> (Op.getNode()->getNumValues() ==
1 && "ShrinkDemandedOp only supports nodes with one result!"
) ? void (0) : __assert_fail ("Op.getNode()->getNumValues() == 1 && \"ShrinkDemandedOp only supports nodes with one result!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 555, __extension__ __PRETTY_FUNCTION__))
;
556
557 SelectionDAG &DAG = TLO.DAG;
558 SDLoc dl(Op);
559
560 // Early return, as this function cannot handle vector types.
561 if (Op.getValueType().isVector())
562 return false;
563
564 // Don't do this if the node has another user, which may require the
565 // full value.
566 if (!Op.getNode()->hasOneUse())
567 return false;
568
569 // Search for the smallest integer type with free casts to and from
570 // Op's type. For expedience, just check power-of-2 integer types.
571 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
572 unsigned DemandedSize = Demanded.getActiveBits();
573 unsigned SmallVTBits = DemandedSize;
574 if (!isPowerOf2_32(SmallVTBits))
575 SmallVTBits = NextPowerOf2(SmallVTBits);
576 for (; SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(SmallVTBits)) {
577 EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), SmallVTBits);
578 if (TLI.isTruncateFree(Op.getValueType(), SmallVT) &&
579 TLI.isZExtFree(SmallVT, Op.getValueType())) {
580 // We found a type with free casts.
581 SDValue X = DAG.getNode(
582 Op.getOpcode(), dl, SmallVT,
583 DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(0)),
584 DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(1)));
585 assert(DemandedSize <= SmallVTBits && "Narrowed below demanded bits?")(static_cast <bool> (DemandedSize <= SmallVTBits &&
"Narrowed below demanded bits?") ? void (0) : __assert_fail (
"DemandedSize <= SmallVTBits && \"Narrowed below demanded bits?\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 585, __extension__ __PRETTY_FUNCTION__))
;
586 SDValue Z = DAG.getNode(ISD::ANY_EXTEND, dl, Op.getValueType(), X);
587 return TLO.CombineTo(Op, Z);
588 }
589 }
590 return false;
591}
592
593bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
594 DAGCombinerInfo &DCI) const {
595 SelectionDAG &DAG = DCI.DAG;
596 TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
597 !DCI.isBeforeLegalizeOps());
598 KnownBits Known;
599
600 bool Simplified = SimplifyDemandedBits(Op, DemandedBits, Known, TLO);
601 if (Simplified) {
602 DCI.AddToWorklist(Op.getNode());
603 DCI.CommitTargetLoweringOpt(TLO);
604 }
605 return Simplified;
606}
607
608bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
609 KnownBits &Known,
610 TargetLoweringOpt &TLO,
611 unsigned Depth,
612 bool AssumeSingleUse) const {
613 EVT VT = Op.getValueType();
614
615 // TODO: We can probably do more work on calculating the known bits and
616 // simplifying the operations for scalable vectors, but for now we just
617 // bail out.
618 if (VT.isScalableVector()) {
619 // Pretend we don't know anything for now.
620 Known = KnownBits(DemandedBits.getBitWidth());
621 return false;
622 }
623
624 APInt DemandedElts = VT.isVector()
625 ? APInt::getAllOnes(VT.getVectorNumElements())
626 : APInt(1, 1);
627 return SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO, Depth,
628 AssumeSingleUse);
629}
630
631// TODO: Can we merge SelectionDAG::GetDemandedBits into this?
632// TODO: Under what circumstances can we create nodes? Constant folding?
633SDValue TargetLowering::SimplifyMultipleUseDemandedBits(
634 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
635 SelectionDAG &DAG, unsigned Depth) const {
636 // Limit search depth.
637 if (Depth >= SelectionDAG::MaxRecursionDepth)
1
Assuming 'Depth' is < 'MaxRecursionDepth'
2
Taking false branch
638 return SDValue();
639
640 // Ignore UNDEFs.
641 if (Op.isUndef())
3
Calling 'SDValue::isUndef'
9
Returning from 'SDValue::isUndef'
642 return SDValue();
643
644 // Not demanding any bits/elts from Op.
645 if (DemandedBits == 0 || DemandedElts == 0)
10
Calling 'APInt::operator=='
13
Returning from 'APInt::operator=='
14
Calling 'APInt::operator=='
17
Returning from 'APInt::operator=='
18
Taking false branch
646 return DAG.getUNDEF(Op.getValueType());
647
648 unsigned NumElts = DemandedElts.getBitWidth();
649 unsigned BitWidth = DemandedBits.getBitWidth();
650 KnownBits LHSKnown, RHSKnown;
651 switch (Op.getOpcode()) {
19
Control jumps to 'case BITCAST:' at line 652
652 case ISD::BITCAST: {
653 SDValue Src = peekThroughBitcasts(Op.getOperand(0));
654 EVT SrcVT = Src.getValueType();
655 EVT DstVT = Op.getValueType();
656 if (SrcVT == DstVT)
20
Calling 'EVT::operator=='
30
Returning from 'EVT::operator=='
31
Taking false branch
657 return Src;
658
659 unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
660 unsigned NumDstEltBits = DstVT.getScalarSizeInBits();
661 if (NumSrcEltBits == NumDstEltBits)
32
Assuming 'NumSrcEltBits' is not equal to 'NumDstEltBits'
662 if (SDValue V = SimplifyMultipleUseDemandedBits(
663 Src, DemandedBits, DemandedElts, DAG, Depth + 1))
664 return DAG.getBitcast(DstVT, V);
665
666 // TODO - bigendian once we have test coverage.
667 if (SrcVT.isVector() && (NumDstEltBits % NumSrcEltBits) == 0 &&
33
Calling 'EVT::isVector'
36
Returning from 'EVT::isVector'
37
Assuming the condition is false
668 DAG.getDataLayout().isLittleEndian()) {
669 unsigned Scale = NumDstEltBits / NumSrcEltBits;
670 unsigned NumSrcElts = SrcVT.getVectorNumElements();
671 APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);
672 APInt DemandedSrcElts = APInt::getZero(NumSrcElts);
673 for (unsigned i = 0; i != Scale; ++i) {
674 unsigned Offset = i * NumSrcEltBits;
675 APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset);
676 if (!Sub.isZero()) {
677 DemandedSrcBits |= Sub;
678 for (unsigned j = 0; j != NumElts; ++j)
679 if (DemandedElts[j])
680 DemandedSrcElts.setBit((j * Scale) + i);
681 }
682 }
683
684 if (SDValue V = SimplifyMultipleUseDemandedBits(
685 Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))
686 return DAG.getBitcast(DstVT, V);
687 }
688
689 // TODO - bigendian once we have test coverage.
690 if ((NumSrcEltBits % NumDstEltBits) == 0 &&
38
Assuming the condition is true
43
Taking true branch
691 DAG.getDataLayout().isLittleEndian()) {
39
Calling 'DataLayout::isLittleEndian'
42
Returning from 'DataLayout::isLittleEndian'
692 unsigned Scale = NumSrcEltBits / NumDstEltBits;
693 unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
44
Assuming the condition is false
45
'?' condition is false
694 APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);
695 APInt DemandedSrcElts = APInt::getZero(NumSrcElts);
696 for (unsigned i = 0; i
45.1
'i' is not equal to 'NumElts'
45.1
'i' is not equal to 'NumElts'
45.1
'i' is not equal to 'NumElts'
45.1
'i' is not equal to 'NumElts'
45.1
'i' is not equal to 'NumElts'
!= NumElts; ++i)
46
Loop condition is true. Entering loop body
697 if (DemandedElts[i]) {
47
Calling 'APInt::operator[]'
51
Returning from 'APInt::operator[]'
52
Taking true branch
698 unsigned Offset = (i % Scale) * NumDstEltBits;
53
The result of the '%' expression is undefined
699 DemandedSrcBits.insertBits(DemandedBits, Offset);
700 DemandedSrcElts.setBit(i / Scale);
701 }
702
703 if (SDValue V = SimplifyMultipleUseDemandedBits(
704 Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))
705 return DAG.getBitcast(DstVT, V);
706 }
707
708 break;
709 }
710 case ISD::AND: {
711 LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
712 RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
713
714 // If all of the demanded bits are known 1 on one side, return the other.
715 // These bits cannot contribute to the result of the 'and' in this
716 // context.
717 if (DemandedBits.isSubsetOf(LHSKnown.Zero | RHSKnown.One))
718 return Op.getOperand(0);
719 if (DemandedBits.isSubsetOf(RHSKnown.Zero | LHSKnown.One))
720 return Op.getOperand(1);
721 break;
722 }
723 case ISD::OR: {
724 LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
725 RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
726
727 // If all of the demanded bits are known zero on one side, return the
728 // other. These bits cannot contribute to the result of the 'or' in this
729 // context.
730 if (DemandedBits.isSubsetOf(LHSKnown.One | RHSKnown.Zero))
731 return Op.getOperand(0);
732 if (DemandedBits.isSubsetOf(RHSKnown.One | LHSKnown.Zero))
733 return Op.getOperand(1);
734 break;
735 }
736 case ISD::XOR: {
737 LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
738 RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
739
740 // If all of the demanded bits are known zero on one side, return the
741 // other.
742 if (DemandedBits.isSubsetOf(RHSKnown.Zero))
743 return Op.getOperand(0);
744 if (DemandedBits.isSubsetOf(LHSKnown.Zero))
745 return Op.getOperand(1);
746 break;
747 }
748 case ISD::SHL: {
749 // If we are only demanding sign bits then we can use the shift source
750 // directly.
751 if (const APInt *MaxSA =
752 DAG.getValidMaximumShiftAmountConstant(Op, DemandedElts)) {
753 SDValue Op0 = Op.getOperand(0);
754 unsigned ShAmt = MaxSA->getZExtValue();
755 unsigned NumSignBits =
756 DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
757 unsigned UpperDemandedBits = BitWidth - DemandedBits.countTrailingZeros();
758 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
759 return Op0;
760 }
761 break;
762 }
763 case ISD::SETCC: {
764 SDValue Op0 = Op.getOperand(0);
765 SDValue Op1 = Op.getOperand(1);
766 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
767 // If (1) we only need the sign-bit, (2) the setcc operands are the same
768 // width as the setcc result, and (3) the result of a setcc conforms to 0 or
769 // -1, we may be able to bypass the setcc.
770 if (DemandedBits.isSignMask() &&
771 Op0.getScalarValueSizeInBits() == BitWidth &&
772 getBooleanContents(Op0.getValueType()) ==
773 BooleanContent::ZeroOrNegativeOneBooleanContent) {
774 // If we're testing X < 0, then this compare isn't needed - just use X!
775 // FIXME: We're limiting to integer types here, but this should also work
776 // if we don't care about FP signed-zero. The use of SETLT with FP means
777 // that we don't care about NaNs.
778 if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
779 (isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode())))
780 return Op0;
781 }
782 break;
783 }
784 case ISD::SIGN_EXTEND_INREG: {
785 // If none of the extended bits are demanded, eliminate the sextinreg.
786 SDValue Op0 = Op.getOperand(0);
787 EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
788 unsigned ExBits = ExVT.getScalarSizeInBits();
789 if (DemandedBits.getActiveBits() <= ExBits)
790 return Op0;
791 // If the input is already sign extended, just drop the extension.
792 unsigned NumSignBits = DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
793 if (NumSignBits >= (BitWidth - ExBits + 1))
794 return Op0;
795 break;
796 }
797 case ISD::ANY_EXTEND_VECTOR_INREG:
798 case ISD::SIGN_EXTEND_VECTOR_INREG:
799 case ISD::ZERO_EXTEND_VECTOR_INREG: {
800 // If we only want the lowest element and none of extended bits, then we can
801 // return the bitcasted source vector.
802 SDValue Src = Op.getOperand(0);
803 EVT SrcVT = Src.getValueType();
804 EVT DstVT = Op.getValueType();
805 if (DemandedElts == 1 && DstVT.getSizeInBits() == SrcVT.getSizeInBits() &&
806 DAG.getDataLayout().isLittleEndian() &&
807 DemandedBits.getActiveBits() <= SrcVT.getScalarSizeInBits()) {
808 return DAG.getBitcast(DstVT, Src);
809 }
810 break;
811 }
812 case ISD::INSERT_VECTOR_ELT: {
813 // If we don't demand the inserted element, return the base vector.
814 SDValue Vec = Op.getOperand(0);
815 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
816 EVT VecVT = Vec.getValueType();
817 if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements()) &&
818 !DemandedElts[CIdx->getZExtValue()])
819 return Vec;
820 break;
821 }
822 case ISD::INSERT_SUBVECTOR: {
823 SDValue Vec = Op.getOperand(0);
824 SDValue Sub = Op.getOperand(1);
825 uint64_t Idx = Op.getConstantOperandVal(2);
826 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
827 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
828 // If we don't demand the inserted subvector, return the base vector.
829 if (DemandedSubElts == 0)
830 return Vec;
831 // If this simply widens the lowest subvector, see if we can do it earlier.
832 if (Idx == 0 && Vec.isUndef()) {
833 if (SDValue NewSub = SimplifyMultipleUseDemandedBits(
834 Sub, DemandedBits, DemandedSubElts, DAG, Depth + 1))
835 return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
836 Op.getOperand(0), NewSub, Op.getOperand(2));
837 }
838 break;
839 }
840 case ISD::VECTOR_SHUFFLE: {
841 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
842
843 // If all the demanded elts are from one operand and are inline,
844 // then we can use the operand directly.
845 bool AllUndef = true, IdentityLHS = true, IdentityRHS = true;
846 for (unsigned i = 0; i != NumElts; ++i) {
847 int M = ShuffleMask[i];
848 if (M < 0 || !DemandedElts[i])
849 continue;
850 AllUndef = false;
851 IdentityLHS &= (M == (int)i);
852 IdentityRHS &= ((M - NumElts) == i);
853 }
854
855 if (AllUndef)
856 return DAG.getUNDEF(Op.getValueType());
857 if (IdentityLHS)
858 return Op.getOperand(0);
859 if (IdentityRHS)
860 return Op.getOperand(1);
861 break;
862 }
863 default:
864 if (Op.getOpcode() >= ISD::BUILTIN_OP_END)
865 if (SDValue V = SimplifyMultipleUseDemandedBitsForTargetNode(
866 Op, DemandedBits, DemandedElts, DAG, Depth))
867 return V;
868 break;
869 }
870 return SDValue();
871}
872
873SDValue TargetLowering::SimplifyMultipleUseDemandedBits(
874 SDValue Op, const APInt &DemandedBits, SelectionDAG &DAG,
875 unsigned Depth) const {
876 EVT VT = Op.getValueType();
877 APInt DemandedElts = VT.isVector()
878 ? APInt::getAllOnes(VT.getVectorNumElements())
879 : APInt(1, 1);
880 return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
881 Depth);
882}
883
884SDValue TargetLowering::SimplifyMultipleUseDemandedVectorElts(
885 SDValue Op, const APInt &DemandedElts, SelectionDAG &DAG,
886 unsigned Depth) const {
887 APInt DemandedBits = APInt::getAllOnes(Op.getScalarValueSizeInBits());
888 return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
889 Depth);
890}
891
892/// Look at Op. At this point, we know that only the OriginalDemandedBits of the
893/// result of Op are ever used downstream. If we can use this information to
894/// simplify Op, create a new simplified DAG node and return true, returning the
895/// original and new nodes in Old and New. Otherwise, analyze the expression and
896/// return a mask of Known bits for the expression (used to simplify the
897/// caller). The Known bits may only be accurate for those bits in the
898/// OriginalDemandedBits and OriginalDemandedElts.
899bool TargetLowering::SimplifyDemandedBits(
900 SDValue Op, const APInt &OriginalDemandedBits,
901 const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
902 unsigned Depth, bool AssumeSingleUse) const {
903 unsigned BitWidth = OriginalDemandedBits.getBitWidth();
904 assert(Op.getScalarValueSizeInBits() == BitWidth &&(static_cast <bool> (Op.getScalarValueSizeInBits() == BitWidth
&& "Mask size mismatches value type size!") ? void (
0) : __assert_fail ("Op.getScalarValueSizeInBits() == BitWidth && \"Mask size mismatches value type size!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 905, __extension__ __PRETTY_FUNCTION__))
905 "Mask size mismatches value type size!")(static_cast <bool> (Op.getScalarValueSizeInBits() == BitWidth
&& "Mask size mismatches value type size!") ? void (
0) : __assert_fail ("Op.getScalarValueSizeInBits() == BitWidth && \"Mask size mismatches value type size!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 905, __extension__ __PRETTY_FUNCTION__))
;
906
907 // Don't know anything.
908 Known = KnownBits(BitWidth);
909
910 // TODO: We can probably do more work on calculating the known bits and
911 // simplifying the operations for scalable vectors, but for now we just
912 // bail out.
913 if (Op.getValueType().isScalableVector())
914 return false;
915
916 unsigned NumElts = OriginalDemandedElts.getBitWidth();
917 assert((!Op.getValueType().isVector() ||(static_cast <bool> ((!Op.getValueType().isVector() || NumElts
== Op.getValueType().getVectorNumElements()) && "Unexpected vector size"
) ? void (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 919, __extension__ __PRETTY_FUNCTION__))
918 NumElts == Op.getValueType().getVectorNumElements()) &&(static_cast <bool> ((!Op.getValueType().isVector() || NumElts
== Op.getValueType().getVectorNumElements()) && "Unexpected vector size"
) ? void (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 919, __extension__ __PRETTY_FUNCTION__))
919 "Unexpected vector size")(static_cast <bool> ((!Op.getValueType().isVector() || NumElts
== Op.getValueType().getVectorNumElements()) && "Unexpected vector size"
) ? void (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 919, __extension__ __PRETTY_FUNCTION__))
;
920
921 APInt DemandedBits = OriginalDemandedBits;
922 APInt DemandedElts = OriginalDemandedElts;
923 SDLoc dl(Op);
924 auto &DL = TLO.DAG.getDataLayout();
925
926 // Undef operand.
927 if (Op.isUndef())
928 return false;
929
930 if (Op.getOpcode() == ISD::Constant) {
931 // We know all of the bits for a constant!
932 Known = KnownBits::makeConstant(cast<ConstantSDNode>(Op)->getAPIntValue());
933 return false;
934 }
935
936 if (Op.getOpcode() == ISD::ConstantFP) {
937 // We know all of the bits for a floating point constant!
938 Known = KnownBits::makeConstant(
939 cast<ConstantFPSDNode>(Op)->getValueAPF().bitcastToAPInt());
940 return false;
941 }
942
943 // Other users may use these bits.
944 EVT VT = Op.getValueType();
945 if (!Op.getNode()->hasOneUse() && !AssumeSingleUse) {
946 if (Depth != 0) {
947 // If not at the root, Just compute the Known bits to
948 // simplify things downstream.
949 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
950 return false;
951 }
952 // If this is the root being simplified, allow it to have multiple uses,
953 // just set the DemandedBits/Elts to all bits.
954 DemandedBits = APInt::getAllOnes(BitWidth);
955 DemandedElts = APInt::getAllOnes(NumElts);
956 } else if (OriginalDemandedBits == 0 || OriginalDemandedElts == 0) {
957 // Not demanding any bits/elts from Op.
958 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
959 } else if (Depth >= SelectionDAG::MaxRecursionDepth) {
960 // Limit search depth.
961 return false;
962 }
963
964 KnownBits Known2;
965 switch (Op.getOpcode()) {
966 case ISD::TargetConstant:
967 llvm_unreachable("Can't simplify this node")::llvm::llvm_unreachable_internal("Can't simplify this node",
"/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 967)
;
968 case ISD::SCALAR_TO_VECTOR: {
969 if (!DemandedElts[0])
970 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
971
972 KnownBits SrcKnown;
973 SDValue Src = Op.getOperand(0);
974 unsigned SrcBitWidth = Src.getScalarValueSizeInBits();
975 APInt SrcDemandedBits = DemandedBits.zextOrSelf(SrcBitWidth);
976 if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcKnown, TLO, Depth + 1))
977 return true;
978
979 // Upper elements are undef, so only get the knownbits if we just demand
980 // the bottom element.
981 if (DemandedElts == 1)
982 Known = SrcKnown.anyextOrTrunc(BitWidth);
983 break;
984 }
985 case ISD::BUILD_VECTOR:
986 // Collect the known bits that are shared by every demanded element.
987 // TODO: Call SimplifyDemandedBits for non-constant demanded elements.
988 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
989 return false; // Don't fall through, will infinitely loop.
990 case ISD::LOAD: {
991 auto *LD = cast<LoadSDNode>(Op);
992 if (getTargetConstantFromLoad(LD)) {
993 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
994 return false; // Don't fall through, will infinitely loop.
995 }
996 if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
997 // If this is a ZEXTLoad and we are looking at the loaded value.
998 EVT MemVT = LD->getMemoryVT();
999 unsigned MemBits = MemVT.getScalarSizeInBits();
1000 Known.Zero.setBitsFrom(MemBits);
1001 return false; // Don't fall through, will infinitely loop.
1002 }
1003 break;
1004 }
1005 case ISD::INSERT_VECTOR_ELT: {
1006 SDValue Vec = Op.getOperand(0);
1007 SDValue Scl = Op.getOperand(1);
1008 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
1009 EVT VecVT = Vec.getValueType();
1010
1011 // If index isn't constant, assume we need all vector elements AND the
1012 // inserted element.
1013 APInt DemandedVecElts(DemandedElts);
1014 if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements())) {
1015 unsigned Idx = CIdx->getZExtValue();
1016 DemandedVecElts.clearBit(Idx);
1017
1018 // Inserted element is not required.
1019 if (!DemandedElts[Idx])
1020 return TLO.CombineTo(Op, Vec);
1021 }
1022
1023 KnownBits KnownScl;
1024 unsigned NumSclBits = Scl.getScalarValueSizeInBits();
1025 APInt DemandedSclBits = DemandedBits.zextOrTrunc(NumSclBits);
1026 if (SimplifyDemandedBits(Scl, DemandedSclBits, KnownScl, TLO, Depth + 1))
1027 return true;
1028
1029 Known = KnownScl.anyextOrTrunc(BitWidth);
1030
1031 KnownBits KnownVec;
1032 if (SimplifyDemandedBits(Vec, DemandedBits, DemandedVecElts, KnownVec, TLO,
1033 Depth + 1))
1034 return true;
1035
1036 if (!!DemandedVecElts)
1037 Known = KnownBits::commonBits(Known, KnownVec);
1038
1039 return false;
1040 }
1041 case ISD::INSERT_SUBVECTOR: {
1042 // Demand any elements from the subvector and the remainder from the src its
1043 // inserted into.
1044 SDValue Src = Op.getOperand(0);
1045 SDValue Sub = Op.getOperand(1);
1046 uint64_t Idx = Op.getConstantOperandVal(2);
1047 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
1048 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
1049 APInt DemandedSrcElts = DemandedElts;
1050 DemandedSrcElts.insertBits(APInt::getZero(NumSubElts), Idx);
1051
1052 KnownBits KnownSub, KnownSrc;
1053 if (SimplifyDemandedBits(Sub, DemandedBits, DemandedSubElts, KnownSub, TLO,
1054 Depth + 1))
1055 return true;
1056 if (SimplifyDemandedBits(Src, DemandedBits, DemandedSrcElts, KnownSrc, TLO,
1057 Depth + 1))
1058 return true;
1059
1060 Known.Zero.setAllBits();
1061 Known.One.setAllBits();
1062 if (!!DemandedSubElts)
1063 Known = KnownBits::commonBits(Known, KnownSub);
1064 if (!!DemandedSrcElts)
1065 Known = KnownBits::commonBits(Known, KnownSrc);
1066
1067 // Attempt to avoid multi-use src if we don't need anything from it.
1068 if (!DemandedBits.isAllOnes() || !DemandedSubElts.isAllOnes() ||
1069 !DemandedSrcElts.isAllOnes()) {
1070 SDValue NewSub = SimplifyMultipleUseDemandedBits(
1071 Sub, DemandedBits, DemandedSubElts, TLO.DAG, Depth + 1);
1072 SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1073 Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);
1074 if (NewSub || NewSrc) {
1075 NewSub = NewSub ? NewSub : Sub;
1076 NewSrc = NewSrc ? NewSrc : Src;
1077 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc, NewSub,
1078 Op.getOperand(2));
1079 return TLO.CombineTo(Op, NewOp);
1080 }
1081 }
1082 break;
1083 }
1084 case ISD::EXTRACT_SUBVECTOR: {
1085 // Offset the demanded elts by the subvector index.
1086 SDValue Src = Op.getOperand(0);
1087 if (Src.getValueType().isScalableVector())
1088 break;
1089 uint64_t Idx = Op.getConstantOperandVal(1);
1090 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
1091 APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
1092
1093 if (SimplifyDemandedBits(Src, DemandedBits, DemandedSrcElts, Known, TLO,
1094 Depth + 1))
1095 return true;
1096
1097 // Attempt to avoid multi-use src if we don't need anything from it.
1098 if (!DemandedBits.isAllOnes() || !DemandedSrcElts.isAllOnes()) {
1099 SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
1100 Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);
1101 if (DemandedSrc) {
1102 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc,
1103 Op.getOperand(1));
1104 return TLO.CombineTo(Op, NewOp);
1105 }
1106 }
1107 break;
1108 }
1109 case ISD::CONCAT_VECTORS: {
1110 Known.Zero.setAllBits();
1111 Known.One.setAllBits();
1112 EVT SubVT = Op.getOperand(0).getValueType();
1113 unsigned NumSubVecs = Op.getNumOperands();
1114 unsigned NumSubElts = SubVT.getVectorNumElements();
1115 for (unsigned i = 0; i != NumSubVecs; ++i) {
1116 APInt DemandedSubElts =
1117 DemandedElts.extractBits(NumSubElts, i * NumSubElts);
1118 if (SimplifyDemandedBits(Op.getOperand(i), DemandedBits, DemandedSubElts,
1119 Known2, TLO, Depth + 1))
1120 return true;
1121 // Known bits are shared by every demanded subvector element.
1122 if (!!DemandedSubElts)
1123 Known = KnownBits::commonBits(Known, Known2);
1124 }
1125 break;
1126 }
1127 case ISD::VECTOR_SHUFFLE: {
1128 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
1129
1130 // Collect demanded elements from shuffle operands..
1131 APInt DemandedLHS(NumElts, 0);
1132 APInt DemandedRHS(NumElts, 0);
1133 for (unsigned i = 0; i != NumElts; ++i) {
1134 if (!DemandedElts[i])
1135 continue;
1136 int M = ShuffleMask[i];
1137 if (M < 0) {
1138 // For UNDEF elements, we don't know anything about the common state of
1139 // the shuffle result.
1140 DemandedLHS.clearAllBits();
1141 DemandedRHS.clearAllBits();
1142 break;
1143 }
1144 assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range")(static_cast <bool> (0 <= M && M < (int)(
2 * NumElts) && "Shuffle index out of range") ? void (
0) : __assert_fail ("0 <= M && M < (int)(2 * NumElts) && \"Shuffle index out of range\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1144, __extension__ __PRETTY_FUNCTION__))
;
1145 if (M < (int)NumElts)
1146 DemandedLHS.setBit(M);
1147 else
1148 DemandedRHS.setBit(M - NumElts);
1149 }
1150
1151 if (!!DemandedLHS || !!DemandedRHS) {
1152 SDValue Op0 = Op.getOperand(0);
1153 SDValue Op1 = Op.getOperand(1);
1154
1155 Known.Zero.setAllBits();
1156 Known.One.setAllBits();
1157 if (!!DemandedLHS) {
1158 if (SimplifyDemandedBits(Op0, DemandedBits, DemandedLHS, Known2, TLO,
1159 Depth + 1))
1160 return true;
1161 Known = KnownBits::commonBits(Known, Known2);
1162 }
1163 if (!!DemandedRHS) {
1164 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedRHS, Known2, TLO,
1165 Depth + 1))
1166 return true;
1167 Known = KnownBits::commonBits(Known, Known2);
1168 }
1169
1170 // Attempt to avoid multi-use ops if we don't need anything from them.
1171 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1172 Op0, DemandedBits, DemandedLHS, TLO.DAG, Depth + 1);
1173 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1174 Op1, DemandedBits, DemandedRHS, TLO.DAG, Depth + 1);
1175 if (DemandedOp0 || DemandedOp1) {
1176 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1177 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1178 SDValue NewOp = TLO.DAG.getVectorShuffle(VT, dl, Op0, Op1, ShuffleMask);
1179 return TLO.CombineTo(Op, NewOp);
1180 }
1181 }
1182 break;
1183 }
1184 case ISD::AND: {
1185 SDValue Op0 = Op.getOperand(0);
1186 SDValue Op1 = Op.getOperand(1);
1187
1188 // If the RHS is a constant, check to see if the LHS would be zero without
1189 // using the bits from the RHS. Below, we use knowledge about the RHS to
1190 // simplify the LHS, here we're using information from the LHS to simplify
1191 // the RHS.
1192 if (ConstantSDNode *RHSC = isConstOrConstSplat(Op1)) {
1193 // Do not increment Depth here; that can cause an infinite loop.
1194 KnownBits LHSKnown = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth);
1195 // If the LHS already has zeros where RHSC does, this 'and' is dead.
1196 if ((LHSKnown.Zero & DemandedBits) ==
1197 (~RHSC->getAPIntValue() & DemandedBits))
1198 return TLO.CombineTo(Op, Op0);
1199
1200 // If any of the set bits in the RHS are known zero on the LHS, shrink
1201 // the constant.
1202 if (ShrinkDemandedConstant(Op, ~LHSKnown.Zero & DemandedBits,
1203 DemandedElts, TLO))
1204 return true;
1205
1206 // Bitwise-not (xor X, -1) is a special case: we don't usually shrink its
1207 // constant, but if this 'and' is only clearing bits that were just set by
1208 // the xor, then this 'and' can be eliminated by shrinking the mask of
1209 // the xor. For example, for a 32-bit X:
1210 // and (xor (srl X, 31), -1), 1 --> xor (srl X, 31), 1
1211 if (isBitwiseNot(Op0) && Op0.hasOneUse() &&
1212 LHSKnown.One == ~RHSC->getAPIntValue()) {
1213 SDValue Xor = TLO.DAG.getNode(ISD::XOR, dl, VT, Op0.getOperand(0), Op1);
1214 return TLO.CombineTo(Op, Xor);
1215 }
1216 }
1217
1218 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1219 Depth + 1))
1220 return true;
1221 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1221, __extension__ __PRETTY_FUNCTION__))
;
1222 if (SimplifyDemandedBits(Op0, ~Known.Zero & DemandedBits, DemandedElts,
1223 Known2, TLO, Depth + 1))
1224 return true;
1225 assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1225, __extension__ __PRETTY_FUNCTION__))
;
1226
1227 // Attempt to avoid multi-use ops if we don't need anything from them.
1228 if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
1229 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1230 Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1231 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1232 Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1233 if (DemandedOp0 || DemandedOp1) {
1234 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1235 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1236 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1237 return TLO.CombineTo(Op, NewOp);
1238 }
1239 }
1240
1241 // If all of the demanded bits are known one on one side, return the other.
1242 // These bits cannot contribute to the result of the 'and'.
1243 if (DemandedBits.isSubsetOf(Known2.Zero | Known.One))
1244 return TLO.CombineTo(Op, Op0);
1245 if (DemandedBits.isSubsetOf(Known.Zero | Known2.One))
1246 return TLO.CombineTo(Op, Op1);
1247 // If all of the demanded bits in the inputs are known zeros, return zero.
1248 if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
1249 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, dl, VT));
1250 // If the RHS is a constant, see if we can simplify it.
1251 if (ShrinkDemandedConstant(Op, ~Known2.Zero & DemandedBits, DemandedElts,
1252 TLO))
1253 return true;
1254 // If the operation can be done in a smaller type, do so.
1255 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1256 return true;
1257
1258 Known &= Known2;
1259 break;
1260 }
1261 case ISD::OR: {
1262 SDValue Op0 = Op.getOperand(0);
1263 SDValue Op1 = Op.getOperand(1);
1264
1265 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1266 Depth + 1))
1267 return true;
1268 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1268, __extension__ __PRETTY_FUNCTION__))
;
1269 if (SimplifyDemandedBits(Op0, ~Known.One & DemandedBits, DemandedElts,
1270 Known2, TLO, Depth + 1))
1271 return true;
1272 assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1272, __extension__ __PRETTY_FUNCTION__))
;
1273
1274 // Attempt to avoid multi-use ops if we don't need anything from them.
1275 if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
1276 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1277 Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1278 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1279 Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1280 if (DemandedOp0 || DemandedOp1) {
1281 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1282 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1283 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1284 return TLO.CombineTo(Op, NewOp);
1285 }
1286 }
1287
1288 // If all of the demanded bits are known zero on one side, return the other.
1289 // These bits cannot contribute to the result of the 'or'.
1290 if (DemandedBits.isSubsetOf(Known2.One | Known.Zero))
1291 return TLO.CombineTo(Op, Op0);
1292 if (DemandedBits.isSubsetOf(Known.One | Known2.Zero))
1293 return TLO.CombineTo(Op, Op1);
1294 // If the RHS is a constant, see if we can simplify it.
1295 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1296 return true;
1297 // If the operation can be done in a smaller type, do so.
1298 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1299 return true;
1300
1301 Known |= Known2;
1302 break;
1303 }
1304 case ISD::XOR: {
1305 SDValue Op0 = Op.getOperand(0);
1306 SDValue Op1 = Op.getOperand(1);
1307
1308 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1309 Depth + 1))
1310 return true;
1311 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1311, __extension__ __PRETTY_FUNCTION__))
;
1312 if (SimplifyDemandedBits(Op0, DemandedBits, DemandedElts, Known2, TLO,
1313 Depth + 1))
1314 return true;
1315 assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1315, __extension__ __PRETTY_FUNCTION__))
;
1316
1317 // Attempt to avoid multi-use ops if we don't need anything from them.
1318 if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
1319 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1320 Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1321 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1322 Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1323 if (DemandedOp0 || DemandedOp1) {
1324 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1325 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1326 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1327 return TLO.CombineTo(Op, NewOp);
1328 }
1329 }
1330
1331 // If all of the demanded bits are known zero on one side, return the other.
1332 // These bits cannot contribute to the result of the 'xor'.
1333 if (DemandedBits.isSubsetOf(Known.Zero))
1334 return TLO.CombineTo(Op, Op0);
1335 if (DemandedBits.isSubsetOf(Known2.Zero))
1336 return TLO.CombineTo(Op, Op1);
1337 // If the operation can be done in a smaller type, do so.
1338 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1339 return true;
1340
1341 // If all of the unknown bits are known to be zero on one side or the other
1342 // turn this into an *inclusive* or.
1343 // e.g. (A & C1)^(B & C2) -> (A & C1)|(B & C2) iff C1&C2 == 0
1344 if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
1345 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::OR, dl, VT, Op0, Op1));
1346
1347 ConstantSDNode* C = isConstOrConstSplat(Op1, DemandedElts);
1348 if (C) {
1349 // If one side is a constant, and all of the set bits in the constant are
1350 // also known set on the other side, turn this into an AND, as we know
1351 // the bits will be cleared.
1352 // e.g. (X | C1) ^ C2 --> (X | C1) & ~C2 iff (C1&C2) == C2
1353 // NB: it is okay if more bits are known than are requested
1354 if (C->getAPIntValue() == Known2.One) {
1355 SDValue ANDC =
1356 TLO.DAG.getConstant(~C->getAPIntValue() & DemandedBits, dl, VT);
1357 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, dl, VT, Op0, ANDC));
1358 }
1359
1360 // If the RHS is a constant, see if we can change it. Don't alter a -1
1361 // constant because that's a 'not' op, and that is better for combining
1362 // and codegen.
1363 if (!C->isAllOnes() && DemandedBits.isSubsetOf(C->getAPIntValue())) {
1364 // We're flipping all demanded bits. Flip the undemanded bits too.
1365 SDValue New = TLO.DAG.getNOT(dl, Op0, VT);
1366 return TLO.CombineTo(Op, New);
1367 }
1368 }
1369
1370 // If we can't turn this into a 'not', try to shrink the constant.
1371 if (!C || !C->isAllOnes())
1372 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1373 return true;
1374
1375 Known ^= Known2;
1376 break;
1377 }
1378 case ISD::SELECT:
1379 if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known, TLO,
1380 Depth + 1))
1381 return true;
1382 if (SimplifyDemandedBits(Op.getOperand(1), DemandedBits, Known2, TLO,
1383 Depth + 1))
1384 return true;
1385 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1385, __extension__ __PRETTY_FUNCTION__))
;
1386 assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1386, __extension__ __PRETTY_FUNCTION__))
;
1387
1388 // If the operands are constants, see if we can simplify them.
1389 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1390 return true;
1391
1392 // Only known if known in both the LHS and RHS.
1393 Known = KnownBits::commonBits(Known, Known2);
1394 break;
1395 case ISD::SELECT_CC:
1396 if (SimplifyDemandedBits(Op.getOperand(3), DemandedBits, Known, TLO,
1397 Depth + 1))
1398 return true;
1399 if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known2, TLO,
1400 Depth + 1))
1401 return true;
1402 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1402, __extension__ __PRETTY_FUNCTION__))
;
1403 assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1403, __extension__ __PRETTY_FUNCTION__))
;
1404
1405 // If the operands are constants, see if we can simplify them.
1406 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1407 return true;
1408
1409 // Only known if known in both the LHS and RHS.
1410 Known = KnownBits::commonBits(Known, Known2);
1411 break;
1412 case ISD::SETCC: {
1413 SDValue Op0 = Op.getOperand(0);
1414 SDValue Op1 = Op.getOperand(1);
1415 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
1416 // If (1) we only need the sign-bit, (2) the setcc operands are the same
1417 // width as the setcc result, and (3) the result of a setcc conforms to 0 or
1418 // -1, we may be able to bypass the setcc.
1419 if (DemandedBits.isSignMask() &&
1420 Op0.getScalarValueSizeInBits() == BitWidth &&
1421 getBooleanContents(Op0.getValueType()) ==
1422 BooleanContent::ZeroOrNegativeOneBooleanContent) {
1423 // If we're testing X < 0, then this compare isn't needed - just use X!
1424 // FIXME: We're limiting to integer types here, but this should also work
1425 // if we don't care about FP signed-zero. The use of SETLT with FP means
1426 // that we don't care about NaNs.
1427 if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
1428 (isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode())))
1429 return TLO.CombineTo(Op, Op0);
1430
1431 // TODO: Should we check for other forms of sign-bit comparisons?
1432 // Examples: X <= -1, X >= 0
1433 }
1434 if (getBooleanContents(Op0.getValueType()) ==
1435 TargetLowering::ZeroOrOneBooleanContent &&
1436 BitWidth > 1)
1437 Known.Zero.setBitsFrom(1);
1438 break;
1439 }
1440 case ISD::SHL: {
1441 SDValue Op0 = Op.getOperand(0);
1442 SDValue Op1 = Op.getOperand(1);
1443 EVT ShiftVT = Op1.getValueType();
1444
1445 if (const APInt *SA =
1446 TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
1447 unsigned ShAmt = SA->getZExtValue();
1448 if (ShAmt == 0)
1449 return TLO.CombineTo(Op, Op0);
1450
1451 // If this is ((X >>u C1) << ShAmt), see if we can simplify this into a
1452 // single shift. We can do this if the bottom bits (which are shifted
1453 // out) are never demanded.
1454 // TODO - support non-uniform vector amounts.
1455 if (Op0.getOpcode() == ISD::SRL) {
1456 if (!DemandedBits.intersects(APInt::getLowBitsSet(BitWidth, ShAmt))) {
1457 if (const APInt *SA2 =
1458 TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {
1459 unsigned C1 = SA2->getZExtValue();
1460 unsigned Opc = ISD::SHL;
1461 int Diff = ShAmt - C1;
1462 if (Diff < 0) {
1463 Diff = -Diff;
1464 Opc = ISD::SRL;
1465 }
1466 SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);
1467 return TLO.CombineTo(
1468 Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));
1469 }
1470 }
1471 }
1472
1473 // Convert (shl (anyext x, c)) to (anyext (shl x, c)) if the high bits
1474 // are not demanded. This will likely allow the anyext to be folded away.
1475 // TODO - support non-uniform vector amounts.
1476 if (Op0.getOpcode() == ISD::ANY_EXTEND) {
1477 SDValue InnerOp = Op0.getOperand(0);
1478 EVT InnerVT = InnerOp.getValueType();
1479 unsigned InnerBits = InnerVT.getScalarSizeInBits();
1480 if (ShAmt < InnerBits && DemandedBits.getActiveBits() <= InnerBits &&
1481 isTypeDesirableForOp(ISD::SHL, InnerVT)) {
1482 EVT ShTy = getShiftAmountTy(InnerVT, DL);
1483 if (!APInt(BitWidth, ShAmt).isIntN(ShTy.getSizeInBits()))
1484 ShTy = InnerVT;
1485 SDValue NarrowShl =
1486 TLO.DAG.getNode(ISD::SHL, dl, InnerVT, InnerOp,
1487 TLO.DAG.getConstant(ShAmt, dl, ShTy));
1488 return TLO.CombineTo(
1489 Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, NarrowShl));
1490 }
1491
1492 // Repeat the SHL optimization above in cases where an extension
1493 // intervenes: (shl (anyext (shr x, c1)), c2) to
1494 // (shl (anyext x), c2-c1). This requires that the bottom c1 bits
1495 // aren't demanded (as above) and that the shifted upper c1 bits of
1496 // x aren't demanded.
1497 // TODO - support non-uniform vector amounts.
1498 if (Op0.hasOneUse() && InnerOp.getOpcode() == ISD::SRL &&
1499 InnerOp.hasOneUse()) {
1500 if (const APInt *SA2 =
1501 TLO.DAG.getValidShiftAmountConstant(InnerOp, DemandedElts)) {
1502 unsigned InnerShAmt = SA2->getZExtValue();
1503 if (InnerShAmt < ShAmt && InnerShAmt < InnerBits &&
1504 DemandedBits.getActiveBits() <=
1505 (InnerBits - InnerShAmt + ShAmt) &&
1506 DemandedBits.countTrailingZeros() >= ShAmt) {
1507 SDValue NewSA =
1508 TLO.DAG.getConstant(ShAmt - InnerShAmt, dl, ShiftVT);
1509 SDValue NewExt = TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT,
1510 InnerOp.getOperand(0));
1511 return TLO.CombineTo(
1512 Op, TLO.DAG.getNode(ISD::SHL, dl, VT, NewExt, NewSA));
1513 }
1514 }
1515 }
1516 }
1517
1518 APInt InDemandedMask = DemandedBits.lshr(ShAmt);
1519 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1520 Depth + 1))
1521 return true;
1522 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1522, __extension__ __PRETTY_FUNCTION__))
;
1523 Known.Zero <<= ShAmt;
1524 Known.One <<= ShAmt;
1525 // low bits known zero.
1526 Known.Zero.setLowBits(ShAmt);
1527
1528 // Try shrinking the operation as long as the shift amount will still be
1529 // in range.
1530 if ((ShAmt < DemandedBits.getActiveBits()) &&
1531 ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1532 return true;
1533 }
1534
1535 // If we are only demanding sign bits then we can use the shift source
1536 // directly.
1537 if (const APInt *MaxSA =
1538 TLO.DAG.getValidMaximumShiftAmountConstant(Op, DemandedElts)) {
1539 unsigned ShAmt = MaxSA->getZExtValue();
1540 unsigned NumSignBits =
1541 TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
1542 unsigned UpperDemandedBits = BitWidth - DemandedBits.countTrailingZeros();
1543 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
1544 return TLO.CombineTo(Op, Op0);
1545 }
1546 break;
1547 }
1548 case ISD::SRL: {
1549 SDValue Op0 = Op.getOperand(0);
1550 SDValue Op1 = Op.getOperand(1);
1551 EVT ShiftVT = Op1.getValueType();
1552
1553 if (const APInt *SA =
1554 TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
1555 unsigned ShAmt = SA->getZExtValue();
1556 if (ShAmt == 0)
1557 return TLO.CombineTo(Op, Op0);
1558
1559 // If this is ((X << C1) >>u ShAmt), see if we can simplify this into a
1560 // single shift. We can do this if the top bits (which are shifted out)
1561 // are never demanded.
1562 // TODO - support non-uniform vector amounts.
1563 if (Op0.getOpcode() == ISD::SHL) {
1564 if (!DemandedBits.intersects(APInt::getHighBitsSet(BitWidth, ShAmt))) {
1565 if (const APInt *SA2 =
1566 TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {
1567 unsigned C1 = SA2->getZExtValue();
1568 unsigned Opc = ISD::SRL;
1569 int Diff = ShAmt - C1;
1570 if (Diff < 0) {
1571 Diff = -Diff;
1572 Opc = ISD::SHL;
1573 }
1574 SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);
1575 return TLO.CombineTo(
1576 Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));
1577 }
1578 }
1579 }
1580
1581 APInt InDemandedMask = (DemandedBits << ShAmt);
1582
1583 // If the shift is exact, then it does demand the low bits (and knows that
1584 // they are zero).
1585 if (Op->getFlags().hasExact())
1586 InDemandedMask.setLowBits(ShAmt);
1587
1588 // Compute the new bits that are at the top now.
1589 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1590 Depth + 1))
1591 return true;
1592 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1592, __extension__ __PRETTY_FUNCTION__))
;
1593 Known.Zero.lshrInPlace(ShAmt);
1594 Known.One.lshrInPlace(ShAmt);
1595 // High bits known zero.
1596 Known.Zero.setHighBits(ShAmt);
1597 }
1598 break;
1599 }
1600 case ISD::SRA: {
1601 SDValue Op0 = Op.getOperand(0);
1602 SDValue Op1 = Op.getOperand(1);
1603 EVT ShiftVT = Op1.getValueType();
1604
1605 // If we only want bits that already match the signbit then we don't need
1606 // to shift.
1607 unsigned NumHiDemandedBits = BitWidth - DemandedBits.countTrailingZeros();
1608 if (TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1) >=
1609 NumHiDemandedBits)
1610 return TLO.CombineTo(Op, Op0);
1611
1612 // If this is an arithmetic shift right and only the low-bit is set, we can
1613 // always convert this into a logical shr, even if the shift amount is
1614 // variable. The low bit of the shift cannot be an input sign bit unless
1615 // the shift amount is >= the size of the datatype, which is undefined.
1616 if (DemandedBits.isOne())
1617 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1));
1618
1619 if (const APInt *SA =
1620 TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
1621 unsigned ShAmt = SA->getZExtValue();
1622 if (ShAmt == 0)
1623 return TLO.CombineTo(Op, Op0);
1624
1625 APInt InDemandedMask = (DemandedBits << ShAmt);
1626
1627 // If the shift is exact, then it does demand the low bits (and knows that
1628 // they are zero).
1629 if (Op->getFlags().hasExact())
1630 InDemandedMask.setLowBits(ShAmt);
1631
1632 // If any of the demanded bits are produced by the sign extension, we also
1633 // demand the input sign bit.
1634 if (DemandedBits.countLeadingZeros() < ShAmt)
1635 InDemandedMask.setSignBit();
1636
1637 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1638 Depth + 1))
1639 return true;
1640 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1640, __extension__ __PRETTY_FUNCTION__))
;
1641 Known.Zero.lshrInPlace(ShAmt);
1642 Known.One.lshrInPlace(ShAmt);
1643
1644 // If the input sign bit is known to be zero, or if none of the top bits
1645 // are demanded, turn this into an unsigned shift right.
1646 if (Known.Zero[BitWidth - ShAmt - 1] ||
1647 DemandedBits.countLeadingZeros() >= ShAmt) {
1648 SDNodeFlags Flags;
1649 Flags.setExact(Op->getFlags().hasExact());
1650 return TLO.CombineTo(
1651 Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1, Flags));
1652 }
1653
1654 int Log2 = DemandedBits.exactLogBase2();
1655 if (Log2 >= 0) {
1656 // The bit must come from the sign.
1657 SDValue NewSA = TLO.DAG.getConstant(BitWidth - 1 - Log2, dl, ShiftVT);
1658 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, NewSA));
1659 }
1660
1661 if (Known.One[BitWidth - ShAmt - 1])
1662 // New bits are known one.
1663 Known.One.setHighBits(ShAmt);
1664
1665 // Attempt to avoid multi-use ops if we don't need anything from them.
1666 if (!InDemandedMask.isAllOnes() || !DemandedElts.isAllOnes()) {
1667 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1668 Op0, InDemandedMask, DemandedElts, TLO.DAG, Depth + 1);
1669 if (DemandedOp0) {
1670 SDValue NewOp = TLO.DAG.getNode(ISD::SRA, dl, VT, DemandedOp0, Op1);
1671 return TLO.CombineTo(Op, NewOp);
1672 }
1673 }
1674 }
1675 break;
1676 }
1677 case ISD::FSHL:
1678 case ISD::FSHR: {
1679 SDValue Op0 = Op.getOperand(0);
1680 SDValue Op1 = Op.getOperand(1);
1681 SDValue Op2 = Op.getOperand(2);
1682 bool IsFSHL = (Op.getOpcode() == ISD::FSHL);
1683
1684 if (ConstantSDNode *SA = isConstOrConstSplat(Op2, DemandedElts)) {
1685 unsigned Amt = SA->getAPIntValue().urem(BitWidth);
1686
1687 // For fshl, 0-shift returns the 1st arg.
1688 // For fshr, 0-shift returns the 2nd arg.
1689 if (Amt == 0) {
1690 if (SimplifyDemandedBits(IsFSHL ? Op0 : Op1, DemandedBits, DemandedElts,
1691 Known, TLO, Depth + 1))
1692 return true;
1693 break;
1694 }
1695
1696 // fshl: (Op0 << Amt) | (Op1 >> (BW - Amt))
1697 // fshr: (Op0 << (BW - Amt)) | (Op1 >> Amt)
1698 APInt Demanded0 = DemandedBits.lshr(IsFSHL ? Amt : (BitWidth - Amt));
1699 APInt Demanded1 = DemandedBits << (IsFSHL ? (BitWidth - Amt) : Amt);
1700 if (SimplifyDemandedBits(Op0, Demanded0, DemandedElts, Known2, TLO,
1701 Depth + 1))
1702 return true;
1703 if (SimplifyDemandedBits(Op1, Demanded1, DemandedElts, Known, TLO,
1704 Depth + 1))
1705 return true;
1706
1707 Known2.One <<= (IsFSHL ? Amt : (BitWidth - Amt));
1708 Known2.Zero <<= (IsFSHL ? Amt : (BitWidth - Amt));
1709 Known.One.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);
1710 Known.Zero.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);
1711 Known.One |= Known2.One;
1712 Known.Zero |= Known2.Zero;
1713 }
1714
1715 // For pow-2 bitwidths we only demand the bottom modulo amt bits.
1716 if (isPowerOf2_32(BitWidth)) {
1717 APInt DemandedAmtBits(Op2.getScalarValueSizeInBits(), BitWidth - 1);
1718 if (SimplifyDemandedBits(Op2, DemandedAmtBits, DemandedElts,
1719 Known2, TLO, Depth + 1))
1720 return true;
1721 }
1722 break;
1723 }
1724 case ISD::ROTL:
1725 case ISD::ROTR: {
1726 SDValue Op0 = Op.getOperand(0);
1727 SDValue Op1 = Op.getOperand(1);
1728
1729 // If we're rotating an 0/-1 value, then it stays an 0/-1 value.
1730 if (BitWidth == TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1))
1731 return TLO.CombineTo(Op, Op0);
1732
1733 // For pow-2 bitwidths we only demand the bottom modulo amt bits.
1734 if (isPowerOf2_32(BitWidth)) {
1735 APInt DemandedAmtBits(Op1.getScalarValueSizeInBits(), BitWidth - 1);
1736 if (SimplifyDemandedBits(Op1, DemandedAmtBits, DemandedElts, Known2, TLO,
1737 Depth + 1))
1738 return true;
1739 }
1740 break;
1741 }
1742 case ISD::UMIN: {
1743 // Check if one arg is always less than (or equal) to the other arg.
1744 SDValue Op0 = Op.getOperand(0);
1745 SDValue Op1 = Op.getOperand(1);
1746 KnownBits Known0 = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth + 1);
1747 KnownBits Known1 = TLO.DAG.computeKnownBits(Op1, DemandedElts, Depth + 1);
1748 Known = KnownBits::umin(Known0, Known1);
1749 if (Optional<bool> IsULE = KnownBits::ule(Known0, Known1))
1750 return TLO.CombineTo(Op, IsULE.getValue() ? Op0 : Op1);
1751 if (Optional<bool> IsULT = KnownBits::ult(Known0, Known1))
1752 return TLO.CombineTo(Op, IsULT.getValue() ? Op0 : Op1);
1753 break;
1754 }
1755 case ISD::UMAX: {
1756 // Check if one arg is always greater than (or equal) to the other arg.
1757 SDValue Op0 = Op.getOperand(0);
1758 SDValue Op1 = Op.getOperand(1);
1759 KnownBits Known0 = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth + 1);
1760 KnownBits Known1 = TLO.DAG.computeKnownBits(Op1, DemandedElts, Depth + 1);
1761 Known = KnownBits::umax(Known0, Known1);
1762 if (Optional<bool> IsUGE = KnownBits::uge(Known0, Known1))
1763 return TLO.CombineTo(Op, IsUGE.getValue() ? Op0 : Op1);
1764 if (Optional<bool> IsUGT = KnownBits::ugt(Known0, Known1))
1765 return TLO.CombineTo(Op, IsUGT.getValue() ? Op0 : Op1);
1766 break;
1767 }
1768 case ISD::BITREVERSE: {
1769 SDValue Src = Op.getOperand(0);
1770 APInt DemandedSrcBits = DemandedBits.reverseBits();
1771 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
1772 Depth + 1))
1773 return true;
1774 Known.One = Known2.One.reverseBits();
1775 Known.Zero = Known2.Zero.reverseBits();
1776 break;
1777 }
1778 case ISD::BSWAP: {
1779 SDValue Src = Op.getOperand(0);
1780 APInt DemandedSrcBits = DemandedBits.byteSwap();
1781 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
1782 Depth + 1))
1783 return true;
1784 Known.One = Known2.One.byteSwap();
1785 Known.Zero = Known2.Zero.byteSwap();
1786 break;
1787 }
1788 case ISD::CTPOP: {
1789 // If only 1 bit is demanded, replace with PARITY as long as we're before
1790 // op legalization.
1791 // FIXME: Limit to scalars for now.
1792 if (DemandedBits.isOne() && !TLO.LegalOps && !VT.isVector())
1793 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::PARITY, dl, VT,
1794 Op.getOperand(0)));
1795
1796 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
1797 break;
1798 }
1799 case ISD::SIGN_EXTEND_INREG: {
1800 SDValue Op0 = Op.getOperand(0);
1801 EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
1802 unsigned ExVTBits = ExVT.getScalarSizeInBits();
1803
1804 // If we only care about the highest bit, don't bother shifting right.
1805 if (DemandedBits.isSignMask()) {
1806 unsigned NumSignBits =
1807 TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
1808 bool AlreadySignExtended = NumSignBits >= BitWidth - ExVTBits + 1;
1809 // However if the input is already sign extended we expect the sign
1810 // extension to be dropped altogether later and do not simplify.
1811 if (!AlreadySignExtended) {
1812 // Compute the correct shift amount type, which must be getShiftAmountTy
1813 // for scalar types after legalization.
1814 EVT ShiftAmtTy = VT;
1815 if (TLO.LegalTypes() && !ShiftAmtTy.isVector())
1816 ShiftAmtTy = getShiftAmountTy(ShiftAmtTy, DL);
1817
1818 SDValue ShiftAmt =
1819 TLO.DAG.getConstant(BitWidth - ExVTBits, dl, ShiftAmtTy);
1820 return TLO.CombineTo(Op,
1821 TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, ShiftAmt));
1822 }
1823 }
1824
1825 // If none of the extended bits are demanded, eliminate the sextinreg.
1826 if (DemandedBits.getActiveBits() <= ExVTBits)
1827 return TLO.CombineTo(Op, Op0);
1828
1829 APInt InputDemandedBits = DemandedBits.getLoBits(ExVTBits);
1830
1831 // Since the sign extended bits are demanded, we know that the sign
1832 // bit is demanded.
1833 InputDemandedBits.setBit(ExVTBits - 1);
1834
1835 if (SimplifyDemandedBits(Op0, InputDemandedBits, Known, TLO, Depth + 1))
1836 return true;
1837 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1837, __extension__ __PRETTY_FUNCTION__))
;
1838
1839 // If the sign bit of the input is known set or clear, then we know the
1840 // top bits of the result.
1841
1842 // If the input sign bit is known zero, convert this into a zero extension.
1843 if (Known.Zero[ExVTBits - 1])
1844 return TLO.CombineTo(Op, TLO.DAG.getZeroExtendInReg(Op0, dl, ExVT));
1845
1846 APInt Mask = APInt::getLowBitsSet(BitWidth, ExVTBits);
1847 if (Known.One[ExVTBits - 1]) { // Input sign bit known set
1848 Known.One.setBitsFrom(ExVTBits);
1849 Known.Zero &= Mask;
1850 } else { // Input sign bit unknown
1851 Known.Zero &= Mask;
1852 Known.One &= Mask;
1853 }
1854 break;
1855 }
1856 case ISD::BUILD_PAIR: {
1857 EVT HalfVT = Op.getOperand(0).getValueType();
1858 unsigned HalfBitWidth = HalfVT.getScalarSizeInBits();
1859
1860 APInt MaskLo = DemandedBits.getLoBits(HalfBitWidth).trunc(HalfBitWidth);
1861 APInt MaskHi = DemandedBits.getHiBits(HalfBitWidth).trunc(HalfBitWidth);
1862
1863 KnownBits KnownLo, KnownHi;
1864
1865 if (SimplifyDemandedBits(Op.getOperand(0), MaskLo, KnownLo, TLO, Depth + 1))
1866 return true;
1867
1868 if (SimplifyDemandedBits(Op.getOperand(1), MaskHi, KnownHi, TLO, Depth + 1))
1869 return true;
1870
1871 Known.Zero = KnownLo.Zero.zext(BitWidth) |
1872 KnownHi.Zero.zext(BitWidth).shl(HalfBitWidth);
1873
1874 Known.One = KnownLo.One.zext(BitWidth) |
1875 KnownHi.One.zext(BitWidth).shl(HalfBitWidth);
1876 break;
1877 }
1878 case ISD::ZERO_EXTEND:
1879 case ISD::ZERO_EXTEND_VECTOR_INREG: {
1880 SDValue Src = Op.getOperand(0);
1881 EVT SrcVT = Src.getValueType();
1882 unsigned InBits = SrcVT.getScalarSizeInBits();
1883 unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
1884 bool IsVecInReg = Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG;
1885
1886 // If none of the top bits are demanded, convert this into an any_extend.
1887 if (DemandedBits.getActiveBits() <= InBits) {
1888 // If we only need the non-extended bits of the bottom element
1889 // then we can just bitcast to the result.
1890 if (IsVecInReg && DemandedElts == 1 &&
1891 VT.getSizeInBits() == SrcVT.getSizeInBits() &&
1892 TLO.DAG.getDataLayout().isLittleEndian())
1893 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
1894
1895 unsigned Opc =
1896 IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;
1897 if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
1898 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
1899 }
1900
1901 APInt InDemandedBits = DemandedBits.trunc(InBits);
1902 APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
1903 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
1904 Depth + 1))
1905 return true;
1906 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1906, __extension__ __PRETTY_FUNCTION__))
;
1907 assert(Known.getBitWidth() == InBits && "Src width has changed?")(static_cast <bool> (Known.getBitWidth() == InBits &&
"Src width has changed?") ? void (0) : __assert_fail ("Known.getBitWidth() == InBits && \"Src width has changed?\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1907, __extension__ __PRETTY_FUNCTION__))
;
1908 Known = Known.zext(BitWidth);
1909
1910 // Attempt to avoid multi-use ops if we don't need anything from them.
1911 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1912 Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
1913 return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
1914 break;
1915 }
1916 case ISD::SIGN_EXTEND:
1917 case ISD::SIGN_EXTEND_VECTOR_INREG: {
1918 SDValue Src = Op.getOperand(0);
1919 EVT SrcVT = Src.getValueType();
1920 unsigned InBits = SrcVT.getScalarSizeInBits();
1921 unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
1922 bool IsVecInReg = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG;
1923
1924 // If none of the top bits are demanded, convert this into an any_extend.
1925 if (DemandedBits.getActiveBits() <= InBits) {
1926 // If we only need the non-extended bits of the bottom element
1927 // then we can just bitcast to the result.
1928 if (IsVecInReg && DemandedElts == 1 &&
1929 VT.getSizeInBits() == SrcVT.getSizeInBits() &&
1930 TLO.DAG.getDataLayout().isLittleEndian())
1931 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
1932
1933 unsigned Opc =
1934 IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;
1935 if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
1936 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
1937 }
1938
1939 APInt InDemandedBits = DemandedBits.trunc(InBits);
1940 APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
1941
1942 // Since some of the sign extended bits are demanded, we know that the sign
1943 // bit is demanded.
1944 InDemandedBits.setBit(InBits - 1);
1945
1946 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
1947 Depth + 1))
1948 return true;
1949 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1949, __extension__ __PRETTY_FUNCTION__))
;
1950 assert(Known.getBitWidth() == InBits && "Src width has changed?")(static_cast <bool> (Known.getBitWidth() == InBits &&
"Src width has changed?") ? void (0) : __assert_fail ("Known.getBitWidth() == InBits && \"Src width has changed?\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1950, __extension__ __PRETTY_FUNCTION__))
;
1951
1952 // If the sign bit is known one, the top bits match.
1953 Known = Known.sext(BitWidth);
1954
1955 // If the sign bit is known zero, convert this to a zero extend.
1956 if (Known.isNonNegative()) {
1957 unsigned Opc =
1958 IsVecInReg ? ISD::ZERO_EXTEND_VECTOR_INREG : ISD::ZERO_EXTEND;
1959 if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
1960 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
1961 }
1962
1963 // Attempt to avoid multi-use ops if we don't need anything from them.
1964 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1965 Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
1966 return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
1967 break;
1968 }
1969 case ISD::ANY_EXTEND:
1970 case ISD::ANY_EXTEND_VECTOR_INREG: {
1971 SDValue Src = Op.getOperand(0);
1972 EVT SrcVT = Src.getValueType();
1973 unsigned InBits = SrcVT.getScalarSizeInBits();
1974 unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
1975 bool IsVecInReg = Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG;
1976
1977 // If we only need the bottom element then we can just bitcast.
1978 // TODO: Handle ANY_EXTEND?
1979 if (IsVecInReg && DemandedElts == 1 &&
1980 VT.getSizeInBits() == SrcVT.getSizeInBits() &&
1981 TLO.DAG.getDataLayout().isLittleEndian())
1982 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
1983
1984 APInt InDemandedBits = DemandedBits.trunc(InBits);
1985 APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
1986 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
1987 Depth + 1))
1988 return true;
1989 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1989, __extension__ __PRETTY_FUNCTION__))
;
1990 assert(Known.getBitWidth() == InBits && "Src width has changed?")(static_cast <bool> (Known.getBitWidth() == InBits &&
"Src width has changed?") ? void (0) : __assert_fail ("Known.getBitWidth() == InBits && \"Src width has changed?\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1990, __extension__ __PRETTY_FUNCTION__))
;
1991 Known = Known.anyext(BitWidth);
1992
1993 // Attempt to avoid multi-use ops if we don't need anything from them.
1994 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1995 Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
1996 return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
1997 break;
1998 }
1999 case ISD::TRUNCATE: {
2000 SDValue Src = Op.getOperand(0);
2001
2002 // Simplify the input, using demanded bit information, and compute the known
2003 // zero/one bits live out.
2004 unsigned OperandBitWidth = Src.getScalarValueSizeInBits();
2005 APInt TruncMask = DemandedBits.zext(OperandBitWidth);
2006 if (SimplifyDemandedBits(Src, TruncMask, DemandedElts, Known, TLO,
2007 Depth + 1))
2008 return true;
2009 Known = Known.trunc(BitWidth);
2010
2011 // Attempt to avoid multi-use ops if we don't need anything from them.
2012 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2013 Src, TruncMask, DemandedElts, TLO.DAG, Depth + 1))
2014 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, NewSrc));
2015
2016 // If the input is only used by this truncate, see if we can shrink it based
2017 // on the known demanded bits.
2018 if (Src.getNode()->hasOneUse()) {
2019 switch (Src.getOpcode()) {
2020 default:
2021 break;
2022 case ISD::SRL:
2023 // Shrink SRL by a constant if none of the high bits shifted in are
2024 // demanded.
2025 if (TLO.LegalTypes() && !isTypeDesirableForOp(ISD::SRL, VT))
2026 // Do not turn (vt1 truncate (vt2 srl)) into (vt1 srl) if vt1 is
2027 // undesirable.
2028 break;
2029
2030 const APInt *ShAmtC =
2031 TLO.DAG.getValidShiftAmountConstant(Src, DemandedElts);
2032 if (!ShAmtC || ShAmtC->uge(BitWidth))
2033 break;
2034 uint64_t ShVal = ShAmtC->getZExtValue();
2035
2036 APInt HighBits =
2037 APInt::getHighBitsSet(OperandBitWidth, OperandBitWidth - BitWidth);
2038 HighBits.lshrInPlace(ShVal);
2039 HighBits = HighBits.trunc(BitWidth);
2040
2041 if (!(HighBits & DemandedBits)) {
2042 // None of the shifted in bits are needed. Add a truncate of the
2043 // shift input, then shift it.
2044 SDValue NewShAmt = TLO.DAG.getConstant(
2045 ShVal, dl, getShiftAmountTy(VT, DL, TLO.LegalTypes()));
2046 SDValue NewTrunc =
2047 TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, Src.getOperand(0));
2048 return TLO.CombineTo(
2049 Op, TLO.DAG.getNode(ISD::SRL, dl, VT, NewTrunc, NewShAmt));
2050 }
2051 break;
2052 }
2053 }
2054
2055 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2055, __extension__ __PRETTY_FUNCTION__))
;
2056 break;
2057 }
2058 case ISD::AssertZext: {
2059 // AssertZext demands all of the high bits, plus any of the low bits
2060 // demanded by its users.
2061 EVT ZVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2062 APInt InMask = APInt::getLowBitsSet(BitWidth, ZVT.getSizeInBits());
2063 if (SimplifyDemandedBits(Op.getOperand(0), ~InMask | DemandedBits, Known,
2064 TLO, Depth + 1))
2065 return true;
2066 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2066, __extension__ __PRETTY_FUNCTION__))
;
2067
2068 Known.Zero |= ~InMask;
2069 break;
2070 }
2071 case ISD::EXTRACT_VECTOR_ELT: {
2072 SDValue Src = Op.getOperand(0);
2073 SDValue Idx = Op.getOperand(1);
2074 ElementCount SrcEltCnt = Src.getValueType().getVectorElementCount();
2075 unsigned EltBitWidth = Src.getScalarValueSizeInBits();
2076
2077 if (SrcEltCnt.isScalable())
2078 return false;
2079
2080 // Demand the bits from every vector element without a constant index.
2081 unsigned NumSrcElts = SrcEltCnt.getFixedValue();
2082 APInt DemandedSrcElts = APInt::getAllOnes(NumSrcElts);
2083 if (auto *CIdx = dyn_cast<ConstantSDNode>(Idx))
2084 if (CIdx->getAPIntValue().ult(NumSrcElts))
2085 DemandedSrcElts = APInt::getOneBitSet(NumSrcElts, CIdx->getZExtValue());
2086
2087 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
2088 // anything about the extended bits.
2089 APInt DemandedSrcBits = DemandedBits;
2090 if (BitWidth > EltBitWidth)
2091 DemandedSrcBits = DemandedSrcBits.trunc(EltBitWidth);
2092
2093 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts, Known2, TLO,
2094 Depth + 1))
2095 return true;
2096
2097 // Attempt to avoid multi-use ops if we don't need anything from them.
2098 if (!DemandedSrcBits.isAllOnes() || !DemandedSrcElts.isAllOnes()) {
2099 if (SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
2100 Src, DemandedSrcBits, DemandedSrcElts, TLO.DAG, Depth + 1)) {
2101 SDValue NewOp =
2102 TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc, Idx);
2103 return TLO.CombineTo(Op, NewOp);
2104 }
2105 }
2106
2107 Known = Known2;
2108 if (BitWidth > EltBitWidth)
2109 Known = Known.anyext(BitWidth);
2110 break;
2111 }
2112 case ISD::BITCAST: {
2113 SDValue Src = Op.getOperand(0);
2114 EVT SrcVT = Src.getValueType();
2115 unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
2116
2117 // If this is an FP->Int bitcast and if the sign bit is the only
2118 // thing demanded, turn this into a FGETSIGN.
2119 if (!TLO.LegalOperations() && !VT.isVector() && !SrcVT.isVector() &&
2120 DemandedBits == APInt::getSignMask(Op.getValueSizeInBits()) &&
2121 SrcVT.isFloatingPoint()) {
2122 bool OpVTLegal = isOperationLegalOrCustom(ISD::FGETSIGN, VT);
2123 bool i32Legal = isOperationLegalOrCustom(ISD::FGETSIGN, MVT::i32);
2124 if ((OpVTLegal || i32Legal) && VT.isSimple() && SrcVT != MVT::f16 &&
2125 SrcVT != MVT::f128) {
2126 // Cannot eliminate/lower SHL for f128 yet.
2127 EVT Ty = OpVTLegal ? VT : MVT::i32;
2128 // Make a FGETSIGN + SHL to move the sign bit into the appropriate
2129 // place. We expect the SHL to be eliminated by other optimizations.
2130 SDValue Sign = TLO.DAG.getNode(ISD::FGETSIGN, dl, Ty, Src);
2131 unsigned OpVTSizeInBits = Op.getValueSizeInBits();
2132 if (!OpVTLegal && OpVTSizeInBits > 32)
2133 Sign = TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Sign);
2134 unsigned ShVal = Op.getValueSizeInBits() - 1;
2135 SDValue ShAmt = TLO.DAG.getConstant(ShVal, dl, VT);
2136 return TLO.CombineTo(Op,
2137 TLO.DAG.getNode(ISD::SHL, dl, VT, Sign, ShAmt));
2138 }
2139 }
2140
2141 // Bitcast from a vector using SimplifyDemanded Bits/VectorElts.
2142 // Demand the elt/bit if any of the original elts/bits are demanded.
2143 // TODO - bigendian once we have test coverage.
2144 if (SrcVT.isVector() && (BitWidth % NumSrcEltBits) == 0 &&
2145 TLO.DAG.getDataLayout().isLittleEndian()) {
2146 unsigned Scale = BitWidth / NumSrcEltBits;
2147 unsigned NumSrcElts = SrcVT.getVectorNumElements();
2148 APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);
2149 APInt DemandedSrcElts = APInt::getZero(NumSrcElts);
2150 for (unsigned i = 0; i != Scale; ++i) {
2151 unsigned Offset = i * NumSrcEltBits;
2152 APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset);
2153 if (!Sub.isZero()) {
2154 DemandedSrcBits |= Sub;
2155 for (unsigned j = 0; j != NumElts; ++j)
2156 if (DemandedElts[j])
2157 DemandedSrcElts.setBit((j * Scale) + i);
2158 }
2159 }
2160
2161 APInt KnownSrcUndef, KnownSrcZero;
2162 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
2163 KnownSrcZero, TLO, Depth + 1))
2164 return true;
2165
2166 KnownBits KnownSrcBits;
2167 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
2168 KnownSrcBits, TLO, Depth + 1))
2169 return true;
2170 } else if ((NumSrcEltBits % BitWidth) == 0 &&
2171 TLO.DAG.getDataLayout().isLittleEndian()) {
2172 unsigned Scale = NumSrcEltBits / BitWidth;
2173 unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
2174 APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);
2175 APInt DemandedSrcElts = APInt::getZero(NumSrcElts);
2176 for (unsigned i = 0; i != NumElts; ++i)
2177 if (DemandedElts[i]) {
2178 unsigned Offset = (i % Scale) * BitWidth;
2179 DemandedSrcBits.insertBits(DemandedBits, Offset);
2180 DemandedSrcElts.setBit(i / Scale);
2181 }
2182
2183 if (SrcVT.isVector()) {
2184 APInt KnownSrcUndef, KnownSrcZero;
2185 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
2186 KnownSrcZero, TLO, Depth + 1))
2187 return true;
2188 }
2189
2190 KnownBits KnownSrcBits;
2191 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
2192 KnownSrcBits, TLO, Depth + 1))
2193 return true;
2194 }
2195
2196 // If this is a bitcast, let computeKnownBits handle it. Only do this on a
2197 // recursive call where Known may be useful to the caller.
2198 if (Depth > 0) {
2199 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2200 return false;
2201 }
2202 break;
2203 }
2204 case ISD::ADD:
2205 case ISD::MUL:
2206 case ISD::SUB: {
2207 // Add, Sub, and Mul don't demand any bits in positions beyond that
2208 // of the highest bit demanded of them.
2209 SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1);
2210 SDNodeFlags Flags = Op.getNode()->getFlags();
2211 unsigned DemandedBitsLZ = DemandedBits.countLeadingZeros();
2212 APInt LoMask = APInt::getLowBitsSet(BitWidth, BitWidth - DemandedBitsLZ);
2213 if (SimplifyDemandedBits(Op0, LoMask, DemandedElts, Known2, TLO,
2214 Depth + 1) ||
2215 SimplifyDemandedBits(Op1, LoMask, DemandedElts, Known2, TLO,
2216 Depth + 1) ||
2217 // See if the operation should be performed at a smaller bit width.
2218 ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO)) {
2219 if (Flags.hasNoSignedWrap() || Flags.hasNoUnsignedWrap()) {
2220 // Disable the nsw and nuw flags. We can no longer guarantee that we
2221 // won't wrap after simplification.
2222 Flags.setNoSignedWrap(false);
2223 Flags.setNoUnsignedWrap(false);
2224 SDValue NewOp =
2225 TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);
2226 return TLO.CombineTo(Op, NewOp);
2227 }
2228 return true;
2229 }
2230
2231 // Attempt to avoid multi-use ops if we don't need anything from them.
2232 if (!LoMask.isAllOnes() || !DemandedElts.isAllOnes()) {
2233 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2234 Op0, LoMask, DemandedElts, TLO.DAG, Depth + 1);
2235 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
2236 Op1, LoMask, DemandedElts, TLO.DAG, Depth + 1);
2237 if (DemandedOp0 || DemandedOp1) {
2238 Flags.setNoSignedWrap(false);
2239 Flags.setNoUnsignedWrap(false);
2240 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
2241 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
2242 SDValue NewOp =
2243 TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);
2244 return TLO.CombineTo(Op, NewOp);
2245 }
2246 }
2247
2248 // If we have a constant operand, we may be able to turn it into -1 if we
2249 // do not demand the high bits. This can make the constant smaller to
2250 // encode, allow more general folding, or match specialized instruction
2251 // patterns (eg, 'blsr' on x86). Don't bother changing 1 to -1 because that
2252 // is probably not useful (and could be detrimental).
2253 ConstantSDNode *C = isConstOrConstSplat(Op1);
2254 APInt HighMask = APInt::getHighBitsSet(BitWidth, DemandedBitsLZ);
2255 if (C && !C->isAllOnes() && !C->isOne() &&
2256 (C->getAPIntValue() | HighMask).isAllOnes()) {
2257 SDValue Neg1 = TLO.DAG.getAllOnesConstant(dl, VT);
2258 // Disable the nsw and nuw flags. We can no longer guarantee that we
2259 // won't wrap after simplification.
2260 Flags.setNoSignedWrap(false);
2261 Flags.setNoUnsignedWrap(false);
2262 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Neg1, Flags);
2263 return TLO.CombineTo(Op, NewOp);
2264 }
2265
2266 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2267 }
2268 default:
2269 if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
2270 if (SimplifyDemandedBitsForTargetNode(Op, DemandedBits, DemandedElts,
2271 Known, TLO, Depth))
2272 return true;
2273 break;
2274 }
2275
2276 // Just use computeKnownBits to compute output bits.
2277 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2278 break;
2279 }
2280
2281 // If we know the value of all of the demanded bits, return this as a
2282 // constant.
2283 if (DemandedBits.isSubsetOf(Known.Zero | Known.One)) {
2284 // Avoid folding to a constant if any OpaqueConstant is involved.
2285 const SDNode *N = Op.getNode();
2286 for (SDNode *Op :
2287 llvm::make_range(SDNodeIterator::begin(N), SDNodeIterator::end(N))) {
2288 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op))
2289 if (C->isOpaque())
2290 return false;
2291 }
2292 if (VT.isInteger())
2293 return TLO.CombineTo(Op, TLO.DAG.getConstant(Known.One, dl, VT));
2294 if (VT.isFloatingPoint())
2295 return TLO.CombineTo(
2296 Op,
2297 TLO.DAG.getConstantFP(
2298 APFloat(TLO.DAG.EVTToAPFloatSemantics(VT), Known.One), dl, VT));
2299 }
2300
2301 return false;
2302}
2303
2304bool TargetLowering::SimplifyDemandedVectorElts(SDValue Op,
2305 const APInt &DemandedElts,
2306 APInt &KnownUndef,
2307 APInt &KnownZero,
2308 DAGCombinerInfo &DCI) const {
2309 SelectionDAG &DAG = DCI.DAG;
2310 TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
2311 !DCI.isBeforeLegalizeOps());
2312
2313 bool Simplified =
2314 SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero, TLO);
2315 if (Simplified) {
2316 DCI.AddToWorklist(Op.getNode());
2317 DCI.CommitTargetLoweringOpt(TLO);
2318 }
2319
2320 return Simplified;
2321}
2322
2323/// Given a vector binary operation and known undefined elements for each input
2324/// operand, compute whether each element of the output is undefined.
2325static APInt getKnownUndefForVectorBinop(SDValue BO, SelectionDAG &DAG,
2326 const APInt &UndefOp0,
2327 const APInt &UndefOp1) {
2328 EVT VT = BO.getValueType();
2329 assert(DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() &&(static_cast <bool> (DAG.getTargetLoweringInfo().isBinOp
(BO.getOpcode()) && VT.isVector() && "Vector binop only"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() && \"Vector binop only\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2330, __extension__ __PRETTY_FUNCTION__))
2330 "Vector binop only")(static_cast <bool> (DAG.getTargetLoweringInfo().isBinOp
(BO.getOpcode()) && VT.isVector() && "Vector binop only"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() && \"Vector binop only\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2330, __extension__ __PRETTY_FUNCTION__))
;
2331
2332 EVT EltVT = VT.getVectorElementType();
2333 unsigned NumElts = VT.getVectorNumElements();
2334 assert(UndefOp0.getBitWidth() == NumElts &&(static_cast <bool> (UndefOp0.getBitWidth() == NumElts &&
UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis"
) ? void (0) : __assert_fail ("UndefOp0.getBitWidth() == NumElts && UndefOp1.getBitWidth() == NumElts && \"Bad type for undef analysis\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2335, __extension__ __PRETTY_FUNCTION__))
2335 UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis")(static_cast <bool> (UndefOp0.getBitWidth() == NumElts &&
UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis"
) ? void (0) : __assert_fail ("UndefOp0.getBitWidth() == NumElts && UndefOp1.getBitWidth() == NumElts && \"Bad type for undef analysis\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2335, __extension__ __PRETTY_FUNCTION__))
;
2336
2337 auto getUndefOrConstantElt = [&](SDValue V, unsigned Index,
2338 const APInt &UndefVals) {
2339 if (UndefVals[Index])
2340 return DAG.getUNDEF(EltVT);
2341
2342 if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
2343 // Try hard to make sure that the getNode() call is not creating temporary
2344 // nodes. Ignore opaque integers because they do not constant fold.
2345 SDValue Elt = BV->getOperand(Index);
2346 auto *C = dyn_cast<ConstantSDNode>(Elt);
2347 if (isa<ConstantFPSDNode>(Elt) || Elt.isUndef() || (C && !C->isOpaque()))
2348 return Elt;
2349 }
2350
2351 return SDValue();
2352 };
2353
2354 APInt KnownUndef = APInt::getZero(NumElts);
2355 for (unsigned i = 0; i != NumElts; ++i) {
2356 // If both inputs for this element are either constant or undef and match
2357 // the element type, compute the constant/undef result for this element of
2358 // the vector.
2359 // TODO: Ideally we would use FoldConstantArithmetic() here, but that does
2360 // not handle FP constants. The code within getNode() should be refactored
2361 // to avoid the danger of creating a bogus temporary node here.
2362 SDValue C0 = getUndefOrConstantElt(BO.getOperand(0), i, UndefOp0);
2363 SDValue C1 = getUndefOrConstantElt(BO.getOperand(1), i, UndefOp1);
2364 if (C0 && C1 && C0.getValueType() == EltVT && C1.getValueType() == EltVT)
2365 if (DAG.getNode(BO.getOpcode(), SDLoc(BO), EltVT, C0, C1).isUndef())
2366 KnownUndef.setBit(i);
2367 }
2368 return KnownUndef;
2369}
2370
2371bool TargetLowering::SimplifyDemandedVectorElts(
2372 SDValue Op, const APInt &OriginalDemandedElts, APInt &KnownUndef,
2373 APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth,
2374 bool AssumeSingleUse) const {
2375 EVT VT = Op.getValueType();
2376 unsigned Opcode = Op.getOpcode();
2377 APInt DemandedElts = OriginalDemandedElts;
2378 unsigned NumElts = DemandedElts.getBitWidth();
2379 assert(VT.isVector() && "Expected vector op")(static_cast <bool> (VT.isVector() && "Expected vector op"
) ? void (0) : __assert_fail ("VT.isVector() && \"Expected vector op\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2379, __extension__ __PRETTY_FUNCTION__))
;
2380
2381 KnownUndef = KnownZero = APInt::getZero(NumElts);
2382
2383 // TODO: For now we assume we know nothing about scalable vectors.
2384 if (VT.isScalableVector())
2385 return false;
2386
2387 assert(VT.getVectorNumElements() == NumElts &&(static_cast <bool> (VT.getVectorNumElements() == NumElts
&& "Mask size mismatches value type element count!")
? void (0) : __assert_fail ("VT.getVectorNumElements() == NumElts && \"Mask size mismatches value type element count!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2388, __extension__ __PRETTY_FUNCTION__))
2388 "Mask size mismatches value type element count!")(static_cast <bool> (VT.getVectorNumElements() == NumElts
&& "Mask size mismatches value type element count!")
? void (0) : __assert_fail ("VT.getVectorNumElements() == NumElts && \"Mask size mismatches value type element count!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2388, __extension__ __PRETTY_FUNCTION__))
;
2389
2390 // Undef operand.
2391 if (Op.isUndef()) {
2392 KnownUndef.setAllBits();
2393 return false;
2394 }
2395
2396 // If Op has other users, assume that all elements are needed.
2397 if (!Op.getNode()->hasOneUse() && !AssumeSingleUse)
2398 DemandedElts.setAllBits();
2399
2400 // Not demanding any elements from Op.
2401 if (DemandedElts == 0) {
2402 KnownUndef.setAllBits();
2403 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
2404 }
2405
2406 // Limit search depth.
2407 if (Depth >= SelectionDAG::MaxRecursionDepth)
2408 return false;
2409
2410 SDLoc DL(Op);
2411 unsigned EltSizeInBits = VT.getScalarSizeInBits();
2412
2413 // Helper for demanding the specified elements and all the bits of both binary
2414 // operands.
2415 auto SimplifyDemandedVectorEltsBinOp = [&](SDValue Op0, SDValue Op1) {
2416 SDValue NewOp0 = SimplifyMultipleUseDemandedVectorElts(Op0, DemandedElts,
2417 TLO.DAG, Depth + 1);
2418 SDValue NewOp1 = SimplifyMultipleUseDemandedVectorElts(Op1, DemandedElts,
2419 TLO.DAG, Depth + 1);
2420 if (NewOp0 || NewOp1) {
2421 SDValue NewOp = TLO.DAG.getNode(
2422 Opcode, SDLoc(Op), VT, NewOp0 ? NewOp0 : Op0, NewOp1 ? NewOp1 : Op1);
2423 return TLO.CombineTo(Op, NewOp);
2424 }
2425 return false;
2426 };
2427
2428 switch (Opcode) {
2429 case ISD::SCALAR_TO_VECTOR: {
2430 if (!DemandedElts[0]) {
2431 KnownUndef.setAllBits();
2432 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
2433 }
2434 SDValue ScalarSrc = Op.getOperand(0);
2435 if (ScalarSrc.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
2436 SDValue Src = ScalarSrc.getOperand(0);
2437 SDValue Idx = ScalarSrc.getOperand(1);
2438 EVT SrcVT = Src.getValueType();
2439
2440 ElementCount SrcEltCnt = SrcVT.getVectorElementCount();
2441
2442 if (SrcEltCnt.isScalable())
2443 return false;
2444
2445 unsigned NumSrcElts = SrcEltCnt.getFixedValue();
2446 if (isNullConstant(Idx)) {
2447 APInt SrcDemandedElts = APInt::getOneBitSet(NumSrcElts, 0);
2448 APInt SrcUndef = KnownUndef.zextOrTrunc(NumSrcElts);
2449 APInt SrcZero = KnownZero.zextOrTrunc(NumSrcElts);
2450 if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2451 TLO, Depth + 1))
2452 return true;
2453 }
2454 }
2455 KnownUndef.setHighBits(NumElts - 1);
2456 break;
2457 }
2458 case ISD::BITCAST: {
2459 SDValue Src = Op.getOperand(0);
2460 EVT SrcVT = Src.getValueType();
2461
2462 // We only handle vectors here.
2463 // TODO - investigate calling SimplifyDemandedBits/ComputeKnownBits?
2464 if (!SrcVT.isVector())
2465 break;
2466
2467 // Fast handling of 'identity' bitcasts.
2468 unsigned NumSrcElts = SrcVT.getVectorNumElements();
2469 if (NumSrcElts == NumElts)
2470 return SimplifyDemandedVectorElts(Src, DemandedElts, KnownUndef,
2471 KnownZero, TLO, Depth + 1);
2472
2473 APInt SrcDemandedElts, SrcZero, SrcUndef;
2474
2475 // Bitcast from 'large element' src vector to 'small element' vector, we
2476 // must demand a source element if any DemandedElt maps to it.
2477 if ((NumElts % NumSrcElts) == 0) {
2478 unsigned Scale = NumElts / NumSrcElts;
2479 SrcDemandedElts = APIntOps::ScaleBitMask(DemandedElts, NumSrcElts);
2480 if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2481 TLO, Depth + 1))
2482 return true;
2483
2484 // Try calling SimplifyDemandedBits, converting demanded elts to the bits
2485 // of the large element.
2486 // TODO - bigendian once we have test coverage.
2487 if (TLO.DAG.getDataLayout().isLittleEndian()) {
2488 unsigned SrcEltSizeInBits = SrcVT.getScalarSizeInBits();
2489 APInt SrcDemandedBits = APInt::getZero(SrcEltSizeInBits);
2490 for (unsigned i = 0; i != NumElts; ++i)
2491 if (DemandedElts[i]) {
2492 unsigned Ofs = (i % Scale) * EltSizeInBits;
2493 SrcDemandedBits.setBits(Ofs, Ofs + EltSizeInBits);
2494 }
2495
2496 KnownBits Known;
2497 if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcDemandedElts, Known,
2498 TLO, Depth + 1))
2499 return true;
2500 }
2501
2502 // If the src element is zero/undef then all the output elements will be -
2503 // only demanded elements are guaranteed to be correct.
2504 for (unsigned i = 0; i != NumSrcElts; ++i) {
2505 if (SrcDemandedElts[i]) {
2506 if (SrcZero[i])
2507 KnownZero.setBits(i * Scale, (i + 1) * Scale);
2508 if (SrcUndef[i])
2509 KnownUndef.setBits(i * Scale, (i + 1) * Scale);
2510 }
2511 }
2512 }
2513
2514 // Bitcast from 'small element' src vector to 'large element' vector, we
2515 // demand all smaller source elements covered by the larger demanded element
2516 // of this vector.
2517 if ((NumSrcElts % NumElts) == 0) {
2518 unsigned Scale = NumSrcElts / NumElts;
2519 SrcDemandedElts = APIntOps::ScaleBitMask(DemandedElts, NumSrcElts);
2520 if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2521 TLO, Depth + 1))
2522 return true;
2523
2524 // If all the src elements covering an output element are zero/undef, then
2525 // the output element will be as well, assuming it was demanded.
2526 for (unsigned i = 0; i != NumElts; ++i) {
2527 if (DemandedElts[i]) {
2528 if (SrcZero.extractBits(Scale, i * Scale).isAllOnes())
2529 KnownZero.setBit(i);
2530 if (SrcUndef.extractBits(Scale, i * Scale).isAllOnes())
2531 KnownUndef.setBit(i);
2532 }
2533 }
2534 }
2535 break;
2536 }
2537 case ISD::BUILD_VECTOR: {
2538 // Check all elements and simplify any unused elements with UNDEF.
2539 if (!DemandedElts.isAllOnes()) {
2540 // Don't simplify BROADCASTS.
2541 if (llvm::any_of(Op->op_values(),
2542 [&](SDValue Elt) { return Op.getOperand(0) != Elt; })) {
2543 SmallVector<SDValue, 32> Ops(Op->op_begin(), Op->op_end());
2544 bool Updated = false;
2545 for (unsigned i = 0; i != NumElts; ++i) {
2546 if (!DemandedElts[i] && !Ops[i].isUndef()) {
2547 Ops[i] = TLO.DAG.getUNDEF(Ops[0].getValueType());
2548 KnownUndef.setBit(i);
2549 Updated = true;
2550 }
2551 }
2552 if (Updated)
2553 return TLO.CombineTo(Op, TLO.DAG.getBuildVector(VT, DL, Ops));
2554 }
2555 }
2556 for (unsigned i = 0; i != NumElts; ++i) {
2557 SDValue SrcOp = Op.getOperand(i);
2558 if (SrcOp.isUndef()) {
2559 KnownUndef.setBit(i);
2560 } else if (EltSizeInBits == SrcOp.getScalarValueSizeInBits() &&
2561 (isNullConstant(SrcOp) || isNullFPConstant(SrcOp))) {
2562 KnownZero.setBit(i);
2563 }
2564 }
2565 break;
2566 }
2567 case ISD::CONCAT_VECTORS: {
2568 EVT SubVT = Op.getOperand(0).getValueType();
2569 unsigned NumSubVecs = Op.getNumOperands();
2570 unsigned NumSubElts = SubVT.getVectorNumElements();
2571 for (unsigned i = 0; i != NumSubVecs; ++i) {
2572 SDValue SubOp = Op.getOperand(i);
2573 APInt SubElts = DemandedElts.extractBits(NumSubElts, i * NumSubElts);
2574 APInt SubUndef, SubZero;
2575 if (SimplifyDemandedVectorElts(SubOp, SubElts, SubUndef, SubZero, TLO,
2576 Depth + 1))
2577 return true;
2578 KnownUndef.insertBits(SubUndef, i * NumSubElts);
2579 KnownZero.insertBits(SubZero, i * NumSubElts);
2580 }
2581 break;
2582 }
2583 case ISD::INSERT_SUBVECTOR: {
2584 // Demand any elements from the subvector and the remainder from the src its
2585 // inserted into.
2586 SDValue Src = Op.getOperand(0);
2587 SDValue Sub = Op.getOperand(1);
2588 uint64_t Idx = Op.getConstantOperandVal(2);
2589 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
2590 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
2591 APInt DemandedSrcElts = DemandedElts;
2592 DemandedSrcElts.insertBits(APInt::getZero(NumSubElts), Idx);
2593
2594 APInt SubUndef, SubZero;
2595 if (SimplifyDemandedVectorElts(Sub, DemandedSubElts, SubUndef, SubZero, TLO,
2596 Depth + 1))
2597 return true;
2598
2599 // If none of the src operand elements are demanded, replace it with undef.
2600 if (!DemandedSrcElts && !Src.isUndef())
2601 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
2602 TLO.DAG.getUNDEF(VT), Sub,
2603 Op.getOperand(2)));
2604
2605 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownUndef, KnownZero,
2606 TLO, Depth + 1))
2607 return true;
2608 KnownUndef.insertBits(SubUndef, Idx);
2609 KnownZero.insertBits(SubZero, Idx);
2610
2611 // Attempt to avoid multi-use ops if we don't need anything from them.
2612 if (!DemandedSrcElts.isAllOnes() || !DemandedSubElts.isAllOnes()) {
2613 SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
2614 Src, DemandedSrcElts, TLO.DAG, Depth + 1);
2615 SDValue NewSub = SimplifyMultipleUseDemandedVectorElts(
2616 Sub, DemandedSubElts, TLO.DAG, Depth + 1);
2617 if (NewSrc || NewSub) {
2618 NewSrc = NewSrc ? NewSrc : Src;
2619 NewSub = NewSub ? NewSub : Sub;
2620 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,
2621 NewSub, Op.getOperand(2));
2622 return TLO.CombineTo(Op, NewOp);
2623 }
2624 }
2625 break;
2626 }
2627 case ISD::EXTRACT_SUBVECTOR: {
2628 // Offset the demanded elts by the subvector index.
2629 SDValue Src = Op.getOperand(0);
2630 if (Src.getValueType().isScalableVector())
2631 break;
2632 uint64_t Idx = Op.getConstantOperandVal(1);
2633 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2634 APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
2635
2636 APInt SrcUndef, SrcZero;
2637 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
2638 Depth + 1))
2639 return true;
2640 KnownUndef = SrcUndef.extractBits(NumElts, Idx);
2641 KnownZero = SrcZero.extractBits(NumElts, Idx);
2642
2643 // Attempt to avoid multi-use ops if we don't need anything from them.
2644 if (!DemandedElts.isAllOnes()) {
2645 SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
2646 Src, DemandedSrcElts, TLO.DAG, Depth + 1);
2647 if (NewSrc) {
2648 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,
2649 Op.getOperand(1));
2650 return TLO.CombineTo(Op, NewOp);
2651 }
2652 }
2653 break;
2654 }
2655 case ISD::INSERT_VECTOR_ELT: {
2656 SDValue Vec = Op.getOperand(0);
2657 SDValue Scl = Op.getOperand(1);
2658 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
2659
2660 // For a legal, constant insertion index, if we don't need this insertion
2661 // then strip it, else remove it from the demanded elts.
2662 if (CIdx && CIdx->getAPIntValue().ult(NumElts)) {
2663 unsigned Idx = CIdx->getZExtValue();
2664 if (!DemandedElts[Idx])
2665 return TLO.CombineTo(Op, Vec);
2666
2667 APInt DemandedVecElts(DemandedElts);
2668 DemandedVecElts.clearBit(Idx);
2669 if (SimplifyDemandedVectorElts(Vec, DemandedVecElts, KnownUndef,
2670 KnownZero, TLO, Depth + 1))
2671 return true;
2672
2673 KnownUndef.setBitVal(Idx, Scl.isUndef());
2674
2675 KnownZero.setBitVal(Idx, isNullConstant(Scl) || isNullFPConstant(Scl));
2676 break;
2677 }
2678
2679 APInt VecUndef, VecZero;
2680 if (SimplifyDemandedVectorElts(Vec, DemandedElts, VecUndef, VecZero, TLO,
2681 Depth + 1))
2682 return true;
2683 // Without knowing the insertion index we can't set KnownUndef/KnownZero.
2684 break;
2685 }
2686 case ISD::VSELECT: {
2687 // Try to transform the select condition based on the current demanded
2688 // elements.
2689 // TODO: If a condition element is undef, we can choose from one arm of the
2690 // select (and if one arm is undef, then we can propagate that to the
2691 // result).
2692 // TODO - add support for constant vselect masks (see IR version of this).
2693 APInt UnusedUndef, UnusedZero;
2694 if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, UnusedUndef,
2695 UnusedZero, TLO, Depth + 1))
2696 return true;
2697
2698 // See if we can simplify either vselect operand.
2699 APInt DemandedLHS(DemandedElts);
2700 APInt DemandedRHS(DemandedElts);
2701 APInt UndefLHS, ZeroLHS;
2702 APInt UndefRHS, ZeroRHS;
2703 if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedLHS, UndefLHS,
2704 ZeroLHS, TLO, Depth + 1))
2705 return true;
2706 if (SimplifyDemandedVectorElts(Op.getOperand(2), DemandedRHS, UndefRHS,
2707 ZeroRHS, TLO, Depth + 1))
2708 return true;
2709
2710 KnownUndef = UndefLHS & UndefRHS;
2711 KnownZero = ZeroLHS & ZeroRHS;
2712 break;
2713 }
2714 case ISD::VECTOR_SHUFFLE: {
2715 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
2716
2717 // Collect demanded elements from shuffle operands..
2718 APInt DemandedLHS(NumElts, 0);
2719 APInt DemandedRHS(NumElts, 0);
2720 for (unsigned i = 0; i != NumElts; ++i) {
2721 int M = ShuffleMask[i];
2722 if (M < 0 || !DemandedElts[i])
2723 continue;
2724 assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range")(static_cast <bool> (0 <= M && M < (int)(
2 * NumElts) && "Shuffle index out of range") ? void (
0) : __assert_fail ("0 <= M && M < (int)(2 * NumElts) && \"Shuffle index out of range\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2724, __extension__ __PRETTY_FUNCTION__))
;
2725 if (M < (int)NumElts)
2726 DemandedLHS.setBit(M);
2727 else
2728 DemandedRHS.setBit(M - NumElts);
2729 }
2730
2731 // See if we can simplify either shuffle operand.
2732 APInt UndefLHS, ZeroLHS;
2733 APInt UndefRHS, ZeroRHS;
2734 if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedLHS, UndefLHS,
2735 ZeroLHS, TLO, Depth + 1))
2736 return true;
2737 if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedRHS, UndefRHS,
2738 ZeroRHS, TLO, Depth + 1))
2739 return true;
2740
2741 // Simplify mask using undef elements from LHS/RHS.
2742 bool Updated = false;
2743 bool IdentityLHS = true, IdentityRHS = true;
2744 SmallVector<int, 32> NewMask(ShuffleMask.begin(), ShuffleMask.end());
2745 for (unsigned i = 0; i != NumElts; ++i) {
2746 int &M = NewMask[i];
2747 if (M < 0)
2748 continue;
2749 if (!DemandedElts[i] || (M < (int)NumElts && UndefLHS[M]) ||
2750 (M >= (int)NumElts && UndefRHS[M - NumElts])) {
2751 Updated = true;
2752 M = -1;
2753 }
2754 IdentityLHS &= (M < 0) || (M == (int)i);
2755 IdentityRHS &= (M < 0) || ((M - NumElts) == i);
2756 }
2757
2758 // Update legal shuffle masks based on demanded elements if it won't reduce
2759 // to Identity which can cause premature removal of the shuffle mask.
2760 if (Updated && !IdentityLHS && !IdentityRHS && !TLO.LegalOps) {
2761 SDValue LegalShuffle =
2762 buildLegalVectorShuffle(VT, DL, Op.getOperand(0), Op.getOperand(1),
2763 NewMask, TLO.DAG);
2764 if (LegalShuffle)
2765 return TLO.CombineTo(Op, LegalShuffle);
2766 }
2767
2768 // Propagate undef/zero elements from LHS/RHS.
2769 for (unsigned i = 0; i != NumElts; ++i) {
2770 int M = ShuffleMask[i];
2771 if (M < 0) {
2772 KnownUndef.setBit(i);
2773 } else if (M < (int)NumElts) {
2774 if (UndefLHS[M])
2775 KnownUndef.setBit(i);
2776 if (ZeroLHS[M])
2777 KnownZero.setBit(i);
2778 } else {
2779 if (UndefRHS[M - NumElts])
2780 KnownUndef.setBit(i);
2781 if (ZeroRHS[M - NumElts])
2782 KnownZero.setBit(i);
2783 }
2784 }
2785 break;
2786 }
2787 case ISD::ANY_EXTEND_VECTOR_INREG:
2788 case ISD::SIGN_EXTEND_VECTOR_INREG:
2789 case ISD::ZERO_EXTEND_VECTOR_INREG: {
2790 APInt SrcUndef, SrcZero;
2791 SDValue Src = Op.getOperand(0);
2792 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2793 APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts);
2794 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
2795 Depth + 1))
2796 return true;
2797 KnownZero = SrcZero.zextOrTrunc(NumElts);
2798 KnownUndef = SrcUndef.zextOrTrunc(NumElts);
2799
2800 if (Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG &&
2801 Op.getValueSizeInBits() == Src.getValueSizeInBits() &&
2802 DemandedSrcElts == 1 && TLO.DAG.getDataLayout().isLittleEndian()) {
2803 // aext - if we just need the bottom element then we can bitcast.
2804 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
2805 }
2806
2807 if (Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) {
2808 // zext(undef) upper bits are guaranteed to be zero.
2809 if (DemandedElts.isSubsetOf(KnownUndef))
2810 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
2811 KnownUndef.clearAllBits();
2812 }
2813 break;
2814 }
2815
2816 // TODO: There are more binop opcodes that could be handled here - MIN,
2817 // MAX, saturated math, etc.
2818 case ISD::OR:
2819 case ISD::XOR:
2820 case ISD::ADD:
2821 case ISD::SUB:
2822 case ISD::FADD:
2823 case ISD::FSUB:
2824 case ISD::FMUL:
2825 case ISD::FDIV:
2826 case ISD::FREM: {
2827 SDValue Op0 = Op.getOperand(0);
2828 SDValue Op1 = Op.getOperand(1);
2829
2830 APInt UndefRHS, ZeroRHS;
2831 if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,
2832 Depth + 1))
2833 return true;
2834 APInt UndefLHS, ZeroLHS;
2835 if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
2836 Depth + 1))
2837 return true;
2838
2839 KnownZero = ZeroLHS & ZeroRHS;
2840 KnownUndef = getKnownUndefForVectorBinop(Op, TLO.DAG, UndefLHS, UndefRHS);
2841
2842 // Attempt to avoid multi-use ops if we don't need anything from them.
2843 // TODO - use KnownUndef to relax the demandedelts?
2844 if (!DemandedElts.isAllOnes())
2845 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
2846 return true;
2847 break;
2848 }
2849 case ISD::SHL:
2850 case ISD::SRL:
2851 case ISD::SRA:
2852 case ISD::ROTL:
2853 case ISD::ROTR: {
2854 SDValue Op0 = Op.getOperand(0);
2855 SDValue Op1 = Op.getOperand(1);
2856
2857 APInt UndefRHS, ZeroRHS;
2858 if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,
2859 Depth + 1))
2860 return true;
2861 APInt UndefLHS, ZeroLHS;
2862 if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
2863 Depth + 1))
2864 return true;
2865
2866 KnownZero = ZeroLHS;
2867 KnownUndef = UndefLHS & UndefRHS; // TODO: use getKnownUndefForVectorBinop?
2868
2869 // Attempt to avoid multi-use ops if we don't need anything from them.
2870 // TODO - use KnownUndef to relax the demandedelts?
2871 if (!DemandedElts.isAllOnes())
2872 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
2873 return true;
2874 break;
2875 }
2876 case ISD::MUL:
2877 case ISD::AND: {
2878 SDValue Op0 = Op.getOperand(0);
2879 SDValue Op1 = Op.getOperand(1);
2880
2881 APInt SrcUndef, SrcZero;
2882 if (SimplifyDemandedVectorElts(Op1, DemandedElts, SrcUndef, SrcZero, TLO,
2883 Depth + 1))
2884 return true;
2885 if (SimplifyDemandedVectorElts(Op0, DemandedElts, KnownUndef, KnownZero,
2886 TLO, Depth + 1))
2887 return true;
2888
2889 // If either side has a zero element, then the result element is zero, even
2890 // if the other is an UNDEF.
2891 // TODO: Extend getKnownUndefForVectorBinop to also deal with known zeros
2892 // and then handle 'and' nodes with the rest of the binop opcodes.
2893 KnownZero |= SrcZero;
2894 KnownUndef &= SrcUndef;
2895 KnownUndef &= ~KnownZero;
2896
2897 // Attempt to avoid multi-use ops if we don't need anything from them.
2898 // TODO - use KnownUndef to relax the demandedelts?
2899 if (!DemandedElts.isAllOnes())
2900 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
2901 return true;
2902 break;
2903 }
2904 case ISD::TRUNCATE:
2905 case ISD::SIGN_EXTEND:
2906 case ISD::ZERO_EXTEND:
2907 if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,
2908 KnownZero, TLO, Depth + 1))
2909 return true;
2910
2911 if (Op.getOpcode() == ISD::ZERO_EXTEND) {
2912 // zext(undef) upper bits are guaranteed to be zero.
2913 if (DemandedElts.isSubsetOf(KnownUndef))
2914 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
2915 KnownUndef.clearAllBits();
2916 }
2917 break;
2918 default: {
2919 if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
2920 if (SimplifyDemandedVectorEltsForTargetNode(Op, DemandedElts, KnownUndef,
2921 KnownZero, TLO, Depth))
2922 return true;
2923 } else {
2924 KnownBits Known;
2925 APInt DemandedBits = APInt::getAllOnes(EltSizeInBits);
2926 if (SimplifyDemandedBits(Op, DemandedBits, OriginalDemandedElts, Known,
2927 TLO, Depth, AssumeSingleUse))
2928 return true;
2929 }
2930 break;
2931 }
2932 }
2933 assert((KnownUndef & KnownZero) == 0 && "Elements flagged as undef AND zero")(static_cast <bool> ((KnownUndef & KnownZero) == 0 &&
"Elements flagged as undef AND zero") ? void (0) : __assert_fail
("(KnownUndef & KnownZero) == 0 && \"Elements flagged as undef AND zero\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2933, __extension__ __PRETTY_FUNCTION__))
;
2934
2935 // Constant fold all undef cases.
2936 // TODO: Handle zero cases as well.
2937 if (DemandedElts.isSubsetOf(KnownUndef))
2938 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
2939
2940 return false;
2941}
2942
2943/// Determine which of the bits specified in Mask are known to be either zero or
2944/// one and return them in the Known.
2945void TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
2946 KnownBits &Known,
2947 const APInt &DemandedElts,
2948 const SelectionDAG &DAG,
2949 unsigned Depth) const {
2950 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2955, __extension__ __PRETTY_FUNCTION__))
2951 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2955, __extension__ __PRETTY_FUNCTION__))
2952 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2955, __extension__ __PRETTY_FUNCTION__))
2953 Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2955, __extension__ __PRETTY_FUNCTION__))
2954 "Should use MaskedValueIsZero if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2955, __extension__ __PRETTY_FUNCTION__))
2955 " is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2955, __extension__ __PRETTY_FUNCTION__))
;
2956 Known.resetAll();
2957}
2958
2959void TargetLowering::computeKnownBitsForTargetInstr(
2960 GISelKnownBits &Analysis, Register R, KnownBits &Known,
2961 const APInt &DemandedElts, const MachineRegisterInfo &MRI,
2962 unsigned Depth) const {
2963 Known.resetAll();
2964}
2965
2966void TargetLowering::computeKnownBitsForFrameIndex(
2967 const int FrameIdx, KnownBits &Known, const MachineFunction &MF) const {
2968 // The low bits are known zero if the pointer is aligned.
2969 Known.Zero.setLowBits(Log2(MF.getFrameInfo().getObjectAlign(FrameIdx)));
2970}
2971
2972Align TargetLowering::computeKnownAlignForTargetInstr(
2973 GISelKnownBits &Analysis, Register R, const MachineRegisterInfo &MRI,
2974 unsigned Depth) const {
2975 return Align(1);
2976}
2977
2978/// This method can be implemented by targets that want to expose additional
2979/// information about sign bits to the DAG Combiner.
2980unsigned TargetLowering::ComputeNumSignBitsForTargetNode(SDValue Op,
2981 const APInt &,
2982 const SelectionDAG &,
2983 unsigned Depth) const {
2984 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2989, __extension__ __PRETTY_FUNCTION__))
2985 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2989, __extension__ __PRETTY_FUNCTION__))
2986 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2989, __extension__ __PRETTY_FUNCTION__))
2987 Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2989, __extension__ __PRETTY_FUNCTION__))
2988 "Should use ComputeNumSignBits if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2989, __extension__ __PRETTY_FUNCTION__))
2989 " is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2989, __extension__ __PRETTY_FUNCTION__))
;
2990 return 1;
2991}
2992
2993unsigned TargetLowering::computeNumSignBitsForTargetInstr(
2994 GISelKnownBits &Analysis, Register R, const APInt &DemandedElts,
2995 const MachineRegisterInfo &MRI, unsigned Depth) const {
2996 return 1;
2997}
2998
2999bool TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
3000 SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,
3001 TargetLoweringOpt &TLO, unsigned Depth) const {
3002 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3007, __extension__ __PRETTY_FUNCTION__))
3003 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3007, __extension__ __PRETTY_FUNCTION__))
3004 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3007, __extension__ __PRETTY_FUNCTION__))
3005 Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3007, __extension__ __PRETTY_FUNCTION__))
3006 "Should use SimplifyDemandedVectorElts if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3007, __extension__ __PRETTY_FUNCTION__))
3007 " is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3007, __extension__ __PRETTY_FUNCTION__))
;
3008 return false;
3009}
3010
3011bool TargetLowering::SimplifyDemandedBitsForTargetNode(
3012 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
3013 KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const {
3014 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3019, __extension__ __PRETTY_FUNCTION__))
3015 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3019, __extension__ __PRETTY_FUNCTION__))
3016 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3019, __extension__ __PRETTY_FUNCTION__))
3017 Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3019, __extension__ __PRETTY_FUNCTION__))
3018 "Should use SimplifyDemandedBits if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3019, __extension__ __PRETTY_FUNCTION__))
3019 " is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3019, __extension__ __PRETTY_FUNCTION__))
;
3020 computeKnownBitsForTargetNode(Op, Known, DemandedElts, TLO.DAG, Depth);
3021 return false;
3022}
3023
3024SDValue TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(
3025 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
3026 SelectionDAG &DAG, unsigned Depth) const {
3027 assert((static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3033, __extension__ __PRETTY_FUNCTION__))
3028 (Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3033, __extension__ __PRETTY_FUNCTION__))
3029 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3033, __extension__ __PRETTY_FUNCTION__))
3030 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3033, __extension__ __PRETTY_FUNCTION__))
3031 Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3033, __extension__ __PRETTY_FUNCTION__))
3032 "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3033, __extension__ __PRETTY_FUNCTION__))
3033 " is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3033, __extension__ __PRETTY_FUNCTION__))
;
3034 return SDValue();
3035}
3036
3037SDValue
3038TargetLowering::buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0,
3039 SDValue N1, MutableArrayRef<int> Mask,
3040 SelectionDAG &DAG) const {
3041 bool LegalMask = isShuffleMaskLegal(Mask, VT);
3042 if (!LegalMask) {
3043 std::swap(N0, N1);
3044 ShuffleVectorSDNode::commuteMask(Mask);
3045 LegalMask = isShuffleMaskLegal(Mask, VT);
3046 }
3047
3048 if (!LegalMask)
3049 return SDValue();
3050
3051 return DAG.getVectorShuffle(VT, DL, N0, N1, Mask);
3052}
3053
3054const Constant *TargetLowering::getTargetConstantFromLoad(LoadSDNode*) const {
3055 return nullptr;
3056}
3057
3058bool TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode(
3059 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
3060 bool PoisonOnly, unsigned Depth) const {
3061 assert((static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3067, __extension__ __PRETTY_FUNCTION__))
3062 (Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3067, __extension__ __PRETTY_FUNCTION__))
3063 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3067, __extension__ __PRETTY_FUNCTION__))
3064 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3067, __extension__ __PRETTY_FUNCTION__))
3065 Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3067, __extension__ __PRETTY_FUNCTION__))
3066 "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3067, __extension__ __PRETTY_FUNCTION__))
3067 " is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3067, __extension__ __PRETTY_FUNCTION__))
;
3068 return false;
3069}
3070
3071bool TargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
3072 const SelectionDAG &DAG,
3073 bool SNaN,
3074 unsigned Depth) const {
3075 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3080, __extension__ __PRETTY_FUNCTION__))
3076 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3080, __extension__ __PRETTY_FUNCTION__))
3077 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3080, __extension__ __PRETTY_FUNCTION__))
3078 Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3080, __extension__ __PRETTY_FUNCTION__))
3079 "Should use isKnownNeverNaN if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3080, __extension__ __PRETTY_FUNCTION__))
3080 " is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3080, __extension__ __PRETTY_FUNCTION__))
;
3081 return false;
3082}
3083
3084// FIXME: Ideally, this would use ISD::isConstantSplatVector(), but that must
3085// work with truncating build vectors and vectors with elements of less than
3086// 8 bits.
3087bool TargetLowering::isConstTrueVal(const SDNode *N) const {
3088 if (!N)
3089 return false;
3090
3091 APInt CVal;
3092 if (auto *CN = dyn_cast<ConstantSDNode>(N)) {
3093 CVal = CN->getAPIntValue();
3094 } else if (auto *BV = dyn_cast<BuildVectorSDNode>(N)) {
3095 auto *CN = BV->getConstantSplatNode();
3096 if (!CN)
3097 return false;
3098
3099 // If this is a truncating build vector, truncate the splat value.
3100 // Otherwise, we may fail to match the expected values below.
3101 unsigned BVEltWidth = BV->getValueType(0).getScalarSizeInBits();
3102 CVal = CN->getAPIntValue();
3103 if (BVEltWidth < CVal.getBitWidth())
3104 CVal = CVal.trunc(BVEltWidth);
3105 } else {
3106 return false;
3107 }
3108
3109 switch (getBooleanContents(N->getValueType(0))) {
3110 case UndefinedBooleanContent:
3111 return CVal[0];
3112 case ZeroOrOneBooleanContent:
3113 return CVal.isOne();
3114 case ZeroOrNegativeOneBooleanContent:
3115 return CVal.isAllOnes();
3116 }
3117
3118 llvm_unreachable("Invalid boolean contents")::llvm::llvm_unreachable_internal("Invalid boolean contents",
"/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3118)
;
3119}
3120
3121bool TargetLowering::isConstFalseVal(const SDNode *N) const {
3122 if (!N)
3123 return false;
3124
3125 const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N);
3126 if (!CN) {
3127 const BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N);
3128 if (!BV)
3129 return false;
3130
3131 // Only interested in constant splats, we don't care about undef
3132 // elements in identifying boolean constants and getConstantSplatNode
3133 // returns NULL if all ops are undef;
3134 CN = BV->getConstantSplatNode();
3135 if (!CN)
3136 return false;
3137 }
3138
3139 if (getBooleanContents(N->getValueType(0)) == UndefinedBooleanContent)
3140 return !CN->getAPIntValue()[0];
3141
3142 return CN->isZero();
3143}
3144
3145bool TargetLowering::isExtendedTrueVal(const ConstantSDNode *N, EVT VT,
3146 bool SExt) const {
3147 if (VT == MVT::i1)
3148 return N->isOne();
3149
3150 TargetLowering::BooleanContent Cnt = getBooleanContents(VT);
3151 switch (Cnt) {
3152 case TargetLowering::ZeroOrOneBooleanContent:
3153 // An extended value of 1 is always true, unless its original type is i1,
3154 // in which case it will be sign extended to -1.
3155 return (N->isOne() && !SExt) || (SExt && (N->getValueType(0) != MVT::i1));
3156 case TargetLowering::UndefinedBooleanContent:
3157 case TargetLowering::ZeroOrNegativeOneBooleanContent:
3158 return N->isAllOnes() && SExt;
3159 }
3160 llvm_unreachable("Unexpected enumeration.")::llvm::llvm_unreachable_internal("Unexpected enumeration.", "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3160)
;
3161}
3162
3163/// This helper function of SimplifySetCC tries to optimize the comparison when
3164/// either operand of the SetCC node is a bitwise-and instruction.
3165SDValue TargetLowering::foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1,
3166 ISD::CondCode Cond, const SDLoc &DL,
3167 DAGCombinerInfo &DCI) const {
3168 // Match these patterns in any of their permutations:
3169 // (X & Y) == Y
3170 // (X & Y) != Y
3171 if (N1.getOpcode() == ISD::AND && N0.getOpcode() != ISD::AND)
3172 std::swap(N0, N1);
3173
3174 EVT OpVT = N0.getValueType();
3175 if (N0.getOpcode() != ISD::AND || !OpVT.isInteger() ||
3176 (Cond != ISD::SETEQ && Cond != ISD::SETNE))
3177 return SDValue();
3178
3179 SDValue X, Y;
3180 if (N0.getOperand(0) == N1) {
3181 X = N0.getOperand(1);
3182 Y = N0.getOperand(0);
3183 } else if (N0.getOperand(1) == N1) {
3184 X = N0.getOperand(0);
3185 Y = N0.getOperand(1);
3186 } else {
3187 return SDValue();
3188 }
3189
3190 SelectionDAG &DAG = DCI.DAG;
3191 SDValue Zero = DAG.getConstant(0, DL, OpVT);
3192 if (DAG.isKnownToBeAPowerOfTwo(Y)) {
3193 // Simplify X & Y == Y to X & Y != 0 if Y has exactly one bit set.
3194 // Note that where Y is variable and is known to have at most one bit set
3195 // (for example, if it is Z & 1) we cannot do this; the expressions are not
3196 // equivalent when Y == 0.
3197 assert(OpVT.isInteger())(static_cast <bool> (OpVT.isInteger()) ? void (0) : __assert_fail
("OpVT.isInteger()", "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3197, __extension__ __PRETTY_FUNCTION__))
;
3198 Cond = ISD::getSetCCInverse(Cond, OpVT);
3199 if (DCI.isBeforeLegalizeOps() ||
3200 isCondCodeLegal(Cond, N0.getSimpleValueType()))
3201 return DAG.getSetCC(DL, VT, N0, Zero, Cond);
3202 } else if (N0.hasOneUse() && hasAndNotCompare(Y)) {
3203 // If the target supports an 'and-not' or 'and-complement' logic operation,
3204 // try to use that to make a comparison operation more efficient.
3205 // But don't do this transform if the mask is a single bit because there are
3206 // more efficient ways to deal with that case (for example, 'bt' on x86 or
3207 // 'rlwinm' on PPC).
3208
3209 // Bail out if the compare operand that we want to turn into a zero is
3210 // already a zero (otherwise, infinite loop).
3211 auto *YConst = dyn_cast<ConstantSDNode>(Y);
3212 if (YConst && YConst->isZero())
3213 return SDValue();
3214
3215 // Transform this into: ~X & Y == 0.
3216 SDValue NotX = DAG.getNOT(SDLoc(X), X, OpVT);
3217 SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, NotX, Y);
3218 return DAG.getSetCC(DL, VT, NewAnd, Zero, Cond);
3219 }
3220
3221 return SDValue();
3222}
3223
3224/// There are multiple IR patterns that could be checking whether certain
3225/// truncation of a signed number would be lossy or not. The pattern which is
3226/// best at IR level, may not lower optimally. Thus, we want to unfold it.
3227/// We are looking for the following pattern: (KeptBits is a constant)
3228/// (add %x, (1 << (KeptBits-1))) srccond (1 << KeptBits)
3229/// KeptBits won't be bitwidth(x), that will be constant-folded to true/false.
3230/// KeptBits also can't be 1, that would have been folded to %x dstcond 0
3231/// We will unfold it into the natural trunc+sext pattern:
3232/// ((%x << C) a>> C) dstcond %x
3233/// Where C = bitwidth(x) - KeptBits and C u< bitwidth(x)
3234SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(
3235 EVT SCCVT, SDValue N0, SDValue N1, ISD::CondCode Cond, DAGCombinerInfo &DCI,
3236 const SDLoc &DL) const {
3237 // We must be comparing with a constant.
3238 ConstantSDNode *C1;
3239 if (!(C1 = dyn_cast<ConstantSDNode>(N1)))
3240 return SDValue();
3241
3242 // N0 should be: add %x, (1 << (KeptBits-1))
3243 if (N0->getOpcode() != ISD::ADD)
3244 return SDValue();
3245
3246 // And we must be 'add'ing a constant.
3247 ConstantSDNode *C01;
3248 if (!(C01 = dyn_cast<ConstantSDNode>(N0->getOperand(1))))
3249 return SDValue();
3250
3251 SDValue X = N0->getOperand(0);
3252 EVT XVT = X.getValueType();
3253
3254 // Validate constants ...
3255
3256 APInt I1 = C1->getAPIntValue();
3257
3258 ISD::CondCode NewCond;
3259 if (Cond == ISD::CondCode::SETULT) {
3260 NewCond = ISD::CondCode::SETEQ;
3261 } else if (Cond == ISD::CondCode::SETULE) {
3262 NewCond = ISD::CondCode::SETEQ;
3263 // But need to 'canonicalize' the constant.
3264 I1 += 1;
3265 } else if (Cond == ISD::CondCode::SETUGT) {
3266 NewCond = ISD::CondCode::SETNE;
3267 // But need to 'canonicalize' the constant.
3268 I1 += 1;
3269 } else if (Cond == ISD::CondCode::SETUGE) {
3270 NewCond = ISD::CondCode::SETNE;
3271 } else
3272 return SDValue();
3273
3274 APInt I01 = C01->getAPIntValue();
3275
3276 auto checkConstants = [&I1, &I01]() -> bool {
3277 // Both of them must be power-of-two, and the constant from setcc is bigger.
3278 return I1.ugt(I01) && I1.isPowerOf2() && I01.isPowerOf2();
3279 };
3280
3281 if (checkConstants()) {
3282 // Great, e.g. got icmp ult i16 (add i16 %x, 128), 256
3283 } else {
3284 // What if we invert constants? (and the target predicate)
3285 I1.negate();
3286 I01.negate();
3287 assert(XVT.isInteger())(static_cast <bool> (XVT.isInteger()) ? void (0) : __assert_fail
("XVT.isInteger()", "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3287, __extension__ __PRETTY_FUNCTION__))
;
3288 NewCond = getSetCCInverse(NewCond, XVT);
3289 if (!checkConstants())
3290 return SDValue();
3291 // Great, e.g. got icmp uge i16 (add i16 %x, -128), -256
3292 }
3293
3294 // They are power-of-two, so which bit is set?
3295 const unsigned KeptBits = I1.logBase2();
3296 const unsigned KeptBitsMinusOne = I01.logBase2();
3297
3298 // Magic!
3299 if (KeptBits != (KeptBitsMinusOne + 1))
3300 return SDValue();
3301 assert(KeptBits > 0 && KeptBits < XVT.getSizeInBits() && "unreachable")(static_cast <bool> (KeptBits > 0 && KeptBits
< XVT.getSizeInBits() && "unreachable") ? void (0
) : __assert_fail ("KeptBits > 0 && KeptBits < XVT.getSizeInBits() && \"unreachable\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3301, __extension__ __PRETTY_FUNCTION__))
;
3302
3303 // We don't want to do this in every single case.
3304 SelectionDAG &DAG = DCI.DAG;
3305 if (!DAG.getTargetLoweringInfo().shouldTransformSignedTruncationCheck(
3306 XVT, KeptBits))
3307 return SDValue();
3308
3309 const unsigned MaskedBits = XVT.getSizeInBits() - KeptBits;
3310 assert(MaskedBits > 0 && MaskedBits < XVT.getSizeInBits() && "unreachable")(static_cast <bool> (MaskedBits > 0 && MaskedBits
< XVT.getSizeInBits() && "unreachable") ? void (0
) : __assert_fail ("MaskedBits > 0 && MaskedBits < XVT.getSizeInBits() && \"unreachable\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3310, __extension__ __PRETTY_FUNCTION__))
;
3311
3312 // Unfold into: ((%x << C) a>> C) cond %x
3313 // Where 'cond' will be either 'eq' or 'ne'.
3314 SDValue ShiftAmt = DAG.getConstant(MaskedBits, DL, XVT);
3315 SDValue T0 = DAG.getNode(ISD::SHL, DL, XVT, X, ShiftAmt);
3316 SDValue T1 = DAG.getNode(ISD::SRA, DL, XVT, T0, ShiftAmt);
3317 SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, X, NewCond);
3318
3319 return T2;
3320}
3321
3322// (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
3323SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(
3324 EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond,
3325 DAGCombinerInfo &DCI, const SDLoc &DL) const {
3326 assert(isConstOrConstSplat(N1C) &&(static_cast <bool> (isConstOrConstSplat(N1C) &&
isConstOrConstSplat(N1C)->getAPIntValue().isZero() &&
"Should be a comparison with 0.") ? void (0) : __assert_fail
("isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->getAPIntValue().isZero() && \"Should be a comparison with 0.\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3328, __extension__ __PRETTY_FUNCTION__))
3327 isConstOrConstSplat(N1C)->getAPIntValue().isZero() &&(static_cast <bool> (isConstOrConstSplat(N1C) &&
isConstOrConstSplat(N1C)->getAPIntValue().isZero() &&
"Should be a comparison with 0.") ? void (0) : __assert_fail
("isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->getAPIntValue().isZero() && \"Should be a comparison with 0.\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3328, __extension__ __PRETTY_FUNCTION__))
3328 "Should be a comparison with 0.")(static_cast <bool> (isConstOrConstSplat(N1C) &&
isConstOrConstSplat(N1C)->getAPIntValue().isZero() &&
"Should be a comparison with 0.") ? void (0) : __assert_fail
("isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->getAPIntValue().isZero() && \"Should be a comparison with 0.\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3328, __extension__ __PRETTY_FUNCTION__))
;
3329 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Valid only for [in]equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Valid only for [in]equality comparisons.\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3330, __extension__ __PRETTY_FUNCTION__))
3330 "Valid only for [in]equality comparisons.")(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Valid only for [in]equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Valid only for [in]equality comparisons.\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3330, __extension__ __PRETTY_FUNCTION__))
;
3331
3332 unsigned NewShiftOpcode;
3333 SDValue X, C, Y;
3334
3335 SelectionDAG &DAG = DCI.DAG;
3336 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3337
3338 // Look for '(C l>>/<< Y)'.
3339 auto Match = [&NewShiftOpcode, &X, &C, &Y, &TLI, &DAG](SDValue V) {
3340 // The shift should be one-use.
3341 if (!V.hasOneUse())
3342 return false;
3343 unsigned OldShiftOpcode = V.getOpcode();
3344 switch (OldShiftOpcode) {
3345 case ISD::SHL:
3346 NewShiftOpcode = ISD::SRL;
3347 break;
3348 case ISD::SRL:
3349 NewShiftOpcode = ISD::SHL;
3350 break;
3351 default:
3352 return false; // must be a logical shift.
3353 }
3354 // We should be shifting a constant.
3355 // FIXME: best to use isConstantOrConstantVector().
3356 C = V.getOperand(0);
3357 ConstantSDNode *CC =
3358 isConstOrConstSplat(C, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
3359 if (!CC)
3360 return false;
3361 Y = V.getOperand(1);
3362
3363 ConstantSDNode *XC =
3364 isConstOrConstSplat(X, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
3365 return TLI.shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
3366 X, XC, CC, Y, OldShiftOpcode, NewShiftOpcode, DAG);
3367 };
3368
3369 // LHS of comparison should be an one-use 'and'.
3370 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())
3371 return SDValue();
3372
3373 X = N0.getOperand(0);
3374 SDValue Mask = N0.getOperand(1);
3375
3376 // 'and' is commutative!
3377 if (!Match(Mask)) {
3378 std::swap(X, Mask);
3379 if (!Match(Mask))
3380 return SDValue();
3381 }
3382
3383 EVT VT = X.getValueType();
3384
3385 // Produce:
3386 // ((X 'OppositeShiftOpcode' Y) & C) Cond 0
3387 SDValue T0 = DAG.getNode(NewShiftOpcode, DL, VT, X, Y);
3388 SDValue T1 = DAG.getNode(ISD::AND, DL, VT, T0, C);
3389 SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, N1C, Cond);
3390 return T2;
3391}
3392
3393/// Try to fold an equality comparison with a {add/sub/xor} binary operation as
3394/// the 1st operand (N0). Callers are expected to swap the N0/N1 parameters to
3395/// handle the commuted versions of these patterns.
3396SDValue TargetLowering::foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1,
3397 ISD::CondCode Cond, const SDLoc &DL,
3398 DAGCombinerInfo &DCI) const {
3399 unsigned BOpcode = N0.getOpcode();
3400 assert((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) &&(static_cast <bool> ((BOpcode == ISD::ADD || BOpcode ==
ISD::SUB || BOpcode == ISD::XOR) && "Unexpected binop"
) ? void (0) : __assert_fail ("(BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) && \"Unexpected binop\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3401, __extension__ __PRETTY_FUNCTION__))
3401 "Unexpected binop")(static_cast <bool> ((BOpcode == ISD::ADD || BOpcode ==
ISD::SUB || BOpcode == ISD::XOR) && "Unexpected binop"
) ? void (0) : __assert_fail ("(BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) && \"Unexpected binop\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3401, __extension__ __PRETTY_FUNCTION__))
;
3402 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Unexpected condcode")(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Unexpected condcode") ? void (0) : __assert_fail
("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Unexpected condcode\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3402, __extension__ __PRETTY_FUNCTION__))
;
3403
3404 // (X + Y) == X --> Y == 0
3405 // (X - Y) == X --> Y == 0
3406 // (X ^ Y) == X --> Y == 0
3407 SelectionDAG &DAG = DCI.DAG;
3408 EVT OpVT = N0.getValueType();
3409 SDValue X = N0.getOperand(0);
3410 SDValue Y = N0.getOperand(1);
3411 if (X == N1)
3412 return DAG.getSetCC(DL, VT, Y, DAG.getConstant(0, DL, OpVT), Cond);
3413
3414 if (Y != N1)
3415 return SDValue();
3416
3417 // (X + Y) == Y --> X == 0
3418 // (X ^ Y) == Y --> X == 0
3419 if (BOpcode == ISD::ADD || BOpcode == ISD::XOR)
3420 return DAG.getSetCC(DL, VT, X, DAG.getConstant(0, DL, OpVT), Cond);
3421
3422 // The shift would not be valid if the operands are boolean (i1).
3423 if (!N0.hasOneUse() || OpVT.getScalarSizeInBits() == 1)
3424 return SDValue();
3425
3426 // (X - Y) == Y --> X == Y << 1
3427 EVT ShiftVT = getShiftAmountTy(OpVT, DAG.getDataLayout(),
3428 !DCI.isBeforeLegalize());
3429 SDValue One = DAG.getConstant(1, DL, ShiftVT);
3430 SDValue YShl1 = DAG.getNode(ISD::SHL, DL, N1.getValueType(), Y, One);
3431 if (!DCI.isCalledByLegalizer())
3432 DCI.AddToWorklist(YShl1.getNode());
3433 return DAG.getSetCC(DL, VT, X, YShl1, Cond);
3434}
3435
3436static SDValue simplifySetCCWithCTPOP(const TargetLowering &TLI, EVT VT,
3437 SDValue N0, const APInt &C1,
3438 ISD::CondCode Cond, const SDLoc &dl,
3439 SelectionDAG &DAG) {
3440 // Look through truncs that don't change the value of a ctpop.
3441 // FIXME: Add vector support? Need to be careful with setcc result type below.
3442 SDValue CTPOP = N0;
3443 if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && !VT.isVector() &&
3444 N0.getScalarValueSizeInBits() > Log2_32(N0.getOperand(0).getScalarValueSizeInBits()))
3445 CTPOP = N0.getOperand(0);
3446
3447 if (CTPOP.getOpcode() != ISD::CTPOP || !CTPOP.hasOneUse())
3448 return SDValue();
3449
3450 EVT CTVT = CTPOP.getValueType();
3451 SDValue CTOp = CTPOP.getOperand(0);
3452
3453 // If this is a vector CTPOP, keep the CTPOP if it is legal.
3454 // TODO: Should we check if CTPOP is legal(or custom) for scalars?
3455 if (VT.isVector() && TLI.isOperationLegal(ISD::CTPOP, CTVT))
3456 return SDValue();
3457
3458 // (ctpop x) u< 2 -> (x & x-1) == 0
3459 // (ctpop x) u> 1 -> (x & x-1) != 0
3460 if (Cond == ISD::SETULT || Cond == ISD::SETUGT) {
3461 unsigned CostLimit = TLI.getCustomCtpopCost(CTVT, Cond);
3462 if (C1.ugt(CostLimit + (Cond == ISD::SETULT)))
3463 return SDValue();
3464 if (C1 == 0 && (Cond == ISD::SETULT))
3465 return SDValue(); // This is handled elsewhere.
3466
3467 unsigned Passes = C1.getLimitedValue() - (Cond == ISD::SETULT);
3468
3469 SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
3470 SDValue Result = CTOp;
3471 for (unsigned i = 0; i < Passes; i++) {
3472 SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, Result, NegOne);
3473 Result = DAG.getNode(ISD::AND, dl, CTVT, Result, Add);
3474 }
3475 ISD::CondCode CC = Cond == ISD::SETULT ? ISD::SETEQ : ISD::SETNE;
3476 return DAG.getSetCC(dl, VT, Result, DAG.getConstant(0, dl, CTVT), CC);
3477 }
3478
3479 // If ctpop is not supported, expand a power-of-2 comparison based on it.
3480 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && C1 == 1) {
3481 // For scalars, keep CTPOP if it is legal or custom.
3482 if (!VT.isVector() && TLI.isOperationLegalOrCustom(ISD::CTPOP, CTVT))
3483 return SDValue();
3484 // This is based on X86's custom lowering for CTPOP which produces more
3485 // instructions than the expansion here.
3486
3487 // (ctpop x) == 1 --> (x != 0) && ((x & x-1) == 0)
3488 // (ctpop x) != 1 --> (x == 0) || ((x & x-1) != 0)
3489 SDValue Zero = DAG.getConstant(0, dl, CTVT);
3490 SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
3491 assert(CTVT.isInteger())(static_cast <bool> (CTVT.isInteger()) ? void (0) : __assert_fail
("CTVT.isInteger()", "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3491, __extension__ __PRETTY_FUNCTION__))
;
3492 ISD::CondCode InvCond = ISD::getSetCCInverse(Cond, CTVT);
3493 SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, CTOp, NegOne);
3494 SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Add);
3495 SDValue LHS = DAG.getSetCC(dl, VT, CTOp, Zero, InvCond);
3496 SDValue RHS = DAG.getSetCC(dl, VT, And, Zero, Cond);
3497 unsigned LogicOpcode = Cond == ISD::SETEQ ? ISD::AND : ISD::OR;
3498 return DAG.getNode(LogicOpcode, dl, VT, LHS, RHS);
3499 }
3500
3501 return SDValue();
3502}
3503
3504/// Try to simplify a setcc built with the specified operands and cc. If it is
3505/// unable to simplify it, return a null SDValue.
3506SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
3507 ISD::CondCode Cond, bool foldBooleans,
3508 DAGCombinerInfo &DCI,
3509 const SDLoc &dl) const {
3510 SelectionDAG &DAG = DCI.DAG;
3511 const DataLayout &Layout = DAG.getDataLayout();
3512 EVT OpVT = N0.getValueType();
3513
3514 // Constant fold or commute setcc.
3515 if (SDValue Fold = DAG.FoldSetCC(VT, N0, N1, Cond, dl))
3516 return Fold;
3517
3518 // Ensure that the constant occurs on the RHS and fold constant comparisons.
3519 // TODO: Handle non-splat vector constants. All undef causes trouble.
3520 // FIXME: We can't yet fold constant scalable vector splats, so avoid an
3521 // infinite loop here when we encounter one.
3522 ISD::CondCode SwappedCC = ISD::getSetCCSwappedOperands(Cond);
3523 if (isConstOrConstSplat(N0) &&
3524 (!OpVT.isScalableVector() || !isConstOrConstSplat(N1)) &&
3525 (DCI.isBeforeLegalizeOps() ||
3526 isCondCodeLegal(SwappedCC, N0.getSimpleValueType())))
3527 return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);
3528
3529 // If we have a subtract with the same 2 non-constant operands as this setcc
3530 // -- but in reverse order -- then try to commute the operands of this setcc
3531 // to match. A matching pair of setcc (cmp) and sub may be combined into 1
3532 // instruction on some targets.
3533 if (!isConstOrConstSplat(N0) && !isConstOrConstSplat(N1) &&
3534 (DCI.isBeforeLegalizeOps() ||
3535 isCondCodeLegal(SwappedCC, N0.getSimpleValueType())) &&
3536 DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N1, N0}) &&
3537 !DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N0, N1}))
3538 return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);
3539
3540 if (auto *N1C = isConstOrConstSplat(N1)) {
3541 const APInt &C1 = N1C->getAPIntValue();
3542
3543 // Optimize some CTPOP cases.
3544 if (SDValue V = simplifySetCCWithCTPOP(*this, VT, N0, C1, Cond, dl, DAG))
3545 return V;
3546
3547 // If the LHS is '(srl (ctlz x), 5)', the RHS is 0/1, and this is an
3548 // equality comparison, then we're just comparing whether X itself is
3549 // zero.
3550 if (N0.getOpcode() == ISD::SRL && (C1.isZero() || C1.isOne()) &&
3551 N0.getOperand(0).getOpcode() == ISD::CTLZ &&
3552 isPowerOf2_32(N0.getScalarValueSizeInBits())) {
3553 if (ConstantSDNode *ShAmt = isConstOrConstSplat(N0.getOperand(1))) {
3554 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3555 ShAmt->getAPIntValue() == Log2_32(N0.getScalarValueSizeInBits())) {
3556 if ((C1 == 0) == (Cond == ISD::SETEQ)) {
3557 // (srl (ctlz x), 5) == 0 -> X != 0
3558 // (srl (ctlz x), 5) != 1 -> X != 0
3559 Cond = ISD::SETNE;
3560 } else {
3561 // (srl (ctlz x), 5) != 0 -> X == 0
3562 // (srl (ctlz x), 5) == 1 -> X == 0
3563 Cond = ISD::SETEQ;
3564 }
3565 SDValue Zero = DAG.getConstant(0, dl, N0.getValueType());
3566 return DAG.getSetCC(dl, VT, N0.getOperand(0).getOperand(0), Zero,
3567 Cond);
3568 }
3569 }
3570 }
3571 }
3572
3573 // FIXME: Support vectors.
3574 if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
3575 const APInt &C1 = N1C->getAPIntValue();
3576
3577 // (zext x) == C --> x == (trunc C)
3578 // (sext x) == C --> x == (trunc C)
3579 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3580 DCI.isBeforeLegalize() && N0->hasOneUse()) {
3581 unsigned MinBits = N0.getValueSizeInBits();
3582 SDValue PreExt;
3583 bool Signed = false;
3584 if (N0->getOpcode() == ISD::ZERO_EXTEND) {
3585 // ZExt
3586 MinBits = N0->getOperand(0).getValueSizeInBits();
3587 PreExt = N0->getOperand(0);
3588 } else if (N0->getOpcode() == ISD::AND) {
3589 // DAGCombine turns costly ZExts into ANDs
3590 if (auto *C = dyn_cast<ConstantSDNode>(N0->getOperand(1)))
3591 if ((C->getAPIntValue()+1).isPowerOf2()) {
3592 MinBits = C->getAPIntValue().countTrailingOnes();
3593 PreExt = N0->getOperand(0);
3594 }
3595 } else if (N0->getOpcode() == ISD::SIGN_EXTEND) {
3596 // SExt
3597 MinBits = N0->getOperand(0).getValueSizeInBits();
3598 PreExt = N0->getOperand(0);
3599 Signed = true;
3600 } else if (auto *LN0 = dyn_cast<LoadSDNode>(N0)) {
3601 // ZEXTLOAD / SEXTLOAD
3602 if (LN0->getExtensionType() == ISD::ZEXTLOAD) {
3603 MinBits = LN0->getMemoryVT().getSizeInBits();
3604 PreExt = N0;
3605 } else if (LN0->getExtensionType() == ISD::SEXTLOAD) {
3606 Signed = true;
3607 MinBits = LN0->getMemoryVT().getSizeInBits();
3608 PreExt = N0;
3609 }
3610 }
3611
3612 // Figure out how many bits we need to preserve this constant.
3613 unsigned ReqdBits = Signed ?
3614 C1.getBitWidth() - C1.getNumSignBits() + 1 :
3615 C1.getActiveBits();
3616
3617 // Make sure we're not losing bits from the constant.
3618 if (MinBits > 0 &&
3619 MinBits < C1.getBitWidth() &&
3620 MinBits >= ReqdBits) {
3621 EVT MinVT = EVT::getIntegerVT(*DAG.getContext(), MinBits);
3622 if (isTypeDesirableForOp(ISD::SETCC, MinVT)) {
3623 // Will get folded away.
3624 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MinVT, PreExt);
3625 if (MinBits == 1 && C1 == 1)
3626 // Invert the condition.
3627 return DAG.getSetCC(dl, VT, Trunc, DAG.getConstant(0, dl, MVT::i1),
3628 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
3629 SDValue C = DAG.getConstant(C1.trunc(MinBits), dl, MinVT);
3630 return DAG.getSetCC(dl, VT, Trunc, C, Cond);
3631 }
3632
3633 // If truncating the setcc operands is not desirable, we can still
3634 // simplify the expression in some cases:
3635 // setcc ([sz]ext (setcc x, y, cc)), 0, setne) -> setcc (x, y, cc)
3636 // setcc ([sz]ext (setcc x, y, cc)), 0, seteq) -> setcc (x, y, inv(cc))
3637 // setcc (zext (setcc x, y, cc)), 1, setne) -> setcc (x, y, inv(cc))
3638 // setcc (zext (setcc x, y, cc)), 1, seteq) -> setcc (x, y, cc)
3639 // setcc (sext (setcc x, y, cc)), -1, setne) -> setcc (x, y, inv(cc))
3640 // setcc (sext (setcc x, y, cc)), -1, seteq) -> setcc (x, y, cc)
3641 SDValue TopSetCC = N0->getOperand(0);
3642 unsigned N0Opc = N0->getOpcode();
3643 bool SExt = (N0Opc == ISD::SIGN_EXTEND);
3644 if (TopSetCC.getValueType() == MVT::i1 && VT == MVT::i1 &&
3645 TopSetCC.getOpcode() == ISD::SETCC &&
3646 (N0Opc == ISD::ZERO_EXTEND || N0Opc == ISD::SIGN_EXTEND) &&
3647 (isConstFalseVal(N1C) ||
3648 isExtendedTrueVal(N1C, N0->getValueType(0), SExt))) {
3649
3650 bool Inverse = (N1C->isZero() && Cond == ISD::SETEQ) ||
3651 (!N1C->isZero() && Cond == ISD::SETNE);
3652
3653 if (!Inverse)
3654 return TopSetCC;
3655
3656 ISD::CondCode InvCond = ISD::getSetCCInverse(
3657 cast<CondCodeSDNode>(TopSetCC.getOperand(2))->get(),
3658 TopSetCC.getOperand(0).getValueType());
3659 return DAG.getSetCC(dl, VT, TopSetCC.getOperand(0),
3660 TopSetCC.getOperand(1),
3661 InvCond);
3662 }
3663 }
3664 }
3665
3666 // If the LHS is '(and load, const)', the RHS is 0, the test is for
3667 // equality or unsigned, and all 1 bits of the const are in the same
3668 // partial word, see if we can shorten the load.
3669 if (DCI.isBeforeLegalize() &&
3670 !ISD::isSignedIntSetCC(Cond) &&
3671 N0.getOpcode() == ISD::AND && C1 == 0 &&
3672 N0.getNode()->hasOneUse() &&
3673 isa<LoadSDNode>(N0.getOperand(0)) &&
3674 N0.getOperand(0).getNode()->hasOneUse() &&
3675 isa<ConstantSDNode>(N0.getOperand(1))) {
3676 LoadSDNode *Lod = cast<LoadSDNode>(N0.getOperand(0));
3677 APInt bestMask;
3678 unsigned bestWidth = 0, bestOffset = 0;
3679 if (Lod->isSimple() && Lod->isUnindexed()) {
3680 unsigned origWidth = N0.getValueSizeInBits();
3681 unsigned maskWidth = origWidth;
3682 // We can narrow (e.g.) 16-bit extending loads on 32-bit target to
3683 // 8 bits, but have to be careful...
3684 if (Lod->getExtensionType() != ISD::NON_EXTLOAD)
3685 origWidth = Lod->getMemoryVT().getSizeInBits();
3686 const APInt &Mask = N0.getConstantOperandAPInt(1);
3687 for (unsigned width = origWidth / 2; width>=8; width /= 2) {
3688 APInt newMask = APInt::getLowBitsSet(maskWidth, width);
3689 for (unsigned offset=0; offset<origWidth/width; offset++) {
3690 if (Mask.isSubsetOf(newMask)) {
3691 if (Layout.isLittleEndian())
3692 bestOffset = (uint64_t)offset * (width/8);
3693 else
3694 bestOffset = (origWidth/width - offset - 1) * (width/8);
3695 bestMask = Mask.lshr(offset * (width/8) * 8);
3696 bestWidth = width;
3697 break;
3698 }
3699 newMask <<= width;
3700 }
3701 }
3702 }
3703 if (bestWidth) {
3704 EVT newVT = EVT::getIntegerVT(*DAG.getContext(), bestWidth);
3705 if (newVT.isRound() &&
3706 shouldReduceLoadWidth(Lod, ISD::NON_EXTLOAD, newVT)) {
3707 SDValue Ptr = Lod->getBasePtr();
3708 if (bestOffset != 0)
3709 Ptr =
3710 DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(bestOffset), dl);
3711 SDValue NewLoad =
3712 DAG.getLoad(newVT, dl, Lod->getChain(), Ptr,
3713 Lod->getPointerInfo().getWithOffset(bestOffset),
3714 Lod->getOriginalAlign());
3715 return DAG.getSetCC(dl, VT,
3716 DAG.getNode(ISD::AND, dl, newVT, NewLoad,
3717 DAG.getConstant(bestMask.trunc(bestWidth),
3718 dl, newVT)),
3719 DAG.getConstant(0LL, dl, newVT), Cond);
3720 }
3721 }
3722 }
3723
3724 // If the LHS is a ZERO_EXTEND, perform the comparison on the input.
3725 if (N0.getOpcode() == ISD::ZERO_EXTEND) {
3726 unsigned InSize = N0.getOperand(0).getValueSizeInBits();
3727
3728 // If the comparison constant has bits in the upper part, the
3729 // zero-extended value could never match.
3730 if (C1.intersects(APInt::getHighBitsSet(C1.getBitWidth(),
3731 C1.getBitWidth() - InSize))) {
3732 switch (Cond) {
3733 case ISD::SETUGT:
3734 case ISD::SETUGE:
3735 case ISD::SETEQ:
3736 return DAG.getConstant(0, dl, VT);
3737 case ISD::SETULT:
3738 case ISD::SETULE:
3739 case ISD::SETNE:
3740 return DAG.getConstant(1, dl, VT);
3741 case ISD::SETGT:
3742 case ISD::SETGE:
3743 // True if the sign bit of C1 is set.
3744 return DAG.getConstant(C1.isNegative(), dl, VT);
3745 case ISD::SETLT:
3746 case ISD::SETLE:
3747 // True if the sign bit of C1 isn't set.
3748 return DAG.getConstant(C1.isNonNegative(), dl, VT);
3749 default:
3750 break;
3751 }
3752 }
3753
3754 // Otherwise, we can perform the comparison with the low bits.
3755 switch (Cond) {
3756 case ISD::SETEQ:
3757 case ISD::SETNE:
3758 case ISD::SETUGT:
3759 case ISD::SETUGE:
3760 case ISD::SETULT:
3761 case ISD::SETULE: {
3762 EVT newVT = N0.getOperand(0).getValueType();
3763 if (DCI.isBeforeLegalizeOps() ||
3764 (isOperationLegal(ISD::SETCC, newVT) &&
3765 isCondCodeLegal(Cond, newVT.getSimpleVT()))) {
3766 EVT NewSetCCVT = getSetCCResultType(Layout, *DAG.getContext(), newVT);
3767 SDValue NewConst = DAG.getConstant(C1.trunc(InSize), dl, newVT);
3768
3769 SDValue NewSetCC = DAG.getSetCC(dl, NewSetCCVT, N0.getOperand(0),
3770 NewConst, Cond);
3771 return DAG.getBoolExtOrTrunc(NewSetCC, dl, VT, N0.getValueType());
3772 }
3773 break;
3774 }
3775 default:
3776 break; // todo, be more careful with signed comparisons
3777 }
3778 } else if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
3779 (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3780 !isSExtCheaperThanZExt(cast<VTSDNode>(N0.getOperand(1))->getVT(),
3781 OpVT)) {
3782 EVT ExtSrcTy = cast<VTSDNode>(N0.getOperand(1))->getVT();
3783 unsigned ExtSrcTyBits = ExtSrcTy.getSizeInBits();
3784 EVT ExtDstTy = N0.getValueType();
3785 unsigned ExtDstTyBits = ExtDstTy.getSizeInBits();
3786
3787 // If the constant doesn't fit into the number of bits for the source of
3788 // the sign extension, it is impossible for both sides to be equal.
3789 if (C1.getMinSignedBits() > ExtSrcTyBits)
3790 return DAG.getBoolConstant(Cond == ISD::SETNE, dl, VT, OpVT);
3791
3792 assert(ExtDstTy == N0.getOperand(0).getValueType() &&(static_cast <bool> (ExtDstTy == N0.getOperand(0).getValueType
() && ExtDstTy != ExtSrcTy && "Unexpected types!"
) ? void (0) : __assert_fail ("ExtDstTy == N0.getOperand(0).getValueType() && ExtDstTy != ExtSrcTy && \"Unexpected types!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3793, __extension__ __PRETTY_FUNCTION__))
3793 ExtDstTy != ExtSrcTy && "Unexpected types!")(static_cast <bool> (ExtDstTy == N0.getOperand(0).getValueType
() && ExtDstTy != ExtSrcTy && "Unexpected types!"
) ? void (0) : __assert_fail ("ExtDstTy == N0.getOperand(0).getValueType() && ExtDstTy != ExtSrcTy && \"Unexpected types!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3793, __extension__ __PRETTY_FUNCTION__))
;
3794 APInt Imm = APInt::getLowBitsSet(ExtDstTyBits, ExtSrcTyBits);
3795 SDValue ZextOp = DAG.getNode(ISD::AND, dl, ExtDstTy, N0.getOperand(0),
3796 DAG.getConstant(Imm, dl, ExtDstTy));
3797 if (!DCI.isCalledByLegalizer())
3798 DCI.AddToWorklist(ZextOp.getNode());
3799 // Otherwise, make this a use of a zext.
3800 return DAG.getSetCC(dl, VT, ZextOp,
3801 DAG.getConstant(C1 & Imm, dl, ExtDstTy), Cond);
3802 } else if ((N1C->isZero() || N1C->isOne()) &&
3803 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
3804 // SETCC (SETCC), [0|1], [EQ|NE] -> SETCC
3805 if (N0.getOpcode() == ISD::SETCC &&
3806 isTypeLegal(VT) && VT.bitsLE(N0.getValueType()) &&
3807 (N0.getValueType() == MVT::i1 ||
3808 getBooleanContents(N0.getOperand(0).getValueType()) ==
3809 ZeroOrOneBooleanContent)) {
3810 bool TrueWhenTrue = (Cond == ISD::SETEQ) ^ (!N1C->isOne());
3811 if (TrueWhenTrue)
3812 return DAG.getNode(ISD::TRUNCATE, dl, VT, N0);
3813 // Invert the condition.
3814 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
3815 CC = ISD::getSetCCInverse(CC, N0.getOperand(0).getValueType());
3816 if (DCI.isBeforeLegalizeOps() ||
3817 isCondCodeLegal(CC, N0.getOperand(0).getSimpleValueType()))
3818 return DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC);
3819 }
3820
3821 if ((N0.getOpcode() == ISD::XOR ||
3822 (N0.getOpcode() == ISD::AND &&
3823 N0.getOperand(0).getOpcode() == ISD::XOR &&
3824 N0.getOperand(1) == N0.getOperand(0).getOperand(1))) &&
3825 isOneConstant(N0.getOperand(1))) {
3826 // If this is (X^1) == 0/1, swap the RHS and eliminate the xor. We
3827 // can only do this if the top bits are known zero.
3828 unsigned BitWidth = N0.getValueSizeInBits();
3829 if (DAG.MaskedValueIsZero(N0,
3830 APInt::getHighBitsSet(BitWidth,
3831 BitWidth-1))) {
3832 // Okay, get the un-inverted input value.
3833 SDValue Val;
3834 if (N0.getOpcode() == ISD::XOR) {
3835 Val = N0.getOperand(0);
3836 } else {
3837 assert(N0.getOpcode() == ISD::AND &&(static_cast <bool> (N0.getOpcode() == ISD::AND &&
N0.getOperand(0).getOpcode() == ISD::XOR) ? void (0) : __assert_fail
("N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::XOR"
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3838, __extension__ __PRETTY_FUNCTION__))
3838 N0.getOperand(0).getOpcode() == ISD::XOR)(static_cast <bool> (N0.getOpcode() == ISD::AND &&
N0.getOperand(0).getOpcode() == ISD::XOR) ? void (0) : __assert_fail
("N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::XOR"
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3838, __extension__ __PRETTY_FUNCTION__))
;
3839 // ((X^1)&1)^1 -> X & 1
3840 Val = DAG.getNode(ISD::AND, dl, N0.getValueType(),
3841 N0.getOperand(0).getOperand(0),
3842 N0.getOperand(1));
3843 }
3844
3845 return DAG.getSetCC(dl, VT, Val, N1,
3846 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
3847 }
3848 } else if (N1C->isOne()) {
3849 SDValue Op0 = N0;
3850 if (Op0.getOpcode() == ISD::TRUNCATE)
3851 Op0 = Op0.getOperand(0);
3852
3853 if ((Op0.getOpcode() == ISD::XOR) &&
3854 Op0.getOperand(0).getOpcode() == ISD::SETCC &&
3855 Op0.getOperand(1).getOpcode() == ISD::SETCC) {
3856 SDValue XorLHS = Op0.getOperand(0);
3857 SDValue XorRHS = Op0.getOperand(1);
3858 // Ensure that the input setccs return an i1 type or 0/1 value.
3859 if (Op0.getValueType() == MVT::i1 ||
3860 (getBooleanContents(XorLHS.getOperand(0).getValueType()) ==
3861 ZeroOrOneBooleanContent &&
3862 getBooleanContents(XorRHS.getOperand(0).getValueType()) ==
3863 ZeroOrOneBooleanContent)) {
3864 // (xor (setcc), (setcc)) == / != 1 -> (setcc) != / == (setcc)
3865 Cond = (Cond == ISD::SETEQ) ? ISD::SETNE : ISD::SETEQ;
3866 return DAG.getSetCC(dl, VT, XorLHS, XorRHS, Cond);
3867 }
3868 }
3869 if (Op0.getOpcode() == ISD::AND && isOneConstant(Op0.getOperand(1))) {
3870 // If this is (X&1) == / != 1, normalize it to (X&1) != / == 0.
3871 if (Op0.getValueType().bitsGT(VT))
3872 Op0 = DAG.getNode(ISD::AND, dl, VT,
3873 DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)),
3874 DAG.getConstant(1, dl, VT));
3875 else if (Op0.getValueType().bitsLT(VT))
3876 Op0 = DAG.getNode(ISD::AND, dl, VT,
3877 DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)),
3878 DAG.getConstant(1, dl, VT));
3879
3880 return DAG.getSetCC(dl, VT, Op0,
3881 DAG.getConstant(0, dl, Op0.getValueType()),
3882 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
3883 }
3884 if (Op0.getOpcode() == ISD::AssertZext &&
3885 cast<VTSDNode>(Op0.getOperand(1))->getVT() == MVT::i1)
3886 return DAG.getSetCC(dl, VT, Op0,
3887 DAG.getConstant(0, dl, Op0.getValueType()),
3888 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
3889 }
3890 }
3891
3892 // Given:
3893 // icmp eq/ne (urem %x, %y), 0
3894 // Iff %x has 0 or 1 bits set, and %y has at least 2 bits set, omit 'urem':
3895 // icmp eq/ne %x, 0
3896 if (N0.getOpcode() == ISD::UREM && N1C->isZero() &&
3897 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
3898 KnownBits XKnown = DAG.computeKnownBits(N0.getOperand(0));
3899 KnownBits YKnown = DAG.computeKnownBits(N0.getOperand(1));
3900 if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2)
3901 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1, Cond);
3902 }
3903
3904 // Fold set_cc seteq (ashr X, BW-1), -1 -> set_cc setlt X, 0
3905 // and set_cc setne (ashr X, BW-1), -1 -> set_cc setge X, 0
3906 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3907 N0.getOpcode() == ISD::SRA && isa<ConstantSDNode>(N0.getOperand(1)) &&
3908 N0.getConstantOperandAPInt(1) == OpVT.getScalarSizeInBits() - 1 &&
3909 N1C && N1C->isAllOnes()) {
3910 return DAG.getSetCC(dl, VT, N0.getOperand(0),
3911 DAG.getConstant(0, dl, OpVT),
3912 Cond == ISD::SETEQ ? ISD::SETLT : ISD::SETGE);
3913 }
3914
3915 if (SDValue V =
3916 optimizeSetCCOfSignedTruncationCheck(VT, N0, N1, Cond, DCI, dl))
3917 return V;
3918 }
3919
3920 // These simplifications apply to splat vectors as well.
3921 // TODO: Handle more splat vector cases.
3922 if (auto *N1C = isConstOrConstSplat(N1)) {
3923 const APInt &C1 = N1C->getAPIntValue();
3924
3925 APInt MinVal, MaxVal;
3926 unsigned OperandBitSize = N1C->getValueType(0).getScalarSizeInBits();
3927 if (ISD::isSignedIntSetCC(Cond)) {
3928 MinVal = APInt::getSignedMinValue(OperandBitSize);
3929 MaxVal = APInt::getSignedMaxValue(OperandBitSize);
3930 } else {
3931 MinVal = APInt::getMinValue(OperandBitSize);
3932 MaxVal = APInt::getMaxValue(OperandBitSize);
3933 }
3934
3935 // Canonicalize GE/LE comparisons to use GT/LT comparisons.
3936 if (Cond == ISD::SETGE || Cond == ISD::SETUGE) {
3937 // X >= MIN --> true
3938 if (C1 == MinVal)
3939 return DAG.getBoolConstant(true, dl, VT, OpVT);
3940
3941 if (!VT.isVector()) { // TODO: Support this for vectors.
3942 // X >= C0 --> X > (C0 - 1)
3943 APInt C = C1 - 1;
3944 ISD::CondCode NewCC = (Cond == ISD::SETGE) ? ISD::SETGT : ISD::SETUGT;
3945 if ((DCI.isBeforeLegalizeOps() ||
3946 isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
3947 (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
3948 isLegalICmpImmediate(C.getSExtValue())))) {
3949 return DAG.getSetCC(dl, VT, N0,
3950 DAG.getConstant(C, dl, N1.getValueType()),
3951 NewCC);
3952 }
3953 }
3954 }
3955
3956 if (Cond == ISD::SETLE || Cond == ISD::SETULE) {
3957 // X <= MAX --> true
3958 if (C1 == MaxVal)
3959 return DAG.getBoolConstant(true, dl, VT, OpVT);
3960
3961 // X <= C0 --> X < (C0 + 1)
3962 if (!VT.isVector()) { // TODO: Support this for vectors.
3963 APInt C = C1 + 1;
3964 ISD::CondCode NewCC = (Cond == ISD::SETLE) ? ISD::SETLT : ISD::SETULT;
3965 if ((DCI.isBeforeLegalizeOps() ||
3966 isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
3967 (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
3968 isLegalICmpImmediate(C.getSExtValue())))) {
3969 return DAG.getSetCC(dl, VT, N0,
3970 DAG.getConstant(C, dl, N1.getValueType()),
3971 NewCC);
3972 }
3973 }
3974 }
3975
3976 if (Cond == ISD::SETLT || Cond == ISD::SETULT) {
3977 if (C1 == MinVal)
3978 return DAG.getBoolConstant(false, dl, VT, OpVT); // X < MIN --> false
3979
3980 // TODO: Support this for vectors after legalize ops.
3981 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
3982 // Canonicalize setlt X, Max --> setne X, Max
3983 if (C1 == MaxVal)
3984 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
3985
3986 // If we have setult X, 1, turn it into seteq X, 0
3987 if (C1 == MinVal+1)
3988 return DAG.getSetCC(dl, VT, N0,
3989 DAG.getConstant(MinVal, dl, N0.getValueType()),
3990 ISD::SETEQ);
3991 }
3992 }
3993
3994 if (Cond == ISD::SETGT || Cond == ISD::SETUGT) {
3995 if (C1 == MaxVal)
3996 return DAG.getBoolConstant(false, dl, VT, OpVT); // X > MAX --> false
3997
3998 // TODO: Support this for vectors after legalize ops.
3999 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
4000 // Canonicalize setgt X, Min --> setne X, Min
4001 if (C1 == MinVal)
4002 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
4003
4004 // If we have setugt X, Max-1, turn it into seteq X, Max
4005 if (C1 == MaxVal-1)
4006 return DAG.getSetCC(dl, VT, N0,
4007 DAG.getConstant(MaxVal, dl, N0.getValueType()),
4008 ISD::SETEQ);
4009 }
4010 }
4011
4012 if (Cond == ISD::SETEQ || Cond == ISD::SETNE) {
4013 // (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
4014 if (C1.isZero())
4015 if (SDValue CC = optimizeSetCCByHoistingAndByConstFromLogicalShift(
4016 VT, N0, N1, Cond, DCI, dl))
4017 return CC;
4018
4019 // For all/any comparisons, replace or(x,shl(y,bw/2)) with and/or(x,y).
4020 // For example, when high 32-bits of i64 X are known clear:
4021 // all bits clear: (X | (Y<<32)) == 0 --> (X | Y) == 0
4022 // all bits set: (X | (Y<<32)) == -1 --> (X & Y) == -1
4023 bool CmpZero = N1C->getAPIntValue().isZero();
4024 bool CmpNegOne = N1C->getAPIntValue().isAllOnes();
4025 if ((CmpZero || CmpNegOne) && N0.hasOneUse()) {
4026 // Match or(lo,shl(hi,bw/2)) pattern.
4027 auto IsConcat = [&](SDValue V, SDValue &Lo, SDValue &Hi) {
4028 unsigned EltBits = V.getScalarValueSizeInBits();
4029 if (V.getOpcode() != ISD::OR || (EltBits % 2) != 0)
4030 return false;
4031 SDValue LHS = V.getOperand(0);
4032 SDValue RHS = V.getOperand(1);
4033 APInt HiBits = APInt::getHighBitsSet(EltBits, EltBits / 2);
4034 // Unshifted element must have zero upperbits.
4035 if (RHS.getOpcode() == ISD::SHL &&
4036 isa<ConstantSDNode>(RHS.getOperand(1)) &&
4037 RHS.getConstantOperandAPInt(1) == (EltBits / 2) &&
4038 DAG.MaskedValueIsZero(LHS, HiBits)) {
4039 Lo = LHS;
4040 Hi = RHS.getOperand(0);
4041 return true;
4042 }
4043 if (LHS.getOpcode() == ISD::SHL &&
4044 isa<ConstantSDNode>(LHS.getOperand(1)) &&
4045 LHS.getConstantOperandAPInt(1) == (EltBits / 2) &&
4046 DAG.MaskedValueIsZero(RHS, HiBits)) {
4047 Lo = RHS;
4048 Hi = LHS.getOperand(0);
4049 return true;
4050 }
4051 return false;
4052 };
4053
4054 auto MergeConcat = [&](SDValue Lo, SDValue Hi) {
4055 unsigned EltBits = N0.getScalarValueSizeInBits();
4056 unsigned HalfBits = EltBits / 2;
4057 APInt HiBits = APInt::getHighBitsSet(EltBits, HalfBits);
4058 SDValue LoBits = DAG.getConstant(~HiBits, dl, OpVT);
4059 SDValue HiMask = DAG.getNode(ISD::AND, dl, OpVT, Hi, LoBits);
4060 SDValue NewN0 =
4061 DAG.getNode(CmpZero ? ISD::OR : ISD::AND, dl, OpVT, Lo, HiMask);
4062 SDValue NewN1 = CmpZero ? DAG.getConstant(0, dl, OpVT) : LoBits;
4063 return DAG.getSetCC(dl, VT, NewN0, NewN1, Cond);
4064 };
4065
4066 SDValue Lo, Hi;
4067 if (IsConcat(N0, Lo, Hi))
4068 return MergeConcat(Lo, Hi);
4069
4070 if (N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR) {
4071 SDValue Lo0, Lo1, Hi0, Hi1;
4072 if (IsConcat(N0.getOperand(0), Lo0, Hi0) &&
4073 IsConcat(N0.getOperand(1), Lo1, Hi1)) {
4074 return MergeConcat(DAG.getNode(N0.getOpcode(), dl, OpVT, Lo0, Lo1),
4075 DAG.getNode(N0.getOpcode(), dl, OpVT, Hi0, Hi1));
4076 }
4077 }
4078 }
4079 }
4080
4081 // If we have "setcc X, C0", check to see if we can shrink the immediate
4082 // by changing cc.
4083 // TODO: Support this for vectors after legalize ops.
4084 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
4085 // SETUGT X, SINTMAX -> SETLT X, 0
4086 // SETUGE X, SINTMIN -> SETLT X, 0
4087 if ((Cond == ISD::SETUGT && C1.isMaxSignedValue()) ||
4088 (Cond == ISD::SETUGE && C1.isMinSignedValue()))
4089 return DAG.getSetCC(dl, VT, N0,
4090 DAG.getConstant(0, dl, N1.getValueType()),
4091 ISD::SETLT);
4092
4093 // SETULT X, SINTMIN -> SETGT X, -1
4094 // SETULE X, SINTMAX -> SETGT X, -1
4095 if ((Cond == ISD::SETULT && C1.isMinSignedValue()) ||
4096 (Cond == ISD::SETULE && C1.isMaxSignedValue()))
4097 return DAG.getSetCC(dl, VT, N0,
4098 DAG.getAllOnesConstant(dl, N1.getValueType()),
4099 ISD::SETGT);
4100 }
4101 }
4102
4103 // Back to non-vector simplifications.
4104 // TODO: Can we do these for vector splats?
4105 if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
4106 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4107 const APInt &C1 = N1C->getAPIntValue();
4108 EVT ShValTy = N0.getValueType();
4109
4110 // Fold bit comparisons when we can. This will result in an
4111 // incorrect value when boolean false is negative one, unless
4112 // the bitsize is 1 in which case the false value is the same
4113 // in practice regardless of the representation.
4114 if ((VT.getSizeInBits() == 1 ||
4115 getBooleanContents(N0.getValueType()) == ZeroOrOneBooleanContent) &&
4116 (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4117 (VT == ShValTy || (isTypeLegal(VT) && VT.bitsLE(ShValTy))) &&
4118 N0.getOpcode() == ISD::AND) {
4119 if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
4120 EVT ShiftTy =
4121 getShiftAmountTy(ShValTy, Layout, !DCI.isBeforeLegalize());
4122 if (Cond == ISD::SETNE && C1 == 0) {// (X & 8) != 0 --> (X & 8) >> 3
4123 // Perform the xform if the AND RHS is a single bit.
4124 unsigned ShCt = AndRHS->getAPIntValue().logBase2();
4125 if (AndRHS->getAPIntValue().isPowerOf2() &&
4126 !TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {
4127 return DAG.getNode(ISD::TRUNCATE, dl, VT,
4128 DAG.getNode(ISD::SRL, dl, ShValTy, N0,
4129 DAG.getConstant(ShCt, dl, ShiftTy)));
4130 }
4131 } else if (Cond == ISD::SETEQ && C1 == AndRHS->getAPIntValue()) {
4132 // (X & 8) == 8 --> (X & 8) >> 3
4133 // Perform the xform if C1 is a single bit.
4134 unsigned ShCt = C1.logBase2();
4135 if (C1.isPowerOf2() &&
4136 !TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {
4137 return DAG.getNode(ISD::TRUNCATE, dl, VT,
4138 DAG.getNode(ISD::SRL, dl, ShValTy, N0,
4139 DAG.getConstant(ShCt, dl, ShiftTy)));
4140 }
4141 }
4142 }
4143 }
4144
4145 if (C1.getMinSignedBits() <= 64 &&
4146 !isLegalICmpImmediate(C1.getSExtValue())) {
4147 EVT ShiftTy = getShiftAmountTy(ShValTy, Layout, !DCI.isBeforeLegalize());
4148 // (X & -256) == 256 -> (X >> 8) == 1
4149 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4150 N0.getOpcode() == ISD::AND && N0.hasOneUse()) {
4151 if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
4152 const APInt &AndRHSC = AndRHS->getAPIntValue();
4153 if ((-AndRHSC).isPowerOf2() && (AndRHSC & C1) == C1) {
4154 unsigned ShiftBits = AndRHSC.countTrailingZeros();
4155 if (!TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
4156 SDValue Shift =
4157 DAG.getNode(ISD::SRL, dl, ShValTy, N0.getOperand(0),
4158 DAG.getConstant(ShiftBits, dl, ShiftTy));
4159 SDValue CmpRHS = DAG.getConstant(C1.lshr(ShiftBits), dl, ShValTy);
4160 return DAG.getSetCC(dl, VT, Shift, CmpRHS, Cond);
4161 }
4162 }
4163 }
4164 } else if (Cond == ISD::SETULT || Cond == ISD::SETUGE ||
4165 Cond == ISD::SETULE || Cond == ISD::SETUGT) {
4166 bool AdjOne = (Cond == ISD::SETULE || Cond == ISD::SETUGT);
4167 // X < 0x100000000 -> (X >> 32) < 1
4168 // X >= 0x100000000 -> (X >> 32) >= 1
4169 // X <= 0x0ffffffff -> (X >> 32) < 1
4170 // X > 0x0ffffffff -> (X >> 32) >= 1
4171 unsigned ShiftBits;
4172 APInt NewC = C1;
4173 ISD::CondCode NewCond = Cond;
4174 if (AdjOne) {
4175 ShiftBits = C1.countTrailingOnes();
4176 NewC = NewC + 1;
4177 NewCond = (Cond == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
4178 } else {
4179 ShiftBits = C1.countTrailingZeros();
4180 }
4181 NewC.lshrInPlace(ShiftBits);
4182 if (ShiftBits && NewC.getMinSignedBits() <= 64 &&
4183 isLegalICmpImmediate(NewC.getSExtValue()) &&
4184 !TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
4185 SDValue Shift = DAG.getNode(ISD::SRL, dl, ShValTy, N0,
4186 DAG.getConstant(ShiftBits, dl, ShiftTy));
4187 SDValue CmpRHS = DAG.getConstant(NewC, dl, ShValTy);
4188 return DAG.getSetCC(dl, VT, Shift, CmpRHS, NewCond);
4189 }
4190 }
4191 }
4192 }
4193
4194 if (!isa<ConstantFPSDNode>(N0) && isa<ConstantFPSDNode>(N1)) {
4195 auto *CFP = cast<ConstantFPSDNode>(N1);
4196 assert(!CFP->getValueAPF().isNaN() && "Unexpected NaN value")(static_cast <bool> (!CFP->getValueAPF().isNaN() &&
"Unexpected NaN value") ? void (0) : __assert_fail ("!CFP->getValueAPF().isNaN() && \"Unexpected NaN value\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4196, __extension__ __PRETTY_FUNCTION__))
;
4197
4198 // Otherwise, we know the RHS is not a NaN. Simplify the node to drop the
4199 // constant if knowing that the operand is non-nan is enough. We prefer to
4200 // have SETO(x,x) instead of SETO(x, 0.0) because this avoids having to
4201 // materialize 0.0.
4202 if (Cond == ISD::SETO || Cond == ISD::SETUO)
4203 return DAG.getSetCC(dl, VT, N0, N0, Cond);
4204
4205 // setcc (fneg x), C -> setcc swap(pred) x, -C
4206 if (N0.getOpcode() == ISD::FNEG) {
4207 ISD::CondCode SwapCond = ISD::getSetCCSwappedOperands(Cond);
4208 if (DCI.isBeforeLegalizeOps() ||
4209 isCondCodeLegal(SwapCond, N0.getSimpleValueType())) {
4210 SDValue NegN1 = DAG.getNode(ISD::FNEG, dl, N0.getValueType(), N1);
4211 return DAG.getSetCC(dl, VT, N0.getOperand(0), NegN1, SwapCond);
4212 }
4213 }
4214
4215 // If the condition is not legal, see if we can find an equivalent one
4216 // which is legal.
4217 if (!isCondCodeLegal(Cond, N0.getSimpleValueType())) {
4218 // If the comparison was an awkward floating-point == or != and one of
4219 // the comparison operands is infinity or negative infinity, convert the
4220 // condition to a less-awkward <= or >=.
4221 if (CFP->getValueAPF().isInfinity()) {
4222 bool IsNegInf = CFP->getValueAPF().isNegative();
4223 ISD::CondCode NewCond = ISD::SETCC_INVALID;
4224 switch (Cond) {
4225 case ISD::SETOEQ: NewCond = IsNegInf ? ISD::SETOLE : ISD::SETOGE; break;
4226 case ISD::SETUEQ: NewCond = IsNegInf ? ISD::SETULE : ISD::SETUGE; break;
4227 case ISD::SETUNE: NewCond = IsNegInf ? ISD::SETUGT : ISD::SETULT; break;
4228 case ISD::SETONE: NewCond = IsNegInf ? ISD::SETOGT : ISD::SETOLT; break;
4229 default: break;
4230 }
4231 if (NewCond != ISD::SETCC_INVALID &&
4232 isCondCodeLegal(NewCond, N0.getSimpleValueType()))
4233 return DAG.getSetCC(dl, VT, N0, N1, NewCond);
4234 }
4235 }
4236 }
4237
4238 if (N0 == N1) {
4239 // The sext(setcc()) => setcc() optimization relies on the appropriate
4240 // constant being emitted.
4241 assert(!N0.getValueType().isInteger() &&(static_cast <bool> (!N0.getValueType().isInteger() &&
"Integer types should be handled by FoldSetCC") ? void (0) :
__assert_fail ("!N0.getValueType().isInteger() && \"Integer types should be handled by FoldSetCC\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4242, __extension__ __PRETTY_FUNCTION__))
4242 "Integer types should be handled by FoldSetCC")(static_cast <bool> (!N0.getValueType().isInteger() &&
"Integer types should be handled by FoldSetCC") ? void (0) :
__assert_fail ("!N0.getValueType().isInteger() && \"Integer types should be handled by FoldSetCC\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4242, __extension__ __PRETTY_FUNCTION__))
;
4243
4244 bool EqTrue = ISD::isTrueWhenEqual(Cond);
4245 unsigned UOF = ISD::getUnorderedFlavor(Cond);
4246 if (UOF == 2) // FP operators that are undefined on NaNs.
4247 return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
4248 if (UOF == unsigned(EqTrue))
4249 return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
4250 // Otherwise, we can't fold it. However, we can simplify it to SETUO/SETO
4251 // if it is not already.
4252 ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO;
4253 if (NewCond != Cond &&
4254 (DCI.isBeforeLegalizeOps() ||
4255 isCondCodeLegal(NewCond, N0.getSimpleValueType())))
4256 return DAG.getSetCC(dl, VT, N0, N1, NewCond);
4257 }
4258
4259 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4260 N0.getValueType().isInteger()) {
4261 if (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB ||
4262 N0.getOpcode() == ISD::XOR) {
4263 // Simplify (X+Y) == (X+Z) --> Y == Z
4264 if (N0.getOpcode() == N1.getOpcode()) {
4265 if (N0.getOperand(0) == N1.getOperand(0))
4266 return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(1), Cond);
4267 if (N0.getOperand(1) == N1.getOperand(1))
4268 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(0), Cond);
4269 if (isCommutativeBinOp(N0.getOpcode())) {
4270 // If X op Y == Y op X, try other combinations.
4271 if (N0.getOperand(0) == N1.getOperand(1))
4272 return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(0),
4273 Cond);
4274 if (N0.getOperand(1) == N1.getOperand(0))
4275 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(1),
4276 Cond);
4277 }
4278 }
4279
4280 // If RHS is a legal immediate value for a compare instruction, we need
4281 // to be careful about increasing register pressure needlessly.
4282 bool LegalRHSImm = false;
4283
4284 if (auto *RHSC = dyn_cast<ConstantSDNode>(N1)) {
4285 if (auto *LHSR = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
4286 // Turn (X+C1) == C2 --> X == C2-C1
4287 if (N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse()) {
4288 return DAG.getSetCC(dl, VT, N0.getOperand(0),
4289 DAG.getConstant(RHSC->getAPIntValue()-
4290 LHSR->getAPIntValue(),
4291 dl, N0.getValueType()), Cond);
4292 }
4293
4294 // Turn (X^C1) == C2 into X == C1^C2 iff X&~C1 = 0.
4295 if (N0.getOpcode() == ISD::XOR)
4296 // If we know that all of the inverted bits are zero, don't bother
4297 // performing the inversion.
4298 if (DAG.MaskedValueIsZero(N0.getOperand(0), ~LHSR->getAPIntValue()))
4299 return
4300 DAG.getSetCC(dl, VT, N0.getOperand(0),
4301 DAG.getConstant(LHSR->getAPIntValue() ^
4302 RHSC->getAPIntValue(),
4303 dl, N0.getValueType()),
4304 Cond);
4305 }
4306
4307 // Turn (C1-X) == C2 --> X == C1-C2
4308 if (auto *SUBC = dyn_cast<ConstantSDNode>(N0.getOperand(0))) {
4309 if (N0.getOpcode() == ISD::SUB && N0.getNode()->hasOneUse()) {
4310 return
4311 DAG.getSetCC(dl, VT, N0.getOperand(1),
4312 DAG.getConstant(SUBC->getAPIntValue() -
4313 RHSC->getAPIntValue(),
4314 dl, N0.getValueType()),
4315 Cond);
4316 }
4317 }
4318
4319 // Could RHSC fold directly into a compare?
4320 if (RHSC->getValueType(0).getSizeInBits() <= 64)
4321 LegalRHSImm = isLegalICmpImmediate(RHSC->getSExtValue());
4322 }
4323
4324 // (X+Y) == X --> Y == 0 and similar folds.
4325 // Don't do this if X is an immediate that can fold into a cmp
4326 // instruction and X+Y has other uses. It could be an induction variable
4327 // chain, and the transform would increase register pressure.
4328 if (!LegalRHSImm || N0.hasOneUse())
4329 if (SDValue V = foldSetCCWithBinOp(VT, N0, N1, Cond, dl, DCI))
4330 return V;
4331 }
4332
4333 if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB ||
4334 N1.getOpcode() == ISD::XOR)
4335 if (SDValue V = foldSetCCWithBinOp(VT, N1, N0, Cond, dl, DCI))
4336 return V;
4337
4338 if (SDValue V = foldSetCCWithAnd(VT, N0, N1, Cond, dl, DCI))
4339 return V;
4340 }
4341
4342 // Fold remainder of division by a constant.
4343 if ((N0.getOpcode() == ISD::UREM || N0.getOpcode() == ISD::SREM) &&
4344 N0.hasOneUse() && (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
4345 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
4346
4347 // When division is cheap or optimizing for minimum size,
4348 // fall through to DIVREM creation by skipping this fold.
4349 if (!isIntDivCheap(VT, Attr) && !Attr.hasFnAttr(Attribute::MinSize)) {
4350 if (N0.getOpcode() == ISD::UREM) {
4351 if (SDValue Folded = buildUREMEqFold(VT, N0, N1, Cond, DCI, dl))
4352 return Folded;
4353 } else if (N0.getOpcode() == ISD::SREM) {
4354 if (SDValue Folded = buildSREMEqFold(VT, N0, N1, Cond, DCI, dl))
4355 return Folded;
4356 }
4357 }
4358 }
4359
4360 // Fold away ALL boolean setcc's.
4361 if (N0.getValueType().getScalarType() == MVT::i1 && foldBooleans) {
4362 SDValue Temp;
4363 switch (Cond) {
4364 default: llvm_unreachable("Unknown integer setcc!")::llvm::llvm_unreachable_internal("Unknown integer setcc!", "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4364)
;
4365 case ISD::SETEQ: // X == Y -> ~(X^Y)
4366 Temp = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
4367 N0 = DAG.getNOT(dl, Temp, OpVT);
4368 if (!DCI.isCalledByLegalizer())
4369 DCI.AddToWorklist(Temp.getNode());
4370 break;
4371 case ISD::SETNE: // X != Y --> (X^Y)
4372 N0 = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
4373 break;
4374 case ISD::SETGT: // X >s Y --> X == 0 & Y == 1 --> ~X & Y
4375 case ISD::SETULT: // X <u Y --> X == 0 & Y == 1 --> ~X & Y
4376 Temp = DAG.getNOT(dl, N0, OpVT);
4377 N0 = DAG.getNode(ISD::AND, dl, OpVT, N1, Temp);
4378 if (!DCI.isCalledByLegalizer())
4379 DCI.AddToWorklist(Temp.getNode());
4380 break;
4381 case ISD::SETLT: // X <s Y --> X == 1 & Y == 0 --> ~Y & X
4382 case ISD::SETUGT: // X >u Y --> X == 1 & Y == 0 --> ~Y & X
4383 Temp = DAG.getNOT(dl, N1, OpVT);
4384 N0 = DAG.getNode(ISD::AND, dl, OpVT, N0, Temp);
4385 if (!DCI.isCalledByLegalizer())
4386 DCI.AddToWorklist(Temp.getNode());
4387 break;
4388 case ISD::SETULE: // X <=u Y --> X == 0 | Y == 1 --> ~X | Y
4389 case ISD::SETGE: // X >=s Y --> X == 0 | Y == 1 --> ~X | Y
4390 Temp = DAG.getNOT(dl, N0, OpVT);
4391 N0 = DAG.getNode(ISD::OR, dl, OpVT, N1, Temp);
4392 if (!DCI.isCalledByLegalizer())
4393 DCI.AddToWorklist(Temp.getNode());
4394 break;
4395 case ISD::SETUGE: // X >=u Y --> X == 1 | Y == 0 --> ~Y | X
4396 case ISD::SETLE: // X <=s Y --> X == 1 | Y == 0 --> ~Y | X
4397 Temp = DAG.getNOT(dl, N1, OpVT);
4398 N0 = DAG.getNode(ISD::OR, dl, OpVT, N0, Temp);
4399 break;
4400 }
4401 if (VT.getScalarType() != MVT::i1) {
4402 if (!DCI.isCalledByLegalizer())
4403 DCI.AddToWorklist(N0.getNode());
4404 // FIXME: If running after legalize, we probably can't do this.
4405 ISD::NodeType ExtendCode = getExtendForContent(getBooleanContents(OpVT));
4406 N0 = DAG.getNode(ExtendCode, dl, VT, N0);
4407 }
4408 return N0;
4409 }
4410
4411 // Could not fold it.
4412 return SDValue();
4413}
4414
4415/// Returns true (and the GlobalValue and the offset) if the node is a
4416/// GlobalAddress + offset.
4417bool TargetLowering::isGAPlusOffset(SDNode *WN, const GlobalValue *&GA,
4418 int64_t &Offset) const {
4419
4420 SDNode *N = unwrapAddress(SDValue(WN, 0)).getNode();
4421
4422 if (auto *GASD = dyn_cast<GlobalAddressSDNode>(N)) {
4423 GA = GASD->getGlobal();
4424 Offset += GASD->getOffset();
4425 return true;
4426 }
4427
4428 if (N->getOpcode() == ISD::ADD) {
4429 SDValue N1 = N->getOperand(0);
4430 SDValue N2 = N->getOperand(1);
4431 if (isGAPlusOffset(N1.getNode(), GA, Offset)) {
4432 if (auto *V = dyn_cast<ConstantSDNode>(N2)) {
4433 Offset += V->getSExtValue();
4434 return true;
4435 }
4436 } else if (isGAPlusOffset(N2.getNode(), GA, Offset)) {
4437 if (auto *V = dyn_cast<ConstantSDNode>(N1)) {
4438 Offset += V->getSExtValue();
4439 return true;
4440 }
4441 }
4442 }
4443
4444 return false;
4445}
4446
4447SDValue TargetLowering::PerformDAGCombine(SDNode *N,
4448 DAGCombinerInfo &DCI) const {
4449 // Default implementation: no optimization.
4450 return SDValue();
4451}
4452
4453//===----------------------------------------------------------------------===//
4454// Inline Assembler Implementation Methods
4455//===----------------------------------------------------------------------===//
4456
4457TargetLowering::ConstraintType
4458TargetLowering::getConstraintType(StringRef Constraint) const {
4459 unsigned S = Constraint.size();
4460
4461 if (S == 1) {
4462 switch (Constraint[0]) {
4463 default: break;
4464 case 'r':
4465 return C_RegisterClass;
4466 case 'm': // memory
4467 case 'o': // offsetable
4468 case 'V': // not offsetable
4469 return C_Memory;
4470 case 'n': // Simple Integer
4471 case 'E': // Floating Point Constant
4472 case 'F': // Floating Point Constant
4473 return C_Immediate;
4474 case 'i': // Simple Integer or Relocatable Constant
4475 case 's': // Relocatable Constant
4476 case 'p': // Address.
4477 case 'X': // Allow ANY value.
4478 case 'I': // Target registers.
4479 case 'J':
4480 case 'K':
4481 case 'L':
4482 case 'M':
4483 case 'N':
4484 case 'O':
4485 case 'P':
4486 case '<':
4487 case '>':
4488 return C_Other;
4489 }
4490 }
4491
4492 if (S > 1 && Constraint[0] == '{' && Constraint[S - 1] == '}') {
4493 if (S == 8 && Constraint.substr(1, 6) == "memory") // "{memory}"
4494 return C_Memory;
4495 return C_Register;
4496 }
4497 return C_Unknown;
4498}
4499
4500/// Try to replace an X constraint, which matches anything, with another that
4501/// has more specific requirements based on the type of the corresponding
4502/// operand.
4503const char *TargetLowering::LowerXConstraint(EVT ConstraintVT) const {
4504 if (ConstraintVT.isInteger())
4505 return "r";
4506 if (ConstraintVT.isFloatingPoint())
4507 return "f"; // works for many targets
4508 return nullptr;
4509}
4510
4511SDValue TargetLowering::LowerAsmOutputForConstraint(
4512 SDValue &Chain, SDValue &Flag, const SDLoc &DL,
4513 const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const {
4514 return SDValue();
4515}
4516
4517/// Lower the specified operand into the Ops vector.
4518/// If it is invalid, don't add anything to Ops.
4519void TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
4520 std::string &Constraint,
4521 std::vector<SDValue> &Ops,
4522 SelectionDAG &DAG) const {
4523
4524 if (Constraint.length() > 1) return;
4525
4526 char ConstraintLetter = Constraint[0];
4527 switch (ConstraintLetter) {
4528 default: break;
4529 case 'X': // Allows any operand; labels (basic block) use this.
4530 if (Op.getOpcode() == ISD::BasicBlock ||
4531 Op.getOpcode() == ISD::TargetBlockAddress) {
4532 Ops.push_back(Op);
4533 return;
4534 }
4535 LLVM_FALLTHROUGH[[gnu::fallthrough]];
4536 case 'i': // Simple Integer or Relocatable Constant
4537 case 'n': // Simple Integer
4538 case 's': { // Relocatable Constant
4539
4540 GlobalAddressSDNode *GA;
4541 ConstantSDNode *C;
4542 BlockAddressSDNode *BA;
4543 uint64_t Offset = 0;
4544
4545 // Match (GA) or (C) or (GA+C) or (GA-C) or ((GA+C)+C) or (((GA+C)+C)+C),
4546 // etc., since getelementpointer is variadic. We can't use
4547 // SelectionDAG::FoldSymbolOffset because it expects the GA to be accessible
4548 // while in this case the GA may be furthest from the root node which is
4549 // likely an ISD::ADD.
4550 while (1) {
4551 if ((GA = dyn_cast<GlobalAddressSDNode>(Op)) && ConstraintLetter != 'n') {
4552 Ops.push_back(DAG.getTargetGlobalAddress(GA->getGlobal(), SDLoc(Op),
4553 GA->getValueType(0),
4554 Offset + GA->getOffset()));
4555 return;
4556 }
4557 if ((C = dyn_cast<ConstantSDNode>(Op)) && ConstraintLetter != 's') {
4558 // gcc prints these as sign extended. Sign extend value to 64 bits
4559 // now; without this it would get ZExt'd later in
4560 // ScheduleDAGSDNodes::EmitNode, which is very generic.
4561 bool IsBool = C->getConstantIntValue()->getBitWidth() == 1;
4562 BooleanContent BCont = getBooleanContents(MVT::i64);
4563 ISD::NodeType ExtOpc =
4564 IsBool ? getExtendForContent(BCont) : ISD::SIGN_EXTEND;
4565 int64_t ExtVal =
4566 ExtOpc == ISD::ZERO_EXTEND ? C->getZExtValue() : C->getSExtValue();
4567 Ops.push_back(
4568 DAG.getTargetConstant(Offset + ExtVal, SDLoc(C), MVT::i64));
4569 return;
4570 }
4571 if ((BA = dyn_cast<BlockAddressSDNode>(Op)) && ConstraintLetter != 'n') {
4572 Ops.push_back(DAG.getTargetBlockAddress(
4573 BA->getBlockAddress(), BA->getValueType(0),
4574 Offset + BA->getOffset(), BA->getTargetFlags()));
4575 return;
4576 }
4577 const unsigned OpCode = Op.getOpcode();
4578 if (OpCode == ISD::ADD || OpCode == ISD::SUB) {
4579 if ((C = dyn_cast<ConstantSDNode>(Op.getOperand(0))))
4580 Op = Op.getOperand(1);
4581 // Subtraction is not commutative.
4582 else if (OpCode == ISD::ADD &&
4583 (C = dyn_cast<ConstantSDNode>(Op.getOperand(1))))
4584 Op = Op.getOperand(0);
4585 else
4586 return;
4587 Offset += (OpCode == ISD::ADD ? 1 : -1) * C->getSExtValue();
4588 continue;
4589 }
4590 return;
4591 }
4592 break;
4593 }
4594 }
4595}
4596
4597std::pair<unsigned, const TargetRegisterClass *>
4598TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *RI,
4599 StringRef Constraint,
4600 MVT VT) const {
4601 if (Constraint.empty() || Constraint[0] != '{')
4602 return std::make_pair(0u, static_cast<TargetRegisterClass *>(nullptr));
4603 assert(*(Constraint.end() - 1) == '}' && "Not a brace enclosed constraint?")(static_cast <bool> (*(Constraint.end() - 1) == '}' &&
"Not a brace enclosed constraint?") ? void (0) : __assert_fail
("*(Constraint.end() - 1) == '}' && \"Not a brace enclosed constraint?\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4603, __extension__ __PRETTY_FUNCTION__))
;
4604
4605 // Remove the braces from around the name.
4606 StringRef RegName(Constraint.data() + 1, Constraint.size() - 2);
4607
4608 std::pair<unsigned, const TargetRegisterClass *> R =
4609 std::make_pair(0u, static_cast<const TargetRegisterClass *>(nullptr));
4610
4611 // Figure out which register class contains this reg.
4612 for (const TargetRegisterClass *RC : RI->regclasses()) {
4613 // If none of the value types for this register class are valid, we
4614 // can't use it. For example, 64-bit reg classes on 32-bit targets.
4615 if (!isLegalRC(*RI, *RC))
4616 continue;
4617
4618 for (const MCPhysReg &PR : *RC) {
4619 if (RegName.equals_insensitive(RI->getRegAsmName(PR))) {
4620 std::pair<unsigned, const TargetRegisterClass *> S =
4621 std::make_pair(PR, RC);
4622
4623 // If this register class has the requested value type, return it,
4624 // otherwise keep searching and return the first class found
4625 // if no other is found which explicitly has the requested type.
4626 if (RI->isTypeLegalForClass(*RC, VT))
4627 return S;
4628 if (!R.second)
4629 R = S;
4630 }
4631 }
4632 }
4633
4634 return R;
4635}
4636
4637//===----------------------------------------------------------------------===//
4638// Constraint Selection.
4639
4640/// Return true of this is an input operand that is a matching constraint like
4641/// "4".
4642bool TargetLowering::AsmOperandInfo::isMatchingInputConstraint() const {
4643 assert(!ConstraintCode.empty() && "No known constraint!")(static_cast <bool> (!ConstraintCode.empty() &&
"No known constraint!") ? void (0) : __assert_fail ("!ConstraintCode.empty() && \"No known constraint!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4643, __extension__ __PRETTY_FUNCTION__))
;
4644 return isdigit(static_cast<unsigned char>(ConstraintCode[0]));
4645}
4646
4647/// If this is an input matching constraint, this method returns the output
4648/// operand it matches.
4649unsigned TargetLowering::AsmOperandInfo::getMatchedOperand() const {
4650 assert(!ConstraintCode.empty() && "No known constraint!")(static_cast <bool> (!ConstraintCode.empty() &&
"No known constraint!") ? void (0) : __assert_fail ("!ConstraintCode.empty() && \"No known constraint!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4650, __extension__ __PRETTY_FUNCTION__))
;
4651 return atoi(ConstraintCode.c_str());
4652}
4653
4654/// Split up the constraint string from the inline assembly value into the
4655/// specific constraints and their prefixes, and also tie in the associated
4656/// operand values.
4657/// If this returns an empty vector, and if the constraint string itself
4658/// isn't empty, there was an error parsing.
4659TargetLowering::AsmOperandInfoVector
4660TargetLowering::ParseConstraints(const DataLayout &DL,
4661 const TargetRegisterInfo *TRI,
4662 const CallBase &Call) const {
4663 /// Information about all of the constraints.
4664 AsmOperandInfoVector ConstraintOperands;
4665 const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
4666 unsigned maCount = 0; // Largest number of multiple alternative constraints.
4667
4668 // Do a prepass over the constraints, canonicalizing them, and building up the
4669 // ConstraintOperands list.
4670 unsigned ArgNo = 0; // ArgNo - The argument of the CallInst.
4671 unsigned ResNo = 0; // ResNo - The result number of the next output.
4672
4673 for (InlineAsm::ConstraintInfo &CI : IA->ParseConstraints()) {
4674 ConstraintOperands.emplace_back(std::move(CI));
4675 AsmOperandInfo &OpInfo = ConstraintOperands.back();
4676
4677 // Update multiple alternative constraint count.
4678 if (OpInfo.multipleAlternatives.size() > maCount)
4679 maCount = OpInfo.multipleAlternatives.size();
4680
4681 OpInfo.ConstraintVT = MVT::Other;
4682
4683 // Compute the value type for each operand.
4684 switch (OpInfo.Type) {
4685 case InlineAsm::isOutput:
4686 // Indirect outputs just consume an argument.
4687 if (OpInfo.isIndirect) {
4688 OpInfo.CallOperandVal = Call.getArgOperand(ArgNo++);
4689 break;
4690 }
4691
4692 // The return value of the call is this value. As such, there is no
4693 // corresponding argument.
4694 assert(!Call.getType()->isVoidTy() && "Bad inline asm!")(static_cast <bool> (!Call.getType()->isVoidTy() &&
"Bad inline asm!") ? void (0) : __assert_fail ("!Call.getType()->isVoidTy() && \"Bad inline asm!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4694, __extension__ __PRETTY_FUNCTION__))
;
4695 if (StructType *STy = dyn_cast<StructType>(Call.getType())) {
4696 OpInfo.ConstraintVT =
4697 getSimpleValueType(DL, STy->getElementType(ResNo));
4698 } else {
4699 assert(ResNo == 0 && "Asm only has one result!")(static_cast <bool> (ResNo == 0 && "Asm only has one result!"
) ? void (0) : __assert_fail ("ResNo == 0 && \"Asm only has one result!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4699, __extension__ __PRETTY_FUNCTION__))
;
4700 OpInfo.ConstraintVT =
4701 getAsmOperandValueType(DL, Call.getType()).getSimpleVT();
4702 }
4703 ++ResNo;
4704 break;
4705 case InlineAsm::isInput:
4706 OpInfo.CallOperandVal = Call.getArgOperand(ArgNo++);
4707 break;
4708 case InlineAsm::isClobber:
4709 // Nothing to do.
4710 break;
4711 }
4712
4713 if (OpInfo.CallOperandVal) {
4714 llvm::Type *OpTy = OpInfo.CallOperandVal->getType();
4715 if (OpInfo.isIndirect) {
4716 llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy);
4717 if (!PtrTy)
4718 report_fatal_error("Indirect operand for inline asm not a pointer!");
4719 OpTy = PtrTy->getElementType();
4720 }
4721
4722 // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
4723 if (StructType *STy = dyn_cast<StructType>(OpTy))
4724 if (STy->getNumElements() == 1)
4725 OpTy = STy->getElementType(0);
4726
4727 // If OpTy is not a single value, it may be a struct/union that we
4728 // can tile with integers.
4729 if (!OpTy->isSingleValueType() && OpTy->isSized()) {
4730 unsigned BitSize = DL.getTypeSizeInBits(OpTy);
4731 switch (BitSize) {
4732 default: break;
4733 case 1:
4734 case 8:
4735 case 16:
4736 case 32:
4737 case 64:
4738 case 128:
4739 OpInfo.ConstraintVT =
4740 MVT::getVT(IntegerType::get(OpTy->getContext(), BitSize), true);
4741 break;
4742 }
4743 } else if (PointerType *PT = dyn_cast<PointerType>(OpTy)) {
4744 unsigned PtrSize = DL.getPointerSizeInBits(PT->getAddressSpace());
4745 OpInfo.ConstraintVT = MVT::getIntegerVT(PtrSize);
4746 } else {
4747 OpInfo.ConstraintVT = MVT::getVT(OpTy, true);
4748 }
4749 }
4750 }
4751
4752 // If we have multiple alternative constraints, select the best alternative.
4753 if (!ConstraintOperands.empty()) {
4754 if (maCount) {
4755 unsigned bestMAIndex = 0;
4756 int bestWeight = -1;
4757 // weight: -1 = invalid match, and 0 = so-so match to 5 = good match.
4758 int weight = -1;
4759 unsigned maIndex;
4760 // Compute the sums of the weights for each alternative, keeping track
4761 // of the best (highest weight) one so far.
4762 for (maIndex = 0; maIndex < maCount; ++maIndex) {
4763 int weightSum = 0;
4764 for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
4765 cIndex != eIndex; ++cIndex) {
4766 AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];
4767 if (OpInfo.Type == InlineAsm::isClobber)
4768 continue;
4769
4770 // If this is an output operand with a matching input operand,
4771 // look up the matching input. If their types mismatch, e.g. one
4772 // is an integer, the other is floating point, or their sizes are
4773 // different, flag it as an maCantMatch.
4774 if (OpInfo.hasMatchingInput()) {
4775 AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
4776 if (OpInfo.ConstraintVT != Input.ConstraintVT) {
4777 if ((OpInfo.ConstraintVT.isInteger() !=
4778 Input.ConstraintVT.isInteger()) ||
4779 (OpInfo.ConstraintVT.getSizeInBits() !=
4780 Input.ConstraintVT.getSizeInBits())) {
4781 weightSum = -1; // Can't match.
4782 break;
4783 }
4784 }
4785 }
4786 weight = getMultipleConstraintMatchWeight(OpInfo, maIndex);
4787 if (weight == -1) {
4788 weightSum = -1;
4789 break;
4790 }
4791 weightSum += weight;
4792 }
4793 // Update best.
4794 if (weightSum > bestWeight) {
4795 bestWeight = weightSum;
4796 bestMAIndex = maIndex;
4797 }
4798 }
4799
4800 // Now select chosen alternative in each constraint.
4801 for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
4802 cIndex != eIndex; ++cIndex) {
4803 AsmOperandInfo &cInfo = ConstraintOperands[cIndex];
4804 if (cInfo.Type == InlineAsm::isClobber)
4805 continue;
4806 cInfo.selectAlternative(bestMAIndex);
4807 }
4808 }
4809 }
4810
4811 // Check and hook up tied operands, choose constraint code to use.
4812 for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
4813 cIndex != eIndex; ++cIndex) {
4814 AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];
4815
4816 // If this is an output operand with a matching input operand, look up the
4817 // matching input. If their types mismatch, e.g. one is an integer, the
4818 // other is floating point, or their sizes are different, flag it as an
4819 // error.
4820 if (OpInfo.hasMatchingInput()) {
4821 AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
4822
4823 if (OpInfo.ConstraintVT != Input.ConstraintVT) {
4824 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
4825 getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,
4826 OpInfo.ConstraintVT);
4827 std::pair<unsigned, const TargetRegisterClass *> InputRC =
4828 getRegForInlineAsmConstraint(TRI, Input.ConstraintCode,
4829 Input.ConstraintVT);
4830 if ((OpInfo.ConstraintVT.isInteger() !=
4831 Input.ConstraintVT.isInteger()) ||
4832 (MatchRC.second != InputRC.second)) {
4833 report_fatal_error("Unsupported asm: input constraint"
4834 " with a matching output constraint of"
4835 " incompatible type!");
4836 }
4837 }
4838 }
4839 }
4840
4841 return ConstraintOperands;
4842}
4843
4844/// Return an integer indicating how general CT is.
4845static unsigned getConstraintGenerality(TargetLowering::ConstraintType CT) {
4846 switch (CT) {
4847 case TargetLowering::C_Immediate:
4848 case TargetLowering::C_Other:
4849 case TargetLowering::C_Unknown:
4850 return 0;
4851 case TargetLowering::C_Register:
4852 return 1;
4853 case TargetLowering::C_RegisterClass:
4854 return 2;
4855 case TargetLowering::C_Memory:
4856 return 3;
4857 }
4858 llvm_unreachable("Invalid constraint type")::llvm::llvm_unreachable_internal("Invalid constraint type", "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4858)
;
4859}
4860
4861/// Examine constraint type and operand type and determine a weight value.
4862/// This object must already have been set up with the operand type
4863/// and the current alternative constraint selected.
4864TargetLowering::ConstraintWeight
4865 TargetLowering::getMultipleConstraintMatchWeight(
4866 AsmOperandInfo &info, int maIndex) const {
4867 InlineAsm::ConstraintCodeVector *rCodes;
4868 if (maIndex >= (int)info.multipleAlternatives.size())
4869 rCodes = &info.Codes;
4870 else
4871 rCodes = &info.multipleAlternatives[maIndex].Codes;
4872 ConstraintWeight BestWeight = CW_Invalid;
4873
4874 // Loop over the options, keeping track of the most general one.
4875 for (unsigned i = 0, e = rCodes->size(); i != e; ++i) {
4876 ConstraintWeight weight =
4877 getSingleConstraintMatchWeight(info, (*rCodes)[i].c_str());
4878 if (weight > BestWeight)
4879 BestWeight = weight;
4880 }
4881
4882 return BestWeight;
4883}
4884
4885/// Examine constraint type and operand type and determine a weight value.
4886/// This object must already have been set up with the operand type
4887/// and the current alternative constraint selected.
4888TargetLowering::ConstraintWeight
4889 TargetLowering::getSingleConstraintMatchWeight(
4890 AsmOperandInfo &info, const char *constraint) const {
4891 ConstraintWeight weight = CW_Invalid;
4892 Value *CallOperandVal = info.CallOperandVal;
4893 // If we don't have a value, we can't do a match,
4894 // but allow it at the lowest weight.
4895 if (!CallOperandVal)
4896 return CW_Default;
4897 // Look at the constraint type.
4898 switch (*constraint) {
4899 case 'i': // immediate integer.
4900 case 'n': // immediate integer with a known value.
4901 if (isa<ConstantInt>(CallOperandVal))
4902 weight = CW_Constant;
4903 break;
4904 case 's': // non-explicit intregal immediate.
4905 if (isa<GlobalValue>(CallOperandVal))
4906 weight = CW_Constant;
4907 break;
4908 case 'E': // immediate float if host format.
4909 case 'F': // immediate float.
4910 if (isa<ConstantFP>(CallOperandVal))
4911 weight = CW_Constant;
4912 break;
4913 case '<': // memory operand with autodecrement.
4914 case '>': // memory operand with autoincrement.
4915 case 'm': // memory operand.
4916 case 'o': // offsettable memory operand
4917 case 'V': // non-offsettable memory operand
4918 weight = CW_Memory;
4919 break;
4920 case 'r': // general register.
4921 case 'g': // general register, memory operand or immediate integer.
4922 // note: Clang converts "g" to "imr".
4923 if (CallOperandVal->getType()->isIntegerTy())
4924 weight = CW_Register;
4925 break;
4926 case 'X': // any operand.
4927 default:
4928 weight = CW_Default;
4929 break;
4930 }
4931 return weight;
4932}
4933
4934/// If there are multiple different constraints that we could pick for this
4935/// operand (e.g. "imr") try to pick the 'best' one.
4936/// This is somewhat tricky: constraints fall into four classes:
4937/// Other -> immediates and magic values
4938/// Register -> one specific register
4939/// RegisterClass -> a group of regs
4940/// Memory -> memory
4941/// Ideally, we would pick the most specific constraint possible: if we have
4942/// something that fits into a register, we would pick it. The problem here
4943/// is that if we have something that could either be in a register or in
4944/// memory that use of the register could cause selection of *other*
4945/// operands to fail: they might only succeed if we pick memory. Because of
4946/// this the heuristic we use is:
4947///
4948/// 1) If there is an 'other' constraint, and if the operand is valid for
4949/// that constraint, use it. This makes us take advantage of 'i'
4950/// constraints when available.
4951/// 2) Otherwise, pick the most general constraint present. This prefers
4952/// 'm' over 'r', for example.
4953///
4954static void ChooseConstraint(TargetLowering::AsmOperandInfo &OpInfo,
4955 const TargetLowering &TLI,
4956 SDValue Op, SelectionDAG *DAG) {
4957 assert(OpInfo.Codes.size() > 1 && "Doesn't have multiple constraint options")(static_cast <bool> (OpInfo.Codes.size() > 1 &&
"Doesn't have multiple constraint options") ? void (0) : __assert_fail
("OpInfo.Codes.size() > 1 && \"Doesn't have multiple constraint options\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4957, __extension__ __PRETTY_FUNCTION__))
;
4958 unsigned BestIdx = 0;
4959 TargetLowering::ConstraintType BestType = TargetLowering::C_Unknown;
4960 int BestGenerality = -1;
4961
4962 // Loop over the options, keeping track of the most general one.
4963 for (unsigned i = 0, e = OpInfo.Codes.size(); i != e; ++i) {
4964 TargetLowering::ConstraintType CType =
4965 TLI.getConstraintType(OpInfo.Codes[i]);
4966
4967 // Indirect 'other' or 'immediate' constraints are not allowed.
4968 if (OpInfo.isIndirect && !(CType == TargetLowering::C_Memory ||
4969 CType == TargetLowering::C_Register ||
4970 CType == TargetLowering::C_RegisterClass))
4971 continue;
4972
4973 // If this is an 'other' or 'immediate' constraint, see if the operand is
4974 // valid for it. For example, on X86 we might have an 'rI' constraint. If
4975 // the operand is an integer in the range [0..31] we want to use I (saving a
4976 // load of a register), otherwise we must use 'r'.
4977 if ((CType == TargetLowering::C_Other ||
4978 CType == TargetLowering::C_Immediate) && Op.getNode()) {
4979 assert(OpInfo.Codes[i].size() == 1 &&(static_cast <bool> (OpInfo.Codes[i].size() == 1 &&
"Unhandled multi-letter 'other' constraint") ? void (0) : __assert_fail
("OpInfo.Codes[i].size() == 1 && \"Unhandled multi-letter 'other' constraint\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4980, __extension__ __PRETTY_FUNCTION__))
4980 "Unhandled multi-letter 'other' constraint")(static_cast <bool> (OpInfo.Codes[i].size() == 1 &&
"Unhandled multi-letter 'other' constraint") ? void (0) : __assert_fail
("OpInfo.Codes[i].size() == 1 && \"Unhandled multi-letter 'other' constraint\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4980, __extension__ __PRETTY_FUNCTION__))
;
4981 std::vector<SDValue> ResultOps;
4982 TLI.LowerAsmOperandForConstraint(Op, OpInfo.Codes[i],
4983 ResultOps, *DAG);
4984 if (!ResultOps.empty()) {
4985 BestType = CType;
4986 BestIdx = i;
4987 break;
4988 }
4989 }
4990
4991 // Things with matching constraints can only be registers, per gcc
4992 // documentation. This mainly affects "g" constraints.
4993 if (CType == TargetLowering::C_Memory && OpInfo.hasMatchingInput())
4994 continue;
4995
4996 // This constraint letter is more general than the previous one, use it.
4997 int Generality = getConstraintGenerality(CType);
4998 if (Generality > BestGenerality) {
4999 BestType = CType;
5000 BestIdx = i;
5001 BestGenerality = Generality;
5002 }
5003 }
5004
5005 OpInfo.ConstraintCode = OpInfo.Codes[BestIdx];
5006 OpInfo.ConstraintType = BestType;
5007}
5008
5009/// Determines the constraint code and constraint type to use for the specific
5010/// AsmOperandInfo, setting OpInfo.ConstraintCode and OpInfo.ConstraintType.
5011void TargetLowering::ComputeConstraintToUse(AsmOperandInfo &OpInfo,
5012 SDValue Op,
5013 SelectionDAG *DAG) const {
5014 assert(!OpInfo.Codes.empty() && "Must have at least one constraint")(static_cast <bool> (!OpInfo.Codes.empty() && "Must have at least one constraint"
) ? void (0) : __assert_fail ("!OpInfo.Codes.empty() && \"Must have at least one constraint\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5014, __extension__ __PRETTY_FUNCTION__))
;
5015
5016 // Single-letter constraints ('r') are very common.
5017 if (OpInfo.Codes.size() == 1) {
5018 OpInfo.ConstraintCode = OpInfo.Codes[0];
5019 OpInfo.ConstraintType = getConstraintType(OpInfo.ConstraintCode);
5020 } else {
5021 ChooseConstraint(OpInfo, *this, Op, DAG);
5022 }
5023
5024 // 'X' matches anything.
5025 if (OpInfo.ConstraintCode == "X" && OpInfo.CallOperandVal) {
5026 // Labels and constants are handled elsewhere ('X' is the only thing
5027 // that matches labels). For Functions, the type here is the type of
5028 // the result, which is not what we want to look at; leave them alone.
5029 Value *v = OpInfo.CallOperandVal;
5030 if (isa<BasicBlock>(v) || isa<ConstantInt>(v) || isa<Function>(v)) {
5031 OpInfo.CallOperandVal = v;
5032 return;
5033 }
5034
5035 if (Op.getNode() && Op.getOpcode() == ISD::TargetBlockAddress)
5036 return;
5037
5038 // Otherwise, try to resolve it to something we know about by looking at
5039 // the actual operand type.
5040 if (const char *Repl = LowerXConstraint(OpInfo.ConstraintVT)) {
5041 OpInfo.ConstraintCode = Repl;
5042 OpInfo.ConstraintType = getConstraintType(OpInfo.ConstraintCode);
5043 }
5044 }
5045}
5046
5047/// Given an exact SDIV by a constant, create a multiplication
5048/// with the multiplicative inverse of the constant.
5049static SDValue BuildExactSDIV(const TargetLowering &TLI, SDNode *N,
5050 const SDLoc &dl, SelectionDAG &DAG,
5051 SmallVectorImpl<SDNode *> &Created) {
5052 SDValue Op0 = N->getOperand(0);
5053 SDValue Op1 = N->getOperand(1);
5054 EVT VT = N->getValueType(0);
5055 EVT SVT = VT.getScalarType();
5056 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
5057 EVT ShSVT = ShVT.getScalarType();
5058
5059 bool UseSRA = false;
5060 SmallVector<SDValue, 16> Shifts, Factors;
5061
5062 auto BuildSDIVPattern = [&](ConstantSDNode *C) {
5063 if (C->isZero())
5064 return false;
5065 APInt Divisor = C->getAPIntValue();
5066 unsigned Shift = Divisor.countTrailingZeros();
5067 if (Shift) {
5068 Divisor.ashrInPlace(Shift);
5069 UseSRA = true;
5070 }
5071 // Calculate the multiplicative inverse, using Newton's method.
5072 APInt t;
5073 APInt Factor = Divisor;
5074 while ((t = Divisor * Factor) != 1)
5075 Factor *= APInt(Divisor.getBitWidth(), 2) - t;
5076 Shifts.push_back(DAG.getConstant(Shift, dl, ShSVT));
5077 Factors.push_back(DAG.getConstant(Factor, dl, SVT));
5078 return true;
5079 };
5080
5081 // Collect all magic values from the build vector.
5082 if (!ISD::matchUnaryPredicate(Op1, BuildSDIVPattern))
5083 return SDValue();
5084
5085 SDValue Shift, Factor;
5086 if (Op1.getOpcode() == ISD::BUILD_VECTOR) {
5087 Shift = DAG.getBuildVector(ShVT, dl, Shifts);
5088 Factor = DAG.getBuildVector(VT, dl, Factors);
5089 } else if (Op1.getOpcode() == ISD::SPLAT_VECTOR) {
5090 assert(Shifts.size() == 1 && Factors.size() == 1 &&(static_cast <bool> (Shifts.size() == 1 && Factors
.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("Shifts.size() == 1 && Factors.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5092, __extension__ __PRETTY_FUNCTION__))
5091 "Expected matchUnaryPredicate to return one element for scalable "(static_cast <bool> (Shifts.size() == 1 && Factors
.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("Shifts.size() == 1 && Factors.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5092, __extension__ __PRETTY_FUNCTION__))
5092 "vectors")(static_cast <bool> (Shifts.size() == 1 && Factors
.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("Shifts.size() == 1 && Factors.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5092, __extension__ __PRETTY_FUNCTION__))
;
5093 Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]);
5094 Factor = DAG.getSplatVector(VT, dl, Factors[0]);
5095 } else {
5096 assert(isa<ConstantSDNode>(Op1) && "Expected a constant")(static_cast <bool> (isa<ConstantSDNode>(Op1) &&
"Expected a constant") ? void (0) : __assert_fail ("isa<ConstantSDNode>(Op1) && \"Expected a constant\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5096, __extension__ __PRETTY_FUNCTION__))
;
5097 Shift = Shifts[0];
5098 Factor = Factors[0];
5099 }
5100
5101 SDValue Res = Op0;
5102
5103 // Shift the value upfront if it is even, so the LSB is one.
5104 if (UseSRA) {
5105 // TODO: For UDIV use SRL instead of SRA.
5106 SDNodeFlags Flags;
5107 Flags.setExact(true);
5108 Res = DAG.getNode(ISD::SRA, dl, VT, Res, Shift, Flags);
5109 Created.push_back(Res.getNode());
5110 }
5111
5112 return DAG.getNode(ISD::MUL, dl, VT, Res, Factor);
5113}
5114
5115SDValue TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
5116 SelectionDAG &DAG,
5117 SmallVectorImpl<SDNode *> &Created) const {
5118 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
5119 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5120 if (TLI.isIntDivCheap(N->getValueType(0), Attr))
5121 return SDValue(N, 0); // Lower SDIV as SDIV
5122 return SDValue();
5123}
5124
5125/// Given an ISD::SDIV node expressing a divide by constant,
5126/// return a DAG expression to select that will generate the same value by
5127/// multiplying by a magic number.
5128/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
5129SDValue TargetLowering::BuildSDIV(SDNode *N, SelectionDAG &DAG,
5130 bool IsAfterLegalization,
5131 SmallVectorImpl<SDNode *> &Created) const {
5132 SDLoc dl(N);
5133 EVT VT = N->getValueType(0);
5134 EVT SVT = VT.getScalarType();
5135 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
5136 EVT ShSVT = ShVT.getScalarType();
5137 unsigned EltBits = VT.getScalarSizeInBits();
5138 EVT MulVT;
5139
5140 // Check to see if we can do this.
5141 // FIXME: We should be more aggressive here.
5142 if (!isTypeLegal(VT)) {
5143 // Limit this to simple scalars for now.
5144 if (VT.isVector() || !VT.isSimple())
5145 return SDValue();
5146
5147 // If this type will be promoted to a large enough type with a legal
5148 // multiply operation, we can go ahead and do this transform.
5149 if (getTypeAction(VT.getSimpleVT()) != TypePromoteInteger)
5150 return SDValue();
5151
5152 MulVT = getTypeToTransformTo(*DAG.getContext(), VT);
5153 if (MulVT.getSizeInBits() < (2 * EltBits) ||
5154 !isOperationLegal(ISD::MUL, MulVT))
5155 return SDValue();
5156 }
5157
5158 // If the sdiv has an 'exact' bit we can use a simpler lowering.
5159 if (N->getFlags().hasExact())
5160 return BuildExactSDIV(*this, N, dl, DAG, Created);
5161
5162 SmallVector<SDValue, 16> MagicFactors, Factors, Shifts, ShiftMasks;
5163
5164 auto BuildSDIVPattern = [&](ConstantSDNode *C) {
5165 if (C->isZero())
5166 return false;
5167
5168 const APInt &Divisor = C->getAPIntValue();
5169 SignedDivisionByConstantInfo magics = SignedDivisionByConstantInfo::get(Divisor);
5170 int NumeratorFactor = 0;
5171 int ShiftMask = -1;
5172
5173 if (Divisor.isOne() || Divisor.isAllOnes()) {
5174 // If d is +1/-1, we just multiply the numerator by +1/-1.
5175 NumeratorFactor = Divisor.getSExtValue();
5176 magics.Magic = 0;
5177 magics.ShiftAmount = 0;
5178 ShiftMask = 0;
5179 } else if (Divisor.isStrictlyPositive() && magics.Magic.isNegative()) {
5180 // If d > 0 and m < 0, add the numerator.
5181 NumeratorFactor = 1;
5182 } else if (Divisor.isNegative() && magics.Magic.isStrictlyPositive()) {
5183 // If d < 0 and m > 0, subtract the numerator.
5184 NumeratorFactor = -1;
5185 }
5186
5187 MagicFactors.push_back(DAG.getConstant(magics.Magic, dl, SVT));
5188 Factors.push_back(DAG.getConstant(NumeratorFactor, dl, SVT));
5189 Shifts.push_back(DAG.getConstant(magics.ShiftAmount, dl, ShSVT));
5190 ShiftMasks.push_back(DAG.getConstant(ShiftMask, dl, SVT));
5191 return true;
5192 };
5193
5194 SDValue N0 = N->getOperand(0);
5195 SDValue N1 = N->getOperand(1);
5196
5197 // Collect the shifts / magic values from each element.
5198 if (!ISD::matchUnaryPredicate(N1, BuildSDIVPattern))
5199 return SDValue();
5200
5201 SDValue MagicFactor, Factor, Shift, ShiftMask;
5202 if (N1.getOpcode() == ISD::BUILD_VECTOR) {
5203 MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);
5204 Factor = DAG.getBuildVector(VT, dl, Factors);
5205 Shift = DAG.getBuildVector(ShVT, dl, Shifts);
5206 ShiftMask = DAG.getBuildVector(VT, dl, ShiftMasks);
5207 } else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {
5208 assert(MagicFactors.size() == 1 && Factors.size() == 1 &&(static_cast <bool> (MagicFactors.size() == 1 &&
Factors.size() == 1 && Shifts.size() == 1 &&
ShiftMasks.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("MagicFactors.size() == 1 && Factors.size() == 1 && Shifts.size() == 1 && ShiftMasks.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5211, __extension__ __PRETTY_FUNCTION__))
5209 Shifts.size() == 1 && ShiftMasks.size() == 1 &&(static_cast <bool> (MagicFactors.size() == 1 &&
Factors.size() == 1 && Shifts.size() == 1 &&
ShiftMasks.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("MagicFactors.size() == 1 && Factors.size() == 1 && Shifts.size() == 1 && ShiftMasks.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5211, __extension__ __PRETTY_FUNCTION__))
5210 "Expected matchUnaryPredicate to return one element for scalable "(static_cast <bool> (MagicFactors.size() == 1 &&
Factors.size() == 1 && Shifts.size() == 1 &&
ShiftMasks.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("MagicFactors.size() == 1 && Factors.size() == 1 && Shifts.size() == 1 && ShiftMasks.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5211, __extension__ __PRETTY_FUNCTION__))
5211 "vectors")(static_cast <bool> (MagicFactors.size() == 1 &&
Factors.size() == 1 && Shifts.size() == 1 &&
ShiftMasks.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("MagicFactors.size() == 1 && Factors.size() == 1 && Shifts.size() == 1 && ShiftMasks.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5211, __extension__ __PRETTY_FUNCTION__))
;
5212 MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]);
5213 Factor = DAG.getSplatVector(VT, dl, Factors[0]);
5214 Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]);
5215 ShiftMask = DAG.getSplatVector(VT, dl, ShiftMasks[0]);
5216 } else {
5217 assert(isa<ConstantSDNode>(N1) && "Expected a constant")(static_cast <bool> (isa<ConstantSDNode>(N1) &&
"Expected a constant") ? void (0) : __assert_fail ("isa<ConstantSDNode>(N1) && \"Expected a constant\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5217, __extension__ __PRETTY_FUNCTION__))
;
5218 MagicFactor = MagicFactors[0];
5219 Factor = Factors[0];
5220 Shift = Shifts[0];
5221 ShiftMask = ShiftMasks[0];
5222 }
5223
5224 // Multiply the numerator (operand 0) by the magic value.
5225 // FIXME: We should support doing a MUL in a wider type.
5226 auto GetMULHS = [&](SDValue X, SDValue Y) {
5227 // If the type isn't legal, use a wider mul of the the type calculated
5228 // earlier.
5229 if (!isTypeLegal(VT)) {
5230 X = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, X);
5231 Y = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, Y);
5232 Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y);
5233 Y = DAG.getNode(ISD::SRL, dl, MulVT, Y,
5234 DAG.getShiftAmountConstant(EltBits, MulVT, dl));
5235 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);
5236 }
5237
5238 if (isOperationLegalOrCustom(ISD::MULHS, VT, IsAfterLegalization))
5239 return DAG.getNode(ISD::MULHS, dl, VT, X, Y);
5240 if (isOperationLegalOrCustom(ISD::SMUL_LOHI, VT, IsAfterLegalization)) {
5241 SDValue LoHi =
5242 DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);
5243 return SDValue(LoHi.getNode(), 1);
5244 }
5245 return SDValue();
5246 };
5247
5248 SDValue Q = GetMULHS(N0, MagicFactor);
5249 if (!Q)
5250 return SDValue();
5251
5252 Created.push_back(Q.getNode());
5253
5254 // (Optionally) Add/subtract the numerator using Factor.
5255 Factor = DAG.getNode(ISD::MUL, dl, VT, N0, Factor);
5256 Created.push_back(Factor.getNode());
5257 Q = DAG.getNode(ISD::ADD, dl, VT, Q, Factor);
5258 Created.push_back(Q.getNode());
5259
5260 // Shift right algebraic by shift value.
5261 Q = DAG.getNode(ISD::SRA, dl, VT, Q, Shift);
5262 Created.push_back(Q.getNode());
5263
5264 // Extract the sign bit, mask it and add it to the quotient.
5265 SDValue SignShift = DAG.getConstant(EltBits - 1, dl, ShVT);
5266 SDValue T = DAG.getNode(ISD::SRL, dl, VT, Q, SignShift);
5267 Created.push_back(T.getNode());
5268 T = DAG.getNode(ISD::AND, dl, VT, T, ShiftMask);
5269 Created.push_back(T.getNode());
5270 return DAG.getNode(ISD::ADD, dl, VT, Q, T);
5271}
5272
5273/// Given an ISD::UDIV node expressing a divide by constant,
5274/// return a DAG expression to select that will generate the same value by
5275/// multiplying by a magic number.
5276/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
5277SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG,
5278 bool IsAfterLegalization,
5279 SmallVectorImpl<SDNode *> &Created) const {
5280 SDLoc dl(N);
5281 EVT VT = N->getValueType(0);
5282 EVT SVT = VT.getScalarType();
5283 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
5284 EVT ShSVT = ShVT.getScalarType();
5285 unsigned EltBits = VT.getScalarSizeInBits();
5286 EVT MulVT;
5287
5288 // Check to see if we can do this.
5289 // FIXME: We should be more aggressive here.
5290 if (!isTypeLegal(VT)) {
5291 // Limit this to simple scalars for now.
5292 if (VT.isVector() || !VT.isSimple())
5293 return SDValue();
5294
5295 // If this type will be promoted to a large enough type with a legal
5296 // multiply operation, we can go ahead and do this transform.
5297 if (getTypeAction(VT.getSimpleVT()) != TypePromoteInteger)
5298 return SDValue();
5299
5300 MulVT = getTypeToTransformTo(*DAG.getContext(), VT);
5301 if (MulVT.getSizeInBits() < (2 * EltBits) ||
5302 !isOperationLegal(ISD::MUL, MulVT))
5303 return SDValue();
5304 }
5305
5306 bool UseNPQ = false;
5307 SmallVector<SDValue, 16> PreShifts, PostShifts, MagicFactors, NPQFactors;
5308
5309 auto BuildUDIVPattern = [&](ConstantSDNode *C) {
5310 if (C->isZero())
5311 return false;
5312 // FIXME: We should use a narrower constant when the upper
5313 // bits are known to be zero.
5314 const APInt& Divisor = C->getAPIntValue();
5315 UnsignedDivisonByConstantInfo magics = UnsignedDivisonByConstantInfo::get(Divisor);
5316 unsigned PreShift = 0, PostShift = 0;
5317
5318 // If the divisor is even, we can avoid using the expensive fixup by
5319 // shifting the divided value upfront.
5320 if (magics.IsAdd != 0 && !Divisor[0]) {
5321 PreShift = Divisor.countTrailingZeros();
5322 // Get magic number for the shifted divisor.
5323 magics = UnsignedDivisonByConstantInfo::get(Divisor.lshr(PreShift), PreShift);
5324 assert(magics.IsAdd == 0 && "Should use cheap fixup now")(static_cast <bool> (magics.IsAdd == 0 && "Should use cheap fixup now"
) ? void (0) : __assert_fail ("magics.IsAdd == 0 && \"Should use cheap fixup now\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5324, __extension__ __PRETTY_FUNCTION__))
;
5325 }
5326
5327 APInt Magic = magics.Magic;
5328
5329 unsigned SelNPQ;
5330 if (magics.IsAdd == 0 || Divisor.isOne()) {
5331 assert(magics.ShiftAmount < Divisor.getBitWidth() &&(static_cast <bool> (magics.ShiftAmount < Divisor.getBitWidth
() && "We shouldn't generate an undefined shift!") ? void
(0) : __assert_fail ("magics.ShiftAmount < Divisor.getBitWidth() && \"We shouldn't generate an undefined shift!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5332, __extension__ __PRETTY_FUNCTION__))
5332 "We shouldn't generate an undefined shift!")(static_cast <bool> (magics.ShiftAmount < Divisor.getBitWidth
() && "We shouldn't generate an undefined shift!") ? void
(0) : __assert_fail ("magics.ShiftAmount < Divisor.getBitWidth() && \"We shouldn't generate an undefined shift!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5332, __extension__ __PRETTY_FUNCTION__))
;
5333 PostShift = magics.ShiftAmount;
5334 SelNPQ = false;
5335 } else {
5336 PostShift = magics.ShiftAmount - 1;
5337 SelNPQ = true;
5338 }
5339
5340 PreShifts.push_back(DAG.getConstant(PreShift, dl, ShSVT));
5341 MagicFactors.push_back(DAG.getConstant(Magic, dl, SVT));
5342 NPQFactors.push_back(
5343 DAG.getConstant(SelNPQ ? APInt::getOneBitSet(EltBits, EltBits - 1)
5344 : APInt::getZero(EltBits),
5345 dl, SVT));
5346 PostShifts.push_back(DAG.getConstant(PostShift, dl, ShSVT));
5347 UseNPQ |= SelNPQ;
5348 return true;
5349 };
5350
5351 SDValue N0 = N->getOperand(0);
5352 SDValue N1 = N->getOperand(1);
5353
5354 // Collect the shifts/magic values from each element.
5355 if (!ISD::matchUnaryPredicate(N1, BuildUDIVPattern))
5356 return SDValue();
5357
5358 SDValue PreShift, PostShift, MagicFactor, NPQFactor;
5359 if (N1.getOpcode() == ISD::BUILD_VECTOR) {
5360 PreShift = DAG.getBuildVector(ShVT, dl, PreShifts);
5361 MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);
5362 NPQFactor = DAG.getBuildVector(VT, dl, NPQFactors);
5363 PostShift = DAG.getBuildVector(ShVT, dl, PostShifts);
5364 } else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {
5365 assert(PreShifts.size() == 1 && MagicFactors.size() == 1 &&(static_cast <bool> (PreShifts.size() == 1 && MagicFactors
.size() == 1 && NPQFactors.size() == 1 && PostShifts
.size() == 1 && "Expected matchUnaryPredicate to return one for scalable vectors"
) ? void (0) : __assert_fail ("PreShifts.size() == 1 && MagicFactors.size() == 1 && NPQFactors.size() == 1 && PostShifts.size() == 1 && \"Expected matchUnaryPredicate to return one for scalable vectors\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5367, __extension__ __PRETTY_FUNCTION__))
5366 NPQFactors.size() == 1 && PostShifts.size() == 1 &&(static_cast <bool> (PreShifts.size() == 1 && MagicFactors
.size() == 1 && NPQFactors.size() == 1 && PostShifts
.size() == 1 && "Expected matchUnaryPredicate to return one for scalable vectors"
) ? void (0) : __assert_fail ("PreShifts.size() == 1 && MagicFactors.size() == 1 && NPQFactors.size() == 1 && PostShifts.size() == 1 && \"Expected matchUnaryPredicate to return one for scalable vectors\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5367, __extension__ __PRETTY_FUNCTION__))
5367 "Expected matchUnaryPredicate to return one for scalable vectors")(static_cast <bool> (PreShifts.size() == 1 && MagicFactors
.size() == 1 && NPQFactors.size() == 1 && PostShifts
.size() == 1 && "Expected matchUnaryPredicate to return one for scalable vectors"
) ? void (0) : __assert_fail ("PreShifts.size() == 1 && MagicFactors.size() == 1 && NPQFactors.size() == 1 && PostShifts.size() == 1 && \"Expected matchUnaryPredicate to return one for scalable vectors\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5367, __extension__ __PRETTY_FUNCTION__))
;
5368 PreShift = DAG.getSplatVector(ShVT, dl, PreShifts[0]);
5369 MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]);
5370 NPQFactor = DAG.getSplatVector(VT, dl, NPQFactors[0]);
5371 PostShift = DAG.getSplatVector(ShVT, dl, PostShifts[0]);
5372 } else {
5373 assert(isa<ConstantSDNode>(N1) && "Expected a constant")(static_cast <bool> (isa<ConstantSDNode>(N1) &&
"Expected a constant") ? void (0) : __assert_fail ("isa<ConstantSDNode>(N1) && \"Expected a constant\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5373, __extension__ __PRETTY_FUNCTION__))
;
5374 PreShift = PreShifts[0];
5375 MagicFactor = MagicFactors[0];
5376 PostShift = PostShifts[0];
5377 }
5378
5379 SDValue Q = N0;
5380 Q = DAG.getNode(ISD::SRL, dl, VT, Q, PreShift);
5381 Created.push_back(Q.getNode());
5382
5383 // FIXME: We should support doing a MUL in a wider type.
5384 auto GetMULHU = [&](SDValue X, SDValue Y) {
5385 // If the type isn't legal, use a wider mul of the the type calculated
5386 // earlier.
5387 if (!isTypeLegal(VT)) {
5388 X = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, X);
5389 Y = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, Y);
5390 Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y);
5391 Y = DAG.getNode(ISD::SRL, dl, MulVT, Y,
5392 DAG.getShiftAmountConstant(EltBits, MulVT, dl));
5393 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);
5394 }
5395
5396 if (isOperationLegalOrCustom(ISD::MULHU, VT, IsAfterLegalization))
5397 return DAG.getNode(ISD::MULHU, dl, VT, X, Y);
5398 if (isOperationLegalOrCustom(ISD::UMUL_LOHI, VT, IsAfterLegalization)) {
5399 SDValue LoHi =
5400 DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);
5401 return SDValue(LoHi.getNode(), 1);
5402 }
5403 return SDValue(); // No mulhu or equivalent
5404 };
5405
5406 // Multiply the numerator (operand 0) by the magic value.
5407 Q = GetMULHU(Q, MagicFactor);
5408 if (!Q)
5409 return SDValue();
5410
5411 Created.push_back(Q.getNode());
5412
5413 if (UseNPQ) {
5414 SDValue NPQ = DAG.getNode(ISD::SUB, dl, VT, N0, Q);
5415 Created.push_back(NPQ.getNode());
5416
5417 // For vectors we might have a mix of non-NPQ/NPQ paths, so use
5418 // MULHU to act as a SRL-by-1 for NPQ, else multiply by zero.
5419 if (VT.isVector())
5420 NPQ = GetMULHU(NPQ, NPQFactor);
5421 else
5422 NPQ = DAG.getNode(ISD::SRL, dl, VT, NPQ, DAG.getConstant(1, dl, ShVT));
5423
5424 Created.push_back(NPQ.getNode());
5425
5426 Q = DAG.getNode(ISD::ADD, dl, VT, NPQ, Q);
5427 Created.push_back(Q.getNode());
5428 }
5429
5430 Q = DAG.getNode(ISD::SRL, dl, VT, Q, PostShift);
5431 Created.push_back(Q.getNode());
5432
5433 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
5434
5435 SDValue One = DAG.getConstant(1, dl, VT);
5436 SDValue IsOne = DAG.getSetCC(dl, SetCCVT, N1, One, ISD::SETEQ);
5437 return DAG.getSelect(dl, VT, IsOne, N0, Q);
5438}
5439
5440/// If all values in Values that *don't* match the predicate are same 'splat'
5441/// value, then replace all values with that splat value.
5442/// Else, if AlternativeReplacement was provided, then replace all values that
5443/// do match predicate with AlternativeReplacement value.
5444static void
5445turnVectorIntoSplatVector(MutableArrayRef<SDValue> Values,
5446 std::function<bool(SDValue)> Predicate,
5447 SDValue AlternativeReplacement = SDValue()) {
5448 SDValue Replacement;
5449 // Is there a value for which the Predicate does *NOT* match? What is it?
5450 auto SplatValue = llvm::find_if_not(Values, Predicate);
5451 if (SplatValue != Values.end()) {
5452 // Does Values consist only of SplatValue's and values matching Predicate?
5453 if (llvm::all_of(Values, [Predicate, SplatValue](SDValue Value) {
5454 return Value == *SplatValue || Predicate(Value);
5455 })) // Then we shall replace values matching predicate with SplatValue.
5456 Replacement = *SplatValue;
5457 }
5458 if (!Replacement) {
5459 // Oops, we did not find the "baseline" splat value.
5460 if (!AlternativeReplacement)
5461 return; // Nothing to do.
5462 // Let's replace with provided value then.
5463 Replacement = AlternativeReplacement;
5464 }
5465 std::replace_if(Values.begin(), Values.end(), Predicate, Replacement);
5466}
5467
5468/// Given an ISD::UREM used only by an ISD::SETEQ or ISD::SETNE
5469/// where the divisor is constant and the comparison target is zero,
5470/// return a DAG expression that will generate the same comparison result
5471/// using only multiplications, additions and shifts/rotations.
5472/// Ref: "Hacker's Delight" 10-17.
5473SDValue TargetLowering::buildUREMEqFold(EVT SETCCVT, SDValue REMNode,
5474 SDValue CompTargetNode,
5475 ISD::CondCode Cond,
5476 DAGCombinerInfo &DCI,
5477 const SDLoc &DL) const {
5478 SmallVector<SDNode *, 5> Built;
5479 if (SDValue Folded = prepareUREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
5480 DCI, DL, Built)) {
5481 for (SDNode *N : Built)
5482 DCI.AddToWorklist(N);
5483 return Folded;
5484 }
5485
5486 return SDValue();
5487}
5488
5489SDValue
5490TargetLowering::prepareUREMEqFold(EVT SETCCVT, SDValue REMNode,
5491 SDValue CompTargetNode, ISD::CondCode Cond,
5492 DAGCombinerInfo &DCI, const SDLoc &DL,
5493 SmallVectorImpl<SDNode *> &Created) const {
5494 // fold (seteq/ne (urem N, D), 0) -> (setule/ugt (rotr (mul N, P), K), Q)
5495 // - D must be constant, with D = D0 * 2^K where D0 is odd
5496 // - P is the multiplicative inverse of D0 modulo 2^W
5497 // - Q = floor(((2^W) - 1) / D)
5498 // where W is the width of the common type of N and D.
5499 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Only applicable for (in)equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5500, __extension__ __PRETTY_FUNCTION__))
5500 "Only applicable for (in)equality comparisons.")(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Only applicable for (in)equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5500, __extension__ __PRETTY_FUNCTION__))
;
5501
5502 SelectionDAG &DAG = DCI.DAG;
5503
5504 EVT VT = REMNode.getValueType();
5505 EVT SVT = VT.getScalarType();
5506 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout(), !DCI.isBeforeLegalize());
5507 EVT ShSVT = ShVT.getScalarType();
5508
5509 // If MUL is unavailable, we cannot proceed in any case.
5510 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT))
5511 return SDValue();
5512
5513 bool ComparingWithAllZeros = true;
5514 bool AllComparisonsWithNonZerosAreTautological = true;
5515 bool HadTautologicalLanes = false;
5516 bool AllLanesAreTautological = true;
5517 bool HadEvenDivisor = false;
5518 bool AllDivisorsArePowerOfTwo = true;
5519 bool HadTautologicalInvertedLanes = false;
5520 SmallVector<SDValue, 16> PAmts, KAmts, QAmts, IAmts;
5521
5522 auto BuildUREMPattern = [&](ConstantSDNode *CDiv, ConstantSDNode *CCmp) {
5523 // Division by 0 is UB. Leave it to be constant-folded elsewhere.
5524 if (CDiv->isZero())
5525 return false;
5526
5527 const APInt &D = CDiv->getAPIntValue();
5528 const APInt &Cmp = CCmp->getAPIntValue();
5529
5530 ComparingWithAllZeros &= Cmp.isZero();
5531
5532 // x u% C1` is *always* less than C1. So given `x u% C1 == C2`,
5533 // if C2 is not less than C1, the comparison is always false.
5534 // But we will only be able to produce the comparison that will give the
5535 // opposive tautological answer. So this lane would need to be fixed up.
5536 bool TautologicalInvertedLane = D.ule(Cmp);
5537 HadTautologicalInve