Bug Summary

File:llvm/include/llvm/CodeGen/SelectionDAGNodes.h
Warning:line 1120, column 10
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name TargetLowering.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-14~++20220119111520+da61cb019eb2/build-llvm -resource-dir /usr/lib/llvm-14/lib/clang/14.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-14~++20220119111520+da61cb019eb2/llvm/lib/CodeGen/SelectionDAG -I include -I /build/llvm-toolchain-snapshot-14~++20220119111520+da61cb019eb2/llvm/include -D _FORTIFY_SOURCE=2 -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-14/lib/clang/14.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -fmacro-prefix-map=/build/llvm-toolchain-snapshot-14~++20220119111520+da61cb019eb2/build-llvm=build-llvm -fmacro-prefix-map=/build/llvm-toolchain-snapshot-14~++20220119111520+da61cb019eb2/= -fcoverage-prefix-map=/build/llvm-toolchain-snapshot-14~++20220119111520+da61cb019eb2/build-llvm=build-llvm -fcoverage-prefix-map=/build/llvm-toolchain-snapshot-14~++20220119111520+da61cb019eb2/= -O3 -Wno-unused-command-line-argument -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-14~++20220119111520+da61cb019eb2/build-llvm -fdebug-prefix-map=/build/llvm-toolchain-snapshot-14~++20220119111520+da61cb019eb2/build-llvm=build-llvm -fdebug-prefix-map=/build/llvm-toolchain-snapshot-14~++20220119111520+da61cb019eb2/= -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2022-01-19-134126-35450-1 -x c++ /build/llvm-toolchain-snapshot-14~++20220119111520+da61cb019eb2/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

/build/llvm-toolchain-snapshot-14~++20220119111520+da61cb019eb2/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

1//===-- TargetLowering.cpp - Implement the TargetLowering class -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the TargetLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/CodeGen/TargetLowering.h"
14#include "llvm/ADT/STLExtras.h"
15#include "llvm/CodeGen/CallingConvLower.h"
16#include "llvm/CodeGen/MachineFrameInfo.h"
17#include "llvm/CodeGen/MachineFunction.h"
18#include "llvm/CodeGen/MachineJumpTableInfo.h"
19#include "llvm/CodeGen/MachineRegisterInfo.h"
20#include "llvm/CodeGen/SelectionDAG.h"
21#include "llvm/CodeGen/TargetRegisterInfo.h"
22#include "llvm/CodeGen/TargetSubtargetInfo.h"
23#include "llvm/IR/DataLayout.h"
24#include "llvm/IR/DerivedTypes.h"
25#include "llvm/IR/GlobalVariable.h"
26#include "llvm/IR/LLVMContext.h"
27#include "llvm/MC/MCAsmInfo.h"
28#include "llvm/MC/MCExpr.h"
29#include "llvm/Support/DivisionByConstantInfo.h"
30#include "llvm/Support/ErrorHandling.h"
31#include "llvm/Support/KnownBits.h"
32#include "llvm/Support/MathExtras.h"
33#include "llvm/Target/TargetLoweringObjectFile.h"
34#include "llvm/Target/TargetMachine.h"
35#include <cctype>
36using namespace llvm;
37
38/// NOTE: The TargetMachine owns TLOF.
39TargetLowering::TargetLowering(const TargetMachine &tm)
40 : TargetLoweringBase(tm) {}
41
42const char *TargetLowering::getTargetNodeName(unsigned Opcode) const {
43 return nullptr;
44}
45
46bool TargetLowering::isPositionIndependent() const {
47 return getTargetMachine().isPositionIndependent();
48}
49
50/// Check whether a given call node is in tail position within its function. If
51/// so, it sets Chain to the input chain of the tail call.
52bool TargetLowering::isInTailCallPosition(SelectionDAG &DAG, SDNode *Node,
53 SDValue &Chain) const {
54 const Function &F = DAG.getMachineFunction().getFunction();
55
56 // First, check if tail calls have been disabled in this function.
57 if (F.getFnAttribute("disable-tail-calls").getValueAsBool())
58 return false;
59
60 // Conservatively require the attributes of the call to match those of
61 // the return. Ignore following attributes because they don't affect the
62 // call sequence.
63 AttrBuilder CallerAttrs(F.getContext(), F.getAttributes().getRetAttrs());
64 for (const auto &Attr : {Attribute::Alignment, Attribute::Dereferenceable,
65 Attribute::DereferenceableOrNull, Attribute::NoAlias,
66 Attribute::NonNull})
67 CallerAttrs.removeAttribute(Attr);
68
69 if (CallerAttrs.hasAttributes())
70 return false;
71
72 // It's not safe to eliminate the sign / zero extension of the return value.
73 if (CallerAttrs.contains(Attribute::ZExt) ||
74 CallerAttrs.contains(Attribute::SExt))
75 return false;
76
77 // Check if the only use is a function return node.
78 return isUsedByReturnOnly(Node, Chain);
79}
80
81bool TargetLowering::parametersInCSRMatch(const MachineRegisterInfo &MRI,
82 const uint32_t *CallerPreservedMask,
83 const SmallVectorImpl<CCValAssign> &ArgLocs,
84 const SmallVectorImpl<SDValue> &OutVals) const {
85 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
86 const CCValAssign &ArgLoc = ArgLocs[I];
87 if (!ArgLoc.isRegLoc())
88 continue;
89 MCRegister Reg = ArgLoc.getLocReg();
90 // Only look at callee saved registers.
91 if (MachineOperand::clobbersPhysReg(CallerPreservedMask, Reg))
92 continue;
93 // Check that we pass the value used for the caller.
94 // (We look for a CopyFromReg reading a virtual register that is used
95 // for the function live-in value of register Reg)
96 SDValue Value = OutVals[I];
97 if (Value->getOpcode() != ISD::CopyFromReg)
98 return false;
99 Register ArgReg = cast<RegisterSDNode>(Value->getOperand(1))->getReg();
100 if (MRI.getLiveInPhysReg(ArgReg) != Reg)
101 return false;
102 }
103 return true;
104}
105
106/// Set CallLoweringInfo attribute flags based on a call instruction
107/// and called function attributes.
108void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call,
109 unsigned ArgIdx) {
110 IsSExt = Call->paramHasAttr(ArgIdx, Attribute::SExt);
111 IsZExt = Call->paramHasAttr(ArgIdx, Attribute::ZExt);
112 IsInReg = Call->paramHasAttr(ArgIdx, Attribute::InReg);
113 IsSRet = Call->paramHasAttr(ArgIdx, Attribute::StructRet);
114 IsNest = Call->paramHasAttr(ArgIdx, Attribute::Nest);
115 IsByVal = Call->paramHasAttr(ArgIdx, Attribute::ByVal);
116 IsPreallocated = Call->paramHasAttr(ArgIdx, Attribute::Preallocated);
117 IsInAlloca = Call->paramHasAttr(ArgIdx, Attribute::InAlloca);
118 IsReturned = Call->paramHasAttr(ArgIdx, Attribute::Returned);
119 IsSwiftSelf = Call->paramHasAttr(ArgIdx, Attribute::SwiftSelf);
120 IsSwiftAsync = Call->paramHasAttr(ArgIdx, Attribute::SwiftAsync);
121 IsSwiftError = Call->paramHasAttr(ArgIdx, Attribute::SwiftError);
122 Alignment = Call->getParamStackAlign(ArgIdx);
123 IndirectType = nullptr;
124 assert(IsByVal + IsPreallocated + IsInAlloca <= 1 &&(static_cast <bool> (IsByVal + IsPreallocated + IsInAlloca
<= 1 && "multiple ABI attributes?") ? void (0) : __assert_fail
("IsByVal + IsPreallocated + IsInAlloca <= 1 && \"multiple ABI attributes?\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 125, __extension__
__PRETTY_FUNCTION__))
125 "multiple ABI attributes?")(static_cast <bool> (IsByVal + IsPreallocated + IsInAlloca
<= 1 && "multiple ABI attributes?") ? void (0) : __assert_fail
("IsByVal + IsPreallocated + IsInAlloca <= 1 && \"multiple ABI attributes?\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 125, __extension__
__PRETTY_FUNCTION__))
;
126 if (IsByVal) {
127 IndirectType = Call->getParamByValType(ArgIdx);
128 if (!Alignment)
129 Alignment = Call->getParamAlign(ArgIdx);
130 }
131 if (IsPreallocated)
132 IndirectType = Call->getParamPreallocatedType(ArgIdx);
133 if (IsInAlloca)
134 IndirectType = Call->getParamInAllocaType(ArgIdx);
135}
136
137/// Generate a libcall taking the given operands as arguments and returning a
138/// result of type RetVT.
139std::pair<SDValue, SDValue>
140TargetLowering::makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT,
141 ArrayRef<SDValue> Ops,
142 MakeLibCallOptions CallOptions,
143 const SDLoc &dl,
144 SDValue InChain) const {
145 if (!InChain)
146 InChain = DAG.getEntryNode();
147
148 TargetLowering::ArgListTy Args;
149 Args.reserve(Ops.size());
150
151 TargetLowering::ArgListEntry Entry;
152 for (unsigned i = 0; i < Ops.size(); ++i) {
153 SDValue NewOp = Ops[i];
154 Entry.Node = NewOp;
155 Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
156 Entry.IsSExt = shouldSignExtendTypeInLibCall(NewOp.getValueType(),
157 CallOptions.IsSExt);
158 Entry.IsZExt = !Entry.IsSExt;
159
160 if (CallOptions.IsSoften &&
161 !shouldExtendTypeInLibCall(CallOptions.OpsVTBeforeSoften[i])) {
162 Entry.IsSExt = Entry.IsZExt = false;
163 }
164 Args.push_back(Entry);
165 }
166
167 if (LC == RTLIB::UNKNOWN_LIBCALL)
168 report_fatal_error("Unsupported library call operation!");
169 SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
170 getPointerTy(DAG.getDataLayout()));
171
172 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
173 TargetLowering::CallLoweringInfo CLI(DAG);
174 bool signExtend = shouldSignExtendTypeInLibCall(RetVT, CallOptions.IsSExt);
175 bool zeroExtend = !signExtend;
176
177 if (CallOptions.IsSoften &&
178 !shouldExtendTypeInLibCall(CallOptions.RetVTBeforeSoften)) {
179 signExtend = zeroExtend = false;
180 }
181
182 CLI.setDebugLoc(dl)
183 .setChain(InChain)
184 .setLibCallee(getLibcallCallingConv(LC), RetTy, Callee, std::move(Args))
185 .setNoReturn(CallOptions.DoesNotReturn)
186 .setDiscardResult(!CallOptions.IsReturnValueUsed)
187 .setIsPostTypeLegalization(CallOptions.IsPostTypeLegalization)
188 .setSExtResult(signExtend)
189 .setZExtResult(zeroExtend);
190 return LowerCallTo(CLI);
191}
192
193bool TargetLowering::findOptimalMemOpLowering(
194 std::vector<EVT> &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS,
195 unsigned SrcAS, const AttributeList &FuncAttributes) const {
196 if (Op.isMemcpyWithFixedDstAlign() && Op.getSrcAlign() < Op.getDstAlign())
197 return false;
198
199 EVT VT = getOptimalMemOpType(Op, FuncAttributes);
200
201 if (VT == MVT::Other) {
202 // Use the largest integer type whose alignment constraints are satisfied.
203 // We only need to check DstAlign here as SrcAlign is always greater or
204 // equal to DstAlign (or zero).
205 VT = MVT::i64;
206 if (Op.isFixedDstAlign())
207 while (Op.getDstAlign() < (VT.getSizeInBits() / 8) &&
208 !allowsMisalignedMemoryAccesses(VT, DstAS, Op.getDstAlign()))
209 VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1);
210 assert(VT.isInteger())(static_cast <bool> (VT.isInteger()) ? void (0) : __assert_fail
("VT.isInteger()", "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 210, __extension__ __PRETTY_FUNCTION__))
;
211
212 // Find the largest legal integer type.
213 MVT LVT = MVT::i64;
214 while (!isTypeLegal(LVT))
215 LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
216 assert(LVT.isInteger())(static_cast <bool> (LVT.isInteger()) ? void (0) : __assert_fail
("LVT.isInteger()", "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 216, __extension__ __PRETTY_FUNCTION__))
;
217
218 // If the type we've chosen is larger than the largest legal integer type
219 // then use that instead.
220 if (VT.bitsGT(LVT))
221 VT = LVT;
222 }
223
224 unsigned NumMemOps = 0;
225 uint64_t Size = Op.size();
226 while (Size) {
227 unsigned VTSize = VT.getSizeInBits() / 8;
228 while (VTSize > Size) {
229 // For now, only use non-vector load / store's for the left-over pieces.
230 EVT NewVT = VT;
231 unsigned NewVTSize;
232
233 bool Found = false;
234 if (VT.isVector() || VT.isFloatingPoint()) {
235 NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;
236 if (isOperationLegalOrCustom(ISD::STORE, NewVT) &&
237 isSafeMemOpType(NewVT.getSimpleVT()))
238 Found = true;
239 else if (NewVT == MVT::i64 &&
240 isOperationLegalOrCustom(ISD::STORE, MVT::f64) &&
241 isSafeMemOpType(MVT::f64)) {
242 // i64 is usually not legal on 32-bit targets, but f64 may be.
243 NewVT = MVT::f64;
244 Found = true;
245 }
246 }
247
248 if (!Found) {
249 do {
250 NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
251 if (NewVT == MVT::i8)
252 break;
253 } while (!isSafeMemOpType(NewVT.getSimpleVT()));
254 }
255 NewVTSize = NewVT.getSizeInBits() / 8;
256
257 // If the new VT cannot cover all of the remaining bits, then consider
258 // issuing a (or a pair of) unaligned and overlapping load / store.
259 bool Fast;
260 if (NumMemOps && Op.allowOverlap() && NewVTSize < Size &&
261 allowsMisalignedMemoryAccesses(
262 VT, DstAS, Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1),
263 MachineMemOperand::MONone, &Fast) &&
264 Fast)
265 VTSize = Size;
266 else {
267 VT = NewVT;
268 VTSize = NewVTSize;
269 }
270 }
271
272 if (++NumMemOps > Limit)
273 return false;
274
275 MemOps.push_back(VT);
276 Size -= VTSize;
277 }
278
279 return true;
280}
281
282/// Soften the operands of a comparison. This code is shared among BR_CC,
283/// SELECT_CC, and SETCC handlers.
284void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,
285 SDValue &NewLHS, SDValue &NewRHS,
286 ISD::CondCode &CCCode,
287 const SDLoc &dl, const SDValue OldLHS,
288 const SDValue OldRHS) const {
289 SDValue Chain;
290 return softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, dl, OldLHS,
291 OldRHS, Chain);
292}
293
294void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,
295 SDValue &NewLHS, SDValue &NewRHS,
296 ISD::CondCode &CCCode,
297 const SDLoc &dl, const SDValue OldLHS,
298 const SDValue OldRHS,
299 SDValue &Chain,
300 bool IsSignaling) const {
301 // FIXME: Currently we cannot really respect all IEEE predicates due to libgcc
302 // not supporting it. We can update this code when libgcc provides such
303 // functions.
304
305 assert((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128)(static_cast <bool> ((VT == MVT::f32 || VT == MVT::f64 ||
VT == MVT::f128 || VT == MVT::ppcf128) && "Unsupported setcc type!"
) ? void (0) : __assert_fail ("(VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128) && \"Unsupported setcc type!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 306, __extension__
__PRETTY_FUNCTION__))
306 && "Unsupported setcc type!")(static_cast <bool> ((VT == MVT::f32 || VT == MVT::f64 ||
VT == MVT::f128 || VT == MVT::ppcf128) && "Unsupported setcc type!"
) ? void (0) : __assert_fail ("(VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128) && \"Unsupported setcc type!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 306, __extension__
__PRETTY_FUNCTION__))
;
307
308 // Expand into one or more soft-fp libcall(s).
309 RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL;
310 bool ShouldInvertCC = false;
311 switch (CCCode) {
312 case ISD::SETEQ:
313 case ISD::SETOEQ:
314 LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
315 (VT == MVT::f64) ? RTLIB::OEQ_F64 :
316 (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
317 break;
318 case ISD::SETNE:
319 case ISD::SETUNE:
320 LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 :
321 (VT == MVT::f64) ? RTLIB::UNE_F64 :
322 (VT == MVT::f128) ? RTLIB::UNE_F128 : RTLIB::UNE_PPCF128;
323 break;
324 case ISD::SETGE:
325 case ISD::SETOGE:
326 LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
327 (VT == MVT::f64) ? RTLIB::OGE_F64 :
328 (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
329 break;
330 case ISD::SETLT:
331 case ISD::SETOLT:
332 LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
333 (VT == MVT::f64) ? RTLIB::OLT_F64 :
334 (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
335 break;
336 case ISD::SETLE:
337 case ISD::SETOLE:
338 LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
339 (VT == MVT::f64) ? RTLIB::OLE_F64 :
340 (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
341 break;
342 case ISD::SETGT:
343 case ISD::SETOGT:
344 LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
345 (VT == MVT::f64) ? RTLIB::OGT_F64 :
346 (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
347 break;
348 case ISD::SETO:
349 ShouldInvertCC = true;
350 LLVM_FALLTHROUGH[[gnu::fallthrough]];
351 case ISD::SETUO:
352 LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
353 (VT == MVT::f64) ? RTLIB::UO_F64 :
354 (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
355 break;
356 case ISD::SETONE:
357 // SETONE = O && UNE
358 ShouldInvertCC = true;
359 LLVM_FALLTHROUGH[[gnu::fallthrough]];
360 case ISD::SETUEQ:
361 LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
362 (VT == MVT::f64) ? RTLIB::UO_F64 :
363 (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
364 LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
365 (VT == MVT::f64) ? RTLIB::OEQ_F64 :
366 (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
367 break;
368 default:
369 // Invert CC for unordered comparisons
370 ShouldInvertCC = true;
371 switch (CCCode) {
372 case ISD::SETULT:
373 LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
374 (VT == MVT::f64) ? RTLIB::OGE_F64 :
375 (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
376 break;
377 case ISD::SETULE:
378 LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
379 (VT == MVT::f64) ? RTLIB::OGT_F64 :
380 (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
381 break;
382 case ISD::SETUGT:
383 LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
384 (VT == MVT::f64) ? RTLIB::OLE_F64 :
385 (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
386 break;
387 case ISD::SETUGE:
388 LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
389 (VT == MVT::f64) ? RTLIB::OLT_F64 :
390 (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
391 break;
392 default: llvm_unreachable("Do not know how to soften this setcc!")::llvm::llvm_unreachable_internal("Do not know how to soften this setcc!"
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 392)
;
393 }
394 }
395
396 // Use the target specific return value for comparions lib calls.
397 EVT RetVT = getCmpLibcallReturnType();
398 SDValue Ops[2] = {NewLHS, NewRHS};
399 TargetLowering::MakeLibCallOptions CallOptions;
400 EVT OpsVT[2] = { OldLHS.getValueType(),
401 OldRHS.getValueType() };
402 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT, true);
403 auto Call = makeLibCall(DAG, LC1, RetVT, Ops, CallOptions, dl, Chain);
404 NewLHS = Call.first;
405 NewRHS = DAG.getConstant(0, dl, RetVT);
406
407 CCCode = getCmpLibcallCC(LC1);
408 if (ShouldInvertCC) {
409 assert(RetVT.isInteger())(static_cast <bool> (RetVT.isInteger()) ? void (0) : __assert_fail
("RetVT.isInteger()", "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 409, __extension__ __PRETTY_FUNCTION__))
;
410 CCCode = getSetCCInverse(CCCode, RetVT);
411 }
412
413 if (LC2 == RTLIB::UNKNOWN_LIBCALL) {
414 // Update Chain.
415 Chain = Call.second;
416 } else {
417 EVT SetCCVT =
418 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), RetVT);
419 SDValue Tmp = DAG.getSetCC(dl, SetCCVT, NewLHS, NewRHS, CCCode);
420 auto Call2 = makeLibCall(DAG, LC2, RetVT, Ops, CallOptions, dl, Chain);
421 CCCode = getCmpLibcallCC(LC2);
422 if (ShouldInvertCC)
423 CCCode = getSetCCInverse(CCCode, RetVT);
424 NewLHS = DAG.getSetCC(dl, SetCCVT, Call2.first, NewRHS, CCCode);
425 if (Chain)
426 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Call.second,
427 Call2.second);
428 NewLHS = DAG.getNode(ShouldInvertCC ? ISD::AND : ISD::OR, dl,
429 Tmp.getValueType(), Tmp, NewLHS);
430 NewRHS = SDValue();
431 }
432}
433
434/// Return the entry encoding for a jump table in the current function. The
435/// returned value is a member of the MachineJumpTableInfo::JTEntryKind enum.
436unsigned TargetLowering::getJumpTableEncoding() const {
437 // In non-pic modes, just use the address of a block.
438 if (!isPositionIndependent())
439 return MachineJumpTableInfo::EK_BlockAddress;
440
441 // In PIC mode, if the target supports a GPRel32 directive, use it.
442 if (getTargetMachine().getMCAsmInfo()->getGPRel32Directive() != nullptr)
443 return MachineJumpTableInfo::EK_GPRel32BlockAddress;
444
445 // Otherwise, use a label difference.
446 return MachineJumpTableInfo::EK_LabelDifference32;
447}
448
449SDValue TargetLowering::getPICJumpTableRelocBase(SDValue Table,
450 SelectionDAG &DAG) const {
451 // If our PIC model is GP relative, use the global offset table as the base.
452 unsigned JTEncoding = getJumpTableEncoding();
453
454 if ((JTEncoding == MachineJumpTableInfo::EK_GPRel64BlockAddress) ||
455 (JTEncoding == MachineJumpTableInfo::EK_GPRel32BlockAddress))
456 return DAG.getGLOBAL_OFFSET_TABLE(getPointerTy(DAG.getDataLayout()));
457
458 return Table;
459}
460
461/// This returns the relocation base for the given PIC jumptable, the same as
462/// getPICJumpTableRelocBase, but as an MCExpr.
463const MCExpr *
464TargetLowering::getPICJumpTableRelocBaseExpr(const MachineFunction *MF,
465 unsigned JTI,MCContext &Ctx) const{
466 // The normal PIC reloc base is the label at the start of the jump table.
467 return MCSymbolRefExpr::create(MF->getJTISymbol(JTI, Ctx), Ctx);
468}
469
470bool
471TargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
472 const TargetMachine &TM = getTargetMachine();
473 const GlobalValue *GV = GA->getGlobal();
474
475 // If the address is not even local to this DSO we will have to load it from
476 // a got and then add the offset.
477 if (!TM.shouldAssumeDSOLocal(*GV->getParent(), GV))
478 return false;
479
480 // If the code is position independent we will have to add a base register.
481 if (isPositionIndependent())
482 return false;
483
484 // Otherwise we can do it.
485 return true;
486}
487
488//===----------------------------------------------------------------------===//
489// Optimization Methods
490//===----------------------------------------------------------------------===//
491
492/// If the specified instruction has a constant integer operand and there are
493/// bits set in that constant that are not demanded, then clear those bits and
494/// return true.
495bool TargetLowering::ShrinkDemandedConstant(SDValue Op,
496 const APInt &DemandedBits,
497 const APInt &DemandedElts,
498 TargetLoweringOpt &TLO) const {
499 SDLoc DL(Op);
500 unsigned Opcode = Op.getOpcode();
501
502 // Do target-specific constant optimization.
503 if (targetShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
504 return TLO.New.getNode();
505
506 // FIXME: ISD::SELECT, ISD::SELECT_CC
507 switch (Opcode) {
508 default:
509 break;
510 case ISD::XOR:
511 case ISD::AND:
512 case ISD::OR: {
513 auto *Op1C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
514 if (!Op1C || Op1C->isOpaque())
515 return false;
516
517 // If this is a 'not' op, don't touch it because that's a canonical form.
518 const APInt &C = Op1C->getAPIntValue();
519 if (Opcode == ISD::XOR && DemandedBits.isSubsetOf(C))
520 return false;
521
522 if (!C.isSubsetOf(DemandedBits)) {
523 EVT VT = Op.getValueType();
524 SDValue NewC = TLO.DAG.getConstant(DemandedBits & C, DL, VT);
525 SDValue NewOp = TLO.DAG.getNode(Opcode, DL, VT, Op.getOperand(0), NewC);
526 return TLO.CombineTo(Op, NewOp);
527 }
528
529 break;
530 }
531 }
532
533 return false;
534}
535
536bool TargetLowering::ShrinkDemandedConstant(SDValue Op,
537 const APInt &DemandedBits,
538 TargetLoweringOpt &TLO) const {
539 EVT VT = Op.getValueType();
540 APInt DemandedElts = VT.isVector()
541 ? APInt::getAllOnes(VT.getVectorNumElements())
542 : APInt(1, 1);
543 return ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO);
544}
545
546/// Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.
547/// This uses isZExtFree and ZERO_EXTEND for the widening cast, but it could be
548/// generalized for targets with other types of implicit widening casts.
549bool TargetLowering::ShrinkDemandedOp(SDValue Op, unsigned BitWidth,
550 const APInt &Demanded,
551 TargetLoweringOpt &TLO) const {
552 assert(Op.getNumOperands() == 2 &&(static_cast <bool> (Op.getNumOperands() == 2 &&
"ShrinkDemandedOp only supports binary operators!") ? void (
0) : __assert_fail ("Op.getNumOperands() == 2 && \"ShrinkDemandedOp only supports binary operators!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 553, __extension__
__PRETTY_FUNCTION__))
553 "ShrinkDemandedOp only supports binary operators!")(static_cast <bool> (Op.getNumOperands() == 2 &&
"ShrinkDemandedOp only supports binary operators!") ? void (
0) : __assert_fail ("Op.getNumOperands() == 2 && \"ShrinkDemandedOp only supports binary operators!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 553, __extension__
__PRETTY_FUNCTION__))
;
554 assert(Op.getNode()->getNumValues() == 1 &&(static_cast <bool> (Op.getNode()->getNumValues() ==
1 && "ShrinkDemandedOp only supports nodes with one result!"
) ? void (0) : __assert_fail ("Op.getNode()->getNumValues() == 1 && \"ShrinkDemandedOp only supports nodes with one result!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 555, __extension__
__PRETTY_FUNCTION__))
555 "ShrinkDemandedOp only supports nodes with one result!")(static_cast <bool> (Op.getNode()->getNumValues() ==
1 && "ShrinkDemandedOp only supports nodes with one result!"
) ? void (0) : __assert_fail ("Op.getNode()->getNumValues() == 1 && \"ShrinkDemandedOp only supports nodes with one result!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 555, __extension__
__PRETTY_FUNCTION__))
;
556
557 SelectionDAG &DAG = TLO.DAG;
558 SDLoc dl(Op);
559
560 // Early return, as this function cannot handle vector types.
561 if (Op.getValueType().isVector())
562 return false;
563
564 // Don't do this if the node has another user, which may require the
565 // full value.
566 if (!Op.getNode()->hasOneUse())
567 return false;
568
569 // Search for the smallest integer type with free casts to and from
570 // Op's type. For expedience, just check power-of-2 integer types.
571 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
572 unsigned DemandedSize = Demanded.getActiveBits();
573 unsigned SmallVTBits = DemandedSize;
574 if (!isPowerOf2_32(SmallVTBits))
575 SmallVTBits = NextPowerOf2(SmallVTBits);
576 for (; SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(SmallVTBits)) {
577 EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), SmallVTBits);
578 if (TLI.isTruncateFree(Op.getValueType(), SmallVT) &&
579 TLI.isZExtFree(SmallVT, Op.getValueType())) {
580 // We found a type with free casts.
581 SDValue X = DAG.getNode(
582 Op.getOpcode(), dl, SmallVT,
583 DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(0)),
584 DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(1)));
585 assert(DemandedSize <= SmallVTBits && "Narrowed below demanded bits?")(static_cast <bool> (DemandedSize <= SmallVTBits &&
"Narrowed below demanded bits?") ? void (0) : __assert_fail (
"DemandedSize <= SmallVTBits && \"Narrowed below demanded bits?\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 585, __extension__
__PRETTY_FUNCTION__))
;
586 SDValue Z = DAG.getNode(ISD::ANY_EXTEND, dl, Op.getValueType(), X);
587 return TLO.CombineTo(Op, Z);
588 }
589 }
590 return false;
591}
592
593bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
594 DAGCombinerInfo &DCI) const {
595 SelectionDAG &DAG = DCI.DAG;
596 TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
597 !DCI.isBeforeLegalizeOps());
598 KnownBits Known;
599
600 bool Simplified = SimplifyDemandedBits(Op, DemandedBits, Known, TLO);
601 if (Simplified) {
602 DCI.AddToWorklist(Op.getNode());
603 DCI.CommitTargetLoweringOpt(TLO);
604 }
605 return Simplified;
606}
607
608bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
609 KnownBits &Known,
610 TargetLoweringOpt &TLO,
611 unsigned Depth,
612 bool AssumeSingleUse) const {
613 EVT VT = Op.getValueType();
614
615 // TODO: We can probably do more work on calculating the known bits and
616 // simplifying the operations for scalable vectors, but for now we just
617 // bail out.
618 if (VT.isScalableVector()) {
619 // Pretend we don't know anything for now.
620 Known = KnownBits(DemandedBits.getBitWidth());
621 return false;
622 }
623
624 APInt DemandedElts = VT.isVector()
625 ? APInt::getAllOnes(VT.getVectorNumElements())
626 : APInt(1, 1);
627 return SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO, Depth,
628 AssumeSingleUse);
629}
630
631// TODO: Can we merge SelectionDAG::GetDemandedBits into this?
632// TODO: Under what circumstances can we create nodes? Constant folding?
633SDValue TargetLowering::SimplifyMultipleUseDemandedBits(
634 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
635 SelectionDAG &DAG, unsigned Depth) const {
636 // Limit search depth.
637 if (Depth >= SelectionDAG::MaxRecursionDepth)
638 return SDValue();
639
640 // Ignore UNDEFs.
641 if (Op.isUndef())
642 return SDValue();
643
644 // Not demanding any bits/elts from Op.
645 if (DemandedBits == 0 || DemandedElts == 0)
646 return DAG.getUNDEF(Op.getValueType());
647
648 bool IsLE = DAG.getDataLayout().isLittleEndian();
649 unsigned NumElts = DemandedElts.getBitWidth();
650 unsigned BitWidth = DemandedBits.getBitWidth();
651 KnownBits LHSKnown, RHSKnown;
652 switch (Op.getOpcode()) {
653 case ISD::BITCAST: {
654 SDValue Src = peekThroughBitcasts(Op.getOperand(0));
655 EVT SrcVT = Src.getValueType();
656 EVT DstVT = Op.getValueType();
657 if (SrcVT == DstVT)
658 return Src;
659
660 unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
661 unsigned NumDstEltBits = DstVT.getScalarSizeInBits();
662 if (NumSrcEltBits == NumDstEltBits)
663 if (SDValue V = SimplifyMultipleUseDemandedBits(
664 Src, DemandedBits, DemandedElts, DAG, Depth + 1))
665 return DAG.getBitcast(DstVT, V);
666
667 if (SrcVT.isVector() && (NumDstEltBits % NumSrcEltBits) == 0) {
668 unsigned Scale = NumDstEltBits / NumSrcEltBits;
669 unsigned NumSrcElts = SrcVT.getVectorNumElements();
670 APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);
671 APInt DemandedSrcElts = APInt::getZero(NumSrcElts);
672 for (unsigned i = 0; i != Scale; ++i) {
673 unsigned EltOffset = IsLE ? i : (Scale - 1 - i);
674 unsigned BitOffset = EltOffset * NumSrcEltBits;
675 APInt Sub = DemandedBits.extractBits(NumSrcEltBits, BitOffset);
676 if (!Sub.isZero()) {
677 DemandedSrcBits |= Sub;
678 for (unsigned j = 0; j != NumElts; ++j)
679 if (DemandedElts[j])
680 DemandedSrcElts.setBit((j * Scale) + i);
681 }
682 }
683
684 if (SDValue V = SimplifyMultipleUseDemandedBits(
685 Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))
686 return DAG.getBitcast(DstVT, V);
687 }
688
689 // TODO - bigendian once we have test coverage.
690 if (IsLE && (NumSrcEltBits % NumDstEltBits) == 0) {
691 unsigned Scale = NumSrcEltBits / NumDstEltBits;
692 unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
693 APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);
694 APInt DemandedSrcElts = APInt::getZero(NumSrcElts);
695 for (unsigned i = 0; i != NumElts; ++i)
696 if (DemandedElts[i]) {
697 unsigned Offset = (i % Scale) * NumDstEltBits;
698 DemandedSrcBits.insertBits(DemandedBits, Offset);
699 DemandedSrcElts.setBit(i / Scale);
700 }
701
702 if (SDValue V = SimplifyMultipleUseDemandedBits(
703 Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))
704 return DAG.getBitcast(DstVT, V);
705 }
706
707 break;
708 }
709 case ISD::AND: {
710 LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
711 RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
712
713 // If all of the demanded bits are known 1 on one side, return the other.
714 // These bits cannot contribute to the result of the 'and' in this
715 // context.
716 if (DemandedBits.isSubsetOf(LHSKnown.Zero | RHSKnown.One))
717 return Op.getOperand(0);
718 if (DemandedBits.isSubsetOf(RHSKnown.Zero | LHSKnown.One))
719 return Op.getOperand(1);
720 break;
721 }
722 case ISD::OR: {
723 LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
724 RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
725
726 // If all of the demanded bits are known zero on one side, return the
727 // other. These bits cannot contribute to the result of the 'or' in this
728 // context.
729 if (DemandedBits.isSubsetOf(LHSKnown.One | RHSKnown.Zero))
730 return Op.getOperand(0);
731 if (DemandedBits.isSubsetOf(RHSKnown.One | LHSKnown.Zero))
732 return Op.getOperand(1);
733 break;
734 }
735 case ISD::XOR: {
736 LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
737 RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
738
739 // If all of the demanded bits are known zero on one side, return the
740 // other.
741 if (DemandedBits.isSubsetOf(RHSKnown.Zero))
742 return Op.getOperand(0);
743 if (DemandedBits.isSubsetOf(LHSKnown.Zero))
744 return Op.getOperand(1);
745 break;
746 }
747 case ISD::SHL: {
748 // If we are only demanding sign bits then we can use the shift source
749 // directly.
750 if (const APInt *MaxSA =
751 DAG.getValidMaximumShiftAmountConstant(Op, DemandedElts)) {
752 SDValue Op0 = Op.getOperand(0);
753 unsigned ShAmt = MaxSA->getZExtValue();
754 unsigned NumSignBits =
755 DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
756 unsigned UpperDemandedBits = BitWidth - DemandedBits.countTrailingZeros();
757 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
758 return Op0;
759 }
760 break;
761 }
762 case ISD::SETCC: {
763 SDValue Op0 = Op.getOperand(0);
764 SDValue Op1 = Op.getOperand(1);
765 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
766 // If (1) we only need the sign-bit, (2) the setcc operands are the same
767 // width as the setcc result, and (3) the result of a setcc conforms to 0 or
768 // -1, we may be able to bypass the setcc.
769 if (DemandedBits.isSignMask() &&
770 Op0.getScalarValueSizeInBits() == BitWidth &&
771 getBooleanContents(Op0.getValueType()) ==
772 BooleanContent::ZeroOrNegativeOneBooleanContent) {
773 // If we're testing X < 0, then this compare isn't needed - just use X!
774 // FIXME: We're limiting to integer types here, but this should also work
775 // if we don't care about FP signed-zero. The use of SETLT with FP means
776 // that we don't care about NaNs.
777 if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
778 (isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode())))
779 return Op0;
780 }
781 break;
782 }
783 case ISD::SIGN_EXTEND_INREG: {
784 // If none of the extended bits are demanded, eliminate the sextinreg.
785 SDValue Op0 = Op.getOperand(0);
786 EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
787 unsigned ExBits = ExVT.getScalarSizeInBits();
788 if (DemandedBits.getActiveBits() <= ExBits)
789 return Op0;
790 // If the input is already sign extended, just drop the extension.
791 unsigned NumSignBits = DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
792 if (NumSignBits >= (BitWidth - ExBits + 1))
793 return Op0;
794 break;
795 }
796 case ISD::ANY_EXTEND_VECTOR_INREG:
797 case ISD::SIGN_EXTEND_VECTOR_INREG:
798 case ISD::ZERO_EXTEND_VECTOR_INREG: {
799 // If we only want the lowest element and none of extended bits, then we can
800 // return the bitcasted source vector.
801 SDValue Src = Op.getOperand(0);
802 EVT SrcVT = Src.getValueType();
803 EVT DstVT = Op.getValueType();
804 if (IsLE && DemandedElts == 1 &&
805 DstVT.getSizeInBits() == SrcVT.getSizeInBits() &&
806 DemandedBits.getActiveBits() <= SrcVT.getScalarSizeInBits()) {
807 return DAG.getBitcast(DstVT, Src);
808 }
809 break;
810 }
811 case ISD::INSERT_VECTOR_ELT: {
812 // If we don't demand the inserted element, return the base vector.
813 SDValue Vec = Op.getOperand(0);
814 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
815 EVT VecVT = Vec.getValueType();
816 if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements()) &&
817 !DemandedElts[CIdx->getZExtValue()])
818 return Vec;
819 break;
820 }
821 case ISD::INSERT_SUBVECTOR: {
822 SDValue Vec = Op.getOperand(0);
823 SDValue Sub = Op.getOperand(1);
824 uint64_t Idx = Op.getConstantOperandVal(2);
825 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
826 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
827 // If we don't demand the inserted subvector, return the base vector.
828 if (DemandedSubElts == 0)
829 return Vec;
830 // If this simply widens the lowest subvector, see if we can do it earlier.
831 if (Idx == 0 && Vec.isUndef()) {
832 if (SDValue NewSub = SimplifyMultipleUseDemandedBits(
833 Sub, DemandedBits, DemandedSubElts, DAG, Depth + 1))
834 return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
835 Op.getOperand(0), NewSub, Op.getOperand(2));
836 }
837 break;
838 }
839 case ISD::VECTOR_SHUFFLE: {
840 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
841
842 // If all the demanded elts are from one operand and are inline,
843 // then we can use the operand directly.
844 bool AllUndef = true, IdentityLHS = true, IdentityRHS = true;
845 for (unsigned i = 0; i != NumElts; ++i) {
846 int M = ShuffleMask[i];
847 if (M < 0 || !DemandedElts[i])
848 continue;
849 AllUndef = false;
850 IdentityLHS &= (M == (int)i);
851 IdentityRHS &= ((M - NumElts) == i);
852 }
853
854 if (AllUndef)
855 return DAG.getUNDEF(Op.getValueType());
856 if (IdentityLHS)
857 return Op.getOperand(0);
858 if (IdentityRHS)
859 return Op.getOperand(1);
860 break;
861 }
862 default:
863 if (Op.getOpcode() >= ISD::BUILTIN_OP_END)
864 if (SDValue V = SimplifyMultipleUseDemandedBitsForTargetNode(
865 Op, DemandedBits, DemandedElts, DAG, Depth))
866 return V;
867 break;
868 }
869 return SDValue();
870}
871
872SDValue TargetLowering::SimplifyMultipleUseDemandedBits(
873 SDValue Op, const APInt &DemandedBits, SelectionDAG &DAG,
874 unsigned Depth) const {
875 EVT VT = Op.getValueType();
876 APInt DemandedElts = VT.isVector()
877 ? APInt::getAllOnes(VT.getVectorNumElements())
878 : APInt(1, 1);
879 return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
880 Depth);
881}
882
883SDValue TargetLowering::SimplifyMultipleUseDemandedVectorElts(
884 SDValue Op, const APInt &DemandedElts, SelectionDAG &DAG,
885 unsigned Depth) const {
886 APInt DemandedBits = APInt::getAllOnes(Op.getScalarValueSizeInBits());
887 return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
888 Depth);
889}
890
891/// Look at Op. At this point, we know that only the OriginalDemandedBits of the
892/// result of Op are ever used downstream. If we can use this information to
893/// simplify Op, create a new simplified DAG node and return true, returning the
894/// original and new nodes in Old and New. Otherwise, analyze the expression and
895/// return a mask of Known bits for the expression (used to simplify the
896/// caller). The Known bits may only be accurate for those bits in the
897/// OriginalDemandedBits and OriginalDemandedElts.
898bool TargetLowering::SimplifyDemandedBits(
899 SDValue Op, const APInt &OriginalDemandedBits,
900 const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
901 unsigned Depth, bool AssumeSingleUse) const {
902 unsigned BitWidth = OriginalDemandedBits.getBitWidth();
903 assert(Op.getScalarValueSizeInBits() == BitWidth &&(static_cast <bool> (Op.getScalarValueSizeInBits() == BitWidth
&& "Mask size mismatches value type size!") ? void (
0) : __assert_fail ("Op.getScalarValueSizeInBits() == BitWidth && \"Mask size mismatches value type size!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 904, __extension__
__PRETTY_FUNCTION__))
904 "Mask size mismatches value type size!")(static_cast <bool> (Op.getScalarValueSizeInBits() == BitWidth
&& "Mask size mismatches value type size!") ? void (
0) : __assert_fail ("Op.getScalarValueSizeInBits() == BitWidth && \"Mask size mismatches value type size!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 904, __extension__
__PRETTY_FUNCTION__))
;
905
906 // Don't know anything.
907 Known = KnownBits(BitWidth);
908
909 // TODO: We can probably do more work on calculating the known bits and
910 // simplifying the operations for scalable vectors, but for now we just
911 // bail out.
912 if (Op.getValueType().isScalableVector())
913 return false;
914
915 bool IsLE = TLO.DAG.getDataLayout().isLittleEndian();
916 unsigned NumElts = OriginalDemandedElts.getBitWidth();
917 assert((!Op.getValueType().isVector() ||(static_cast <bool> ((!Op.getValueType().isVector() || NumElts
== Op.getValueType().getVectorNumElements()) && "Unexpected vector size"
) ? void (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 919, __extension__
__PRETTY_FUNCTION__))
918 NumElts == Op.getValueType().getVectorNumElements()) &&(static_cast <bool> ((!Op.getValueType().isVector() || NumElts
== Op.getValueType().getVectorNumElements()) && "Unexpected vector size"
) ? void (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 919, __extension__
__PRETTY_FUNCTION__))
919 "Unexpected vector size")(static_cast <bool> ((!Op.getValueType().isVector() || NumElts
== Op.getValueType().getVectorNumElements()) && "Unexpected vector size"
) ? void (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 919, __extension__
__PRETTY_FUNCTION__))
;
920
921 APInt DemandedBits = OriginalDemandedBits;
922 APInt DemandedElts = OriginalDemandedElts;
923 SDLoc dl(Op);
924 auto &DL = TLO.DAG.getDataLayout();
925
926 // Undef operand.
927 if (Op.isUndef())
928 return false;
929
930 if (Op.getOpcode() == ISD::Constant) {
931 // We know all of the bits for a constant!
932 Known = KnownBits::makeConstant(cast<ConstantSDNode>(Op)->getAPIntValue());
933 return false;
934 }
935
936 if (Op.getOpcode() == ISD::ConstantFP) {
937 // We know all of the bits for a floating point constant!
938 Known = KnownBits::makeConstant(
939 cast<ConstantFPSDNode>(Op)->getValueAPF().bitcastToAPInt());
940 return false;
941 }
942
943 // Other users may use these bits.
944 EVT VT = Op.getValueType();
945 if (!Op.getNode()->hasOneUse() && !AssumeSingleUse) {
946 if (Depth != 0) {
947 // If not at the root, Just compute the Known bits to
948 // simplify things downstream.
949 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
950 return false;
951 }
952 // If this is the root being simplified, allow it to have multiple uses,
953 // just set the DemandedBits/Elts to all bits.
954 DemandedBits = APInt::getAllOnes(BitWidth);
955 DemandedElts = APInt::getAllOnes(NumElts);
956 } else if (OriginalDemandedBits == 0 || OriginalDemandedElts == 0) {
957 // Not demanding any bits/elts from Op.
958 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
959 } else if (Depth >= SelectionDAG::MaxRecursionDepth) {
960 // Limit search depth.
961 return false;
962 }
963
964 KnownBits Known2;
965 switch (Op.getOpcode()) {
966 case ISD::TargetConstant:
967 llvm_unreachable("Can't simplify this node")::llvm::llvm_unreachable_internal("Can't simplify this node",
"llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 967)
;
968 case ISD::SCALAR_TO_VECTOR: {
969 if (!DemandedElts[0])
970 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
971
972 KnownBits SrcKnown;
973 SDValue Src = Op.getOperand(0);
974 unsigned SrcBitWidth = Src.getScalarValueSizeInBits();
975 APInt SrcDemandedBits = DemandedBits.zextOrSelf(SrcBitWidth);
976 if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcKnown, TLO, Depth + 1))
977 return true;
978
979 // Upper elements are undef, so only get the knownbits if we just demand
980 // the bottom element.
981 if (DemandedElts == 1)
982 Known = SrcKnown.anyextOrTrunc(BitWidth);
983 break;
984 }
985 case ISD::BUILD_VECTOR:
986 // Collect the known bits that are shared by every demanded element.
987 // TODO: Call SimplifyDemandedBits for non-constant demanded elements.
988 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
989 return false; // Don't fall through, will infinitely loop.
990 case ISD::LOAD: {
991 auto *LD = cast<LoadSDNode>(Op);
992 if (getTargetConstantFromLoad(LD)) {
993 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
994 return false; // Don't fall through, will infinitely loop.
995 }
996 if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
997 // If this is a ZEXTLoad and we are looking at the loaded value.
998 EVT MemVT = LD->getMemoryVT();
999 unsigned MemBits = MemVT.getScalarSizeInBits();
1000 Known.Zero.setBitsFrom(MemBits);
1001 return false; // Don't fall through, will infinitely loop.
1002 }
1003 break;
1004 }
1005 case ISD::INSERT_VECTOR_ELT: {
1006 SDValue Vec = Op.getOperand(0);
1007 SDValue Scl = Op.getOperand(1);
1008 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
1009 EVT VecVT = Vec.getValueType();
1010
1011 // If index isn't constant, assume we need all vector elements AND the
1012 // inserted element.
1013 APInt DemandedVecElts(DemandedElts);
1014 if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements())) {
1015 unsigned Idx = CIdx->getZExtValue();
1016 DemandedVecElts.clearBit(Idx);
1017
1018 // Inserted element is not required.
1019 if (!DemandedElts[Idx])
1020 return TLO.CombineTo(Op, Vec);
1021 }
1022
1023 KnownBits KnownScl;
1024 unsigned NumSclBits = Scl.getScalarValueSizeInBits();
1025 APInt DemandedSclBits = DemandedBits.zextOrTrunc(NumSclBits);
1026 if (SimplifyDemandedBits(Scl, DemandedSclBits, KnownScl, TLO, Depth + 1))
1027 return true;
1028
1029 Known = KnownScl.anyextOrTrunc(BitWidth);
1030
1031 KnownBits KnownVec;
1032 if (SimplifyDemandedBits(Vec, DemandedBits, DemandedVecElts, KnownVec, TLO,
1033 Depth + 1))
1034 return true;
1035
1036 if (!!DemandedVecElts)
1037 Known = KnownBits::commonBits(Known, KnownVec);
1038
1039 return false;
1040 }
1041 case ISD::INSERT_SUBVECTOR: {
1042 // Demand any elements from the subvector and the remainder from the src its
1043 // inserted into.
1044 SDValue Src = Op.getOperand(0);
1045 SDValue Sub = Op.getOperand(1);
1046 uint64_t Idx = Op.getConstantOperandVal(2);
1047 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
1048 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
1049 APInt DemandedSrcElts = DemandedElts;
1050 DemandedSrcElts.insertBits(APInt::getZero(NumSubElts), Idx);
1051
1052 KnownBits KnownSub, KnownSrc;
1053 if (SimplifyDemandedBits(Sub, DemandedBits, DemandedSubElts, KnownSub, TLO,
1054 Depth + 1))
1055 return true;
1056 if (SimplifyDemandedBits(Src, DemandedBits, DemandedSrcElts, KnownSrc, TLO,
1057 Depth + 1))
1058 return true;
1059
1060 Known.Zero.setAllBits();
1061 Known.One.setAllBits();
1062 if (!!DemandedSubElts)
1063 Known = KnownBits::commonBits(Known, KnownSub);
1064 if (!!DemandedSrcElts)
1065 Known = KnownBits::commonBits(Known, KnownSrc);
1066
1067 // Attempt to avoid multi-use src if we don't need anything from it.
1068 if (!DemandedBits.isAllOnes() || !DemandedSubElts.isAllOnes() ||
1069 !DemandedSrcElts.isAllOnes()) {
1070 SDValue NewSub = SimplifyMultipleUseDemandedBits(
1071 Sub, DemandedBits, DemandedSubElts, TLO.DAG, Depth + 1);
1072 SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1073 Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);
1074 if (NewSub || NewSrc) {
1075 NewSub = NewSub ? NewSub : Sub;
1076 NewSrc = NewSrc ? NewSrc : Src;
1077 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc, NewSub,
1078 Op.getOperand(2));
1079 return TLO.CombineTo(Op, NewOp);
1080 }
1081 }
1082 break;
1083 }
1084 case ISD::EXTRACT_SUBVECTOR: {
1085 // Offset the demanded elts by the subvector index.
1086 SDValue Src = Op.getOperand(0);
1087 if (Src.getValueType().isScalableVector())
1088 break;
1089 uint64_t Idx = Op.getConstantOperandVal(1);
1090 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
1091 APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
1092
1093 if (SimplifyDemandedBits(Src, DemandedBits, DemandedSrcElts, Known, TLO,
1094 Depth + 1))
1095 return true;
1096
1097 // Attempt to avoid multi-use src if we don't need anything from it.
1098 if (!DemandedBits.isAllOnes() || !DemandedSrcElts.isAllOnes()) {
1099 SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
1100 Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);
1101 if (DemandedSrc) {
1102 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc,
1103 Op.getOperand(1));
1104 return TLO.CombineTo(Op, NewOp);
1105 }
1106 }
1107 break;
1108 }
1109 case ISD::CONCAT_VECTORS: {
1110 Known.Zero.setAllBits();
1111 Known.One.setAllBits();
1112 EVT SubVT = Op.getOperand(0).getValueType();
1113 unsigned NumSubVecs = Op.getNumOperands();
1114 unsigned NumSubElts = SubVT.getVectorNumElements();
1115 for (unsigned i = 0; i != NumSubVecs; ++i) {
1116 APInt DemandedSubElts =
1117 DemandedElts.extractBits(NumSubElts, i * NumSubElts);
1118 if (SimplifyDemandedBits(Op.getOperand(i), DemandedBits, DemandedSubElts,
1119 Known2, TLO, Depth + 1))
1120 return true;
1121 // Known bits are shared by every demanded subvector element.
1122 if (!!DemandedSubElts)
1123 Known = KnownBits::commonBits(Known, Known2);
1124 }
1125 break;
1126 }
1127 case ISD::VECTOR_SHUFFLE: {
1128 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
1129
1130 // Collect demanded elements from shuffle operands..
1131 APInt DemandedLHS(NumElts, 0);
1132 APInt DemandedRHS(NumElts, 0);
1133 for (unsigned i = 0; i != NumElts; ++i) {
1134 if (!DemandedElts[i])
1135 continue;
1136 int M = ShuffleMask[i];
1137 if (M < 0) {
1138 // For UNDEF elements, we don't know anything about the common state of
1139 // the shuffle result.
1140 DemandedLHS.clearAllBits();
1141 DemandedRHS.clearAllBits();
1142 break;
1143 }
1144 assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range")(static_cast <bool> (0 <= M && M < (int)(
2 * NumElts) && "Shuffle index out of range") ? void (
0) : __assert_fail ("0 <= M && M < (int)(2 * NumElts) && \"Shuffle index out of range\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 1144, __extension__
__PRETTY_FUNCTION__))
;
1145 if (M < (int)NumElts)
1146 DemandedLHS.setBit(M);
1147 else
1148 DemandedRHS.setBit(M - NumElts);
1149 }
1150
1151 if (!!DemandedLHS || !!DemandedRHS) {
1152 SDValue Op0 = Op.getOperand(0);
1153 SDValue Op1 = Op.getOperand(1);
1154
1155 Known.Zero.setAllBits();
1156 Known.One.setAllBits();
1157 if (!!DemandedLHS) {
1158 if (SimplifyDemandedBits(Op0, DemandedBits, DemandedLHS, Known2, TLO,
1159 Depth + 1))
1160 return true;
1161 Known = KnownBits::commonBits(Known, Known2);
1162 }
1163 if (!!DemandedRHS) {
1164 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedRHS, Known2, TLO,
1165 Depth + 1))
1166 return true;
1167 Known = KnownBits::commonBits(Known, Known2);
1168 }
1169
1170 // Attempt to avoid multi-use ops if we don't need anything from them.
1171 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1172 Op0, DemandedBits, DemandedLHS, TLO.DAG, Depth + 1);
1173 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1174 Op1, DemandedBits, DemandedRHS, TLO.DAG, Depth + 1);
1175 if (DemandedOp0 || DemandedOp1) {
1176 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1177 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1178 SDValue NewOp = TLO.DAG.getVectorShuffle(VT, dl, Op0, Op1, ShuffleMask);
1179 return TLO.CombineTo(Op, NewOp);
1180 }
1181 }
1182 break;
1183 }
1184 case ISD::AND: {
1185 SDValue Op0 = Op.getOperand(0);
1186 SDValue Op1 = Op.getOperand(1);
1187
1188 // If the RHS is a constant, check to see if the LHS would be zero without
1189 // using the bits from the RHS. Below, we use knowledge about the RHS to
1190 // simplify the LHS, here we're using information from the LHS to simplify
1191 // the RHS.
1192 if (ConstantSDNode *RHSC = isConstOrConstSplat(Op1)) {
1193 // Do not increment Depth here; that can cause an infinite loop.
1194 KnownBits LHSKnown = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth);
1195 // If the LHS already has zeros where RHSC does, this 'and' is dead.
1196 if ((LHSKnown.Zero & DemandedBits) ==
1197 (~RHSC->getAPIntValue() & DemandedBits))
1198 return TLO.CombineTo(Op, Op0);
1199
1200 // If any of the set bits in the RHS are known zero on the LHS, shrink
1201 // the constant.
1202 if (ShrinkDemandedConstant(Op, ~LHSKnown.Zero & DemandedBits,
1203 DemandedElts, TLO))
1204 return true;
1205
1206 // Bitwise-not (xor X, -1) is a special case: we don't usually shrink its
1207 // constant, but if this 'and' is only clearing bits that were just set by
1208 // the xor, then this 'and' can be eliminated by shrinking the mask of
1209 // the xor. For example, for a 32-bit X:
1210 // and (xor (srl X, 31), -1), 1 --> xor (srl X, 31), 1
1211 if (isBitwiseNot(Op0) && Op0.hasOneUse() &&
1212 LHSKnown.One == ~RHSC->getAPIntValue()) {
1213 SDValue Xor = TLO.DAG.getNode(ISD::XOR, dl, VT, Op0.getOperand(0), Op1);
1214 return TLO.CombineTo(Op, Xor);
1215 }
1216 }
1217
1218 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1219 Depth + 1))
1220 return true;
1221 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 1221, __extension__
__PRETTY_FUNCTION__))
;
1222 if (SimplifyDemandedBits(Op0, ~Known.Zero & DemandedBits, DemandedElts,
1223 Known2, TLO, Depth + 1))
1224 return true;
1225 assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 1225, __extension__
__PRETTY_FUNCTION__))
;
1226
1227 // Attempt to avoid multi-use ops if we don't need anything from them.
1228 if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
1229 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1230 Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1231 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1232 Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1233 if (DemandedOp0 || DemandedOp1) {
1234 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1235 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1236 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1237 return TLO.CombineTo(Op, NewOp);
1238 }
1239 }
1240
1241 // If all of the demanded bits are known one on one side, return the other.
1242 // These bits cannot contribute to the result of the 'and'.
1243 if (DemandedBits.isSubsetOf(Known2.Zero | Known.One))
1244 return TLO.CombineTo(Op, Op0);
1245 if (DemandedBits.isSubsetOf(Known.Zero | Known2.One))
1246 return TLO.CombineTo(Op, Op1);
1247 // If all of the demanded bits in the inputs are known zeros, return zero.
1248 if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
1249 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, dl, VT));
1250 // If the RHS is a constant, see if we can simplify it.
1251 if (ShrinkDemandedConstant(Op, ~Known2.Zero & DemandedBits, DemandedElts,
1252 TLO))
1253 return true;
1254 // If the operation can be done in a smaller type, do so.
1255 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1256 return true;
1257
1258 Known &= Known2;
1259 break;
1260 }
1261 case ISD::OR: {
1262 SDValue Op0 = Op.getOperand(0);
1263 SDValue Op1 = Op.getOperand(1);
1264
1265 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1266 Depth + 1))
1267 return true;
1268 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 1268, __extension__
__PRETTY_FUNCTION__))
;
1269 if (SimplifyDemandedBits(Op0, ~Known.One & DemandedBits, DemandedElts,
1270 Known2, TLO, Depth + 1))
1271 return true;
1272 assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 1272, __extension__
__PRETTY_FUNCTION__))
;
1273
1274 // Attempt to avoid multi-use ops if we don't need anything from them.
1275 if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
1276 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1277 Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1278 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1279 Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1280 if (DemandedOp0 || DemandedOp1) {
1281 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1282 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1283 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1284 return TLO.CombineTo(Op, NewOp);
1285 }
1286 }
1287
1288 // If all of the demanded bits are known zero on one side, return the other.
1289 // These bits cannot contribute to the result of the 'or'.
1290 if (DemandedBits.isSubsetOf(Known2.One | Known.Zero))
1291 return TLO.CombineTo(Op, Op0);
1292 if (DemandedBits.isSubsetOf(Known.One | Known2.Zero))
1293 return TLO.CombineTo(Op, Op1);
1294 // If the RHS is a constant, see if we can simplify it.
1295 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1296 return true;
1297 // If the operation can be done in a smaller type, do so.
1298 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1299 return true;
1300
1301 Known |= Known2;
1302 break;
1303 }
1304 case ISD::XOR: {
1305 SDValue Op0 = Op.getOperand(0);
1306 SDValue Op1 = Op.getOperand(1);
1307
1308 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1309 Depth + 1))
1310 return true;
1311 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 1311, __extension__
__PRETTY_FUNCTION__))
;
1312 if (SimplifyDemandedBits(Op0, DemandedBits, DemandedElts, Known2, TLO,
1313 Depth + 1))
1314 return true;
1315 assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 1315, __extension__
__PRETTY_FUNCTION__))
;
1316
1317 // Attempt to avoid multi-use ops if we don't need anything from them.
1318 if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
1319 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1320 Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1321 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1322 Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1323 if (DemandedOp0 || DemandedOp1) {
1324 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1325 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1326 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1327 return TLO.CombineTo(Op, NewOp);
1328 }
1329 }
1330
1331 // If all of the demanded bits are known zero on one side, return the other.
1332 // These bits cannot contribute to the result of the 'xor'.
1333 if (DemandedBits.isSubsetOf(Known.Zero))
1334 return TLO.CombineTo(Op, Op0);
1335 if (DemandedBits.isSubsetOf(Known2.Zero))
1336 return TLO.CombineTo(Op, Op1);
1337 // If the operation can be done in a smaller type, do so.
1338 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1339 return true;
1340
1341 // If all of the unknown bits are known to be zero on one side or the other
1342 // turn this into an *inclusive* or.
1343 // e.g. (A & C1)^(B & C2) -> (A & C1)|(B & C2) iff C1&C2 == 0
1344 if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
1345 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::OR, dl, VT, Op0, Op1));
1346
1347 ConstantSDNode* C = isConstOrConstSplat(Op1, DemandedElts);
1348 if (C) {
1349 // If one side is a constant, and all of the set bits in the constant are
1350 // also known set on the other side, turn this into an AND, as we know
1351 // the bits will be cleared.
1352 // e.g. (X | C1) ^ C2 --> (X | C1) & ~C2 iff (C1&C2) == C2
1353 // NB: it is okay if more bits are known than are requested
1354 if (C->getAPIntValue() == Known2.One) {
1355 SDValue ANDC =
1356 TLO.DAG.getConstant(~C->getAPIntValue() & DemandedBits, dl, VT);
1357 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, dl, VT, Op0, ANDC));
1358 }
1359
1360 // If the RHS is a constant, see if we can change it. Don't alter a -1
1361 // constant because that's a 'not' op, and that is better for combining
1362 // and codegen.
1363 if (!C->isAllOnes() && DemandedBits.isSubsetOf(C->getAPIntValue())) {
1364 // We're flipping all demanded bits. Flip the undemanded bits too.
1365 SDValue New = TLO.DAG.getNOT(dl, Op0, VT);
1366 return TLO.CombineTo(Op, New);
1367 }
1368 }
1369
1370 // If we can't turn this into a 'not', try to shrink the constant.
1371 if (!C || !C->isAllOnes())
1372 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1373 return true;
1374
1375 Known ^= Known2;
1376 break;
1377 }
1378 case ISD::SELECT:
1379 if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known, TLO,
1380 Depth + 1))
1381 return true;
1382 if (SimplifyDemandedBits(Op.getOperand(1), DemandedBits, Known2, TLO,
1383 Depth + 1))
1384 return true;
1385 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 1385, __extension__
__PRETTY_FUNCTION__))
;
1386 assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 1386, __extension__
__PRETTY_FUNCTION__))
;
1387
1388 // If the operands are constants, see if we can simplify them.
1389 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1390 return true;
1391
1392 // Only known if known in both the LHS and RHS.
1393 Known = KnownBits::commonBits(Known, Known2);
1394 break;
1395 case ISD::SELECT_CC:
1396 if (SimplifyDemandedBits(Op.getOperand(3), DemandedBits, Known, TLO,
1397 Depth + 1))
1398 return true;
1399 if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known2, TLO,
1400 Depth + 1))
1401 return true;
1402 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 1402, __extension__
__PRETTY_FUNCTION__))
;
1403 assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 1403, __extension__
__PRETTY_FUNCTION__))
;
1404
1405 // If the operands are constants, see if we can simplify them.
1406 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1407 return true;
1408
1409 // Only known if known in both the LHS and RHS.
1410 Known = KnownBits::commonBits(Known, Known2);
1411 break;
1412 case ISD::SETCC: {
1413 SDValue Op0 = Op.getOperand(0);
1414 SDValue Op1 = Op.getOperand(1);
1415 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
1416 // If (1) we only need the sign-bit, (2) the setcc operands are the same
1417 // width as the setcc result, and (3) the result of a setcc conforms to 0 or
1418 // -1, we may be able to bypass the setcc.
1419 if (DemandedBits.isSignMask() &&
1420 Op0.getScalarValueSizeInBits() == BitWidth &&
1421 getBooleanContents(Op0.getValueType()) ==
1422 BooleanContent::ZeroOrNegativeOneBooleanContent) {
1423 // If we're testing X < 0, then this compare isn't needed - just use X!
1424 // FIXME: We're limiting to integer types here, but this should also work
1425 // if we don't care about FP signed-zero. The use of SETLT with FP means
1426 // that we don't care about NaNs.
1427 if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
1428 (isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode())))
1429 return TLO.CombineTo(Op, Op0);
1430
1431 // TODO: Should we check for other forms of sign-bit comparisons?
1432 // Examples: X <= -1, X >= 0
1433 }
1434 if (getBooleanContents(Op0.getValueType()) ==
1435 TargetLowering::ZeroOrOneBooleanContent &&
1436 BitWidth > 1)
1437 Known.Zero.setBitsFrom(1);
1438 break;
1439 }
1440 case ISD::SHL: {
1441 SDValue Op0 = Op.getOperand(0);
1442 SDValue Op1 = Op.getOperand(1);
1443 EVT ShiftVT = Op1.getValueType();
1444
1445 if (const APInt *SA =
1446 TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
1447 unsigned ShAmt = SA->getZExtValue();
1448 if (ShAmt == 0)
1449 return TLO.CombineTo(Op, Op0);
1450
1451 // If this is ((X >>u C1) << ShAmt), see if we can simplify this into a
1452 // single shift. We can do this if the bottom bits (which are shifted
1453 // out) are never demanded.
1454 // TODO - support non-uniform vector amounts.
1455 if (Op0.getOpcode() == ISD::SRL) {
1456 if (!DemandedBits.intersects(APInt::getLowBitsSet(BitWidth, ShAmt))) {
1457 if (const APInt *SA2 =
1458 TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {
1459 unsigned C1 = SA2->getZExtValue();
1460 unsigned Opc = ISD::SHL;
1461 int Diff = ShAmt - C1;
1462 if (Diff < 0) {
1463 Diff = -Diff;
1464 Opc = ISD::SRL;
1465 }
1466 SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);
1467 return TLO.CombineTo(
1468 Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));
1469 }
1470 }
1471 }
1472
1473 // Convert (shl (anyext x, c)) to (anyext (shl x, c)) if the high bits
1474 // are not demanded. This will likely allow the anyext to be folded away.
1475 // TODO - support non-uniform vector amounts.
1476 if (Op0.getOpcode() == ISD::ANY_EXTEND) {
1477 SDValue InnerOp = Op0.getOperand(0);
1478 EVT InnerVT = InnerOp.getValueType();
1479 unsigned InnerBits = InnerVT.getScalarSizeInBits();
1480 if (ShAmt < InnerBits && DemandedBits.getActiveBits() <= InnerBits &&
1481 isTypeDesirableForOp(ISD::SHL, InnerVT)) {
1482 EVT ShTy = getShiftAmountTy(InnerVT, DL);
1483 if (!APInt(BitWidth, ShAmt).isIntN(ShTy.getSizeInBits()))
1484 ShTy = InnerVT;
1485 SDValue NarrowShl =
1486 TLO.DAG.getNode(ISD::SHL, dl, InnerVT, InnerOp,
1487 TLO.DAG.getConstant(ShAmt, dl, ShTy));
1488 return TLO.CombineTo(
1489 Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, NarrowShl));
1490 }
1491
1492 // Repeat the SHL optimization above in cases where an extension
1493 // intervenes: (shl (anyext (shr x, c1)), c2) to
1494 // (shl (anyext x), c2-c1). This requires that the bottom c1 bits
1495 // aren't demanded (as above) and that the shifted upper c1 bits of
1496 // x aren't demanded.
1497 // TODO - support non-uniform vector amounts.
1498 if (Op0.hasOneUse() && InnerOp.getOpcode() == ISD::SRL &&
1499 InnerOp.hasOneUse()) {
1500 if (const APInt *SA2 =
1501 TLO.DAG.getValidShiftAmountConstant(InnerOp, DemandedElts)) {
1502 unsigned InnerShAmt = SA2->getZExtValue();
1503 if (InnerShAmt < ShAmt && InnerShAmt < InnerBits &&
1504 DemandedBits.getActiveBits() <=
1505 (InnerBits - InnerShAmt + ShAmt) &&
1506 DemandedBits.countTrailingZeros() >= ShAmt) {
1507 SDValue NewSA =
1508 TLO.DAG.getConstant(ShAmt - InnerShAmt, dl, ShiftVT);
1509 SDValue NewExt = TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT,
1510 InnerOp.getOperand(0));
1511 return TLO.CombineTo(
1512 Op, TLO.DAG.getNode(ISD::SHL, dl, VT, NewExt, NewSA));
1513 }
1514 }
1515 }
1516 }
1517
1518 APInt InDemandedMask = DemandedBits.lshr(ShAmt);
1519 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1520 Depth + 1))
1521 return true;
1522 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 1522, __extension__
__PRETTY_FUNCTION__))
;
1523 Known.Zero <<= ShAmt;
1524 Known.One <<= ShAmt;
1525 // low bits known zero.
1526 Known.Zero.setLowBits(ShAmt);
1527
1528 // Try shrinking the operation as long as the shift amount will still be
1529 // in range.
1530 if ((ShAmt < DemandedBits.getActiveBits()) &&
1531 ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1532 return true;
1533 }
1534
1535 // If we are only demanding sign bits then we can use the shift source
1536 // directly.
1537 if (const APInt *MaxSA =
1538 TLO.DAG.getValidMaximumShiftAmountConstant(Op, DemandedElts)) {
1539 unsigned ShAmt = MaxSA->getZExtValue();
1540 unsigned NumSignBits =
1541 TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
1542 unsigned UpperDemandedBits = BitWidth - DemandedBits.countTrailingZeros();
1543 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
1544 return TLO.CombineTo(Op, Op0);
1545 }
1546 break;
1547 }
1548 case ISD::SRL: {
1549 SDValue Op0 = Op.getOperand(0);
1550 SDValue Op1 = Op.getOperand(1);
1551 EVT ShiftVT = Op1.getValueType();
1552
1553 if (const APInt *SA =
1554 TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
1555 unsigned ShAmt = SA->getZExtValue();
1556 if (ShAmt == 0)
1557 return TLO.CombineTo(Op, Op0);
1558
1559 // If this is ((X << C1) >>u ShAmt), see if we can simplify this into a
1560 // single shift. We can do this if the top bits (which are shifted out)
1561 // are never demanded.
1562 // TODO - support non-uniform vector amounts.
1563 if (Op0.getOpcode() == ISD::SHL) {
1564 if (!DemandedBits.intersects(APInt::getHighBitsSet(BitWidth, ShAmt))) {
1565 if (const APInt *SA2 =
1566 TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {
1567 unsigned C1 = SA2->getZExtValue();
1568 unsigned Opc = ISD::SRL;
1569 int Diff = ShAmt - C1;
1570 if (Diff < 0) {
1571 Diff = -Diff;
1572 Opc = ISD::SHL;
1573 }
1574 SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);
1575 return TLO.CombineTo(
1576 Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));
1577 }
1578 }
1579 }
1580
1581 APInt InDemandedMask = (DemandedBits << ShAmt);
1582
1583 // If the shift is exact, then it does demand the low bits (and knows that
1584 // they are zero).
1585 if (Op->getFlags().hasExact())
1586 InDemandedMask.setLowBits(ShAmt);
1587
1588 // Compute the new bits that are at the top now.
1589 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1590 Depth + 1))
1591 return true;
1592 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 1592, __extension__
__PRETTY_FUNCTION__))
;
1593 Known.Zero.lshrInPlace(ShAmt);
1594 Known.One.lshrInPlace(ShAmt);
1595 // High bits known zero.
1596 Known.Zero.setHighBits(ShAmt);
1597 }
1598 break;
1599 }
1600 case ISD::SRA: {
1601 SDValue Op0 = Op.getOperand(0);
1602 SDValue Op1 = Op.getOperand(1);
1603 EVT ShiftVT = Op1.getValueType();
1604
1605 // If we only want bits that already match the signbit then we don't need
1606 // to shift.
1607 unsigned NumHiDemandedBits = BitWidth - DemandedBits.countTrailingZeros();
1608 if (TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1) >=
1609 NumHiDemandedBits)
1610 return TLO.CombineTo(Op, Op0);
1611
1612 // If this is an arithmetic shift right and only the low-bit is set, we can
1613 // always convert this into a logical shr, even if the shift amount is
1614 // variable. The low bit of the shift cannot be an input sign bit unless
1615 // the shift amount is >= the size of the datatype, which is undefined.
1616 if (DemandedBits.isOne())
1617 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1));
1618
1619 if (const APInt *SA =
1620 TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
1621 unsigned ShAmt = SA->getZExtValue();
1622 if (ShAmt == 0)
1623 return TLO.CombineTo(Op, Op0);
1624
1625 APInt InDemandedMask = (DemandedBits << ShAmt);
1626
1627 // If the shift is exact, then it does demand the low bits (and knows that
1628 // they are zero).
1629 if (Op->getFlags().hasExact())
1630 InDemandedMask.setLowBits(ShAmt);
1631
1632 // If any of the demanded bits are produced by the sign extension, we also
1633 // demand the input sign bit.
1634 if (DemandedBits.countLeadingZeros() < ShAmt)
1635 InDemandedMask.setSignBit();
1636
1637 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1638 Depth + 1))
1639 return true;
1640 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 1640, __extension__
__PRETTY_FUNCTION__))
;
1641 Known.Zero.lshrInPlace(ShAmt);
1642 Known.One.lshrInPlace(ShAmt);
1643
1644 // If the input sign bit is known to be zero, or if none of the top bits
1645 // are demanded, turn this into an unsigned shift right.
1646 if (Known.Zero[BitWidth - ShAmt - 1] ||
1647 DemandedBits.countLeadingZeros() >= ShAmt) {
1648 SDNodeFlags Flags;
1649 Flags.setExact(Op->getFlags().hasExact());
1650 return TLO.CombineTo(
1651 Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1, Flags));
1652 }
1653
1654 int Log2 = DemandedBits.exactLogBase2();
1655 if (Log2 >= 0) {
1656 // The bit must come from the sign.
1657 SDValue NewSA = TLO.DAG.getConstant(BitWidth - 1 - Log2, dl, ShiftVT);
1658 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, NewSA));
1659 }
1660
1661 if (Known.One[BitWidth - ShAmt - 1])
1662 // New bits are known one.
1663 Known.One.setHighBits(ShAmt);
1664
1665 // Attempt to avoid multi-use ops if we don't need anything from them.
1666 if (!InDemandedMask.isAllOnes() || !DemandedElts.isAllOnes()) {
1667 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1668 Op0, InDemandedMask, DemandedElts, TLO.DAG, Depth + 1);
1669 if (DemandedOp0) {
1670 SDValue NewOp = TLO.DAG.getNode(ISD::SRA, dl, VT, DemandedOp0, Op1);
1671 return TLO.CombineTo(Op, NewOp);
1672 }
1673 }
1674 }
1675 break;
1676 }
1677 case ISD::FSHL:
1678 case ISD::FSHR: {
1679 SDValue Op0 = Op.getOperand(0);
1680 SDValue Op1 = Op.getOperand(1);
1681 SDValue Op2 = Op.getOperand(2);
1682 bool IsFSHL = (Op.getOpcode() == ISD::FSHL);
1683
1684 if (ConstantSDNode *SA = isConstOrConstSplat(Op2, DemandedElts)) {
1685 unsigned Amt = SA->getAPIntValue().urem(BitWidth);
1686
1687 // For fshl, 0-shift returns the 1st arg.
1688 // For fshr, 0-shift returns the 2nd arg.
1689 if (Amt == 0) {
1690 if (SimplifyDemandedBits(IsFSHL ? Op0 : Op1, DemandedBits, DemandedElts,
1691 Known, TLO, Depth + 1))
1692 return true;
1693 break;
1694 }
1695
1696 // fshl: (Op0 << Amt) | (Op1 >> (BW - Amt))
1697 // fshr: (Op0 << (BW - Amt)) | (Op1 >> Amt)
1698 APInt Demanded0 = DemandedBits.lshr(IsFSHL ? Amt : (BitWidth - Amt));
1699 APInt Demanded1 = DemandedBits << (IsFSHL ? (BitWidth - Amt) : Amt);
1700 if (SimplifyDemandedBits(Op0, Demanded0, DemandedElts, Known2, TLO,
1701 Depth + 1))
1702 return true;
1703 if (SimplifyDemandedBits(Op1, Demanded1, DemandedElts, Known, TLO,
1704 Depth + 1))
1705 return true;
1706
1707 Known2.One <<= (IsFSHL ? Amt : (BitWidth - Amt));
1708 Known2.Zero <<= (IsFSHL ? Amt : (BitWidth - Amt));
1709 Known.One.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);
1710 Known.Zero.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);
1711 Known.One |= Known2.One;
1712 Known.Zero |= Known2.Zero;
1713 }
1714
1715 // For pow-2 bitwidths we only demand the bottom modulo amt bits.
1716 if (isPowerOf2_32(BitWidth)) {
1717 APInt DemandedAmtBits(Op2.getScalarValueSizeInBits(), BitWidth - 1);
1718 if (SimplifyDemandedBits(Op2, DemandedAmtBits, DemandedElts,
1719 Known2, TLO, Depth + 1))
1720 return true;
1721 }
1722 break;
1723 }
1724 case ISD::ROTL:
1725 case ISD::ROTR: {
1726 SDValue Op0 = Op.getOperand(0);
1727 SDValue Op1 = Op.getOperand(1);
1728 bool IsROTL = (Op.getOpcode() == ISD::ROTL);
1729
1730 // If we're rotating an 0/-1 value, then it stays an 0/-1 value.
1731 if (BitWidth == TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1))
1732 return TLO.CombineTo(Op, Op0);
1733
1734 if (ConstantSDNode *SA = isConstOrConstSplat(Op1, DemandedElts)) {
1735 unsigned Amt = SA->getAPIntValue().urem(BitWidth);
1736 unsigned RevAmt = BitWidth - Amt;
1737
1738 // rotl: (Op0 << Amt) | (Op0 >> (BW - Amt))
1739 // rotr: (Op0 << (BW - Amt)) | (Op0 >> Amt)
1740 APInt Demanded0 = DemandedBits.rotr(IsROTL ? Amt : RevAmt);
1741 if (SimplifyDemandedBits(Op0, Demanded0, DemandedElts, Known2, TLO,
1742 Depth + 1))
1743 return true;
1744
1745 // rot*(x, 0) --> x
1746 if (Amt == 0)
1747 return TLO.CombineTo(Op, Op0);
1748
1749 // See if we don't demand either half of the rotated bits.
1750 if ((!TLO.LegalOperations() || isOperationLegal(ISD::SHL, VT)) &&
1751 DemandedBits.countTrailingZeros() >= (IsROTL ? Amt : RevAmt)) {
1752 Op1 = TLO.DAG.getConstant(IsROTL ? Amt : RevAmt, dl, Op1.getValueType());
1753 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, Op1));
1754 }
1755 if ((!TLO.LegalOperations() || isOperationLegal(ISD::SRL, VT)) &&
1756 DemandedBits.countLeadingZeros() >= (IsROTL ? RevAmt : Amt)) {
1757 Op1 = TLO.DAG.getConstant(IsROTL ? RevAmt : Amt, dl, Op1.getValueType());
1758 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1));
1759 }
1760 }
1761
1762 // For pow-2 bitwidths we only demand the bottom modulo amt bits.
1763 if (isPowerOf2_32(BitWidth)) {
1764 APInt DemandedAmtBits(Op1.getScalarValueSizeInBits(), BitWidth - 1);
1765 if (SimplifyDemandedBits(Op1, DemandedAmtBits, DemandedElts, Known2, TLO,
1766 Depth + 1))
1767 return true;
1768 }
1769 break;
1770 }
1771 case ISD::UMIN: {
1772 // Check if one arg is always less than (or equal) to the other arg.
1773 SDValue Op0 = Op.getOperand(0);
1774 SDValue Op1 = Op.getOperand(1);
1775 KnownBits Known0 = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth + 1);
1776 KnownBits Known1 = TLO.DAG.computeKnownBits(Op1, DemandedElts, Depth + 1);
1777 Known = KnownBits::umin(Known0, Known1);
1778 if (Optional<bool> IsULE = KnownBits::ule(Known0, Known1))
1779 return TLO.CombineTo(Op, IsULE.getValue() ? Op0 : Op1);
1780 if (Optional<bool> IsULT = KnownBits::ult(Known0, Known1))
1781 return TLO.CombineTo(Op, IsULT.getValue() ? Op0 : Op1);
1782 break;
1783 }
1784 case ISD::UMAX: {
1785 // Check if one arg is always greater than (or equal) to the other arg.
1786 SDValue Op0 = Op.getOperand(0);
1787 SDValue Op1 = Op.getOperand(1);
1788 KnownBits Known0 = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth + 1);
1789 KnownBits Known1 = TLO.DAG.computeKnownBits(Op1, DemandedElts, Depth + 1);
1790 Known = KnownBits::umax(Known0, Known1);
1791 if (Optional<bool> IsUGE = KnownBits::uge(Known0, Known1))
1792 return TLO.CombineTo(Op, IsUGE.getValue() ? Op0 : Op1);
1793 if (Optional<bool> IsUGT = KnownBits::ugt(Known0, Known1))
1794 return TLO.CombineTo(Op, IsUGT.getValue() ? Op0 : Op1);
1795 break;
1796 }
1797 case ISD::BITREVERSE: {
1798 SDValue Src = Op.getOperand(0);
1799 APInt DemandedSrcBits = DemandedBits.reverseBits();
1800 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
1801 Depth + 1))
1802 return true;
1803 Known.One = Known2.One.reverseBits();
1804 Known.Zero = Known2.Zero.reverseBits();
1805 break;
1806 }
1807 case ISD::BSWAP: {
1808 SDValue Src = Op.getOperand(0);
1809
1810 // If the only bits demanded come from one byte of the bswap result,
1811 // just shift the input byte into position to eliminate the bswap.
1812 unsigned NLZ = DemandedBits.countLeadingZeros();
1813 unsigned NTZ = DemandedBits.countTrailingZeros();
1814
1815 // Round NTZ down to the next byte. If we have 11 trailing zeros, then
1816 // we need all the bits down to bit 8. Likewise, round NLZ. If we
1817 // have 14 leading zeros, round to 8.
1818 NLZ &= ~7;
1819 NTZ &= ~7;
1820 // If we need exactly one byte, we can do this transformation.
1821 if (BitWidth - NLZ - NTZ == 8) {
1822 unsigned ResultBit = NTZ;
1823 unsigned InputBit = BitWidth - NTZ - 8;
1824
1825 // Replace this with either a left or right shift to get the byte into
1826 // the right place.
1827 unsigned ShiftOpcode = InputBit > ResultBit ? ISD::SRL : ISD::SHL;
1828 if (!TLO.LegalOperations() || isOperationLegal(ShiftOpcode, VT)) {
1829 EVT ShiftAmtTy = getShiftAmountTy(VT, DL);
1830 unsigned ShiftAmount =
1831 InputBit > ResultBit ? InputBit - ResultBit : ResultBit - InputBit;
1832 SDValue ShAmt = TLO.DAG.getConstant(ShiftAmount, dl, ShiftAmtTy);
1833 SDValue NewOp = TLO.DAG.getNode(ShiftOpcode, dl, VT, Src, ShAmt);
1834 return TLO.CombineTo(Op, NewOp);
1835 }
1836 }
1837
1838 APInt DemandedSrcBits = DemandedBits.byteSwap();
1839 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
1840 Depth + 1))
1841 return true;
1842 Known.One = Known2.One.byteSwap();
1843 Known.Zero = Known2.Zero.byteSwap();
1844 break;
1845 }
1846 case ISD::CTPOP: {
1847 // If only 1 bit is demanded, replace with PARITY as long as we're before
1848 // op legalization.
1849 // FIXME: Limit to scalars for now.
1850 if (DemandedBits.isOne() && !TLO.LegalOps && !VT.isVector())
1851 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::PARITY, dl, VT,
1852 Op.getOperand(0)));
1853
1854 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
1855 break;
1856 }
1857 case ISD::SIGN_EXTEND_INREG: {
1858 SDValue Op0 = Op.getOperand(0);
1859 EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
1860 unsigned ExVTBits = ExVT.getScalarSizeInBits();
1861
1862 // If we only care about the highest bit, don't bother shifting right.
1863 if (DemandedBits.isSignMask()) {
1864 unsigned MinSignedBits =
1865 TLO.DAG.ComputeMaxSignificantBits(Op0, DemandedElts, Depth + 1);
1866 bool AlreadySignExtended = ExVTBits >= MinSignedBits;
1867 // However if the input is already sign extended we expect the sign
1868 // extension to be dropped altogether later and do not simplify.
1869 if (!AlreadySignExtended) {
1870 // Compute the correct shift amount type, which must be getShiftAmountTy
1871 // for scalar types after legalization.
1872 SDValue ShiftAmt = TLO.DAG.getConstant(BitWidth - ExVTBits, dl,
1873 getShiftAmountTy(VT, DL));
1874 return TLO.CombineTo(Op,
1875 TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, ShiftAmt));
1876 }
1877 }
1878
1879 // If none of the extended bits are demanded, eliminate the sextinreg.
1880 if (DemandedBits.getActiveBits() <= ExVTBits)
1881 return TLO.CombineTo(Op, Op0);
1882
1883 APInt InputDemandedBits = DemandedBits.getLoBits(ExVTBits);
1884
1885 // Since the sign extended bits are demanded, we know that the sign
1886 // bit is demanded.
1887 InputDemandedBits.setBit(ExVTBits - 1);
1888
1889 if (SimplifyDemandedBits(Op0, InputDemandedBits, Known, TLO, Depth + 1))
1890 return true;
1891 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 1891, __extension__
__PRETTY_FUNCTION__))
;
1892
1893 // If the sign bit of the input is known set or clear, then we know the
1894 // top bits of the result.
1895
1896 // If the input sign bit is known zero, convert this into a zero extension.
1897 if (Known.Zero[ExVTBits - 1])
1898 return TLO.CombineTo(Op, TLO.DAG.getZeroExtendInReg(Op0, dl, ExVT));
1899
1900 APInt Mask = APInt::getLowBitsSet(BitWidth, ExVTBits);
1901 if (Known.One[ExVTBits - 1]) { // Input sign bit known set
1902 Known.One.setBitsFrom(ExVTBits);
1903 Known.Zero &= Mask;
1904 } else { // Input sign bit unknown
1905 Known.Zero &= Mask;
1906 Known.One &= Mask;
1907 }
1908 break;
1909 }
1910 case ISD::BUILD_PAIR: {
1911 EVT HalfVT = Op.getOperand(0).getValueType();
1912 unsigned HalfBitWidth = HalfVT.getScalarSizeInBits();
1913
1914 APInt MaskLo = DemandedBits.getLoBits(HalfBitWidth).trunc(HalfBitWidth);
1915 APInt MaskHi = DemandedBits.getHiBits(HalfBitWidth).trunc(HalfBitWidth);
1916
1917 KnownBits KnownLo, KnownHi;
1918
1919 if (SimplifyDemandedBits(Op.getOperand(0), MaskLo, KnownLo, TLO, Depth + 1))
1920 return true;
1921
1922 if (SimplifyDemandedBits(Op.getOperand(1), MaskHi, KnownHi, TLO, Depth + 1))
1923 return true;
1924
1925 Known.Zero = KnownLo.Zero.zext(BitWidth) |
1926 KnownHi.Zero.zext(BitWidth).shl(HalfBitWidth);
1927
1928 Known.One = KnownLo.One.zext(BitWidth) |
1929 KnownHi.One.zext(BitWidth).shl(HalfBitWidth);
1930 break;
1931 }
1932 case ISD::ZERO_EXTEND:
1933 case ISD::ZERO_EXTEND_VECTOR_INREG: {
1934 SDValue Src = Op.getOperand(0);
1935 EVT SrcVT = Src.getValueType();
1936 unsigned InBits = SrcVT.getScalarSizeInBits();
1937 unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
1938 bool IsVecInReg = Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG;
1939
1940 // If none of the top bits are demanded, convert this into an any_extend.
1941 if (DemandedBits.getActiveBits() <= InBits) {
1942 // If we only need the non-extended bits of the bottom element
1943 // then we can just bitcast to the result.
1944 if (IsLE && IsVecInReg && DemandedElts == 1 &&
1945 VT.getSizeInBits() == SrcVT.getSizeInBits())
1946 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
1947
1948 unsigned Opc =
1949 IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;
1950 if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
1951 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
1952 }
1953
1954 APInt InDemandedBits = DemandedBits.trunc(InBits);
1955 APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
1956 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
1957 Depth + 1))
1958 return true;
1959 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 1959, __extension__
__PRETTY_FUNCTION__))
;
1960 assert(Known.getBitWidth() == InBits && "Src width has changed?")(static_cast <bool> (Known.getBitWidth() == InBits &&
"Src width has changed?") ? void (0) : __assert_fail ("Known.getBitWidth() == InBits && \"Src width has changed?\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 1960, __extension__
__PRETTY_FUNCTION__))
;
1961 Known = Known.zext(BitWidth);
1962
1963 // Attempt to avoid multi-use ops if we don't need anything from them.
1964 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1965 Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
1966 return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
1967 break;
1968 }
1969 case ISD::SIGN_EXTEND:
1970 case ISD::SIGN_EXTEND_VECTOR_INREG: {
1971 SDValue Src = Op.getOperand(0);
1972 EVT SrcVT = Src.getValueType();
1973 unsigned InBits = SrcVT.getScalarSizeInBits();
1974 unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
1975 bool IsVecInReg = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG;
1976
1977 // If none of the top bits are demanded, convert this into an any_extend.
1978 if (DemandedBits.getActiveBits() <= InBits) {
1979 // If we only need the non-extended bits of the bottom element
1980 // then we can just bitcast to the result.
1981 if (IsLE && IsVecInReg && DemandedElts == 1 &&
1982 VT.getSizeInBits() == SrcVT.getSizeInBits())
1983 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
1984
1985 unsigned Opc =
1986 IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;
1987 if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
1988 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
1989 }
1990
1991 APInt InDemandedBits = DemandedBits.trunc(InBits);
1992 APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
1993
1994 // Since some of the sign extended bits are demanded, we know that the sign
1995 // bit is demanded.
1996 InDemandedBits.setBit(InBits - 1);
1997
1998 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
1999 Depth + 1))
2000 return true;
2001 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 2001, __extension__
__PRETTY_FUNCTION__))
;
2002 assert(Known.getBitWidth() == InBits && "Src width has changed?")(static_cast <bool> (Known.getBitWidth() == InBits &&
"Src width has changed?") ? void (0) : __assert_fail ("Known.getBitWidth() == InBits && \"Src width has changed?\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 2002, __extension__
__PRETTY_FUNCTION__))
;
2003
2004 // If the sign bit is known one, the top bits match.
2005 Known = Known.sext(BitWidth);
2006
2007 // If the sign bit is known zero, convert this to a zero extend.
2008 if (Known.isNonNegative()) {
2009 unsigned Opc =
2010 IsVecInReg ? ISD::ZERO_EXTEND_VECTOR_INREG : ISD::ZERO_EXTEND;
2011 if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
2012 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
2013 }
2014
2015 // Attempt to avoid multi-use ops if we don't need anything from them.
2016 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2017 Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
2018 return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
2019 break;
2020 }
2021 case ISD::ANY_EXTEND:
2022 case ISD::ANY_EXTEND_VECTOR_INREG: {
2023 SDValue Src = Op.getOperand(0);
2024 EVT SrcVT = Src.getValueType();
2025 unsigned InBits = SrcVT.getScalarSizeInBits();
2026 unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
2027 bool IsVecInReg = Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG;
2028
2029 // If we only need the bottom element then we can just bitcast.
2030 // TODO: Handle ANY_EXTEND?
2031 if (IsLE && IsVecInReg && DemandedElts == 1 &&
2032 VT.getSizeInBits() == SrcVT.getSizeInBits())
2033 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
2034
2035 APInt InDemandedBits = DemandedBits.trunc(InBits);
2036 APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
2037 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
2038 Depth + 1))
2039 return true;
2040 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 2040, __extension__
__PRETTY_FUNCTION__))
;
2041 assert(Known.getBitWidth() == InBits && "Src width has changed?")(static_cast <bool> (Known.getBitWidth() == InBits &&
"Src width has changed?") ? void (0) : __assert_fail ("Known.getBitWidth() == InBits && \"Src width has changed?\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 2041, __extension__
__PRETTY_FUNCTION__))
;
2042 Known = Known.anyext(BitWidth);
2043
2044 // Attempt to avoid multi-use ops if we don't need anything from them.
2045 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2046 Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
2047 return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
2048 break;
2049 }
2050 case ISD::TRUNCATE: {
2051 SDValue Src = Op.getOperand(0);
2052
2053 // Simplify the input, using demanded bit information, and compute the known
2054 // zero/one bits live out.
2055 unsigned OperandBitWidth = Src.getScalarValueSizeInBits();
2056 APInt TruncMask = DemandedBits.zext(OperandBitWidth);
2057 if (SimplifyDemandedBits(Src, TruncMask, DemandedElts, Known, TLO,
2058 Depth + 1))
2059 return true;
2060 Known = Known.trunc(BitWidth);
2061
2062 // Attempt to avoid multi-use ops if we don't need anything from them.
2063 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2064 Src, TruncMask, DemandedElts, TLO.DAG, Depth + 1))
2065 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, NewSrc));
2066
2067 // If the input is only used by this truncate, see if we can shrink it based
2068 // on the known demanded bits.
2069 if (Src.getNode()->hasOneUse()) {
2070 switch (Src.getOpcode()) {
2071 default:
2072 break;
2073 case ISD::SRL:
2074 // Shrink SRL by a constant if none of the high bits shifted in are
2075 // demanded.
2076 if (TLO.LegalTypes() && !isTypeDesirableForOp(ISD::SRL, VT))
2077 // Do not turn (vt1 truncate (vt2 srl)) into (vt1 srl) if vt1 is
2078 // undesirable.
2079 break;
2080
2081 const APInt *ShAmtC =
2082 TLO.DAG.getValidShiftAmountConstant(Src, DemandedElts);
2083 if (!ShAmtC || ShAmtC->uge(BitWidth))
2084 break;
2085 uint64_t ShVal = ShAmtC->getZExtValue();
2086
2087 APInt HighBits =
2088 APInt::getHighBitsSet(OperandBitWidth, OperandBitWidth - BitWidth);
2089 HighBits.lshrInPlace(ShVal);
2090 HighBits = HighBits.trunc(BitWidth);
2091
2092 if (!(HighBits & DemandedBits)) {
2093 // None of the shifted in bits are needed. Add a truncate of the
2094 // shift input, then shift it.
2095 SDValue NewShAmt = TLO.DAG.getConstant(
2096 ShVal, dl, getShiftAmountTy(VT, DL, TLO.LegalTypes()));
2097 SDValue NewTrunc =
2098 TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, Src.getOperand(0));
2099 return TLO.CombineTo(
2100 Op, TLO.DAG.getNode(ISD::SRL, dl, VT, NewTrunc, NewShAmt));
2101 }
2102 break;
2103 }
2104 }
2105
2106 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 2106, __extension__
__PRETTY_FUNCTION__))
;
2107 break;
2108 }
2109 case ISD::AssertZext: {
2110 // AssertZext demands all of the high bits, plus any of the low bits
2111 // demanded by its users.
2112 EVT ZVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2113 APInt InMask = APInt::getLowBitsSet(BitWidth, ZVT.getSizeInBits());
2114 if (SimplifyDemandedBits(Op.getOperand(0), ~InMask | DemandedBits, Known,
2115 TLO, Depth + 1))
2116 return true;
2117 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 2117, __extension__
__PRETTY_FUNCTION__))
;
2118
2119 Known.Zero |= ~InMask;
2120 break;
2121 }
2122 case ISD::EXTRACT_VECTOR_ELT: {
2123 SDValue Src = Op.getOperand(0);
2124 SDValue Idx = Op.getOperand(1);
2125 ElementCount SrcEltCnt = Src.getValueType().getVectorElementCount();
2126 unsigned EltBitWidth = Src.getScalarValueSizeInBits();
2127
2128 if (SrcEltCnt.isScalable())
2129 return false;
2130
2131 // Demand the bits from every vector element without a constant index.
2132 unsigned NumSrcElts = SrcEltCnt.getFixedValue();
2133 APInt DemandedSrcElts = APInt::getAllOnes(NumSrcElts);
2134 if (auto *CIdx = dyn_cast<ConstantSDNode>(Idx))
2135 if (CIdx->getAPIntValue().ult(NumSrcElts))
2136 DemandedSrcElts = APInt::getOneBitSet(NumSrcElts, CIdx->getZExtValue());
2137
2138 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
2139 // anything about the extended bits.
2140 APInt DemandedSrcBits = DemandedBits;
2141 if (BitWidth > EltBitWidth)
2142 DemandedSrcBits = DemandedSrcBits.trunc(EltBitWidth);
2143
2144 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts, Known2, TLO,
2145 Depth + 1))
2146 return true;
2147
2148 // Attempt to avoid multi-use ops if we don't need anything from them.
2149 if (!DemandedSrcBits.isAllOnes() || !DemandedSrcElts.isAllOnes()) {
2150 if (SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
2151 Src, DemandedSrcBits, DemandedSrcElts, TLO.DAG, Depth + 1)) {
2152 SDValue NewOp =
2153 TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc, Idx);
2154 return TLO.CombineTo(Op, NewOp);
2155 }
2156 }
2157
2158 Known = Known2;
2159 if (BitWidth > EltBitWidth)
2160 Known = Known.anyext(BitWidth);
2161 break;
2162 }
2163 case ISD::BITCAST: {
2164 SDValue Src = Op.getOperand(0);
2165 EVT SrcVT = Src.getValueType();
2166 unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
2167
2168 // If this is an FP->Int bitcast and if the sign bit is the only
2169 // thing demanded, turn this into a FGETSIGN.
2170 if (!TLO.LegalOperations() && !VT.isVector() && !SrcVT.isVector() &&
2171 DemandedBits == APInt::getSignMask(Op.getValueSizeInBits()) &&
2172 SrcVT.isFloatingPoint()) {
2173 bool OpVTLegal = isOperationLegalOrCustom(ISD::FGETSIGN, VT);
2174 bool i32Legal = isOperationLegalOrCustom(ISD::FGETSIGN, MVT::i32);
2175 if ((OpVTLegal || i32Legal) && VT.isSimple() && SrcVT != MVT::f16 &&
2176 SrcVT != MVT::f128) {
2177 // Cannot eliminate/lower SHL for f128 yet.
2178 EVT Ty = OpVTLegal ? VT : MVT::i32;
2179 // Make a FGETSIGN + SHL to move the sign bit into the appropriate
2180 // place. We expect the SHL to be eliminated by other optimizations.
2181 SDValue Sign = TLO.DAG.getNode(ISD::FGETSIGN, dl, Ty, Src);
2182 unsigned OpVTSizeInBits = Op.getValueSizeInBits();
2183 if (!OpVTLegal && OpVTSizeInBits > 32)
2184 Sign = TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Sign);
2185 unsigned ShVal = Op.getValueSizeInBits() - 1;
2186 SDValue ShAmt = TLO.DAG.getConstant(ShVal, dl, VT);
2187 return TLO.CombineTo(Op,
2188 TLO.DAG.getNode(ISD::SHL, dl, VT, Sign, ShAmt));
2189 }
2190 }
2191
2192 // Bitcast from a vector using SimplifyDemanded Bits/VectorElts.
2193 // Demand the elt/bit if any of the original elts/bits are demanded.
2194 if (SrcVT.isVector() && (BitWidth % NumSrcEltBits) == 0) {
2195 unsigned Scale = BitWidth / NumSrcEltBits;
2196 unsigned NumSrcElts = SrcVT.getVectorNumElements();
2197 APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);
2198 APInt DemandedSrcElts = APInt::getZero(NumSrcElts);
2199 for (unsigned i = 0; i != Scale; ++i) {
2200 unsigned EltOffset = IsLE ? i : (Scale - 1 - i);
2201 unsigned BitOffset = EltOffset * NumSrcEltBits;
2202 APInt Sub = DemandedBits.extractBits(NumSrcEltBits, BitOffset);
2203 if (!Sub.isZero()) {
2204 DemandedSrcBits |= Sub;
2205 for (unsigned j = 0; j != NumElts; ++j)
2206 if (DemandedElts[j])
2207 DemandedSrcElts.setBit((j * Scale) + i);
2208 }
2209 }
2210
2211 APInt KnownSrcUndef, KnownSrcZero;
2212 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
2213 KnownSrcZero, TLO, Depth + 1))
2214 return true;
2215
2216 KnownBits KnownSrcBits;
2217 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
2218 KnownSrcBits, TLO, Depth + 1))
2219 return true;
2220 } else if (IsLE && (NumSrcEltBits % BitWidth) == 0) {
2221 // TODO - bigendian once we have test coverage.
2222 unsigned Scale = NumSrcEltBits / BitWidth;
2223 unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
2224 APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);
2225 APInt DemandedSrcElts = APInt::getZero(NumSrcElts);
2226 for (unsigned i = 0; i != NumElts; ++i)
2227 if (DemandedElts[i]) {
2228 unsigned Offset = (i % Scale) * BitWidth;
2229 DemandedSrcBits.insertBits(DemandedBits, Offset);
2230 DemandedSrcElts.setBit(i / Scale);
2231 }
2232
2233 if (SrcVT.isVector()) {
2234 APInt KnownSrcUndef, KnownSrcZero;
2235 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
2236 KnownSrcZero, TLO, Depth + 1))
2237 return true;
2238 }
2239
2240 KnownBits KnownSrcBits;
2241 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
2242 KnownSrcBits, TLO, Depth + 1))
2243 return true;
2244 }
2245
2246 // If this is a bitcast, let computeKnownBits handle it. Only do this on a
2247 // recursive call where Known may be useful to the caller.
2248 if (Depth > 0) {
2249 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2250 return false;
2251 }
2252 break;
2253 }
2254 case ISD::ADD:
2255 case ISD::MUL:
2256 case ISD::SUB: {
2257 // Add, Sub, and Mul don't demand any bits in positions beyond that
2258 // of the highest bit demanded of them.
2259 SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1);
2260 SDNodeFlags Flags = Op.getNode()->getFlags();
2261 unsigned DemandedBitsLZ = DemandedBits.countLeadingZeros();
2262 APInt LoMask = APInt::getLowBitsSet(BitWidth, BitWidth - DemandedBitsLZ);
2263 if (SimplifyDemandedBits(Op0, LoMask, DemandedElts, Known2, TLO,
2264 Depth + 1) ||
2265 SimplifyDemandedBits(Op1, LoMask, DemandedElts, Known2, TLO,
2266 Depth + 1) ||
2267 // See if the operation should be performed at a smaller bit width.
2268 ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO)) {
2269 if (Flags.hasNoSignedWrap() || Flags.hasNoUnsignedWrap()) {
2270 // Disable the nsw and nuw flags. We can no longer guarantee that we
2271 // won't wrap after simplification.
2272 Flags.setNoSignedWrap(false);
2273 Flags.setNoUnsignedWrap(false);
2274 SDValue NewOp =
2275 TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);
2276 return TLO.CombineTo(Op, NewOp);
2277 }
2278 return true;
2279 }
2280
2281 // Attempt to avoid multi-use ops if we don't need anything from them.
2282 if (!LoMask.isAllOnes() || !DemandedElts.isAllOnes()) {
2283 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2284 Op0, LoMask, DemandedElts, TLO.DAG, Depth + 1);
2285 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
2286 Op1, LoMask, DemandedElts, TLO.DAG, Depth + 1);
2287 if (DemandedOp0 || DemandedOp1) {
2288 Flags.setNoSignedWrap(false);
2289 Flags.setNoUnsignedWrap(false);
2290 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
2291 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
2292 SDValue NewOp =
2293 TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);
2294 return TLO.CombineTo(Op, NewOp);
2295 }
2296 }
2297
2298 // If we have a constant operand, we may be able to turn it into -1 if we
2299 // do not demand the high bits. This can make the constant smaller to
2300 // encode, allow more general folding, or match specialized instruction
2301 // patterns (eg, 'blsr' on x86). Don't bother changing 1 to -1 because that
2302 // is probably not useful (and could be detrimental).
2303 ConstantSDNode *C = isConstOrConstSplat(Op1);
2304 APInt HighMask = APInt::getHighBitsSet(BitWidth, DemandedBitsLZ);
2305 if (C && !C->isAllOnes() && !C->isOne() &&
2306 (C->getAPIntValue() | HighMask).isAllOnes()) {
2307 SDValue Neg1 = TLO.DAG.getAllOnesConstant(dl, VT);
2308 // Disable the nsw and nuw flags. We can no longer guarantee that we
2309 // won't wrap after simplification.
2310 Flags.setNoSignedWrap(false);
2311 Flags.setNoUnsignedWrap(false);
2312 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Neg1, Flags);
2313 return TLO.CombineTo(Op, NewOp);
2314 }
2315
2316 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2317 }
2318 default:
2319 if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
2320 if (SimplifyDemandedBitsForTargetNode(Op, DemandedBits, DemandedElts,
2321 Known, TLO, Depth))
2322 return true;
2323 break;
2324 }
2325
2326 // Just use computeKnownBits to compute output bits.
2327 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2328 break;
2329 }
2330
2331 // If we know the value of all of the demanded bits, return this as a
2332 // constant.
2333 if (DemandedBits.isSubsetOf(Known.Zero | Known.One)) {
2334 // Avoid folding to a constant if any OpaqueConstant is involved.
2335 const SDNode *N = Op.getNode();
2336 for (SDNode *Op :
2337 llvm::make_range(SDNodeIterator::begin(N), SDNodeIterator::end(N))) {
2338 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op))
2339 if (C->isOpaque())
2340 return false;
2341 }
2342 if (VT.isInteger())
2343 return TLO.CombineTo(Op, TLO.DAG.getConstant(Known.One, dl, VT));
2344 if (VT.isFloatingPoint())
2345 return TLO.CombineTo(
2346 Op,
2347 TLO.DAG.getConstantFP(
2348 APFloat(TLO.DAG.EVTToAPFloatSemantics(VT), Known.One), dl, VT));
2349 }
2350
2351 return false;
2352}
2353
2354bool TargetLowering::SimplifyDemandedVectorElts(SDValue Op,
2355 const APInt &DemandedElts,
2356 APInt &KnownUndef,
2357 APInt &KnownZero,
2358 DAGCombinerInfo &DCI) const {
2359 SelectionDAG &DAG = DCI.DAG;
2360 TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
2361 !DCI.isBeforeLegalizeOps());
2362
2363 bool Simplified =
2364 SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero, TLO);
2365 if (Simplified) {
2366 DCI.AddToWorklist(Op.getNode());
2367 DCI.CommitTargetLoweringOpt(TLO);
2368 }
2369
2370 return Simplified;
2371}
2372
2373/// Given a vector binary operation and known undefined elements for each input
2374/// operand, compute whether each element of the output is undefined.
2375static APInt getKnownUndefForVectorBinop(SDValue BO, SelectionDAG &DAG,
2376 const APInt &UndefOp0,
2377 const APInt &UndefOp1) {
2378 EVT VT = BO.getValueType();
2379 assert(DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() &&(static_cast <bool> (DAG.getTargetLoweringInfo().isBinOp
(BO.getOpcode()) && VT.isVector() && "Vector binop only"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() && \"Vector binop only\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 2380, __extension__
__PRETTY_FUNCTION__))
2380 "Vector binop only")(static_cast <bool> (DAG.getTargetLoweringInfo().isBinOp
(BO.getOpcode()) && VT.isVector() && "Vector binop only"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() && \"Vector binop only\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 2380, __extension__
__PRETTY_FUNCTION__))
;
2381
2382 EVT EltVT = VT.getVectorElementType();
2383 unsigned NumElts = VT.getVectorNumElements();
2384 assert(UndefOp0.getBitWidth() == NumElts &&(static_cast <bool> (UndefOp0.getBitWidth() == NumElts &&
UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis"
) ? void (0) : __assert_fail ("UndefOp0.getBitWidth() == NumElts && UndefOp1.getBitWidth() == NumElts && \"Bad type for undef analysis\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 2385, __extension__
__PRETTY_FUNCTION__))
2385 UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis")(static_cast <bool> (UndefOp0.getBitWidth() == NumElts &&
UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis"
) ? void (0) : __assert_fail ("UndefOp0.getBitWidth() == NumElts && UndefOp1.getBitWidth() == NumElts && \"Bad type for undef analysis\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 2385, __extension__
__PRETTY_FUNCTION__))
;
2386
2387 auto getUndefOrConstantElt = [&](SDValue V, unsigned Index,
2388 const APInt &UndefVals) {
2389 if (UndefVals[Index])
2390 return DAG.getUNDEF(EltVT);
2391
2392 if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
2393 // Try hard to make sure that the getNode() call is not creating temporary
2394 // nodes. Ignore opaque integers because they do not constant fold.
2395 SDValue Elt = BV->getOperand(Index);
2396 auto *C = dyn_cast<ConstantSDNode>(Elt);
2397 if (isa<ConstantFPSDNode>(Elt) || Elt.isUndef() || (C && !C->isOpaque()))
2398 return Elt;
2399 }
2400
2401 return SDValue();
2402 };
2403
2404 APInt KnownUndef = APInt::getZero(NumElts);
2405 for (unsigned i = 0; i != NumElts; ++i) {
2406 // If both inputs for this element are either constant or undef and match
2407 // the element type, compute the constant/undef result for this element of
2408 // the vector.
2409 // TODO: Ideally we would use FoldConstantArithmetic() here, but that does
2410 // not handle FP constants. The code within getNode() should be refactored
2411 // to avoid the danger of creating a bogus temporary node here.
2412 SDValue C0 = getUndefOrConstantElt(BO.getOperand(0), i, UndefOp0);
2413 SDValue C1 = getUndefOrConstantElt(BO.getOperand(1), i, UndefOp1);
2414 if (C0 && C1 && C0.getValueType() == EltVT && C1.getValueType() == EltVT)
2415 if (DAG.getNode(BO.getOpcode(), SDLoc(BO), EltVT, C0, C1).isUndef())
2416 KnownUndef.setBit(i);
2417 }
2418 return KnownUndef;
2419}
2420
2421bool TargetLowering::SimplifyDemandedVectorElts(
2422 SDValue Op, const APInt &OriginalDemandedElts, APInt &KnownUndef,
2423 APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth,
2424 bool AssumeSingleUse) const {
2425 EVT VT = Op.getValueType();
2426 unsigned Opcode = Op.getOpcode();
2427 APInt DemandedElts = OriginalDemandedElts;
2428 unsigned NumElts = DemandedElts.getBitWidth();
2429 assert(VT.isVector() && "Expected vector op")(static_cast <bool> (VT.isVector() && "Expected vector op"
) ? void (0) : __assert_fail ("VT.isVector() && \"Expected vector op\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 2429, __extension__
__PRETTY_FUNCTION__))
;
2430
2431 KnownUndef = KnownZero = APInt::getZero(NumElts);
2432
2433 // TODO: For now we assume we know nothing about scalable vectors.
2434 if (VT.isScalableVector())
2435 return false;
2436
2437 assert(VT.getVectorNumElements() == NumElts &&(static_cast <bool> (VT.getVectorNumElements() == NumElts
&& "Mask size mismatches value type element count!")
? void (0) : __assert_fail ("VT.getVectorNumElements() == NumElts && \"Mask size mismatches value type element count!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 2438, __extension__
__PRETTY_FUNCTION__))
2438 "Mask size mismatches value type element count!")(static_cast <bool> (VT.getVectorNumElements() == NumElts
&& "Mask size mismatches value type element count!")
? void (0) : __assert_fail ("VT.getVectorNumElements() == NumElts && \"Mask size mismatches value type element count!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 2438, __extension__
__PRETTY_FUNCTION__))
;
2439
2440 // Undef operand.
2441 if (Op.isUndef()) {
2442 KnownUndef.setAllBits();
2443 return false;
2444 }
2445
2446 // If Op has other users, assume that all elements are needed.
2447 if (!Op.getNode()->hasOneUse() && !AssumeSingleUse)
2448 DemandedElts.setAllBits();
2449
2450 // Not demanding any elements from Op.
2451 if (DemandedElts == 0) {
2452 KnownUndef.setAllBits();
2453 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
2454 }
2455
2456 // Limit search depth.
2457 if (Depth >= SelectionDAG::MaxRecursionDepth)
2458 return false;
2459
2460 SDLoc DL(Op);
2461 unsigned EltSizeInBits = VT.getScalarSizeInBits();
2462 bool IsLE = TLO.DAG.getDataLayout().isLittleEndian();
2463
2464 // Helper for demanding the specified elements and all the bits of both binary
2465 // operands.
2466 auto SimplifyDemandedVectorEltsBinOp = [&](SDValue Op0, SDValue Op1) {
2467 SDValue NewOp0 = SimplifyMultipleUseDemandedVectorElts(Op0, DemandedElts,
2468 TLO.DAG, Depth + 1);
2469 SDValue NewOp1 = SimplifyMultipleUseDemandedVectorElts(Op1, DemandedElts,
2470 TLO.DAG, Depth + 1);
2471 if (NewOp0 || NewOp1) {
2472 SDValue NewOp = TLO.DAG.getNode(
2473 Opcode, SDLoc(Op), VT, NewOp0 ? NewOp0 : Op0, NewOp1 ? NewOp1 : Op1);
2474 return TLO.CombineTo(Op, NewOp);
2475 }
2476 return false;
2477 };
2478
2479 switch (Opcode) {
2480 case ISD::SCALAR_TO_VECTOR: {
2481 if (!DemandedElts[0]) {
2482 KnownUndef.setAllBits();
2483 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
2484 }
2485 SDValue ScalarSrc = Op.getOperand(0);
2486 if (ScalarSrc.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
2487 SDValue Src = ScalarSrc.getOperand(0);
2488 SDValue Idx = ScalarSrc.getOperand(1);
2489 EVT SrcVT = Src.getValueType();
2490
2491 ElementCount SrcEltCnt = SrcVT.getVectorElementCount();
2492
2493 if (SrcEltCnt.isScalable())
2494 return false;
2495
2496 unsigned NumSrcElts = SrcEltCnt.getFixedValue();
2497 if (isNullConstant(Idx)) {
2498 APInt SrcDemandedElts = APInt::getOneBitSet(NumSrcElts, 0);
2499 APInt SrcUndef = KnownUndef.zextOrTrunc(NumSrcElts);
2500 APInt SrcZero = KnownZero.zextOrTrunc(NumSrcElts);
2501 if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2502 TLO, Depth + 1))
2503 return true;
2504 }
2505 }
2506 KnownUndef.setHighBits(NumElts - 1);
2507 break;
2508 }
2509 case ISD::BITCAST: {
2510 SDValue Src = Op.getOperand(0);
2511 EVT SrcVT = Src.getValueType();
2512
2513 // We only handle vectors here.
2514 // TODO - investigate calling SimplifyDemandedBits/ComputeKnownBits?
2515 if (!SrcVT.isVector())
2516 break;
2517
2518 // Fast handling of 'identity' bitcasts.
2519 unsigned NumSrcElts = SrcVT.getVectorNumElements();
2520 if (NumSrcElts == NumElts)
2521 return SimplifyDemandedVectorElts(Src, DemandedElts, KnownUndef,
2522 KnownZero, TLO, Depth + 1);
2523
2524 APInt SrcDemandedElts, SrcZero, SrcUndef;
2525
2526 // Bitcast from 'large element' src vector to 'small element' vector, we
2527 // must demand a source element if any DemandedElt maps to it.
2528 if ((NumElts % NumSrcElts) == 0) {
2529 unsigned Scale = NumElts / NumSrcElts;
2530 SrcDemandedElts = APIntOps::ScaleBitMask(DemandedElts, NumSrcElts);
2531 if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2532 TLO, Depth + 1))
2533 return true;
2534
2535 // Try calling SimplifyDemandedBits, converting demanded elts to the bits
2536 // of the large element.
2537 // TODO - bigendian once we have test coverage.
2538 if (IsLE) {
2539 unsigned SrcEltSizeInBits = SrcVT.getScalarSizeInBits();
2540 APInt SrcDemandedBits = APInt::getZero(SrcEltSizeInBits);
2541 for (unsigned i = 0; i != NumElts; ++i)
2542 if (DemandedElts[i]) {
2543 unsigned Ofs = (i % Scale) * EltSizeInBits;
2544 SrcDemandedBits.setBits(Ofs, Ofs + EltSizeInBits);
2545 }
2546
2547 KnownBits Known;
2548 if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcDemandedElts, Known,
2549 TLO, Depth + 1))
2550 return true;
2551 }
2552
2553 // If the src element is zero/undef then all the output elements will be -
2554 // only demanded elements are guaranteed to be correct.
2555 for (unsigned i = 0; i != NumSrcElts; ++i) {
2556 if (SrcDemandedElts[i]) {
2557 if (SrcZero[i])
2558 KnownZero.setBits(i * Scale, (i + 1) * Scale);
2559 if (SrcUndef[i])
2560 KnownUndef.setBits(i * Scale, (i + 1) * Scale);
2561 }
2562 }
2563 }
2564
2565 // Bitcast from 'small element' src vector to 'large element' vector, we
2566 // demand all smaller source elements covered by the larger demanded element
2567 // of this vector.
2568 if ((NumSrcElts % NumElts) == 0) {
2569 unsigned Scale = NumSrcElts / NumElts;
2570 SrcDemandedElts = APIntOps::ScaleBitMask(DemandedElts, NumSrcElts);
2571 if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2572 TLO, Depth + 1))
2573 return true;
2574
2575 // If all the src elements covering an output element are zero/undef, then
2576 // the output element will be as well, assuming it was demanded.
2577 for (unsigned i = 0; i != NumElts; ++i) {
2578 if (DemandedElts[i]) {
2579 if (SrcZero.extractBits(Scale, i * Scale).isAllOnes())
2580 KnownZero.setBit(i);
2581 if (SrcUndef.extractBits(Scale, i * Scale).isAllOnes())
2582 KnownUndef.setBit(i);
2583 }
2584 }
2585 }
2586 break;
2587 }
2588 case ISD::BUILD_VECTOR: {
2589 // Check all elements and simplify any unused elements with UNDEF.
2590 if (!DemandedElts.isAllOnes()) {
2591 // Don't simplify BROADCASTS.
2592 if (llvm::any_of(Op->op_values(),
2593 [&](SDValue Elt) { return Op.getOperand(0) != Elt; })) {
2594 SmallVector<SDValue, 32> Ops(Op->op_begin(), Op->op_end());
2595 bool Updated = false;
2596 for (unsigned i = 0; i != NumElts; ++i) {
2597 if (!DemandedElts[i] && !Ops[i].isUndef()) {
2598 Ops[i] = TLO.DAG.getUNDEF(Ops[0].getValueType());
2599 KnownUndef.setBit(i);
2600 Updated = true;
2601 }
2602 }
2603 if (Updated)
2604 return TLO.CombineTo(Op, TLO.DAG.getBuildVector(VT, DL, Ops));
2605 }
2606 }
2607 for (unsigned i = 0; i != NumElts; ++i) {
2608 SDValue SrcOp = Op.getOperand(i);
2609 if (SrcOp.isUndef()) {
2610 KnownUndef.setBit(i);
2611 } else if (EltSizeInBits == SrcOp.getScalarValueSizeInBits() &&
2612 (isNullConstant(SrcOp) || isNullFPConstant(SrcOp))) {
2613 KnownZero.setBit(i);
2614 }
2615 }
2616 break;
2617 }
2618 case ISD::CONCAT_VECTORS: {
2619 EVT SubVT = Op.getOperand(0).getValueType();
2620 unsigned NumSubVecs = Op.getNumOperands();
2621 unsigned NumSubElts = SubVT.getVectorNumElements();
2622 for (unsigned i = 0; i != NumSubVecs; ++i) {
2623 SDValue SubOp = Op.getOperand(i);
2624 APInt SubElts = DemandedElts.extractBits(NumSubElts, i * NumSubElts);
2625 APInt SubUndef, SubZero;
2626 if (SimplifyDemandedVectorElts(SubOp, SubElts, SubUndef, SubZero, TLO,
2627 Depth + 1))
2628 return true;
2629 KnownUndef.insertBits(SubUndef, i * NumSubElts);
2630 KnownZero.insertBits(SubZero, i * NumSubElts);
2631 }
2632 break;
2633 }
2634 case ISD::INSERT_SUBVECTOR: {
2635 // Demand any elements from the subvector and the remainder from the src its
2636 // inserted into.
2637 SDValue Src = Op.getOperand(0);
2638 SDValue Sub = Op.getOperand(1);
2639 uint64_t Idx = Op.getConstantOperandVal(2);
2640 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
2641 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
2642 APInt DemandedSrcElts = DemandedElts;
2643 DemandedSrcElts.insertBits(APInt::getZero(NumSubElts), Idx);
2644
2645 APInt SubUndef, SubZero;
2646 if (SimplifyDemandedVectorElts(Sub, DemandedSubElts, SubUndef, SubZero, TLO,
2647 Depth + 1))
2648 return true;
2649
2650 // If none of the src operand elements are demanded, replace it with undef.
2651 if (!DemandedSrcElts && !Src.isUndef())
2652 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
2653 TLO.DAG.getUNDEF(VT), Sub,
2654 Op.getOperand(2)));
2655
2656 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownUndef, KnownZero,
2657 TLO, Depth + 1))
2658 return true;
2659 KnownUndef.insertBits(SubUndef, Idx);
2660 KnownZero.insertBits(SubZero, Idx);
2661
2662 // Attempt to avoid multi-use ops if we don't need anything from them.
2663 if (!DemandedSrcElts.isAllOnes() || !DemandedSubElts.isAllOnes()) {
2664 SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
2665 Src, DemandedSrcElts, TLO.DAG, Depth + 1);
2666 SDValue NewSub = SimplifyMultipleUseDemandedVectorElts(
2667 Sub, DemandedSubElts, TLO.DAG, Depth + 1);
2668 if (NewSrc || NewSub) {
2669 NewSrc = NewSrc ? NewSrc : Src;
2670 NewSub = NewSub ? NewSub : Sub;
2671 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,
2672 NewSub, Op.getOperand(2));
2673 return TLO.CombineTo(Op, NewOp);
2674 }
2675 }
2676 break;
2677 }
2678 case ISD::EXTRACT_SUBVECTOR: {
2679 // Offset the demanded elts by the subvector index.
2680 SDValue Src = Op.getOperand(0);
2681 if (Src.getValueType().isScalableVector())
2682 break;
2683 uint64_t Idx = Op.getConstantOperandVal(1);
2684 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2685 APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
2686
2687 APInt SrcUndef, SrcZero;
2688 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
2689 Depth + 1))
2690 return true;
2691 KnownUndef = SrcUndef.extractBits(NumElts, Idx);
2692 KnownZero = SrcZero.extractBits(NumElts, Idx);
2693
2694 // Attempt to avoid multi-use ops if we don't need anything from them.
2695 if (!DemandedElts.isAllOnes()) {
2696 SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
2697 Src, DemandedSrcElts, TLO.DAG, Depth + 1);
2698 if (NewSrc) {
2699 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,
2700 Op.getOperand(1));
2701 return TLO.CombineTo(Op, NewOp);
2702 }
2703 }
2704 break;
2705 }
2706 case ISD::INSERT_VECTOR_ELT: {
2707 SDValue Vec = Op.getOperand(0);
2708 SDValue Scl = Op.getOperand(1);
2709 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
2710
2711 // For a legal, constant insertion index, if we don't need this insertion
2712 // then strip it, else remove it from the demanded elts.
2713 if (CIdx && CIdx->getAPIntValue().ult(NumElts)) {
2714 unsigned Idx = CIdx->getZExtValue();
2715 if (!DemandedElts[Idx])
2716 return TLO.CombineTo(Op, Vec);
2717
2718 APInt DemandedVecElts(DemandedElts);
2719 DemandedVecElts.clearBit(Idx);
2720 if (SimplifyDemandedVectorElts(Vec, DemandedVecElts, KnownUndef,
2721 KnownZero, TLO, Depth + 1))
2722 return true;
2723
2724 KnownUndef.setBitVal(Idx, Scl.isUndef());
2725
2726 KnownZero.setBitVal(Idx, isNullConstant(Scl) || isNullFPConstant(Scl));
2727 break;
2728 }
2729
2730 APInt VecUndef, VecZero;
2731 if (SimplifyDemandedVectorElts(Vec, DemandedElts, VecUndef, VecZero, TLO,
2732 Depth + 1))
2733 return true;
2734 // Without knowing the insertion index we can't set KnownUndef/KnownZero.
2735 break;
2736 }
2737 case ISD::VSELECT: {
2738 // Try to transform the select condition based on the current demanded
2739 // elements.
2740 // TODO: If a condition element is undef, we can choose from one arm of the
2741 // select (and if one arm is undef, then we can propagate that to the
2742 // result).
2743 // TODO - add support for constant vselect masks (see IR version of this).
2744 APInt UnusedUndef, UnusedZero;
2745 if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, UnusedUndef,
2746 UnusedZero, TLO, Depth + 1))
2747 return true;
2748
2749 // See if we can simplify either vselect operand.
2750 APInt DemandedLHS(DemandedElts);
2751 APInt DemandedRHS(DemandedElts);
2752 APInt UndefLHS, ZeroLHS;
2753 APInt UndefRHS, ZeroRHS;
2754 if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedLHS, UndefLHS,
2755 ZeroLHS, TLO, Depth + 1))
2756 return true;
2757 if (SimplifyDemandedVectorElts(Op.getOperand(2), DemandedRHS, UndefRHS,
2758 ZeroRHS, TLO, Depth + 1))
2759 return true;
2760
2761 KnownUndef = UndefLHS & UndefRHS;
2762 KnownZero = ZeroLHS & ZeroRHS;
2763 break;
2764 }
2765 case ISD::VECTOR_SHUFFLE: {
2766 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
2767
2768 // Collect demanded elements from shuffle operands..
2769 APInt DemandedLHS(NumElts, 0);
2770 APInt DemandedRHS(NumElts, 0);
2771 for (unsigned i = 0; i != NumElts; ++i) {
2772 int M = ShuffleMask[i];
2773 if (M < 0 || !DemandedElts[i])
2774 continue;
2775 assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range")(static_cast <bool> (0 <= M && M < (int)(
2 * NumElts) && "Shuffle index out of range") ? void (
0) : __assert_fail ("0 <= M && M < (int)(2 * NumElts) && \"Shuffle index out of range\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 2775, __extension__
__PRETTY_FUNCTION__))
;
2776 if (M < (int)NumElts)
2777 DemandedLHS.setBit(M);
2778 else
2779 DemandedRHS.setBit(M - NumElts);
2780 }
2781
2782 // See if we can simplify either shuffle operand.
2783 APInt UndefLHS, ZeroLHS;
2784 APInt UndefRHS, ZeroRHS;
2785 if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedLHS, UndefLHS,
2786 ZeroLHS, TLO, Depth + 1))
2787 return true;
2788 if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedRHS, UndefRHS,
2789 ZeroRHS, TLO, Depth + 1))
2790 return true;
2791
2792 // Simplify mask using undef elements from LHS/RHS.
2793 bool Updated = false;
2794 bool IdentityLHS = true, IdentityRHS = true;
2795 SmallVector<int, 32> NewMask(ShuffleMask.begin(), ShuffleMask.end());
2796 for (unsigned i = 0; i != NumElts; ++i) {
2797 int &M = NewMask[i];
2798 if (M < 0)
2799 continue;
2800 if (!DemandedElts[i] || (M < (int)NumElts && UndefLHS[M]) ||
2801 (M >= (int)NumElts && UndefRHS[M - NumElts])) {
2802 Updated = true;
2803 M = -1;
2804 }
2805 IdentityLHS &= (M < 0) || (M == (int)i);
2806 IdentityRHS &= (M < 0) || ((M - NumElts) == i);
2807 }
2808
2809 // Update legal shuffle masks based on demanded elements if it won't reduce
2810 // to Identity which can cause premature removal of the shuffle mask.
2811 if (Updated && !IdentityLHS && !IdentityRHS && !TLO.LegalOps) {
2812 SDValue LegalShuffle =
2813 buildLegalVectorShuffle(VT, DL, Op.getOperand(0), Op.getOperand(1),
2814 NewMask, TLO.DAG);
2815 if (LegalShuffle)
2816 return TLO.CombineTo(Op, LegalShuffle);
2817 }
2818
2819 // Propagate undef/zero elements from LHS/RHS.
2820 for (unsigned i = 0; i != NumElts; ++i) {
2821 int M = ShuffleMask[i];
2822 if (M < 0) {
2823 KnownUndef.setBit(i);
2824 } else if (M < (int)NumElts) {
2825 if (UndefLHS[M])
2826 KnownUndef.setBit(i);
2827 if (ZeroLHS[M])
2828 KnownZero.setBit(i);
2829 } else {
2830 if (UndefRHS[M - NumElts])
2831 KnownUndef.setBit(i);
2832 if (ZeroRHS[M - NumElts])
2833 KnownZero.setBit(i);
2834 }
2835 }
2836 break;
2837 }
2838 case ISD::ANY_EXTEND_VECTOR_INREG:
2839 case ISD::SIGN_EXTEND_VECTOR_INREG:
2840 case ISD::ZERO_EXTEND_VECTOR_INREG: {
2841 APInt SrcUndef, SrcZero;
2842 SDValue Src = Op.getOperand(0);
2843 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2844 APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts);
2845 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
2846 Depth + 1))
2847 return true;
2848 KnownZero = SrcZero.zextOrTrunc(NumElts);
2849 KnownUndef = SrcUndef.zextOrTrunc(NumElts);
2850
2851 if (IsLE && Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG &&
2852 Op.getValueSizeInBits() == Src.getValueSizeInBits() &&
2853 DemandedSrcElts == 1) {
2854 // aext - if we just need the bottom element then we can bitcast.
2855 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
2856 }
2857
2858 if (Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) {
2859 // zext(undef) upper bits are guaranteed to be zero.
2860 if (DemandedElts.isSubsetOf(KnownUndef))
2861 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
2862 KnownUndef.clearAllBits();
2863
2864 // zext - if we just need the bottom element then we can mask:
2865 // zext(and(x,c)) -> and(x,c') iff the zext is the only user of the and.
2866 if (IsLE && DemandedSrcElts == 1 && Src.getOpcode() == ISD::AND &&
2867 Op->isOnlyUserOf(Src.getNode()) &&
2868 Op.getValueSizeInBits() == Src.getValueSizeInBits()) {
2869 SDLoc DL(Op);
2870 EVT SrcVT = Src.getValueType();
2871 EVT SrcSVT = SrcVT.getScalarType();
2872 SmallVector<SDValue> MaskElts;
2873 MaskElts.push_back(TLO.DAG.getAllOnesConstant(DL, SrcSVT));
2874 MaskElts.append(NumSrcElts - 1, TLO.DAG.getConstant(0, DL, SrcSVT));
2875 SDValue Mask = TLO.DAG.getBuildVector(SrcVT, DL, MaskElts);
2876 if (SDValue Fold = TLO.DAG.FoldConstantArithmetic(
2877 ISD::AND, DL, SrcVT, {Src.getOperand(1), Mask})) {
2878 Fold = TLO.DAG.getNode(ISD::AND, DL, SrcVT, Src.getOperand(0), Fold);
2879 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Fold));
2880 }
2881 }
2882 }
2883 break;
2884 }
2885
2886 // TODO: There are more binop opcodes that could be handled here - MIN,
2887 // MAX, saturated math, etc.
2888 case ISD::ADD: {
2889 SDValue Op0 = Op.getOperand(0);
2890 SDValue Op1 = Op.getOperand(1);
2891 if (Op0 == Op1 && Op->isOnlyUserOf(Op0.getNode())) {
2892 APInt UndefLHS, ZeroLHS;
2893 if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
2894 Depth + 1, /*AssumeSingleUse*/ true))
2895 return true;
2896 }
2897 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2898 }
2899 case ISD::OR:
2900 case ISD::XOR:
2901 case ISD::SUB:
2902 case ISD::FADD:
2903 case ISD::FSUB:
2904 case ISD::FMUL:
2905 case ISD::FDIV:
2906 case ISD::FREM: {
2907 SDValue Op0 = Op.getOperand(0);
2908 SDValue Op1 = Op.getOperand(1);
2909
2910 APInt UndefRHS, ZeroRHS;
2911 if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,
2912 Depth + 1))
2913 return true;
2914 APInt UndefLHS, ZeroLHS;
2915 if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
2916 Depth + 1))
2917 return true;
2918
2919 KnownZero = ZeroLHS & ZeroRHS;
2920 KnownUndef = getKnownUndefForVectorBinop(Op, TLO.DAG, UndefLHS, UndefRHS);
2921
2922 // Attempt to avoid multi-use ops if we don't need anything from them.
2923 // TODO - use KnownUndef to relax the demandedelts?
2924 if (!DemandedElts.isAllOnes())
2925 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
2926 return true;
2927 break;
2928 }
2929 case ISD::SHL:
2930 case ISD::SRL:
2931 case ISD::SRA:
2932 case ISD::ROTL:
2933 case ISD::ROTR: {
2934 SDValue Op0 = Op.getOperand(0);
2935 SDValue Op1 = Op.getOperand(1);
2936
2937 APInt UndefRHS, ZeroRHS;
2938 if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,
2939 Depth + 1))
2940 return true;
2941 APInt UndefLHS, ZeroLHS;
2942 if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
2943 Depth + 1))
2944 return true;
2945
2946 KnownZero = ZeroLHS;
2947 KnownUndef = UndefLHS & UndefRHS; // TODO: use getKnownUndefForVectorBinop?
2948
2949 // Attempt to avoid multi-use ops if we don't need anything from them.
2950 // TODO - use KnownUndef to relax the demandedelts?
2951 if (!DemandedElts.isAllOnes())
2952 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
2953 return true;
2954 break;
2955 }
2956 case ISD::MUL:
2957 case ISD::AND: {
2958 SDValue Op0 = Op.getOperand(0);
2959 SDValue Op1 = Op.getOperand(1);
2960
2961 APInt SrcUndef, SrcZero;
2962 if (SimplifyDemandedVectorElts(Op1, DemandedElts, SrcUndef, SrcZero, TLO,
2963 Depth + 1))
2964 return true;
2965 if (SimplifyDemandedVectorElts(Op0, DemandedElts, KnownUndef, KnownZero,
2966 TLO, Depth + 1))
2967 return true;
2968
2969 // If either side has a zero element, then the result element is zero, even
2970 // if the other is an UNDEF.
2971 // TODO: Extend getKnownUndefForVectorBinop to also deal with known zeros
2972 // and then handle 'and' nodes with the rest of the binop opcodes.
2973 KnownZero |= SrcZero;
2974 KnownUndef &= SrcUndef;
2975 KnownUndef &= ~KnownZero;
2976
2977 // Attempt to avoid multi-use ops if we don't need anything from them.
2978 // TODO - use KnownUndef to relax the demandedelts?
2979 if (!DemandedElts.isAllOnes())
2980 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
2981 return true;
2982 break;
2983 }
2984 case ISD::TRUNCATE:
2985 case ISD::SIGN_EXTEND:
2986 case ISD::ZERO_EXTEND:
2987 if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,
2988 KnownZero, TLO, Depth + 1))
2989 return true;
2990
2991 if (Op.getOpcode() == ISD::ZERO_EXTEND) {
2992 // zext(undef) upper bits are guaranteed to be zero.
2993 if (DemandedElts.isSubsetOf(KnownUndef))
2994 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
2995 KnownUndef.clearAllBits();
2996 }
2997 break;
2998 default: {
2999 if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
3000 if (SimplifyDemandedVectorEltsForTargetNode(Op, DemandedElts, KnownUndef,
3001 KnownZero, TLO, Depth))
3002 return true;
3003 } else {
3004 KnownBits Known;
3005 APInt DemandedBits = APInt::getAllOnes(EltSizeInBits);
3006 if (SimplifyDemandedBits(Op, DemandedBits, OriginalDemandedElts, Known,
3007 TLO, Depth, AssumeSingleUse))
3008 return true;
3009 }
3010 break;
3011 }
3012 }
3013 assert((KnownUndef & KnownZero) == 0 && "Elements flagged as undef AND zero")(static_cast <bool> ((KnownUndef & KnownZero) == 0 &&
"Elements flagged as undef AND zero") ? void (0) : __assert_fail
("(KnownUndef & KnownZero) == 0 && \"Elements flagged as undef AND zero\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3013, __extension__
__PRETTY_FUNCTION__))
;
3014
3015 // Constant fold all undef cases.
3016 // TODO: Handle zero cases as well.
3017 if (DemandedElts.isSubsetOf(KnownUndef))
3018 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
3019
3020 return false;
3021}
3022
3023/// Determine which of the bits specified in Mask are known to be either zero or
3024/// one and return them in the Known.
3025void TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
3026 KnownBits &Known,
3027 const APInt &DemandedElts,
3028 const SelectionDAG &DAG,
3029 unsigned Depth) const {
3030 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3035, __extension__
__PRETTY_FUNCTION__))
3031 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3035, __extension__
__PRETTY_FUNCTION__))
3032 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3035, __extension__
__PRETTY_FUNCTION__))
3033 Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3035, __extension__
__PRETTY_FUNCTION__))
3034 "Should use MaskedValueIsZero if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3035, __extension__
__PRETTY_FUNCTION__))
3035 " is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3035, __extension__
__PRETTY_FUNCTION__))
;
3036 Known.resetAll();
3037}
3038
3039void TargetLowering::computeKnownBitsForTargetInstr(
3040 GISelKnownBits &Analysis, Register R, KnownBits &Known,
3041 const APInt &DemandedElts, const MachineRegisterInfo &MRI,
3042 unsigned Depth) const {
3043 Known.resetAll();
3044}
3045
3046void TargetLowering::computeKnownBitsForFrameIndex(
3047 const int FrameIdx, KnownBits &Known, const MachineFunction &MF) const {
3048 // The low bits are known zero if the pointer is aligned.
3049 Known.Zero.setLowBits(Log2(MF.getFrameInfo().getObjectAlign(FrameIdx)));
3050}
3051
3052Align TargetLowering::computeKnownAlignForTargetInstr(
3053 GISelKnownBits &Analysis, Register R, const MachineRegisterInfo &MRI,
3054 unsigned Depth) const {
3055 return Align(1);
3056}
3057
3058/// This method can be implemented by targets that want to expose additional
3059/// information about sign bits to the DAG Combiner.
3060unsigned TargetLowering::ComputeNumSignBitsForTargetNode(SDValue Op,
3061 const APInt &,
3062 const SelectionDAG &,
3063 unsigned Depth) const {
3064 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3069, __extension__
__PRETTY_FUNCTION__))
3065 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3069, __extension__
__PRETTY_FUNCTION__))
3066 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3069, __extension__
__PRETTY_FUNCTION__))
3067 Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3069, __extension__
__PRETTY_FUNCTION__))
3068 "Should use ComputeNumSignBits if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3069, __extension__
__PRETTY_FUNCTION__))
3069 " is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3069, __extension__
__PRETTY_FUNCTION__))
;
3070 return 1;
3071}
3072
3073unsigned TargetLowering::computeNumSignBitsForTargetInstr(
3074 GISelKnownBits &Analysis, Register R, const APInt &DemandedElts,
3075 const MachineRegisterInfo &MRI, unsigned Depth) const {
3076 return 1;
3077}
3078
3079bool TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
3080 SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,
3081 TargetLoweringOpt &TLO, unsigned Depth) const {
3082 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3087, __extension__
__PRETTY_FUNCTION__))
3083 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3087, __extension__
__PRETTY_FUNCTION__))
3084 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3087, __extension__
__PRETTY_FUNCTION__))
3085 Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3087, __extension__
__PRETTY_FUNCTION__))
3086 "Should use SimplifyDemandedVectorElts if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3087, __extension__
__PRETTY_FUNCTION__))
3087 " is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3087, __extension__
__PRETTY_FUNCTION__))
;
3088 return false;
3089}
3090
3091bool TargetLowering::SimplifyDemandedBitsForTargetNode(
3092 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
3093 KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const {
3094 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3099, __extension__
__PRETTY_FUNCTION__))
3095 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3099, __extension__
__PRETTY_FUNCTION__))
3096 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3099, __extension__
__PRETTY_FUNCTION__))
3097 Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3099, __extension__
__PRETTY_FUNCTION__))
3098 "Should use SimplifyDemandedBits if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3099, __extension__
__PRETTY_FUNCTION__))
3099 " is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3099, __extension__
__PRETTY_FUNCTION__))
;
3100 computeKnownBitsForTargetNode(Op, Known, DemandedElts, TLO.DAG, Depth);
3101 return false;
3102}
3103
3104SDValue TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(
3105 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
3106 SelectionDAG &DAG, unsigned Depth) const {
3107 assert((static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3113, __extension__
__PRETTY_FUNCTION__))
3108 (Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3113, __extension__
__PRETTY_FUNCTION__))
3109 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3113, __extension__
__PRETTY_FUNCTION__))
3110 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3113, __extension__
__PRETTY_FUNCTION__))
3111 Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3113, __extension__
__PRETTY_FUNCTION__))
3112 "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3113, __extension__
__PRETTY_FUNCTION__))
3113 " is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3113, __extension__
__PRETTY_FUNCTION__))
;
3114 return SDValue();
3115}
3116
3117SDValue
3118TargetLowering::buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0,
3119 SDValue N1, MutableArrayRef<int> Mask,
3120 SelectionDAG &DAG) const {
3121 bool LegalMask = isShuffleMaskLegal(Mask, VT);
3122 if (!LegalMask) {
3123 std::swap(N0, N1);
3124 ShuffleVectorSDNode::commuteMask(Mask);
3125 LegalMask = isShuffleMaskLegal(Mask, VT);
3126 }
3127
3128 if (!LegalMask)
3129 return SDValue();
3130
3131 return DAG.getVectorShuffle(VT, DL, N0, N1, Mask);
3132}
3133
3134const Constant *TargetLowering::getTargetConstantFromLoad(LoadSDNode*) const {
3135 return nullptr;
3136}
3137
3138bool TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode(
3139 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
3140 bool PoisonOnly, unsigned Depth) const {
3141 assert((static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3147, __extension__
__PRETTY_FUNCTION__))
3142 (Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3147, __extension__
__PRETTY_FUNCTION__))
3143 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3147, __extension__
__PRETTY_FUNCTION__))
3144 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3147, __extension__
__PRETTY_FUNCTION__))
3145 Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3147, __extension__
__PRETTY_FUNCTION__))
3146 "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3147, __extension__
__PRETTY_FUNCTION__))
3147 " is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3147, __extension__
__PRETTY_FUNCTION__))
;
3148 return false;
3149}
3150
3151bool TargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
3152 const SelectionDAG &DAG,
3153 bool SNaN,
3154 unsigned Depth) const {
3155 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3160, __extension__
__PRETTY_FUNCTION__))
3156 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3160, __extension__
__PRETTY_FUNCTION__))
3157 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3160, __extension__
__PRETTY_FUNCTION__))
3158 Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3160, __extension__
__PRETTY_FUNCTION__))
3159 "Should use isKnownNeverNaN if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3160, __extension__
__PRETTY_FUNCTION__))
3160 " is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3160, __extension__
__PRETTY_FUNCTION__))
;
3161 return false;
3162}
3163
3164bool TargetLowering::isSplatValueForTargetNode(SDValue Op,
3165 const APInt &DemandedElts,
3166 APInt &UndefElts,
3167 unsigned Depth) const {
3168 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isSplatValue if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isSplatValue if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3173, __extension__
__PRETTY_FUNCTION__))
3169 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isSplatValue if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isSplatValue if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3173, __extension__
__PRETTY_FUNCTION__))
3170 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isSplatValue if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isSplatValue if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3173, __extension__
__PRETTY_FUNCTION__))
3171 Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isSplatValue if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isSplatValue if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3173, __extension__
__PRETTY_FUNCTION__))
3172 "Should use isSplatValue if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isSplatValue if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isSplatValue if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3173, __extension__
__PRETTY_FUNCTION__))
3173 " is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isSplatValue if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isSplatValue if you don't know whether Op\" \" is a target node!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3173, __extension__
__PRETTY_FUNCTION__))
;
3174 return false;
3175}
3176
3177// FIXME: Ideally, this would use ISD::isConstantSplatVector(), but that must
3178// work with truncating build vectors and vectors with elements of less than
3179// 8 bits.
3180bool TargetLowering::isConstTrueVal(const SDNode *N) const {
3181 if (!N)
3182 return false;
3183
3184 APInt CVal;
3185 if (auto *CN = dyn_cast<ConstantSDNode>(N)) {
3186 CVal = CN->getAPIntValue();
3187 } else if (auto *BV = dyn_cast<BuildVectorSDNode>(N)) {
3188 auto *CN = BV->getConstantSplatNode();
3189 if (!CN)
3190 return false;
3191
3192 // If this is a truncating build vector, truncate the splat value.
3193 // Otherwise, we may fail to match the expected values below.
3194 unsigned BVEltWidth = BV->getValueType(0).getScalarSizeInBits();
3195 CVal = CN->getAPIntValue();
3196 if (BVEltWidth < CVal.getBitWidth())
3197 CVal = CVal.trunc(BVEltWidth);
3198 } else {
3199 return false;
3200 }
3201
3202 switch (getBooleanContents(N->getValueType(0))) {
3203 case UndefinedBooleanContent:
3204 return CVal[0];
3205 case ZeroOrOneBooleanContent:
3206 return CVal.isOne();
3207 case ZeroOrNegativeOneBooleanContent:
3208 return CVal.isAllOnes();
3209 }
3210
3211 llvm_unreachable("Invalid boolean contents")::llvm::llvm_unreachable_internal("Invalid boolean contents",
"llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3211)
;
3212}
3213
3214bool TargetLowering::isConstFalseVal(const SDNode *N) const {
3215 if (!N)
3216 return false;
3217
3218 const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N);
3219 if (!CN) {
3220 const BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N);
3221 if (!BV)
3222 return false;
3223
3224 // Only interested in constant splats, we don't care about undef
3225 // elements in identifying boolean constants and getConstantSplatNode
3226 // returns NULL if all ops are undef;
3227 CN = BV->getConstantSplatNode();
3228 if (!CN)
3229 return false;
3230 }
3231
3232 if (getBooleanContents(N->getValueType(0)) == UndefinedBooleanContent)
3233 return !CN->getAPIntValue()[0];
3234
3235 return CN->isZero();
3236}
3237
3238bool TargetLowering::isExtendedTrueVal(const ConstantSDNode *N, EVT VT,
3239 bool SExt) const {
3240 if (VT == MVT::i1)
3241 return N->isOne();
3242
3243 TargetLowering::BooleanContent Cnt = getBooleanContents(VT);
3244 switch (Cnt) {
3245 case TargetLowering::ZeroOrOneBooleanContent:
3246 // An extended value of 1 is always true, unless its original type is i1,
3247 // in which case it will be sign extended to -1.
3248 return (N->isOne() && !SExt) || (SExt && (N->getValueType(0) != MVT::i1));
3249 case TargetLowering::UndefinedBooleanContent:
3250 case TargetLowering::ZeroOrNegativeOneBooleanContent:
3251 return N->isAllOnes() && SExt;
3252 }
3253 llvm_unreachable("Unexpected enumeration.")::llvm::llvm_unreachable_internal("Unexpected enumeration.", "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3253)
;
3254}
3255
3256/// This helper function of SimplifySetCC tries to optimize the comparison when
3257/// either operand of the SetCC node is a bitwise-and instruction.
3258SDValue TargetLowering::foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1,
3259 ISD::CondCode Cond, const SDLoc &DL,
3260 DAGCombinerInfo &DCI) const {
3261 // Match these patterns in any of their permutations:
3262 // (X & Y) == Y
3263 // (X & Y) != Y
3264 if (N1.getOpcode() == ISD::AND && N0.getOpcode() != ISD::AND)
3265 std::swap(N0, N1);
3266
3267 EVT OpVT = N0.getValueType();
3268 if (N0.getOpcode() != ISD::AND || !OpVT.isInteger() ||
3269 (Cond != ISD::SETEQ && Cond != ISD::SETNE))
3270 return SDValue();
3271
3272 SDValue X, Y;
3273 if (N0.getOperand(0) == N1) {
3274 X = N0.getOperand(1);
3275 Y = N0.getOperand(0);
3276 } else if (N0.getOperand(1) == N1) {
3277 X = N0.getOperand(0);
3278 Y = N0.getOperand(1);
3279 } else {
3280 return SDValue();
3281 }
3282
3283 SelectionDAG &DAG = DCI.DAG;
3284 SDValue Zero = DAG.getConstant(0, DL, OpVT);
3285 if (DAG.isKnownToBeAPowerOfTwo(Y)) {
3286 // Simplify X & Y == Y to X & Y != 0 if Y has exactly one bit set.
3287 // Note that where Y is variable and is known to have at most one bit set
3288 // (for example, if it is Z & 1) we cannot do this; the expressions are not
3289 // equivalent when Y == 0.
3290 assert(OpVT.isInteger())(static_cast <bool> (OpVT.isInteger()) ? void (0) : __assert_fail
("OpVT.isInteger()", "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3290, __extension__ __PRETTY_FUNCTION__))
;
3291 Cond = ISD::getSetCCInverse(Cond, OpVT);
3292 if (DCI.isBeforeLegalizeOps() ||
3293 isCondCodeLegal(Cond, N0.getSimpleValueType()))
3294 return DAG.getSetCC(DL, VT, N0, Zero, Cond);
3295 } else if (N0.hasOneUse() && hasAndNotCompare(Y)) {
3296 // If the target supports an 'and-not' or 'and-complement' logic operation,
3297 // try to use that to make a comparison operation more efficient.
3298 // But don't do this transform if the mask is a single bit because there are
3299 // more efficient ways to deal with that case (for example, 'bt' on x86 or
3300 // 'rlwinm' on PPC).
3301
3302 // Bail out if the compare operand that we want to turn into a zero is
3303 // already a zero (otherwise, infinite loop).
3304 auto *YConst = dyn_cast<ConstantSDNode>(Y);
3305 if (YConst && YConst->isZero())
3306 return SDValue();
3307
3308 // Transform this into: ~X & Y == 0.
3309 SDValue NotX = DAG.getNOT(SDLoc(X), X, OpVT);
3310 SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, NotX, Y);
3311 return DAG.getSetCC(DL, VT, NewAnd, Zero, Cond);
3312 }
3313
3314 return SDValue();
3315}
3316
3317/// There are multiple IR patterns that could be checking whether certain
3318/// truncation of a signed number would be lossy or not. The pattern which is
3319/// best at IR level, may not lower optimally. Thus, we want to unfold it.
3320/// We are looking for the following pattern: (KeptBits is a constant)
3321/// (add %x, (1 << (KeptBits-1))) srccond (1 << KeptBits)
3322/// KeptBits won't be bitwidth(x), that will be constant-folded to true/false.
3323/// KeptBits also can't be 1, that would have been folded to %x dstcond 0
3324/// We will unfold it into the natural trunc+sext pattern:
3325/// ((%x << C) a>> C) dstcond %x
3326/// Where C = bitwidth(x) - KeptBits and C u< bitwidth(x)
3327SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(
3328 EVT SCCVT, SDValue N0, SDValue N1, ISD::CondCode Cond, DAGCombinerInfo &DCI,
3329 const SDLoc &DL) const {
3330 // We must be comparing with a constant.
3331 ConstantSDNode *C1;
3332 if (!(C1 = dyn_cast<ConstantSDNode>(N1)))
3333 return SDValue();
3334
3335 // N0 should be: add %x, (1 << (KeptBits-1))
3336 if (N0->getOpcode() != ISD::ADD)
3337 return SDValue();
3338
3339 // And we must be 'add'ing a constant.
3340 ConstantSDNode *C01;
3341 if (!(C01 = dyn_cast<ConstantSDNode>(N0->getOperand(1))))
3342 return SDValue();
3343
3344 SDValue X = N0->getOperand(0);
3345 EVT XVT = X.getValueType();
3346
3347 // Validate constants ...
3348
3349 APInt I1 = C1->getAPIntValue();
3350
3351 ISD::CondCode NewCond;
3352 if (Cond == ISD::CondCode::SETULT) {
3353 NewCond = ISD::CondCode::SETEQ;
3354 } else if (Cond == ISD::CondCode::SETULE) {
3355 NewCond = ISD::CondCode::SETEQ;
3356 // But need to 'canonicalize' the constant.
3357 I1 += 1;
3358 } else if (Cond == ISD::CondCode::SETUGT) {
3359 NewCond = ISD::CondCode::SETNE;
3360 // But need to 'canonicalize' the constant.
3361 I1 += 1;
3362 } else if (Cond == ISD::CondCode::SETUGE) {
3363 NewCond = ISD::CondCode::SETNE;
3364 } else
3365 return SDValue();
3366
3367 APInt I01 = C01->getAPIntValue();
3368
3369 auto checkConstants = [&I1, &I01]() -> bool {
3370 // Both of them must be power-of-two, and the constant from setcc is bigger.
3371 return I1.ugt(I01) && I1.isPowerOf2() && I01.isPowerOf2();
3372 };
3373
3374 if (checkConstants()) {
3375 // Great, e.g. got icmp ult i16 (add i16 %x, 128), 256
3376 } else {
3377 // What if we invert constants? (and the target predicate)
3378 I1.negate();
3379 I01.negate();
3380 assert(XVT.isInteger())(static_cast <bool> (XVT.isInteger()) ? void (0) : __assert_fail
("XVT.isInteger()", "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3380, __extension__ __PRETTY_FUNCTION__))
;
3381 NewCond = getSetCCInverse(NewCond, XVT);
3382 if (!checkConstants())
3383 return SDValue();
3384 // Great, e.g. got icmp uge i16 (add i16 %x, -128), -256
3385 }
3386
3387 // They are power-of-two, so which bit is set?
3388 const unsigned KeptBits = I1.logBase2();
3389 const unsigned KeptBitsMinusOne = I01.logBase2();
3390
3391 // Magic!
3392 if (KeptBits != (KeptBitsMinusOne + 1))
3393 return SDValue();
3394 assert(KeptBits > 0 && KeptBits < XVT.getSizeInBits() && "unreachable")(static_cast <bool> (KeptBits > 0 && KeptBits
< XVT.getSizeInBits() && "unreachable") ? void (0
) : __assert_fail ("KeptBits > 0 && KeptBits < XVT.getSizeInBits() && \"unreachable\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3394, __extension__
__PRETTY_FUNCTION__))
;
3395
3396 // We don't want to do this in every single case.
3397 SelectionDAG &DAG = DCI.DAG;
3398 if (!DAG.getTargetLoweringInfo().shouldTransformSignedTruncationCheck(
3399 XVT, KeptBits))
3400 return SDValue();
3401
3402 const unsigned MaskedBits = XVT.getSizeInBits() - KeptBits;
3403 assert(MaskedBits > 0 && MaskedBits < XVT.getSizeInBits() && "unreachable")(static_cast <bool> (MaskedBits > 0 && MaskedBits
< XVT.getSizeInBits() && "unreachable") ? void (0
) : __assert_fail ("MaskedBits > 0 && MaskedBits < XVT.getSizeInBits() && \"unreachable\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3403, __extension__
__PRETTY_FUNCTION__))
;
3404
3405 // Unfold into: ((%x << C) a>> C) cond %x
3406 // Where 'cond' will be either 'eq' or 'ne'.
3407 SDValue ShiftAmt = DAG.getConstant(MaskedBits, DL, XVT);
3408 SDValue T0 = DAG.getNode(ISD::SHL, DL, XVT, X, ShiftAmt);
3409 SDValue T1 = DAG.getNode(ISD::SRA, DL, XVT, T0, ShiftAmt);
3410 SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, X, NewCond);
3411
3412 return T2;
3413}
3414
3415// (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
3416SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(
3417 EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond,
3418 DAGCombinerInfo &DCI, const SDLoc &DL) const {
3419 assert(isConstOrConstSplat(N1C) &&(static_cast <bool> (isConstOrConstSplat(N1C) &&
isConstOrConstSplat(N1C)->getAPIntValue().isZero() &&
"Should be a comparison with 0.") ? void (0) : __assert_fail
("isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->getAPIntValue().isZero() && \"Should be a comparison with 0.\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3421, __extension__
__PRETTY_FUNCTION__))
3420 isConstOrConstSplat(N1C)->getAPIntValue().isZero() &&(static_cast <bool> (isConstOrConstSplat(N1C) &&
isConstOrConstSplat(N1C)->getAPIntValue().isZero() &&
"Should be a comparison with 0.") ? void (0) : __assert_fail
("isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->getAPIntValue().isZero() && \"Should be a comparison with 0.\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3421, __extension__
__PRETTY_FUNCTION__))
3421 "Should be a comparison with 0.")(static_cast <bool> (isConstOrConstSplat(N1C) &&
isConstOrConstSplat(N1C)->getAPIntValue().isZero() &&
"Should be a comparison with 0.") ? void (0) : __assert_fail
("isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->getAPIntValue().isZero() && \"Should be a comparison with 0.\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3421, __extension__
__PRETTY_FUNCTION__))
;
3422 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Valid only for [in]equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Valid only for [in]equality comparisons.\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3423, __extension__
__PRETTY_FUNCTION__))
3423 "Valid only for [in]equality comparisons.")(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Valid only for [in]equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Valid only for [in]equality comparisons.\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3423, __extension__
__PRETTY_FUNCTION__))
;
3424
3425 unsigned NewShiftOpcode;
3426 SDValue X, C, Y;
3427
3428 SelectionDAG &DAG = DCI.DAG;
3429 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3430
3431 // Look for '(C l>>/<< Y)'.
3432 auto Match = [&NewShiftOpcode, &X, &C, &Y, &TLI, &DAG](SDValue V) {
3433 // The shift should be one-use.
3434 if (!V.hasOneUse())
3435 return false;
3436 unsigned OldShiftOpcode = V.getOpcode();
3437 switch (OldShiftOpcode) {
3438 case ISD::SHL:
3439 NewShiftOpcode = ISD::SRL;
3440 break;
3441 case ISD::SRL:
3442 NewShiftOpcode = ISD::SHL;
3443 break;
3444 default:
3445 return false; // must be a logical shift.
3446 }
3447 // We should be shifting a constant.
3448 // FIXME: best to use isConstantOrConstantVector().
3449 C = V.getOperand(0);
3450 ConstantSDNode *CC =
3451 isConstOrConstSplat(C, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
3452 if (!CC)
3453 return false;
3454 Y = V.getOperand(1);
3455
3456 ConstantSDNode *XC =
3457 isConstOrConstSplat(X, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
3458 return TLI.shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
3459 X, XC, CC, Y, OldShiftOpcode, NewShiftOpcode, DAG);
3460 };
3461
3462 // LHS of comparison should be an one-use 'and'.
3463 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())
3464 return SDValue();
3465
3466 X = N0.getOperand(0);
3467 SDValue Mask = N0.getOperand(1);
3468
3469 // 'and' is commutative!
3470 if (!Match(Mask)) {
3471 std::swap(X, Mask);
3472 if (!Match(Mask))
3473 return SDValue();
3474 }
3475
3476 EVT VT = X.getValueType();
3477
3478 // Produce:
3479 // ((X 'OppositeShiftOpcode' Y) & C) Cond 0
3480 SDValue T0 = DAG.getNode(NewShiftOpcode, DL, VT, X, Y);
3481 SDValue T1 = DAG.getNode(ISD::AND, DL, VT, T0, C);
3482 SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, N1C, Cond);
3483 return T2;
3484}
3485
3486/// Try to fold an equality comparison with a {add/sub/xor} binary operation as
3487/// the 1st operand (N0). Callers are expected to swap the N0/N1 parameters to
3488/// handle the commuted versions of these patterns.
3489SDValue TargetLowering::foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1,
3490 ISD::CondCode Cond, const SDLoc &DL,
3491 DAGCombinerInfo &DCI) const {
3492 unsigned BOpcode = N0.getOpcode();
3493 assert((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) &&(static_cast <bool> ((BOpcode == ISD::ADD || BOpcode ==
ISD::SUB || BOpcode == ISD::XOR) && "Unexpected binop"
) ? void (0) : __assert_fail ("(BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) && \"Unexpected binop\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3494, __extension__
__PRETTY_FUNCTION__))
3494 "Unexpected binop")(static_cast <bool> ((BOpcode == ISD::ADD || BOpcode ==
ISD::SUB || BOpcode == ISD::XOR) && "Unexpected binop"
) ? void (0) : __assert_fail ("(BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) && \"Unexpected binop\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3494, __extension__
__PRETTY_FUNCTION__))
;
3495 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Unexpected condcode")(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Unexpected condcode") ? void (0) : __assert_fail
("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Unexpected condcode\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3495, __extension__
__PRETTY_FUNCTION__))
;
3496
3497 // (X + Y) == X --> Y == 0
3498 // (X - Y) == X --> Y == 0
3499 // (X ^ Y) == X --> Y == 0
3500 SelectionDAG &DAG = DCI.DAG;
3501 EVT OpVT = N0.getValueType();
3502 SDValue X = N0.getOperand(0);
3503 SDValue Y = N0.getOperand(1);
3504 if (X == N1)
3505 return DAG.getSetCC(DL, VT, Y, DAG.getConstant(0, DL, OpVT), Cond);
3506
3507 if (Y != N1)
3508 return SDValue();
3509
3510 // (X + Y) == Y --> X == 0
3511 // (X ^ Y) == Y --> X == 0
3512 if (BOpcode == ISD::ADD || BOpcode == ISD::XOR)
3513 return DAG.getSetCC(DL, VT, X, DAG.getConstant(0, DL, OpVT), Cond);
3514
3515 // The shift would not be valid if the operands are boolean (i1).
3516 if (!N0.hasOneUse() || OpVT.getScalarSizeInBits() == 1)
3517 return SDValue();
3518
3519 // (X - Y) == Y --> X == Y << 1
3520 EVT ShiftVT = getShiftAmountTy(OpVT, DAG.getDataLayout(),
3521 !DCI.isBeforeLegalize());
3522 SDValue One = DAG.getConstant(1, DL, ShiftVT);
3523 SDValue YShl1 = DAG.getNode(ISD::SHL, DL, N1.getValueType(), Y, One);
3524 if (!DCI.isCalledByLegalizer())
3525 DCI.AddToWorklist(YShl1.getNode());
3526 return DAG.getSetCC(DL, VT, X, YShl1, Cond);
3527}
3528
3529static SDValue simplifySetCCWithCTPOP(const TargetLowering &TLI, EVT VT,
3530 SDValue N0, const APInt &C1,
3531 ISD::CondCode Cond, const SDLoc &dl,
3532 SelectionDAG &DAG) {
3533 // Look through truncs that don't change the value of a ctpop.
3534 // FIXME: Add vector support? Need to be careful with setcc result type below.
3535 SDValue CTPOP = N0;
3536 if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && !VT.isVector() &&
3537 N0.getScalarValueSizeInBits() > Log2_32(N0.getOperand(0).getScalarValueSizeInBits()))
3538 CTPOP = N0.getOperand(0);
3539
3540 if (CTPOP.getOpcode() != ISD::CTPOP || !CTPOP.hasOneUse())
3541 return SDValue();
3542
3543 EVT CTVT = CTPOP.getValueType();
3544 SDValue CTOp = CTPOP.getOperand(0);
3545
3546 // If this is a vector CTPOP, keep the CTPOP if it is legal.
3547 // TODO: Should we check if CTPOP is legal(or custom) for scalars?
3548 if (VT.isVector() && TLI.isOperationLegal(ISD::CTPOP, CTVT))
3549 return SDValue();
3550
3551 // (ctpop x) u< 2 -> (x & x-1) == 0
3552 // (ctpop x) u> 1 -> (x & x-1) != 0
3553 if (Cond == ISD::SETULT || Cond == ISD::SETUGT) {
3554 unsigned CostLimit = TLI.getCustomCtpopCost(CTVT, Cond);
3555 if (C1.ugt(CostLimit + (Cond == ISD::SETULT)))
3556 return SDValue();
3557 if (C1 == 0 && (Cond == ISD::SETULT))
3558 return SDValue(); // This is handled elsewhere.
3559
3560 unsigned Passes = C1.getLimitedValue() - (Cond == ISD::SETULT);
3561
3562 SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
3563 SDValue Result = CTOp;
3564 for (unsigned i = 0; i < Passes; i++) {
3565 SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, Result, NegOne);
3566 Result = DAG.getNode(ISD::AND, dl, CTVT, Result, Add);
3567 }
3568 ISD::CondCode CC = Cond == ISD::SETULT ? ISD::SETEQ : ISD::SETNE;
3569 return DAG.getSetCC(dl, VT, Result, DAG.getConstant(0, dl, CTVT), CC);
3570 }
3571
3572 // If ctpop is not supported, expand a power-of-2 comparison based on it.
3573 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && C1 == 1) {
3574 // For scalars, keep CTPOP if it is legal or custom.
3575 if (!VT.isVector() && TLI.isOperationLegalOrCustom(ISD::CTPOP, CTVT))
3576 return SDValue();
3577 // This is based on X86's custom lowering for CTPOP which produces more
3578 // instructions than the expansion here.
3579
3580 // (ctpop x) == 1 --> (x != 0) && ((x & x-1) == 0)
3581 // (ctpop x) != 1 --> (x == 0) || ((x & x-1) != 0)
3582 SDValue Zero = DAG.getConstant(0, dl, CTVT);
3583 SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
3584 assert(CTVT.isInteger())(static_cast <bool> (CTVT.isInteger()) ? void (0) : __assert_fail
("CTVT.isInteger()", "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3584, __extension__ __PRETTY_FUNCTION__))
;
3585 ISD::CondCode InvCond = ISD::getSetCCInverse(Cond, CTVT);
3586 SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, CTOp, NegOne);
3587 SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Add);
3588 SDValue LHS = DAG.getSetCC(dl, VT, CTOp, Zero, InvCond);
3589 SDValue RHS = DAG.getSetCC(dl, VT, And, Zero, Cond);
3590 unsigned LogicOpcode = Cond == ISD::SETEQ ? ISD::AND : ISD::OR;
3591 return DAG.getNode(LogicOpcode, dl, VT, LHS, RHS);
3592 }
3593
3594 return SDValue();
3595}
3596
3597/// Try to simplify a setcc built with the specified operands and cc. If it is
3598/// unable to simplify it, return a null SDValue.
3599SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
3600 ISD::CondCode Cond, bool foldBooleans,
3601 DAGCombinerInfo &DCI,
3602 const SDLoc &dl) const {
3603 SelectionDAG &DAG = DCI.DAG;
3604 const DataLayout &Layout = DAG.getDataLayout();
3605 EVT OpVT = N0.getValueType();
3606
3607 // Constant fold or commute setcc.
3608 if (SDValue Fold = DAG.FoldSetCC(VT, N0, N1, Cond, dl))
3609 return Fold;
3610
3611 // Ensure that the constant occurs on the RHS and fold constant comparisons.
3612 // TODO: Handle non-splat vector constants. All undef causes trouble.
3613 // FIXME: We can't yet fold constant scalable vector splats, so avoid an
3614 // infinite loop here when we encounter one.
3615 ISD::CondCode SwappedCC = ISD::getSetCCSwappedOperands(Cond);
3616 if (isConstOrConstSplat(N0) &&
3617 (!OpVT.isScalableVector() || !isConstOrConstSplat(N1)) &&
3618 (DCI.isBeforeLegalizeOps() ||
3619 isCondCodeLegal(SwappedCC, N0.getSimpleValueType())))
3620 return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);
3621
3622 // If we have a subtract with the same 2 non-constant operands as this setcc
3623 // -- but in reverse order -- then try to commute the operands of this setcc
3624 // to match. A matching pair of setcc (cmp) and sub may be combined into 1
3625 // instruction on some targets.
3626 if (!isConstOrConstSplat(N0) && !isConstOrConstSplat(N1) &&
3627 (DCI.isBeforeLegalizeOps() ||
3628 isCondCodeLegal(SwappedCC, N0.getSimpleValueType())) &&
3629 DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N1, N0}) &&
3630 !DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N0, N1}))
3631 return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);
3632
3633 if (auto *N1C = isConstOrConstSplat(N1)) {
3634 const APInt &C1 = N1C->getAPIntValue();
3635
3636 // Optimize some CTPOP cases.
3637 if (SDValue V = simplifySetCCWithCTPOP(*this, VT, N0, C1, Cond, dl, DAG))
3638 return V;
3639
3640 // If the LHS is '(srl (ctlz x), 5)', the RHS is 0/1, and this is an
3641 // equality comparison, then we're just comparing whether X itself is
3642 // zero.
3643 if (N0.getOpcode() == ISD::SRL && (C1.isZero() || C1.isOne()) &&
3644 N0.getOperand(0).getOpcode() == ISD::CTLZ &&
3645 isPowerOf2_32(N0.getScalarValueSizeInBits())) {
3646 if (ConstantSDNode *ShAmt = isConstOrConstSplat(N0.getOperand(1))) {
3647 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3648 ShAmt->getAPIntValue() == Log2_32(N0.getScalarValueSizeInBits())) {
3649 if ((C1 == 0) == (Cond == ISD::SETEQ)) {
3650 // (srl (ctlz x), 5) == 0 -> X != 0
3651 // (srl (ctlz x), 5) != 1 -> X != 0
3652 Cond = ISD::SETNE;
3653 } else {
3654 // (srl (ctlz x), 5) != 0 -> X == 0
3655 // (srl (ctlz x), 5) == 1 -> X == 0
3656 Cond = ISD::SETEQ;
3657 }
3658 SDValue Zero = DAG.getConstant(0, dl, N0.getValueType());
3659 return DAG.getSetCC(dl, VT, N0.getOperand(0).getOperand(0), Zero,
3660 Cond);
3661 }
3662 }
3663 }
3664 }
3665
3666 // FIXME: Support vectors.
3667 if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
3668 const APInt &C1 = N1C->getAPIntValue();
3669
3670 // (zext x) == C --> x == (trunc C)
3671 // (sext x) == C --> x == (trunc C)
3672 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3673 DCI.isBeforeLegalize() && N0->hasOneUse()) {
3674 unsigned MinBits = N0.getValueSizeInBits();
3675 SDValue PreExt;
3676 bool Signed = false;
3677 if (N0->getOpcode() == ISD::ZERO_EXTEND) {
3678 // ZExt
3679 MinBits = N0->getOperand(0).getValueSizeInBits();
3680 PreExt = N0->getOperand(0);
3681 } else if (N0->getOpcode() == ISD::AND) {
3682 // DAGCombine turns costly ZExts into ANDs
3683 if (auto *C = dyn_cast<ConstantSDNode>(N0->getOperand(1)))
3684 if ((C->getAPIntValue()+1).isPowerOf2()) {
3685 MinBits = C->getAPIntValue().countTrailingOnes();
3686 PreExt = N0->getOperand(0);
3687 }
3688 } else if (N0->getOpcode() == ISD::SIGN_EXTEND) {
3689 // SExt
3690 MinBits = N0->getOperand(0).getValueSizeInBits();
3691 PreExt = N0->getOperand(0);
3692 Signed = true;
3693 } else if (auto *LN0 = dyn_cast<LoadSDNode>(N0)) {
3694 // ZEXTLOAD / SEXTLOAD
3695 if (LN0->getExtensionType() == ISD::ZEXTLOAD) {
3696 MinBits = LN0->getMemoryVT().getSizeInBits();
3697 PreExt = N0;
3698 } else if (LN0->getExtensionType() == ISD::SEXTLOAD) {
3699 Signed = true;
3700 MinBits = LN0->getMemoryVT().getSizeInBits();
3701 PreExt = N0;
3702 }
3703 }
3704
3705 // Figure out how many bits we need to preserve this constant.
3706 unsigned ReqdBits = Signed ? C1.getMinSignedBits() : C1.getActiveBits();
3707
3708 // Make sure we're not losing bits from the constant.
3709 if (MinBits > 0 &&
3710 MinBits < C1.getBitWidth() &&
3711 MinBits >= ReqdBits) {
3712 EVT MinVT = EVT::getIntegerVT(*DAG.getContext(), MinBits);
3713 if (isTypeDesirableForOp(ISD::SETCC, MinVT)) {
3714 // Will get folded away.
3715 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MinVT, PreExt);
3716 if (MinBits == 1 && C1 == 1)
3717 // Invert the condition.
3718 return DAG.getSetCC(dl, VT, Trunc, DAG.getConstant(0, dl, MVT::i1),
3719 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
3720 SDValue C = DAG.getConstant(C1.trunc(MinBits), dl, MinVT);
3721 return DAG.getSetCC(dl, VT, Trunc, C, Cond);
3722 }
3723
3724 // If truncating the setcc operands is not desirable, we can still
3725 // simplify the expression in some cases:
3726 // setcc ([sz]ext (setcc x, y, cc)), 0, setne) -> setcc (x, y, cc)
3727 // setcc ([sz]ext (setcc x, y, cc)), 0, seteq) -> setcc (x, y, inv(cc))
3728 // setcc (zext (setcc x, y, cc)), 1, setne) -> setcc (x, y, inv(cc))
3729 // setcc (zext (setcc x, y, cc)), 1, seteq) -> setcc (x, y, cc)
3730 // setcc (sext (setcc x, y, cc)), -1, setne) -> setcc (x, y, inv(cc))
3731 // setcc (sext (setcc x, y, cc)), -1, seteq) -> setcc (x, y, cc)
3732 SDValue TopSetCC = N0->getOperand(0);
3733 unsigned N0Opc = N0->getOpcode();
3734 bool SExt = (N0Opc == ISD::SIGN_EXTEND);
3735 if (TopSetCC.getValueType() == MVT::i1 && VT == MVT::i1 &&
3736 TopSetCC.getOpcode() == ISD::SETCC &&
3737 (N0Opc == ISD::ZERO_EXTEND || N0Opc == ISD::SIGN_EXTEND) &&
3738 (isConstFalseVal(N1C) ||
3739 isExtendedTrueVal(N1C, N0->getValueType(0), SExt))) {
3740
3741 bool Inverse = (N1C->isZero() && Cond == ISD::SETEQ) ||
3742 (!N1C->isZero() && Cond == ISD::SETNE);
3743
3744 if (!Inverse)
3745 return TopSetCC;
3746
3747 ISD::CondCode InvCond = ISD::getSetCCInverse(
3748 cast<CondCodeSDNode>(TopSetCC.getOperand(2))->get(),
3749 TopSetCC.getOperand(0).getValueType());
3750 return DAG.getSetCC(dl, VT, TopSetCC.getOperand(0),
3751 TopSetCC.getOperand(1),
3752 InvCond);
3753 }
3754 }
3755 }
3756
3757 // If the LHS is '(and load, const)', the RHS is 0, the test is for
3758 // equality or unsigned, and all 1 bits of the const are in the same
3759 // partial word, see if we can shorten the load.
3760 if (DCI.isBeforeLegalize() &&
3761 !ISD::isSignedIntSetCC(Cond) &&
3762 N0.getOpcode() == ISD::AND && C1 == 0 &&
3763 N0.getNode()->hasOneUse() &&
3764 isa<LoadSDNode>(N0.getOperand(0)) &&
3765 N0.getOperand(0).getNode()->hasOneUse() &&
3766 isa<ConstantSDNode>(N0.getOperand(1))) {
3767 LoadSDNode *Lod = cast<LoadSDNode>(N0.getOperand(0));
3768 APInt bestMask;
3769 unsigned bestWidth = 0, bestOffset = 0;
3770 if (Lod->isSimple() && Lod->isUnindexed()) {
3771 unsigned origWidth = N0.getValueSizeInBits();
3772 unsigned maskWidth = origWidth;
3773 // We can narrow (e.g.) 16-bit extending loads on 32-bit target to
3774 // 8 bits, but have to be careful...
3775 if (Lod->getExtensionType() != ISD::NON_EXTLOAD)
3776 origWidth = Lod->getMemoryVT().getSizeInBits();
3777 const APInt &Mask = N0.getConstantOperandAPInt(1);
3778 for (unsigned width = origWidth / 2; width>=8; width /= 2) {
3779 APInt newMask = APInt::getLowBitsSet(maskWidth, width);
3780 for (unsigned offset=0; offset<origWidth/width; offset++) {
3781 if (Mask.isSubsetOf(newMask)) {
3782 if (Layout.isLittleEndian())
3783 bestOffset = (uint64_t)offset * (width/8);
3784 else
3785 bestOffset = (origWidth/width - offset - 1) * (width/8);
3786 bestMask = Mask.lshr(offset * (width/8) * 8);
3787 bestWidth = width;
3788 break;
3789 }
3790 newMask <<= width;
3791 }
3792 }
3793 }
3794 if (bestWidth) {
3795 EVT newVT = EVT::getIntegerVT(*DAG.getContext(), bestWidth);
3796 if (newVT.isRound() &&
3797 shouldReduceLoadWidth(Lod, ISD::NON_EXTLOAD, newVT)) {
3798 SDValue Ptr = Lod->getBasePtr();
3799 if (bestOffset != 0)
3800 Ptr =
3801 DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(bestOffset), dl);
3802 SDValue NewLoad =
3803 DAG.getLoad(newVT, dl, Lod->getChain(), Ptr,
3804 Lod->getPointerInfo().getWithOffset(bestOffset),
3805 Lod->getOriginalAlign());
3806 return DAG.getSetCC(dl, VT,
3807 DAG.getNode(ISD::AND, dl, newVT, NewLoad,
3808 DAG.getConstant(bestMask.trunc(bestWidth),
3809 dl, newVT)),
3810 DAG.getConstant(0LL, dl, newVT), Cond);
3811 }
3812 }
3813 }
3814
3815 // If the LHS is a ZERO_EXTEND, perform the comparison on the input.
3816 if (N0.getOpcode() == ISD::ZERO_EXTEND) {
3817 unsigned InSize = N0.getOperand(0).getValueSizeInBits();
3818
3819 // If the comparison constant has bits in the upper part, the
3820 // zero-extended value could never match.
3821 if (C1.intersects(APInt::getHighBitsSet(C1.getBitWidth(),
3822 C1.getBitWidth() - InSize))) {
3823 switch (Cond) {
3824 case ISD::SETUGT:
3825 case ISD::SETUGE:
3826 case ISD::SETEQ:
3827 return DAG.getConstant(0, dl, VT);
3828 case ISD::SETULT:
3829 case ISD::SETULE:
3830 case ISD::SETNE:
3831 return DAG.getConstant(1, dl, VT);
3832 case ISD::SETGT:
3833 case ISD::SETGE:
3834 // True if the sign bit of C1 is set.
3835 return DAG.getConstant(C1.isNegative(), dl, VT);
3836 case ISD::SETLT:
3837 case ISD::SETLE:
3838 // True if the sign bit of C1 isn't set.
3839 return DAG.getConstant(C1.isNonNegative(), dl, VT);
3840 default:
3841 break;
3842 }
3843 }
3844
3845 // Otherwise, we can perform the comparison with the low bits.
3846 switch (Cond) {
3847 case ISD::SETEQ:
3848 case ISD::SETNE:
3849 case ISD::SETUGT:
3850 case ISD::SETUGE:
3851 case ISD::SETULT:
3852 case ISD::SETULE: {
3853 EVT newVT = N0.getOperand(0).getValueType();
3854 if (DCI.isBeforeLegalizeOps() ||
3855 (isOperationLegal(ISD::SETCC, newVT) &&
3856 isCondCodeLegal(Cond, newVT.getSimpleVT()))) {
3857 EVT NewSetCCVT = getSetCCResultType(Layout, *DAG.getContext(), newVT);
3858 SDValue NewConst = DAG.getConstant(C1.trunc(InSize), dl, newVT);
3859
3860 SDValue NewSetCC = DAG.getSetCC(dl, NewSetCCVT, N0.getOperand(0),
3861 NewConst, Cond);
3862 return DAG.getBoolExtOrTrunc(NewSetCC, dl, VT, N0.getValueType());
3863 }
3864 break;
3865 }
3866 default:
3867 break; // todo, be more careful with signed comparisons
3868 }
3869 } else if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
3870 (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3871 !isSExtCheaperThanZExt(cast<VTSDNode>(N0.getOperand(1))->getVT(),
3872 OpVT)) {
3873 EVT ExtSrcTy = cast<VTSDNode>(N0.getOperand(1))->getVT();
3874 unsigned ExtSrcTyBits = ExtSrcTy.getSizeInBits();
3875 EVT ExtDstTy = N0.getValueType();
3876 unsigned ExtDstTyBits = ExtDstTy.getSizeInBits();
3877
3878 // If the constant doesn't fit into the number of bits for the source of
3879 // the sign extension, it is impossible for both sides to be equal.
3880 if (C1.getMinSignedBits() > ExtSrcTyBits)
3881 return DAG.getBoolConstant(Cond == ISD::SETNE, dl, VT, OpVT);
3882
3883 assert(ExtDstTy == N0.getOperand(0).getValueType() &&(static_cast <bool> (ExtDstTy == N0.getOperand(0).getValueType
() && ExtDstTy != ExtSrcTy && "Unexpected types!"
) ? void (0) : __assert_fail ("ExtDstTy == N0.getOperand(0).getValueType() && ExtDstTy != ExtSrcTy && \"Unexpected types!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3884, __extension__
__PRETTY_FUNCTION__))
3884 ExtDstTy != ExtSrcTy && "Unexpected types!")(static_cast <bool> (ExtDstTy == N0.getOperand(0).getValueType
() && ExtDstTy != ExtSrcTy && "Unexpected types!"
) ? void (0) : __assert_fail ("ExtDstTy == N0.getOperand(0).getValueType() && ExtDstTy != ExtSrcTy && \"Unexpected types!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3884, __extension__
__PRETTY_FUNCTION__))
;
3885 APInt Imm = APInt::getLowBitsSet(ExtDstTyBits, ExtSrcTyBits);
3886 SDValue ZextOp = DAG.getNode(ISD::AND, dl, ExtDstTy, N0.getOperand(0),
3887 DAG.getConstant(Imm, dl, ExtDstTy));
3888 if (!DCI.isCalledByLegalizer())
3889 DCI.AddToWorklist(ZextOp.getNode());
3890 // Otherwise, make this a use of a zext.
3891 return DAG.getSetCC(dl, VT, ZextOp,
3892 DAG.getConstant(C1 & Imm, dl, ExtDstTy), Cond);
3893 } else if ((N1C->isZero() || N1C->isOne()) &&
3894 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
3895 // SETCC (SETCC), [0|1], [EQ|NE] -> SETCC
3896 if (N0.getOpcode() == ISD::SETCC &&
3897 isTypeLegal(VT) && VT.bitsLE(N0.getValueType()) &&
3898 (N0.getValueType() == MVT::i1 ||
3899 getBooleanContents(N0.getOperand(0).getValueType()) ==
3900 ZeroOrOneBooleanContent)) {
3901 bool TrueWhenTrue = (Cond == ISD::SETEQ) ^ (!N1C->isOne());
3902 if (TrueWhenTrue)
3903 return DAG.getNode(ISD::TRUNCATE, dl, VT, N0);
3904 // Invert the condition.
3905 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
3906 CC = ISD::getSetCCInverse(CC, N0.getOperand(0).getValueType());
3907 if (DCI.isBeforeLegalizeOps() ||
3908 isCondCodeLegal(CC, N0.getOperand(0).getSimpleValueType()))
3909 return DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC);
3910 }
3911
3912 if ((N0.getOpcode() == ISD::XOR ||
3913 (N0.getOpcode() == ISD::AND &&
3914 N0.getOperand(0).getOpcode() == ISD::XOR &&
3915 N0.getOperand(1) == N0.getOperand(0).getOperand(1))) &&
3916 isOneConstant(N0.getOperand(1))) {
3917 // If this is (X^1) == 0/1, swap the RHS and eliminate the xor. We
3918 // can only do this if the top bits are known zero.
3919 unsigned BitWidth = N0.getValueSizeInBits();
3920 if (DAG.MaskedValueIsZero(N0,
3921 APInt::getHighBitsSet(BitWidth,
3922 BitWidth-1))) {
3923 // Okay, get the un-inverted input value.
3924 SDValue Val;
3925 if (N0.getOpcode() == ISD::XOR) {
3926 Val = N0.getOperand(0);
3927 } else {
3928 assert(N0.getOpcode() == ISD::AND &&(static_cast <bool> (N0.getOpcode() == ISD::AND &&
N0.getOperand(0).getOpcode() == ISD::XOR) ? void (0) : __assert_fail
("N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::XOR"
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3929, __extension__
__PRETTY_FUNCTION__))
3929 N0.getOperand(0).getOpcode() == ISD::XOR)(static_cast <bool> (N0.getOpcode() == ISD::AND &&
N0.getOperand(0).getOpcode() == ISD::XOR) ? void (0) : __assert_fail
("N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::XOR"
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 3929, __extension__
__PRETTY_FUNCTION__))
;
3930 // ((X^1)&1)^1 -> X & 1
3931 Val = DAG.getNode(ISD::AND, dl, N0.getValueType(),
3932 N0.getOperand(0).getOperand(0),
3933 N0.getOperand(1));
3934 }
3935
3936 return DAG.getSetCC(dl, VT, Val, N1,
3937 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
3938 }
3939 } else if (N1C->isOne()) {
3940 SDValue Op0 = N0;
3941 if (Op0.getOpcode() == ISD::TRUNCATE)
3942 Op0 = Op0.getOperand(0);
3943
3944 if ((Op0.getOpcode() == ISD::XOR) &&
3945 Op0.getOperand(0).getOpcode() == ISD::SETCC &&
3946 Op0.getOperand(1).getOpcode() == ISD::SETCC) {
3947 SDValue XorLHS = Op0.getOperand(0);
3948 SDValue XorRHS = Op0.getOperand(1);
3949 // Ensure that the input setccs return an i1 type or 0/1 value.
3950 if (Op0.getValueType() == MVT::i1 ||
3951 (getBooleanContents(XorLHS.getOperand(0).getValueType()) ==
3952 ZeroOrOneBooleanContent &&
3953 getBooleanContents(XorRHS.getOperand(0).getValueType()) ==
3954 ZeroOrOneBooleanContent)) {
3955 // (xor (setcc), (setcc)) == / != 1 -> (setcc) != / == (setcc)
3956 Cond = (Cond == ISD::SETEQ) ? ISD::SETNE : ISD::SETEQ;
3957 return DAG.getSetCC(dl, VT, XorLHS, XorRHS, Cond);
3958 }
3959 }
3960 if (Op0.getOpcode() == ISD::AND && isOneConstant(Op0.getOperand(1))) {
3961 // If this is (X&1) == / != 1, normalize it to (X&1) != / == 0.
3962 if (Op0.getValueType().bitsGT(VT))
3963 Op0 = DAG.getNode(ISD::AND, dl, VT,
3964 DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)),
3965 DAG.getConstant(1, dl, VT));
3966 else if (Op0.getValueType().bitsLT(VT))
3967 Op0 = DAG.getNode(ISD::AND, dl, VT,
3968 DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)),
3969 DAG.getConstant(1, dl, VT));
3970
3971 return DAG.getSetCC(dl, VT, Op0,
3972 DAG.getConstant(0, dl, Op0.getValueType()),
3973 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
3974 }
3975 if (Op0.getOpcode() == ISD::AssertZext &&
3976 cast<VTSDNode>(Op0.getOperand(1))->getVT() == MVT::i1)
3977 return DAG.getSetCC(dl, VT, Op0,
3978 DAG.getConstant(0, dl, Op0.getValueType()),
3979 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
3980 }
3981 }
3982
3983 // Given:
3984 // icmp eq/ne (urem %x, %y), 0
3985 // Iff %x has 0 or 1 bits set, and %y has at least 2 bits set, omit 'urem':
3986 // icmp eq/ne %x, 0
3987 if (N0.getOpcode() == ISD::UREM && N1C->isZero() &&
3988 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
3989 KnownBits XKnown = DAG.computeKnownBits(N0.getOperand(0));
3990 KnownBits YKnown = DAG.computeKnownBits(N0.getOperand(1));
3991 if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2)
3992 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1, Cond);
3993 }
3994
3995 // Fold set_cc seteq (ashr X, BW-1), -1 -> set_cc setlt X, 0
3996 // and set_cc setne (ashr X, BW-1), -1 -> set_cc setge X, 0
3997 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3998 N0.getOpcode() == ISD::SRA && isa<ConstantSDNode>(N0.getOperand(1)) &&
3999 N0.getConstantOperandAPInt(1) == OpVT.getScalarSizeInBits() - 1 &&
4000 N1C && N1C->isAllOnes()) {
4001 return DAG.getSetCC(dl, VT, N0.getOperand(0),
4002 DAG.getConstant(0, dl, OpVT),
4003 Cond == ISD::SETEQ ? ISD::SETLT : ISD::SETGE);
4004 }
4005
4006 if (SDValue V =
4007 optimizeSetCCOfSignedTruncationCheck(VT, N0, N1, Cond, DCI, dl))
4008 return V;
4009 }
4010
4011 // These simplifications apply to splat vectors as well.
4012 // TODO: Handle more splat vector cases.
4013 if (auto *N1C = isConstOrConstSplat(N1)) {
4014 const APInt &C1 = N1C->getAPIntValue();
4015
4016 APInt MinVal, MaxVal;
4017 unsigned OperandBitSize = N1C->getValueType(0).getScalarSizeInBits();
4018 if (ISD::isSignedIntSetCC(Cond)) {
4019 MinVal = APInt::getSignedMinValue(OperandBitSize);
4020 MaxVal = APInt::getSignedMaxValue(OperandBitSize);
4021 } else {
4022 MinVal = APInt::getMinValue(OperandBitSize);
4023 MaxVal = APInt::getMaxValue(OperandBitSize);
4024 }
4025
4026 // Canonicalize GE/LE comparisons to use GT/LT comparisons.
4027 if (Cond == ISD::SETGE || Cond == ISD::SETUGE) {
4028 // X >= MIN --> true
4029 if (C1 == MinVal)
4030 return DAG.getBoolConstant(true, dl, VT, OpVT);
4031
4032 if (!VT.isVector()) { // TODO: Support this for vectors.
4033 // X >= C0 --> X > (C0 - 1)
4034 APInt C = C1 - 1;
4035 ISD::CondCode NewCC = (Cond == ISD::SETGE) ? ISD::SETGT : ISD::SETUGT;
4036 if ((DCI.isBeforeLegalizeOps() ||
4037 isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
4038 (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
4039 isLegalICmpImmediate(C.getSExtValue())))) {
4040 return DAG.getSetCC(dl, VT, N0,
4041 DAG.getConstant(C, dl, N1.getValueType()),
4042 NewCC);
4043 }
4044 }
4045 }
4046
4047 if (Cond == ISD::SETLE || Cond == ISD::SETULE) {
4048 // X <= MAX --> true
4049 if (C1 == MaxVal)
4050 return DAG.getBoolConstant(true, dl, VT, OpVT);
4051
4052 // X <= C0 --> X < (C0 + 1)
4053 if (!VT.isVector()) { // TODO: Support this for vectors.
4054 APInt C = C1 + 1;
4055 ISD::CondCode NewCC = (Cond == ISD::SETLE) ? ISD::SETLT : ISD::SETULT;
4056 if ((DCI.isBeforeLegalizeOps() ||
4057 isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
4058 (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
4059 isLegalICmpImmediate(C.getSExtValue())))) {
4060 return DAG.getSetCC(dl, VT, N0,
4061 DAG.getConstant(C, dl, N1.getValueType()),
4062 NewCC);
4063 }
4064 }
4065 }
4066
4067 if (Cond == ISD::SETLT || Cond == ISD::SETULT) {
4068 if (C1 == MinVal)
4069 return DAG.getBoolConstant(false, dl, VT, OpVT); // X < MIN --> false
4070
4071 // TODO: Support this for vectors after legalize ops.
4072 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
4073 // Canonicalize setlt X, Max --> setne X, Max
4074 if (C1 == MaxVal)
4075 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
4076
4077 // If we have setult X, 1, turn it into seteq X, 0
4078 if (C1 == MinVal+1)
4079 return DAG.getSetCC(dl, VT, N0,
4080 DAG.getConstant(MinVal, dl, N0.getValueType()),
4081 ISD::SETEQ);
4082 }
4083 }
4084
4085 if (Cond == ISD::SETGT || Cond == ISD::SETUGT) {
4086 if (C1 == MaxVal)
4087 return DAG.getBoolConstant(false, dl, VT, OpVT); // X > MAX --> false
4088
4089 // TODO: Support this for vectors after legalize ops.
4090 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
4091 // Canonicalize setgt X, Min --> setne X, Min
4092 if (C1 == MinVal)
4093 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
4094
4095 // If we have setugt X, Max-1, turn it into seteq X, Max
4096 if (C1 == MaxVal-1)
4097 return DAG.getSetCC(dl, VT, N0,
4098 DAG.getConstant(MaxVal, dl, N0.getValueType()),
4099 ISD::SETEQ);
4100 }
4101 }
4102
4103 if (Cond == ISD::SETEQ || Cond == ISD::SETNE) {
4104 // (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
4105 if (C1.isZero())
4106 if (SDValue CC = optimizeSetCCByHoistingAndByConstFromLogicalShift(
4107 VT, N0, N1, Cond, DCI, dl))
4108 return CC;
4109
4110 // For all/any comparisons, replace or(x,shl(y,bw/2)) with and/or(x,y).
4111 // For example, when high 32-bits of i64 X are known clear:
4112 // all bits clear: (X | (Y<<32)) == 0 --> (X | Y) == 0
4113 // all bits set: (X | (Y<<32)) == -1 --> (X & Y) == -1
4114 bool CmpZero = N1C->getAPIntValue().isZero();
4115 bool CmpNegOne = N1C->getAPIntValue().isAllOnes();
4116 if ((CmpZero || CmpNegOne) && N0.hasOneUse()) {
4117 // Match or(lo,shl(hi,bw/2)) pattern.
4118 auto IsConcat = [&](SDValue V, SDValue &Lo, SDValue &Hi) {
4119 unsigned EltBits = V.getScalarValueSizeInBits();
4120 if (V.getOpcode() != ISD::OR || (EltBits % 2) != 0)
4121 return false;
4122 SDValue LHS = V.getOperand(0);
4123 SDValue RHS = V.getOperand(1);
4124 APInt HiBits = APInt::getHighBitsSet(EltBits, EltBits / 2);
4125 // Unshifted element must have zero upperbits.
4126 if (RHS.getOpcode() == ISD::SHL &&
4127 isa<ConstantSDNode>(RHS.getOperand(1)) &&
4128 RHS.getConstantOperandAPInt(1) == (EltBits / 2) &&
4129 DAG.MaskedValueIsZero(LHS, HiBits)) {
4130 Lo = LHS;
4131 Hi = RHS.getOperand(0);
4132 return true;
4133 }
4134 if (LHS.getOpcode() == ISD::SHL &&
4135 isa<ConstantSDNode>(LHS.getOperand(1)) &&
4136 LHS.getConstantOperandAPInt(1) == (EltBits / 2) &&
4137 DAG.MaskedValueIsZero(RHS, HiBits)) {
4138 Lo = RHS;
4139 Hi = LHS.getOperand(0);
4140 return true;
4141 }
4142 return false;
4143 };
4144
4145 auto MergeConcat = [&](SDValue Lo, SDValue Hi) {
4146 unsigned EltBits = N0.getScalarValueSizeInBits();
4147 unsigned HalfBits = EltBits / 2;
4148 APInt HiBits = APInt::getHighBitsSet(EltBits, HalfBits);
4149 SDValue LoBits = DAG.getConstant(~HiBits, dl, OpVT);
4150 SDValue HiMask = DAG.getNode(ISD::AND, dl, OpVT, Hi, LoBits);
4151 SDValue NewN0 =
4152 DAG.getNode(CmpZero ? ISD::OR : ISD::AND, dl, OpVT, Lo, HiMask);
4153 SDValue NewN1 = CmpZero ? DAG.getConstant(0, dl, OpVT) : LoBits;
4154 return DAG.getSetCC(dl, VT, NewN0, NewN1, Cond);
4155 };
4156
4157 SDValue Lo, Hi;
4158 if (IsConcat(N0, Lo, Hi))
4159 return MergeConcat(Lo, Hi);
4160
4161 if (N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR) {
4162 SDValue Lo0, Lo1, Hi0, Hi1;
4163 if (IsConcat(N0.getOperand(0), Lo0, Hi0) &&
4164 IsConcat(N0.getOperand(1), Lo1, Hi1)) {
4165 return MergeConcat(DAG.getNode(N0.getOpcode(), dl, OpVT, Lo0, Lo1),
4166 DAG.getNode(N0.getOpcode(), dl, OpVT, Hi0, Hi1));
4167 }
4168 }
4169 }
4170 }
4171
4172 // If we have "setcc X, C0", check to see if we can shrink the immediate
4173 // by changing cc.
4174 // TODO: Support this for vectors after legalize ops.
4175 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
4176 // SETUGT X, SINTMAX -> SETLT X, 0
4177 // SETUGE X, SINTMIN -> SETLT X, 0
4178 if ((Cond == ISD::SETUGT && C1.isMaxSignedValue()) ||
4179 (Cond == ISD::SETUGE && C1.isMinSignedValue()))
4180 return DAG.getSetCC(dl, VT, N0,
4181 DAG.getConstant(0, dl, N1.getValueType()),
4182 ISD::SETLT);
4183
4184 // SETULT X, SINTMIN -> SETGT X, -1
4185 // SETULE X, SINTMAX -> SETGT X, -1
4186 if ((Cond == ISD::SETULT && C1.isMinSignedValue()) ||
4187 (Cond == ISD::SETULE && C1.isMaxSignedValue()))
4188 return DAG.getSetCC(dl, VT, N0,
4189 DAG.getAllOnesConstant(dl, N1.getValueType()),
4190 ISD::SETGT);
4191 }
4192 }
4193
4194 // Back to non-vector simplifications.
4195 // TODO: Can we do these for vector splats?
4196 if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
4197 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4198 const APInt &C1 = N1C->getAPIntValue();
4199 EVT ShValTy = N0.getValueType();
4200
4201 // Fold bit comparisons when we can. This will result in an
4202 // incorrect value when boolean false is negative one, unless
4203 // the bitsize is 1 in which case the false value is the same
4204 // in practice regardless of the representation.
4205 if ((VT.getSizeInBits() == 1 ||
4206 getBooleanContents(N0.getValueType()) == ZeroOrOneBooleanContent) &&
4207 (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4208 (VT == ShValTy || (isTypeLegal(VT) && VT.bitsLE(ShValTy))) &&
4209 N0.getOpcode() == ISD::AND) {
4210 if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
4211 EVT ShiftTy =
4212 getShiftAmountTy(ShValTy, Layout, !DCI.isBeforeLegalize());
4213 if (Cond == ISD::SETNE && C1 == 0) {// (X & 8) != 0 --> (X & 8) >> 3
4214 // Perform the xform if the AND RHS is a single bit.
4215 unsigned ShCt = AndRHS->getAPIntValue().logBase2();
4216 if (AndRHS->getAPIntValue().isPowerOf2() &&
4217 !TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {
4218 return DAG.getNode(ISD::TRUNCATE, dl, VT,
4219 DAG.getNode(ISD::SRL, dl, ShValTy, N0,
4220 DAG.getConstant(ShCt, dl, ShiftTy)));
4221 }
4222 } else if (Cond == ISD::SETEQ && C1 == AndRHS->getAPIntValue()) {
4223 // (X & 8) == 8 --> (X & 8) >> 3
4224 // Perform the xform if C1 is a single bit.
4225 unsigned ShCt = C1.logBase2();
4226 if (C1.isPowerOf2() &&
4227 !TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {
4228 return DAG.getNode(ISD::TRUNCATE, dl, VT,
4229 DAG.getNode(ISD::SRL, dl, ShValTy, N0,
4230 DAG.getConstant(ShCt, dl, ShiftTy)));
4231 }
4232 }
4233 }
4234 }
4235
4236 if (C1.getMinSignedBits() <= 64 &&
4237 !isLegalICmpImmediate(C1.getSExtValue())) {
4238 EVT ShiftTy = getShiftAmountTy(ShValTy, Layout, !DCI.isBeforeLegalize());
4239 // (X & -256) == 256 -> (X >> 8) == 1
4240 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4241 N0.getOpcode() == ISD::AND && N0.hasOneUse()) {
4242 if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
4243 const APInt &AndRHSC = AndRHS->getAPIntValue();
4244 if (AndRHSC.isNegatedPowerOf2() && (AndRHSC & C1) == C1) {
4245 unsigned ShiftBits = AndRHSC.countTrailingZeros();
4246 if (!TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
4247 SDValue Shift =
4248 DAG.getNode(ISD::SRL, dl, ShValTy, N0.getOperand(0),
4249 DAG.getConstant(ShiftBits, dl, ShiftTy));
4250 SDValue CmpRHS = DAG.getConstant(C1.lshr(ShiftBits), dl, ShValTy);
4251 return DAG.getSetCC(dl, VT, Shift, CmpRHS, Cond);
4252 }
4253 }
4254 }
4255 } else if (Cond == ISD::SETULT || Cond == ISD::SETUGE ||
4256 Cond == ISD::SETULE || Cond == ISD::SETUGT) {
4257 bool AdjOne = (Cond == ISD::SETULE || Cond == ISD::SETUGT);
4258 // X < 0x100000000 -> (X >> 32) < 1
4259 // X >= 0x100000000 -> (X >> 32) >= 1
4260 // X <= 0x0ffffffff -> (X >> 32) < 1
4261 // X > 0x0ffffffff -> (X >> 32) >= 1
4262 unsigned ShiftBits;
4263 APInt NewC = C1;
4264 ISD::CondCode NewCond = Cond;
4265 if (AdjOne) {
4266 ShiftBits = C1.countTrailingOnes();
4267 NewC = NewC + 1;
4268 NewCond = (Cond == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
4269 } else {
4270 ShiftBits = C1.countTrailingZeros();
4271 }
4272 NewC.lshrInPlace(ShiftBits);
4273 if (ShiftBits && NewC.getMinSignedBits() <= 64 &&
4274 isLegalICmpImmediate(NewC.getSExtValue()) &&
4275 !TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
4276 SDValue Shift = DAG.getNode(ISD::SRL, dl, ShValTy, N0,
4277 DAG.getConstant(ShiftBits, dl, ShiftTy));
4278 SDValue CmpRHS = DAG.getConstant(NewC, dl, ShValTy);
4279 return DAG.getSetCC(dl, VT, Shift, CmpRHS, NewCond);
4280 }
4281 }
4282 }
4283 }
4284
4285 if (!isa<ConstantFPSDNode>(N0) && isa<ConstantFPSDNode>(N1)) {
4286 auto *CFP = cast<ConstantFPSDNode>(N1);
4287 assert(!CFP->getValueAPF().isNaN() && "Unexpected NaN value")(static_cast <bool> (!CFP->getValueAPF().isNaN() &&
"Unexpected NaN value") ? void (0) : __assert_fail ("!CFP->getValueAPF().isNaN() && \"Unexpected NaN value\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 4287, __extension__
__PRETTY_FUNCTION__))
;
4288
4289 // Otherwise, we know the RHS is not a NaN. Simplify the node to drop the
4290 // constant if knowing that the operand is non-nan is enough. We prefer to
4291 // have SETO(x,x) instead of SETO(x, 0.0) because this avoids having to
4292 // materialize 0.0.
4293 if (Cond == ISD::SETO || Cond == ISD::SETUO)
4294 return DAG.getSetCC(dl, VT, N0, N0, Cond);
4295
4296 // setcc (fneg x), C -> setcc swap(pred) x, -C
4297 if (N0.getOpcode() == ISD::FNEG) {
4298 ISD::CondCode SwapCond = ISD::getSetCCSwappedOperands(Cond);
4299 if (DCI.isBeforeLegalizeOps() ||
4300 isCondCodeLegal(SwapCond, N0.getSimpleValueType())) {
4301 SDValue NegN1 = DAG.getNode(ISD::FNEG, dl, N0.getValueType(), N1);
4302 return DAG.getSetCC(dl, VT, N0.getOperand(0), NegN1, SwapCond);
4303 }
4304 }
4305
4306 // If the condition is not legal, see if we can find an equivalent one
4307 // which is legal.
4308 if (!isCondCodeLegal(Cond, N0.getSimpleValueType())) {
4309 // If the comparison was an awkward floating-point == or != and one of
4310 // the comparison operands is infinity or negative infinity, convert the
4311 // condition to a less-awkward <= or >=.
4312 if (CFP->getValueAPF().isInfinity()) {
4313 bool IsNegInf = CFP->getValueAPF().isNegative();
4314 ISD::CondCode NewCond = ISD::SETCC_INVALID;
4315 switch (Cond) {
4316 case ISD::SETOEQ: NewCond = IsNegInf ? ISD::SETOLE : ISD::SETOGE; break;
4317 case ISD::SETUEQ: NewCond = IsNegInf ? ISD::SETULE : ISD::SETUGE; break;
4318 case ISD::SETUNE: NewCond = IsNegInf ? ISD::SETUGT : ISD::SETULT; break;
4319 case ISD::SETONE: NewCond = IsNegInf ? ISD::SETOGT : ISD::SETOLT; break;
4320 default: break;
4321 }
4322 if (NewCond != ISD::SETCC_INVALID &&
4323 isCondCodeLegal(NewCond, N0.getSimpleValueType()))
4324 return DAG.getSetCC(dl, VT, N0, N1, NewCond);
4325 }
4326 }
4327 }
4328
4329 if (N0 == N1) {
4330 // The sext(setcc()) => setcc() optimization relies on the appropriate
4331 // constant being emitted.
4332 assert(!N0.getValueType().isInteger() &&(static_cast <bool> (!N0.getValueType().isInteger() &&
"Integer types should be handled by FoldSetCC") ? void (0) :
__assert_fail ("!N0.getValueType().isInteger() && \"Integer types should be handled by FoldSetCC\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 4333, __extension__
__PRETTY_FUNCTION__))
4333 "Integer types should be handled by FoldSetCC")(static_cast <bool> (!N0.getValueType().isInteger() &&
"Integer types should be handled by FoldSetCC") ? void (0) :
__assert_fail ("!N0.getValueType().isInteger() && \"Integer types should be handled by FoldSetCC\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 4333, __extension__
__PRETTY_FUNCTION__))
;
4334
4335 bool EqTrue = ISD::isTrueWhenEqual(Cond);
4336 unsigned UOF = ISD::getUnorderedFlavor(Cond);
4337 if (UOF == 2) // FP operators that are undefined on NaNs.
4338 return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
4339 if (UOF == unsigned(EqTrue))
4340 return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
4341 // Otherwise, we can't fold it. However, we can simplify it to SETUO/SETO
4342 // if it is not already.
4343 ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO;
4344 if (NewCond != Cond &&
4345 (DCI.isBeforeLegalizeOps() ||
4346 isCondCodeLegal(NewCond, N0.getSimpleValueType())))
4347 return DAG.getSetCC(dl, VT, N0, N1, NewCond);
4348 }
4349
4350 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4351 N0.getValueType().isInteger()) {
4352 if (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB ||
4353 N0.getOpcode() == ISD::XOR) {
4354 // Simplify (X+Y) == (X+Z) --> Y == Z
4355 if (N0.getOpcode() == N1.getOpcode()) {
4356 if (N0.getOperand(0) == N1.getOperand(0))
4357 return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(1), Cond);
4358 if (N0.getOperand(1) == N1.getOperand(1))
4359 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(0), Cond);
4360 if (isCommutativeBinOp(N0.getOpcode())) {
4361 // If X op Y == Y op X, try other combinations.
4362 if (N0.getOperand(0) == N1.getOperand(1))
4363 return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(0),
4364 Cond);
4365 if (N0.getOperand(1) == N1.getOperand(0))
4366 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(1),
4367 Cond);
4368 }
4369 }
4370
4371 // If RHS is a legal immediate value for a compare instruction, we need
4372 // to be careful about increasing register pressure needlessly.
4373 bool LegalRHSImm = false;
4374
4375 if (auto *RHSC = dyn_cast<ConstantSDNode>(N1)) {
4376 if (auto *LHSR = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
4377 // Turn (X+C1) == C2 --> X == C2-C1
4378 if (N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse()) {
4379 return DAG.getSetCC(dl, VT, N0.getOperand(0),
4380 DAG.getConstant(RHSC->getAPIntValue()-
4381 LHSR->getAPIntValue(),
4382 dl, N0.getValueType()), Cond);
4383 }
4384
4385 // Turn (X^C1) == C2 into X == C1^C2 iff X&~C1 = 0.
4386 if (N0.getOpcode() == ISD::XOR)
4387 // If we know that all of the inverted bits are zero, don't bother
4388 // performing the inversion.
4389 if (DAG.MaskedValueIsZero(N0.getOperand(0), ~LHSR->getAPIntValue()))
4390 return
4391 DAG.getSetCC(dl, VT, N0.getOperand(0),
4392 DAG.getConstant(LHSR->getAPIntValue() ^
4393 RHSC->getAPIntValue(),
4394 dl, N0.getValueType()),
4395 Cond);
4396 }
4397
4398 // Turn (C1-X) == C2 --> X == C1-C2
4399 if (auto *SUBC = dyn_cast<ConstantSDNode>(N0.getOperand(0))) {
4400 if (N0.getOpcode() == ISD::SUB && N0.getNode()->hasOneUse()) {
4401 return
4402 DAG.getSetCC(dl, VT, N0.getOperand(1),
4403 DAG.getConstant(SUBC->getAPIntValue() -
4404 RHSC->getAPIntValue(),
4405 dl, N0.getValueType()),
4406 Cond);
4407 }
4408 }
4409
4410 // Could RHSC fold directly into a compare?
4411 if (RHSC->getValueType(0).getSizeInBits() <= 64)
4412 LegalRHSImm = isLegalICmpImmediate(RHSC->getSExtValue());
4413 }
4414
4415 // (X+Y) == X --> Y == 0 and similar folds.
4416 // Don't do this if X is an immediate that can fold into a cmp
4417 // instruction and X+Y has other uses. It could be an induction variable
4418 // chain, and the transform would increase register pressure.
4419 if (!LegalRHSImm || N0.hasOneUse())
4420 if (SDValue V = foldSetCCWithBinOp(VT, N0, N1, Cond, dl, DCI))
4421 return V;
4422 }
4423
4424 if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB ||
4425 N1.getOpcode() == ISD::XOR)
4426 if (SDValue V = foldSetCCWithBinOp(VT, N1, N0, Cond, dl, DCI))
4427 return V;
4428
4429 if (SDValue V = foldSetCCWithAnd(VT, N0, N1, Cond, dl, DCI))
4430 return V;
4431 }
4432
4433 // Fold remainder of division by a constant.
4434 if ((N0.getOpcode() == ISD::UREM || N0.getOpcode() == ISD::SREM) &&
4435 N0.hasOneUse() && (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
4436 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
4437
4438 // When division is cheap or optimizing for minimum size,
4439 // fall through to DIVREM creation by skipping this fold.
4440 if (!isIntDivCheap(VT, Attr) && !Attr.hasFnAttr(Attribute::MinSize)) {
4441 if (N0.getOpcode() == ISD::UREM) {
4442 if (SDValue Folded = buildUREMEqFold(VT, N0, N1, Cond, DCI, dl))
4443 return Folded;
4444 } else if (N0.getOpcode() == ISD::SREM) {
4445 if (SDValue Folded = buildSREMEqFold(VT, N0, N1, Cond, DCI, dl))
4446 return Folded;
4447 }
4448 }
4449 }
4450
4451 // Fold away ALL boolean setcc's.
4452 if (N0.getValueType().getScalarType() == MVT::i1 && foldBooleans) {
4453 SDValue Temp;
4454 switch (Cond) {
4455 default: llvm_unreachable("Unknown integer setcc!")::llvm::llvm_unreachable_internal("Unknown integer setcc!", "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4455)
;
4456 case ISD::SETEQ: // X == Y -> ~(X^Y)
4457 Temp = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
4458 N0 = DAG.getNOT(dl, Temp, OpVT);
4459 if (!DCI.isCalledByLegalizer())
4460 DCI.AddToWorklist(Temp.getNode());
4461 break;
4462 case ISD::SETNE: // X != Y --> (X^Y)
4463 N0 = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
4464 break;
4465 case ISD::SETGT: // X >s Y --> X == 0 & Y == 1 --> ~X & Y
4466 case ISD::SETULT: // X <u Y --> X == 0 & Y == 1 --> ~X & Y
4467 Temp = DAG.getNOT(dl, N0, OpVT);
4468 N0 = DAG.getNode(ISD::AND, dl, OpVT, N1, Temp);
4469 if (!DCI.isCalledByLegalizer())
4470 DCI.AddToWorklist(Temp.getNode());
4471 break;
4472 case ISD::SETLT: // X <s Y --> X == 1 & Y == 0 --> ~Y & X
4473 case ISD::SETUGT: // X >u Y --> X == 1 & Y == 0 --> ~Y & X
4474 Temp = DAG.getNOT(dl, N1, OpVT);
4475 N0 = DAG.getNode(ISD::AND, dl, OpVT, N0, Temp);
4476 if (!DCI.isCalledByLegalizer())
4477 DCI.AddToWorklist(Temp.getNode());
4478 break;
4479 case ISD::SETULE: // X <=u Y --> X == 0 | Y == 1 --> ~X | Y
4480 case ISD::SETGE: // X >=s Y --> X == 0 | Y == 1 --> ~X | Y
4481 Temp = DAG.getNOT(dl, N0, OpVT);
4482 N0 = DAG.getNode(ISD::OR, dl, OpVT, N1, Temp);
4483 if (!DCI.isCalledByLegalizer())
4484 DCI.AddToWorklist(Temp.getNode());
4485 break;
4486 case ISD::SETUGE: // X >=u Y --> X == 1 | Y == 0 --> ~Y | X
4487 case ISD::SETLE: // X <=s Y --> X == 1 | Y == 0 --> ~Y | X
4488 Temp = DAG.getNOT(dl, N1, OpVT);
4489 N0 = DAG.getNode(ISD::OR, dl, OpVT, N0, Temp);
4490 break;
4491 }
4492 if (VT.getScalarType() != MVT::i1) {
4493 if (!DCI.isCalledByLegalizer())
4494 DCI.AddToWorklist(N0.getNode());
4495 // FIXME: If running after legalize, we probably can't do this.
4496 ISD::NodeType ExtendCode = getExtendForContent(getBooleanContents(OpVT));
4497 N0 = DAG.getNode(ExtendCode, dl, VT, N0);
4498 }
4499 return N0;
4500 }
4501
4502 // Could not fold it.
4503 return SDValue();
4504}
4505
4506/// Returns true (and the GlobalValue and the offset) if the node is a
4507/// GlobalAddress + offset.
4508bool TargetLowering::isGAPlusOffset(SDNode *WN, const GlobalValue *&GA,
4509 int64_t &Offset) const {
4510
4511 SDNode *N = unwrapAddress(SDValue(WN, 0)).getNode();
4512
4513 if (auto *GASD = dyn_cast<GlobalAddressSDNode>(N)) {
4514 GA = GASD->getGlobal();
4515 Offset += GASD->getOffset();
4516 return true;
4517 }
4518
4519 if (N->getOpcode() == ISD::ADD) {
4520 SDValue N1 = N->getOperand(0);
4521 SDValue N2 = N->getOperand(1);
4522 if (isGAPlusOffset(N1.getNode(), GA, Offset)) {
4523 if (auto *V = dyn_cast<ConstantSDNode>(N2)) {
4524 Offset += V->getSExtValue();
4525 return true;
4526 }
4527 } else if (isGAPlusOffset(N2.getNode(), GA, Offset)) {
4528 if (auto *V = dyn_cast<ConstantSDNode>(N1)) {
4529 Offset += V->getSExtValue();
4530 return true;
4531 }
4532 }
4533 }
4534
4535 return false;
4536}
4537
4538SDValue TargetLowering::PerformDAGCombine(SDNode *N,
4539 DAGCombinerInfo &DCI) const {
4540 // Default implementation: no optimization.
4541 return SDValue();
4542}
4543
4544//===----------------------------------------------------------------------===//
4545// Inline Assembler Implementation Methods
4546//===----------------------------------------------------------------------===//
4547
4548TargetLowering::ConstraintType
4549TargetLowering::getConstraintType(StringRef Constraint) const {
4550 unsigned S = Constraint.size();
4551
4552 if (S == 1) {
4553 switch (Constraint[0]) {
4554 default: break;
4555 case 'r':
4556 return C_RegisterClass;
4557 case 'm': // memory
4558 case 'o': // offsetable
4559 case 'V': // not offsetable
4560 return C_Memory;
4561 case 'n': // Simple Integer
4562 case 'E': // Floating Point Constant
4563 case 'F': // Floating Point Constant
4564 return C_Immediate;
4565 case 'i': // Simple Integer or Relocatable Constant
4566 case 's': // Relocatable Constant
4567 case 'p': // Address.
4568 case 'X': // Allow ANY value.
4569 case 'I': // Target registers.
4570 case 'J':
4571 case 'K':
4572 case 'L':
4573 case 'M':
4574 case 'N':
4575 case 'O':
4576 case 'P':
4577 case '<':
4578 case '>':
4579 return C_Other;
4580 }
4581 }
4582
4583 if (S > 1 && Constraint[0] == '{' && Constraint[S - 1] == '}') {
4584 if (S == 8 && Constraint.substr(1, 6) == "memory") // "{memory}"
4585 return C_Memory;
4586 return C_Register;
4587 }
4588 return C_Unknown;
4589}
4590
4591/// Try to replace an X constraint, which matches anything, with another that
4592/// has more specific requirements based on the type of the corresponding
4593/// operand.
4594const char *TargetLowering::LowerXConstraint(EVT ConstraintVT) const {
4595 if (ConstraintVT.isInteger())
4596 return "r";
4597 if (ConstraintVT.isFloatingPoint())
4598 return "f"; // works for many targets
4599 return nullptr;
4600}
4601
4602SDValue TargetLowering::LowerAsmOutputForConstraint(
4603 SDValue &Chain, SDValue &Flag, const SDLoc &DL,
4604 const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const {
4605 return SDValue();
4606}
4607
4608/// Lower the specified operand into the Ops vector.
4609/// If it is invalid, don't add anything to Ops.
4610void TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
4611 std::string &Constraint,
4612 std::vector<SDValue> &Ops,
4613 SelectionDAG &DAG) const {
4614
4615 if (Constraint.length() > 1) return;
16
Assuming the condition is false
17
Taking false branch
4616
4617 char ConstraintLetter = Constraint[0];
4618 switch (ConstraintLetter) {
18
Control jumps to 'case 110:' at line 4622
4619 default: break;
4620 case 'X': // Allows any operand
4621 case 'i': // Simple Integer or Relocatable Constant
4622 case 'n': // Simple Integer
4623 case 's': { // Relocatable Constant
4624
4625 ConstantSDNode *C;
4626 uint64_t Offset = 0;
4627
4628 // Match (GA) or (C) or (GA+C) or (GA-C) or ((GA+C)+C) or (((GA+C)+C)+C),
4629 // etc., since getelementpointer is variadic. We can't use
4630 // SelectionDAG::FoldSymbolOffset because it expects the GA to be accessible
4631 // while in this case the GA may be furthest from the root node which is
4632 // likely an ISD::ADD.
4633 while (true) {
31
Execution continues on line 4633
4634 if ((C = dyn_cast<ConstantSDNode>(Op)) && ConstraintLetter != 's') {
19
Calling 'dyn_cast<llvm::ConstantSDNode, llvm::SDValue>'
23
Returning from 'dyn_cast<llvm::ConstantSDNode, llvm::SDValue>'
24
Assuming 'C' is null
32
Calling 'dyn_cast<llvm::ConstantSDNode, llvm::SDValue>'
47
Returning from 'dyn_cast<llvm::ConstantSDNode, llvm::SDValue>'
48
Assuming 'C' is null
49
Assuming pointer value is null
4635 // gcc prints these as sign extended. Sign extend value to 64 bits
4636 // now; without this it would get ZExt'd later in
4637 // ScheduleDAGSDNodes::EmitNode, which is very generic.
4638 bool IsBool = C->getConstantIntValue()->getBitWidth() == 1;
4639 BooleanContent BCont = getBooleanContents(MVT::i64);
4640 ISD::NodeType ExtOpc =
4641 IsBool ? getExtendForContent(BCont) : ISD::SIGN_EXTEND;
4642 int64_t ExtVal =
4643 ExtOpc == ISD::ZERO_EXTEND ? C->getZExtValue() : C->getSExtValue();
4644 Ops.push_back(
4645 DAG.getTargetConstant(Offset + ExtVal, SDLoc(C), MVT::i64));
4646 return;
4647 }
4648 if (ConstraintLetter != 'n') {
25
Taking false branch
50
Taking false branch
4649 if (const auto *GA = dyn_cast<GlobalAddressSDNode>(Op)) {
4650 Ops.push_back(DAG.getTargetGlobalAddress(GA->getGlobal(), SDLoc(Op),
4651 GA->getValueType(0),
4652 Offset + GA->getOffset()));
4653 return;
4654 }
4655 if (const auto *BA = dyn_cast<BlockAddressSDNode>(Op)) {
4656 Ops.push_back(DAG.getTargetBlockAddress(
4657 BA->getBlockAddress(), BA->getValueType(0),
4658 Offset + BA->getOffset(), BA->getTargetFlags()));
4659 return;
4660 }
4661 if (isa<BasicBlockSDNode>(Op)) {
4662 Ops.push_back(Op);
4663 return;
4664 }
4665 }
4666 const unsigned OpCode = Op.getOpcode();
51
Calling 'SDValue::getOpcode'
4667 if (OpCode == ISD::ADD || OpCode == ISD::SUB) {
26
Assuming 'OpCode' is equal to ADD
4668 if ((C = dyn_cast<ConstantSDNode>(Op.getOperand(0))))
27
Assuming 'C' is non-null
28
Taking true branch
4669 Op = Op.getOperand(1);
29
Value assigned to 'Op.Node'
4670 // Subtraction is not commutative.
4671 else if (OpCode == ISD::ADD &&
4672 (C = dyn_cast<ConstantSDNode>(Op.getOperand(1))))
4673 Op = Op.getOperand(0);
4674 else
4675 return;
4676 Offset += (OpCode
29.1
'OpCode' is equal to ADD
29.1
'OpCode' is equal to ADD
29.1
'OpCode' is equal to ADD
== ISD::ADD ? 1 : -1) * C->getSExtValue();
30
'?' condition is true
4677 continue;
4678 }
4679 return;
4680 }
4681 break;
4682 }
4683 }
4684}
4685
4686std::pair<unsigned, const TargetRegisterClass *>
4687TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *RI,
4688 StringRef Constraint,
4689 MVT VT) const {
4690 if (Constraint.empty() || Constraint[0] != '{')
4691 return std::make_pair(0u, static_cast<TargetRegisterClass *>(nullptr));
4692 assert(*(Constraint.end() - 1) == '}' && "Not a brace enclosed constraint?")(static_cast <bool> (*(Constraint.end() - 1) == '}' &&
"Not a brace enclosed constraint?") ? void (0) : __assert_fail
("*(Constraint.end() - 1) == '}' && \"Not a brace enclosed constraint?\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 4692, __extension__
__PRETTY_FUNCTION__))
;
4693
4694 // Remove the braces from around the name.
4695 StringRef RegName(Constraint.data() + 1, Constraint.size() - 2);
4696
4697 std::pair<unsigned, const TargetRegisterClass *> R =
4698 std::make_pair(0u, static_cast<const TargetRegisterClass *>(nullptr));
4699
4700 // Figure out which register class contains this reg.
4701 for (const TargetRegisterClass *RC : RI->regclasses()) {
4702 // If none of the value types for this register class are valid, we
4703 // can't use it. For example, 64-bit reg classes on 32-bit targets.
4704 if (!isLegalRC(*RI, *RC))
4705 continue;
4706
4707 for (const MCPhysReg &PR : *RC) {
4708 if (RegName.equals_insensitive(RI->getRegAsmName(PR))) {
4709 std::pair<unsigned, const TargetRegisterClass *> S =
4710 std::make_pair(PR, RC);
4711
4712 // If this register class has the requested value type, return it,
4713 // otherwise keep searching and return the first class found
4714 // if no other is found which explicitly has the requested type.
4715 if (RI->isTypeLegalForClass(*RC, VT))
4716 return S;
4717 if (!R.second)
4718 R = S;
4719 }
4720 }
4721 }
4722
4723 return R;
4724}
4725
4726//===----------------------------------------------------------------------===//
4727// Constraint Selection.
4728
4729/// Return true of this is an input operand that is a matching constraint like
4730/// "4".
4731bool TargetLowering::AsmOperandInfo::isMatchingInputConstraint() const {
4732 assert(!ConstraintCode.empty() && "No known constraint!")(static_cast <bool> (!ConstraintCode.empty() &&
"No known constraint!") ? void (0) : __assert_fail ("!ConstraintCode.empty() && \"No known constraint!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 4732, __extension__
__PRETTY_FUNCTION__))
;
4733 return isdigit(static_cast<unsigned char>(ConstraintCode[0]));
4734}
4735
4736/// If this is an input matching constraint, this method returns the output
4737/// operand it matches.
4738unsigned TargetLowering::AsmOperandInfo::getMatchedOperand() const {
4739 assert(!ConstraintCode.empty() && "No known constraint!")(static_cast <bool> (!ConstraintCode.empty() &&
"No known constraint!") ? void (0) : __assert_fail ("!ConstraintCode.empty() && \"No known constraint!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 4739, __extension__
__PRETTY_FUNCTION__))
;
4740 return atoi(ConstraintCode.c_str());
4741}
4742
4743/// Split up the constraint string from the inline assembly value into the
4744/// specific constraints and their prefixes, and also tie in the associated
4745/// operand values.
4746/// If this returns an empty vector, and if the constraint string itself
4747/// isn't empty, there was an error parsing.
4748TargetLowering::AsmOperandInfoVector
4749TargetLowering::ParseConstraints(const DataLayout &DL,
4750 const TargetRegisterInfo *TRI,
4751 const CallBase &Call) const {
4752 /// Information about all of the constraints.
4753 AsmOperandInfoVector ConstraintOperands;
4754 const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
4755 unsigned maCount = 0; // Largest number of multiple alternative constraints.
4756
4757 // Do a prepass over the constraints, canonicalizing them, and building up the
4758 // ConstraintOperands list.
4759 unsigned ArgNo = 0; // ArgNo - The argument of the CallInst.
4760 unsigned ResNo = 0; // ResNo - The result number of the next output.
4761
4762 for (InlineAsm::ConstraintInfo &CI : IA->ParseConstraints()) {
4763 ConstraintOperands.emplace_back(std::move(CI));
4764 AsmOperandInfo &OpInfo = ConstraintOperands.back();
4765
4766 // Update multiple alternative constraint count.
4767 if (OpInfo.multipleAlternatives.size() > maCount)
4768 maCount = OpInfo.multipleAlternatives.size();
4769
4770 OpInfo.ConstraintVT = MVT::Other;
4771
4772 // Compute the value type for each operand.
4773 switch (OpInfo.Type) {
4774 case InlineAsm::isOutput:
4775 // Indirect outputs just consume an argument.
4776 if (OpInfo.isIndirect) {
4777 OpInfo.CallOperandVal = Call.getArgOperand(ArgNo);
4778 break;
4779 }
4780
4781 // The return value of the call is this value. As such, there is no
4782 // corresponding argument.
4783 assert(!Call.getType()->isVoidTy() && "Bad inline asm!")(static_cast <bool> (!Call.getType()->isVoidTy() &&
"Bad inline asm!") ? void (0) : __assert_fail ("!Call.getType()->isVoidTy() && \"Bad inline asm!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 4783, __extension__
__PRETTY_FUNCTION__))
;
4784 if (StructType *STy = dyn_cast<StructType>(Call.getType())) {
4785 OpInfo.ConstraintVT =
4786 getSimpleValueType(DL, STy->getElementType(ResNo));
4787 } else {
4788 assert(ResNo == 0 && "Asm only has one result!")(static_cast <bool> (ResNo == 0 && "Asm only has one result!"
) ? void (0) : __assert_fail ("ResNo == 0 && \"Asm only has one result!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 4788, __extension__
__PRETTY_FUNCTION__))
;
4789 OpInfo.ConstraintVT =
4790 getAsmOperandValueType(DL, Call.getType()).getSimpleVT();
4791 }
4792 ++ResNo;
4793 break;
4794 case InlineAsm::isInput:
4795 OpInfo.CallOperandVal = Call.getArgOperand(ArgNo);
4796 break;
4797 case InlineAsm::isClobber:
4798 // Nothing to do.
4799 break;
4800 }
4801
4802 if (OpInfo.CallOperandVal) {
4803 llvm::Type *OpTy = OpInfo.CallOperandVal->getType();
4804 if (OpInfo.isIndirect) {
4805 OpTy = Call.getAttributes().getParamElementType(ArgNo);
4806 assert(OpTy && "Indirect opernad must have elementtype attribute")(static_cast <bool> (OpTy && "Indirect opernad must have elementtype attribute"
) ? void (0) : __assert_fail ("OpTy && \"Indirect opernad must have elementtype attribute\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 4806, __extension__
__PRETTY_FUNCTION__))
;
4807 }
4808
4809 // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
4810 if (StructType *STy = dyn_cast<StructType>(OpTy))
4811 if (STy->getNumElements() == 1)
4812 OpTy = STy->getElementType(0);
4813
4814 // If OpTy is not a single value, it may be a struct/union that we
4815 // can tile with integers.
4816 if (!OpTy->isSingleValueType() && OpTy->isSized()) {
4817 unsigned BitSize = DL.getTypeSizeInBits(OpTy);
4818 switch (BitSize) {
4819 default: break;
4820 case 1:
4821 case 8:
4822 case 16:
4823 case 32:
4824 case 64:
4825 case 128:
4826 OpInfo.ConstraintVT =
4827 MVT::getVT(IntegerType::get(OpTy->getContext(), BitSize), true);
4828 break;
4829 }
4830 } else if (PointerType *PT = dyn_cast<PointerType>(OpTy)) {
4831 unsigned PtrSize = DL.getPointerSizeInBits(PT->getAddressSpace());
4832 OpInfo.ConstraintVT = MVT::getIntegerVT(PtrSize);
4833 } else {
4834 OpInfo.ConstraintVT = MVT::getVT(OpTy, true);
4835 }
4836
4837 ArgNo++;
4838 }
4839 }
4840
4841 // If we have multiple alternative constraints, select the best alternative.
4842 if (!ConstraintOperands.empty()) {
4843 if (maCount) {
4844 unsigned bestMAIndex = 0;
4845 int bestWeight = -1;
4846 // weight: -1 = invalid match, and 0 = so-so match to 5 = good match.
4847 int weight = -1;
4848 unsigned maIndex;
4849 // Compute the sums of the weights for each alternative, keeping track
4850 // of the best (highest weight) one so far.
4851 for (maIndex = 0; maIndex < maCount; ++maIndex) {
4852 int weightSum = 0;
4853 for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
4854 cIndex != eIndex; ++cIndex) {
4855 AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];
4856 if (OpInfo.Type == InlineAsm::isClobber)
4857 continue;
4858
4859 // If this is an output operand with a matching input operand,
4860 // look up the matching input. If their types mismatch, e.g. one
4861 // is an integer, the other is floating point, or their sizes are
4862 // different, flag it as an maCantMatch.
4863 if (OpInfo.hasMatchingInput()) {
4864 AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
4865 if (OpInfo.ConstraintVT != Input.ConstraintVT) {
4866 if ((OpInfo.ConstraintVT.isInteger() !=
4867 Input.ConstraintVT.isInteger()) ||
4868 (OpInfo.ConstraintVT.getSizeInBits() !=
4869 Input.ConstraintVT.getSizeInBits())) {
4870 weightSum = -1; // Can't match.
4871 break;
4872 }
4873 }
4874 }
4875 weight = getMultipleConstraintMatchWeight(OpInfo, maIndex);
4876 if (weight == -1) {
4877 weightSum = -1;
4878 break;
4879 }
4880 weightSum += weight;
4881 }
4882 // Update best.
4883 if (weightSum > bestWeight) {
4884 bestWeight = weightSum;
4885 bestMAIndex = maIndex;
4886 }
4887 }
4888
4889 // Now select chosen alternative in each constraint.
4890 for (AsmOperandInfo &cInfo : ConstraintOperands)
4891 if (cInfo.Type != InlineAsm::isClobber)
4892 cInfo.selectAlternative(bestMAIndex);
4893 }
4894 }
4895
4896 // Check and hook up tied operands, choose constraint code to use.
4897 for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
4898 cIndex != eIndex; ++cIndex) {
4899 AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];
4900
4901 // If this is an output operand with a matching input operand, look up the
4902 // matching input. If their types mismatch, e.g. one is an integer, the
4903 // other is floating point, or their sizes are different, flag it as an
4904 // error.
4905 if (OpInfo.hasMatchingInput()) {
4906 AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
4907
4908 if (OpInfo.ConstraintVT != Input.ConstraintVT) {
4909 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
4910 getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,
4911 OpInfo.ConstraintVT);
4912 std::pair<unsigned, const TargetRegisterClass *> InputRC =
4913 getRegForInlineAsmConstraint(TRI, Input.ConstraintCode,
4914 Input.ConstraintVT);
4915 if ((OpInfo.ConstraintVT.isInteger() !=
4916 Input.ConstraintVT.isInteger()) ||
4917 (MatchRC.second != InputRC.second)) {
4918 report_fatal_error("Unsupported asm: input constraint"
4919 " with a matching output constraint of"
4920 " incompatible type!");
4921 }
4922 }
4923 }
4924 }
4925
4926 return ConstraintOperands;
4927}
4928
4929/// Return an integer indicating how general CT is.
4930static unsigned getConstraintGenerality(TargetLowering::ConstraintType CT) {
4931 switch (CT) {
4932 case TargetLowering::C_Immediate:
4933 case TargetLowering::C_Other:
4934 case TargetLowering::C_Unknown:
4935 return 0;
4936 case TargetLowering::C_Register:
4937 return 1;
4938 case TargetLowering::C_RegisterClass:
4939 return 2;
4940 case TargetLowering::C_Memory:
4941 return 3;
4942 }
4943 llvm_unreachable("Invalid constraint type")::llvm::llvm_unreachable_internal("Invalid constraint type", "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4943)
;
4944}
4945
4946/// Examine constraint type and operand type and determine a weight value.
4947/// This object must already have been set up with the operand type
4948/// and the current alternative constraint selected.
4949TargetLowering::ConstraintWeight
4950 TargetLowering::getMultipleConstraintMatchWeight(
4951 AsmOperandInfo &info, int maIndex) const {
4952 InlineAsm::ConstraintCodeVector *rCodes;
4953 if (maIndex >= (int)info.multipleAlternatives.size())
4954 rCodes = &info.Codes;
4955 else
4956 rCodes = &info.multipleAlternatives[maIndex].Codes;
4957 ConstraintWeight BestWeight = CW_Invalid;
4958
4959 // Loop over the options, keeping track of the most general one.
4960 for (const std::string &rCode : *rCodes) {
4961 ConstraintWeight weight =
4962 getSingleConstraintMatchWeight(info, rCode.c_str());
4963 if (weight > BestWeight)
4964 BestWeight = weight;
4965 }
4966
4967 return BestWeight;
4968}
4969
4970/// Examine constraint type and operand type and determine a weight value.
4971/// This object must already have been set up with the operand type
4972/// and the current alternative constraint selected.
4973TargetLowering::ConstraintWeight
4974 TargetLowering::getSingleConstraintMatchWeight(
4975 AsmOperandInfo &info, const char *constraint) const {
4976 ConstraintWeight weight = CW_Invalid;
4977 Value *CallOperandVal = info.CallOperandVal;
4978 // If we don't have a value, we can't do a match,
4979 // but allow it at the lowest weight.
4980 if (!CallOperandVal)
4981 return CW_Default;
4982 // Look at the constraint type.
4983 switch (*constraint) {
4984 case 'i': // immediate integer.
4985 case 'n': // immediate integer with a known value.
4986 if (isa<ConstantInt>(CallOperandVal))
4987 weight = CW_Constant;
4988 break;
4989 case 's': // non-explicit intregal immediate.
4990 if (isa<GlobalValue>(CallOperandVal))
4991 weight = CW_Constant;
4992 break;
4993 case 'E': // immediate float if host format.
4994 case 'F': // immediate float.
4995 if (isa<ConstantFP>(CallOperandVal))
4996 weight = CW_Constant;
4997 break;
4998 case '<': // memory operand with autodecrement.
4999 case '>': // memory operand with autoincrement.
5000 case 'm': // memory operand.
5001 case 'o': // offsettable memory operand
5002 case 'V': // non-offsettable memory operand
5003 weight = CW_Memory;
5004 break;
5005 case 'r': // general register.
5006 case 'g': // general register, memory operand or immediate integer.
5007 // note: Clang converts "g" to "imr".
5008 if (CallOperandVal->getType()->isIntegerTy())
5009 weight = CW_Register;
5010 break;
5011 case 'X': // any operand.
5012 default:
5013 weight = CW_Default;
5014 break;
5015 }
5016 return weight;
5017}
5018
5019/// If there are multiple different constraints that we could pick for this
5020/// operand (e.g. "imr") try to pick the 'best' one.
5021/// This is somewhat tricky: constraints fall into four classes:
5022/// Other -> immediates and magic values
5023/// Register -> one specific register
5024/// RegisterClass -> a group of regs
5025/// Memory -> memory
5026/// Ideally, we would pick the most specific constraint possible: if we have
5027/// something that fits into a register, we would pick it. The problem here
5028/// is that if we have something that could either be in a register or in
5029/// memory that use of the register could cause selection of *other*
5030/// operands to fail: they might only succeed if we pick memory. Because of
5031/// this the heuristic we use is:
5032///
5033/// 1) If there is an 'other' constraint, and if the operand is valid for
5034/// that constraint, use it. This makes us take advantage of 'i'
5035/// constraints when available.
5036/// 2) Otherwise, pick the most general constraint present. This prefers
5037/// 'm' over 'r', for example.
5038///
5039static void ChooseConstraint(TargetLowering::AsmOperandInfo &OpInfo,
5040 const TargetLowering &TLI,
5041 SDValue Op, SelectionDAG *DAG) {
5042 assert(OpInfo.Codes.size() > 1 && "Doesn't have multiple constraint options")(static_cast <bool> (OpInfo.Codes.size() > 1 &&
"Doesn't have multiple constraint options") ? void (0) : __assert_fail
("OpInfo.Codes.size() > 1 && \"Doesn't have multiple constraint options\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 5042, __extension__
__PRETTY_FUNCTION__))
;
6
Assuming the condition is true
7
'?' condition is true
5043 unsigned BestIdx = 0;
5044 TargetLowering::ConstraintType BestType = TargetLowering::C_Unknown;
5045 int BestGenerality = -1;
5046
5047 // Loop over the options, keeping track of the most general one.
5048 for (unsigned i = 0, e = OpInfo.Codes.size(); i != e; ++i) {
8
Assuming 'i' is not equal to 'e'
9
Loop condition is true. Entering loop body
5049 TargetLowering::ConstraintType CType =
5050 TLI.getConstraintType(OpInfo.Codes[i]);
5051
5052 // Indirect 'other' or 'immediate' constraints are not allowed.
5053 if (OpInfo.isIndirect && !(CType == TargetLowering::C_Memory ||
10
Assuming field 'isIndirect' is false
5054 CType == TargetLowering::C_Register ||
5055 CType == TargetLowering::C_RegisterClass))
5056 continue;
5057
5058 // If this is an 'other' or 'immediate' constraint, see if the operand is
5059 // valid for it. For example, on X86 we might have an 'rI' constraint. If
5060 // the operand is an integer in the range [0..31] we want to use I (saving a
5061 // load of a register), otherwise we must use 'r'.
5062 if ((CType
10.1
'CType' is equal to C_Other
10.1
'CType' is equal to C_Other
10.1
'CType' is equal to C_Other
== TargetLowering::C_Other ||
5063 CType == TargetLowering::C_Immediate) && Op.getNode()) {
11
Assuming the condition is true
5064 assert(OpInfo.Codes[i].size() == 1 &&(static_cast <bool> (OpInfo.Codes[i].size() == 1 &&
"Unhandled multi-letter 'other' constraint") ? void (0) : __assert_fail
("OpInfo.Codes[i].size() == 1 && \"Unhandled multi-letter 'other' constraint\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 5065, __extension__
__PRETTY_FUNCTION__))
12
Taking true branch
13
Assuming the condition is true
14
'?' condition is true
5065 "Unhandled multi-letter 'other' constraint")(static_cast <bool> (OpInfo.Codes[i].size() == 1 &&
"Unhandled multi-letter 'other' constraint") ? void (0) : __assert_fail
("OpInfo.Codes[i].size() == 1 && \"Unhandled multi-letter 'other' constraint\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 5065, __extension__
__PRETTY_FUNCTION__))
;
5066 std::vector<SDValue> ResultOps;
5067 TLI.LowerAsmOperandForConstraint(Op, OpInfo.Codes[i],
15
Calling 'TargetLowering::LowerAsmOperandForConstraint'
5068 ResultOps, *DAG);
5069 if (!ResultOps.empty()) {
5070 BestType = CType;
5071 BestIdx = i;
5072 break;
5073 }
5074 }
5075
5076 // Things with matching constraints can only be registers, per gcc
5077 // documentation. This mainly affects "g" constraints.
5078 if (CType == TargetLowering::C_Memory && OpInfo.hasMatchingInput())
5079 continue;
5080
5081 // This constraint letter is more general than the previous one, use it.
5082 int Generality = getConstraintGenerality(CType);
5083 if (Generality > BestGenerality) {
5084 BestType = CType;
5085 BestIdx = i;
5086 BestGenerality = Generality;
5087 }
5088 }
5089
5090 OpInfo.ConstraintCode = OpInfo.Codes[BestIdx];
5091 OpInfo.ConstraintType = BestType;
5092}
5093
5094/// Determines the constraint code and constraint type to use for the specific
5095/// AsmOperandInfo, setting OpInfo.ConstraintCode and OpInfo.ConstraintType.
5096void TargetLowering::ComputeConstraintToUse(AsmOperandInfo &OpInfo,
5097 SDValue Op,
5098 SelectionDAG *DAG) const {
5099 assert(!OpInfo.Codes.empty() && "Must have at least one constraint")(static_cast <bool> (!OpInfo.Codes.empty() && "Must have at least one constraint"
) ? void (0) : __assert_fail ("!OpInfo.Codes.empty() && \"Must have at least one constraint\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 5099, __extension__
__PRETTY_FUNCTION__))
;
1
Assuming the condition is true
2
'?' condition is true
5100
5101 // Single-letter constraints ('r') are very common.
5102 if (OpInfo.Codes.size() == 1) {
3
Assuming the condition is false
4
Taking false branch
5103 OpInfo.ConstraintCode = OpInfo.Codes[0];
5104 OpInfo.ConstraintType = getConstraintType(OpInfo.ConstraintCode);
5105 } else {
5106 ChooseConstraint(OpInfo, *this, Op, DAG);
5
Calling 'ChooseConstraint'
5107 }
5108
5109 // 'X' matches anything.
5110 if (OpInfo.ConstraintCode == "X" && OpInfo.CallOperandVal) {
5111 // Constants are handled elsewhere. For Functions, the type here is the
5112 // type of the result, which is not what we want to look at; leave them
5113 // alone.
5114 Value *v = OpInfo.CallOperandVal;
5115 if (isa<ConstantInt>(v) || isa<Function>(v)) {
5116 return;
5117 }
5118
5119 if (isa<BasicBlock>(v) || isa<BlockAddress>(v)) {
5120 OpInfo.ConstraintCode = "i";
5121 return;
5122 }
5123
5124 // Otherwise, try to resolve it to something we know about by looking at
5125 // the actual operand type.
5126 if (const char *Repl = LowerXConstraint(OpInfo.ConstraintVT)) {
5127 OpInfo.ConstraintCode = Repl;
5128 OpInfo.ConstraintType = getConstraintType(OpInfo.ConstraintCode);
5129 }
5130 }
5131}
5132
5133/// Given an exact SDIV by a constant, create a multiplication
5134/// with the multiplicative inverse of the constant.
5135static SDValue BuildExactSDIV(const TargetLowering &TLI, SDNode *N,
5136 const SDLoc &dl, SelectionDAG &DAG,
5137 SmallVectorImpl<SDNode *> &Created) {
5138 SDValue Op0 = N->getOperand(0);
5139 SDValue Op1 = N->getOperand(1);
5140 EVT VT = N->getValueType(0);
5141 EVT SVT = VT.getScalarType();
5142 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
5143 EVT ShSVT = ShVT.getScalarType();
5144
5145 bool UseSRA = false;
5146 SmallVector<SDValue, 16> Shifts, Factors;
5147
5148 auto BuildSDIVPattern = [&](ConstantSDNode *C) {
5149 if (C->isZero())
5150 return false;
5151 APInt Divisor = C->getAPIntValue();
5152 unsigned Shift = Divisor.countTrailingZeros();
5153 if (Shift) {
5154 Divisor.ashrInPlace(Shift);
5155 UseSRA = true;
5156 }
5157 // Calculate the multiplicative inverse, using Newton's method.
5158 APInt t;
5159 APInt Factor = Divisor;
5160 while ((t = Divisor * Factor) != 1)
5161 Factor *= APInt(Divisor.getBitWidth(), 2) - t;
5162 Shifts.push_back(DAG.getConstant(Shift, dl, ShSVT));
5163 Factors.push_back(DAG.getConstant(Factor, dl, SVT));
5164 return true;
5165 };
5166
5167 // Collect all magic values from the build vector.
5168 if (!ISD::matchUnaryPredicate(Op1, BuildSDIVPattern))
5169 return SDValue();
5170
5171 SDValue Shift, Factor;
5172 if (Op1.getOpcode() == ISD::BUILD_VECTOR) {
5173 Shift = DAG.getBuildVector(ShVT, dl, Shifts);
5174 Factor = DAG.getBuildVector(VT, dl, Factors);
5175 } else if (Op1.getOpcode() == ISD::SPLAT_VECTOR) {
5176 assert(Shifts.size() == 1 && Factors.size() == 1 &&(static_cast <bool> (Shifts.size() == 1 && Factors
.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("Shifts.size() == 1 && Factors.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 5178, __extension__
__PRETTY_FUNCTION__))
5177 "Expected matchUnaryPredicate to return one element for scalable "(static_cast <bool> (Shifts.size() == 1 && Factors
.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("Shifts.size() == 1 && Factors.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 5178, __extension__
__PRETTY_FUNCTION__))
5178 "vectors")(static_cast <bool> (Shifts.size() == 1 && Factors
.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("Shifts.size() == 1 && Factors.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 5178, __extension__
__PRETTY_FUNCTION__))
;
5179 Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]);
5180 Factor = DAG.getSplatVector(VT, dl, Factors[0]);
5181 } else {
5182 assert(isa<ConstantSDNode>(Op1) && "Expected a constant")(static_cast <bool> (isa<ConstantSDNode>(Op1) &&
"Expected a constant") ? void (0) : __assert_fail ("isa<ConstantSDNode>(Op1) && \"Expected a constant\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 5182, __extension__
__PRETTY_FUNCTION__))
;
5183 Shift = Shifts[0];
5184 Factor = Factors[0];
5185 }
5186
5187 SDValue Res = Op0;
5188
5189 // Shift the value upfront if it is even, so the LSB is one.
5190 if (UseSRA) {
5191 // TODO: For UDIV use SRL instead of SRA.
5192 SDNodeFlags Flags;
5193 Flags.setExact(true);
5194 Res = DAG.getNode(ISD::SRA, dl, VT, Res, Shift, Flags);
5195 Created.push_back(Res.getNode());
5196 }
5197
5198 return DAG.getNode(ISD::MUL, dl, VT, Res, Factor);
5199}
5200
5201SDValue TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
5202 SelectionDAG &DAG,
5203 SmallVectorImpl<SDNode *> &Created) const {
5204 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
5205 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5206 if (TLI.isIntDivCheap(N->getValueType(0), Attr))
5207 return SDValue(N, 0); // Lower SDIV as SDIV
5208 return SDValue();
5209}
5210
5211/// Given an ISD::SDIV node expressing a divide by constant,
5212/// return a DAG expression to select that will generate the same value by
5213/// multiplying by a magic number.
5214/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
5215SDValue TargetLowering::BuildSDIV(SDNode *N, SelectionDAG &DAG,
5216 bool IsAfterLegalization,
5217 SmallVectorImpl<SDNode *> &Created) const {
5218 SDLoc dl(N);
5219 EVT VT = N->getValueType(0);
5220 EVT SVT = VT.getScalarType();
5221 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
5222 EVT ShSVT = ShVT.getScalarType();
5223 unsigned EltBits = VT.getScalarSizeInBits();
5224 EVT MulVT;
5225
5226 // Check to see if we can do this.
5227 // FIXME: We should be more aggressive here.
5228 if (!isTypeLegal(VT)) {
5229 // Limit this to simple scalars for now.
5230 if (VT.isVector() || !VT.isSimple())
5231 return SDValue();
5232
5233 // If this type will be promoted to a large enough type with a legal
5234 // multiply operation, we can go ahead and do this transform.
5235 if (getTypeAction(VT.getSimpleVT()) != TypePromoteInteger)
5236 return SDValue();
5237
5238 MulVT = getTypeToTransformTo(*DAG.getContext(), VT);
5239 if (MulVT.getSizeInBits() < (2 * EltBits) ||
5240 !isOperationLegal(ISD::MUL, MulVT))
5241 return SDValue();
5242 }
5243
5244 // If the sdiv has an 'exact' bit we can use a simpler lowering.
5245 if (N->getFlags().hasExact())
5246 return BuildExactSDIV(*this, N, dl, DAG, Created);
5247
5248 SmallVector<SDValue, 16> MagicFactors, Factors, Shifts, ShiftMasks;
5249
5250 auto BuildSDIVPattern = [&](ConstantSDNode *C) {
5251 if (C->isZero())
5252 return false;
5253
5254 const APInt &Divisor = C->getAPIntValue();
5255 SignedDivisionByConstantInfo magics = SignedDivisionByConstantInfo::get(Divisor);
5256 int NumeratorFactor = 0;
5257 int ShiftMask = -1;
5258
5259 if (Divisor.isOne() || Divisor.isAllOnes()) {
5260 // If d is +1/-1, we just multiply the numerator by +1/-1.
5261 NumeratorFactor = Divisor.getSExtValue();
5262 magics.Magic = 0;
5263 magics.ShiftAmount = 0;
5264 ShiftMask = 0;
5265 } else if (Divisor.isStrictlyPositive() && magics.Magic.isNegative()) {
5266 // If d > 0 and m < 0, add the numerator.
5267 NumeratorFactor = 1;
5268 } else if (Divisor.isNegative() && magics.Magic.isStrictlyPositive()) {
5269 // If d < 0 and m > 0, subtract the numerator.
5270 NumeratorFactor = -1;
5271 }
5272
5273 MagicFactors.push_back(DAG.getConstant(magics.Magic, dl, SVT));
5274 Factors.push_back(DAG.getConstant(NumeratorFactor, dl, SVT));
5275 Shifts.push_back(DAG.getConstant(magics.ShiftAmount, dl, ShSVT));
5276 ShiftMasks.push_back(DAG.getConstant(ShiftMask, dl, SVT));
5277 return true;
5278 };
5279
5280 SDValue N0 = N->getOperand(0);
5281 SDValue N1 = N->getOperand(1);
5282
5283 // Collect the shifts / magic values from each element.
5284 if (!ISD::matchUnaryPredicate(N1, BuildSDIVPattern))
5285 return SDValue();
5286
5287 SDValue MagicFactor, Factor, Shift, ShiftMask;
5288 if (N1.getOpcode() == ISD::BUILD_VECTOR) {
5289 MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);
5290 Factor = DAG.getBuildVector(VT, dl, Factors);
5291 Shift = DAG.getBuildVector(ShVT, dl, Shifts);
5292 ShiftMask = DAG.getBuildVector(VT, dl, ShiftMasks);
5293 } else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {
5294 assert(MagicFactors.size() == 1 && Factors.size() == 1 &&(static_cast <bool> (MagicFactors.size() == 1 &&
Factors.size() == 1 && Shifts.size() == 1 &&
ShiftMasks.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("MagicFactors.size() == 1 && Factors.size() == 1 && Shifts.size() == 1 && ShiftMasks.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 5297, __extension__
__PRETTY_FUNCTION__))
5295 Shifts.size() == 1 && ShiftMasks.size() == 1 &&(static_cast <bool> (MagicFactors.size() == 1 &&
Factors.size() == 1 && Shifts.size() == 1 &&
ShiftMasks.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("MagicFactors.size() == 1 && Factors.size() == 1 && Shifts.size() == 1 && ShiftMasks.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 5297, __extension__
__PRETTY_FUNCTION__))
5296 "Expected matchUnaryPredicate to return one element for scalable "(static_cast <bool> (MagicFactors.size() == 1 &&
Factors.size() == 1 && Shifts.size() == 1 &&
ShiftMasks.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("MagicFactors.size() == 1 && Factors.size() == 1 && Shifts.size() == 1 && ShiftMasks.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 5297, __extension__
__PRETTY_FUNCTION__))
5297 "vectors")(static_cast <bool> (MagicFactors.size() == 1 &&
Factors.size() == 1 && Shifts.size() == 1 &&
ShiftMasks.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("MagicFactors.size() == 1 && Factors.size() == 1 && Shifts.size() == 1 && ShiftMasks.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 5297, __extension__
__PRETTY_FUNCTION__))
;
5298 MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]);
5299 Factor = DAG.getSplatVector(VT, dl, Factors[0]);
5300 Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]);
5301 ShiftMask = DAG.getSplatVector(VT, dl, ShiftMasks[0]);
5302 } else {
5303 assert(isa<ConstantSDNode>(N1) && "Expected a constant")(static_cast <bool> (isa<ConstantSDNode>(N1) &&
"Expected a constant") ? void (0) : __assert_fail ("isa<ConstantSDNode>(N1) && \"Expected a constant\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 5303, __extension__
__PRETTY_FUNCTION__))
;
5304 MagicFactor = MagicFactors[0];
5305 Factor = Factors[0];
5306 Shift = Shifts[0];
5307 ShiftMask = ShiftMasks[0];
5308 }
5309
5310 // Multiply the numerator (operand 0) by the magic value.
5311 // FIXME: We should support doing a MUL in a wider type.
5312 auto GetMULHS = [&](SDValue X, SDValue Y) {
5313 // If the type isn't legal, use a wider mul of the the type calculated
5314 // earlier.
5315 if (!isTypeLegal(VT)) {
5316 X = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, X);
5317 Y = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, Y);
5318 Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y);
5319 Y = DAG.getNode(ISD::SRL, dl, MulVT, Y,
5320 DAG.getShiftAmountConstant(EltBits, MulVT, dl));
5321 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);
5322 }
5323
5324 if (isOperationLegalOrCustom(ISD::MULHS, VT, IsAfterLegalization))
5325 return DAG.getNode(ISD::MULHS, dl, VT, X, Y);
5326 if (isOperationLegalOrCustom(ISD::SMUL_LOHI, VT, IsAfterLegalization)) {
5327 SDValue LoHi =
5328 DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);
5329 return SDValue(LoHi.getNode(), 1);
5330 }
5331 return SDValue();
5332 };
5333
5334 SDValue Q = GetMULHS(N0, MagicFactor);
5335 if (!Q)
5336 return SDValue();
5337
5338 Created.push_back(Q.getNode());
5339
5340 // (Optionally) Add/subtract the numerator using Factor.
5341 Factor = DAG.getNode(ISD::MUL, dl, VT, N0, Factor);
5342 Created.push_back(Factor.getNode());
5343 Q = DAG.getNode(ISD::ADD, dl, VT, Q, Factor);
5344 Created.push_back(Q.getNode());
5345
5346 // Shift right algebraic by shift value.
5347 Q = DAG.getNode(ISD::SRA, dl, VT, Q, Shift);
5348 Created.push_back(Q.getNode());
5349
5350 // Extract the sign bit, mask it and add it to the quotient.
5351 SDValue SignShift = DAG.getConstant(EltBits - 1, dl, ShVT);
5352 SDValue T = DAG.getNode(ISD::SRL, dl, VT, Q, SignShift);
5353 Created.push_back(T.getNode());
5354 T = DAG.getNode(ISD::AND, dl, VT, T, ShiftMask);
5355 Created.push_back(T.getNode());
5356 return DAG.getNode(ISD::ADD, dl, VT, Q, T);
5357}
5358
5359/// Given an ISD::UDIV node expressing a divide by constant,
5360/// return a DAG expression to select that will generate the same value by
5361/// multiplying by a magic number.
5362/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
5363SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG,
5364 bool IsAfterLegalization,
5365 SmallVectorImpl<SDNode *> &Created) const {
5366 SDLoc dl(N);
5367 EVT VT = N->getValueType(0);
5368 EVT SVT = VT.getScalarType();
5369 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
5370 EVT ShSVT = ShVT.getScalarType();
5371 unsigned EltBits = VT.getScalarSizeInBits();
5372 EVT MulVT;
5373
5374 // Check to see if we can do this.
5375 // FIXME: We should be more aggressive here.
5376 if (!isTypeLegal(VT)) {
5377 // Limit this to simple scalars for now.
5378 if (VT.isVector() || !VT.isSimple())
5379 return SDValue();
5380
5381 // If this type will be promoted to a large enough type with a legal
5382 // multiply operation, we can go ahead and do this transform.
5383 if (getTypeAction(VT.getSimpleVT()) != TypePromoteInteger)
5384 return SDValue();
5385
5386 MulVT = getTypeToTransformTo(*DAG.getContext(), VT);
5387 if (MulVT.getSizeInBits() < (2 * EltBits) ||
5388 !isOperationLegal(ISD::MUL, MulVT))
5389 return SDValue();
5390 }
5391
5392 bool UseNPQ = false;
5393 SmallVector<SDValue, 16> PreShifts, PostShifts, MagicFactors, NPQFactors;
5394
5395 auto BuildUDIVPattern = [&](ConstantSDNode *C) {
5396 if (C->isZero())
5397 return false;
5398 // FIXME: We should use a narrower constant when the upper
5399 // bits are known to be zero.
5400 const APInt& Divisor = C->getAPIntValue();
5401 UnsignedDivisonByConstantInfo magics = UnsignedDivisonByConstantInfo::get(Divisor);
5402 unsigned PreShift = 0, PostShift = 0;
5403
5404 // If the divisor is even, we can avoid using the expensive fixup by
5405 // shifting the divided value upfront.
5406 if (magics.IsAdd != 0 && !Divisor[0]) {
5407 PreShift = Divisor.countTrailingZeros();
5408 // Get magic number for the shifted divisor.
5409 magics = UnsignedDivisonByConstantInfo::get(Divisor.lshr(PreShift), PreShift);
5410 assert(magics.IsAdd == 0 && "Should use cheap fixup now")(static_cast <bool> (magics.IsAdd == 0 && "Should use cheap fixup now"
) ? void (0) : __assert_fail ("magics.IsAdd == 0 && \"Should use cheap fixup now\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 5410, __extension__
__PRETTY_FUNCTION__))
;
5411 }
5412
5413 APInt Magic = magics.Magic;
5414
5415 unsigned SelNPQ;
5416 if (magics.IsAdd == 0 || Divisor.isOne()) {
5417 assert(magics.ShiftAmount < Divisor.getBitWidth() &&(static_cast <bool> (magics.ShiftAmount < Divisor.getBitWidth
() && "We shouldn't generate an undefined shift!") ? void
(0) : __assert_fail ("magics.ShiftAmount < Divisor.getBitWidth() && \"We shouldn't generate an undefined shift!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 5418, __extension__
__PRETTY_FUNCTION__))
5418 "We shouldn't generate an undefined shift!")(static_cast <bool> (magics.ShiftAmount < Divisor.getBitWidth
() && "We shouldn't generate an undefined shift!") ? void
(0) : __assert_fail ("magics.ShiftAmount < Divisor.getBitWidth() && \"We shouldn't generate an undefined shift!\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 5418, __extension__
__PRETTY_FUNCTION__))
;
5419 PostShift = magics.ShiftAmount;
5420 SelNPQ = false;
5421 } else {
5422 PostShift = magics.ShiftAmount - 1;
5423 SelNPQ = true;
5424 }
5425
5426 PreShifts.push_back(DAG.getConstant(PreShift, dl, ShSVT));
5427 MagicFactors.push_back(DAG.getConstant(Magic, dl, SVT));
5428 NPQFactors.push_back(
5429 DAG.getConstant(SelNPQ ? APInt::getOneBitSet(EltBits, EltBits - 1)
5430 : APInt::getZero(EltBits),
5431 dl, SVT));
5432 PostShifts.push_back(DAG.getConstant(PostShift, dl, ShSVT));
5433 UseNPQ |= SelNPQ;
5434 return true;
5435 };
5436
5437 SDValue N0 = N->getOperand(0);
5438 SDValue N1 = N->getOperand(1);
5439
5440 // Collect the shifts/magic values from each element.
5441 if (!ISD::matchUnaryPredicate(N1, BuildUDIVPattern))
5442 return SDValue();
5443
5444 SDValue PreShift, PostShift, MagicFactor, NPQFactor;
5445 if (N1.getOpcode() == ISD::BUILD_VECTOR) {
5446 PreShift = DAG.getBuildVector(ShVT, dl, PreShifts);
5447 MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);
5448 NPQFactor = DAG.getBuildVector(VT, dl, NPQFactors);
5449 PostShift = DAG.getBuildVector(ShVT, dl, PostShifts);
5450 } else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {
5451 assert(PreShifts.size() == 1 && MagicFactors.size() == 1 &&(static_cast <bool> (PreShifts.size() == 1 && MagicFactors
.size() == 1 && NPQFactors.size() == 1 && PostShifts
.size() == 1 && "Expected matchUnaryPredicate to return one for scalable vectors"
) ? void (0) : __assert_fail ("PreShifts.size() == 1 && MagicFactors.size() == 1 && NPQFactors.size() == 1 && PostShifts.size() == 1 && \"Expected matchUnaryPredicate to return one for scalable vectors\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 5453, __extension__
__PRETTY_FUNCTION__))
5452 NPQFactors.size() == 1 && PostShifts.size() == 1 &&(static_cast <bool> (PreShifts.size() == 1 && MagicFactors
.size() == 1 && NPQFactors.size() == 1 && PostShifts
.size() == 1 && "Expected matchUnaryPredicate to return one for scalable vectors"
) ? void (0) : __assert_fail ("PreShifts.size() == 1 && MagicFactors.size() == 1 && NPQFactors.size() == 1 && PostShifts.size() == 1 && \"Expected matchUnaryPredicate to return one for scalable vectors\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 5453, __extension__
__PRETTY_FUNCTION__))
5453 "Expected matchUnaryPredicate to return one for scalable vectors")(static_cast <bool> (PreShifts.size() == 1 && MagicFactors
.size() == 1 && NPQFactors.size() == 1 && PostShifts
.size() == 1 && "Expected matchUnaryPredicate to return one for scalable vectors"
) ? void (0) : __assert_fail ("PreShifts.size() == 1 && MagicFactors.size() == 1 && NPQFactors.size() == 1 && PostShifts.size() == 1 && \"Expected matchUnaryPredicate to return one for scalable vectors\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 5453, __extension__
__PRETTY_FUNCTION__))
;
5454 PreShift = DAG.getSplatVector(ShVT, dl, PreShifts[0]);
5455 MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]);
5456 NPQFactor = DAG.getSplatVector(VT, dl, NPQFactors[0]);
5457 PostShift = DAG.getSplatVector(ShVT, dl, PostShifts[0]);
5458 } else {
5459 assert(isa<ConstantSDNode>(N1) && "Expected a constant")(static_cast <bool> (isa<ConstantSDNode>(N1) &&
"Expected a constant") ? void (0) : __assert_fail ("isa<ConstantSDNode>(N1) && \"Expected a constant\""
, "llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp", 5459, __extension__
__PRETTY_FUNCTION__))
;
5460 PreShift = PreShifts[0];
5461 MagicFactor = MagicFactors[0];
5462 PostShift = PostShifts[0];
5463 }
5464
5465 SDValue Q = N0;
5466 Q = DAG.getNode(ISD::SRL, dl, VT, Q, PreShift);
5467 Created.push_back(Q.getNode());
5468
5469 // FIXME: We should support doing a MUL in a wider type.
5470 auto GetMULHU = [&](SDValue X, SDValue Y) {
5471 // If the type isn't legal, use a wider mul of the the type calculated
5472 // earlier.
5473 if (!isTypeLegal(VT)) {
5474 X = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, X);
5475 Y = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, Y);
5476 Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y);
5477 Y = DAG.getNode(ISD::SRL, dl, MulVT, Y,
5478 DAG.getShiftAmountConstant(EltBits, MulVT, dl));
5479 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);
5480 }
5481
5482 if (isOperationLegalOrCustom(ISD::MULHU, VT, IsAfterLegalization))
5483 return DAG.getNode(ISD::MULHU, dl, VT, X, Y);
5484 if (isOperationLegalOrCustom(ISD::UMUL_LOHI, VT, IsAfterLegalization)) {
5485 SDValue LoHi =
5486 DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);
5487 return SDValue(LoHi.getNode(), 1);
5488 }
5489 return SDValue(); // No mulhu or equivalent
5490 };
5491
5492 // Multiply the numerator (operand 0) by the magic value.
5493 Q = GetMULHU(Q, MagicFactor);
5494 if (!Q)
5495 return SDValue();
5496
5497 Created.push_back(Q.getNode());
5498
5499 if (UseNPQ) {
5500 SDValue NPQ = DAG.getNode(ISD::SUB, dl, VT, N0, Q);
5501 Created.push_back(NPQ.getNode());
5502
5503 // For vectors we might have a mix of non-NPQ/NPQ paths, so use
5504 // MULHU to act as a SRL-by-1 for NPQ, else multiply by zero.
5505 if (VT.isVector())
5506 NPQ = GetMULHU(NPQ, NPQFactor);
5507 else
5508 NPQ = DAG.getNode(ISD::SRL, dl, VT, NPQ, DAG.getConstant(1, dl, ShVT));
5509
5510 Created.push_back(NPQ.getNode());
5511
5512 Q = DAG.getNode(ISD::ADD, dl, VT, NPQ, Q);
5513 Created.push_back(Q.getNode());
5514 }
5515
5516 Q = DAG.getNode(ISD::SRL, dl, VT, Q, PostShift);
5517 Created.push_back(Q.getNode());
5518
5519 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
5520
5521 SDValue One = DAG.getConstant(1, dl, VT);
5522 SDValue IsOne = DAG.getSetCC(dl, SetCCVT, N1, One, ISD::SETEQ);
5523 return DAG.getSelect(dl, VT, IsOne, N0, Q);
5524}
5525
5526/// If all values in Values that *don't* match the predicate are same 'splat'
5527/// value, then replace all values with that splat value.
5528/// Else, if AlternativeReplacement was provided, then replace all values that
5529/// do match predicate with AlternativeReplacement value.
5530static void
5531turnVectorIntoSplatVector(MutableArrayRef<SDValue> Values,
5532 std::function<bool(SDValue)> Predicate,
5533 SDValue AlternativeReplacement = SDValue()) {
5534 SDValue Replacement;
5535 // Is there a value for which the Predi