Bug Summary

File:llvm/include/llvm/ADT/APInt.h
Warning:line 449, column 36
The result of the left shift is undefined due to shifting by '4294967295', which is greater or equal to the width of type 'llvm::APInt::WordType'

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name TargetLowering.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -mframe-pointer=none -fmath-errno -fno-rounding-math -masm-verbose -mconstructor-aliases -munwind-tables -target-cpu x86-64 -dwarf-column-info -fno-split-dwarf-inlining -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-11/lib/clang/11.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/build-llvm/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/build-llvm/include -I /build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-11/lib/clang/11.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/build-llvm/lib/CodeGen/SelectionDAG -fdebug-prefix-map=/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347=. -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2020-03-09-184146-41876-1 -x c++ /build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

1//===-- TargetLowering.cpp - Implement the TargetLowering class -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the TargetLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/CodeGen/TargetLowering.h"
14#include "llvm/ADT/STLExtras.h"
15#include "llvm/CodeGen/CallingConvLower.h"
16#include "llvm/CodeGen/MachineFrameInfo.h"
17#include "llvm/CodeGen/MachineFunction.h"
18#include "llvm/CodeGen/MachineJumpTableInfo.h"
19#include "llvm/CodeGen/MachineRegisterInfo.h"
20#include "llvm/CodeGen/SelectionDAG.h"
21#include "llvm/CodeGen/TargetRegisterInfo.h"
22#include "llvm/CodeGen/TargetSubtargetInfo.h"
23#include "llvm/IR/DataLayout.h"
24#include "llvm/IR/DerivedTypes.h"
25#include "llvm/IR/GlobalVariable.h"
26#include "llvm/IR/LLVMContext.h"
27#include "llvm/MC/MCAsmInfo.h"
28#include "llvm/MC/MCExpr.h"
29#include "llvm/Support/ErrorHandling.h"
30#include "llvm/Support/KnownBits.h"
31#include "llvm/Support/MathExtras.h"
32#include "llvm/Target/TargetLoweringObjectFile.h"
33#include "llvm/Target/TargetMachine.h"
34#include <cctype>
35using namespace llvm;
36
37/// NOTE: The TargetMachine owns TLOF.
38TargetLowering::TargetLowering(const TargetMachine &tm)
39 : TargetLoweringBase(tm) {}
40
41const char *TargetLowering::getTargetNodeName(unsigned Opcode) const {
42 return nullptr;
43}
44
45bool TargetLowering::isPositionIndependent() const {
46 return getTargetMachine().isPositionIndependent();
47}
48
49/// Check whether a given call node is in tail position within its function. If
50/// so, it sets Chain to the input chain of the tail call.
51bool TargetLowering::isInTailCallPosition(SelectionDAG &DAG, SDNode *Node,
52 SDValue &Chain) const {
53 const Function &F = DAG.getMachineFunction().getFunction();
54
55 // First, check if tail calls have been disabled in this function.
56 if (F.getFnAttribute("disable-tail-calls").getValueAsString() == "true")
57 return false;
58
59 // Conservatively require the attributes of the call to match those of
60 // the return. Ignore NoAlias and NonNull because they don't affect the
61 // call sequence.
62 AttributeList CallerAttrs = F.getAttributes();
63 if (AttrBuilder(CallerAttrs, AttributeList::ReturnIndex)
64 .removeAttribute(Attribute::NoAlias)
65 .removeAttribute(Attribute::NonNull)
66 .hasAttributes())
67 return false;
68
69 // It's not safe to eliminate the sign / zero extension of the return value.
70 if (CallerAttrs.hasAttribute(AttributeList::ReturnIndex, Attribute::ZExt) ||
71 CallerAttrs.hasAttribute(AttributeList::ReturnIndex, Attribute::SExt))
72 return false;
73
74 // Check if the only use is a function return node.
75 return isUsedByReturnOnly(Node, Chain);
76}
77
78bool TargetLowering::parametersInCSRMatch(const MachineRegisterInfo &MRI,
79 const uint32_t *CallerPreservedMask,
80 const SmallVectorImpl<CCValAssign> &ArgLocs,
81 const SmallVectorImpl<SDValue> &OutVals) const {
82 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
83 const CCValAssign &ArgLoc = ArgLocs[I];
84 if (!ArgLoc.isRegLoc())
85 continue;
86 Register Reg = ArgLoc.getLocReg();
87 // Only look at callee saved registers.
88 if (MachineOperand::clobbersPhysReg(CallerPreservedMask, Reg))
89 continue;
90 // Check that we pass the value used for the caller.
91 // (We look for a CopyFromReg reading a virtual register that is used
92 // for the function live-in value of register Reg)
93 SDValue Value = OutVals[I];
94 if (Value->getOpcode() != ISD::CopyFromReg)
95 return false;
96 unsigned ArgReg = cast<RegisterSDNode>(Value->getOperand(1))->getReg();
97 if (MRI.getLiveInPhysReg(ArgReg) != Reg)
98 return false;
99 }
100 return true;
101}
102
103/// Set CallLoweringInfo attribute flags based on a call instruction
104/// and called function attributes.
105void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call,
106 unsigned ArgIdx) {
107 IsSExt = Call->paramHasAttr(ArgIdx, Attribute::SExt);
108 IsZExt = Call->paramHasAttr(ArgIdx, Attribute::ZExt);
109 IsInReg = Call->paramHasAttr(ArgIdx, Attribute::InReg);
110 IsSRet = Call->paramHasAttr(ArgIdx, Attribute::StructRet);
111 IsNest = Call->paramHasAttr(ArgIdx, Attribute::Nest);
112 IsByVal = Call->paramHasAttr(ArgIdx, Attribute::ByVal);
113 IsInAlloca = Call->paramHasAttr(ArgIdx, Attribute::InAlloca);
114 IsReturned = Call->paramHasAttr(ArgIdx, Attribute::Returned);
115 IsSwiftSelf = Call->paramHasAttr(ArgIdx, Attribute::SwiftSelf);
116 IsSwiftError = Call->paramHasAttr(ArgIdx, Attribute::SwiftError);
117 Alignment = Call->getParamAlignment(ArgIdx);
118 ByValType = nullptr;
119 if (Call->paramHasAttr(ArgIdx, Attribute::ByVal))
120 ByValType = Call->getParamByValType(ArgIdx);
121}
122
123/// Generate a libcall taking the given operands as arguments and returning a
124/// result of type RetVT.
125std::pair<SDValue, SDValue>
126TargetLowering::makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT,
127 ArrayRef<SDValue> Ops,
128 MakeLibCallOptions CallOptions,
129 const SDLoc &dl,
130 SDValue InChain) const {
131 if (!InChain)
132 InChain = DAG.getEntryNode();
133
134 TargetLowering::ArgListTy Args;
135 Args.reserve(Ops.size());
136
137 TargetLowering::ArgListEntry Entry;
138 for (unsigned i = 0; i < Ops.size(); ++i) {
139 SDValue NewOp = Ops[i];
140 Entry.Node = NewOp;
141 Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
142 Entry.IsSExt = shouldSignExtendTypeInLibCall(NewOp.getValueType(),
143 CallOptions.IsSExt);
144 Entry.IsZExt = !Entry.IsSExt;
145
146 if (CallOptions.IsSoften &&
147 !shouldExtendTypeInLibCall(CallOptions.OpsVTBeforeSoften[i])) {
148 Entry.IsSExt = Entry.IsZExt = false;
149 }
150 Args.push_back(Entry);
151 }
152
153 if (LC == RTLIB::UNKNOWN_LIBCALL)
154 report_fatal_error("Unsupported library call operation!");
155 SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
156 getPointerTy(DAG.getDataLayout()));
157
158 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
159 TargetLowering::CallLoweringInfo CLI(DAG);
160 bool signExtend = shouldSignExtendTypeInLibCall(RetVT, CallOptions.IsSExt);
161 bool zeroExtend = !signExtend;
162
163 if (CallOptions.IsSoften &&
164 !shouldExtendTypeInLibCall(CallOptions.RetVTBeforeSoften)) {
165 signExtend = zeroExtend = false;
166 }
167
168 CLI.setDebugLoc(dl)
169 .setChain(InChain)
170 .setLibCallee(getLibcallCallingConv(LC), RetTy, Callee, std::move(Args))
171 .setNoReturn(CallOptions.DoesNotReturn)
172 .setDiscardResult(!CallOptions.IsReturnValueUsed)
173 .setIsPostTypeLegalization(CallOptions.IsPostTypeLegalization)
174 .setSExtResult(signExtend)
175 .setZExtResult(zeroExtend);
176 return LowerCallTo(CLI);
177}
178
179bool TargetLowering::findOptimalMemOpLowering(
180 std::vector<EVT> &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS,
181 unsigned SrcAS, const AttributeList &FuncAttributes) const {
182 if (Op.isMemcpyWithFixedDstAlign() && Op.getSrcAlign() < Op.getDstAlign())
183 return false;
184
185 EVT VT = getOptimalMemOpType(Op, FuncAttributes);
186
187 if (VT == MVT::Other) {
188 // Use the largest integer type whose alignment constraints are satisfied.
189 // We only need to check DstAlign here as SrcAlign is always greater or
190 // equal to DstAlign (or zero).
191 VT = MVT::i64;
192 if (Op.isFixedDstAlign())
193 while (
194 Op.getDstAlign() < (VT.getSizeInBits() / 8) &&
195 !allowsMisalignedMemoryAccesses(VT, DstAS, Op.getDstAlign().value()))
196 VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1);
197 assert(VT.isInteger())((VT.isInteger()) ? static_cast<void> (0) : __assert_fail
("VT.isInteger()", "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 197, __PRETTY_FUNCTION__))
;
198
199 // Find the largest legal integer type.
200 MVT LVT = MVT::i64;
201 while (!isTypeLegal(LVT))
202 LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
203 assert(LVT.isInteger())((LVT.isInteger()) ? static_cast<void> (0) : __assert_fail
("LVT.isInteger()", "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 203, __PRETTY_FUNCTION__))
;
204
205 // If the type we've chosen is larger than the largest legal integer type
206 // then use that instead.
207 if (VT.bitsGT(LVT))
208 VT = LVT;
209 }
210
211 unsigned NumMemOps = 0;
212 uint64_t Size = Op.size();
213 while (Size) {
214 unsigned VTSize = VT.getSizeInBits() / 8;
215 while (VTSize > Size) {
216 // For now, only use non-vector load / store's for the left-over pieces.
217 EVT NewVT = VT;
218 unsigned NewVTSize;
219
220 bool Found = false;
221 if (VT.isVector() || VT.isFloatingPoint()) {
222 NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;
223 if (isOperationLegalOrCustom(ISD::STORE, NewVT) &&
224 isSafeMemOpType(NewVT.getSimpleVT()))
225 Found = true;
226 else if (NewVT == MVT::i64 &&
227 isOperationLegalOrCustom(ISD::STORE, MVT::f64) &&
228 isSafeMemOpType(MVT::f64)) {
229 // i64 is usually not legal on 32-bit targets, but f64 may be.
230 NewVT = MVT::f64;
231 Found = true;
232 }
233 }
234
235 if (!Found) {
236 do {
237 NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
238 if (NewVT == MVT::i8)
239 break;
240 } while (!isSafeMemOpType(NewVT.getSimpleVT()));
241 }
242 NewVTSize = NewVT.getSizeInBits() / 8;
243
244 // If the new VT cannot cover all of the remaining bits, then consider
245 // issuing a (or a pair of) unaligned and overlapping load / store.
246 bool Fast;
247 if (NumMemOps && Op.allowOverlap() && NewVTSize < Size &&
248 allowsMisalignedMemoryAccesses(
249 VT, DstAS, Op.isFixedDstAlign() ? Op.getDstAlign().value() : 0,
250 MachineMemOperand::MONone, &Fast) &&
251 Fast)
252 VTSize = Size;
253 else {
254 VT = NewVT;
255 VTSize = NewVTSize;
256 }
257 }
258
259 if (++NumMemOps > Limit)
260 return false;
261
262 MemOps.push_back(VT);
263 Size -= VTSize;
264 }
265
266 return true;
267}
268
269/// Soften the operands of a comparison. This code is shared among BR_CC,
270/// SELECT_CC, and SETCC handlers.
271void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,
272 SDValue &NewLHS, SDValue &NewRHS,
273 ISD::CondCode &CCCode,
274 const SDLoc &dl, const SDValue OldLHS,
275 const SDValue OldRHS) const {
276 SDValue Chain;
277 return softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, dl, OldLHS,
278 OldRHS, Chain);
279}
280
281void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,
282 SDValue &NewLHS, SDValue &NewRHS,
283 ISD::CondCode &CCCode,
284 const SDLoc &dl, const SDValue OldLHS,
285 const SDValue OldRHS,
286 SDValue &Chain,
287 bool IsSignaling) const {
288 // FIXME: Currently we cannot really respect all IEEE predicates due to libgcc
289 // not supporting it. We can update this code when libgcc provides such
290 // functions.
291
292 assert((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128)(((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT ==
MVT::ppcf128) && "Unsupported setcc type!") ? static_cast
<void> (0) : __assert_fail ("(VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128) && \"Unsupported setcc type!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 293, __PRETTY_FUNCTION__))
293 && "Unsupported setcc type!")(((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT ==
MVT::ppcf128) && "Unsupported setcc type!") ? static_cast
<void> (0) : __assert_fail ("(VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128) && \"Unsupported setcc type!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 293, __PRETTY_FUNCTION__))
;
294
295 // Expand into one or more soft-fp libcall(s).
296 RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL;
297 bool ShouldInvertCC = false;
298 switch (CCCode) {
299 case ISD::SETEQ:
300 case ISD::SETOEQ:
301 LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
302 (VT == MVT::f64) ? RTLIB::OEQ_F64 :
303 (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
304 break;
305 case ISD::SETNE:
306 case ISD::SETUNE:
307 LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 :
308 (VT == MVT::f64) ? RTLIB::UNE_F64 :
309 (VT == MVT::f128) ? RTLIB::UNE_F128 : RTLIB::UNE_PPCF128;
310 break;
311 case ISD::SETGE:
312 case ISD::SETOGE:
313 LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
314 (VT == MVT::f64) ? RTLIB::OGE_F64 :
315 (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
316 break;
317 case ISD::SETLT:
318 case ISD::SETOLT:
319 LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
320 (VT == MVT::f64) ? RTLIB::OLT_F64 :
321 (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
322 break;
323 case ISD::SETLE:
324 case ISD::SETOLE:
325 LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
326 (VT == MVT::f64) ? RTLIB::OLE_F64 :
327 (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
328 break;
329 case ISD::SETGT:
330 case ISD::SETOGT:
331 LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
332 (VT == MVT::f64) ? RTLIB::OGT_F64 :
333 (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
334 break;
335 case ISD::SETO:
336 ShouldInvertCC = true;
337 LLVM_FALLTHROUGH[[gnu::fallthrough]];
338 case ISD::SETUO:
339 LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
340 (VT == MVT::f64) ? RTLIB::UO_F64 :
341 (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
342 break;
343 case ISD::SETONE:
344 // SETONE = O && UNE
345 ShouldInvertCC = true;
346 LLVM_FALLTHROUGH[[gnu::fallthrough]];
347 case ISD::SETUEQ:
348 LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
349 (VT == MVT::f64) ? RTLIB::UO_F64 :
350 (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
351 LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
352 (VT == MVT::f64) ? RTLIB::OEQ_F64 :
353 (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
354 break;
355 default:
356 // Invert CC for unordered comparisons
357 ShouldInvertCC = true;
358 switch (CCCode) {
359 case ISD::SETULT:
360 LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
361 (VT == MVT::f64) ? RTLIB::OGE_F64 :
362 (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
363 break;
364 case ISD::SETULE:
365 LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
366 (VT == MVT::f64) ? RTLIB::OGT_F64 :
367 (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
368 break;
369 case ISD::SETUGT:
370 LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
371 (VT == MVT::f64) ? RTLIB::OLE_F64 :
372 (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
373 break;
374 case ISD::SETUGE:
375 LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
376 (VT == MVT::f64) ? RTLIB::OLT_F64 :
377 (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
378 break;
379 default: llvm_unreachable("Do not know how to soften this setcc!")::llvm::llvm_unreachable_internal("Do not know how to soften this setcc!"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 379)
;
380 }
381 }
382
383 // Use the target specific return value for comparions lib calls.
384 EVT RetVT = getCmpLibcallReturnType();
385 SDValue Ops[2] = {NewLHS, NewRHS};
386 TargetLowering::MakeLibCallOptions CallOptions;
387 EVT OpsVT[2] = { OldLHS.getValueType(),
388 OldRHS.getValueType() };
389 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT, true);
390 auto Call = makeLibCall(DAG, LC1, RetVT, Ops, CallOptions, dl, Chain);
391 NewLHS = Call.first;
392 NewRHS = DAG.getConstant(0, dl, RetVT);
393
394 CCCode = getCmpLibcallCC(LC1);
395 if (ShouldInvertCC) {
396 assert(RetVT.isInteger())((RetVT.isInteger()) ? static_cast<void> (0) : __assert_fail
("RetVT.isInteger()", "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 396, __PRETTY_FUNCTION__))
;
397 CCCode = getSetCCInverse(CCCode, RetVT);
398 }
399
400 if (LC2 == RTLIB::UNKNOWN_LIBCALL) {
401 // Update Chain.
402 Chain = Call.second;
403 } else {
404 EVT SetCCVT =
405 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), RetVT);
406 SDValue Tmp = DAG.getSetCC(dl, SetCCVT, NewLHS, NewRHS, CCCode);
407 auto Call2 = makeLibCall(DAG, LC2, RetVT, Ops, CallOptions, dl, Chain);
408 CCCode = getCmpLibcallCC(LC2);
409 if (ShouldInvertCC)
410 CCCode = getSetCCInverse(CCCode, RetVT);
411 NewLHS = DAG.getSetCC(dl, SetCCVT, Call2.first, NewRHS, CCCode);
412 if (Chain)
413 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Call.second,
414 Call2.second);
415 NewLHS = DAG.getNode(ShouldInvertCC ? ISD::AND : ISD::OR, dl,
416 Tmp.getValueType(), Tmp, NewLHS);
417 NewRHS = SDValue();
418 }
419}
420
421/// Return the entry encoding for a jump table in the current function. The
422/// returned value is a member of the MachineJumpTableInfo::JTEntryKind enum.
423unsigned TargetLowering::getJumpTableEncoding() const {
424 // In non-pic modes, just use the address of a block.
425 if (!isPositionIndependent())
426 return MachineJumpTableInfo::EK_BlockAddress;
427
428 // In PIC mode, if the target supports a GPRel32 directive, use it.
429 if (getTargetMachine().getMCAsmInfo()->getGPRel32Directive() != nullptr)
430 return MachineJumpTableInfo::EK_GPRel32BlockAddress;
431
432 // Otherwise, use a label difference.
433 return MachineJumpTableInfo::EK_LabelDifference32;
434}
435
436SDValue TargetLowering::getPICJumpTableRelocBase(SDValue Table,
437 SelectionDAG &DAG) const {
438 // If our PIC model is GP relative, use the global offset table as the base.
439 unsigned JTEncoding = getJumpTableEncoding();
440
441 if ((JTEncoding == MachineJumpTableInfo::EK_GPRel64BlockAddress) ||
442 (JTEncoding == MachineJumpTableInfo::EK_GPRel32BlockAddress))
443 return DAG.getGLOBAL_OFFSET_TABLE(getPointerTy(DAG.getDataLayout()));
444
445 return Table;
446}
447
448/// This returns the relocation base for the given PIC jumptable, the same as
449/// getPICJumpTableRelocBase, but as an MCExpr.
450const MCExpr *
451TargetLowering::getPICJumpTableRelocBaseExpr(const MachineFunction *MF,
452 unsigned JTI,MCContext &Ctx) const{
453 // The normal PIC reloc base is the label at the start of the jump table.
454 return MCSymbolRefExpr::create(MF->getJTISymbol(JTI, Ctx), Ctx);
455}
456
457bool
458TargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
459 const TargetMachine &TM = getTargetMachine();
460 const GlobalValue *GV = GA->getGlobal();
461
462 // If the address is not even local to this DSO we will have to load it from
463 // a got and then add the offset.
464 if (!TM.shouldAssumeDSOLocal(*GV->getParent(), GV))
465 return false;
466
467 // If the code is position independent we will have to add a base register.
468 if (isPositionIndependent())
469 return false;
470
471 // Otherwise we can do it.
472 return true;
473}
474
475//===----------------------------------------------------------------------===//
476// Optimization Methods
477//===----------------------------------------------------------------------===//
478
479/// If the specified instruction has a constant integer operand and there are
480/// bits set in that constant that are not demanded, then clear those bits and
481/// return true.
482bool TargetLowering::ShrinkDemandedConstant(SDValue Op, const APInt &Demanded,
483 TargetLoweringOpt &TLO) const {
484 SDLoc DL(Op);
485 unsigned Opcode = Op.getOpcode();
486
487 // Do target-specific constant optimization.
488 if (targetShrinkDemandedConstant(Op, Demanded, TLO))
489 return TLO.New.getNode();
490
491 // FIXME: ISD::SELECT, ISD::SELECT_CC
492 switch (Opcode) {
493 default:
494 break;
495 case ISD::XOR:
496 case ISD::AND:
497 case ISD::OR: {
498 auto *Op1C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
499 if (!Op1C)
500 return false;
501
502 // If this is a 'not' op, don't touch it because that's a canonical form.
503 const APInt &C = Op1C->getAPIntValue();
504 if (Opcode == ISD::XOR && Demanded.isSubsetOf(C))
505 return false;
506
507 if (!C.isSubsetOf(Demanded)) {
508 EVT VT = Op.getValueType();
509 SDValue NewC = TLO.DAG.getConstant(Demanded & C, DL, VT);
510 SDValue NewOp = TLO.DAG.getNode(Opcode, DL, VT, Op.getOperand(0), NewC);
511 return TLO.CombineTo(Op, NewOp);
512 }
513
514 break;
515 }
516 }
517
518 return false;
519}
520
521/// Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.
522/// This uses isZExtFree and ZERO_EXTEND for the widening cast, but it could be
523/// generalized for targets with other types of implicit widening casts.
524bool TargetLowering::ShrinkDemandedOp(SDValue Op, unsigned BitWidth,
525 const APInt &Demanded,
526 TargetLoweringOpt &TLO) const {
527 assert(Op.getNumOperands() == 2 &&((Op.getNumOperands() == 2 && "ShrinkDemandedOp only supports binary operators!"
) ? static_cast<void> (0) : __assert_fail ("Op.getNumOperands() == 2 && \"ShrinkDemandedOp only supports binary operators!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 528, __PRETTY_FUNCTION__))
528 "ShrinkDemandedOp only supports binary operators!")((Op.getNumOperands() == 2 && "ShrinkDemandedOp only supports binary operators!"
) ? static_cast<void> (0) : __assert_fail ("Op.getNumOperands() == 2 && \"ShrinkDemandedOp only supports binary operators!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 528, __PRETTY_FUNCTION__))
;
529 assert(Op.getNode()->getNumValues() == 1 &&((Op.getNode()->getNumValues() == 1 && "ShrinkDemandedOp only supports nodes with one result!"
) ? static_cast<void> (0) : __assert_fail ("Op.getNode()->getNumValues() == 1 && \"ShrinkDemandedOp only supports nodes with one result!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 530, __PRETTY_FUNCTION__))
530 "ShrinkDemandedOp only supports nodes with one result!")((Op.getNode()->getNumValues() == 1 && "ShrinkDemandedOp only supports nodes with one result!"
) ? static_cast<void> (0) : __assert_fail ("Op.getNode()->getNumValues() == 1 && \"ShrinkDemandedOp only supports nodes with one result!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 530, __PRETTY_FUNCTION__))
;
531
532 SelectionDAG &DAG = TLO.DAG;
533 SDLoc dl(Op);
534
535 // Early return, as this function cannot handle vector types.
536 if (Op.getValueType().isVector())
537 return false;
538
539 // Don't do this if the node has another user, which may require the
540 // full value.
541 if (!Op.getNode()->hasOneUse())
542 return false;
543
544 // Search for the smallest integer type with free casts to and from
545 // Op's type. For expedience, just check power-of-2 integer types.
546 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
547 unsigned DemandedSize = Demanded.getActiveBits();
548 unsigned SmallVTBits = DemandedSize;
549 if (!isPowerOf2_32(SmallVTBits))
550 SmallVTBits = NextPowerOf2(SmallVTBits);
551 for (; SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(SmallVTBits)) {
552 EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), SmallVTBits);
553 if (TLI.isTruncateFree(Op.getValueType(), SmallVT) &&
554 TLI.isZExtFree(SmallVT, Op.getValueType())) {
555 // We found a type with free casts.
556 SDValue X = DAG.getNode(
557 Op.getOpcode(), dl, SmallVT,
558 DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(0)),
559 DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(1)));
560 assert(DemandedSize <= SmallVTBits && "Narrowed below demanded bits?")((DemandedSize <= SmallVTBits && "Narrowed below demanded bits?"
) ? static_cast<void> (0) : __assert_fail ("DemandedSize <= SmallVTBits && \"Narrowed below demanded bits?\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 560, __PRETTY_FUNCTION__))
;
561 SDValue Z = DAG.getNode(ISD::ANY_EXTEND, dl, Op.getValueType(), X);
562 return TLO.CombineTo(Op, Z);
563 }
564 }
565 return false;
566}
567
568bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
569 DAGCombinerInfo &DCI) const {
570 SelectionDAG &DAG = DCI.DAG;
571 TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
572 !DCI.isBeforeLegalizeOps());
573 KnownBits Known;
574
575 bool Simplified = SimplifyDemandedBits(Op, DemandedBits, Known, TLO);
576 if (Simplified) {
577 DCI.AddToWorklist(Op.getNode());
578 DCI.CommitTargetLoweringOpt(TLO);
579 }
580 return Simplified;
581}
582
583bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
584 KnownBits &Known,
585 TargetLoweringOpt &TLO,
586 unsigned Depth,
587 bool AssumeSingleUse) const {
588 EVT VT = Op.getValueType();
589 APInt DemandedElts = VT.isVector()
590 ? APInt::getAllOnesValue(VT.getVectorNumElements())
591 : APInt(1, 1);
592 return SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO, Depth,
593 AssumeSingleUse);
594}
595
596// TODO: Can we merge SelectionDAG::GetDemandedBits into this?
597// TODO: Under what circumstances can we create nodes? Constant folding?
598SDValue TargetLowering::SimplifyMultipleUseDemandedBits(
599 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
600 SelectionDAG &DAG, unsigned Depth) const {
601 // Limit search depth.
602 if (Depth >= SelectionDAG::MaxRecursionDepth)
603 return SDValue();
604
605 // Ignore UNDEFs.
606 if (Op.isUndef())
607 return SDValue();
608
609 // Not demanding any bits/elts from Op.
610 if (DemandedBits == 0 || DemandedElts == 0)
611 return DAG.getUNDEF(Op.getValueType());
612
613 unsigned NumElts = DemandedElts.getBitWidth();
614 KnownBits LHSKnown, RHSKnown;
615 switch (Op.getOpcode()) {
616 case ISD::BITCAST: {
617 SDValue Src = peekThroughBitcasts(Op.getOperand(0));
618 EVT SrcVT = Src.getValueType();
619 EVT DstVT = Op.getValueType();
620 unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
621 unsigned NumDstEltBits = DstVT.getScalarSizeInBits();
622
623 if (NumSrcEltBits == NumDstEltBits)
624 if (SDValue V = SimplifyMultipleUseDemandedBits(
625 Src, DemandedBits, DemandedElts, DAG, Depth + 1))
626 return DAG.getBitcast(DstVT, V);
627
628 // TODO - bigendian once we have test coverage.
629 if (SrcVT.isVector() && (NumDstEltBits % NumSrcEltBits) == 0 &&
630 DAG.getDataLayout().isLittleEndian()) {
631 unsigned Scale = NumDstEltBits / NumSrcEltBits;
632 unsigned NumSrcElts = SrcVT.getVectorNumElements();
633 APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);
634 APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);
635 for (unsigned i = 0; i != Scale; ++i) {
636 unsigned Offset = i * NumSrcEltBits;
637 APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset);
638 if (!Sub.isNullValue()) {
639 DemandedSrcBits |= Sub;
640 for (unsigned j = 0; j != NumElts; ++j)
641 if (DemandedElts[j])
642 DemandedSrcElts.setBit((j * Scale) + i);
643 }
644 }
645
646 if (SDValue V = SimplifyMultipleUseDemandedBits(
647 Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))
648 return DAG.getBitcast(DstVT, V);
649 }
650
651 // TODO - bigendian once we have test coverage.
652 if ((NumSrcEltBits % NumDstEltBits) == 0 &&
653 DAG.getDataLayout().isLittleEndian()) {
654 unsigned Scale = NumSrcEltBits / NumDstEltBits;
655 unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
656 APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);
657 APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);
658 for (unsigned i = 0; i != NumElts; ++i)
659 if (DemandedElts[i]) {
660 unsigned Offset = (i % Scale) * NumDstEltBits;
661 DemandedSrcBits.insertBits(DemandedBits, Offset);
662 DemandedSrcElts.setBit(i / Scale);
663 }
664
665 if (SDValue V = SimplifyMultipleUseDemandedBits(
666 Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))
667 return DAG.getBitcast(DstVT, V);
668 }
669
670 break;
671 }
672 case ISD::AND: {
673 LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
674 RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
675
676 // If all of the demanded bits are known 1 on one side, return the other.
677 // These bits cannot contribute to the result of the 'and' in this
678 // context.
679 if (DemandedBits.isSubsetOf(LHSKnown.Zero | RHSKnown.One))
680 return Op.getOperand(0);
681 if (DemandedBits.isSubsetOf(RHSKnown.Zero | LHSKnown.One))
682 return Op.getOperand(1);
683 break;
684 }
685 case ISD::OR: {
686 LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
687 RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
688
689 // If all of the demanded bits are known zero on one side, return the
690 // other. These bits cannot contribute to the result of the 'or' in this
691 // context.
692 if (DemandedBits.isSubsetOf(LHSKnown.One | RHSKnown.Zero))
693 return Op.getOperand(0);
694 if (DemandedBits.isSubsetOf(RHSKnown.One | LHSKnown.Zero))
695 return Op.getOperand(1);
696 break;
697 }
698 case ISD::XOR: {
699 LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
700 RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
701
702 // If all of the demanded bits are known zero on one side, return the
703 // other.
704 if (DemandedBits.isSubsetOf(RHSKnown.Zero))
705 return Op.getOperand(0);
706 if (DemandedBits.isSubsetOf(LHSKnown.Zero))
707 return Op.getOperand(1);
708 break;
709 }
710 case ISD::SETCC: {
711 SDValue Op0 = Op.getOperand(0);
712 SDValue Op1 = Op.getOperand(1);
713 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
714 // If (1) we only need the sign-bit, (2) the setcc operands are the same
715 // width as the setcc result, and (3) the result of a setcc conforms to 0 or
716 // -1, we may be able to bypass the setcc.
717 if (DemandedBits.isSignMask() &&
718 Op0.getScalarValueSizeInBits() == DemandedBits.getBitWidth() &&
719 getBooleanContents(Op0.getValueType()) ==
720 BooleanContent::ZeroOrNegativeOneBooleanContent) {
721 // If we're testing X < 0, then this compare isn't needed - just use X!
722 // FIXME: We're limiting to integer types here, but this should also work
723 // if we don't care about FP signed-zero. The use of SETLT with FP means
724 // that we don't care about NaNs.
725 if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
726 (isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode())))
727 return Op0;
728 }
729 break;
730 }
731 case ISD::SIGN_EXTEND_INREG: {
732 // If none of the extended bits are demanded, eliminate the sextinreg.
733 EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
734 if (DemandedBits.getActiveBits() <= ExVT.getScalarSizeInBits())
735 return Op.getOperand(0);
736 break;
737 }
738 case ISD::INSERT_VECTOR_ELT: {
739 // If we don't demand the inserted element, return the base vector.
740 SDValue Vec = Op.getOperand(0);
741 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
742 EVT VecVT = Vec.getValueType();
743 if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements()) &&
744 !DemandedElts[CIdx->getZExtValue()])
745 return Vec;
746 break;
747 }
748 case ISD::INSERT_SUBVECTOR: {
749 // If we don't demand the inserted subvector, return the base vector.
750 SDValue Vec = Op.getOperand(0);
751 SDValue Sub = Op.getOperand(1);
752 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
753 unsigned NumVecElts = Vec.getValueType().getVectorNumElements();
754 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
755 if (CIdx && CIdx->getAPIntValue().ule(NumVecElts - NumSubElts))
756 if (DemandedElts.extractBits(NumSubElts, CIdx->getZExtValue()) == 0)
757 return Vec;
758 break;
759 }
760 case ISD::VECTOR_SHUFFLE: {
761 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
762
763 // If all the demanded elts are from one operand and are inline,
764 // then we can use the operand directly.
765 bool AllUndef = true, IdentityLHS = true, IdentityRHS = true;
766 for (unsigned i = 0; i != NumElts; ++i) {
767 int M = ShuffleMask[i];
768 if (M < 0 || !DemandedElts[i])
769 continue;
770 AllUndef = false;
771 IdentityLHS &= (M == (int)i);
772 IdentityRHS &= ((M - NumElts) == i);
773 }
774
775 if (AllUndef)
776 return DAG.getUNDEF(Op.getValueType());
777 if (IdentityLHS)
778 return Op.getOperand(0);
779 if (IdentityRHS)
780 return Op.getOperand(1);
781 break;
782 }
783 default:
784 if (Op.getOpcode() >= ISD::BUILTIN_OP_END)
785 if (SDValue V = SimplifyMultipleUseDemandedBitsForTargetNode(
786 Op, DemandedBits, DemandedElts, DAG, Depth))
787 return V;
788 break;
789 }
790 return SDValue();
791}
792
793SDValue TargetLowering::SimplifyMultipleUseDemandedBits(
794 SDValue Op, const APInt &DemandedBits, SelectionDAG &DAG,
795 unsigned Depth) const {
796 EVT VT = Op.getValueType();
797 APInt DemandedElts = VT.isVector()
798 ? APInt::getAllOnesValue(VT.getVectorNumElements())
799 : APInt(1, 1);
800 return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
801 Depth);
802}
803
804/// Look at Op. At this point, we know that only the OriginalDemandedBits of the
805/// result of Op are ever used downstream. If we can use this information to
806/// simplify Op, create a new simplified DAG node and return true, returning the
807/// original and new nodes in Old and New. Otherwise, analyze the expression and
808/// return a mask of Known bits for the expression (used to simplify the
809/// caller). The Known bits may only be accurate for those bits in the
810/// OriginalDemandedBits and OriginalDemandedElts.
811bool TargetLowering::SimplifyDemandedBits(
812 SDValue Op, const APInt &OriginalDemandedBits,
813 const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
814 unsigned Depth, bool AssumeSingleUse) const {
815 unsigned BitWidth = OriginalDemandedBits.getBitWidth();
816 assert(Op.getScalarValueSizeInBits() == BitWidth &&((Op.getScalarValueSizeInBits() == BitWidth && "Mask size mismatches value type size!"
) ? static_cast<void> (0) : __assert_fail ("Op.getScalarValueSizeInBits() == BitWidth && \"Mask size mismatches value type size!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 817, __PRETTY_FUNCTION__))
817 "Mask size mismatches value type size!")((Op.getScalarValueSizeInBits() == BitWidth && "Mask size mismatches value type size!"
) ? static_cast<void> (0) : __assert_fail ("Op.getScalarValueSizeInBits() == BitWidth && \"Mask size mismatches value type size!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 817, __PRETTY_FUNCTION__))
;
818
819 unsigned NumElts = OriginalDemandedElts.getBitWidth();
820 assert((!Op.getValueType().isVector() ||(((!Op.getValueType().isVector() || NumElts == Op.getValueType
().getVectorNumElements()) && "Unexpected vector size"
) ? static_cast<void> (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 822, __PRETTY_FUNCTION__))
821 NumElts == Op.getValueType().getVectorNumElements()) &&(((!Op.getValueType().isVector() || NumElts == Op.getValueType
().getVectorNumElements()) && "Unexpected vector size"
) ? static_cast<void> (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 822, __PRETTY_FUNCTION__))
822 "Unexpected vector size")(((!Op.getValueType().isVector() || NumElts == Op.getValueType
().getVectorNumElements()) && "Unexpected vector size"
) ? static_cast<void> (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 822, __PRETTY_FUNCTION__))
;
823
824 APInt DemandedBits = OriginalDemandedBits;
825 APInt DemandedElts = OriginalDemandedElts;
826 SDLoc dl(Op);
827 auto &DL = TLO.DAG.getDataLayout();
828
829 // Don't know anything.
830 Known = KnownBits(BitWidth);
831
832 // Undef operand.
833 if (Op.isUndef())
834 return false;
835
836 if (Op.getOpcode() == ISD::Constant) {
837 // We know all of the bits for a constant!
838 Known.One = cast<ConstantSDNode>(Op)->getAPIntValue();
839 Known.Zero = ~Known.One;
840 return false;
841 }
842
843 // Other users may use these bits.
844 EVT VT = Op.getValueType();
845 if (!Op.getNode()->hasOneUse() && !AssumeSingleUse) {
846 if (Depth != 0) {
847 // If not at the root, Just compute the Known bits to
848 // simplify things downstream.
849 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
850 return false;
851 }
852 // If this is the root being simplified, allow it to have multiple uses,
853 // just set the DemandedBits/Elts to all bits.
854 DemandedBits = APInt::getAllOnesValue(BitWidth);
855 DemandedElts = APInt::getAllOnesValue(NumElts);
856 } else if (OriginalDemandedBits == 0 || OriginalDemandedElts == 0) {
857 // Not demanding any bits/elts from Op.
858 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
859 } else if (Depth >= SelectionDAG::MaxRecursionDepth) {
860 // Limit search depth.
861 return false;
862 }
863
864 KnownBits Known2, KnownOut;
865 switch (Op.getOpcode()) {
866 case ISD::TargetConstant:
867 llvm_unreachable("Can't simplify this node")::llvm::llvm_unreachable_internal("Can't simplify this node",
"/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 867)
;
868 case ISD::SCALAR_TO_VECTOR: {
869 if (!DemandedElts[0])
870 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
871
872 KnownBits SrcKnown;
873 SDValue Src = Op.getOperand(0);
874 unsigned SrcBitWidth = Src.getScalarValueSizeInBits();
875 APInt SrcDemandedBits = DemandedBits.zextOrSelf(SrcBitWidth);
876 if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcKnown, TLO, Depth + 1))
877 return true;
878
879 // Upper elements are undef, so only get the knownbits if we just demand
880 // the bottom element.
881 if (DemandedElts == 1)
882 Known = SrcKnown.anyextOrTrunc(BitWidth);
883 break;
884 }
885 case ISD::BUILD_VECTOR:
886 // Collect the known bits that are shared by every demanded element.
887 // TODO: Call SimplifyDemandedBits for non-constant demanded elements.
888 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
889 return false; // Don't fall through, will infinitely loop.
890 case ISD::LOAD: {
891 LoadSDNode *LD = cast<LoadSDNode>(Op);
892 if (getTargetConstantFromLoad(LD)) {
893 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
894 return false; // Don't fall through, will infinitely loop.
895 } else if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
896 // If this is a ZEXTLoad and we are looking at the loaded value.
897 EVT MemVT = LD->getMemoryVT();
898 unsigned MemBits = MemVT.getScalarSizeInBits();
899 Known.Zero.setBitsFrom(MemBits);
900 return false; // Don't fall through, will infinitely loop.
901 }
902 break;
903 }
904 case ISD::INSERT_VECTOR_ELT: {
905 SDValue Vec = Op.getOperand(0);
906 SDValue Scl = Op.getOperand(1);
907 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
908 EVT VecVT = Vec.getValueType();
909
910 // If index isn't constant, assume we need all vector elements AND the
911 // inserted element.
912 APInt DemandedVecElts(DemandedElts);
913 if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements())) {
914 unsigned Idx = CIdx->getZExtValue();
915 DemandedVecElts.clearBit(Idx);
916
917 // Inserted element is not required.
918 if (!DemandedElts[Idx])
919 return TLO.CombineTo(Op, Vec);
920 }
921
922 KnownBits KnownScl;
923 unsigned NumSclBits = Scl.getScalarValueSizeInBits();
924 APInt DemandedSclBits = DemandedBits.zextOrTrunc(NumSclBits);
925 if (SimplifyDemandedBits(Scl, DemandedSclBits, KnownScl, TLO, Depth + 1))
926 return true;
927
928 Known = KnownScl.anyextOrTrunc(BitWidth);
929
930 KnownBits KnownVec;
931 if (SimplifyDemandedBits(Vec, DemandedBits, DemandedVecElts, KnownVec, TLO,
932 Depth + 1))
933 return true;
934
935 if (!!DemandedVecElts) {
936 Known.One &= KnownVec.One;
937 Known.Zero &= KnownVec.Zero;
938 }
939
940 return false;
941 }
942 case ISD::INSERT_SUBVECTOR: {
943 SDValue Base = Op.getOperand(0);
944 SDValue Sub = Op.getOperand(1);
945 EVT SubVT = Sub.getValueType();
946 unsigned NumSubElts = SubVT.getVectorNumElements();
947
948 // If index isn't constant, assume we need the original demanded base
949 // elements and ALL the inserted subvector elements.
950 APInt BaseElts = DemandedElts;
951 APInt SubElts = APInt::getAllOnesValue(NumSubElts);
952 if (isa<ConstantSDNode>(Op.getOperand(2))) {
953 const APInt &Idx = Op.getConstantOperandAPInt(2);
954 if (Idx.ule(NumElts - NumSubElts)) {
955 unsigned SubIdx = Idx.getZExtValue();
956 SubElts = DemandedElts.extractBits(NumSubElts, SubIdx);
957 BaseElts.insertBits(APInt::getNullValue(NumSubElts), SubIdx);
958 }
959 }
960
961 KnownBits KnownSub, KnownBase;
962 if (SimplifyDemandedBits(Sub, DemandedBits, SubElts, KnownSub, TLO,
963 Depth + 1))
964 return true;
965 if (SimplifyDemandedBits(Base, DemandedBits, BaseElts, KnownBase, TLO,
966 Depth + 1))
967 return true;
968
969 Known.Zero.setAllBits();
970 Known.One.setAllBits();
971 if (!!SubElts) {
972 Known.One &= KnownSub.One;
973 Known.Zero &= KnownSub.Zero;
974 }
975 if (!!BaseElts) {
976 Known.One &= KnownBase.One;
977 Known.Zero &= KnownBase.Zero;
978 }
979
980 // Attempt to avoid multi-use src if we don't need anything from it.
981 if (!DemandedBits.isAllOnesValue() || !SubElts.isAllOnesValue() ||
982 !BaseElts.isAllOnesValue()) {
983 SDValue NewSub = SimplifyMultipleUseDemandedBits(
984 Sub, DemandedBits, SubElts, TLO.DAG, Depth + 1);
985 SDValue NewBase = SimplifyMultipleUseDemandedBits(
986 Base, DemandedBits, BaseElts, TLO.DAG, Depth + 1);
987 if (NewSub || NewBase) {
988 NewSub = NewSub ? NewSub : Sub;
989 NewBase = NewBase ? NewBase : Base;
990 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewBase, NewSub,
991 Op.getOperand(2));
992 return TLO.CombineTo(Op, NewOp);
993 }
994 }
995 break;
996 }
997 case ISD::EXTRACT_SUBVECTOR: {
998 // If index isn't constant, assume we need all the source vector elements.
999 SDValue Src = Op.getOperand(0);
1000 ConstantSDNode *SubIdx = dyn_cast<ConstantSDNode>(Op.getOperand(1));
1001 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
1002 APInt SrcElts = APInt::getAllOnesValue(NumSrcElts);
1003 if (SubIdx && SubIdx->getAPIntValue().ule(NumSrcElts - NumElts)) {
1004 // Offset the demanded elts by the subvector index.
1005 uint64_t Idx = SubIdx->getZExtValue();
1006 SrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
1007 }
1008 if (SimplifyDemandedBits(Src, DemandedBits, SrcElts, Known, TLO, Depth + 1))
1009 return true;
1010
1011 // Attempt to avoid multi-use src if we don't need anything from it.
1012 if (!DemandedBits.isAllOnesValue() || !SrcElts.isAllOnesValue()) {
1013 SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
1014 Src, DemandedBits, SrcElts, TLO.DAG, Depth + 1);
1015 if (DemandedSrc) {
1016 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc,
1017 Op.getOperand(1));
1018 return TLO.CombineTo(Op, NewOp);
1019 }
1020 }
1021 break;
1022 }
1023 case ISD::CONCAT_VECTORS: {
1024 Known.Zero.setAllBits();
1025 Known.One.setAllBits();
1026 EVT SubVT = Op.getOperand(0).getValueType();
1027 unsigned NumSubVecs = Op.getNumOperands();
1028 unsigned NumSubElts = SubVT.getVectorNumElements();
1029 for (unsigned i = 0; i != NumSubVecs; ++i) {
1030 APInt DemandedSubElts =
1031 DemandedElts.extractBits(NumSubElts, i * NumSubElts);
1032 if (SimplifyDemandedBits(Op.getOperand(i), DemandedBits, DemandedSubElts,
1033 Known2, TLO, Depth + 1))
1034 return true;
1035 // Known bits are shared by every demanded subvector element.
1036 if (!!DemandedSubElts) {
1037 Known.One &= Known2.One;
1038 Known.Zero &= Known2.Zero;
1039 }
1040 }
1041 break;
1042 }
1043 case ISD::VECTOR_SHUFFLE: {
1044 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
1045
1046 // Collect demanded elements from shuffle operands..
1047 APInt DemandedLHS(NumElts, 0);
1048 APInt DemandedRHS(NumElts, 0);
1049 for (unsigned i = 0; i != NumElts; ++i) {
1050 if (!DemandedElts[i])
1051 continue;
1052 int M = ShuffleMask[i];
1053 if (M < 0) {
1054 // For UNDEF elements, we don't know anything about the common state of
1055 // the shuffle result.
1056 DemandedLHS.clearAllBits();
1057 DemandedRHS.clearAllBits();
1058 break;
1059 }
1060 assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range")((0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range"
) ? static_cast<void> (0) : __assert_fail ("0 <= M && M < (int)(2 * NumElts) && \"Shuffle index out of range\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1060, __PRETTY_FUNCTION__))
;
1061 if (M < (int)NumElts)
1062 DemandedLHS.setBit(M);
1063 else
1064 DemandedRHS.setBit(M - NumElts);
1065 }
1066
1067 if (!!DemandedLHS || !!DemandedRHS) {
1068 SDValue Op0 = Op.getOperand(0);
1069 SDValue Op1 = Op.getOperand(1);
1070
1071 Known.Zero.setAllBits();
1072 Known.One.setAllBits();
1073 if (!!DemandedLHS) {
1074 if (SimplifyDemandedBits(Op0, DemandedBits, DemandedLHS, Known2, TLO,
1075 Depth + 1))
1076 return true;
1077 Known.One &= Known2.One;
1078 Known.Zero &= Known2.Zero;
1079 }
1080 if (!!DemandedRHS) {
1081 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedRHS, Known2, TLO,
1082 Depth + 1))
1083 return true;
1084 Known.One &= Known2.One;
1085 Known.Zero &= Known2.Zero;
1086 }
1087
1088 // Attempt to avoid multi-use ops if we don't need anything from them.
1089 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1090 Op0, DemandedBits, DemandedLHS, TLO.DAG, Depth + 1);
1091 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1092 Op1, DemandedBits, DemandedRHS, TLO.DAG, Depth + 1);
1093 if (DemandedOp0 || DemandedOp1) {
1094 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1095 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1096 SDValue NewOp = TLO.DAG.getVectorShuffle(VT, dl, Op0, Op1, ShuffleMask);
1097 return TLO.CombineTo(Op, NewOp);
1098 }
1099 }
1100 break;
1101 }
1102 case ISD::AND: {
1103 SDValue Op0 = Op.getOperand(0);
1104 SDValue Op1 = Op.getOperand(1);
1105
1106 // If the RHS is a constant, check to see if the LHS would be zero without
1107 // using the bits from the RHS. Below, we use knowledge about the RHS to
1108 // simplify the LHS, here we're using information from the LHS to simplify
1109 // the RHS.
1110 if (ConstantSDNode *RHSC = isConstOrConstSplat(Op1)) {
1111 // Do not increment Depth here; that can cause an infinite loop.
1112 KnownBits LHSKnown = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth);
1113 // If the LHS already has zeros where RHSC does, this 'and' is dead.
1114 if ((LHSKnown.Zero & DemandedBits) ==
1115 (~RHSC->getAPIntValue() & DemandedBits))
1116 return TLO.CombineTo(Op, Op0);
1117
1118 // If any of the set bits in the RHS are known zero on the LHS, shrink
1119 // the constant.
1120 if (ShrinkDemandedConstant(Op, ~LHSKnown.Zero & DemandedBits, TLO))
1121 return true;
1122
1123 // Bitwise-not (xor X, -1) is a special case: we don't usually shrink its
1124 // constant, but if this 'and' is only clearing bits that were just set by
1125 // the xor, then this 'and' can be eliminated by shrinking the mask of
1126 // the xor. For example, for a 32-bit X:
1127 // and (xor (srl X, 31), -1), 1 --> xor (srl X, 31), 1
1128 if (isBitwiseNot(Op0) && Op0.hasOneUse() &&
1129 LHSKnown.One == ~RHSC->getAPIntValue()) {
1130 SDValue Xor = TLO.DAG.getNode(ISD::XOR, dl, VT, Op0.getOperand(0), Op1);
1131 return TLO.CombineTo(Op, Xor);
1132 }
1133 }
1134
1135 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1136 Depth + 1))
1137 return true;
1138 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1138, __PRETTY_FUNCTION__))
;
1139 if (SimplifyDemandedBits(Op0, ~Known.Zero & DemandedBits, DemandedElts,
1140 Known2, TLO, Depth + 1))
1141 return true;
1142 assert(!Known2.hasConflict() && "Bits known to be one AND zero?")((!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1142, __PRETTY_FUNCTION__))
;
1143
1144 // Attempt to avoid multi-use ops if we don't need anything from them.
1145 if (!DemandedBits.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {
1146 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1147 Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1148 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1149 Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1150 if (DemandedOp0 || DemandedOp1) {
1151 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1152 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1153 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1154 return TLO.CombineTo(Op, NewOp);
1155 }
1156 }
1157
1158 // If all of the demanded bits are known one on one side, return the other.
1159 // These bits cannot contribute to the result of the 'and'.
1160 if (DemandedBits.isSubsetOf(Known2.Zero | Known.One))
1161 return TLO.CombineTo(Op, Op0);
1162 if (DemandedBits.isSubsetOf(Known.Zero | Known2.One))
1163 return TLO.CombineTo(Op, Op1);
1164 // If all of the demanded bits in the inputs are known zeros, return zero.
1165 if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
1166 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, dl, VT));
1167 // If the RHS is a constant, see if we can simplify it.
1168 if (ShrinkDemandedConstant(Op, ~Known2.Zero & DemandedBits, TLO))
1169 return true;
1170 // If the operation can be done in a smaller type, do so.
1171 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1172 return true;
1173
1174 // Output known-1 bits are only known if set in both the LHS & RHS.
1175 Known.One &= Known2.One;
1176 // Output known-0 are known to be clear if zero in either the LHS | RHS.
1177 Known.Zero |= Known2.Zero;
1178 break;
1179 }
1180 case ISD::OR: {
1181 SDValue Op0 = Op.getOperand(0);
1182 SDValue Op1 = Op.getOperand(1);
1183
1184 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1185 Depth + 1))
1186 return true;
1187 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1187, __PRETTY_FUNCTION__))
;
1188 if (SimplifyDemandedBits(Op0, ~Known.One & DemandedBits, DemandedElts,
1189 Known2, TLO, Depth + 1))
1190 return true;
1191 assert(!Known2.hasConflict() && "Bits known to be one AND zero?")((!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1191, __PRETTY_FUNCTION__))
;
1192
1193 // Attempt to avoid multi-use ops if we don't need anything from them.
1194 if (!DemandedBits.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {
1195 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1196 Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1197 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1198 Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1199 if (DemandedOp0 || DemandedOp1) {
1200 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1201 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1202 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1203 return TLO.CombineTo(Op, NewOp);
1204 }
1205 }
1206
1207 // If all of the demanded bits are known zero on one side, return the other.
1208 // These bits cannot contribute to the result of the 'or'.
1209 if (DemandedBits.isSubsetOf(Known2.One | Known.Zero))
1210 return TLO.CombineTo(Op, Op0);
1211 if (DemandedBits.isSubsetOf(Known.One | Known2.Zero))
1212 return TLO.CombineTo(Op, Op1);
1213 // If the RHS is a constant, see if we can simplify it.
1214 if (ShrinkDemandedConstant(Op, DemandedBits, TLO))
1215 return true;
1216 // If the operation can be done in a smaller type, do so.
1217 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1218 return true;
1219
1220 // Output known-0 bits are only known if clear in both the LHS & RHS.
1221 Known.Zero &= Known2.Zero;
1222 // Output known-1 are known to be set if set in either the LHS | RHS.
1223 Known.One |= Known2.One;
1224 break;
1225 }
1226 case ISD::XOR: {
1227 SDValue Op0 = Op.getOperand(0);
1228 SDValue Op1 = Op.getOperand(1);
1229
1230 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1231 Depth + 1))
1232 return true;
1233 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1233, __PRETTY_FUNCTION__))
;
1234 if (SimplifyDemandedBits(Op0, DemandedBits, DemandedElts, Known2, TLO,
1235 Depth + 1))
1236 return true;
1237 assert(!Known2.hasConflict() && "Bits known to be one AND zero?")((!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1237, __PRETTY_FUNCTION__))
;
1238
1239 // Attempt to avoid multi-use ops if we don't need anything from them.
1240 if (!DemandedBits.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {
1241 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1242 Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1243 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1244 Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1245 if (DemandedOp0 || DemandedOp1) {
1246 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1247 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1248 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1249 return TLO.CombineTo(Op, NewOp);
1250 }
1251 }
1252
1253 // If all of the demanded bits are known zero on one side, return the other.
1254 // These bits cannot contribute to the result of the 'xor'.
1255 if (DemandedBits.isSubsetOf(Known.Zero))
1256 return TLO.CombineTo(Op, Op0);
1257 if (DemandedBits.isSubsetOf(Known2.Zero))
1258 return TLO.CombineTo(Op, Op1);
1259 // If the operation can be done in a smaller type, do so.
1260 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1261 return true;
1262
1263 // If all of the unknown bits are known to be zero on one side or the other
1264 // (but not both) turn this into an *inclusive* or.
1265 // e.g. (A & C1)^(B & C2) -> (A & C1)|(B & C2) iff C1&C2 == 0
1266 if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
1267 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::OR, dl, VT, Op0, Op1));
1268
1269 // Output known-0 bits are known if clear or set in both the LHS & RHS.
1270 KnownOut.Zero = (Known.Zero & Known2.Zero) | (Known.One & Known2.One);
1271 // Output known-1 are known to be set if set in only one of the LHS, RHS.
1272 KnownOut.One = (Known.Zero & Known2.One) | (Known.One & Known2.Zero);
1273
1274 if (ConstantSDNode *C = isConstOrConstSplat(Op1)) {
1275 // If one side is a constant, and all of the known set bits on the other
1276 // side are also set in the constant, turn this into an AND, as we know
1277 // the bits will be cleared.
1278 // e.g. (X | C1) ^ C2 --> (X | C1) & ~C2 iff (C1&C2) == C2
1279 // NB: it is okay if more bits are known than are requested
1280 if (C->getAPIntValue() == Known2.One) {
1281 SDValue ANDC =
1282 TLO.DAG.getConstant(~C->getAPIntValue() & DemandedBits, dl, VT);
1283 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, dl, VT, Op0, ANDC));
1284 }
1285
1286 // If the RHS is a constant, see if we can change it. Don't alter a -1
1287 // constant because that's a 'not' op, and that is better for combining
1288 // and codegen.
1289 if (!C->isAllOnesValue()) {
1290 if (DemandedBits.isSubsetOf(C->getAPIntValue())) {
1291 // We're flipping all demanded bits. Flip the undemanded bits too.
1292 SDValue New = TLO.DAG.getNOT(dl, Op0, VT);
1293 return TLO.CombineTo(Op, New);
1294 }
1295 // If we can't turn this into a 'not', try to shrink the constant.
1296 if (ShrinkDemandedConstant(Op, DemandedBits, TLO))
1297 return true;
1298 }
1299 }
1300
1301 Known = std::move(KnownOut);
1302 break;
1303 }
1304 case ISD::SELECT:
1305 if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known, TLO,
1306 Depth + 1))
1307 return true;
1308 if (SimplifyDemandedBits(Op.getOperand(1), DemandedBits, Known2, TLO,
1309 Depth + 1))
1310 return true;
1311 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1311, __PRETTY_FUNCTION__))
;
1312 assert(!Known2.hasConflict() && "Bits known to be one AND zero?")((!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1312, __PRETTY_FUNCTION__))
;
1313
1314 // If the operands are constants, see if we can simplify them.
1315 if (ShrinkDemandedConstant(Op, DemandedBits, TLO))
1316 return true;
1317
1318 // Only known if known in both the LHS and RHS.
1319 Known.One &= Known2.One;
1320 Known.Zero &= Known2.Zero;
1321 break;
1322 case ISD::SELECT_CC:
1323 if (SimplifyDemandedBits(Op.getOperand(3), DemandedBits, Known, TLO,
1324 Depth + 1))
1325 return true;
1326 if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known2, TLO,
1327 Depth + 1))
1328 return true;
1329 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1329, __PRETTY_FUNCTION__))
;
1330 assert(!Known2.hasConflict() && "Bits known to be one AND zero?")((!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1330, __PRETTY_FUNCTION__))
;
1331
1332 // If the operands are constants, see if we can simplify them.
1333 if (ShrinkDemandedConstant(Op, DemandedBits, TLO))
1334 return true;
1335
1336 // Only known if known in both the LHS and RHS.
1337 Known.One &= Known2.One;
1338 Known.Zero &= Known2.Zero;
1339 break;
1340 case ISD::SETCC: {
1341 SDValue Op0 = Op.getOperand(0);
1342 SDValue Op1 = Op.getOperand(1);
1343 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
1344 // If (1) we only need the sign-bit, (2) the setcc operands are the same
1345 // width as the setcc result, and (3) the result of a setcc conforms to 0 or
1346 // -1, we may be able to bypass the setcc.
1347 if (DemandedBits.isSignMask() &&
1348 Op0.getScalarValueSizeInBits() == BitWidth &&
1349 getBooleanContents(Op0.getValueType()) ==
1350 BooleanContent::ZeroOrNegativeOneBooleanContent) {
1351 // If we're testing X < 0, then this compare isn't needed - just use X!
1352 // FIXME: We're limiting to integer types here, but this should also work
1353 // if we don't care about FP signed-zero. The use of SETLT with FP means
1354 // that we don't care about NaNs.
1355 if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
1356 (isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode())))
1357 return TLO.CombineTo(Op, Op0);
1358
1359 // TODO: Should we check for other forms of sign-bit comparisons?
1360 // Examples: X <= -1, X >= 0
1361 }
1362 if (getBooleanContents(Op0.getValueType()) ==
1363 TargetLowering::ZeroOrOneBooleanContent &&
1364 BitWidth > 1)
1365 Known.Zero.setBitsFrom(1);
1366 break;
1367 }
1368 case ISD::SHL: {
1369 SDValue Op0 = Op.getOperand(0);
1370 SDValue Op1 = Op.getOperand(1);
1371 EVT ShiftVT = Op1.getValueType();
1372
1373 if (const APInt *SA =
1374 TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
1375 unsigned ShAmt = SA->getZExtValue();
1376 if (ShAmt == 0)
1377 return TLO.CombineTo(Op, Op0);
1378
1379 // If this is ((X >>u C1) << ShAmt), see if we can simplify this into a
1380 // single shift. We can do this if the bottom bits (which are shifted
1381 // out) are never demanded.
1382 // TODO - support non-uniform vector amounts.
1383 if (Op0.getOpcode() == ISD::SRL) {
1384 if (!DemandedBits.intersects(APInt::getLowBitsSet(BitWidth, ShAmt))) {
1385 if (const APInt *SA2 =
1386 TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {
1387 if (SA2->ult(BitWidth)) {
1388 unsigned C1 = SA2->getZExtValue();
1389 unsigned Opc = ISD::SHL;
1390 int Diff = ShAmt - C1;
1391 if (Diff < 0) {
1392 Diff = -Diff;
1393 Opc = ISD::SRL;
1394 }
1395
1396 SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);
1397 return TLO.CombineTo(
1398 Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));
1399 }
1400 }
1401 }
1402 }
1403
1404 // Convert (shl (anyext x, c)) to (anyext (shl x, c)) if the high bits
1405 // are not demanded. This will likely allow the anyext to be folded away.
1406 // TODO - support non-uniform vector amounts.
1407 if (Op0.getOpcode() == ISD::ANY_EXTEND) {
1408 SDValue InnerOp = Op0.getOperand(0);
1409 EVT InnerVT = InnerOp.getValueType();
1410 unsigned InnerBits = InnerVT.getScalarSizeInBits();
1411 if (ShAmt < InnerBits && DemandedBits.getActiveBits() <= InnerBits &&
1412 isTypeDesirableForOp(ISD::SHL, InnerVT)) {
1413 EVT ShTy = getShiftAmountTy(InnerVT, DL);
1414 if (!APInt(BitWidth, ShAmt).isIntN(ShTy.getSizeInBits()))
1415 ShTy = InnerVT;
1416 SDValue NarrowShl =
1417 TLO.DAG.getNode(ISD::SHL, dl, InnerVT, InnerOp,
1418 TLO.DAG.getConstant(ShAmt, dl, ShTy));
1419 return TLO.CombineTo(
1420 Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, NarrowShl));
1421 }
1422
1423 // Repeat the SHL optimization above in cases where an extension
1424 // intervenes: (shl (anyext (shr x, c1)), c2) to
1425 // (shl (anyext x), c2-c1). This requires that the bottom c1 bits
1426 // aren't demanded (as above) and that the shifted upper c1 bits of
1427 // x aren't demanded.
1428 // TODO - support non-uniform vector amounts.
1429 if (Op0.hasOneUse() && InnerOp.getOpcode() == ISD::SRL &&
1430 InnerOp.hasOneUse()) {
1431 if (const APInt *SA2 =
1432 TLO.DAG.getValidShiftAmountConstant(InnerOp, DemandedElts)) {
1433 unsigned InnerShAmt = SA2->getLimitedValue(InnerBits);
1434 if (InnerShAmt < ShAmt && InnerShAmt < InnerBits &&
1435 DemandedBits.getActiveBits() <=
1436 (InnerBits - InnerShAmt + ShAmt) &&
1437 DemandedBits.countTrailingZeros() >= ShAmt) {
1438 SDValue NewSA =
1439 TLO.DAG.getConstant(ShAmt - InnerShAmt, dl, ShiftVT);
1440 SDValue NewExt = TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT,
1441 InnerOp.getOperand(0));
1442 return TLO.CombineTo(
1443 Op, TLO.DAG.getNode(ISD::SHL, dl, VT, NewExt, NewSA));
1444 }
1445 }
1446 }
1447 }
1448
1449 APInt InDemandedMask = DemandedBits.lshr(ShAmt);
1450 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1451 Depth + 1))
1452 return true;
1453 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1453, __PRETTY_FUNCTION__))
;
1454 Known.Zero <<= ShAmt;
1455 Known.One <<= ShAmt;
1456 // low bits known zero.
1457 Known.Zero.setLowBits(ShAmt);
1458
1459 // Try shrinking the operation as long as the shift amount will still be
1460 // in range.
1461 if ((ShAmt < DemandedBits.getActiveBits()) &&
1462 ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1463 return true;
1464 }
1465 break;
1466 }
1467 case ISD::SRL: {
1468 SDValue Op0 = Op.getOperand(0);
1469 SDValue Op1 = Op.getOperand(1);
1470 EVT ShiftVT = Op1.getValueType();
1471
1472 if (const APInt *SA =
1473 TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
1474 unsigned ShAmt = SA->getZExtValue();
1475 if (ShAmt == 0)
1476 return TLO.CombineTo(Op, Op0);
1477
1478 // If this is ((X << C1) >>u ShAmt), see if we can simplify this into a
1479 // single shift. We can do this if the top bits (which are shifted out)
1480 // are never demanded.
1481 // TODO - support non-uniform vector amounts.
1482 if (Op0.getOpcode() == ISD::SHL) {
1483 if (const APInt *SA2 =
1484 TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {
1485 if (!DemandedBits.intersects(
1486 APInt::getHighBitsSet(BitWidth, ShAmt))) {
1487 if (SA2->ult(BitWidth)) {
1488 unsigned C1 = SA2->getZExtValue();
1489 unsigned Opc = ISD::SRL;
1490 int Diff = ShAmt - C1;
1491 if (Diff < 0) {
1492 Diff = -Diff;
1493 Opc = ISD::SHL;
1494 }
1495
1496 SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);
1497 return TLO.CombineTo(
1498 Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));
1499 }
1500 }
1501 }
1502 }
1503
1504 APInt InDemandedMask = (DemandedBits << ShAmt);
1505
1506 // If the shift is exact, then it does demand the low bits (and knows that
1507 // they are zero).
1508 if (Op->getFlags().hasExact())
1509 InDemandedMask.setLowBits(ShAmt);
1510
1511 // Compute the new bits that are at the top now.
1512 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1513 Depth + 1))
1514 return true;
1515 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1515, __PRETTY_FUNCTION__))
;
1516 Known.Zero.lshrInPlace(ShAmt);
1517 Known.One.lshrInPlace(ShAmt);
1518 // High bits known zero.
1519 Known.Zero.setHighBits(ShAmt);
1520 }
1521 break;
1522 }
1523 case ISD::SRA: {
1524 SDValue Op0 = Op.getOperand(0);
1525 SDValue Op1 = Op.getOperand(1);
1526 EVT ShiftVT = Op1.getValueType();
1527
1528 // If we only want bits that already match the signbit then we don't need
1529 // to shift.
1530 unsigned NumHiDemandedBits = BitWidth - DemandedBits.countTrailingZeros();
1531 if (TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1) >=
1532 NumHiDemandedBits)
1533 return TLO.CombineTo(Op, Op0);
1534
1535 // If this is an arithmetic shift right and only the low-bit is set, we can
1536 // always convert this into a logical shr, even if the shift amount is
1537 // variable. The low bit of the shift cannot be an input sign bit unless
1538 // the shift amount is >= the size of the datatype, which is undefined.
1539 if (DemandedBits.isOneValue())
1540 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1));
1541
1542 if (const APInt *SA =
1543 TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
1544 unsigned ShAmt = SA->getZExtValue();
1545 if (ShAmt == 0)
1546 return TLO.CombineTo(Op, Op0);
1547
1548 APInt InDemandedMask = (DemandedBits << ShAmt);
1549
1550 // If the shift is exact, then it does demand the low bits (and knows that
1551 // they are zero).
1552 if (Op->getFlags().hasExact())
1553 InDemandedMask.setLowBits(ShAmt);
1554
1555 // If any of the demanded bits are produced by the sign extension, we also
1556 // demand the input sign bit.
1557 if (DemandedBits.countLeadingZeros() < ShAmt)
1558 InDemandedMask.setSignBit();
1559
1560 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1561 Depth + 1))
1562 return true;
1563 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1563, __PRETTY_FUNCTION__))
;
1564 Known.Zero.lshrInPlace(ShAmt);
1565 Known.One.lshrInPlace(ShAmt);
1566
1567 // If the input sign bit is known to be zero, or if none of the top bits
1568 // are demanded, turn this into an unsigned shift right.
1569 if (Known.Zero[BitWidth - ShAmt - 1] ||
1570 DemandedBits.countLeadingZeros() >= ShAmt) {
1571 SDNodeFlags Flags;
1572 Flags.setExact(Op->getFlags().hasExact());
1573 return TLO.CombineTo(
1574 Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1, Flags));
1575 }
1576
1577 int Log2 = DemandedBits.exactLogBase2();
1578 if (Log2 >= 0) {
1579 // The bit must come from the sign.
1580 SDValue NewSA = TLO.DAG.getConstant(BitWidth - 1 - Log2, dl, ShiftVT);
1581 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, NewSA));
1582 }
1583
1584 if (Known.One[BitWidth - ShAmt - 1])
1585 // New bits are known one.
1586 Known.One.setHighBits(ShAmt);
1587
1588 // Attempt to avoid multi-use ops if we don't need anything from them.
1589 if (!InDemandedMask.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {
1590 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1591 Op0, InDemandedMask, DemandedElts, TLO.DAG, Depth + 1);
1592 if (DemandedOp0) {
1593 SDValue NewOp = TLO.DAG.getNode(ISD::SRA, dl, VT, DemandedOp0, Op1);
1594 return TLO.CombineTo(Op, NewOp);
1595 }
1596 }
1597 }
1598 break;
1599 }
1600 case ISD::FSHL:
1601 case ISD::FSHR: {
1602 SDValue Op0 = Op.getOperand(0);
1603 SDValue Op1 = Op.getOperand(1);
1604 SDValue Op2 = Op.getOperand(2);
1605 bool IsFSHL = (Op.getOpcode() == ISD::FSHL);
1606
1607 if (ConstantSDNode *SA = isConstOrConstSplat(Op2, DemandedElts)) {
1608 unsigned Amt = SA->getAPIntValue().urem(BitWidth);
1609
1610 // For fshl, 0-shift returns the 1st arg.
1611 // For fshr, 0-shift returns the 2nd arg.
1612 if (Amt == 0) {
1613 if (SimplifyDemandedBits(IsFSHL ? Op0 : Op1, DemandedBits, DemandedElts,
1614 Known, TLO, Depth + 1))
1615 return true;
1616 break;
1617 }
1618
1619 // fshl: (Op0 << Amt) | (Op1 >> (BW - Amt))
1620 // fshr: (Op0 << (BW - Amt)) | (Op1 >> Amt)
1621 APInt Demanded0 = DemandedBits.lshr(IsFSHL ? Amt : (BitWidth - Amt));
1622 APInt Demanded1 = DemandedBits << (IsFSHL ? (BitWidth - Amt) : Amt);
1623 if (SimplifyDemandedBits(Op0, Demanded0, DemandedElts, Known2, TLO,
1624 Depth + 1))
1625 return true;
1626 if (SimplifyDemandedBits(Op1, Demanded1, DemandedElts, Known, TLO,
1627 Depth + 1))
1628 return true;
1629
1630 Known2.One <<= (IsFSHL ? Amt : (BitWidth - Amt));
1631 Known2.Zero <<= (IsFSHL ? Amt : (BitWidth - Amt));
1632 Known.One.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);
1633 Known.Zero.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);
1634 Known.One |= Known2.One;
1635 Known.Zero |= Known2.Zero;
1636 }
1637 break;
1638 }
1639 case ISD::ROTL:
1640 case ISD::ROTR: {
1641 SDValue Op0 = Op.getOperand(0);
1642
1643 // If we're rotating an 0/-1 value, then it stays an 0/-1 value.
1644 if (BitWidth == TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1))
1645 return TLO.CombineTo(Op, Op0);
1646 break;
1647 }
1648 case ISD::BITREVERSE: {
1649 SDValue Src = Op.getOperand(0);
1650 APInt DemandedSrcBits = DemandedBits.reverseBits();
1651 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
1652 Depth + 1))
1653 return true;
1654 Known.One = Known2.One.reverseBits();
1655 Known.Zero = Known2.Zero.reverseBits();
1656 break;
1657 }
1658 case ISD::BSWAP: {
1659 SDValue Src = Op.getOperand(0);
1660 APInt DemandedSrcBits = DemandedBits.byteSwap();
1661 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
1662 Depth + 1))
1663 return true;
1664 Known.One = Known2.One.byteSwap();
1665 Known.Zero = Known2.Zero.byteSwap();
1666 break;
1667 }
1668 case ISD::SIGN_EXTEND_INREG: {
1669 SDValue Op0 = Op.getOperand(0);
1670 EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
1671 unsigned ExVTBits = ExVT.getScalarSizeInBits();
1672
1673 // If we only care about the highest bit, don't bother shifting right.
1674 if (DemandedBits.isSignMask()) {
1675 unsigned NumSignBits =
1676 TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
1677 bool AlreadySignExtended = NumSignBits >= BitWidth - ExVTBits + 1;
1678 // However if the input is already sign extended we expect the sign
1679 // extension to be dropped altogether later and do not simplify.
1680 if (!AlreadySignExtended) {
1681 // Compute the correct shift amount type, which must be getShiftAmountTy
1682 // for scalar types after legalization.
1683 EVT ShiftAmtTy = VT;
1684 if (TLO.LegalTypes() && !ShiftAmtTy.isVector())
1685 ShiftAmtTy = getShiftAmountTy(ShiftAmtTy, DL);
1686
1687 SDValue ShiftAmt =
1688 TLO.DAG.getConstant(BitWidth - ExVTBits, dl, ShiftAmtTy);
1689 return TLO.CombineTo(Op,
1690 TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, ShiftAmt));
1691 }
1692 }
1693
1694 // If none of the extended bits are demanded, eliminate the sextinreg.
1695 if (DemandedBits.getActiveBits() <= ExVTBits)
1696 return TLO.CombineTo(Op, Op0);
1697
1698 APInt InputDemandedBits = DemandedBits.getLoBits(ExVTBits);
1699
1700 // Since the sign extended bits are demanded, we know that the sign
1701 // bit is demanded.
1702 InputDemandedBits.setBit(ExVTBits - 1);
1703
1704 if (SimplifyDemandedBits(Op0, InputDemandedBits, Known, TLO, Depth + 1))
1705 return true;
1706 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1706, __PRETTY_FUNCTION__))
;
1707
1708 // If the sign bit of the input is known set or clear, then we know the
1709 // top bits of the result.
1710
1711 // If the input sign bit is known zero, convert this into a zero extension.
1712 if (Known.Zero[ExVTBits - 1])
1713 return TLO.CombineTo(
1714 Op, TLO.DAG.getZeroExtendInReg(Op0, dl, ExVT.getScalarType()));
1715
1716 APInt Mask = APInt::getLowBitsSet(BitWidth, ExVTBits);
1717 if (Known.One[ExVTBits - 1]) { // Input sign bit known set
1718 Known.One.setBitsFrom(ExVTBits);
1719 Known.Zero &= Mask;
1720 } else { // Input sign bit unknown
1721 Known.Zero &= Mask;
1722 Known.One &= Mask;
1723 }
1724 break;
1725 }
1726 case ISD::BUILD_PAIR: {
1727 EVT HalfVT = Op.getOperand(0).getValueType();
1728 unsigned HalfBitWidth = HalfVT.getScalarSizeInBits();
1729
1730 APInt MaskLo = DemandedBits.getLoBits(HalfBitWidth).trunc(HalfBitWidth);
1731 APInt MaskHi = DemandedBits.getHiBits(HalfBitWidth).trunc(HalfBitWidth);
1732
1733 KnownBits KnownLo, KnownHi;
1734
1735 if (SimplifyDemandedBits(Op.getOperand(0), MaskLo, KnownLo, TLO, Depth + 1))
1736 return true;
1737
1738 if (SimplifyDemandedBits(Op.getOperand(1), MaskHi, KnownHi, TLO, Depth + 1))
1739 return true;
1740
1741 Known.Zero = KnownLo.Zero.zext(BitWidth) |
1742 KnownHi.Zero.zext(BitWidth).shl(HalfBitWidth);
1743
1744 Known.One = KnownLo.One.zext(BitWidth) |
1745 KnownHi.One.zext(BitWidth).shl(HalfBitWidth);
1746 break;
1747 }
1748 case ISD::ZERO_EXTEND:
1749 case ISD::ZERO_EXTEND_VECTOR_INREG: {
1750 SDValue Src = Op.getOperand(0);
1751 EVT SrcVT = Src.getValueType();
1752 unsigned InBits = SrcVT.getScalarSizeInBits();
1753 unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
1754 bool IsVecInReg = Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG;
1755
1756 // If none of the top bits are demanded, convert this into an any_extend.
1757 if (DemandedBits.getActiveBits() <= InBits) {
1758 // If we only need the non-extended bits of the bottom element
1759 // then we can just bitcast to the result.
1760 if (IsVecInReg && DemandedElts == 1 &&
1761 VT.getSizeInBits() == SrcVT.getSizeInBits() &&
1762 TLO.DAG.getDataLayout().isLittleEndian())
1763 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
1764
1765 unsigned Opc =
1766 IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;
1767 if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
1768 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
1769 }
1770
1771 APInt InDemandedBits = DemandedBits.trunc(InBits);
1772 APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
1773 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
1774 Depth + 1))
1775 return true;
1776 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1776, __PRETTY_FUNCTION__))
;
1777 assert(Known.getBitWidth() == InBits && "Src width has changed?")((Known.getBitWidth() == InBits && "Src width has changed?"
) ? static_cast<void> (0) : __assert_fail ("Known.getBitWidth() == InBits && \"Src width has changed?\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1777, __PRETTY_FUNCTION__))
;
1778 Known = Known.zext(BitWidth);
1779 break;
1780 }
1781 case ISD::SIGN_EXTEND:
1782 case ISD::SIGN_EXTEND_VECTOR_INREG: {
1783 SDValue Src = Op.getOperand(0);
1784 EVT SrcVT = Src.getValueType();
1785 unsigned InBits = SrcVT.getScalarSizeInBits();
1786 unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
1787 bool IsVecInReg = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG;
1788
1789 // If none of the top bits are demanded, convert this into an any_extend.
1790 if (DemandedBits.getActiveBits() <= InBits) {
1791 // If we only need the non-extended bits of the bottom element
1792 // then we can just bitcast to the result.
1793 if (IsVecInReg && DemandedElts == 1 &&
1794 VT.getSizeInBits() == SrcVT.getSizeInBits() &&
1795 TLO.DAG.getDataLayout().isLittleEndian())
1796 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
1797
1798 unsigned Opc =
1799 IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;
1800 if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
1801 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
1802 }
1803
1804 APInt InDemandedBits = DemandedBits.trunc(InBits);
1805 APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
1806
1807 // Since some of the sign extended bits are demanded, we know that the sign
1808 // bit is demanded.
1809 InDemandedBits.setBit(InBits - 1);
1810
1811 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
1812 Depth + 1))
1813 return true;
1814 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1814, __PRETTY_FUNCTION__))
;
1815 assert(Known.getBitWidth() == InBits && "Src width has changed?")((Known.getBitWidth() == InBits && "Src width has changed?"
) ? static_cast<void> (0) : __assert_fail ("Known.getBitWidth() == InBits && \"Src width has changed?\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1815, __PRETTY_FUNCTION__))
;
1816
1817 // If the sign bit is known one, the top bits match.
1818 Known = Known.sext(BitWidth);
1819
1820 // If the sign bit is known zero, convert this to a zero extend.
1821 if (Known.isNonNegative()) {
1822 unsigned Opc =
1823 IsVecInReg ? ISD::ZERO_EXTEND_VECTOR_INREG : ISD::ZERO_EXTEND;
1824 if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
1825 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
1826 }
1827 break;
1828 }
1829 case ISD::ANY_EXTEND:
1830 case ISD::ANY_EXTEND_VECTOR_INREG: {
1831 SDValue Src = Op.getOperand(0);
1832 EVT SrcVT = Src.getValueType();
1833 unsigned InBits = SrcVT.getScalarSizeInBits();
1834 unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
1835 bool IsVecInReg = Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG;
1836
1837 // If we only need the bottom element then we can just bitcast.
1838 // TODO: Handle ANY_EXTEND?
1839 if (IsVecInReg && DemandedElts == 1 &&
1840 VT.getSizeInBits() == SrcVT.getSizeInBits() &&
1841 TLO.DAG.getDataLayout().isLittleEndian())
1842 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
1843
1844 APInt InDemandedBits = DemandedBits.trunc(InBits);
1845 APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
1846 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
1847 Depth + 1))
1848 return true;
1849 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1849, __PRETTY_FUNCTION__))
;
1850 assert(Known.getBitWidth() == InBits && "Src width has changed?")((Known.getBitWidth() == InBits && "Src width has changed?"
) ? static_cast<void> (0) : __assert_fail ("Known.getBitWidth() == InBits && \"Src width has changed?\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1850, __PRETTY_FUNCTION__))
;
1851 Known = Known.anyext(BitWidth);
1852
1853 // Attempt to avoid multi-use ops if we don't need anything from them.
1854 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1855 Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
1856 return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
1857 break;
1858 }
1859 case ISD::TRUNCATE: {
1860 SDValue Src = Op.getOperand(0);
1861
1862 // Simplify the input, using demanded bit information, and compute the known
1863 // zero/one bits live out.
1864 unsigned OperandBitWidth = Src.getScalarValueSizeInBits();
1865 APInt TruncMask = DemandedBits.zext(OperandBitWidth);
1866 if (SimplifyDemandedBits(Src, TruncMask, Known, TLO, Depth + 1))
1867 return true;
1868 Known = Known.trunc(BitWidth);
1869
1870 // Attempt to avoid multi-use ops if we don't need anything from them.
1871 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1872 Src, TruncMask, DemandedElts, TLO.DAG, Depth + 1))
1873 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, NewSrc));
1874
1875 // If the input is only used by this truncate, see if we can shrink it based
1876 // on the known demanded bits.
1877 if (Src.getNode()->hasOneUse()) {
1878 switch (Src.getOpcode()) {
1879 default:
1880 break;
1881 case ISD::SRL:
1882 // Shrink SRL by a constant if none of the high bits shifted in are
1883 // demanded.
1884 if (TLO.LegalTypes() && !isTypeDesirableForOp(ISD::SRL, VT))
1885 // Do not turn (vt1 truncate (vt2 srl)) into (vt1 srl) if vt1 is
1886 // undesirable.
1887 break;
1888
1889 SDValue ShAmt = Src.getOperand(1);
1890 auto *ShAmtC = dyn_cast<ConstantSDNode>(ShAmt);
1891 if (!ShAmtC || ShAmtC->getAPIntValue().uge(BitWidth))
1892 break;
1893 uint64_t ShVal = ShAmtC->getZExtValue();
1894
1895 APInt HighBits =
1896 APInt::getHighBitsSet(OperandBitWidth, OperandBitWidth - BitWidth);
1897 HighBits.lshrInPlace(ShVal);
1898 HighBits = HighBits.trunc(BitWidth);
1899
1900 if (!(HighBits & DemandedBits)) {
1901 // None of the shifted in bits are needed. Add a truncate of the
1902 // shift input, then shift it.
1903 if (TLO.LegalTypes())
1904 ShAmt = TLO.DAG.getConstant(ShVal, dl, getShiftAmountTy(VT, DL));
1905 SDValue NewTrunc =
1906 TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, Src.getOperand(0));
1907 return TLO.CombineTo(
1908 Op, TLO.DAG.getNode(ISD::SRL, dl, VT, NewTrunc, ShAmt));
1909 }
1910 break;
1911 }
1912 }
1913
1914 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1914, __PRETTY_FUNCTION__))
;
1915 break;
1916 }
1917 case ISD::AssertZext: {
1918 // AssertZext demands all of the high bits, plus any of the low bits
1919 // demanded by its users.
1920 EVT ZVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
1921 APInt InMask = APInt::getLowBitsSet(BitWidth, ZVT.getSizeInBits());
1922 if (SimplifyDemandedBits(Op.getOperand(0), ~InMask | DemandedBits, Known,
1923 TLO, Depth + 1))
1924 return true;
1925 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1925, __PRETTY_FUNCTION__))
;
1926
1927 Known.Zero |= ~InMask;
1928 break;
1929 }
1930 case ISD::EXTRACT_VECTOR_ELT: {
1931 SDValue Src = Op.getOperand(0);
1932 SDValue Idx = Op.getOperand(1);
1933 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
1934 unsigned EltBitWidth = Src.getScalarValueSizeInBits();
1935
1936 // Demand the bits from every vector element without a constant index.
1937 APInt DemandedSrcElts = APInt::getAllOnesValue(NumSrcElts);
1938 if (auto *CIdx = dyn_cast<ConstantSDNode>(Idx))
1939 if (CIdx->getAPIntValue().ult(NumSrcElts))
1940 DemandedSrcElts = APInt::getOneBitSet(NumSrcElts, CIdx->getZExtValue());
1941
1942 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
1943 // anything about the extended bits.
1944 APInt DemandedSrcBits = DemandedBits;
1945 if (BitWidth > EltBitWidth)
1946 DemandedSrcBits = DemandedSrcBits.trunc(EltBitWidth);
1947
1948 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts, Known2, TLO,
1949 Depth + 1))
1950 return true;
1951
1952 // Attempt to avoid multi-use ops if we don't need anything from them.
1953 if (!DemandedSrcBits.isAllOnesValue() ||
1954 !DemandedSrcElts.isAllOnesValue()) {
1955 if (SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
1956 Src, DemandedSrcBits, DemandedSrcElts, TLO.DAG, Depth + 1)) {
1957 SDValue NewOp =
1958 TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc, Idx);
1959 return TLO.CombineTo(Op, NewOp);
1960 }
1961 }
1962
1963 Known = Known2;
1964 if (BitWidth > EltBitWidth)
1965 Known = Known.anyext(BitWidth);
1966 break;
1967 }
1968 case ISD::BITCAST: {
1969 SDValue Src = Op.getOperand(0);
1970 EVT SrcVT = Src.getValueType();
1971 unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
1972
1973 // If this is an FP->Int bitcast and if the sign bit is the only
1974 // thing demanded, turn this into a FGETSIGN.
1975 if (!TLO.LegalOperations() && !VT.isVector() && !SrcVT.isVector() &&
1976 DemandedBits == APInt::getSignMask(Op.getValueSizeInBits()) &&
1977 SrcVT.isFloatingPoint()) {
1978 bool OpVTLegal = isOperationLegalOrCustom(ISD::FGETSIGN, VT);
1979 bool i32Legal = isOperationLegalOrCustom(ISD::FGETSIGN, MVT::i32);
1980 if ((OpVTLegal || i32Legal) && VT.isSimple() && SrcVT != MVT::f16 &&
1981 SrcVT != MVT::f128) {
1982 // Cannot eliminate/lower SHL for f128 yet.
1983 EVT Ty = OpVTLegal ? VT : MVT::i32;
1984 // Make a FGETSIGN + SHL to move the sign bit into the appropriate
1985 // place. We expect the SHL to be eliminated by other optimizations.
1986 SDValue Sign = TLO.DAG.getNode(ISD::FGETSIGN, dl, Ty, Src);
1987 unsigned OpVTSizeInBits = Op.getValueSizeInBits();
1988 if (!OpVTLegal && OpVTSizeInBits > 32)
1989 Sign = TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Sign);
1990 unsigned ShVal = Op.getValueSizeInBits() - 1;
1991 SDValue ShAmt = TLO.DAG.getConstant(ShVal, dl, VT);
1992 return TLO.CombineTo(Op,
1993 TLO.DAG.getNode(ISD::SHL, dl, VT, Sign, ShAmt));
1994 }
1995 }
1996
1997 // Bitcast from a vector using SimplifyDemanded Bits/VectorElts.
1998 // Demand the elt/bit if any of the original elts/bits are demanded.
1999 // TODO - bigendian once we have test coverage.
2000 if (SrcVT.isVector() && (BitWidth % NumSrcEltBits) == 0 &&
2001 TLO.DAG.getDataLayout().isLittleEndian()) {
2002 unsigned Scale = BitWidth / NumSrcEltBits;
2003 unsigned NumSrcElts = SrcVT.getVectorNumElements();
2004 APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);
2005 APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);
2006 for (unsigned i = 0; i != Scale; ++i) {
2007 unsigned Offset = i * NumSrcEltBits;
2008 APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset);
2009 if (!Sub.isNullValue()) {
2010 DemandedSrcBits |= Sub;
2011 for (unsigned j = 0; j != NumElts; ++j)
2012 if (DemandedElts[j])
2013 DemandedSrcElts.setBit((j * Scale) + i);
2014 }
2015 }
2016
2017 APInt KnownSrcUndef, KnownSrcZero;
2018 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
2019 KnownSrcZero, TLO, Depth + 1))
2020 return true;
2021
2022 KnownBits KnownSrcBits;
2023 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
2024 KnownSrcBits, TLO, Depth + 1))
2025 return true;
2026 } else if ((NumSrcEltBits % BitWidth) == 0 &&
2027 TLO.DAG.getDataLayout().isLittleEndian()) {
2028 unsigned Scale = NumSrcEltBits / BitWidth;
2029 unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
2030 APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);
2031 APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);
2032 for (unsigned i = 0; i != NumElts; ++i)
2033 if (DemandedElts[i]) {
2034 unsigned Offset = (i % Scale) * BitWidth;
2035 DemandedSrcBits.insertBits(DemandedBits, Offset);
2036 DemandedSrcElts.setBit(i / Scale);
2037 }
2038
2039 if (SrcVT.isVector()) {
2040 APInt KnownSrcUndef, KnownSrcZero;
2041 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
2042 KnownSrcZero, TLO, Depth + 1))
2043 return true;
2044 }
2045
2046 KnownBits KnownSrcBits;
2047 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
2048 KnownSrcBits, TLO, Depth + 1))
2049 return true;
2050 }
2051
2052 // If this is a bitcast, let computeKnownBits handle it. Only do this on a
2053 // recursive call where Known may be useful to the caller.
2054 if (Depth > 0) {
2055 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2056 return false;
2057 }
2058 break;
2059 }
2060 case ISD::ADD:
2061 case ISD::MUL:
2062 case ISD::SUB: {
2063 // Add, Sub, and Mul don't demand any bits in positions beyond that
2064 // of the highest bit demanded of them.
2065 SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1);
2066 SDNodeFlags Flags = Op.getNode()->getFlags();
2067 unsigned DemandedBitsLZ = DemandedBits.countLeadingZeros();
2068 APInt LoMask = APInt::getLowBitsSet(BitWidth, BitWidth - DemandedBitsLZ);
2069 if (SimplifyDemandedBits(Op0, LoMask, DemandedElts, Known2, TLO,
2070 Depth + 1) ||
2071 SimplifyDemandedBits(Op1, LoMask, DemandedElts, Known2, TLO,
2072 Depth + 1) ||
2073 // See if the operation should be performed at a smaller bit width.
2074 ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO)) {
2075 if (Flags.hasNoSignedWrap() || Flags.hasNoUnsignedWrap()) {
2076 // Disable the nsw and nuw flags. We can no longer guarantee that we
2077 // won't wrap after simplification.
2078 Flags.setNoSignedWrap(false);
2079 Flags.setNoUnsignedWrap(false);
2080 SDValue NewOp =
2081 TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);
2082 return TLO.CombineTo(Op, NewOp);
2083 }
2084 return true;
2085 }
2086
2087 // Attempt to avoid multi-use ops if we don't need anything from them.
2088 if (!LoMask.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {
2089 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2090 Op0, LoMask, DemandedElts, TLO.DAG, Depth + 1);
2091 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
2092 Op1, LoMask, DemandedElts, TLO.DAG, Depth + 1);
2093 if (DemandedOp0 || DemandedOp1) {
2094 Flags.setNoSignedWrap(false);
2095 Flags.setNoUnsignedWrap(false);
2096 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
2097 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
2098 SDValue NewOp =
2099 TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);
2100 return TLO.CombineTo(Op, NewOp);
2101 }
2102 }
2103
2104 // If we have a constant operand, we may be able to turn it into -1 if we
2105 // do not demand the high bits. This can make the constant smaller to
2106 // encode, allow more general folding, or match specialized instruction
2107 // patterns (eg, 'blsr' on x86). Don't bother changing 1 to -1 because that
2108 // is probably not useful (and could be detrimental).
2109 ConstantSDNode *C = isConstOrConstSplat(Op1);
2110 APInt HighMask = APInt::getHighBitsSet(BitWidth, DemandedBitsLZ);
2111 if (C && !C->isAllOnesValue() && !C->isOne() &&
2112 (C->getAPIntValue() | HighMask).isAllOnesValue()) {
2113 SDValue Neg1 = TLO.DAG.getAllOnesConstant(dl, VT);
2114 // Disable the nsw and nuw flags. We can no longer guarantee that we
2115 // won't wrap after simplification.
2116 Flags.setNoSignedWrap(false);
2117 Flags.setNoUnsignedWrap(false);
2118 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Neg1, Flags);
2119 return TLO.CombineTo(Op, NewOp);
2120 }
2121
2122 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2123 }
2124 default:
2125 if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
2126 if (SimplifyDemandedBitsForTargetNode(Op, DemandedBits, DemandedElts,
2127 Known, TLO, Depth))
2128 return true;
2129 break;
2130 }
2131
2132 // Just use computeKnownBits to compute output bits.
2133 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2134 break;
2135 }
2136
2137 // If we know the value of all of the demanded bits, return this as a
2138 // constant.
2139 if (DemandedBits.isSubsetOf(Known.Zero | Known.One)) {
2140 // Avoid folding to a constant if any OpaqueConstant is involved.
2141 const SDNode *N = Op.getNode();
2142 for (SDNodeIterator I = SDNodeIterator::begin(N),
2143 E = SDNodeIterator::end(N);
2144 I != E; ++I) {
2145 SDNode *Op = *I;
2146 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op))
2147 if (C->isOpaque())
2148 return false;
2149 }
2150 // TODO: Handle float bits as well.
2151 if (VT.isInteger())
2152 return TLO.CombineTo(Op, TLO.DAG.getConstant(Known.One, dl, VT));
2153 }
2154
2155 return false;
2156}
2157
2158bool TargetLowering::SimplifyDemandedVectorElts(SDValue Op,
2159 const APInt &DemandedElts,
2160 APInt &KnownUndef,
2161 APInt &KnownZero,
2162 DAGCombinerInfo &DCI) const {
2163 SelectionDAG &DAG = DCI.DAG;
2164 TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
2165 !DCI.isBeforeLegalizeOps());
2166
2167 bool Simplified =
2168 SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero, TLO);
2169 if (Simplified) {
2170 DCI.AddToWorklist(Op.getNode());
2171 DCI.CommitTargetLoweringOpt(TLO);
2172 }
2173
2174 return Simplified;
2175}
2176
2177/// Given a vector binary operation and known undefined elements for each input
2178/// operand, compute whether each element of the output is undefined.
2179static APInt getKnownUndefForVectorBinop(SDValue BO, SelectionDAG &DAG,
2180 const APInt &UndefOp0,
2181 const APInt &UndefOp1) {
2182 EVT VT = BO.getValueType();
2183 assert(DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() &&((DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) &&
VT.isVector() && "Vector binop only") ? static_cast<
void> (0) : __assert_fail ("DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() && \"Vector binop only\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2184, __PRETTY_FUNCTION__))
2184 "Vector binop only")((DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) &&
VT.isVector() && "Vector binop only") ? static_cast<
void> (0) : __assert_fail ("DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() && \"Vector binop only\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2184, __PRETTY_FUNCTION__))
;
2185
2186 EVT EltVT = VT.getVectorElementType();
2187 unsigned NumElts = VT.getVectorNumElements();
2188 assert(UndefOp0.getBitWidth() == NumElts &&((UndefOp0.getBitWidth() == NumElts && UndefOp1.getBitWidth
() == NumElts && "Bad type for undef analysis") ? static_cast
<void> (0) : __assert_fail ("UndefOp0.getBitWidth() == NumElts && UndefOp1.getBitWidth() == NumElts && \"Bad type for undef analysis\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2189, __PRETTY_FUNCTION__))
2189 UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis")((UndefOp0.getBitWidth() == NumElts && UndefOp1.getBitWidth
() == NumElts && "Bad type for undef analysis") ? static_cast
<void> (0) : __assert_fail ("UndefOp0.getBitWidth() == NumElts && UndefOp1.getBitWidth() == NumElts && \"Bad type for undef analysis\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2189, __PRETTY_FUNCTION__))
;
2190
2191 auto getUndefOrConstantElt = [&](SDValue V, unsigned Index,
2192 const APInt &UndefVals) {
2193 if (UndefVals[Index])
2194 return DAG.getUNDEF(EltVT);
2195
2196 if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
2197 // Try hard to make sure that the getNode() call is not creating temporary
2198 // nodes. Ignore opaque integers because they do not constant fold.
2199 SDValue Elt = BV->getOperand(Index);
2200 auto *C = dyn_cast<ConstantSDNode>(Elt);
2201 if (isa<ConstantFPSDNode>(Elt) || Elt.isUndef() || (C && !C->isOpaque()))
2202 return Elt;
2203 }
2204
2205 return SDValue();
2206 };
2207
2208 APInt KnownUndef = APInt::getNullValue(NumElts);
2209 for (unsigned i = 0; i != NumElts; ++i) {
2210 // If both inputs for this element are either constant or undef and match
2211 // the element type, compute the constant/undef result for this element of
2212 // the vector.
2213 // TODO: Ideally we would use FoldConstantArithmetic() here, but that does
2214 // not handle FP constants. The code within getNode() should be refactored
2215 // to avoid the danger of creating a bogus temporary node here.
2216 SDValue C0 = getUndefOrConstantElt(BO.getOperand(0), i, UndefOp0);
2217 SDValue C1 = getUndefOrConstantElt(BO.getOperand(1), i, UndefOp1);
2218 if (C0 && C1 && C0.getValueType() == EltVT && C1.getValueType() == EltVT)
2219 if (DAG.getNode(BO.getOpcode(), SDLoc(BO), EltVT, C0, C1).isUndef())
2220 KnownUndef.setBit(i);
2221 }
2222 return KnownUndef;
2223}
2224
2225bool TargetLowering::SimplifyDemandedVectorElts(
2226 SDValue Op, const APInt &OriginalDemandedElts, APInt &KnownUndef,
2227 APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth,
2228 bool AssumeSingleUse) const {
2229 EVT VT = Op.getValueType();
2230 APInt DemandedElts = OriginalDemandedElts;
2231 unsigned NumElts = DemandedElts.getBitWidth();
2232 assert(VT.isVector() && "Expected vector op")((VT.isVector() && "Expected vector op") ? static_cast
<void> (0) : __assert_fail ("VT.isVector() && \"Expected vector op\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2232, __PRETTY_FUNCTION__))
;
2233 assert(VT.getVectorNumElements() == NumElts &&((VT.getVectorNumElements() == NumElts && "Mask size mismatches value type element count!"
) ? static_cast<void> (0) : __assert_fail ("VT.getVectorNumElements() == NumElts && \"Mask size mismatches value type element count!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2234, __PRETTY_FUNCTION__))
2234 "Mask size mismatches value type element count!")((VT.getVectorNumElements() == NumElts && "Mask size mismatches value type element count!"
) ? static_cast<void> (0) : __assert_fail ("VT.getVectorNumElements() == NumElts && \"Mask size mismatches value type element count!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2234, __PRETTY_FUNCTION__))
;
2235
2236 KnownUndef = KnownZero = APInt::getNullValue(NumElts);
2237
2238 // Undef operand.
2239 if (Op.isUndef()) {
2240 KnownUndef.setAllBits();
2241 return false;
2242 }
2243
2244 // If Op has other users, assume that all elements are needed.
2245 if (!Op.getNode()->hasOneUse() && !AssumeSingleUse)
2246 DemandedElts.setAllBits();
2247
2248 // Not demanding any elements from Op.
2249 if (DemandedElts == 0) {
2250 KnownUndef.setAllBits();
2251 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
2252 }
2253
2254 // Limit search depth.
2255 if (Depth >= SelectionDAG::MaxRecursionDepth)
2256 return false;
2257
2258 SDLoc DL(Op);
2259 unsigned EltSizeInBits = VT.getScalarSizeInBits();
2260
2261 switch (Op.getOpcode()) {
2262 case ISD::SCALAR_TO_VECTOR: {
2263 if (!DemandedElts[0]) {
2264 KnownUndef.setAllBits();
2265 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
2266 }
2267 KnownUndef.setHighBits(NumElts - 1);
2268 break;
2269 }
2270 case ISD::BITCAST: {
2271 SDValue Src = Op.getOperand(0);
2272 EVT SrcVT = Src.getValueType();
2273
2274 // We only handle vectors here.
2275 // TODO - investigate calling SimplifyDemandedBits/ComputeKnownBits?
2276 if (!SrcVT.isVector())
2277 break;
2278
2279 // Fast handling of 'identity' bitcasts.
2280 unsigned NumSrcElts = SrcVT.getVectorNumElements();
2281 if (NumSrcElts == NumElts)
2282 return SimplifyDemandedVectorElts(Src, DemandedElts, KnownUndef,
2283 KnownZero, TLO, Depth + 1);
2284
2285 APInt SrcZero, SrcUndef;
2286 APInt SrcDemandedElts = APInt::getNullValue(NumSrcElts);
2287
2288 // Bitcast from 'large element' src vector to 'small element' vector, we
2289 // must demand a source element if any DemandedElt maps to it.
2290 if ((NumElts % NumSrcElts) == 0) {
2291 unsigned Scale = NumElts / NumSrcElts;
2292 for (unsigned i = 0; i != NumElts; ++i)
2293 if (DemandedElts[i])
2294 SrcDemandedElts.setBit(i / Scale);
2295
2296 if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2297 TLO, Depth + 1))
2298 return true;
2299
2300 // Try calling SimplifyDemandedBits, converting demanded elts to the bits
2301 // of the large element.
2302 // TODO - bigendian once we have test coverage.
2303 if (TLO.DAG.getDataLayout().isLittleEndian()) {
2304 unsigned SrcEltSizeInBits = SrcVT.getScalarSizeInBits();
2305 APInt SrcDemandedBits = APInt::getNullValue(SrcEltSizeInBits);
2306 for (unsigned i = 0; i != NumElts; ++i)
2307 if (DemandedElts[i]) {
2308 unsigned Ofs = (i % Scale) * EltSizeInBits;
2309 SrcDemandedBits.setBits(Ofs, Ofs + EltSizeInBits);
2310 }
2311
2312 KnownBits Known;
2313 if (SimplifyDemandedBits(Src, SrcDemandedBits, Known, TLO, Depth + 1))
2314 return true;
2315 }
2316
2317 // If the src element is zero/undef then all the output elements will be -
2318 // only demanded elements are guaranteed to be correct.
2319 for (unsigned i = 0; i != NumSrcElts; ++i) {
2320 if (SrcDemandedElts[i]) {
2321 if (SrcZero[i])
2322 KnownZero.setBits(i * Scale, (i + 1) * Scale);
2323 if (SrcUndef[i])
2324 KnownUndef.setBits(i * Scale, (i + 1) * Scale);
2325 }
2326 }
2327 }
2328
2329 // Bitcast from 'small element' src vector to 'large element' vector, we
2330 // demand all smaller source elements covered by the larger demanded element
2331 // of this vector.
2332 if ((NumSrcElts % NumElts) == 0) {
2333 unsigned Scale = NumSrcElts / NumElts;
2334 for (unsigned i = 0; i != NumElts; ++i)
2335 if (DemandedElts[i])
2336 SrcDemandedElts.setBits(i * Scale, (i + 1) * Scale);
2337
2338 if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2339 TLO, Depth + 1))
2340 return true;
2341
2342 // If all the src elements covering an output element are zero/undef, then
2343 // the output element will be as well, assuming it was demanded.
2344 for (unsigned i = 0; i != NumElts; ++i) {
2345 if (DemandedElts[i]) {
2346 if (SrcZero.extractBits(Scale, i * Scale).isAllOnesValue())
2347 KnownZero.setBit(i);
2348 if (SrcUndef.extractBits(Scale, i * Scale).isAllOnesValue())
2349 KnownUndef.setBit(i);
2350 }
2351 }
2352 }
2353 break;
2354 }
2355 case ISD::BUILD_VECTOR: {
2356 // Check all elements and simplify any unused elements with UNDEF.
2357 if (!DemandedElts.isAllOnesValue()) {
2358 // Don't simplify BROADCASTS.
2359 if (llvm::any_of(Op->op_values(),
2360 [&](SDValue Elt) { return Op.getOperand(0) != Elt; })) {
2361 SmallVector<SDValue, 32> Ops(Op->op_begin(), Op->op_end());
2362 bool Updated = false;
2363 for (unsigned i = 0; i != NumElts; ++i) {
2364 if (!DemandedElts[i] && !Ops[i].isUndef()) {
2365 Ops[i] = TLO.DAG.getUNDEF(Ops[0].getValueType());
2366 KnownUndef.setBit(i);
2367 Updated = true;
2368 }
2369 }
2370 if (Updated)
2371 return TLO.CombineTo(Op, TLO.DAG.getBuildVector(VT, DL, Ops));
2372 }
2373 }
2374 for (unsigned i = 0; i != NumElts; ++i) {
2375 SDValue SrcOp = Op.getOperand(i);
2376 if (SrcOp.isUndef()) {
2377 KnownUndef.setBit(i);
2378 } else if (EltSizeInBits == SrcOp.getScalarValueSizeInBits() &&
2379 (isNullConstant(SrcOp) || isNullFPConstant(SrcOp))) {
2380 KnownZero.setBit(i);
2381 }
2382 }
2383 break;
2384 }
2385 case ISD::CONCAT_VECTORS: {
2386 EVT SubVT = Op.getOperand(0).getValueType();
2387 unsigned NumSubVecs = Op.getNumOperands();
2388 unsigned NumSubElts = SubVT.getVectorNumElements();
2389 for (unsigned i = 0; i != NumSubVecs; ++i) {
2390 SDValue SubOp = Op.getOperand(i);
2391 APInt SubElts = DemandedElts.extractBits(NumSubElts, i * NumSubElts);
2392 APInt SubUndef, SubZero;
2393 if (SimplifyDemandedVectorElts(SubOp, SubElts, SubUndef, SubZero, TLO,
2394 Depth + 1))
2395 return true;
2396 KnownUndef.insertBits(SubUndef, i * NumSubElts);
2397 KnownZero.insertBits(SubZero, i * NumSubElts);
2398 }
2399 break;
2400 }
2401 case ISD::INSERT_SUBVECTOR: {
2402 if (!isa<ConstantSDNode>(Op.getOperand(2)))
2403 break;
2404 SDValue Base = Op.getOperand(0);
2405 SDValue Sub = Op.getOperand(1);
2406 EVT SubVT = Sub.getValueType();
2407 unsigned NumSubElts = SubVT.getVectorNumElements();
2408 const APInt &Idx = Op.getConstantOperandAPInt(2);
2409 if (Idx.ugt(NumElts - NumSubElts))
2410 break;
2411 unsigned SubIdx = Idx.getZExtValue();
2412 APInt SubElts = DemandedElts.extractBits(NumSubElts, SubIdx);
2413 APInt SubUndef, SubZero;
2414 if (SimplifyDemandedVectorElts(Sub, SubElts, SubUndef, SubZero, TLO,
2415 Depth + 1))
2416 return true;
2417 APInt BaseElts = DemandedElts;
2418 BaseElts.insertBits(APInt::getNullValue(NumSubElts), SubIdx);
2419
2420 // If none of the base operand elements are demanded, replace it with undef.
2421 if (!BaseElts && !Base.isUndef())
2422 return TLO.CombineTo(Op,
2423 TLO.DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
2424 TLO.DAG.getUNDEF(VT),
2425 Op.getOperand(1),
2426 Op.getOperand(2)));
2427
2428 if (SimplifyDemandedVectorElts(Base, BaseElts, KnownUndef, KnownZero, TLO,
2429 Depth + 1))
2430 return true;
2431 KnownUndef.insertBits(SubUndef, SubIdx);
2432 KnownZero.insertBits(SubZero, SubIdx);
2433 break;
2434 }
2435 case ISD::EXTRACT_SUBVECTOR: {
2436 SDValue Src = Op.getOperand(0);
2437 ConstantSDNode *SubIdx = dyn_cast<ConstantSDNode>(Op.getOperand(1));
2438 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2439 if (SubIdx && SubIdx->getAPIntValue().ule(NumSrcElts - NumElts)) {
2440 // Offset the demanded elts by the subvector index.
2441 uint64_t Idx = SubIdx->getZExtValue();
2442 APInt SrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
2443 APInt SrcUndef, SrcZero;
2444 if (SimplifyDemandedVectorElts(Src, SrcElts, SrcUndef, SrcZero, TLO,
2445 Depth + 1))
2446 return true;
2447 KnownUndef = SrcUndef.extractBits(NumElts, Idx);
2448 KnownZero = SrcZero.extractBits(NumElts, Idx);
2449 }
2450 break;
2451 }
2452 case ISD::INSERT_VECTOR_ELT: {
2453 SDValue Vec = Op.getOperand(0);
2454 SDValue Scl = Op.getOperand(1);
2455 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
2456
2457 // For a legal, constant insertion index, if we don't need this insertion
2458 // then strip it, else remove it from the demanded elts.
2459 if (CIdx && CIdx->getAPIntValue().ult(NumElts)) {
2460 unsigned Idx = CIdx->getZExtValue();
2461 if (!DemandedElts[Idx])
2462 return TLO.CombineTo(Op, Vec);
2463
2464 APInt DemandedVecElts(DemandedElts);
2465 DemandedVecElts.clearBit(Idx);
2466 if (SimplifyDemandedVectorElts(Vec, DemandedVecElts, KnownUndef,
2467 KnownZero, TLO, Depth + 1))
2468 return true;
2469
2470 KnownUndef.clearBit(Idx);
2471 if (Scl.isUndef())
2472 KnownUndef.setBit(Idx);
2473
2474 KnownZero.clearBit(Idx);
2475 if (isNullConstant(Scl) || isNullFPConstant(Scl))
2476 KnownZero.setBit(Idx);
2477 break;
2478 }
2479
2480 APInt VecUndef, VecZero;
2481 if (SimplifyDemandedVectorElts(Vec, DemandedElts, VecUndef, VecZero, TLO,
2482 Depth + 1))
2483 return true;
2484 // Without knowing the insertion index we can't set KnownUndef/KnownZero.
2485 break;
2486 }
2487 case ISD::VSELECT: {
2488 // Try to transform the select condition based on the current demanded
2489 // elements.
2490 // TODO: If a condition element is undef, we can choose from one arm of the
2491 // select (and if one arm is undef, then we can propagate that to the
2492 // result).
2493 // TODO - add support for constant vselect masks (see IR version of this).
2494 APInt UnusedUndef, UnusedZero;
2495 if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, UnusedUndef,
2496 UnusedZero, TLO, Depth + 1))
2497 return true;
2498
2499 // See if we can simplify either vselect operand.
2500 APInt DemandedLHS(DemandedElts);
2501 APInt DemandedRHS(DemandedElts);
2502 APInt UndefLHS, ZeroLHS;
2503 APInt UndefRHS, ZeroRHS;
2504 if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedLHS, UndefLHS,
2505 ZeroLHS, TLO, Depth + 1))
2506 return true;
2507 if (SimplifyDemandedVectorElts(Op.getOperand(2), DemandedRHS, UndefRHS,
2508 ZeroRHS, TLO, Depth + 1))
2509 return true;
2510
2511 KnownUndef = UndefLHS & UndefRHS;
2512 KnownZero = ZeroLHS & ZeroRHS;
2513 break;
2514 }
2515 case ISD::VECTOR_SHUFFLE: {
2516 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
2517
2518 // Collect demanded elements from shuffle operands..
2519 APInt DemandedLHS(NumElts, 0);
2520 APInt DemandedRHS(NumElts, 0);
2521 for (unsigned i = 0; i != NumElts; ++i) {
2522 int M = ShuffleMask[i];
2523 if (M < 0 || !DemandedElts[i])
2524 continue;
2525 assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range")((0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range"
) ? static_cast<void> (0) : __assert_fail ("0 <= M && M < (int)(2 * NumElts) && \"Shuffle index out of range\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2525, __PRETTY_FUNCTION__))
;
2526 if (M < (int)NumElts)
2527 DemandedLHS.setBit(M);
2528 else
2529 DemandedRHS.setBit(M - NumElts);
2530 }
2531
2532 // See if we can simplify either shuffle operand.
2533 APInt UndefLHS, ZeroLHS;
2534 APInt UndefRHS, ZeroRHS;
2535 if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedLHS, UndefLHS,
2536 ZeroLHS, TLO, Depth + 1))
2537 return true;
2538 if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedRHS, UndefRHS,
2539 ZeroRHS, TLO, Depth + 1))
2540 return true;
2541
2542 // Simplify mask using undef elements from LHS/RHS.
2543 bool Updated = false;
2544 bool IdentityLHS = true, IdentityRHS = true;
2545 SmallVector<int, 32> NewMask(ShuffleMask.begin(), ShuffleMask.end());
2546 for (unsigned i = 0; i != NumElts; ++i) {
2547 int &M = NewMask[i];
2548 if (M < 0)
2549 continue;
2550 if (!DemandedElts[i] || (M < (int)NumElts && UndefLHS[M]) ||
2551 (M >= (int)NumElts && UndefRHS[M - NumElts])) {
2552 Updated = true;
2553 M = -1;
2554 }
2555 IdentityLHS &= (M < 0) || (M == (int)i);
2556 IdentityRHS &= (M < 0) || ((M - NumElts) == i);
2557 }
2558
2559 // Update legal shuffle masks based on demanded elements if it won't reduce
2560 // to Identity which can cause premature removal of the shuffle mask.
2561 if (Updated && !IdentityLHS && !IdentityRHS && !TLO.LegalOps) {
2562 SDValue LegalShuffle =
2563 buildLegalVectorShuffle(VT, DL, Op.getOperand(0), Op.getOperand(1),
2564 NewMask, TLO.DAG);
2565 if (LegalShuffle)
2566 return TLO.CombineTo(Op, LegalShuffle);
2567 }
2568
2569 // Propagate undef/zero elements from LHS/RHS.
2570 for (unsigned i = 0; i != NumElts; ++i) {
2571 int M = ShuffleMask[i];
2572 if (M < 0) {
2573 KnownUndef.setBit(i);
2574 } else if (M < (int)NumElts) {
2575 if (UndefLHS[M])
2576 KnownUndef.setBit(i);
2577 if (ZeroLHS[M])
2578 KnownZero.setBit(i);
2579 } else {
2580 if (UndefRHS[M - NumElts])
2581 KnownUndef.setBit(i);
2582 if (ZeroRHS[M - NumElts])
2583 KnownZero.setBit(i);
2584 }
2585 }
2586 break;
2587 }
2588 case ISD::ANY_EXTEND_VECTOR_INREG:
2589 case ISD::SIGN_EXTEND_VECTOR_INREG:
2590 case ISD::ZERO_EXTEND_VECTOR_INREG: {
2591 APInt SrcUndef, SrcZero;
2592 SDValue Src = Op.getOperand(0);
2593 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2594 APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts);
2595 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
2596 Depth + 1))
2597 return true;
2598 KnownZero = SrcZero.zextOrTrunc(NumElts);
2599 KnownUndef = SrcUndef.zextOrTrunc(NumElts);
2600
2601 if (Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG &&
2602 Op.getValueSizeInBits() == Src.getValueSizeInBits() &&
2603 DemandedSrcElts == 1 && TLO.DAG.getDataLayout().isLittleEndian()) {
2604 // aext - if we just need the bottom element then we can bitcast.
2605 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
2606 }
2607
2608 if (Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) {
2609 // zext(undef) upper bits are guaranteed to be zero.
2610 if (DemandedElts.isSubsetOf(KnownUndef))
2611 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
2612 KnownUndef.clearAllBits();
2613 }
2614 break;
2615 }
2616
2617 // TODO: There are more binop opcodes that could be handled here - MUL, MIN,
2618 // MAX, saturated math, etc.
2619 case ISD::OR:
2620 case ISD::XOR:
2621 case ISD::ADD:
2622 case ISD::SUB:
2623 case ISD::FADD:
2624 case ISD::FSUB:
2625 case ISD::FMUL:
2626 case ISD::FDIV:
2627 case ISD::FREM: {
2628 APInt UndefRHS, ZeroRHS;
2629 if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedElts, UndefRHS,
2630 ZeroRHS, TLO, Depth + 1))
2631 return true;
2632 APInt UndefLHS, ZeroLHS;
2633 if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, UndefLHS,
2634 ZeroLHS, TLO, Depth + 1))
2635 return true;
2636
2637 KnownZero = ZeroLHS & ZeroRHS;
2638 KnownUndef = getKnownUndefForVectorBinop(Op, TLO.DAG, UndefLHS, UndefRHS);
2639 break;
2640 }
2641 case ISD::SHL:
2642 case ISD::SRL:
2643 case ISD::SRA:
2644 case ISD::ROTL:
2645 case ISD::ROTR: {
2646 APInt UndefRHS, ZeroRHS;
2647 if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedElts, UndefRHS,
2648 ZeroRHS, TLO, Depth + 1))
2649 return true;
2650 APInt UndefLHS, ZeroLHS;
2651 if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, UndefLHS,
2652 ZeroLHS, TLO, Depth + 1))
2653 return true;
2654
2655 KnownZero = ZeroLHS;
2656 KnownUndef = UndefLHS & UndefRHS; // TODO: use getKnownUndefForVectorBinop?
2657 break;
2658 }
2659 case ISD::MUL:
2660 case ISD::AND: {
2661 APInt SrcUndef, SrcZero;
2662 if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedElts, SrcUndef,
2663 SrcZero, TLO, Depth + 1))
2664 return true;
2665 if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,
2666 KnownZero, TLO, Depth + 1))
2667 return true;
2668
2669 // If either side has a zero element, then the result element is zero, even
2670 // if the other is an UNDEF.
2671 // TODO: Extend getKnownUndefForVectorBinop to also deal with known zeros
2672 // and then handle 'and' nodes with the rest of the binop opcodes.
2673 KnownZero |= SrcZero;
2674 KnownUndef &= SrcUndef;
2675 KnownUndef &= ~KnownZero;
2676 break;
2677 }
2678 case ISD::TRUNCATE:
2679 case ISD::SIGN_EXTEND:
2680 case ISD::ZERO_EXTEND:
2681 if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,
2682 KnownZero, TLO, Depth + 1))
2683 return true;
2684
2685 if (Op.getOpcode() == ISD::ZERO_EXTEND) {
2686 // zext(undef) upper bits are guaranteed to be zero.
2687 if (DemandedElts.isSubsetOf(KnownUndef))
2688 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
2689 KnownUndef.clearAllBits();
2690 }
2691 break;
2692 default: {
2693 if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
2694 if (SimplifyDemandedVectorEltsForTargetNode(Op, DemandedElts, KnownUndef,
2695 KnownZero, TLO, Depth))
2696 return true;
2697 } else {
2698 KnownBits Known;
2699 APInt DemandedBits = APInt::getAllOnesValue(EltSizeInBits);
2700 if (SimplifyDemandedBits(Op, DemandedBits, OriginalDemandedElts, Known,
2701 TLO, Depth, AssumeSingleUse))
2702 return true;
2703 }
2704 break;
2705 }
2706 }
2707 assert((KnownUndef & KnownZero) == 0 && "Elements flagged as undef AND zero")(((KnownUndef & KnownZero) == 0 && "Elements flagged as undef AND zero"
) ? static_cast<void> (0) : __assert_fail ("(KnownUndef & KnownZero) == 0 && \"Elements flagged as undef AND zero\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2707, __PRETTY_FUNCTION__))
;
2708
2709 // Constant fold all undef cases.
2710 // TODO: Handle zero cases as well.
2711 if (DemandedElts.isSubsetOf(KnownUndef))
2712 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
2713
2714 return false;
2715}
2716
2717/// Determine which of the bits specified in Mask are known to be either zero or
2718/// one and return them in the Known.
2719void TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
2720 KnownBits &Known,
2721 const APInt &DemandedElts,
2722 const SelectionDAG &DAG,
2723 unsigned Depth) const {
2724 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2729, __PRETTY_FUNCTION__))
2725 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2729, __PRETTY_FUNCTION__))
2726 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2729, __PRETTY_FUNCTION__))
2727 Op.getOpcode() == ISD::INTRINSIC_VOID) &&(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2729, __PRETTY_FUNCTION__))
2728 "Should use MaskedValueIsZero if you don't know whether Op"(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2729, __PRETTY_FUNCTION__))
2729 " is a target node!")(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2729, __PRETTY_FUNCTION__))
;
2730 Known.resetAll();
2731}
2732
2733void TargetLowering::computeKnownBitsForTargetInstr(
2734 GISelKnownBits &Analysis, Register R, KnownBits &Known,
2735 const APInt &DemandedElts, const MachineRegisterInfo &MRI,
2736 unsigned Depth) const {
2737 Known.resetAll();
2738}
2739
2740void TargetLowering::computeKnownBitsForFrameIndex(const SDValue Op,
2741 KnownBits &Known,
2742 const APInt &DemandedElts,
2743 const SelectionDAG &DAG,
2744 unsigned Depth) const {
2745 assert(isa<FrameIndexSDNode>(Op) && "expected FrameIndex")((isa<FrameIndexSDNode>(Op) && "expected FrameIndex"
) ? static_cast<void> (0) : __assert_fail ("isa<FrameIndexSDNode>(Op) && \"expected FrameIndex\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2745, __PRETTY_FUNCTION__))
;
2746
2747 if (unsigned Align = DAG.InferPtrAlignment(Op)) {
2748 // The low bits are known zero if the pointer is aligned.
2749 Known.Zero.setLowBits(Log2_32(Align));
2750 }
2751}
2752
2753/// This method can be implemented by targets that want to expose additional
2754/// information about sign bits to the DAG Combiner.
2755unsigned TargetLowering::ComputeNumSignBitsForTargetNode(SDValue Op,
2756 const APInt &,
2757 const SelectionDAG &,
2758 unsigned Depth) const {
2759 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2764, __PRETTY_FUNCTION__))
2760 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2764, __PRETTY_FUNCTION__))
2761 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2764, __PRETTY_FUNCTION__))
2762 Op.getOpcode() == ISD::INTRINSIC_VOID) &&(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2764, __PRETTY_FUNCTION__))
2763 "Should use ComputeNumSignBits if you don't know whether Op"(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2764, __PRETTY_FUNCTION__))
2764 " is a target node!")(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2764, __PRETTY_FUNCTION__))
;
2765 return 1;
2766}
2767
2768bool TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
2769 SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,
2770 TargetLoweringOpt &TLO, unsigned Depth) const {
2771 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2776, __PRETTY_FUNCTION__))
2772 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2776, __PRETTY_FUNCTION__))
2773 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2776, __PRETTY_FUNCTION__))
2774 Op.getOpcode() == ISD::INTRINSIC_VOID) &&(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2776, __PRETTY_FUNCTION__))
2775 "Should use SimplifyDemandedVectorElts if you don't know whether Op"(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2776, __PRETTY_FUNCTION__))
2776 " is a target node!")(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2776, __PRETTY_FUNCTION__))
;
2777 return false;
2778}
2779
2780bool TargetLowering::SimplifyDemandedBitsForTargetNode(
2781 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
2782 KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const {
2783 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2788, __PRETTY_FUNCTION__))
2784 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2788, __PRETTY_FUNCTION__))
2785 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2788, __PRETTY_FUNCTION__))
2786 Op.getOpcode() == ISD::INTRINSIC_VOID) &&(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2788, __PRETTY_FUNCTION__))
2787 "Should use SimplifyDemandedBits if you don't know whether Op"(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2788, __PRETTY_FUNCTION__))
2788 " is a target node!")(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2788, __PRETTY_FUNCTION__))
;
2789 computeKnownBitsForTargetNode(Op, Known, DemandedElts, TLO.DAG, Depth);
2790 return false;
2791}
2792
2793SDValue TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(
2794 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
2795 SelectionDAG &DAG, unsigned Depth) const {
2796 assert((((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2802, __PRETTY_FUNCTION__))
2797 (Op.getOpcode() >= ISD::BUILTIN_OP_END ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2802, __PRETTY_FUNCTION__))
2798 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2802, __PRETTY_FUNCTION__))
2799 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2802, __PRETTY_FUNCTION__))
2800 Op.getOpcode() == ISD::INTRINSIC_VOID) &&(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2802, __PRETTY_FUNCTION__))
2801 "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2802, __PRETTY_FUNCTION__))
2802 " is a target node!")(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2802, __PRETTY_FUNCTION__))
;
2803 return SDValue();
2804}
2805
2806SDValue
2807TargetLowering::buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0,
2808 SDValue N1, MutableArrayRef<int> Mask,
2809 SelectionDAG &DAG) const {
2810 bool LegalMask = isShuffleMaskLegal(Mask, VT);
2811 if (!LegalMask) {
2812 std::swap(N0, N1);
2813 ShuffleVectorSDNode::commuteMask(Mask);
2814 LegalMask = isShuffleMaskLegal(Mask, VT);
2815 }
2816
2817 if (!LegalMask)
2818 return SDValue();
2819
2820 return DAG.getVectorShuffle(VT, DL, N0, N1, Mask);
2821}
2822
2823const Constant *TargetLowering::getTargetConstantFromLoad(LoadSDNode*) const {
2824 return nullptr;
2825}
2826
2827bool TargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
2828 const SelectionDAG &DAG,
2829 bool SNaN,
2830 unsigned Depth) const {
2831 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2836, __PRETTY_FUNCTION__))
2832 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2836, __PRETTY_FUNCTION__))
2833 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2836, __PRETTY_FUNCTION__))
2834 Op.getOpcode() == ISD::INTRINSIC_VOID) &&(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2836, __PRETTY_FUNCTION__))
2835 "Should use isKnownNeverNaN if you don't know whether Op"(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2836, __PRETTY_FUNCTION__))
2836 " is a target node!")(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2836, __PRETTY_FUNCTION__))
;
2837 return false;
2838}
2839
2840// FIXME: Ideally, this would use ISD::isConstantSplatVector(), but that must
2841// work with truncating build vectors and vectors with elements of less than
2842// 8 bits.
2843bool TargetLowering::isConstTrueVal(const SDNode *N) const {
2844 if (!N)
2845 return false;
2846
2847 APInt CVal;
2848 if (auto *CN = dyn_cast<ConstantSDNode>(N)) {
2849 CVal = CN->getAPIntValue();
2850 } else if (auto *BV = dyn_cast<BuildVectorSDNode>(N)) {
2851 auto *CN = BV->getConstantSplatNode();
2852 if (!CN)
2853 return false;
2854
2855 // If this is a truncating build vector, truncate the splat value.
2856 // Otherwise, we may fail to match the expected values below.
2857 unsigned BVEltWidth = BV->getValueType(0).getScalarSizeInBits();
2858 CVal = CN->getAPIntValue();
2859 if (BVEltWidth < CVal.getBitWidth())
2860 CVal = CVal.trunc(BVEltWidth);
2861 } else {
2862 return false;
2863 }
2864
2865 switch (getBooleanContents(N->getValueType(0))) {
2866 case UndefinedBooleanContent:
2867 return CVal[0];
2868 case ZeroOrOneBooleanContent:
2869 return CVal.isOneValue();
2870 case ZeroOrNegativeOneBooleanContent:
2871 return CVal.isAllOnesValue();
2872 }
2873
2874 llvm_unreachable("Invalid boolean contents")::llvm::llvm_unreachable_internal("Invalid boolean contents",
"/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2874)
;
2875}
2876
2877bool TargetLowering::isConstFalseVal(const SDNode *N) const {
2878 if (!N)
2879 return false;
2880
2881 const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N);
2882 if (!CN) {
2883 const BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N);
2884 if (!BV)
2885 return false;
2886
2887 // Only interested in constant splats, we don't care about undef
2888 // elements in identifying boolean constants and getConstantSplatNode
2889 // returns NULL if all ops are undef;
2890 CN = BV->getConstantSplatNode();
2891 if (!CN)
2892 return false;
2893 }
2894
2895 if (getBooleanContents(N->getValueType(0)) == UndefinedBooleanContent)
2896 return !CN->getAPIntValue()[0];
2897
2898 return CN->isNullValue();
2899}
2900
2901bool TargetLowering::isExtendedTrueVal(const ConstantSDNode *N, EVT VT,
2902 bool SExt) const {
2903 if (VT == MVT::i1)
2904 return N->isOne();
2905
2906 TargetLowering::BooleanContent Cnt = getBooleanContents(VT);
2907 switch (Cnt) {
2908 case TargetLowering::ZeroOrOneBooleanContent:
2909 // An extended value of 1 is always true, unless its original type is i1,
2910 // in which case it will be sign extended to -1.
2911 return (N->isOne() && !SExt) || (SExt && (N->getValueType(0) != MVT::i1));
2912 case TargetLowering::UndefinedBooleanContent:
2913 case TargetLowering::ZeroOrNegativeOneBooleanContent:
2914 return N->isAllOnesValue() && SExt;
2915 }
2916 llvm_unreachable("Unexpected enumeration.")::llvm::llvm_unreachable_internal("Unexpected enumeration.", "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2916)
;
2917}
2918
2919/// This helper function of SimplifySetCC tries to optimize the comparison when
2920/// either operand of the SetCC node is a bitwise-and instruction.
2921SDValue TargetLowering::foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1,
2922 ISD::CondCode Cond, const SDLoc &DL,
2923 DAGCombinerInfo &DCI) const {
2924 // Match these patterns in any of their permutations:
2925 // (X & Y) == Y
2926 // (X & Y) != Y
2927 if (N1.getOpcode() == ISD::AND && N0.getOpcode() != ISD::AND)
2928 std::swap(N0, N1);
2929
2930 EVT OpVT = N0.getValueType();
2931 if (N0.getOpcode() != ISD::AND || !OpVT.isInteger() ||
2932 (Cond != ISD::SETEQ && Cond != ISD::SETNE))
2933 return SDValue();
2934
2935 SDValue X, Y;
2936 if (N0.getOperand(0) == N1) {
2937 X = N0.getOperand(1);
2938 Y = N0.getOperand(0);
2939 } else if (N0.getOperand(1) == N1) {
2940 X = N0.getOperand(0);
2941 Y = N0.getOperand(1);
2942 } else {
2943 return SDValue();
2944 }
2945
2946 SelectionDAG &DAG = DCI.DAG;
2947 SDValue Zero = DAG.getConstant(0, DL, OpVT);
2948 if (DAG.isKnownToBeAPowerOfTwo(Y)) {
2949 // Simplify X & Y == Y to X & Y != 0 if Y has exactly one bit set.
2950 // Note that where Y is variable and is known to have at most one bit set
2951 // (for example, if it is Z & 1) we cannot do this; the expressions are not
2952 // equivalent when Y == 0.
2953 assert(OpVT.isInteger())((OpVT.isInteger()) ? static_cast<void> (0) : __assert_fail
("OpVT.isInteger()", "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2953, __PRETTY_FUNCTION__))
;
2954 Cond = ISD::getSetCCInverse(Cond, OpVT);
2955 if (DCI.isBeforeLegalizeOps() ||
2956 isCondCodeLegal(Cond, N0.getSimpleValueType()))
2957 return DAG.getSetCC(DL, VT, N0, Zero, Cond);
2958 } else if (N0.hasOneUse() && hasAndNotCompare(Y)) {
2959 // If the target supports an 'and-not' or 'and-complement' logic operation,
2960 // try to use that to make a comparison operation more efficient.
2961 // But don't do this transform if the mask is a single bit because there are
2962 // more efficient ways to deal with that case (for example, 'bt' on x86 or
2963 // 'rlwinm' on PPC).
2964
2965 // Bail out if the compare operand that we want to turn into a zero is
2966 // already a zero (otherwise, infinite loop).
2967 auto *YConst = dyn_cast<ConstantSDNode>(Y);
2968 if (YConst && YConst->isNullValue())
2969 return SDValue();
2970
2971 // Transform this into: ~X & Y == 0.
2972 SDValue NotX = DAG.getNOT(SDLoc(X), X, OpVT);
2973 SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, NotX, Y);
2974 return DAG.getSetCC(DL, VT, NewAnd, Zero, Cond);
2975 }
2976
2977 return SDValue();
2978}
2979
2980/// There are multiple IR patterns that could be checking whether certain
2981/// truncation of a signed number would be lossy or not. The pattern which is
2982/// best at IR level, may not lower optimally. Thus, we want to unfold it.
2983/// We are looking for the following pattern: (KeptBits is a constant)
2984/// (add %x, (1 << (KeptBits-1))) srccond (1 << KeptBits)
2985/// KeptBits won't be bitwidth(x), that will be constant-folded to true/false.
2986/// KeptBits also can't be 1, that would have been folded to %x dstcond 0
2987/// We will unfold it into the natural trunc+sext pattern:
2988/// ((%x << C) a>> C) dstcond %x
2989/// Where C = bitwidth(x) - KeptBits and C u< bitwidth(x)
2990SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(
2991 EVT SCCVT, SDValue N0, SDValue N1, ISD::CondCode Cond, DAGCombinerInfo &DCI,
2992 const SDLoc &DL) const {
2993 // We must be comparing with a constant.
2994 ConstantSDNode *C1;
2995 if (!(C1 = dyn_cast<ConstantSDNode>(N1)))
2996 return SDValue();
2997
2998 // N0 should be: add %x, (1 << (KeptBits-1))
2999 if (N0->getOpcode() != ISD::ADD)
3000 return SDValue();
3001
3002 // And we must be 'add'ing a constant.
3003 ConstantSDNode *C01;
3004 if (!(C01 = dyn_cast<ConstantSDNode>(N0->getOperand(1))))
3005 return SDValue();
3006
3007 SDValue X = N0->getOperand(0);
3008 EVT XVT = X.getValueType();
3009
3010 // Validate constants ...
3011
3012 APInt I1 = C1->getAPIntValue();
3013
3014 ISD::CondCode NewCond;
3015 if (Cond == ISD::CondCode::SETULT) {
3016 NewCond = ISD::CondCode::SETEQ;
3017 } else if (Cond == ISD::CondCode::SETULE) {
3018 NewCond = ISD::CondCode::SETEQ;
3019 // But need to 'canonicalize' the constant.
3020 I1 += 1;
3021 } else if (Cond == ISD::CondCode::SETUGT) {
3022 NewCond = ISD::CondCode::SETNE;
3023 // But need to 'canonicalize' the constant.
3024 I1 += 1;
3025 } else if (Cond == ISD::CondCode::SETUGE) {
3026 NewCond = ISD::CondCode::SETNE;
3027 } else
3028 return SDValue();
3029
3030 APInt I01 = C01->getAPIntValue();
3031
3032 auto checkConstants = [&I1, &I01]() -> bool {
3033 // Both of them must be power-of-two, and the constant from setcc is bigger.
3034 return I1.ugt(I01) && I1.isPowerOf2() && I01.isPowerOf2();
3035 };
3036
3037 if (checkConstants()) {
3038 // Great, e.g. got icmp ult i16 (add i16 %x, 128), 256
3039 } else {
3040 // What if we invert constants? (and the target predicate)
3041 I1.negate();
3042 I01.negate();
3043 assert(XVT.isInteger())((XVT.isInteger()) ? static_cast<void> (0) : __assert_fail
("XVT.isInteger()", "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3043, __PRETTY_FUNCTION__))
;
3044 NewCond = getSetCCInverse(NewCond, XVT);
3045 if (!checkConstants())
3046 return SDValue();
3047 // Great, e.g. got icmp uge i16 (add i16 %x, -128), -256
3048 }
3049
3050 // They are power-of-two, so which bit is set?
3051 const unsigned KeptBits = I1.logBase2();
3052 const unsigned KeptBitsMinusOne = I01.logBase2();
3053
3054 // Magic!
3055 if (KeptBits != (KeptBitsMinusOne + 1))
3056 return SDValue();
3057 assert(KeptBits > 0 && KeptBits < XVT.getSizeInBits() && "unreachable")((KeptBits > 0 && KeptBits < XVT.getSizeInBits(
) && "unreachable") ? static_cast<void> (0) : __assert_fail
("KeptBits > 0 && KeptBits < XVT.getSizeInBits() && \"unreachable\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3057, __PRETTY_FUNCTION__))
;
3058
3059 // We don't want to do this in every single case.
3060 SelectionDAG &DAG = DCI.DAG;
3061 if (!DAG.getTargetLoweringInfo().shouldTransformSignedTruncationCheck(
3062 XVT, KeptBits))
3063 return SDValue();
3064
3065 const unsigned MaskedBits = XVT.getSizeInBits() - KeptBits;
3066 assert(MaskedBits > 0 && MaskedBits < XVT.getSizeInBits() && "unreachable")((MaskedBits > 0 && MaskedBits < XVT.getSizeInBits
() && "unreachable") ? static_cast<void> (0) : __assert_fail
("MaskedBits > 0 && MaskedBits < XVT.getSizeInBits() && \"unreachable\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3066, __PRETTY_FUNCTION__))
;
3067
3068 // Unfold into: ((%x << C) a>> C) cond %x
3069 // Where 'cond' will be either 'eq' or 'ne'.
3070 SDValue ShiftAmt = DAG.getConstant(MaskedBits, DL, XVT);
3071 SDValue T0 = DAG.getNode(ISD::SHL, DL, XVT, X, ShiftAmt);
3072 SDValue T1 = DAG.getNode(ISD::SRA, DL, XVT, T0, ShiftAmt);
3073 SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, X, NewCond);
3074
3075 return T2;
3076}
3077
3078// (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
3079SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(
3080 EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond,
3081 DAGCombinerInfo &DCI, const SDLoc &DL) const {
3082 assert(isConstOrConstSplat(N1C) &&((isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C
)->getAPIntValue().isNullValue() && "Should be a comparison with 0."
) ? static_cast<void> (0) : __assert_fail ("isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() && \"Should be a comparison with 0.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3084, __PRETTY_FUNCTION__))
3083 isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() &&((isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C
)->getAPIntValue().isNullValue() && "Should be a comparison with 0."
) ? static_cast<void> (0) : __assert_fail ("isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() && \"Should be a comparison with 0.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3084, __PRETTY_FUNCTION__))
3084 "Should be a comparison with 0.")((isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C
)->getAPIntValue().isNullValue() && "Should be a comparison with 0."
) ? static_cast<void> (0) : __assert_fail ("isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() && \"Should be a comparison with 0.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3084, __PRETTY_FUNCTION__))
;
3085 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&(((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Valid only for [in]equality comparisons."
) ? static_cast<void> (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Valid only for [in]equality comparisons.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3086, __PRETTY_FUNCTION__))
3086 "Valid only for [in]equality comparisons.")(((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Valid only for [in]equality comparisons."
) ? static_cast<void> (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Valid only for [in]equality comparisons.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3086, __PRETTY_FUNCTION__))
;
3087
3088 unsigned NewShiftOpcode;
3089 SDValue X, C, Y;
3090
3091 SelectionDAG &DAG = DCI.DAG;
3092 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3093
3094 // Look for '(C l>>/<< Y)'.
3095 auto Match = [&NewShiftOpcode, &X, &C, &Y, &TLI, &DAG](SDValue V) {
3096 // The shift should be one-use.
3097 if (!V.hasOneUse())
3098 return false;
3099 unsigned OldShiftOpcode = V.getOpcode();
3100 switch (OldShiftOpcode) {
3101 case ISD::SHL:
3102 NewShiftOpcode = ISD::SRL;
3103 break;
3104 case ISD::SRL:
3105 NewShiftOpcode = ISD::SHL;
3106 break;
3107 default:
3108 return false; // must be a logical shift.
3109 }
3110 // We should be shifting a constant.
3111 // FIXME: best to use isConstantOrConstantVector().
3112 C = V.getOperand(0);
3113 ConstantSDNode *CC =
3114 isConstOrConstSplat(C, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
3115 if (!CC)
3116 return false;
3117 Y = V.getOperand(1);
3118
3119 ConstantSDNode *XC =
3120 isConstOrConstSplat(X, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
3121 return TLI.shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
3122 X, XC, CC, Y, OldShiftOpcode, NewShiftOpcode, DAG);
3123 };
3124
3125 // LHS of comparison should be an one-use 'and'.
3126 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())
3127 return SDValue();
3128
3129 X = N0.getOperand(0);
3130 SDValue Mask = N0.getOperand(1);
3131
3132 // 'and' is commutative!
3133 if (!Match(Mask)) {
3134 std::swap(X, Mask);
3135 if (!Match(Mask))
3136 return SDValue();
3137 }
3138
3139 EVT VT = X.getValueType();
3140
3141 // Produce:
3142 // ((X 'OppositeShiftOpcode' Y) & C) Cond 0
3143 SDValue T0 = DAG.getNode(NewShiftOpcode, DL, VT, X, Y);
3144 SDValue T1 = DAG.getNode(ISD::AND, DL, VT, T0, C);
3145 SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, N1C, Cond);
3146 return T2;
3147}
3148
3149/// Try to fold an equality comparison with a {add/sub/xor} binary operation as
3150/// the 1st operand (N0). Callers are expected to swap the N0/N1 parameters to
3151/// handle the commuted versions of these patterns.
3152SDValue TargetLowering::foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1,
3153 ISD::CondCode Cond, const SDLoc &DL,
3154 DAGCombinerInfo &DCI) const {
3155 unsigned BOpcode = N0.getOpcode();
3156 assert((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) &&(((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD
::XOR) && "Unexpected binop") ? static_cast<void>
(0) : __assert_fail ("(BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) && \"Unexpected binop\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3157, __PRETTY_FUNCTION__))
3157 "Unexpected binop")(((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD
::XOR) && "Unexpected binop") ? static_cast<void>
(0) : __assert_fail ("(BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) && \"Unexpected binop\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3157, __PRETTY_FUNCTION__))
;
3158 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Unexpected condcode")(((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Unexpected condcode"
) ? static_cast<void> (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Unexpected condcode\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3158, __PRETTY_FUNCTION__))
;
3159
3160 // (X + Y) == X --> Y == 0
3161 // (X - Y) == X --> Y == 0
3162 // (X ^ Y) == X --> Y == 0
3163 SelectionDAG &DAG = DCI.DAG;
3164 EVT OpVT = N0.getValueType();
3165 SDValue X = N0.getOperand(0);
3166 SDValue Y = N0.getOperand(1);
3167 if (X == N1)
3168 return DAG.getSetCC(DL, VT, Y, DAG.getConstant(0, DL, OpVT), Cond);
3169
3170 if (Y != N1)
3171 return SDValue();
3172
3173 // (X + Y) == Y --> X == 0
3174 // (X ^ Y) == Y --> X == 0
3175 if (BOpcode == ISD::ADD || BOpcode == ISD::XOR)
3176 return DAG.getSetCC(DL, VT, X, DAG.getConstant(0, DL, OpVT), Cond);
3177
3178 // The shift would not be valid if the operands are boolean (i1).
3179 if (!N0.hasOneUse() || OpVT.getScalarSizeInBits() == 1)
3180 return SDValue();
3181
3182 // (X - Y) == Y --> X == Y << 1
3183 EVT ShiftVT = getShiftAmountTy(OpVT, DAG.getDataLayout(),
3184 !DCI.isBeforeLegalize());
3185 SDValue One = DAG.getConstant(1, DL, ShiftVT);
3186 SDValue YShl1 = DAG.getNode(ISD::SHL, DL, N1.getValueType(), Y, One);
3187 if (!DCI.isCalledByLegalizer())
3188 DCI.AddToWorklist(YShl1.getNode());
3189 return DAG.getSetCC(DL, VT, X, YShl1, Cond);
3190}
3191
3192/// Try to simplify a setcc built with the specified operands and cc. If it is
3193/// unable to simplify it, return a null SDValue.
3194SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
3195 ISD::CondCode Cond, bool foldBooleans,
3196 DAGCombinerInfo &DCI,
3197 const SDLoc &dl) const {
3198 SelectionDAG &DAG = DCI.DAG;
3199 const DataLayout &Layout = DAG.getDataLayout();
3200 EVT OpVT = N0.getValueType();
3201
3202 // Constant fold or commute setcc.
3203 if (SDValue Fold = DAG.FoldSetCC(VT, N0, N1, Cond, dl))
3204 return Fold;
3205
3206 // Ensure that the constant occurs on the RHS and fold constant comparisons.
3207 // TODO: Handle non-splat vector constants. All undef causes trouble.
3208 ISD::CondCode SwappedCC = ISD::getSetCCSwappedOperands(Cond);
3209 if (isConstOrConstSplat(N0) &&
3210 (DCI.isBeforeLegalizeOps() ||
3211 isCondCodeLegal(SwappedCC, N0.getSimpleValueType())))
3212 return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);
3213
3214 // If we have a subtract with the same 2 non-constant operands as this setcc
3215 // -- but in reverse order -- then try to commute the operands of this setcc
3216 // to match. A matching pair of setcc (cmp) and sub may be combined into 1
3217 // instruction on some targets.
3218 if (!isConstOrConstSplat(N0) && !isConstOrConstSplat(N1) &&
3219 (DCI.isBeforeLegalizeOps() ||
3220 isCondCodeLegal(SwappedCC, N0.getSimpleValueType())) &&
3221 DAG.getNodeIfExists(ISD::SUB, DAG.getVTList(OpVT), { N1, N0 } ) &&
3222 !DAG.getNodeIfExists(ISD::SUB, DAG.getVTList(OpVT), { N0, N1 } ))
3223 return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);
3224
3225 if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
3226 const APInt &C1 = N1C->getAPIntValue();
3227
3228 // If the LHS is '(srl (ctlz x), 5)', the RHS is 0/1, and this is an
3229 // equality comparison, then we're just comparing whether X itself is
3230 // zero.
3231 if (N0.getOpcode() == ISD::SRL && (C1.isNullValue() || C1.isOneValue()) &&
3232 N0.getOperand(0).getOpcode() == ISD::CTLZ &&
3233 N0.getOperand(1).getOpcode() == ISD::Constant) {
3234 const APInt &ShAmt = N0.getConstantOperandAPInt(1);
3235 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3236 ShAmt == Log2_32(N0.getValueSizeInBits())) {
3237 if ((C1 == 0) == (Cond == ISD::SETEQ)) {
3238 // (srl (ctlz x), 5) == 0 -> X != 0
3239 // (srl (ctlz x), 5) != 1 -> X != 0
3240 Cond = ISD::SETNE;
3241 } else {
3242 // (srl (ctlz x), 5) != 0 -> X == 0
3243 // (srl (ctlz x), 5) == 1 -> X == 0
3244 Cond = ISD::SETEQ;
3245 }
3246 SDValue Zero = DAG.getConstant(0, dl, N0.getValueType());
3247 return DAG.getSetCC(dl, VT, N0.getOperand(0).getOperand(0),
3248 Zero, Cond);
3249 }
3250 }
3251
3252 SDValue CTPOP = N0;
3253 // Look through truncs that don't change the value of a ctpop.
3254 if (N0.hasOneUse() && N0.getOpcode() == ISD::TRUNCATE)
3255 CTPOP = N0.getOperand(0);
3256
3257 if (CTPOP.hasOneUse() && CTPOP.getOpcode() == ISD::CTPOP &&
3258 (N0 == CTPOP ||
3259 N0.getValueSizeInBits() > Log2_32_Ceil(CTPOP.getValueSizeInBits()))) {
3260 EVT CTVT = CTPOP.getValueType();
3261 SDValue CTOp = CTPOP.getOperand(0);
3262
3263 // (ctpop x) u< 2 -> (x & x-1) == 0
3264 // (ctpop x) u> 1 -> (x & x-1) != 0
3265 if ((Cond == ISD::SETULT && C1 == 2) || (Cond == ISD::SETUGT && C1 == 1)){
3266 SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
3267 SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, CTOp, NegOne);
3268 SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Add);
3269 ISD::CondCode CC = Cond == ISD::SETULT ? ISD::SETEQ : ISD::SETNE;
3270 return DAG.getSetCC(dl, VT, And, DAG.getConstant(0, dl, CTVT), CC);
3271 }
3272
3273 // If ctpop is not supported, expand a power-of-2 comparison based on it.
3274 if (C1 == 1 && !isOperationLegalOrCustom(ISD::CTPOP, CTVT) &&
3275 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
3276 // (ctpop x) == 1 --> (x != 0) && ((x & x-1) == 0)
3277 // (ctpop x) != 1 --> (x == 0) || ((x & x-1) != 0)
3278 SDValue Zero = DAG.getConstant(0, dl, CTVT);
3279 SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
3280 assert(CTVT.isInteger())((CTVT.isInteger()) ? static_cast<void> (0) : __assert_fail
("CTVT.isInteger()", "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3280, __PRETTY_FUNCTION__))
;
3281 ISD::CondCode InvCond = ISD::getSetCCInverse(Cond, CTVT);
3282 SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, CTOp, NegOne);
3283 SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Add);
3284 SDValue LHS = DAG.getSetCC(dl, VT, CTOp, Zero, InvCond);
3285 SDValue RHS = DAG.getSetCC(dl, VT, And, Zero, Cond);
3286 unsigned LogicOpcode = Cond == ISD::SETEQ ? ISD::AND : ISD::OR;
3287 return DAG.getNode(LogicOpcode, dl, VT, LHS, RHS);
3288 }
3289 }
3290
3291 // (zext x) == C --> x == (trunc C)
3292 // (sext x) == C --> x == (trunc C)
3293 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3294 DCI.isBeforeLegalize() && N0->hasOneUse()) {
3295 unsigned MinBits = N0.getValueSizeInBits();
3296 SDValue PreExt;
3297 bool Signed = false;
3298 if (N0->getOpcode() == ISD::ZERO_EXTEND) {
3299 // ZExt
3300 MinBits = N0->getOperand(0).getValueSizeInBits();
3301 PreExt = N0->getOperand(0);
3302 } else if (N0->getOpcode() == ISD::AND) {
3303 // DAGCombine turns costly ZExts into ANDs
3304 if (auto *C = dyn_cast<ConstantSDNode>(N0->getOperand(1)))
3305 if ((C->getAPIntValue()+1).isPowerOf2()) {
3306 MinBits = C->getAPIntValue().countTrailingOnes();
3307 PreExt = N0->getOperand(0);
3308 }
3309 } else if (N0->getOpcode() == ISD::SIGN_EXTEND) {
3310 // SExt
3311 MinBits = N0->getOperand(0).getValueSizeInBits();
3312 PreExt = N0->getOperand(0);
3313 Signed = true;
3314 } else if (auto *LN0 = dyn_cast<LoadSDNode>(N0)) {
3315 // ZEXTLOAD / SEXTLOAD
3316 if (LN0->getExtensionType() == ISD::ZEXTLOAD) {
3317 MinBits = LN0->getMemoryVT().getSizeInBits();
3318 PreExt = N0;
3319 } else if (LN0->getExtensionType() == ISD::SEXTLOAD) {
3320 Signed = true;
3321 MinBits = LN0->getMemoryVT().getSizeInBits();
3322 PreExt = N0;
3323 }
3324 }
3325
3326 // Figure out how many bits we need to preserve this constant.
3327 unsigned ReqdBits = Signed ?
3328 C1.getBitWidth() - C1.getNumSignBits() + 1 :
3329 C1.getActiveBits();
3330
3331 // Make sure we're not losing bits from the constant.
3332 if (MinBits > 0 &&
3333 MinBits < C1.getBitWidth() &&
3334 MinBits >= ReqdBits) {
3335 EVT MinVT = EVT::getIntegerVT(*DAG.getContext(), MinBits);
3336 if (isTypeDesirableForOp(ISD::SETCC, MinVT)) {
3337 // Will get folded away.
3338 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MinVT, PreExt);
3339 if (MinBits == 1 && C1 == 1)
3340 // Invert the condition.
3341 return DAG.getSetCC(dl, VT, Trunc, DAG.getConstant(0, dl, MVT::i1),
3342 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
3343 SDValue C = DAG.getConstant(C1.trunc(MinBits), dl, MinVT);
3344 return DAG.getSetCC(dl, VT, Trunc, C, Cond);
3345 }
3346
3347 // If truncating the setcc operands is not desirable, we can still
3348 // simplify the expression in some cases:
3349 // setcc ([sz]ext (setcc x, y, cc)), 0, setne) -> setcc (x, y, cc)
3350 // setcc ([sz]ext (setcc x, y, cc)), 0, seteq) -> setcc (x, y, inv(cc))
3351 // setcc (zext (setcc x, y, cc)), 1, setne) -> setcc (x, y, inv(cc))
3352 // setcc (zext (setcc x, y, cc)), 1, seteq) -> setcc (x, y, cc)
3353 // setcc (sext (setcc x, y, cc)), -1, setne) -> setcc (x, y, inv(cc))
3354 // setcc (sext (setcc x, y, cc)), -1, seteq) -> setcc (x, y, cc)
3355 SDValue TopSetCC = N0->getOperand(0);
3356 unsigned N0Opc = N0->getOpcode();
3357 bool SExt = (N0Opc == ISD::SIGN_EXTEND);
3358 if (TopSetCC.getValueType() == MVT::i1 && VT == MVT::i1 &&
3359 TopSetCC.getOpcode() == ISD::SETCC &&
3360 (N0Opc == ISD::ZERO_EXTEND || N0Opc == ISD::SIGN_EXTEND) &&
3361 (isConstFalseVal(N1C) ||
3362 isExtendedTrueVal(N1C, N0->getValueType(0), SExt))) {
3363
3364 bool Inverse = (N1C->isNullValue() && Cond == ISD::SETEQ) ||
3365 (!N1C->isNullValue() && Cond == ISD::SETNE);
3366
3367 if (!Inverse)
3368 return TopSetCC;
3369
3370 ISD::CondCode InvCond = ISD::getSetCCInverse(
3371 cast<CondCodeSDNode>(TopSetCC.getOperand(2))->get(),
3372 TopSetCC.getOperand(0).getValueType());
3373 return DAG.getSetCC(dl, VT, TopSetCC.getOperand(0),
3374 TopSetCC.getOperand(1),
3375 InvCond);
3376 }
3377 }
3378 }
3379
3380 // If the LHS is '(and load, const)', the RHS is 0, the test is for
3381 // equality or unsigned, and all 1 bits of the const are in the same
3382 // partial word, see if we can shorten the load.
3383 if (DCI.isBeforeLegalize() &&
3384 !ISD::isSignedIntSetCC(Cond) &&
3385 N0.getOpcode() == ISD::AND && C1 == 0 &&
3386 N0.getNode()->hasOneUse() &&
3387 isa<LoadSDNode>(N0.getOperand(0)) &&
3388 N0.getOperand(0).getNode()->hasOneUse() &&
3389 isa<ConstantSDNode>(N0.getOperand(1))) {
3390 LoadSDNode *Lod = cast<LoadSDNode>(N0.getOperand(0));
3391 APInt bestMask;
3392 unsigned bestWidth = 0, bestOffset = 0;
3393 if (Lod->isSimple() && Lod->isUnindexed()) {
3394 unsigned origWidth = N0.getValueSizeInBits();
3395 unsigned maskWidth = origWidth;
3396 // We can narrow (e.g.) 16-bit extending loads on 32-bit target to
3397 // 8 bits, but have to be careful...
3398 if (Lod->getExtensionType() != ISD::NON_EXTLOAD)
3399 origWidth = Lod->getMemoryVT().getSizeInBits();
3400 const APInt &Mask = N0.getConstantOperandAPInt(1);
3401 for (unsigned width = origWidth / 2; width>=8; width /= 2) {
3402 APInt newMask = APInt::getLowBitsSet(maskWidth, width);
3403 for (unsigned offset=0; offset<origWidth/width; offset++) {
3404 if (Mask.isSubsetOf(newMask)) {
3405 if (Layout.isLittleEndian())
3406 bestOffset = (uint64_t)offset * (width/8);
3407 else
3408 bestOffset = (origWidth/width - offset - 1) * (width/8);
3409 bestMask = Mask.lshr(offset * (width/8) * 8);
3410 bestWidth = width;
3411 break;
3412 }
3413 newMask <<= width;
3414 }
3415 }
3416 }
3417 if (bestWidth) {
3418 EVT newVT = EVT::getIntegerVT(*DAG.getContext(), bestWidth);
3419 if (newVT.isRound() &&
3420 shouldReduceLoadWidth(Lod, ISD::NON_EXTLOAD, newVT)) {
3421 SDValue Ptr = Lod->getBasePtr();
3422 if (bestOffset != 0)
3423 Ptr = DAG.getMemBasePlusOffset(Ptr, bestOffset, dl);
3424 unsigned NewAlign = MinAlign(Lod->getAlignment(), bestOffset);
3425 SDValue NewLoad = DAG.getLoad(
3426 newVT, dl, Lod->getChain(), Ptr,
3427 Lod->getPointerInfo().getWithOffset(bestOffset), NewAlign);
3428 return DAG.getSetCC(dl, VT,
3429 DAG.getNode(ISD::AND, dl, newVT, NewLoad,
3430 DAG.getConstant(bestMask.trunc(bestWidth),
3431 dl, newVT)),
3432 DAG.getConstant(0LL, dl, newVT), Cond);
3433 }
3434 }
3435 }
3436
3437 // If the LHS is a ZERO_EXTEND, perform the comparison on the input.
3438 if (N0.getOpcode() == ISD::ZERO_EXTEND) {
3439 unsigned InSize = N0.getOperand(0).getValueSizeInBits();
3440
3441 // If the comparison constant has bits in the upper part, the
3442 // zero-extended value could never match.
3443 if (C1.intersects(APInt::getHighBitsSet(C1.getBitWidth(),
3444 C1.getBitWidth() - InSize))) {
3445 switch (Cond) {
3446 case ISD::SETUGT:
3447 case ISD::SETUGE:
3448 case ISD::SETEQ:
3449 return DAG.getConstant(0, dl, VT);
3450 case ISD::SETULT:
3451 case ISD::SETULE:
3452 case ISD::SETNE:
3453 return DAG.getConstant(1, dl, VT);
3454 case ISD::SETGT:
3455 case ISD::SETGE:
3456 // True if the sign bit of C1 is set.
3457 return DAG.getConstant(C1.isNegative(), dl, VT);
3458 case ISD::SETLT:
3459 case ISD::SETLE:
3460 // True if the sign bit of C1 isn't set.
3461 return DAG.getConstant(C1.isNonNegative(), dl, VT);
3462 default:
3463 break;
3464 }
3465 }
3466
3467 // Otherwise, we can perform the comparison with the low bits.
3468 switch (Cond) {
3469 case ISD::SETEQ:
3470 case ISD::SETNE:
3471 case ISD::SETUGT:
3472 case ISD::SETUGE:
3473 case ISD::SETULT:
3474 case ISD::SETULE: {
3475 EVT newVT = N0.getOperand(0).getValueType();
3476 if (DCI.isBeforeLegalizeOps() ||
3477 (isOperationLegal(ISD::SETCC, newVT) &&
3478 isCondCodeLegal(Cond, newVT.getSimpleVT()))) {
3479 EVT NewSetCCVT = getSetCCResultType(Layout, *DAG.getContext(), newVT);
3480 SDValue NewConst = DAG.getConstant(C1.trunc(InSize), dl, newVT);
3481
3482 SDValue NewSetCC = DAG.getSetCC(dl, NewSetCCVT, N0.getOperand(0),
3483 NewConst, Cond);
3484 return DAG.getBoolExtOrTrunc(NewSetCC, dl, VT, N0.getValueType());
3485 }
3486 break;
3487 }
3488 default:
3489 break; // todo, be more careful with signed comparisons
3490 }
3491 } else if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
3492 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
3493 EVT ExtSrcTy = cast<VTSDNode>(N0.getOperand(1))->getVT();
3494 unsigned ExtSrcTyBits = ExtSrcTy.getSizeInBits();
3495 EVT ExtDstTy = N0.getValueType();
3496 unsigned ExtDstTyBits = ExtDstTy.getSizeInBits();
3497
3498 // If the constant doesn't fit into the number of bits for the source of
3499 // the sign extension, it is impossible for both sides to be equal.
3500 if (C1.getMinSignedBits() > ExtSrcTyBits)
3501 return DAG.getConstant(Cond == ISD::SETNE, dl, VT);
3502
3503 SDValue ZextOp;
3504 EVT Op0Ty = N0.getOperand(0).getValueType();
3505 if (Op0Ty == ExtSrcTy) {
3506 ZextOp = N0.getOperand(0);
3507 } else {
3508 APInt Imm = APInt::getLowBitsSet(ExtDstTyBits, ExtSrcTyBits);
3509 ZextOp = DAG.getNode(ISD::AND, dl, Op0Ty, N0.getOperand(0),
3510 DAG.getConstant(Imm, dl, Op0Ty));
3511 }
3512 if (!DCI.isCalledByLegalizer())
3513 DCI.AddToWorklist(ZextOp.getNode());
3514 // Otherwise, make this a use of a zext.
3515 return DAG.getSetCC(dl, VT, ZextOp,
3516 DAG.getConstant(C1 & APInt::getLowBitsSet(
3517 ExtDstTyBits,
3518 ExtSrcTyBits),
3519 dl, ExtDstTy),
3520 Cond);
3521 } else if ((N1C->isNullValue() || N1C->isOne()) &&
3522 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
3523 // SETCC (SETCC), [0|1], [EQ|NE] -> SETCC
3524 if (N0.getOpcode() == ISD::SETCC &&
3525 isTypeLegal(VT) && VT.bitsLE(N0.getValueType()) &&
3526 (N0.getValueType() == MVT::i1 ||
3527 getBooleanContents(N0.getOperand(0).getValueType()) ==
3528 ZeroOrOneBooleanContent)) {
3529 bool TrueWhenTrue = (Cond == ISD::SETEQ) ^ (!N1C->isOne());
3530 if (TrueWhenTrue)
3531 return DAG.getNode(ISD::TRUNCATE, dl, VT, N0);
3532 // Invert the condition.
3533 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
3534 CC = ISD::getSetCCInverse(CC, N0.getOperand(0).getValueType());
3535 if (DCI.isBeforeLegalizeOps() ||
3536 isCondCodeLegal(CC, N0.getOperand(0).getSimpleValueType()))
3537 return DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC);
3538 }
3539
3540 if ((N0.getOpcode() == ISD::XOR ||
3541 (N0.getOpcode() == ISD::AND &&
3542 N0.getOperand(0).getOpcode() == ISD::XOR &&
3543 N0.getOperand(1) == N0.getOperand(0).getOperand(1))) &&
3544 isa<ConstantSDNode>(N0.getOperand(1)) &&
3545 cast<ConstantSDNode>(N0.getOperand(1))->isOne()) {
3546 // If this is (X^1) == 0/1, swap the RHS and eliminate the xor. We
3547 // can only do this if the top bits are known zero.
3548 unsigned BitWidth = N0.getValueSizeInBits();
3549 if (DAG.MaskedValueIsZero(N0,
3550 APInt::getHighBitsSet(BitWidth,
3551 BitWidth-1))) {
3552 // Okay, get the un-inverted input value.
3553 SDValue Val;
3554 if (N0.getOpcode() == ISD::XOR) {
3555 Val = N0.getOperand(0);
3556 } else {
3557 assert(N0.getOpcode() == ISD::AND &&((N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode
() == ISD::XOR) ? static_cast<void> (0) : __assert_fail
("N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::XOR"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3558, __PRETTY_FUNCTION__))
3558 N0.getOperand(0).getOpcode() == ISD::XOR)((N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode
() == ISD::XOR) ? static_cast<void> (0) : __assert_fail
("N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::XOR"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3558, __PRETTY_FUNCTION__))
;
3559 // ((X^1)&1)^1 -> X & 1
3560 Val = DAG.getNode(ISD::AND, dl, N0.getValueType(),
3561 N0.getOperand(0).getOperand(0),
3562 N0.getOperand(1));
3563 }
3564
3565 return DAG.getSetCC(dl, VT, Val, N1,
3566 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
3567 }
3568 } else if (N1C->isOne()) {
3569 SDValue Op0 = N0;
3570 if (Op0.getOpcode() == ISD::TRUNCATE)
3571 Op0 = Op0.getOperand(0);
3572
3573 if ((Op0.getOpcode() == ISD::XOR) &&
3574 Op0.getOperand(0).getOpcode() == ISD::SETCC &&
3575 Op0.getOperand(1).getOpcode() == ISD::SETCC) {
3576 SDValue XorLHS = Op0.getOperand(0);
3577 SDValue XorRHS = Op0.getOperand(1);
3578 // Ensure that the input setccs return an i1 type or 0/1 value.
3579 if (Op0.getValueType() == MVT::i1 ||
3580 (getBooleanContents(XorLHS.getOperand(0).getValueType()) ==
3581 ZeroOrOneBooleanContent &&
3582 getBooleanContents(XorRHS.getOperand(0).getValueType()) ==
3583 ZeroOrOneBooleanContent)) {
3584 // (xor (setcc), (setcc)) == / != 1 -> (setcc) != / == (setcc)
3585 Cond = (Cond == ISD::SETEQ) ? ISD::SETNE : ISD::SETEQ;
3586 return DAG.getSetCC(dl, VT, XorLHS, XorRHS, Cond);
3587 }
3588 }
3589 if (Op0.getOpcode() == ISD::AND &&
3590 isa<ConstantSDNode>(Op0.getOperand(1)) &&
3591 cast<ConstantSDNode>(Op0.getOperand(1))->isOne()) {
3592 // If this is (X&1) == / != 1, normalize it to (X&1) != / == 0.
3593 if (Op0.getValueType().bitsGT(VT))
3594 Op0 = DAG.getNode(ISD::AND, dl, VT,
3595 DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)),
3596 DAG.getConstant(1, dl, VT));
3597 else if (Op0.getValueType().bitsLT(VT))
3598 Op0 = DAG.getNode(ISD::AND, dl, VT,
3599 DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)),
3600 DAG.getConstant(1, dl, VT));
3601
3602 return DAG.getSetCC(dl, VT, Op0,
3603 DAG.getConstant(0, dl, Op0.getValueType()),
3604 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
3605 }
3606 if (Op0.getOpcode() == ISD::AssertZext &&
3607 cast<VTSDNode>(Op0.getOperand(1))->getVT() == MVT::i1)
3608 return DAG.getSetCC(dl, VT, Op0,
3609 DAG.getConstant(0, dl, Op0.getValueType()),
3610 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
3611 }
3612 }
3613
3614 // Given:
3615 // icmp eq/ne (urem %x, %y), 0
3616 // Iff %x has 0 or 1 bits set, and %y has at least 2 bits set, omit 'urem':
3617 // icmp eq/ne %x, 0
3618 if (N0.getOpcode() == ISD::UREM && N1C->isNullValue() &&
3619 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
3620 KnownBits XKnown = DAG.computeKnownBits(N0.getOperand(0));
3621 KnownBits YKnown = DAG.computeKnownBits(N0.getOperand(1));
3622 if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2)
3623 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1, Cond);
3624 }
3625
3626 if (SDValue V =
3627 optimizeSetCCOfSignedTruncationCheck(VT, N0, N1, Cond, DCI, dl))
3628 return V;
3629 }
3630
3631 // These simplifications apply to splat vectors as well.
3632 // TODO: Handle more splat vector cases.
3633 if (auto *N1C = isConstOrConstSplat(N1)) {
3634 const APInt &C1 = N1C->getAPIntValue();
3635
3636 APInt MinVal, MaxVal;
3637 unsigned OperandBitSize = N1C->getValueType(0).getScalarSizeInBits();
3638 if (ISD::isSignedIntSetCC(Cond)) {
3639 MinVal = APInt::getSignedMinValue(OperandBitSize);
3640 MaxVal = APInt::getSignedMaxValue(OperandBitSize);
3641 } else {
3642 MinVal = APInt::getMinValue(OperandBitSize);
3643 MaxVal = APInt::getMaxValue(OperandBitSize);
3644 }
3645
3646 // Canonicalize GE/LE comparisons to use GT/LT comparisons.
3647 if (Cond == ISD::SETGE || Cond == ISD::SETUGE) {
3648 // X >= MIN --> true
3649 if (C1 == MinVal)
3650 return DAG.getBoolConstant(true, dl, VT, OpVT);
3651
3652 if (!VT.isVector()) { // TODO: Support this for vectors.
3653 // X >= C0 --> X > (C0 - 1)
3654 APInt C = C1 - 1;
3655 ISD::CondCode NewCC = (Cond == ISD::SETGE) ? ISD::SETGT : ISD::SETUGT;
3656 if ((DCI.isBeforeLegalizeOps() ||
3657 isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
3658 (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
3659 isLegalICmpImmediate(C.getSExtValue())))) {
3660 return DAG.getSetCC(dl, VT, N0,
3661 DAG.getConstant(C, dl, N1.getValueType()),
3662 NewCC);
3663 }
3664 }
3665 }
3666
3667 if (Cond == ISD::SETLE || Cond == ISD::SETULE) {
3668 // X <= MAX --> true
3669 if (C1 == MaxVal)
3670 return DAG.getBoolConstant(true, dl, VT, OpVT);
3671
3672 // X <= C0 --> X < (C0 + 1)
3673 if (!VT.isVector()) { // TODO: Support this for vectors.
3674 APInt C = C1 + 1;
3675 ISD::CondCode NewCC = (Cond == ISD::SETLE) ? ISD::SETLT : ISD::SETULT;
3676 if ((DCI.isBeforeLegalizeOps() ||
3677 isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
3678 (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
3679 isLegalICmpImmediate(C.getSExtValue())))) {
3680 return DAG.getSetCC(dl, VT, N0,
3681 DAG.getConstant(C, dl, N1.getValueType()),
3682 NewCC);
3683 }
3684 }
3685 }
3686
3687 if (Cond == ISD::SETLT || Cond == ISD::SETULT) {
3688 if (C1 == MinVal)
3689 return DAG.getBoolConstant(false, dl, VT, OpVT); // X < MIN --> false
3690
3691 // TODO: Support this for vectors after legalize ops.
3692 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
3693 // Canonicalize setlt X, Max --> setne X, Max
3694 if (C1 == MaxVal)
3695 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
3696
3697 // If we have setult X, 1, turn it into seteq X, 0
3698 if (C1 == MinVal+1)
3699 return DAG.getSetCC(dl, VT, N0,
3700 DAG.getConstant(MinVal, dl, N0.getValueType()),
3701 ISD::SETEQ);
3702 }
3703 }
3704
3705 if (Cond == ISD::SETGT || Cond == ISD::SETUGT) {
3706 if (C1 == MaxVal)
3707 return DAG.getBoolConstant(false, dl, VT, OpVT); // X > MAX --> false
3708
3709 // TODO: Support this for vectors after legalize ops.
3710 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
3711 // Canonicalize setgt X, Min --> setne X, Min
3712 if (C1 == MinVal)
3713 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
3714
3715 // If we have setugt X, Max-1, turn it into seteq X, Max
3716 if (C1 == MaxVal-1)
3717 return DAG.getSetCC(dl, VT, N0,
3718 DAG.getConstant(MaxVal, dl, N0.getValueType()),
3719 ISD::SETEQ);
3720 }
3721 }
3722
3723 if (Cond == ISD::SETEQ || Cond == ISD::SETNE) {
3724 // (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
3725 if (C1.isNullValue())
3726 if (SDValue CC = optimizeSetCCByHoistingAndByConstFromLogicalShift(
3727 VT, N0, N1, Cond, DCI, dl))
3728 return CC;
3729 }
3730
3731 // If we have "setcc X, C0", check to see if we can shrink the immediate
3732 // by changing cc.
3733 // TODO: Support this for vectors after legalize ops.
3734 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
3735 // SETUGT X, SINTMAX -> SETLT X, 0
3736 if (Cond == ISD::SETUGT &&
3737 C1 == APInt::getSignedMaxValue(OperandBitSize))
3738 return DAG.getSetCC(dl, VT, N0,
3739 DAG.getConstant(0, dl, N1.getValueType()),
3740 ISD::SETLT);
3741
3742 // SETULT X, SINTMIN -> SETGT X, -1
3743 if (Cond == ISD::SETULT &&
3744 C1 == APInt::getSignedMinValue(OperandBitSize)) {
3745 SDValue ConstMinusOne =
3746 DAG.getConstant(APInt::getAllOnesValue(OperandBitSize), dl,
3747 N1.getValueType());
3748 return DAG.getSetCC(dl, VT, N0, ConstMinusOne, ISD::SETGT);
3749 }
3750 }
3751 }
3752
3753 // Back to non-vector simplifications.
3754 // TODO: Can we do these for vector splats?
3755 if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
3756 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3757 const APInt &C1 = N1C->getAPIntValue();
3758 EVT ShValTy = N0.getValueType();
3759
3760 // Fold bit comparisons when we can.
3761 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3762 (VT == ShValTy || (isTypeLegal(VT) && VT.bitsLE(ShValTy))) &&
3763 N0.getOpcode() == ISD::AND) {
3764 if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
3765 EVT ShiftTy =
3766 getShiftAmountTy(ShValTy, Layout, !DCI.isBeforeLegalize());
3767 if (Cond == ISD::SETNE && C1 == 0) {// (X & 8) != 0 --> (X & 8) >> 3
3768 // Perform the xform if the AND RHS is a single bit.
3769 unsigned ShCt = AndRHS->getAPIntValue().logBase2();
3770 if (AndRHS->getAPIntValue().isPowerOf2() &&
3771 !TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {
3772 return DAG.getNode(ISD::TRUNCATE, dl, VT,
3773 DAG.getNode(ISD::SRL, dl, ShValTy, N0,
3774 DAG.getConstant(ShCt, dl, ShiftTy)));
3775 }
3776 } else if (Cond == ISD::SETEQ && C1 == AndRHS->getAPIntValue()) {
3777 // (X & 8) == 8 --> (X & 8) >> 3
3778 // Perform the xform if C1 is a single bit.
3779 unsigned ShCt = C1.logBase2();
3780 if (C1.isPowerOf2() &&
3781 !TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {
3782 return DAG.getNode(ISD::TRUNCATE, dl, VT,
3783 DAG.getNode(ISD::SRL, dl, ShValTy, N0,
3784 DAG.getConstant(ShCt, dl, ShiftTy)));
3785 }
3786 }
3787 }
3788 }
3789
3790 if (C1.getMinSignedBits() <= 64 &&
3791 !isLegalICmpImmediate(C1.getSExtValue())) {
3792 EVT ShiftTy = getShiftAmountTy(ShValTy, Layout, !DCI.isBeforeLegalize());
3793 // (X & -256) == 256 -> (X >> 8) == 1
3794 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3795 N0.getOpcode() == ISD::AND && N0.hasOneUse()) {
3796 if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
3797 const APInt &AndRHSC = AndRHS->getAPIntValue();
3798 if ((-AndRHSC).isPowerOf2() && (AndRHSC & C1) == C1) {
3799 unsigned ShiftBits = AndRHSC.countTrailingZeros();
3800 if (!TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
3801 SDValue Shift =
3802 DAG.getNode(ISD::SRL, dl, ShValTy, N0.getOperand(0),
3803 DAG.getConstant(ShiftBits, dl, ShiftTy));
3804 SDValue CmpRHS = DAG.getConstant(C1.lshr(ShiftBits), dl, ShValTy);
3805 return DAG.getSetCC(dl, VT, Shift, CmpRHS, Cond);
3806 }
3807 }
3808 }
3809 } else if (Cond == ISD::SETULT || Cond == ISD::SETUGE ||
3810 Cond == ISD::SETULE || Cond == ISD::SETUGT) {
3811 bool AdjOne = (Cond == ISD::SETULE || Cond == ISD::SETUGT);
3812 // X < 0x100000000 -> (X >> 32) < 1
3813 // X >= 0x100000000 -> (X >> 32) >= 1
3814 // X <= 0x0ffffffff -> (X >> 32) < 1
3815 // X > 0x0ffffffff -> (X >> 32) >= 1
3816 unsigned ShiftBits;
3817 APInt NewC = C1;
3818 ISD::CondCode NewCond = Cond;
3819 if (AdjOne) {
3820 ShiftBits = C1.countTrailingOnes();
3821 NewC = NewC + 1;
3822 NewCond = (Cond == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
3823 } else {
3824 ShiftBits = C1.countTrailingZeros();
3825 }
3826 NewC.lshrInPlace(ShiftBits);
3827 if (ShiftBits && NewC.getMinSignedBits() <= 64 &&
3828 isLegalICmpImmediate(NewC.getSExtValue()) &&
3829 !TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
3830 SDValue Shift = DAG.getNode(ISD::SRL, dl, ShValTy, N0,
3831 DAG.getConstant(ShiftBits, dl, ShiftTy));
3832 SDValue CmpRHS = DAG.getConstant(NewC, dl, ShValTy);
3833 return DAG.getSetCC(dl, VT, Shift, CmpRHS, NewCond);
3834 }
3835 }
3836 }
3837 }
3838
3839 if (!isa<ConstantFPSDNode>(N0) && isa<ConstantFPSDNode>(N1)) {
3840 auto *CFP = cast<ConstantFPSDNode>(N1);
3841 assert(!CFP->getValueAPF().isNaN() && "Unexpected NaN value")((!CFP->getValueAPF().isNaN() && "Unexpected NaN value"
) ? static_cast<void> (0) : __assert_fail ("!CFP->getValueAPF().isNaN() && \"Unexpected NaN value\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3841, __PRETTY_FUNCTION__))
;
3842
3843 // Otherwise, we know the RHS is not a NaN. Simplify the node to drop the
3844 // constant if knowing that the operand is non-nan is enough. We prefer to
3845 // have SETO(x,x) instead of SETO(x, 0.0) because this avoids having to
3846 // materialize 0.0.
3847 if (Cond == ISD::SETO || Cond == ISD::SETUO)
3848 return DAG.getSetCC(dl, VT, N0, N0, Cond);
3849
3850 // setcc (fneg x), C -> setcc swap(pred) x, -C
3851 if (N0.getOpcode() == ISD::FNEG) {
3852 ISD::CondCode SwapCond = ISD::getSetCCSwappedOperands(Cond);
3853 if (DCI.isBeforeLegalizeOps() ||
3854 isCondCodeLegal(SwapCond, N0.getSimpleValueType())) {
3855 SDValue NegN1 = DAG.getNode(ISD::FNEG, dl, N0.getValueType(), N1);
3856 return DAG.getSetCC(dl, VT, N0.getOperand(0), NegN1, SwapCond);
3857 }
3858 }
3859
3860 // If the condition is not legal, see if we can find an equivalent one
3861 // which is legal.
3862 if (!isCondCodeLegal(Cond, N0.getSimpleValueType())) {
3863 // If the comparison was an awkward floating-point == or != and one of
3864 // the comparison operands is infinity or negative infinity, convert the
3865 // condition to a less-awkward <= or >=.
3866 if (CFP->getValueAPF().isInfinity()) {
3867 bool IsNegInf = CFP->getValueAPF().isNegative();
3868 ISD::CondCode NewCond = ISD::SETCC_INVALID;
3869 switch (Cond) {
3870 case ISD::SETOEQ: NewCond = IsNegInf ? ISD::SETOLE : ISD::SETOGE; break;
3871 case ISD::SETUEQ: NewCond = IsNegInf ? ISD::SETULE : ISD::SETUGE; break;
3872 case ISD::SETUNE: NewCond = IsNegInf ? ISD::SETUGT : ISD::SETULT; break;
3873 case ISD::SETONE: NewCond = IsNegInf ? ISD::SETOGT : ISD::SETOLT; break;
3874 default: break;
3875 }
3876 if (NewCond != ISD::SETCC_INVALID &&
3877 isCondCodeLegal(NewCond, N0.getSimpleValueType()))
3878 return DAG.getSetCC(dl, VT, N0, N1, NewCond);
3879 }
3880 }
3881 }
3882
3883 if (N0 == N1) {
3884 // The sext(setcc()) => setcc() optimization relies on the appropriate
3885 // constant being emitted.
3886 assert(!N0.getValueType().isInteger() &&((!N0.getValueType().isInteger() && "Integer types should be handled by FoldSetCC"
) ? static_cast<void> (0) : __assert_fail ("!N0.getValueType().isInteger() && \"Integer types should be handled by FoldSetCC\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3887, __PRETTY_FUNCTION__))
3887 "Integer types should be handled by FoldSetCC")((!N0.getValueType().isInteger() && "Integer types should be handled by FoldSetCC"
) ? static_cast<void> (0) : __assert_fail ("!N0.getValueType().isInteger() && \"Integer types should be handled by FoldSetCC\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3887, __PRETTY_FUNCTION__))
;
3888
3889 bool EqTrue = ISD::isTrueWhenEqual(Cond);
3890 unsigned UOF = ISD::getUnorderedFlavor(Cond);
3891 if (UOF == 2) // FP operators that are undefined on NaNs.
3892 return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
3893 if (UOF == unsigned(EqTrue))
3894 return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
3895 // Otherwise, we can't fold it. However, we can simplify it to SETUO/SETO
3896 // if it is not already.
3897 ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO;
3898 if (NewCond != Cond &&
3899 (DCI.isBeforeLegalizeOps() ||
3900 isCondCodeLegal(NewCond, N0.getSimpleValueType())))
3901 return DAG.getSetCC(dl, VT, N0, N1, NewCond);
3902 }
3903
3904 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3905 N0.getValueType().isInteger()) {
3906 if (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB ||
3907 N0.getOpcode() == ISD::XOR) {
3908 // Simplify (X+Y) == (X+Z) --> Y == Z
3909 if (N0.getOpcode() == N1.getOpcode()) {
3910 if (N0.getOperand(0) == N1.getOperand(0))
3911 return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(1), Cond);
3912 if (N0.getOperand(1) == N1.getOperand(1))
3913 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(0), Cond);
3914 if (isCommutativeBinOp(N0.getOpcode())) {
3915 // If X op Y == Y op X, try other combinations.
3916 if (N0.getOperand(0) == N1.getOperand(1))
3917 return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(0),
3918 Cond);
3919 if (N0.getOperand(1) == N1.getOperand(0))
3920 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(1),
3921 Cond);
3922 }
3923 }
3924
3925 // If RHS is a legal immediate value for a compare instruction, we need
3926 // to be careful about increasing register pressure needlessly.
3927 bool LegalRHSImm = false;
3928
3929 if (auto *RHSC = dyn_cast<ConstantSDNode>(N1)) {
3930 if (auto *LHSR = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
3931 // Turn (X+C1) == C2 --> X == C2-C1
3932 if (N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse()) {
3933 return DAG.getSetCC(dl, VT, N0.getOperand(0),
3934 DAG.getConstant(RHSC->getAPIntValue()-
3935 LHSR->getAPIntValue(),
3936 dl, N0.getValueType()), Cond);
3937 }
3938
3939 // Turn (X^C1) == C2 into X == C1^C2 iff X&~C1 = 0.
3940 if (N0.getOpcode() == ISD::XOR)
3941 // If we know that all of the inverted bits are zero, don't bother
3942 // performing the inversion.
3943 if (DAG.MaskedValueIsZero(N0.getOperand(0), ~LHSR->getAPIntValue()))
3944 return
3945 DAG.getSetCC(dl, VT, N0.getOperand(0),
3946 DAG.getConstant(LHSR->getAPIntValue() ^
3947 RHSC->getAPIntValue(),
3948 dl, N0.getValueType()),
3949 Cond);
3950 }
3951
3952 // Turn (C1-X) == C2 --> X == C1-C2
3953 if (auto *SUBC = dyn_cast<ConstantSDNode>(N0.getOperand(0))) {
3954 if (N0.getOpcode() == ISD::SUB && N0.getNode()->hasOneUse()) {
3955 return
3956 DAG.getSetCC(dl, VT, N0.getOperand(1),
3957 DAG.getConstant(SUBC->getAPIntValue() -
3958 RHSC->getAPIntValue(),
3959 dl, N0.getValueType()),
3960 Cond);
3961 }
3962 }
3963
3964 // Could RHSC fold directly into a compare?
3965 if (RHSC->getValueType(0).getSizeInBits() <= 64)
3966 LegalRHSImm = isLegalICmpImmediate(RHSC->getSExtValue());
3967 }
3968
3969 // (X+Y) == X --> Y == 0 and similar folds.
3970 // Don't do this if X is an immediate that can fold into a cmp
3971 // instruction and X+Y has other uses. It could be an induction variable
3972 // chain, and the transform would increase register pressure.
3973 if (!LegalRHSImm || N0.hasOneUse())
3974 if (SDValue V = foldSetCCWithBinOp(VT, N0, N1, Cond, dl, DCI))
3975 return V;
3976 }
3977
3978 if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB ||
3979 N1.getOpcode() == ISD::XOR)
3980 if (SDValue V = foldSetCCWithBinOp(VT, N1, N0, Cond, dl, DCI))
3981 return V;
3982
3983 if (SDValue V = foldSetCCWithAnd(VT, N0, N1, Cond, dl, DCI))
3984 return V;
3985 }
3986
3987 // Fold remainder of division by a constant.
3988 if ((N0.getOpcode() == ISD::UREM || N0.getOpcode() == ISD::SREM) &&
3989 N0.hasOneUse() && (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
3990 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
3991
3992 // When division is cheap or optimizing for minimum size,
3993 // fall through to DIVREM creation by skipping this fold.
3994 if (!isIntDivCheap(VT, Attr) && !Attr.hasFnAttribute(Attribute::MinSize)) {
3995 if (N0.getOpcode() == ISD::UREM) {
3996 if (SDValue Folded = buildUREMEqFold(VT, N0, N1, Cond, DCI, dl))
3997 return Folded;
3998 } else if (N0.getOpcode() == ISD::SREM) {
3999 if (SDValue Folded = buildSREMEqFold(VT, N0, N1, Cond, DCI, dl))
4000 return Folded;
4001 }
4002 }
4003 }
4004
4005 // Fold away ALL boolean setcc's.
4006 if (N0.getValueType().getScalarType() == MVT::i1 && foldBooleans) {
4007 SDValue Temp;
4008 switch (Cond) {
4009 default: llvm_unreachable("Unknown integer setcc!")::llvm::llvm_unreachable_internal("Unknown integer setcc!", "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4009)
;
4010 case ISD::SETEQ: // X == Y -> ~(X^Y)
4011 Temp = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
4012 N0 = DAG.getNOT(dl, Temp, OpVT);
4013 if (!DCI.isCalledByLegalizer())
4014 DCI.AddToWorklist(Temp.getNode());
4015 break;
4016 case ISD::SETNE: // X != Y --> (X^Y)
4017 N0 = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
4018 break;
4019 case ISD::SETGT: // X >s Y --> X == 0 & Y == 1 --> ~X & Y
4020 case ISD::SETULT: // X <u Y --> X == 0 & Y == 1 --> ~X & Y
4021 Temp = DAG.getNOT(dl, N0, OpVT);
4022 N0 = DAG.getNode(ISD::AND, dl, OpVT, N1, Temp);
4023 if (!DCI.isCalledByLegalizer())
4024 DCI.AddToWorklist(Temp.getNode());
4025 break;
4026 case ISD::SETLT: // X <s Y --> X == 1 & Y == 0 --> ~Y & X
4027 case ISD::SETUGT: // X >u Y --> X == 1 & Y == 0 --> ~Y & X
4028 Temp = DAG.getNOT(dl, N1, OpVT);
4029 N0 = DAG.getNode(ISD::AND, dl, OpVT, N0, Temp);
4030 if (!DCI.isCalledByLegalizer())
4031 DCI.AddToWorklist(Temp.getNode());
4032 break;
4033 case ISD::SETULE: // X <=u Y --> X == 0 | Y == 1 --> ~X | Y
4034 case ISD::SETGE: // X >=s Y --> X == 0 | Y == 1 --> ~X | Y
4035 Temp = DAG.getNOT(dl, N0, OpVT);
4036 N0 = DAG.getNode(ISD::OR, dl, OpVT, N1, Temp);
4037 if (!DCI.isCalledByLegalizer())
4038 DCI.AddToWorklist(Temp.getNode());
4039 break;
4040 case ISD::SETUGE: // X >=u Y --> X == 1 | Y == 0 --> ~Y | X
4041 case ISD::SETLE: // X <=s Y --> X == 1 | Y == 0 --> ~Y | X
4042 Temp = DAG.getNOT(dl, N1, OpVT);
4043 N0 = DAG.getNode(ISD::OR, dl, OpVT, N0, Temp);
4044 break;
4045 }
4046 if (VT.getScalarType() != MVT::i1) {
4047 if (!DCI.isCalledByLegalizer())
4048 DCI.AddToWorklist(N0.getNode());
4049 // FIXME: If running after legalize, we probably can't do this.
4050 ISD::NodeType ExtendCode = getExtendForContent(getBooleanContents(OpVT));
4051 N0 = DAG.getNode(ExtendCode, dl, VT, N0);
4052 }
4053 return N0;
4054 }
4055
4056 // Could not fold it.
4057 return SDValue();
4058}
4059
4060/// Returns true (and the GlobalValue and the offset) if the node is a
4061/// GlobalAddress + offset.
4062bool TargetLowering::isGAPlusOffset(SDNode *WN, const GlobalValue *&GA,
4063 int64_t &Offset) const {
4064
4065 SDNode *N = unwrapAddress(SDValue(WN, 0)).getNode();
4066
4067 if (auto *GASD = dyn_cast<GlobalAddressSDNode>(N)) {
4068 GA = GASD->getGlobal();
4069 Offset += GASD->getOffset();
4070 return true;
4071 }
4072
4073 if (N->getOpcode() == ISD::ADD) {
4074 SDValue N1 = N->getOperand(0);
4075 SDValue N2 = N->getOperand(1);
4076 if (isGAPlusOffset(N1.getNode(), GA, Offset)) {
4077 if (auto *V = dyn_cast<ConstantSDNode>(N2)) {
4078 Offset += V->getSExtValue();
4079 return true;
4080 }
4081 } else if (isGAPlusOffset(N2.getNode(), GA, Offset)) {
4082 if (auto *V = dyn_cast<ConstantSDNode>(N1)) {
4083 Offset += V->getSExtValue();
4084 return true;
4085 }
4086 }
4087 }
4088
4089 return false;
4090}
4091
4092SDValue TargetLowering::PerformDAGCombine(SDNode *N,
4093 DAGCombinerInfo &DCI) const {
4094 // Default implementation: no optimization.
4095 return SDValue();
4096}
4097
4098//===----------------------------------------------------------------------===//
4099// Inline Assembler Implementation Methods
4100//===----------------------------------------------------------------------===//
4101
4102TargetLowering::ConstraintType
4103TargetLowering::getConstraintType(StringRef Constraint) const {
4104 unsigned S = Constraint.size();
4105
4106 if (S == 1) {
4107 switch (Constraint[0]) {
4108 default: break;
4109 case 'r':
4110 return C_RegisterClass;
4111 case 'm': // memory
4112 case 'o': // offsetable
4113 case 'V': // not offsetable
4114 return C_Memory;
4115 case 'n': // Simple Integer
4116 case 'E': // Floating Point Constant
4117 case 'F': // Floating Point Constant
4118 return C_Immediate;
4119 case 'i': // Simple Integer or Relocatable Constant
4120 case 's': // Relocatable Constant
4121 case 'p': // Address.
4122 case 'X': // Allow ANY value.
4123 case 'I': // Target registers.
4124 case 'J':
4125 case 'K':
4126 case 'L':
4127 case 'M':
4128 case 'N':
4129 case 'O':
4130 case 'P':
4131 case '<':
4132 case '>':
4133 return C_Other;
4134 }
4135 }
4136
4137 if (S > 1 && Constraint[0] == '{' && Constraint[S - 1] == '}') {
4138 if (S == 8 && Constraint.substr(1, 6) == "memory") // "{memory}"
4139 return C_Memory;
4140 return C_Register;
4141 }
4142 return C_Unknown;
4143}
4144
4145/// Try to replace an X constraint, which matches anything, with another that
4146/// has more specific requirements based on the type of the corresponding
4147/// operand.
4148const char *TargetLowering::LowerXConstraint(EVT ConstraintVT) const {
4149 if (ConstraintVT.isInteger())
4150 return "r";
4151 if (ConstraintVT.isFloatingPoint())
4152 return "f"; // works for many targets
4153 return nullptr;
4154}
4155
4156SDValue TargetLowering::LowerAsmOutputForConstraint(
4157 SDValue &Chain, SDValue &Flag, SDLoc DL, const AsmOperandInfo &OpInfo,
4158 SelectionDAG &DAG) const {
4159 return SDValue();
4160}
4161
4162/// Lower the specified operand into the Ops vector.
4163/// If it is invalid, don't add anything to Ops.
4164void TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
4165 std::string &Constraint,
4166 std::vector<SDValue> &Ops,
4167 SelectionDAG &DAG) const {
4168
4169 if (Constraint.length() > 1) return;
4170
4171 char ConstraintLetter = Constraint[0];
4172 switch (ConstraintLetter) {
4173 default: break;
4174 case 'X': // Allows any operand; labels (basic block) use this.
4175 if (Op.getOpcode() == ISD::BasicBlock ||
4176 Op.getOpcode() == ISD::TargetBlockAddress) {
4177 Ops.push_back(Op);
4178 return;
4179 }
4180 LLVM_FALLTHROUGH[[gnu::fallthrough]];
4181 case 'i': // Simple Integer or Relocatable Constant
4182 case 'n': // Simple Integer
4183 case 's': { // Relocatable Constant
4184
4185 GlobalAddressSDNode *GA;
4186 ConstantSDNode *C;
4187 BlockAddressSDNode *BA;
4188 uint64_t Offset = 0;
4189
4190 // Match (GA) or (C) or (GA+C) or (GA-C) or ((GA+C)+C) or (((GA+C)+C)+C),
4191 // etc., since getelementpointer is variadic. We can't use
4192 // SelectionDAG::FoldSymbolOffset because it expects the GA to be accessible
4193 // while in this case the GA may be furthest from the root node which is
4194 // likely an ISD::ADD.
4195 while (1) {
4196 if ((GA = dyn_cast<GlobalAddressSDNode>(Op)) && ConstraintLetter != 'n') {
4197 Ops.push_back(DAG.getTargetGlobalAddress(GA->getGlobal(), SDLoc(Op),
4198 GA->getValueType(0),
4199 Offset + GA->getOffset()));
4200 return;
4201 } else if ((C = dyn_cast<ConstantSDNode>(Op)) &&
4202 ConstraintLetter != 's') {
4203 // gcc prints these as sign extended. Sign extend value to 64 bits
4204 // now; without this it would get ZExt'd later in
4205 // ScheduleDAGSDNodes::EmitNode, which is very generic.
4206 bool IsBool = C->getConstantIntValue()->getBitWidth() == 1;
4207 BooleanContent BCont = getBooleanContents(MVT::i64);
4208 ISD::NodeType ExtOpc = IsBool ? getExtendForContent(BCont)
4209 : ISD::SIGN_EXTEND;
4210 int64_t ExtVal = ExtOpc == ISD::ZERO_EXTEND ? C->getZExtValue()
4211 : C->getSExtValue();
4212 Ops.push_back(DAG.getTargetConstant(Offset + ExtVal,
4213 SDLoc(C), MVT::i64));
4214 return;
4215 } else if ((BA = dyn_cast<BlockAddressSDNode>(Op)) &&
4216 ConstraintLetter != 'n') {
4217 Ops.push_back(DAG.getTargetBlockAddress(
4218 BA->getBlockAddress(), BA->getValueType(0),
4219 Offset + BA->getOffset(), BA->getTargetFlags()));
4220 return;
4221 } else {
4222 const unsigned OpCode = Op.getOpcode();
4223 if (OpCode == ISD::ADD || OpCode == ISD::SUB) {
4224 if ((C = dyn_cast<ConstantSDNode>(Op.getOperand(0))))
4225 Op = Op.getOperand(1);
4226 // Subtraction is not commutative.
4227 else if (OpCode == ISD::ADD &&
4228 (C = dyn_cast<ConstantSDNode>(Op.getOperand(1))))
4229 Op = Op.getOperand(0);
4230 else
4231 return;
4232 Offset += (OpCode == ISD::ADD ? 1 : -1) * C->getSExtValue();
4233 continue;
4234 }
4235 }
4236 return;
4237 }
4238 break;
4239 }
4240 }
4241}
4242
4243std::pair<unsigned, const TargetRegisterClass *>
4244TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *RI,
4245 StringRef Constraint,
4246 MVT VT) const {
4247 if (Constraint.empty() || Constraint[0] != '{')
4248 return std::make_pair(0u, static_cast<TargetRegisterClass *>(nullptr));
4249 assert(*(Constraint.end() - 1) == '}' && "Not a brace enclosed constraint?")((*(Constraint.end() - 1) == '}' && "Not a brace enclosed constraint?"
) ? static_cast<void> (0) : __assert_fail ("*(Constraint.end() - 1) == '}' && \"Not a brace enclosed constraint?\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4249, __PRETTY_FUNCTION__))
;
4250
4251 // Remove the braces from around the name.
4252 StringRef RegName(Constraint.data() + 1, Constraint.size() - 2);
4253
4254 std::pair<unsigned, const TargetRegisterClass *> R =
4255 std::make_pair(0u, static_cast<const TargetRegisterClass *>(nullptr));
4256
4257 // Figure out which register class contains this reg.
4258 for (const TargetRegisterClass *RC : RI->regclasses()) {
4259 // If none of the value types for this register class are valid, we
4260 // can't use it. For example, 64-bit reg classes on 32-bit targets.
4261 if (!isLegalRC(*RI, *RC))
4262 continue;
4263
4264 for (TargetRegisterClass::iterator I = RC->begin(), E = RC->end();
4265 I != E; ++I) {
4266 if (RegName.equals_lower(RI->getRegAsmName(*I))) {
4267 std::pair<unsigned, const TargetRegisterClass *> S =
4268 std::make_pair(*I, RC);
4269
4270 // If this register class has the requested value type, return it,
4271 // otherwise keep searching and return the first class found
4272 // if no other is found which explicitly has the requested type.
4273 if (RI->isTypeLegalForClass(*RC, VT))
4274 return S;
4275 if (!R.second)
4276 R = S;
4277 }
4278 }
4279 }
4280
4281 return R;
4282}
4283
4284//===----------------------------------------------------------------------===//
4285// Constraint Selection.
4286
4287/// Return true of this is an input operand that is a matching constraint like
4288/// "4".
4289bool TargetLowering::AsmOperandInfo::isMatchingInputConstraint() const {
4290 assert(!ConstraintCode.empty() && "No known constraint!")((!ConstraintCode.empty() && "No known constraint!") ?
static_cast<void> (0) : __assert_fail ("!ConstraintCode.empty() && \"No known constraint!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4290, __PRETTY_FUNCTION__))
;
4291 return isdigit(static_cast<unsigned char>(ConstraintCode[0]));
4292}
4293
4294/// If this is an input matching constraint, this method returns the output
4295/// operand it matches.
4296unsigned TargetLowering::AsmOperandInfo::getMatchedOperand() const {
4297 assert(!ConstraintCode.empty() && "No known constraint!")((!ConstraintCode.empty() && "No known constraint!") ?
static_cast<void> (0) : __assert_fail ("!ConstraintCode.empty() && \"No known constraint!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4297, __PRETTY_FUNCTION__))
;
4298 return atoi(ConstraintCode.c_str());
4299}
4300
4301/// Split up the constraint string from the inline assembly value into the
4302/// specific constraints and their prefixes, and also tie in the associated
4303/// operand values.
4304/// If this returns an empty vector, and if the constraint string itself
4305/// isn't empty, there was an error parsing.
4306TargetLowering::AsmOperandInfoVector
4307TargetLowering::ParseConstraints(const DataLayout &DL,
4308 const TargetRegisterInfo *TRI,
4309 ImmutableCallSite CS) const {
4310 /// Information about all of the constraints.
4311 AsmOperandInfoVector ConstraintOperands;
4312 const InlineAsm *IA = cast<InlineAsm>(CS.getCalledValue());
4313 unsigned maCount = 0; // Largest number of multiple alternative constraints.
4314
4315 // Do a prepass over the constraints, canonicalizing them, and building up the
4316 // ConstraintOperands list.
4317 unsigned ArgNo = 0; // ArgNo - The argument of the CallInst.
4318 unsigned ResNo = 0; // ResNo - The result number of the next output.
4319
4320 for (InlineAsm::ConstraintInfo &CI : IA->ParseConstraints()) {
4321 ConstraintOperands.emplace_back(std::move(CI));
4322 AsmOperandInfo &OpInfo = ConstraintOperands.back();
4323
4324 // Update multiple alternative constraint count.
4325 if (OpInfo.multipleAlternatives.size() > maCount)
4326 maCount = OpInfo.multipleAlternatives.size();
4327
4328 OpInfo.ConstraintVT = MVT::Other;
4329
4330 // Compute the value type for each operand.
4331 switch (OpInfo.Type) {
4332 case InlineAsm::isOutput:
4333 // Indirect outputs just consume an argument.
4334 if (OpInfo.isIndirect) {
4335 OpInfo.CallOperandVal = const_cast<Value *>(CS.getArgument(ArgNo++));
4336 break;
4337 }
4338
4339 // The return value of the call is this value. As such, there is no
4340 // corresponding argument.
4341 assert(!CS.getType()->isVoidTy() &&((!CS.getType()->isVoidTy() && "Bad inline asm!") ?
static_cast<void> (0) : __assert_fail ("!CS.getType()->isVoidTy() && \"Bad inline asm!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4342, __PRETTY_FUNCTION__))
4342 "Bad inline asm!")((!CS.getType()->isVoidTy() && "Bad inline asm!") ?
static_cast<void> (0) : __assert_fail ("!CS.getType()->isVoidTy() && \"Bad inline asm!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4342, __PRETTY_FUNCTION__))
;
4343 if (StructType *STy = dyn_cast<StructType>(CS.getType())) {
4344 OpInfo.ConstraintVT =
4345 getSimpleValueType(DL, STy->getElementType(ResNo));
4346 } else {
4347 assert(ResNo == 0 && "Asm only has one result!")((ResNo == 0 && "Asm only has one result!") ? static_cast
<void> (0) : __assert_fail ("ResNo == 0 && \"Asm only has one result!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4347, __PRETTY_FUNCTION__))
;
4348 OpInfo.ConstraintVT = getSimpleValueType(DL, CS.getType());
4349 }
4350 ++ResNo;
4351 break;
4352 case InlineAsm::isInput:
4353 OpInfo.CallOperandVal = const_cast<Value *>(CS.getArgument(ArgNo++));
4354 break;
4355 case InlineAsm::isClobber:
4356 // Nothing to do.
4357 break;
4358 }
4359
4360 if (OpInfo.CallOperandVal) {
4361 llvm::Type *OpTy = OpInfo.CallOperandVal->getType();
4362 if (OpInfo.isIndirect) {
4363 llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy);
4364 if (!PtrTy)
4365 report_fatal_error("Indirect operand for inline asm not a pointer!");
4366 OpTy = PtrTy->getElementType();
4367 }
4368
4369 // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
4370 if (StructType *STy = dyn_cast<StructType>(OpTy))
4371 if (STy->getNumElements() == 1)
4372 OpTy = STy->getElementType(0);
4373
4374 // If OpTy is not a single value, it may be a struct/union that we
4375 // can tile with integers.
4376 if (!OpTy->isSingleValueType() && OpTy->isSized()) {
4377 unsigned BitSize = DL.getTypeSizeInBits(OpTy);
4378 switch (BitSize) {
4379 default: break;
4380 case 1:
4381 case 8:
4382 case 16:
4383 case 32:
4384 case 64:
4385 case 128:
4386 OpInfo.ConstraintVT =
4387 MVT::getVT(IntegerType::get(OpTy->getContext(), BitSize), true);
4388 break;
4389 }
4390 } else if (PointerType *PT = dyn_cast<PointerType>(OpTy)) {
4391 unsigned PtrSize = DL.getPointerSizeInBits(PT->getAddressSpace());
4392 OpInfo.ConstraintVT = MVT::getIntegerVT(PtrSize);
4393 } else {
4394 OpInfo.ConstraintVT = MVT::getVT(OpTy, true);
4395 }
4396 }
4397 }
4398
4399 // If we have multiple alternative constraints, select the best alternative.
4400 if (!ConstraintOperands.empty()) {
4401 if (maCount) {
4402 unsigned bestMAIndex = 0;
4403 int bestWeight = -1;
4404 // weight: -1 = invalid match, and 0 = so-so match to 5 = good match.
4405 int weight = -1;
4406 unsigned maIndex;
4407 // Compute the sums of the weights for each alternative, keeping track
4408 // of the best (highest weight) one so far.
4409 for (maIndex = 0; maIndex < maCount; ++maIndex) {
4410 int weightSum = 0;
4411 for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
4412 cIndex != eIndex; ++cIndex) {
4413 AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];
4414 if (OpInfo.Type == InlineAsm::isClobber)
4415 continue;
4416
4417 // If this is an output operand with a matching input operand,
4418 // look up the matching input. If their types mismatch, e.g. one
4419 // is an integer, the other is floating point, or their sizes are
4420 // different, flag it as an maCantMatch.
4421 if (OpInfo.hasMatchingInput()) {
4422 AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
4423 if (OpInfo.ConstraintVT != Input.ConstraintVT) {
4424 if ((OpInfo.ConstraintVT.isInteger() !=
4425 Input.ConstraintVT.isInteger()) ||
4426 (OpInfo.ConstraintVT.getSizeInBits() !=
4427 Input.ConstraintVT.getSizeInBits())) {
4428 weightSum = -1; // Can't match.
4429 break;
4430 }
4431 }
4432 }
4433 weight = getMultipleConstraintMatchWeight(OpInfo, maIndex);
4434 if (weight == -1) {
4435 weightSum = -1;
4436 break;
4437 }
4438 weightSum += weight;
4439 }
4440 // Update best.
4441 if (weightSum > bestWeight) {
4442 bestWeight = weightSum;
4443 bestMAIndex = maIndex;
4444 }
4445 }
4446
4447 // Now select chosen alternative in each constraint.
4448 for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
4449 cIndex != eIndex; ++cIndex) {
4450 AsmOperandInfo &cInfo = ConstraintOperands[cIndex];
4451 if (cInfo.Type == InlineAsm::isClobber)
4452 continue;
4453 cInfo.selectAlternative(bestMAIndex);
4454 }
4455 }
4456 }
4457
4458 // Check and hook up tied operands, choose constraint code to use.
4459 for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
4460 cIndex != eIndex; ++cIndex) {
4461 AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];
4462
4463 // If this is an output operand with a matching input operand, look up the
4464 // matching input. If their types mismatch, e.g. one is an integer, the
4465 // other is floating point, or their sizes are different, flag it as an
4466 // error.
4467 if (OpInfo.hasMatchingInput()) {
4468 AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
4469
4470 if (OpInfo.ConstraintVT != Input.ConstraintVT) {
4471 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
4472 getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,
4473 OpInfo.ConstraintVT);
4474 std::pair<unsigned, const TargetRegisterClass *> InputRC =
4475 getRegForInlineAsmConstraint(TRI, Input.ConstraintCode,
4476 Input.ConstraintVT);
4477 if ((OpInfo.ConstraintVT.isInteger() !=
4478 Input.ConstraintVT.isInteger()) ||
4479 (MatchRC.second != InputRC.second)) {
4480 report_fatal_error("Unsupported asm: input constraint"
4481 " with a matching output constraint of"
4482 " incompatible type!");
4483 }
4484 }
4485 }
4486 }
4487
4488 return ConstraintOperands;
4489}
4490
4491/// Return an integer indicating how general CT is.
4492static unsigned getConstraintGenerality(TargetLowering::ConstraintType CT) {
4493 switch (CT) {
4494 case TargetLowering::C_Immediate:
4495 case TargetLowering::C_Other:
4496 case TargetLowering::C_Unknown:
4497 return 0;
4498 case TargetLowering::C_Register:
4499 return 1;
4500 case TargetLowering::C_RegisterClass:
4501 return 2;
4502 case TargetLowering::C_Memory:
4503 return 3;
4504 }
4505 llvm_unreachable("Invalid constraint type")::llvm::llvm_unreachable_internal("Invalid constraint type", "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4505)
;
4506}
4507
4508/// Examine constraint type and operand type and determine a weight value.
4509/// This object must already have been set up with the operand type
4510/// and the current alternative constraint selected.
4511TargetLowering::ConstraintWeight
4512 TargetLowering::getMultipleConstraintMatchWeight(
4513 AsmOperandInfo &info, int maIndex) const {
4514 InlineAsm::ConstraintCodeVector *rCodes;
4515 if (maIndex >= (int)info.multipleAlternatives.size())
4516 rCodes = &info.Codes;
4517 else
4518 rCodes = &info.multipleAlternatives[maIndex].Codes;
4519 ConstraintWeight BestWeight = CW_Invalid;
4520
4521 // Loop over the options, keeping track of the most general one.
4522 for (unsigned i = 0, e = rCodes->size(); i != e; ++i) {
4523 ConstraintWeight weight =
4524 getSingleConstraintMatchWeight(info, (*rCodes)[i].c_str());
4525 if (weight > BestWeight)
4526 BestWeight = weight;
4527 }
4528
4529 return BestWeight;
4530}
4531
4532/// Examine constraint type and operand type and determine a weight value.
4533/// This object must already have been set up with the operand type
4534/// and the current alternative constraint selected.
4535TargetLowering::ConstraintWeight
4536 TargetLowering::getSingleConstraintMatchWeight(
4537 AsmOperandInfo &info, const char *constraint) const {
4538 ConstraintWeight weight = CW_Invalid;
4539 Value *CallOperandVal = info.CallOperandVal;
4540 // If we don't have a value, we can't do a match,
4541 // but allow it at the lowest weight.
4542 if (!CallOperandVal)
4543 return CW_Default;
4544 // Look at the constraint type.
4545 switch (*constraint) {
4546 case 'i': // immediate integer.
4547 case 'n': // immediate integer with a known value.
4548 if (isa<ConstantInt>(CallOperandVal))
4549 weight = CW_Constant;
4550 break;
4551 case 's': // non-explicit intregal immediate.
4552 if (isa<GlobalValue>(CallOperandVal))
4553 weight = CW_Constant;
4554 break;
4555 case 'E': // immediate float if host format.
4556 case 'F': // immediate float.
4557 if (isa<ConstantFP>(CallOperandVal))
4558 weight = CW_Constant;
4559 break;
4560 case '<': // memory operand with autodecrement.
4561 case '>': // memory operand with autoincrement.
4562 case 'm': // memory operand.
4563 case 'o': // offsettable memory operand
4564 case 'V': // non-offsettable memory operand
4565 weight = CW_Memory;
4566 break;
4567 case 'r': // general register.
4568 case 'g': // general register, memory operand or immediate integer.
4569 // note: Clang converts "g" to "imr".
4570 if (CallOperandVal->getType()->isIntegerTy())
4571 weight = CW_Register;
4572 break;
4573 case 'X': // any operand.
4574 default:
4575 weight = CW_Default;
4576 break;
4577 }
4578 return weight;
4579}
4580
4581/// If there are multiple different constraints that we could pick for this
4582/// operand (e.g. "imr") try to pick the 'best' one.
4583/// This is somewhat tricky: constraints fall into four classes:
4584/// Other -> immediates and magic values
4585/// Register -> one specific register
4586/// RegisterClass -> a group of regs
4587/// Memory -> memory
4588/// Ideally, we would pick the most specific constraint possible: if we have
4589/// something that fits into a register, we would pick it. The problem here
4590/// is that if we have something that could either be in a register or in
4591/// memory that use of the register could cause selection of *other*
4592/// operands to fail: they might only succeed if we pick memory. Because of
4593/// this the heuristic we use is:
4594///
4595/// 1) If there is an 'other' constraint, and if the operand is valid for
4596/// that constraint, use it. This makes us take advantage of 'i'
4597/// constraints when available.
4598/// 2) Otherwise, pick the most general constraint present. This prefers
4599/// 'm' over 'r', for example.
4600///
4601static void ChooseConstraint(TargetLowering::AsmOperandInfo &OpInfo,
4602 const TargetLowering &TLI,
4603 SDValue Op, SelectionDAG *DAG) {
4604 assert(OpInfo.Codes.size() > 1 && "Doesn't have multiple constraint options")((OpInfo.Codes.size() > 1 && "Doesn't have multiple constraint options"
) ? static_cast<void> (0) : __assert_fail ("OpInfo.Codes.size() > 1 && \"Doesn't have multiple constraint options\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4604, __PRETTY_FUNCTION__))
;
4605 unsigned BestIdx = 0;
4606 TargetLowering::ConstraintType BestType = TargetLowering::C_Unknown;
4607 int BestGenerality = -1;
4608
4609 // Loop over the options, keeping track of the most general one.
4610 for (unsigned i = 0, e = OpInfo.Codes.size(); i != e; ++i) {
4611 TargetLowering::ConstraintType CType =
4612 TLI.getConstraintType(OpInfo.Codes[i]);
4613
4614 // Indirect 'other' or 'immediate' constraints are not allowed.
4615 if (OpInfo.isIndirect && !(CType == TargetLowering::C_Memory ||
4616 CType == TargetLowering::C_Register ||
4617 CType == TargetLowering::C_RegisterClass))
4618 continue;
4619
4620 // If this is an 'other' or 'immediate' constraint, see if the operand is
4621 // valid for it. For example, on X86 we might have an 'rI' constraint. If
4622 // the operand is an integer in the range [0..31] we want to use I (saving a
4623 // load of a register), otherwise we must use 'r'.
4624 if ((CType == TargetLowering::C_Other ||
4625 CType == TargetLowering::C_Immediate) && Op.getNode()) {
4626 assert(OpInfo.Codes[i].size() == 1 &&((OpInfo.Codes[i].size() == 1 && "Unhandled multi-letter 'other' constraint"
) ? static_cast<void> (0) : __assert_fail ("OpInfo.Codes[i].size() == 1 && \"Unhandled multi-letter 'other' constraint\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4627, __PRETTY_FUNCTION__))
4627 "Unhandled multi-letter 'other' constraint")((OpInfo.Codes[i].size() == 1 && "Unhandled multi-letter 'other' constraint"
) ? static_cast<void> (0) : __assert_fail ("OpInfo.Codes[i].size() == 1 && \"Unhandled multi-letter 'other' constraint\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4627, __PRETTY_FUNCTION__))
;
4628 std::vector<SDValue> ResultOps;
4629 TLI.LowerAsmOperandForConstraint(Op, OpInfo.Codes[i],
4630 ResultOps, *DAG);
4631 if (!ResultOps.empty()) {
4632 BestType = CType;
4633 BestIdx = i;
4634 break;
4635 }
4636 }
4637
4638 // Things with matching constraints can only be registers, per gcc
4639 // documentation. This mainly affects "g" constraints.
4640 if (CType == TargetLowering::C_Memory && OpInfo.hasMatchingInput())
4641 continue;
4642
4643 // This constraint letter is more general than the previous one, use it.
4644 int Generality = getConstraintGenerality(CType);
4645 if (Generality > BestGenerality) {
4646 BestType = CType;
4647 BestIdx = i;
4648 BestGenerality = Generality;
4649 }
4650 }
4651
4652 OpInfo.ConstraintCode = OpInfo.Codes[BestIdx];
4653 OpInfo.ConstraintType = BestType;
4654}
4655
4656/// Determines the constraint code and constraint type to use for the specific
4657/// AsmOperandInfo, setting OpInfo.ConstraintCode and OpInfo.ConstraintType.
4658void TargetLowering::ComputeConstraintToUse(AsmOperandInfo &OpInfo,
4659 SDValue Op,
4660 SelectionDAG *DAG) const {
4661 assert(!OpInfo.Codes.empty() && "Must have at least one constraint")((!OpInfo.Codes.empty() && "Must have at least one constraint"
) ? static_cast<void> (0) : __assert_fail ("!OpInfo.Codes.empty() && \"Must have at least one constraint\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4661, __PRETTY_FUNCTION__))
;
4662
4663 // Single-letter constraints ('r') are very common.
4664 if (OpInfo.Codes.size() == 1) {
4665 OpInfo.ConstraintCode = OpInfo.Codes[0];
4666 OpInfo.ConstraintType = getConstraintType(OpInfo.ConstraintCode);
4667 } else {
4668 ChooseConstraint(OpInfo, *this, Op, DAG);
4669 }
4670
4671 // 'X' matches anything.
4672 if (OpInfo.ConstraintCode == "X" && OpInfo.CallOperandVal) {
4673 // Labels and constants are handled elsewhere ('X' is the only thing
4674 // that matches labels). For Functions, the type here is the type of
4675 // the result, which is not what we want to look at; leave them alone.
4676 Value *v = OpInfo.CallOperandVal;
4677 if (isa<BasicBlock>(v) || isa<ConstantInt>(v) || isa<Function>(v)) {
4678 OpInfo.CallOperandVal = v;
4679 return;
4680 }
4681
4682 if (Op.getNode() && Op.getOpcode() == ISD::TargetBlockAddress)
4683 return;
4684
4685 // Otherwise, try to resolve it to something we know about by looking at
4686 // the actual operand type.
4687 if (const char *Repl = LowerXConstraint(OpInfo.ConstraintVT)) {
4688 OpInfo.ConstraintCode = Repl;
4689 OpInfo.ConstraintType = getConstraintType(OpInfo.ConstraintCode);
4690 }
4691 }
4692}
4693
4694/// Given an exact SDIV by a constant, create a multiplication
4695/// with the multiplicative inverse of the constant.
4696static SDValue BuildExactSDIV(const TargetLowering &TLI, SDNode *N,
4697 const SDLoc &dl, SelectionDAG &DAG,
4698 SmallVectorImpl<SDNode *> &Created) {
4699 SDValue Op0 = N->getOperand(0);
4700 SDValue Op1 = N->getOperand(1);
4701 EVT VT = N->getValueType(0);
4702 EVT SVT = VT.getScalarType();
4703 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
4704 EVT ShSVT = ShVT.getScalarType();
4705
4706 bool UseSRA = false;
4707 SmallVector<SDValue, 16> Shifts, Factors;
4708
4709 auto BuildSDIVPattern = [&](ConstantSDNode *C) {
4710 if (C->isNullValue())
4711 return false;
4712 APInt Divisor = C->getAPIntValue();
4713 unsigned Shift = Divisor.countTrailingZeros();
4714 if (Shift) {
4715 Divisor.ashrInPlace(Shift);
4716 UseSRA = true;
4717 }
4718 // Calculate the multiplicative inverse, using Newton's method.
4719 APInt t;
4720 APInt Factor = Divisor;
4721 while ((t = Divisor * Factor) != 1)
4722 Factor *= APInt(Divisor.getBitWidth(), 2) - t;
4723 Shifts.push_back(DAG.getConstant(Shift, dl, ShSVT));
4724 Factors.push_back(DAG.getConstant(Factor, dl, SVT));
4725 return true;
4726 };
4727
4728 // Collect all magic values from the build vector.
4729 if (!ISD::matchUnaryPredicate(Op1, BuildSDIVPattern))
4730 return SDValue();
4731
4732 SDValue Shift, Factor;
4733 if (VT.isVector()) {
4734 Shift = DAG.getBuildVector(ShVT, dl, Shifts);
4735 Factor = DAG.getBuildVector(VT, dl, Factors);
4736 } else {
4737 Shift = Shifts[0];
4738 Factor = Factors[0];
4739 }
4740
4741 SDValue Res = Op0;
4742
4743 // Shift the value upfront if it is even, so the LSB is one.
4744 if (UseSRA) {
4745 // TODO: For UDIV use SRL instead of SRA.
4746 SDNodeFlags Flags;
4747 Flags.setExact(true);
4748 Res = DAG.getNode(ISD::SRA, dl, VT, Res, Shift, Flags);
4749 Created.push_back(Res.getNode());
4750 }
4751
4752 return DAG.getNode(ISD::MUL, dl, VT, Res, Factor);
4753}
4754
4755SDValue TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
4756 SelectionDAG &DAG,
4757 SmallVectorImpl<SDNode *> &Created) const {
4758 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
4759 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4760 if (TLI.isIntDivCheap(N->getValueType(0), Attr))
4761 return SDValue(N, 0); // Lower SDIV as SDIV
4762 return SDValue();
4763}
4764
4765/// Given an ISD::SDIV node expressing a divide by constant,
4766/// return a DAG expression to select that will generate the same value by
4767/// multiplying by a magic number.
4768/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
4769SDValue TargetLowering::BuildSDIV(SDNode *N, SelectionDAG &DAG,
4770 bool IsAfterLegalization,
4771 SmallVectorImpl<SDNode *> &Created) const {
4772 SDLoc dl(N);
4773 EVT VT = N->getValueType(0);
4774 EVT SVT = VT.getScalarType();
4775 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
4776 EVT ShSVT = ShVT.getScalarType();
4777 unsigned EltBits = VT.getScalarSizeInBits();
4778
4779 // Check to see if we can do this.
4780 // FIXME: We should be more aggressive here.
4781 if (!isTypeLegal(VT))
4782 return SDValue();
4783
4784 // If the sdiv has an 'exact' bit we can use a simpler lowering.
4785 if (N->getFlags().hasExact())
4786 return BuildExactSDIV(*this, N, dl, DAG, Created);
4787
4788 SmallVector<SDValue, 16> MagicFactors, Factors, Shifts, ShiftMasks;
4789
4790 auto BuildSDIVPattern = [&](ConstantSDNode *C) {
4791 if (C->isNullValue())
4792 return false;
4793
4794 const APInt &Divisor = C->getAPIntValue();
4795 APInt::ms magics = Divisor.magic();
4796 int NumeratorFactor = 0;
4797 int ShiftMask = -1;
4798
4799 if (Divisor.isOneValue() || Divisor.isAllOnesValue()) {
4800 // If d is +1/-1, we just multiply the numerator by +1/-1.
4801 NumeratorFactor = Divisor.getSExtValue();
4802 magics.m = 0;
4803 magics.s = 0;
4804 ShiftMask = 0;
4805 } else if (Divisor.isStrictlyPositive() && magics.m.isNegative()) {
4806 // If d > 0 and m < 0, add the numerator.
4807 NumeratorFactor = 1;
4808 } else if (Divisor.isNegative() && magics.m.isStrictlyPositive()) {
4809 // If d < 0 and m > 0, subtract the numerator.
4810 NumeratorFactor = -1;
4811 }
4812
4813 MagicFactors.push_back(DAG.getConstant(magics.m, dl, SVT));
4814 Factors.push_back(DAG.getConstant(NumeratorFactor, dl, SVT));
4815 Shifts.push_back(DAG.getConstant(magics.s, dl, ShSVT));
4816 ShiftMasks.push_back(DAG.getConstant(ShiftMask, dl, SVT));
4817 return true;
4818 };
4819
4820 SDValue N0 = N->getOperand(0);
4821 SDValue N1 = N->getOperand(1);
4822
4823 // Collect the shifts / magic values from each element.
4824 if (!ISD::matchUnaryPredicate(N1, BuildSDIVPattern))
4825 return SDValue();
4826
4827 SDValue MagicFactor, Factor, Shift, ShiftMask;
4828 if (VT.isVector()) {
4829 MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);
4830 Factor = DAG.getBuildVector(VT, dl, Factors);
4831 Shift = DAG.getBuildVector(ShVT, dl, Shifts);
4832 ShiftMask = DAG.getBuildVector(VT, dl, ShiftMasks);
4833 } else {
4834 MagicFactor = MagicFactors[0];
4835 Factor = Factors[0];
4836 Shift = Shifts[0];
4837 ShiftMask = ShiftMasks[0];
4838 }
4839
4840 // Multiply the numerator (operand 0) by the magic value.
4841 // FIXME: We should support doing a MUL in a wider type.
4842 SDValue Q;
4843 if (IsAfterLegalization ? isOperationLegal(ISD::MULHS, VT)
4844 : isOperationLegalOrCustom(ISD::MULHS, VT))
4845 Q = DAG.getNode(ISD::MULHS, dl, VT, N0, MagicFactor);
4846 else if (IsAfterLegalization ? isOperationLegal(ISD::SMUL_LOHI, VT)
4847 : isOperationLegalOrCustom(ISD::SMUL_LOHI, VT)) {
4848 SDValue LoHi =
4849 DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT), N0, MagicFactor);
4850 Q = SDValue(LoHi.getNode(), 1);
4851 } else
4852 return SDValue(); // No mulhs or equivalent.
4853 Created.push_back(Q.getNode());
4854
4855 // (Optionally) Add/subtract the numerator using Factor.
4856 Factor = DAG.getNode(ISD::MUL, dl, VT, N0, Factor);
4857 Created.push_back(Factor.getNode());
4858 Q = DAG.getNode(ISD::ADD, dl, VT, Q, Factor);
4859 Created.push_back(Q.getNode());
4860
4861 // Shift right algebraic by shift value.
4862 Q = DAG.getNode(ISD::SRA, dl, VT, Q, Shift);
4863 Created.push_back(Q.getNode());
4864
4865 // Extract the sign bit, mask it and add it to the quotient.
4866 SDValue SignShift = DAG.getConstant(EltBits - 1, dl, ShVT);
4867 SDValue T = DAG.getNode(ISD::SRL, dl, VT, Q, SignShift);
4868 Created.push_back(T.getNode());
4869 T = DAG.getNode(ISD::AND, dl, VT, T, ShiftMask);
4870 Created.push_back(T.getNode());
4871 return DAG.getNode(ISD::ADD, dl, VT, Q, T);
4872}
4873
4874/// Given an ISD::UDIV node expressing a divide by constant,
4875/// return a DAG expression to select that will generate the same value by
4876/// multiplying by a magic number.
4877/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
4878SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG,
4879 bool IsAfterLegalization,
4880 SmallVectorImpl<SDNode *> &Created) const {
4881 SDLoc dl(N);
4882 EVT VT = N->getValueType(0);
4883 EVT SVT = VT.getScalarType();
4884 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
4885 EVT ShSVT = ShVT.getScalarType();
4886 unsigned EltBits = VT.getScalarSizeInBits();
4887
4888 // Check to see if we can do this.
4889 // FIXME: We should be more aggressive here.
4890 if (!isTypeLegal(VT))
4891 return SDValue();
4892
4893 bool UseNPQ = false;
4894 SmallVector<SDValue, 16> PreShifts, PostShifts, MagicFactors, NPQFactors;
4895
4896 auto BuildUDIVPattern = [&](ConstantSDNode *C) {
4897 if (C->isNullValue())
4898 return false;
4899 // FIXME: We should use a narrower constant when the upper
4900 // bits are known to be zero.
4901 APInt Divisor = C->getAPIntValue();
4902 APInt::mu magics = Divisor.magicu();
4903 unsigned PreShift = 0, PostShift = 0;
4904
4905 // If the divisor is even, we can avoid using the expensive fixup by
4906 // shifting the divided value upfront.
4907 if (magics.a != 0 && !Divisor[0]) {
4908 PreShift = Divisor.countTrailingZeros();
4909 // Get magic number for the shifted divisor.
4910 magics = Divisor.lshr(PreShift).magicu(PreShift);
4911 assert(magics.a == 0 && "Should use cheap fixup now")((magics.a == 0 && "Should use cheap fixup now") ? static_cast
<void> (0) : __assert_fail ("magics.a == 0 && \"Should use cheap fixup now\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4911, __PRETTY_FUNCTION__))
;
4912 }
4913
4914 APInt Magic = magics.m;
4915
4916 unsigned SelNPQ;
4917 if (magics.a == 0 || Divisor.isOneValue()) {
4918 assert(magics.s < Divisor.getBitWidth() &&((magics.s < Divisor.getBitWidth() && "We shouldn't generate an undefined shift!"
) ? static_cast<void> (0) : __assert_fail ("magics.s < Divisor.getBitWidth() && \"We shouldn't generate an undefined shift!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4919, __PRETTY_FUNCTION__))
4919 "We shouldn't generate an undefined shift!")((magics.s < Divisor.getBitWidth() && "We shouldn't generate an undefined shift!"
) ? static_cast<void> (0) : __assert_fail ("magics.s < Divisor.getBitWidth() && \"We shouldn't generate an undefined shift!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4919, __PRETTY_FUNCTION__))
;
4920 PostShift = magics.s;
4921 SelNPQ = false;
4922 } else {
4923 PostShift = magics.s - 1;
4924 SelNPQ = true;
4925 }
4926
4927 PreShifts.push_back(DAG.getConstant(PreShift, dl, ShSVT));
4928 MagicFactors.push_back(DAG.getConstant(Magic, dl, SVT));
4929 NPQFactors.push_back(
4930 DAG.getConstant(SelNPQ ? APInt::getOneBitSet(EltBits, EltBits - 1)
4931 : APInt::getNullValue(EltBits),
4932 dl, SVT));
4933 PostShifts.push_back(DAG.getConstant(PostShift, dl, ShSVT));
4934 UseNPQ |= SelNPQ;
4935 return true;
4936 };
4937
4938 SDValue N0 = N->getOperand(0);
4939 SDValue N1 = N->getOperand(1);
4940
4941 // Collect the shifts/magic values from each element.
4942 if (!ISD::matchUnaryPredicate(N1, BuildUDIVPattern))
4943 return SDValue();
4944
4945 SDValue PreShift, PostShift, MagicFactor, NPQFactor;
4946 if (VT.isVector()) {
4947 PreShift = DAG.getBuildVector(ShVT, dl, PreShifts);
4948 MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);
4949 NPQFactor = DAG.getBuildVector(VT, dl, NPQFactors);
4950 PostShift = DAG.getBuildVector(ShVT, dl, PostShifts);
4951 } else {
4952 PreShift = PreShifts[0];
4953 MagicFactor = MagicFactors[0];
4954 PostShift = PostShifts[0];
4955 }
4956
4957 SDValue Q = N0;
4958 Q = DAG.getNode(ISD::SRL, dl, VT, Q, PreShift);
4959 Created.push_back(Q.getNode());
4960
4961 // FIXME: We should support doing a MUL in a wider type.
4962 auto GetMULHU = [&](SDValue X, SDValue Y) {
4963 if (IsAfterLegalization ? isOperationLegal(ISD::MULHU, VT)
4964 : isOperationLegalOrCustom(ISD::MULHU, VT))
4965 return DAG.getNode(ISD::MULHU, dl, VT, X, Y);
4966 if (IsAfterLegalization ? isOperationLegal(ISD::UMUL_LOHI, VT)
4967 : isOperationLegalOrCustom(ISD::UMUL_LOHI, VT)) {
4968 SDValue LoHi =
4969 DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);
4970 return SDValue(LoHi.getNode(), 1);
4971 }
4972 return SDValue(); // No mulhu or equivalent
4973 };
4974
4975 // Multiply the numerator (operand 0) by the magic value.
4976 Q = GetMULHU(Q, MagicFactor);
4977 if (!Q)
4978 return SDValue();
4979
4980 Created.push_back(Q.getNode());
4981
4982 if (UseNPQ) {
4983 SDValue NPQ = DAG.getNode(ISD::SUB, dl, VT, N0, Q);
4984 Created.push_back(NPQ.getNode());
4985
4986 // For vectors we might have a mix of non-NPQ/NPQ paths, so use
4987 // MULHU to act as a SRL-by-1 for NPQ, else multiply by zero.
4988 if (VT.isVector())
4989 NPQ = GetMULHU(NPQ, NPQFactor);
4990 else
4991 NPQ = DAG.getNode(ISD::SRL, dl, VT, NPQ, DAG.getConstant(1, dl, ShVT));
4992
4993 Created.push_back(NPQ.getNode());
4994
4995 Q = DAG.getNode(ISD::ADD, dl, VT, NPQ, Q);
4996 Created.push_back(Q.getNode());
4997 }
4998
4999 Q = DAG.getNode(ISD::SRL, dl, VT, Q, PostShift);
5000 Created.push_back(Q.getNode());
5001
5002 SDValue One = DAG.getConstant(1, dl, VT);
5003 SDValue IsOne = DAG.getSetCC(dl, VT, N1, One, ISD::SETEQ);
5004 return DAG.getSelect(dl, VT, IsOne, N0, Q);
5005}
5006
5007/// If all values in Values that *don't* match the predicate are same 'splat'
5008/// value, then replace all values with that splat value.
5009/// Else, if AlternativeReplacement was provided, then replace all values that
5010/// do match predicate with AlternativeReplacement value.
5011static void
5012turnVectorIntoSplatVector(MutableArrayRef<SDValue> Values,
5013 std::function<bool(SDValue)> Predicate,
5014 SDValue AlternativeReplacement = SDValue()) {
5015 SDValue Replacement;
5016 // Is there a value for which the Predicate does *NOT* match? What is it?
5017 auto SplatValue = llvm::find_if_not(Values, Predicate);
5018 if (SplatValue != Values.end()) {
5019 // Does Values consist only of SplatValue's and values matching Predicate?
5020 if (llvm::all_of(Values, [Predicate, SplatValue](SDValue Value) {
5021 return Value == *SplatValue || Predicate(Value);
5022 })) // Then we shall replace values matching predicate with SplatValue.
5023 Replacement = *SplatValue;
5024 }
5025 if (!Replacement) {
5026 // Oops, we did not find the "baseline" splat value.
5027 if (!AlternativeReplacement)
5028 return; // Nothing to do.
5029 // Let's replace with provided value then.
5030 Replacement = AlternativeReplacement;
5031 }
5032 std::replace_if(Values.begin(), Values.end(), Predicate, Replacement);
5033}
5034
5035/// Given an ISD::UREM used only by an ISD::SETEQ or ISD::SETNE
5036/// where the divisor is constant and the comparison target is zero,
5037/// return a DAG expression that will generate the same comparison result
5038/// using only multiplications, additions and shifts/rotations.
5039/// Ref: "Hacker's Delight" 10-17.
5040SDValue TargetLowering::buildUREMEqFold(EVT SETCCVT, SDValue REMNode,
5041 SDValue CompTargetNode,
5042 ISD::CondCode Cond,
5043 DAGCombinerInfo &DCI,
5044 const SDLoc &DL) const {
5045 SmallVector<SDNode *, 5> Built;
5046 if (SDValue Folded = prepareUREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
5047 DCI, DL, Built)) {
5048 for (SDNode *N : Built)
5049 DCI.AddToWorklist(N);
5050 return Folded;
5051 }
5052
5053 return SDValue();
5054}
5055
5056SDValue
5057TargetLowering::prepareUREMEqFold(EVT SETCCVT, SDValue REMNode,
5058 SDValue CompTargetNode, ISD::CondCode Cond,
5059 DAGCombinerInfo &DCI, const SDLoc &DL,
5060 SmallVectorImpl<SDNode *> &Created) const {
5061 // fold (seteq/ne (urem N, D), 0) -> (setule/ugt (rotr (mul N, P), K), Q)
5062 // - D must be constant, with D = D0 * 2^K where D0 is odd
5063 // - P is the multiplicative inverse of D0 modulo 2^W
5064 // - Q = floor(((2^W) - 1) / D)
5065 // where W is the width of the common type of N and D.
5066 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&(((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Only applicable for (in)equality comparisons."
) ? static_cast<void> (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5067, __PRETTY_FUNCTION__))
5067 "Only applicable for (in)equality comparisons.")(((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Only applicable for (in)equality comparisons."
) ? static_cast<void> (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5067, __PRETTY_FUNCTION__))
;
5068
5069 SelectionDAG &DAG = DCI.DAG;
5070
5071 EVT VT = REMNode.getValueType();
5072 EVT SVT = VT.getScalarType();
5073 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
5074 EVT ShSVT = ShVT.getScalarType();
5075
5076 // If MUL is unavailable, we cannot proceed in any case.
5077 if (!isOperationLegalOrCustom(ISD::MUL, VT))
5078 return SDValue();
5079
5080 bool ComparingWithAllZeros = true;
5081 bool AllComparisonsWithNonZerosAreTautological = true;
5082 bool HadTautologicalLanes = false;
5083 bool AllLanesAreTautological = true;
5084 bool HadEvenDivisor = false;
5085 bool AllDivisorsArePowerOfTwo = true;
5086 bool HadTautologicalInvertedLanes = false;
5087 SmallVector<SDValue, 16> PAmts, KAmts, QAmts, IAmts;
5088
5089 auto BuildUREMPattern = [&](ConstantSDNode *CDiv, ConstantSDNode *CCmp) {
5090 // Division by 0 is UB. Leave it to be constant-folded elsewhere.
5091 if (CDiv->isNullValue())
5092 return false;
5093
5094 const APInt &D = CDiv->getAPIntValue();
5095 const APInt &Cmp = CCmp->getAPIntValue();
5096
5097 ComparingWithAllZeros &= Cmp.isNullValue();
5098
5099 // x u% C1` is *always* less than C1. So given `x u% C1 == C2`,
5100 // if C2 is not less than C1, the comparison is always false.
5101 // But we will only be able to produce the comparison that will give the
5102 // opposive tautological answer. So this lane would need to be fixed up.
5103 bool TautologicalInvertedLane = D.ule(Cmp);
5104 HadTautologicalInvertedLanes |= TautologicalInvertedLane;
5105
5106 // If all lanes are tautological (either all divisors are ones, or divisor
5107 // is not greater than the constant we are comparing with),
5108 // we will prefer to avoid the fold.
5109 bool TautologicalLane = D.isOneValue() || TautologicalInvertedLane;
5110 HadTautologicalLanes |= TautologicalLane;
5111 AllLanesAreTautological &= TautologicalLane;
5112
5113 // If we are comparing with non-zero, we need'll need to subtract said
5114 // comparison value from the LHS. But there is no point in doing that if
5115 // every lane where we are comparing with non-zero is tautological..
5116 if (!Cmp.isNullValue())
5117 AllComparisonsWithNonZerosAreTautological &= TautologicalLane;
5118
5119 // Decompose D into D0 * 2^K
5120 unsigned K = D.countTrailingZeros();
5121 assert((!D.isOneValue() || (K == 0)) && "For divisor '1' we won't rotate.")(((!D.isOneValue() || (K == 0)) && "For divisor '1' we won't rotate."
) ? static_cast<void> (0) : __assert_fail ("(!D.isOneValue() || (K == 0)) && \"For divisor '1' we won't rotate.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5121, __PRETTY_FUNCTION__))
;
5122 APInt D0 = D.lshr(K);
5123
5124 // D is even if it has trailing zeros.
5125 HadEvenDivisor |= (K != 0);
5126 // D is a power-of-two if D0 is one.
5127 // If all divisors are power-of-two, we will prefer to avoid the fold.
5128 AllDivisorsArePowerOfTwo &= D0.isOneValue();
5129
5130 // P = inv(D0, 2^W)
5131 // 2^W requires W + 1 bits, so we have to extend and then truncate.
5132 unsigned W = D.getBitWidth();
5133 APInt P = D0.zext(W + 1)
5134 .multiplicativeInverse(APInt::getSignedMinValue(W + 1))
5135 .trunc(W);
5136 assert(!P.isNullValue() && "No multiplicative inverse!")((!P.isNullValue() && "No multiplicative inverse!") ?
static_cast<void> (0) : __assert_fail ("!P.isNullValue() && \"No multiplicative inverse!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5136, __PRETTY_FUNCTION__))
; // unreachable
5137 assert((D0 * P).isOneValue() && "Multiplicative inverse sanity check.")(((D0 * P).isOneValue() && "Multiplicative inverse sanity check."
) ? static_cast<void> (0) : __assert_fail ("(D0 * P).isOneValue() && \"Multiplicative inverse sanity check.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5137, __PRETTY_FUNCTION__))
;
5138
5139 // Q = floor((2^W - 1) u/ D)
5140 // R = ((2^W - 1) u% D)
5141 APInt Q, R;
5142 APInt::udivrem(APInt::getAllOnesValue(W), D, Q, R);
5143
5144 // If we are comparing with zero, then that comparison constant is okay,
5145 // else it may need to be one less than that.
5146 if (Cmp.ugt(R))
5147 Q -= 1;
5148
5149 assert(APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) &&((APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) &&
"We are expecting that K is always less than all-ones for ShSVT"
) ? static_cast<void> (0) : __assert_fail ("APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) && \"We are expecting that K is always less than all-ones for ShSVT\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5150, __PRETTY_FUNCTION__))
5150 "We are expecting that K is always less than all-ones for ShSVT")((APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) &&
"We are expecting that K is always less than all-ones for ShSVT"
) ? static_cast<void> (0) : __assert_fail ("APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) && \"We are expecting that K is always less than all-ones for ShSVT\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5150, __PRETTY_FUNCTION__))
;
5151
5152 // If the lane is tautological the result can be constant-folded.
5153 if (TautologicalLane) {
5154 // Set P and K amount to a bogus values so we can try to splat them.
5155 P = 0;
5156 K = -1;
5157 // And ensure that comparison constant is tautological,
5158 // it will always compare true/false.
5159 Q = -1;
5160 }
5161
5162 PAmts.push_back(DAG.getConstant(P, DL, SVT));
5163 KAmts.push_back(
5164 DAG.getConstant(APInt(ShSVT.getSizeInBits(), K), DL, ShSVT));
5165 QAmts.push_back(DAG.getConstant(Q, DL, SVT));
5166 return true;
5167 };
5168
5169 SDValue N = REMNode.getOperand(0);
5170 SDValue D = REMNode.getOperand(1);
5171
5172 // Collect the values from each element.
5173 if (!ISD::matchBinaryPredicate(D, CompTargetNode, BuildUREMPattern))
5174 return SDValue();
5175
5176 // If all lanes are tautological, the result can be constant-folded.
5177 if (AllLanesAreTautological)
5178 return SDValue();
5179
5180 // If this is a urem by a powers-of-two, avoid the fold since it can be
5181 // best implemented as a bit test.
5182 if (AllDivisorsArePowerOfTwo)
5183 return SDValue();
5184
5185 SDValue PVal, KVal, QVal;
5186 if (VT.isVector()) {
5187 if (HadTautologicalLanes) {
5188 // Try to turn PAmts into a splat, since we don't care about the values
5189 // that are currently '0'. If we can't, just keep '0'`s.
5190 turnVectorIntoSplatVector(PAmts, isNullConstant);
5191 // Try to turn KAmts into a splat, since we don't care about the values
5192 // that are currently '-1'. If we can't, change them to '0'`s.
5193 turnVectorIntoSplatVector(KAmts, isAllOnesConstant,
5194 DAG.getConstant(0, DL, ShSVT));
5195 }
5196
5197 PVal = DAG.getBuildVector(VT, DL, PAmts);
5198 KVal = DAG.getBuildVector(ShVT, DL, KAmts);
5199 QVal = DAG.getBuildVector(VT, DL, QAmts);
5200 } else {
5201 PVal = PAmts[0];
5202 KVal = KAmts[0];
5203 QVal = QAmts[0];
5204 }
5205
5206 if (!ComparingWithAllZeros && !AllComparisonsWithNonZerosAreTautological) {
5207 if (!isOperationLegalOrCustom(ISD::SUB, VT))
5208 return SDValue(); // FIXME: Could/should use `ISD::ADD`?
5209 assert(CompTargetNode.getValueType() == N.getValueType() &&((CompTargetNode.getValueType() == N.getValueType() &&
"Expecting that the types on LHS and RHS of comparisons match."
) ? static_cast<void> (0) : __assert_fail ("CompTargetNode.getValueType() == N.getValueType() && \"Expecting that the types on LHS and RHS of comparisons match.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5210, __PRETTY_FUNCTION__))
5210 "Expecting that the types on LHS and RHS of comparisons match.")((CompTargetNode.getValueType() == N.getValueType() &&
"Expecting that the types on LHS and RHS of comparisons match."
) ? static_cast<void> (0) : __assert_fail ("CompTargetNode.getValueType() == N.getValueType() && \"Expecting that the types on LHS and RHS of comparisons match.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5210, __PRETTY_FUNCTION__))
;
5211 N = DAG.getNode(ISD::SUB, DL, VT, N, CompTargetNode);
5212 }
5213
5214 // (mul N, P)
5215 SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal);
5216 Created.push_back(Op0.getNode());
5217
5218 // Rotate right only if any divisor was even. We avoid rotates for all-odd
5219 // divisors as a performance improvement, since rotating by 0 is a no-op.
5220 if (HadEvenDivisor) {
5221 // We need ROTR to do this.
5222 if (!isOperationLegalOrCustom(ISD::ROTR, VT))
5223 return SDValue();
5224 SDNodeFlags Flags;
5225 Flags.setExact(true);
5226 // UREM: (rotr (mul N, P), K)
5227 Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal, Flags);
5228 Created.push_back(Op0.getNode());
5229 }
5230
5231 // UREM: (setule/setugt (rotr (mul N, P), K), Q)
5232 SDValue NewCC =
5233 DAG.getSetCC(DL, SETCCVT, Op0, QVal,
5234 ((Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT));
5235 if (!HadTautologicalInvertedLanes)
5236 return NewCC;
5237
5238 // If any lanes previously compared always-false, the NewCC will give
5239 // always-true result for them, so we need to fixup those lanes.
5240 // Or the other way around for inequality predicate.
5241 assert(VT.isVector() && "Can/should only get here for vectors.")((VT.isVector() && "Can/should only get here for vectors."
) ? static_cast<void> (0) : __assert_fail ("VT.isVector() && \"Can/should only get here for vectors.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5241, __PRETTY_FUNCTION__))
;
5242 Created.push_back(NewCC.getNode());
5243
5244 // x u% C1` is *always* less than C1. So given `x u% C1 == C2`,
5245 // if C2 is not less than C1, the comparison is always false.
5246 // But we have produced the comparison that will give the
5247 // opposive tautological answer. So these lanes would need to be fixed up.
5248 SDValue TautologicalInvertedChannels =
5249 DAG.getSetCC(DL, SETCCVT, D, CompTargetNode, ISD::SETULE);
5250 Created.push_back(TautologicalInvertedChannels.getNode());
5251
5252 if (isOperationLegalOrCustom(ISD::VSELECT, SETCCVT)) {
5253 // If we have a vector select, let's replace the comparison results in the
5254 // affected lanes with the correct tautological result.
5255 SDValue Replacement = DAG.getBoolConstant(Cond == ISD::SETEQ ? false : true,
5256 DL, SETCCVT, SETCCVT);
5257 return DAG.getNode(ISD::VSELECT, DL, SETCCVT, TautologicalInvertedChannels,
5258 Replacement, NewCC);
5259 }
5260
5261 // Else, we can just invert the comparison result in the appropriate lanes.
5262 if (isOperationLegalOrCustom(ISD::XOR, SETCCVT))
5263 return DAG.getNode(ISD::XOR, DL, SETCCVT, NewCC,
5264 TautologicalInvertedChannels);
5265
5266 return SDValue(); // Don't know how to lower.
5267}
5268
5269/// Given an ISD::SREM used only by an ISD::SETEQ or ISD::SETNE
5270/// where the divisor is constant and the comparison target is zero,
5271/// return a DAG expression that will generate the same comparison result
5272/// using only multiplications, additions and shifts/rotations.
5273/// Ref: "Hacker's Delight" 10-17.
5274SDValue TargetLowering::buildSREMEqFold(EVT SETCCVT, SDValue REMNode,
5275 SDValue CompTargetNode,
5276 ISD::CondCode Cond,
5277 DAGCombinerInfo &DCI,
5278 const SDLoc &DL) const {
5279 SmallVector<SDNode *, 7> Built;
5280 if (SDValue Folded = prepareSREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
5281 DCI, DL, Built)) {
5282 assert(Built.size() <= 7 && "Max size prediction failed.")((Built.size() <= 7 && "Max size prediction failed."
) ? static_cast<void> (0) : __assert_fail ("Built.size() <= 7 && \"Max size prediction failed.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5282, __PRETTY_FUNCTION__))
;
5283 for (SDNode *N : Built)
5284 DCI.AddToWorklist(N);
5285 return Folded;
5286 }
5287
5288 return SDValue();
5289}
5290
5291SDValue
5292TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode,
5293 SDValue CompTargetNode, ISD::CondCode Cond,
5294 DAGCombinerInfo &DCI, const SDLoc &DL,
5295 SmallVectorImpl<SDNode *> &Created) const {
5296 // Fold:
5297 // (seteq/ne (srem N, D), 0)
5298 // To:
5299 // (setule/ugt (rotr (add (mul N, P), A), K), Q)
5300 //
5301 // - D must be constant, with D = D0 * 2^K where D0 is odd
5302 // - P is the multiplicative inverse of D0 modulo 2^W
5303 // - A = bitwiseand(floor((2^(W - 1) - 1) / D0), (-(2^k)))
5304 // - Q = floor((2 * A) / (2^K))
5305 // where W is the width of the common type of N and D.
5306 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&(((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Only applicable for (in)equality comparisons."
) ? static_cast<void> (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5307, __PRETTY_FUNCTION__))
5307 "Only applicable for (in)equality comparisons.")(((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Only applicable for (in)equality comparisons."
) ? static_cast<void> (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5307, __PRETTY_FUNCTION__))
;
5308
5309 SelectionDAG &DAG = DCI.DAG;
5310
5311 EVT VT = REMNode.getValueType();
5312 EVT SVT = VT.getScalarType();
5313 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
5314 EVT ShSVT = ShVT.getScalarType();
5315
5316 // If MUL is unavailable, we cannot proceed in any case.
5317 if (!isOperationLegalOrCustom(ISD::MUL, VT))
5318 return SDValue();
5319
5320 // TODO: Could support comparing with non-zero too.
5321 ConstantSDNode *CompTarget = isConstOrConstSplat(CompTargetNode);
5322 if (!CompTarget || !CompTarget->isNullValue())
5323 return SDValue();
5324
5325 bool HadIntMinDivisor = false;
5326 bool HadOneDivisor = false;
5327 bool AllDivisorsAreOnes = true;
5328 bool HadEvenDivisor = false;
5329 bool NeedToApplyOffset = false;
5330 bool AllDivisorsArePowerOfTwo = true;
5331 SmallVector<SDValue, 16> PAmts, AAmts, KAmts, QAmts;
5332
5333 auto BuildSREMPattern = [&](ConstantSDNode *C) {
5334 // Division by 0 is UB. Leave it to be constant-folded elsewhere.
5335 if (C->isNullValue())
1
Taking false branch
5336 return false;
5337
5338 // FIXME: we don't fold `rem %X, -C` to `rem %X, C` in DAGCombine.
5339
5340 // WARNING: this fold is only valid for positive divisors!
5341 APInt D = C->getAPIntValue();
5342 if (D.isNegative())
2
Taking false branch
5343 D.negate(); // `rem %X, -C` is equivalent to `rem %X, C`
5344
5345 HadIntMinDivisor |= D.isMinSignedValue();
5346
5347 // If all divisors are ones, we will prefer to avoid the fold.
5348 HadOneDivisor |= D.isOneValue();
5349 AllDivisorsAreOnes &= D.isOneValue();
5350
5351 // Decompose D into D0 * 2^K
5352 unsigned K = D.countTrailingZeros();
5353 assert((!D.isOneValue() || (K == 0)) && "For divisor '1' we won't rotate.")(((!D.isOneValue() || (K == 0)) && "For divisor '1' we won't rotate."
) ? static_cast<void> (0) : __assert_fail ("(!D.isOneValue() || (K == 0)) && \"For divisor '1' we won't rotate.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5353, __PRETTY_FUNCTION__))
;
3
Assuming 'K' is equal to 0
4
'?' condition is true
5354 APInt D0 = D.lshr(K);
5355
5356 if (!D.isMinSignedValue()) {
5
Calling 'APInt::isMinSignedValue'
5357 // D is even if it has trailing zeros; unless it's INT_MIN, in which case
5358 // we don't care about this lane in this fold, we'll special-handle it.
5359 HadEvenDivisor |= (K != 0);
5360 }
5361
5362 // D is a power-of-two if D0 is one. This includes INT_MIN.
5363 // If all divisors are power-of-two, we will prefer to avoid the fold.
5364 AllDivisorsArePowerOfTwo &= D0.isOneValue();
5365
5366 // P = inv(D0, 2^W)
5367 // 2^W requires W + 1 bits, so we have to extend and then truncate.
5368 unsigned W = D.getBitWidth();
5369 APInt P = D0.zext(W + 1)
5370 .multiplicativeInverse(APInt::getSignedMinValue(W + 1))
5371 .trunc(W);
5372 assert(!P.isNullValue() && "No multiplicative inverse!")((!P.isNullValue() && "No multiplicative inverse!") ?
static_cast<void> (0) : __assert_fail ("!P.isNullValue() && \"No multiplicative inverse!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5372, __PRETTY_FUNCTION__))
; // unreachable
5373 assert((D0 * P).isOneValue() && "Multiplicative inverse sanity check.")(((D0 * P).isOneValue() && "Multiplicative inverse sanity check."
) ? static_cast<void> (0) : __assert_fail ("(D0 * P).isOneValue() && \"Multiplicative inverse sanity check.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5373, __PRETTY_FUNCTION__))
;
5374
5375 // A = floor((2^(W - 1) - 1) / D0) & -2^K
5376 APInt A = APInt::getSignedMaxValue(W).udiv(D0);
5377 A.clearLowBits(K);
5378
5379 if (!D.isMinSignedValue()) {
5380 // If divisor INT_MIN, then we don't care about this lane in this fold,
5381 // we'll special-handle it.
5382 NeedToApplyOffset |= A != 0;
5383 }
5384
5385 // Q = floor((2 * A) / (2^K))
5386 APInt Q = (2 * A).udiv(APInt::getOneBitSet(W, K));
5387
5388 assert(APInt::getAllOnesValue(SVT.getSizeInBits()).ugt(A) &&((APInt::getAllOnesValue(SVT.getSizeInBits()).ugt(A) &&
"We are expecting that A is always less than all-ones for SVT"
) ? static_cast<void> (0) : __assert_fail ("APInt::getAllOnesValue(SVT.getSizeInBits()).ugt(A) && \"We are expecting that A is always less than all-ones for SVT\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5389, __PRETTY_FUNCTION__))
5389 "We are expecting that A is always less than all-ones for SVT")((APInt::getAllOnesValue(SVT.getSizeInBits()).ugt(A) &&
"We are expecting that A is always less than all-ones for SVT"
) ? static_cast<void> (0) : __assert_fail ("APInt::getAllOnesValue(SVT.getSizeInBits()).ugt(A) && \"We are expecting that A is always less than all-ones for SVT\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5389, __PRETTY_FUNCTION__))
;
5390 assert(APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) &&((APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) &&
"We are expecting that K is always less than all-ones for ShSVT"
) ? static_cast<void> (0) : __assert_fail ("APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) && \"We are expecting that K is always less than all-ones for ShSVT\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5391, __PRETTY_FUNCTION__))
5391 "We are expecting that K is always less than all-ones for ShSVT")((APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) &&
"We are expecting that K is always less than all-ones for ShSVT"
) ? static_cast<void> (0) : __assert_fail ("APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) && \"We are expecting that K is always less than all-ones for ShSVT\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5391, __PRETTY_FUNCTION__))
;
5392
5393 // If the divisor is 1 the result can be constant-folded. Likewise, we
5394 // don't care about INT_MIN lanes, those can be set to undef if appropriate.
5395 if (D.isOneValue()) {
5396 // Set P, A and K to a bogus values so we can try to splat them.
5397 P = 0;
5398 A = -1;
5399 K = -1;
5400
5401 // x ?% 1 == 0 <--> true <--> x u<= -1
5402 Q = -1;
5403 }
5404
5405 PAmts.push_back(DAG.getConstant(P, DL, SVT));
5406 AAmts.push_back(DAG.getConstant(A, DL, SVT));
5407 KAmts.push_back(
5408 DAG.getConstant(APInt(ShSVT.getSizeInBits(), K), DL, ShSVT));
5409 QAmts.push_back(DAG.getConstant(Q, DL, SVT));
5410 return true;
5411 };
5412
5413 SDValue N = REMNode.getOperand(0);
5414 SDValue D = REMNode.getOperand(1);
5415
5416 // Collect the values from each element.
5417 if (!ISD::matchUnaryPredicate(D, BuildSREMPattern))
5418 return SDValue();
5419
5420 // If this is a srem by a one, avoid the fold since it can be constant-folded.
5421 if (AllDivisorsAreOnes)
5422 return SDValue();
5423
5424 // If this is a srem by a powers-of-two (including INT_MIN), avoid the fold
5425 // since it can be best implemented as a bit test.
5426 if (AllDivisorsArePowerOfTwo)
5427 return SDValue();
5428
5429 SDValue PVal, AVal, KVal, QVal;
5430 if (VT.isVector()) {
5431 if (HadOneDivisor) {
5432 // Try to turn PAmts into a splat, since we don't care about the values
5433 // that are currently '0'. If we can't, just keep '0'`s.
5434 turnVectorIntoSplatVector(PAmts, isNullConstant);
5435 // Try to turn AAmts into a splat, since we don't care about the
5436 // values that are currently '-1'. If we can't, change them to '0'`s.
5437 turnVectorIntoSplatVector(AAmts, isAllOnesConstant,
5438 DAG.getConstant(0, DL, SVT));
5439 // Try to turn KAmts into a splat, since we don't care about the values
5440 // that are currently '-1'. If we can't, change them to '0'`s.
5441 turnVectorIntoSplatVector(KAmts, isAllOnesConstant,
5442 DAG.getConstant(0, DL, ShSVT));
5443 }
5444
5445 PVal = DAG.getBuildVector(VT, DL, PAmts);
5446 AVal = DAG.getBuildVector(VT, DL, AAmts);
5447 KVal = DAG.getBuildVector(ShVT, DL, KAmts);
5448 QVal = DAG.getBuildVector(VT, DL, QAmts);
5449 } else {
5450 PVal = PAmts[0];
5451 AVal = AAmts[0];
5452 KVal = KAmts[0];
5453 QVal = QAmts[0];
5454 }
5455
5456 // (mul N, P)
5457 SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal);
5458 Created.push_back(Op0.getNode());
5459
5460 if (NeedToApplyOffset) {
5461 // We need ADD to do this.
5462 if (!isOperationLegalOrCustom(ISD::ADD, VT))
5463 return SDValue();
5464
5465 // (add (mul N, P), A)
5466 Op0 = DAG.getNode(ISD::ADD, DL, VT, Op0, AVal);
5467 Created.push_back(Op0.getNode());
5468 }
5469
5470 // Rotate right only if any divisor was even. We avoid rotates for all-odd
5471 // divisors as a performance improvement, since rotating by 0 is a no-op.
5472 if (HadEvenDivisor) {
5473 // We need ROTR to do this.
5474 if (!isOperationLegalOrCustom(ISD::ROTR, VT))
5475 return SDValue();
5476 SDNodeFlags Flags;
5477 Flags.setExact(true);
5478 // SREM: (rotr (add (mul N, P), A), K)
5479 Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal, Flags);
5480 Created.push_back(Op0.getNode());
5481 }
5482
5483 // SREM: (setule/setugt (rotr (add (mul N, P), A), K), Q)
5484 SDValue Fold =
5485 DAG.getSetCC(DL, SETCCVT, Op0, QVal,
5486 ((Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT));
5487
5488 // If we didn't have lanes with INT_MIN divisor, then we're done.
5489 if (!HadIntMinDivisor)
5490 return Fold;
5491
5492 // That fold is only valid for positive divisors. Which effectively means,
5493 // it is invalid for INT_MIN divisors. So if we have such a lane,
5494 // we must fix-up results for said lanes.
5495 assert(VT.isVector() && "Can/should only get here for vectors.")((VT.isVector() && "Can/should only get here for vectors."
) ? static_cast<void> (0) : __assert_fail ("VT.isVector() && \"Can/should only get here for vectors.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5495, __PRETTY_FUNCTION__))
;
5496
5497 if (!isOperationLegalOrCustom(ISD::SETEQ, VT) ||
5498 !isOperationLegalOrCustom(ISD::AND, VT) ||
5499 !isOperationLegalOrCustom(Cond, VT) ||
5500 !isOperationLegalOrCustom(ISD::VSELECT, VT))
5501 return SDValue();
5502
5503 Created.push_back(Fold.getNode());
5504
5505 SDValue IntMin = DAG.getConstant(
5506 APInt::getSignedMinValue(SVT.getScalarSizeInBits()), DL, VT);
5507 SDValue IntMax = DAG.getConstant(
5508 APInt::getSignedMaxValue(SVT.getScalarSizeInBits()), DL, VT);
5509 SDValue Zero =
5510 DAG.getConstant(APInt::getNullValue(SVT.getScalarSizeInBits()), DL, VT);
5511
5512 // Which lanes had INT_MIN divisors? Divisor is constant, so const-folded.
5513 SDValue DivisorIsIntMin = DAG.getSetCC(DL, SETCCVT, D, IntMin, ISD::SETEQ);
5514 Created.push_back(DivisorIsIntMin.getNode());
5515
5516 // (N s% INT_MIN) ==/!= 0 <--> (N & INT_MAX) ==/!= 0
5517 SDValue Masked = DAG.getNode(ISD::AND, DL, VT, N, IntMax);
5518 Created.push_back(Masked.getNode());
5519 SDValue MaskedIsZero = DAG.getSetCC(DL, SETCCVT, Masked, Zero, Cond);
5520 Created.push_back(MaskedIsZero.getNode());
5521
5522 // To produce final result we need to blend 2 vectors: 'SetCC' and
5523 // 'MaskedIsZero'. If the divisor for channel was *NOT* INT_MIN, we pick
5524 // from 'Fold', else pick from 'MaskedIsZero'. Since 'DivisorIsIntMin' is
5525 // constant-folded, select can get lowered to a shuffle with constant mask.
5526 SDValue Blended =
5527 DAG.getNode(ISD::VSELECT, DL, VT, DivisorIsIntMin, MaskedIsZero, Fold);
5528
5529 return Blended;
5530}
5531
5532bool TargetLowering::
5533verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const {
5534 if (!isa<ConstantSDNode>(Op.getOperand(0))) {
5535 DAG.getContext()->emitError("argument to '__builtin_return_address' must "
5536 "be a constant integer");
5537 return true;
5538 }
5539
5540 return false;
5541}
5542
5543TargetLowering::NegatibleCost
5544TargetLowering::getNegatibleCost(SDValue Op, SelectionDAG &DAG,
5545 bool LegalOperations, bool ForCodeSize,
5546 unsigned Depth) const {
5547 // fneg is removable even if it has multiple uses.
5548 if (Op.getOpcode() == ISD::FNEG)
5549 return NegatibleCost::Cheaper;
5550
5551 // Don't allow anything with multiple uses unless we know it is free.
5552 EVT VT = Op.getValueType();
5553 const SDNodeFlags Flags = Op->getFlags();
5554 const TargetOptions &Options = DAG.getTarget().Options;
5555 if (!Op.hasOneUse()) {
5556 bool IsFreeExtend = Op.getOpcode() == ISD::FP_EXTEND &&
5557 isFPExtFree(VT, Op.getOperand(0).getValueType());
5558
5559 // If we already have the use of the negated floating constant, it is free
5560 // to negate it even it has multiple uses.
5561 bool IsFreeConstant =
5562 Op.getOpcode() == ISD::ConstantFP &&
5563 !getNegatedExpression(Op, DAG, LegalOperations, ForCodeSize)
5564 .use_empty();
5565
5566 if (!IsFreeExtend && !IsFreeConstant)
5567 return NegatibleCost::Expensive;
5568 }
5569
5570 // Don't recurse exponentially.
5571 if (Depth > SelectionDAG::MaxRecursionDepth)
5572 return NegatibleCost::Expensive;
5573
5574 switch (Op.getOpcode()) {
5575 case ISD::ConstantFP: {
5576 if (!LegalOperations)
5577 return NegatibleCost::Neutral;
5578
5579 // Don't invert constant FP values after legalization unless the target says
5580 // the negated constant is legal.
5581 if (isOperationLegal(ISD::ConstantFP, VT) ||
5582 isFPImmLegal(neg(cast<ConstantFPSDNode>(Op)->getValueAPF()), VT,
5583 ForCodeSize))
5584 return NegatibleCost::Neutral;
5585 break;
5586 }
5587 case ISD::BUILD_VECTOR: {
5588 // Only permit BUILD_VECTOR of constants.
5589 if (llvm::any_of(Op->op_values(), [&](SDValue N) {
5590 return !N.isUndef() && !isa<ConstantFPSDNode>(N);
5591 }))
5592 return NegatibleCost::Expensive;
5593 if (!LegalOperations)
5594 return NegatibleCost::Neutral;
5595 if (isOperationLegal(ISD::ConstantFP, VT) &&
5596 isOperationLegal(ISD::BUILD_VECTOR, VT))
5597 return NegatibleCost::Neutral;
5598 if (llvm::all_of(Op->op_values(), [&](SDValue N) {
5599 return N.isUndef() ||
5600 isFPImmLegal(neg(cast<ConstantFPSDNode>(N)->getValueAPF()), VT,
5601 ForCodeSize);
5602 }))
5603 return NegatibleCost::Neutral;
5604 break;
5605 }
5606 case ISD::FADD: {
5607 if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())
5608 return NegatibleCost::Expensive;
5609
5610 // After operation legalization, it might not be legal to create new FSUBs.
5611 if (LegalOperations && !isOperationLegalOrCustom(ISD::FSUB, VT))
5612 return NegatibleCost::Expensive;
5613
5614 // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
5615 NegatibleCost V0 = getNegatibleCost(Op.getOperand(0), DAG, LegalOperations,
5616 ForCodeSize, Depth + 1);
5617 if (V0 != NegatibleCost::Expensive)
5618 return V0;
5619 // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
5620 return getNegatibleCost(Op.getOperand(1), DAG, LegalOperations, ForCodeSize,
5621 Depth + 1);
5622 }
5623 case ISD::FSUB:
5624 // We can't turn -(A-B) into B-A when we honor signed zeros.
5625 if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())
5626 return NegatibleCost::Expensive;
5627
5628 // fold (fneg (fsub A, B)) -> (fsub B, A)
5629 return NegatibleCost::Neutral;
5630 case ISD::FMUL:
5631 case ISD::FDIV: {
5632 // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y) or (fmul X, (fneg Y))
5633 NegatibleCost V0 = getNegatibleCost(Op.getOperand(0), DAG, LegalOperations,
5634 ForCodeSize, Depth + 1);
5635 if (V0 != NegatibleCost::Expensive)
5636 return V0;
5637
5638 // Ignore X * 2.0 because that is expected to be canonicalized to X + X.
5639 if (auto *C = isConstOrConstSplatFP(Op.getOperand(1)))
5640 if (C->isExactlyValue(2.0) && Op.getOpcode() == ISD::FMUL)
5641 return NegatibleCost::Expensive;
5642
5643 return getNegatibleCost(Op.getOperand(1), DAG, LegalOperations, ForCodeSize,
5644 Depth + 1);
5645 }
5646 case ISD::FMA:
5647 case ISD::FMAD: {
5648 if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())
5649 return NegatibleCost::Expensive;
5650
5651 // fold (fneg (fma X, Y, Z)) -> (fma (fneg X), Y, (fneg Z))
5652 // fold (fneg (fma X, Y, Z)) -> (fma X, (fneg Y), (fneg Z))
5653 NegatibleCost V2 = getNegatibleCost(Op.getOperand(2), DAG, LegalOperations,
5654 ForCodeSize, Depth + 1);
5655 if (NegatibleCost::Expensive == V2)
5656 return NegatibleCost::Expensive;
5657
5658 // One of Op0/Op1 must be cheaply negatible, then select the cheapest.
5659 NegatibleCost V0 = getNegatibleCost(Op.getOperand(0), DAG, LegalOperations,
5660 ForCodeSize, Depth + 1);
5661 NegatibleCost V1 = getNegatibleCost(Op.getOperand(1), DAG, LegalOperations,
5662 ForCodeSize, Depth + 1);
5663 NegatibleCost V01 = std::max(V0, V1);
5664 if (V01 == NegatibleCost::Expensive)
5665 return NegatibleCost::Expensive;
5666 return std::max(V01, V2);
5667 }
5668
5669 case ISD::FP_EXTEND:
5670 case ISD::FP_ROUND:
5671 case ISD::FSIN:
5672 return getNegatibleCost(Op.getOperand(0), DAG, LegalOperations, ForCodeSize,
5673 Depth + 1);
5674 }
5675
5676 return NegatibleCost::Expensive;
5677}
5678
5679SDValue TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,
5680 bool LegalOperations,
5681 bool ForCodeSize,
5682 unsigned Depth) const {
5683 // fneg is removable even if it has multiple uses.
5684 if (Op.getOpcode() == ISD::FNEG)
5685 return Op.getOperand(0);
5686
5687 assert(Depth <= SelectionDAG::MaxRecursionDepth &&((Depth <= SelectionDAG::MaxRecursionDepth && "getNegatedExpression doesn't match getNegatibleCost"
) ? static_cast<void> (0) : __assert_fail ("Depth <= SelectionDAG::MaxRecursionDepth && \"getNegatedExpression doesn't match getNegatibleCost\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5688, __PRETTY_FUNCTION__))
5688 "getNegatedExpression doesn't match getNegatibleCost")((Depth <= SelectionDAG::MaxRecursionDepth && "getNegatedExpression doesn't match getNegatibleCost"
) ? static_cast<void> (0) : __assert_fail ("Depth <= SelectionDAG::MaxRecursionDepth && \"getNegatedExpression doesn't match getNegatibleCost\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5688, __PRETTY_FUNCTION__))
;
5689 const SDNodeFlags Flags = Op->getFlags();
5690
5691 switch (Op.getOpcode()) {
5692 case ISD::ConstantFP: {
5693 APFloat V = cast<ConstantFPSDNode>(Op)->getValueAPF();
5694 V.changeSign();
5695 return DAG.getConstantFP(V, SDLoc(Op), Op.getValueType());
5696 }
5697 case ISD::BUILD_VECTOR: {
5698 SmallVector<SDValue, 4> Ops;
5699 for (SDValue C : Op->op_values()) {
5700 if (C.isUndef()) {
5701 Ops.push_back(C);
5702 continue;
5703 }
5704 APFloat V = cast<ConstantFPSDNode>(C)->getValueAPF();
5705 V.changeSign();
5706 Ops.push_back(DAG.getConstantFP(V, SDLoc(Op), C.getValueType()));
5707 }
5708 return DAG.getBuildVector(Op.getValueType(), SDLoc(Op), Ops);
5709 }
5710 case ISD::FADD: {
5711 assert((DAG.getTarget().Options.NoSignedZerosFPMath ||(((DAG.getTarget().Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros
()) && "Expected NSZ fp-flag") ? static_cast<void>
(0) : __assert_fail ("(DAG.getTarget().Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros()) && \"Expected NSZ fp-flag\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5713, __PRETTY_FUNCTION__))
5712 Flags.hasNoSignedZeros()) &&(((DAG.getTarget().Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros
()) && "Expected NSZ fp-flag") ? static_cast<void>
(0) : __assert_fail ("(DAG.getTarget().Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros()) && \"Expected NSZ fp-flag\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5713, __PRETTY_FUNCTION__))
5713 "Expected NSZ fp-flag")(((DAG.getTarget().Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros
()) && "Expected NSZ fp-flag") ? static_cast<void>
(0) : __assert_fail ("(DAG.getTarget().Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros()) && \"Expected NSZ fp-flag\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5713, __PRETTY_FUNCTION__))
;
5714
5715 // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
5716 NegatibleCost V0 = getNegatibleCost(Op.getOperand(0), DAG, LegalOperations,
5717 ForCodeSize, Depth + 1);
5718 if (V0 != NegatibleCost::Expensive)
5719 return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
5720 getNegatedExpression(Op.getOperand(0), DAG,
5721 LegalOperations, ForCodeSize,
5722 Depth + 1),
5723 Op.getOperand(1), Flags);
5724 // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
5725 return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
5726 getNegatedExpression(Op.getOperand(1), DAG,
5727 LegalOperations, ForCodeSize,
5728 Depth + 1),
5729 Op.getOperand(0), Flags);
5730 }
5731 case ISD::FSUB:
5732 // fold (fneg (fsub 0, B)) -> B
5733 if (ConstantFPSDNode *N0CFP =
5734 isConstOrConstSplatFP(Op.getOperand(0), /*AllowUndefs*/ true))
5735 if (N0CFP->isZero())
5736 return Op.getOperand(1);
5737
5738 // fold (fneg (fsub A, B)) -> (fsub B, A)
5739 return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
5740 Op.getOperand(1), Op.getOperand(0), Flags);
5741
5742 case ISD::FMUL:
5743 case ISD::FDIV: {
5744 // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y)
5745 NegatibleCost V0 = getNegatibleCost(Op.getOperand(0), DAG, LegalOperations,
5746 ForCodeSize, Depth + 1);
5747 if (V0 != NegatibleCost::Expensive)
5748 return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
5749 getNegatedExpression(Op.getOperand(0), DAG,
5750 LegalOperations, ForCodeSize,
5751 Depth + 1),
5752 Op.getOperand(1), Flags);
5753
5754 // fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y))
5755 return DAG.getNode(
5756 Op.getOpcode(), SDLoc(Op), Op.getValueType(), Op.getOperand(0),
5757 getNegatedExpression(Op.getOperand(1), DAG, LegalOperations,
5758 ForCodeSize, Depth + 1),