Bug Summary

File:llvm/include/llvm/ADT/APInt.h
Warning:line 449, column 36
The result of the left shift is undefined due to shifting by '4294967295', which is greater or equal to the width of type 'llvm::APInt::WordType'

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name TargetLowering.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -mframe-pointer=none -fmath-errno -fno-rounding-math -masm-verbose -mconstructor-aliases -munwind-tables -target-cpu x86-64 -dwarf-column-info -fno-split-dwarf-inlining -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-10/lib/clang/10.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/build-llvm/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/build-llvm/include -I /build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-10/lib/clang/10.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/build-llvm/lib/CodeGen/SelectionDAG -fdebug-prefix-map=/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd=. -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2020-01-13-084841-49055-1 -x c++ /build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

1//===-- TargetLowering.cpp - Implement the TargetLowering class -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the TargetLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/CodeGen/TargetLowering.h"
14#include "llvm/ADT/STLExtras.h"
15#include "llvm/CodeGen/CallingConvLower.h"
16#include "llvm/CodeGen/MachineFrameInfo.h"
17#include "llvm/CodeGen/MachineFunction.h"
18#include "llvm/CodeGen/MachineJumpTableInfo.h"
19#include "llvm/CodeGen/MachineRegisterInfo.h"
20#include "llvm/CodeGen/SelectionDAG.h"
21#include "llvm/CodeGen/TargetRegisterInfo.h"
22#include "llvm/CodeGen/TargetSubtargetInfo.h"
23#include "llvm/IR/DataLayout.h"
24#include "llvm/IR/DerivedTypes.h"
25#include "llvm/IR/GlobalVariable.h"
26#include "llvm/IR/LLVMContext.h"
27#include "llvm/MC/MCAsmInfo.h"
28#include "llvm/MC/MCExpr.h"
29#include "llvm/Support/ErrorHandling.h"
30#include "llvm/Support/KnownBits.h"
31#include "llvm/Support/MathExtras.h"
32#include "llvm/Target/TargetLoweringObjectFile.h"
33#include "llvm/Target/TargetMachine.h"
34#include <cctype>
35using namespace llvm;
36
37/// NOTE: The TargetMachine owns TLOF.
38TargetLowering::TargetLowering(const TargetMachine &tm)
39 : TargetLoweringBase(tm) {}
40
41const char *TargetLowering::getTargetNodeName(unsigned Opcode) const {
42 return nullptr;
43}
44
45bool TargetLowering::isPositionIndependent() const {
46 return getTargetMachine().isPositionIndependent();
47}
48
49/// Check whether a given call node is in tail position within its function. If
50/// so, it sets Chain to the input chain of the tail call.
51bool TargetLowering::isInTailCallPosition(SelectionDAG &DAG, SDNode *Node,
52 SDValue &Chain) const {
53 const Function &F = DAG.getMachineFunction().getFunction();
54
55 // First, check if tail calls have been disabled in this function.
56 if (F.getFnAttribute("disable-tail-calls").getValueAsString() == "true")
57 return false;
58
59 // Conservatively require the attributes of the call to match those of
60 // the return. Ignore NoAlias and NonNull because they don't affect the
61 // call sequence.
62 AttributeList CallerAttrs = F.getAttributes();
63 if (AttrBuilder(CallerAttrs, AttributeList::ReturnIndex)
64 .removeAttribute(Attribute::NoAlias)
65 .removeAttribute(Attribute::NonNull)
66 .hasAttributes())
67 return false;
68
69 // It's not safe to eliminate the sign / zero extension of the return value.
70 if (CallerAttrs.hasAttribute(AttributeList::ReturnIndex, Attribute::ZExt) ||
71 CallerAttrs.hasAttribute(AttributeList::ReturnIndex, Attribute::SExt))
72 return false;
73
74 // Check if the only use is a function return node.
75 return isUsedByReturnOnly(Node, Chain);
76}
77
78bool TargetLowering::parametersInCSRMatch(const MachineRegisterInfo &MRI,
79 const uint32_t *CallerPreservedMask,
80 const SmallVectorImpl<CCValAssign> &ArgLocs,
81 const SmallVectorImpl<SDValue> &OutVals) const {
82 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
83 const CCValAssign &ArgLoc = ArgLocs[I];
84 if (!ArgLoc.isRegLoc())
85 continue;
86 Register Reg = ArgLoc.getLocReg();
87 // Only look at callee saved registers.
88 if (MachineOperand::clobbersPhysReg(CallerPreservedMask, Reg))
89 continue;
90 // Check that we pass the value used for the caller.
91 // (We look for a CopyFromReg reading a virtual register that is used
92 // for the function live-in value of register Reg)
93 SDValue Value = OutVals[I];
94 if (Value->getOpcode() != ISD::CopyFromReg)
95 return false;
96 unsigned ArgReg = cast<RegisterSDNode>(Value->getOperand(1))->getReg();
97 if (MRI.getLiveInPhysReg(ArgReg) != Reg)
98 return false;
99 }
100 return true;
101}
102
103/// Set CallLoweringInfo attribute flags based on a call instruction
104/// and called function attributes.
105void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call,
106 unsigned ArgIdx) {
107 IsSExt = Call->paramHasAttr(ArgIdx, Attribute::SExt);
108 IsZExt = Call->paramHasAttr(ArgIdx, Attribute::ZExt);
109 IsInReg = Call->paramHasAttr(ArgIdx, Attribute::InReg);
110 IsSRet = Call->paramHasAttr(ArgIdx, Attribute::StructRet);
111 IsNest = Call->paramHasAttr(ArgIdx, Attribute::Nest);
112 IsByVal = Call->paramHasAttr(ArgIdx, Attribute::ByVal);
113 IsInAlloca = Call->paramHasAttr(ArgIdx, Attribute::InAlloca);
114 IsReturned = Call->paramHasAttr(ArgIdx, Attribute::Returned);
115 IsSwiftSelf = Call->paramHasAttr(ArgIdx, Attribute::SwiftSelf);
116 IsSwiftError = Call->paramHasAttr(ArgIdx, Attribute::SwiftError);
117 Alignment = Call->getParamAlignment(ArgIdx);
118 ByValType = nullptr;
119 if (Call->paramHasAttr(ArgIdx, Attribute::ByVal))
120 ByValType = Call->getParamByValType(ArgIdx);
121}
122
123/// Generate a libcall taking the given operands as arguments and returning a
124/// result of type RetVT.
125std::pair<SDValue, SDValue>
126TargetLowering::makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT,
127 ArrayRef<SDValue> Ops,
128 MakeLibCallOptions CallOptions,
129 const SDLoc &dl,
130 SDValue InChain) const {
131 if (!InChain)
132 InChain = DAG.getEntryNode();
133
134 TargetLowering::ArgListTy Args;
135 Args.reserve(Ops.size());
136
137 TargetLowering::ArgListEntry Entry;
138 for (unsigned i = 0; i < Ops.size(); ++i) {
139 SDValue NewOp = Ops[i];
140 Entry.Node = NewOp;
141 Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
142 Entry.IsSExt = shouldSignExtendTypeInLibCall(NewOp.getValueType(),
143 CallOptions.IsSExt);
144 Entry.IsZExt = !Entry.IsSExt;
145
146 if (CallOptions.IsSoften &&
147 !shouldExtendTypeInLibCall(CallOptions.OpsVTBeforeSoften[i])) {
148 Entry.IsSExt = Entry.IsZExt = false;
149 }
150 Args.push_back(Entry);
151 }
152
153 if (LC == RTLIB::UNKNOWN_LIBCALL)
154 report_fatal_error("Unsupported library call operation!");
155 SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
156 getPointerTy(DAG.getDataLayout()));
157
158 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
159 TargetLowering::CallLoweringInfo CLI(DAG);
160 bool signExtend = shouldSignExtendTypeInLibCall(RetVT, CallOptions.IsSExt);
161 bool zeroExtend = !signExtend;
162
163 if (CallOptions.IsSoften &&
164 !shouldExtendTypeInLibCall(CallOptions.RetVTBeforeSoften)) {
165 signExtend = zeroExtend = false;
166 }
167
168 CLI.setDebugLoc(dl)
169 .setChain(InChain)
170 .setLibCallee(getLibcallCallingConv(LC), RetTy, Callee, std::move(Args))
171 .setNoReturn(CallOptions.DoesNotReturn)
172 .setDiscardResult(!CallOptions.IsReturnValueUsed)
173 .setIsPostTypeLegalization(CallOptions.IsPostTypeLegalization)
174 .setSExtResult(signExtend)
175 .setZExtResult(zeroExtend);
176 return LowerCallTo(CLI);
177}
178
179bool
180TargetLowering::findOptimalMemOpLowering(std::vector<EVT> &MemOps,
181 unsigned Limit, uint64_t Size,
182 unsigned DstAlign, unsigned SrcAlign,
183 bool IsMemset,
184 bool ZeroMemset,
185 bool MemcpyStrSrc,
186 bool AllowOverlap,
187 unsigned DstAS, unsigned SrcAS,
188 const AttributeList &FuncAttributes) const {
189 // If 'SrcAlign' is zero, that means the memory operation does not need to
190 // load the value, i.e. memset or memcpy from constant string. Otherwise,
191 // it's the inferred alignment of the source. 'DstAlign', on the other hand,
192 // is the specified alignment of the memory operation. If it is zero, that
193 // means it's possible to change the alignment of the destination.
194 // 'MemcpyStrSrc' indicates whether the memcpy source is constant so it does
195 // not need to be loaded.
196 if (!(SrcAlign == 0 || SrcAlign >= DstAlign))
197 return false;
198
199 EVT VT = getOptimalMemOpType(Size, DstAlign, SrcAlign,
200 IsMemset, ZeroMemset, MemcpyStrSrc,
201 FuncAttributes);
202
203 if (VT == MVT::Other) {
204 // Use the largest integer type whose alignment constraints are satisfied.
205 // We only need to check DstAlign here as SrcAlign is always greater or
206 // equal to DstAlign (or zero).
207 VT = MVT::i64;
208 while (DstAlign && DstAlign < VT.getSizeInBits() / 8 &&
209 !allowsMisalignedMemoryAccesses(VT, DstAS, DstAlign))
210 VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1);
211 assert(VT.isInteger())((VT.isInteger()) ? static_cast<void> (0) : __assert_fail
("VT.isInteger()", "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 211, __PRETTY_FUNCTION__))
;
212
213 // Find the largest legal integer type.
214 MVT LVT = MVT::i64;
215 while (!isTypeLegal(LVT))
216 LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
217 assert(LVT.isInteger())((LVT.isInteger()) ? static_cast<void> (0) : __assert_fail
("LVT.isInteger()", "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 217, __PRETTY_FUNCTION__))
;
218
219 // If the type we've chosen is larger than the largest legal integer type
220 // then use that instead.
221 if (VT.bitsGT(LVT))
222 VT = LVT;
223 }
224
225 unsigned NumMemOps = 0;
226 while (Size != 0) {
227 unsigned VTSize = VT.getSizeInBits() / 8;
228 while (VTSize > Size) {
229 // For now, only use non-vector load / store's for the left-over pieces.
230 EVT NewVT = VT;
231 unsigned NewVTSize;
232
233 bool Found = false;
234 if (VT.isVector() || VT.isFloatingPoint()) {
235 NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;
236 if (isOperationLegalOrCustom(ISD::STORE, NewVT) &&
237 isSafeMemOpType(NewVT.getSimpleVT()))
238 Found = true;
239 else if (NewVT == MVT::i64 &&
240 isOperationLegalOrCustom(ISD::STORE, MVT::f64) &&
241 isSafeMemOpType(MVT::f64)) {
242 // i64 is usually not legal on 32-bit targets, but f64 may be.
243 NewVT = MVT::f64;
244 Found = true;
245 }
246 }
247
248 if (!Found) {
249 do {
250 NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
251 if (NewVT == MVT::i8)
252 break;
253 } while (!isSafeMemOpType(NewVT.getSimpleVT()));
254 }
255 NewVTSize = NewVT.getSizeInBits() / 8;
256
257 // If the new VT cannot cover all of the remaining bits, then consider
258 // issuing a (or a pair of) unaligned and overlapping load / store.
259 bool Fast;
260 if (NumMemOps && AllowOverlap && NewVTSize < Size &&
261 allowsMisalignedMemoryAccesses(VT, DstAS, DstAlign,
262 MachineMemOperand::MONone, &Fast) &&
263 Fast)
264 VTSize = Size;
265 else {
266 VT = NewVT;
267 VTSize = NewVTSize;
268 }
269 }
270
271 if (++NumMemOps > Limit)
272 return false;
273
274 MemOps.push_back(VT);
275 Size -= VTSize;
276 }
277
278 return true;
279}
280
281/// Soften the operands of a comparison. This code is shared among BR_CC,
282/// SELECT_CC, and SETCC handlers.
283void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,
284 SDValue &NewLHS, SDValue &NewRHS,
285 ISD::CondCode &CCCode,
286 const SDLoc &dl, const SDValue OldLHS,
287 const SDValue OldRHS) const {
288 SDValue Chain;
289 return softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, dl, OldLHS,
290 OldRHS, Chain);
291}
292
293void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,
294 SDValue &NewLHS, SDValue &NewRHS,
295 ISD::CondCode &CCCode,
296 const SDLoc &dl, const SDValue OldLHS,
297 const SDValue OldRHS,
298 SDValue &Chain,
299 bool IsSignaling) const {
300 // FIXME: Currently we cannot really respect all IEEE predicates due to libgcc
301 // not supporting it. We can update this code when libgcc provides such
302 // functions.
303
304 assert((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128)(((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT ==
MVT::ppcf128) && "Unsupported setcc type!") ? static_cast
<void> (0) : __assert_fail ("(VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128) && \"Unsupported setcc type!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 305, __PRETTY_FUNCTION__))
305 && "Unsupported setcc type!")(((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT ==
MVT::ppcf128) && "Unsupported setcc type!") ? static_cast
<void> (0) : __assert_fail ("(VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128) && \"Unsupported setcc type!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 305, __PRETTY_FUNCTION__))
;
306
307 // Expand into one or more soft-fp libcall(s).
308 RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL;
309 bool ShouldInvertCC = false;
310 switch (CCCode) {
311 case ISD::SETEQ:
312 case ISD::SETOEQ:
313 LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
314 (VT == MVT::f64) ? RTLIB::OEQ_F64 :
315 (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
316 break;
317 case ISD::SETNE:
318 case ISD::SETUNE:
319 LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 :
320 (VT == MVT::f64) ? RTLIB::UNE_F64 :
321 (VT == MVT::f128) ? RTLIB::UNE_F128 : RTLIB::UNE_PPCF128;
322 break;
323 case ISD::SETGE:
324 case ISD::SETOGE:
325 LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
326 (VT == MVT::f64) ? RTLIB::OGE_F64 :
327 (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
328 break;
329 case ISD::SETLT:
330 case ISD::SETOLT:
331 LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
332 (VT == MVT::f64) ? RTLIB::OLT_F64 :
333 (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
334 break;
335 case ISD::SETLE:
336 case ISD::SETOLE:
337 LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
338 (VT == MVT::f64) ? RTLIB::OLE_F64 :
339 (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
340 break;
341 case ISD::SETGT:
342 case ISD::SETOGT:
343 LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
344 (VT == MVT::f64) ? RTLIB::OGT_F64 :
345 (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
346 break;
347 case ISD::SETO:
348 ShouldInvertCC = true;
349 LLVM_FALLTHROUGH[[gnu::fallthrough]];
350 case ISD::SETUO:
351 LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
352 (VT == MVT::f64) ? RTLIB::UO_F64 :
353 (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
354 break;
355 case ISD::SETONE:
356 // SETONE = O && UNE
357 ShouldInvertCC = true;
358 LLVM_FALLTHROUGH[[gnu::fallthrough]];
359 case ISD::SETUEQ:
360 LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
361 (VT == MVT::f64) ? RTLIB::UO_F64 :
362 (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
363 LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
364 (VT == MVT::f64) ? RTLIB::OEQ_F64 :
365 (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
366 break;
367 default:
368 // Invert CC for unordered comparisons
369 ShouldInvertCC = true;
370 switch (CCCode) {
371 case ISD::SETULT:
372 LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
373 (VT == MVT::f64) ? RTLIB::OGE_F64 :
374 (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
375 break;
376 case ISD::SETULE:
377 LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
378 (VT == MVT::f64) ? RTLIB::OGT_F64 :
379 (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
380 break;
381 case ISD::SETUGT:
382 LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
383 (VT == MVT::f64) ? RTLIB::OLE_F64 :
384 (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
385 break;
386 case ISD::SETUGE:
387 LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
388 (VT == MVT::f64) ? RTLIB::OLT_F64 :
389 (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
390 break;
391 default: llvm_unreachable("Do not know how to soften this setcc!")::llvm::llvm_unreachable_internal("Do not know how to soften this setcc!"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 391)
;
392 }
393 }
394
395 // Use the target specific return value for comparions lib calls.
396 EVT RetVT = getCmpLibcallReturnType();
397 SDValue Ops[2] = {NewLHS, NewRHS};
398 TargetLowering::MakeLibCallOptions CallOptions;
399 EVT OpsVT[2] = { OldLHS.getValueType(),
400 OldRHS.getValueType() };
401 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT, true);
402 auto Call = makeLibCall(DAG, LC1, RetVT, Ops, CallOptions, dl, Chain);
403 NewLHS = Call.first;
404 NewRHS = DAG.getConstant(0, dl, RetVT);
405
406 CCCode = getCmpLibcallCC(LC1);
407 if (ShouldInvertCC) {
408 assert(RetVT.isInteger())((RetVT.isInteger()) ? static_cast<void> (0) : __assert_fail
("RetVT.isInteger()", "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 408, __PRETTY_FUNCTION__))
;
409 CCCode = getSetCCInverse(CCCode, RetVT);
410 }
411
412 if (LC2 == RTLIB::UNKNOWN_LIBCALL) {
413 // Update Chain.
414 Chain = Call.second;
415 } else {
416 EVT SetCCVT =
417 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), RetVT);
418 SDValue Tmp = DAG.getSetCC(dl, SetCCVT, NewLHS, NewRHS, CCCode);
419 auto Call2 = makeLibCall(DAG, LC2, RetVT, Ops, CallOptions, dl, Chain);
420 CCCode = getCmpLibcallCC(LC2);
421 if (ShouldInvertCC)
422 CCCode = getSetCCInverse(CCCode, RetVT);
423 NewLHS = DAG.getSetCC(dl, SetCCVT, Call2.first, NewRHS, CCCode);
424 if (Chain)
425 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Call.second,
426 Call2.second);
427 NewLHS = DAG.getNode(ShouldInvertCC ? ISD::AND : ISD::OR, dl,
428 Tmp.getValueType(), Tmp, NewLHS);
429 NewRHS = SDValue();
430 }
431}
432
433/// Return the entry encoding for a jump table in the current function. The
434/// returned value is a member of the MachineJumpTableInfo::JTEntryKind enum.
435unsigned TargetLowering::getJumpTableEncoding() const {
436 // In non-pic modes, just use the address of a block.
437 if (!isPositionIndependent())
438 return MachineJumpTableInfo::EK_BlockAddress;
439
440 // In PIC mode, if the target supports a GPRel32 directive, use it.
441 if (getTargetMachine().getMCAsmInfo()->getGPRel32Directive() != nullptr)
442 return MachineJumpTableInfo::EK_GPRel32BlockAddress;
443
444 // Otherwise, use a label difference.
445 return MachineJumpTableInfo::EK_LabelDifference32;
446}
447
448SDValue TargetLowering::getPICJumpTableRelocBase(SDValue Table,
449 SelectionDAG &DAG) const {
450 // If our PIC model is GP relative, use the global offset table as the base.
451 unsigned JTEncoding = getJumpTableEncoding();
452
453 if ((JTEncoding == MachineJumpTableInfo::EK_GPRel64BlockAddress) ||
454 (JTEncoding == MachineJumpTableInfo::EK_GPRel32BlockAddress))
455 return DAG.getGLOBAL_OFFSET_TABLE(getPointerTy(DAG.getDataLayout()));
456
457 return Table;
458}
459
460/// This returns the relocation base for the given PIC jumptable, the same as
461/// getPICJumpTableRelocBase, but as an MCExpr.
462const MCExpr *
463TargetLowering::getPICJumpTableRelocBaseExpr(const MachineFunction *MF,
464 unsigned JTI,MCContext &Ctx) const{
465 // The normal PIC reloc base is the label at the start of the jump table.
466 return MCSymbolRefExpr::create(MF->getJTISymbol(JTI, Ctx), Ctx);
467}
468
469bool
470TargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
471 const TargetMachine &TM = getTargetMachine();
472 const GlobalValue *GV = GA->getGlobal();
473
474 // If the address is not even local to this DSO we will have to load it from
475 // a got and then add the offset.
476 if (!TM.shouldAssumeDSOLocal(*GV->getParent(), GV))
477 return false;
478
479 // If the code is position independent we will have to add a base register.
480 if (isPositionIndependent())
481 return false;
482
483 // Otherwise we can do it.
484 return true;
485}
486
487//===----------------------------------------------------------------------===//
488// Optimization Methods
489//===----------------------------------------------------------------------===//
490
491/// If the specified instruction has a constant integer operand and there are
492/// bits set in that constant that are not demanded, then clear those bits and
493/// return true.
494bool TargetLowering::ShrinkDemandedConstant(SDValue Op, const APInt &Demanded,
495 TargetLoweringOpt &TLO) const {
496 SDLoc DL(Op);
497 unsigned Opcode = Op.getOpcode();
498
499 // Do target-specific constant optimization.
500 if (targetShrinkDemandedConstant(Op, Demanded, TLO))
501 return TLO.New.getNode();
502
503 // FIXME: ISD::SELECT, ISD::SELECT_CC
504 switch (Opcode) {
505 default:
506 break;
507 case ISD::XOR:
508 case ISD::AND:
509 case ISD::OR: {
510 auto *Op1C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
511 if (!Op1C)
512 return false;
513
514 // If this is a 'not' op, don't touch it because that's a canonical form.
515 const APInt &C = Op1C->getAPIntValue();
516 if (Opcode == ISD::XOR && Demanded.isSubsetOf(C))
517 return false;
518
519 if (!C.isSubsetOf(Demanded)) {
520 EVT VT = Op.getValueType();
521 SDValue NewC = TLO.DAG.getConstant(Demanded & C, DL, VT);
522 SDValue NewOp = TLO.DAG.getNode(Opcode, DL, VT, Op.getOperand(0), NewC);
523 return TLO.CombineTo(Op, NewOp);
524 }
525
526 break;
527 }
528 }
529
530 return false;
531}
532
533/// Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.
534/// This uses isZExtFree and ZERO_EXTEND for the widening cast, but it could be
535/// generalized for targets with other types of implicit widening casts.
536bool TargetLowering::ShrinkDemandedOp(SDValue Op, unsigned BitWidth,
537 const APInt &Demanded,
538 TargetLoweringOpt &TLO) const {
539 assert(Op.getNumOperands() == 2 &&((Op.getNumOperands() == 2 && "ShrinkDemandedOp only supports binary operators!"
) ? static_cast<void> (0) : __assert_fail ("Op.getNumOperands() == 2 && \"ShrinkDemandedOp only supports binary operators!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 540, __PRETTY_FUNCTION__))
540 "ShrinkDemandedOp only supports binary operators!")((Op.getNumOperands() == 2 && "ShrinkDemandedOp only supports binary operators!"
) ? static_cast<void> (0) : __assert_fail ("Op.getNumOperands() == 2 && \"ShrinkDemandedOp only supports binary operators!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 540, __PRETTY_FUNCTION__))
;
541 assert(Op.getNode()->getNumValues() == 1 &&((Op.getNode()->getNumValues() == 1 && "ShrinkDemandedOp only supports nodes with one result!"
) ? static_cast<void> (0) : __assert_fail ("Op.getNode()->getNumValues() == 1 && \"ShrinkDemandedOp only supports nodes with one result!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 542, __PRETTY_FUNCTION__))
542 "ShrinkDemandedOp only supports nodes with one result!")((Op.getNode()->getNumValues() == 1 && "ShrinkDemandedOp only supports nodes with one result!"
) ? static_cast<void> (0) : __assert_fail ("Op.getNode()->getNumValues() == 1 && \"ShrinkDemandedOp only supports nodes with one result!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 542, __PRETTY_FUNCTION__))
;
543
544 SelectionDAG &DAG = TLO.DAG;
545 SDLoc dl(Op);
546
547 // Early return, as this function cannot handle vector types.
548 if (Op.getValueType().isVector())
549 return false;
550
551 // Don't do this if the node has another user, which may require the
552 // full value.
553 if (!Op.getNode()->hasOneUse())
554 return false;
555
556 // Search for the smallest integer type with free casts to and from
557 // Op's type. For expedience, just check power-of-2 integer types.
558 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
559 unsigned DemandedSize = Demanded.getActiveBits();
560 unsigned SmallVTBits = DemandedSize;
561 if (!isPowerOf2_32(SmallVTBits))
562 SmallVTBits = NextPowerOf2(SmallVTBits);
563 for (; SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(SmallVTBits)) {
564 EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), SmallVTBits);
565 if (TLI.isTruncateFree(Op.getValueType(), SmallVT) &&
566 TLI.isZExtFree(SmallVT, Op.getValueType())) {
567 // We found a type with free casts.
568 SDValue X = DAG.getNode(
569 Op.getOpcode(), dl, SmallVT,
570 DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(0)),
571 DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(1)));
572 assert(DemandedSize <= SmallVTBits && "Narrowed below demanded bits?")((DemandedSize <= SmallVTBits && "Narrowed below demanded bits?"
) ? static_cast<void> (0) : __assert_fail ("DemandedSize <= SmallVTBits && \"Narrowed below demanded bits?\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 572, __PRETTY_FUNCTION__))
;
573 SDValue Z = DAG.getNode(ISD::ANY_EXTEND, dl, Op.getValueType(), X);
574 return TLO.CombineTo(Op, Z);
575 }
576 }
577 return false;
578}
579
580bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
581 DAGCombinerInfo &DCI) const {
582 SelectionDAG &DAG = DCI.DAG;
583 TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
584 !DCI.isBeforeLegalizeOps());
585 KnownBits Known;
586
587 bool Simplified = SimplifyDemandedBits(Op, DemandedBits, Known, TLO);
588 if (Simplified) {
589 DCI.AddToWorklist(Op.getNode());
590 DCI.CommitTargetLoweringOpt(TLO);
591 }
592 return Simplified;
593}
594
595bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
596 KnownBits &Known,
597 TargetLoweringOpt &TLO,
598 unsigned Depth,
599 bool AssumeSingleUse) const {
600 EVT VT = Op.getValueType();
601 APInt DemandedElts = VT.isVector()
602 ? APInt::getAllOnesValue(VT.getVectorNumElements())
603 : APInt(1, 1);
604 return SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO, Depth,
605 AssumeSingleUse);
606}
607
608// TODO: Can we merge SelectionDAG::GetDemandedBits into this?
609// TODO: Under what circumstances can we create nodes? Constant folding?
610SDValue TargetLowering::SimplifyMultipleUseDemandedBits(
611 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
612 SelectionDAG &DAG, unsigned Depth) const {
613 // Limit search depth.
614 if (Depth >= SelectionDAG::MaxRecursionDepth)
615 return SDValue();
616
617 // Ignore UNDEFs.
618 if (Op.isUndef())
619 return SDValue();
620
621 // Not demanding any bits/elts from Op.
622 if (DemandedBits == 0 || DemandedElts == 0)
623 return DAG.getUNDEF(Op.getValueType());
624
625 unsigned NumElts = DemandedElts.getBitWidth();
626 KnownBits LHSKnown, RHSKnown;
627 switch (Op.getOpcode()) {
628 case ISD::BITCAST: {
629 SDValue Src = peekThroughBitcasts(Op.getOperand(0));
630 EVT SrcVT = Src.getValueType();
631 EVT DstVT = Op.getValueType();
632 unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
633 unsigned NumDstEltBits = DstVT.getScalarSizeInBits();
634
635 if (NumSrcEltBits == NumDstEltBits)
636 if (SDValue V = SimplifyMultipleUseDemandedBits(
637 Src, DemandedBits, DemandedElts, DAG, Depth + 1))
638 return DAG.getBitcast(DstVT, V);
639
640 // TODO - bigendian once we have test coverage.
641 if (SrcVT.isVector() && (NumDstEltBits % NumSrcEltBits) == 0 &&
642 DAG.getDataLayout().isLittleEndian()) {
643 unsigned Scale = NumDstEltBits / NumSrcEltBits;
644 unsigned NumSrcElts = SrcVT.getVectorNumElements();
645 APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);
646 APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);
647 for (unsigned i = 0; i != Scale; ++i) {
648 unsigned Offset = i * NumSrcEltBits;
649 APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset);
650 if (!Sub.isNullValue()) {
651 DemandedSrcBits |= Sub;
652 for (unsigned j = 0; j != NumElts; ++j)
653 if (DemandedElts[j])
654 DemandedSrcElts.setBit((j * Scale) + i);
655 }
656 }
657
658 if (SDValue V = SimplifyMultipleUseDemandedBits(
659 Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))
660 return DAG.getBitcast(DstVT, V);
661 }
662
663 // TODO - bigendian once we have test coverage.
664 if ((NumSrcEltBits % NumDstEltBits) == 0 &&
665 DAG.getDataLayout().isLittleEndian()) {
666 unsigned Scale = NumSrcEltBits / NumDstEltBits;
667 unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
668 APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);
669 APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);
670 for (unsigned i = 0; i != NumElts; ++i)
671 if (DemandedElts[i]) {
672 unsigned Offset = (i % Scale) * NumDstEltBits;
673 DemandedSrcBits.insertBits(DemandedBits, Offset);
674 DemandedSrcElts.setBit(i / Scale);
675 }
676
677 if (SDValue V = SimplifyMultipleUseDemandedBits(
678 Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))
679 return DAG.getBitcast(DstVT, V);
680 }
681
682 break;
683 }
684 case ISD::AND: {
685 LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
686 RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
687
688 // If all of the demanded bits are known 1 on one side, return the other.
689 // These bits cannot contribute to the result of the 'and' in this
690 // context.
691 if (DemandedBits.isSubsetOf(LHSKnown.Zero | RHSKnown.One))
692 return Op.getOperand(0);
693 if (DemandedBits.isSubsetOf(RHSKnown.Zero | LHSKnown.One))
694 return Op.getOperand(1);
695 break;
696 }
697 case ISD::OR: {
698 LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
699 RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
700
701 // If all of the demanded bits are known zero on one side, return the
702 // other. These bits cannot contribute to the result of the 'or' in this
703 // context.
704 if (DemandedBits.isSubsetOf(LHSKnown.One | RHSKnown.Zero))
705 return Op.getOperand(0);
706 if (DemandedBits.isSubsetOf(RHSKnown.One | LHSKnown.Zero))
707 return Op.getOperand(1);
708 break;
709 }
710 case ISD::XOR: {
711 LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
712 RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
713
714 // If all of the demanded bits are known zero on one side, return the
715 // other.
716 if (DemandedBits.isSubsetOf(RHSKnown.Zero))
717 return Op.getOperand(0);
718 if (DemandedBits.isSubsetOf(LHSKnown.Zero))
719 return Op.getOperand(1);
720 break;
721 }
722 case ISD::SETCC: {
723 SDValue Op0 = Op.getOperand(0);
724 SDValue Op1 = Op.getOperand(1);
725 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
726 // If (1) we only need the sign-bit, (2) the setcc operands are the same
727 // width as the setcc result, and (3) the result of a setcc conforms to 0 or
728 // -1, we may be able to bypass the setcc.
729 if (DemandedBits.isSignMask() &&
730 Op0.getScalarValueSizeInBits() == DemandedBits.getBitWidth() &&
731 getBooleanContents(Op0.getValueType()) ==
732 BooleanContent::ZeroOrNegativeOneBooleanContent) {
733 // If we're testing X < 0, then this compare isn't needed - just use X!
734 // FIXME: We're limiting to integer types here, but this should also work
735 // if we don't care about FP signed-zero. The use of SETLT with FP means
736 // that we don't care about NaNs.
737 if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
738 (isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode())))
739 return Op0;
740 }
741 break;
742 }
743 case ISD::SIGN_EXTEND_INREG: {
744 // If none of the extended bits are demanded, eliminate the sextinreg.
745 EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
746 if (DemandedBits.getActiveBits() <= ExVT.getScalarSizeInBits())
747 return Op.getOperand(0);
748 break;
749 }
750 case ISD::INSERT_VECTOR_ELT: {
751 // If we don't demand the inserted element, return the base vector.
752 SDValue Vec = Op.getOperand(0);
753 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
754 EVT VecVT = Vec.getValueType();
755 if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements()) &&
756 !DemandedElts[CIdx->getZExtValue()])
757 return Vec;
758 break;
759 }
760 case ISD::VECTOR_SHUFFLE: {
761 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
762
763 // If all the demanded elts are from one operand and are inline,
764 // then we can use the operand directly.
765 bool AllUndef = true, IdentityLHS = true, IdentityRHS = true;
766 for (unsigned i = 0; i != NumElts; ++i) {
767 int M = ShuffleMask[i];
768 if (M < 0 || !DemandedElts[i])
769 continue;
770 AllUndef = false;
771 IdentityLHS &= (M == (int)i);
772 IdentityRHS &= ((M - NumElts) == i);
773 }
774
775 if (AllUndef)
776 return DAG.getUNDEF(Op.getValueType());
777 if (IdentityLHS)
778 return Op.getOperand(0);
779 if (IdentityRHS)
780 return Op.getOperand(1);
781 break;
782 }
783 default:
784 if (Op.getOpcode() >= ISD::BUILTIN_OP_END)
785 if (SDValue V = SimplifyMultipleUseDemandedBitsForTargetNode(
786 Op, DemandedBits, DemandedElts, DAG, Depth))
787 return V;
788 break;
789 }
790 return SDValue();
791}
792
793/// Look at Op. At this point, we know that only the OriginalDemandedBits of the
794/// result of Op are ever used downstream. If we can use this information to
795/// simplify Op, create a new simplified DAG node and return true, returning the
796/// original and new nodes in Old and New. Otherwise, analyze the expression and
797/// return a mask of Known bits for the expression (used to simplify the
798/// caller). The Known bits may only be accurate for those bits in the
799/// OriginalDemandedBits and OriginalDemandedElts.
800bool TargetLowering::SimplifyDemandedBits(
801 SDValue Op, const APInt &OriginalDemandedBits,
802 const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
803 unsigned Depth, bool AssumeSingleUse) const {
804 unsigned BitWidth = OriginalDemandedBits.getBitWidth();
805 assert(Op.getScalarValueSizeInBits() == BitWidth &&((Op.getScalarValueSizeInBits() == BitWidth && "Mask size mismatches value type size!"
) ? static_cast<void> (0) : __assert_fail ("Op.getScalarValueSizeInBits() == BitWidth && \"Mask size mismatches value type size!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 806, __PRETTY_FUNCTION__))
806 "Mask size mismatches value type size!")((Op.getScalarValueSizeInBits() == BitWidth && "Mask size mismatches value type size!"
) ? static_cast<void> (0) : __assert_fail ("Op.getScalarValueSizeInBits() == BitWidth && \"Mask size mismatches value type size!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 806, __PRETTY_FUNCTION__))
;
807
808 unsigned NumElts = OriginalDemandedElts.getBitWidth();
809 assert((!Op.getValueType().isVector() ||(((!Op.getValueType().isVector() || NumElts == Op.getValueType
().getVectorNumElements()) && "Unexpected vector size"
) ? static_cast<void> (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 811, __PRETTY_FUNCTION__))
810 NumElts == Op.getValueType().getVectorNumElements()) &&(((!Op.getValueType().isVector() || NumElts == Op.getValueType
().getVectorNumElements()) && "Unexpected vector size"
) ? static_cast<void> (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 811, __PRETTY_FUNCTION__))
811 "Unexpected vector size")(((!Op.getValueType().isVector() || NumElts == Op.getValueType
().getVectorNumElements()) && "Unexpected vector size"
) ? static_cast<void> (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 811, __PRETTY_FUNCTION__))
;
812
813 APInt DemandedBits = OriginalDemandedBits;
814 APInt DemandedElts = OriginalDemandedElts;
815 SDLoc dl(Op);
816 auto &DL = TLO.DAG.getDataLayout();
817
818 // Don't know anything.
819 Known = KnownBits(BitWidth);
820
821 // Undef operand.
822 if (Op.isUndef())
823 return false;
824
825 if (Op.getOpcode() == ISD::Constant) {
826 // We know all of the bits for a constant!
827 Known.One = cast<ConstantSDNode>(Op)->getAPIntValue();
828 Known.Zero = ~Known.One;
829 return false;
830 }
831
832 // Other users may use these bits.
833 EVT VT = Op.getValueType();
834 if (!Op.getNode()->hasOneUse() && !AssumeSingleUse) {
835 if (Depth != 0) {
836 // If not at the root, Just compute the Known bits to
837 // simplify things downstream.
838 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
839 return false;
840 }
841 // If this is the root being simplified, allow it to have multiple uses,
842 // just set the DemandedBits/Elts to all bits.
843 DemandedBits = APInt::getAllOnesValue(BitWidth);
844 DemandedElts = APInt::getAllOnesValue(NumElts);
845 } else if (OriginalDemandedBits == 0 || OriginalDemandedElts == 0) {
846 // Not demanding any bits/elts from Op.
847 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
848 } else if (Depth >= SelectionDAG::MaxRecursionDepth) {
849 // Limit search depth.
850 return false;
851 }
852
853 KnownBits Known2, KnownOut;
854 switch (Op.getOpcode()) {
855 case ISD::TargetConstant:
856 llvm_unreachable("Can't simplify this node")::llvm::llvm_unreachable_internal("Can't simplify this node",
"/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 856)
;
857 case ISD::SCALAR_TO_VECTOR: {
858 if (!DemandedElts[0])
859 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
860
861 KnownBits SrcKnown;
862 SDValue Src = Op.getOperand(0);
863 unsigned SrcBitWidth = Src.getScalarValueSizeInBits();
864 APInt SrcDemandedBits = DemandedBits.zextOrSelf(SrcBitWidth);
865 if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcKnown, TLO, Depth + 1))
866 return true;
867 Known = SrcKnown.zextOrTrunc(BitWidth, false);
868 break;
869 }
870 case ISD::BUILD_VECTOR:
871 // Collect the known bits that are shared by every demanded element.
872 // TODO: Call SimplifyDemandedBits for non-constant demanded elements.
873 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
874 return false; // Don't fall through, will infinitely loop.
875 case ISD::LOAD: {
876 LoadSDNode *LD = cast<LoadSDNode>(Op);
877 if (getTargetConstantFromLoad(LD)) {
878 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
879 return false; // Don't fall through, will infinitely loop.
880 }
881 break;
882 }
883 case ISD::INSERT_VECTOR_ELT: {
884 SDValue Vec = Op.getOperand(0);
885 SDValue Scl = Op.getOperand(1);
886 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
887 EVT VecVT = Vec.getValueType();
888
889 // If index isn't constant, assume we need all vector elements AND the
890 // inserted element.
891 APInt DemandedVecElts(DemandedElts);
892 if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements())) {
893 unsigned Idx = CIdx->getZExtValue();
894 DemandedVecElts.clearBit(Idx);
895
896 // Inserted element is not required.
897 if (!DemandedElts[Idx])
898 return TLO.CombineTo(Op, Vec);
899 }
900
901 KnownBits KnownScl;
902 unsigned NumSclBits = Scl.getScalarValueSizeInBits();
903 APInt DemandedSclBits = DemandedBits.zextOrTrunc(NumSclBits);
904 if (SimplifyDemandedBits(Scl, DemandedSclBits, KnownScl, TLO, Depth + 1))
905 return true;
906
907 Known = KnownScl.zextOrTrunc(BitWidth, false);
908
909 KnownBits KnownVec;
910 if (SimplifyDemandedBits(Vec, DemandedBits, DemandedVecElts, KnownVec, TLO,
911 Depth + 1))
912 return true;
913
914 if (!!DemandedVecElts) {
915 Known.One &= KnownVec.One;
916 Known.Zero &= KnownVec.Zero;
917 }
918
919 return false;
920 }
921 case ISD::INSERT_SUBVECTOR: {
922 SDValue Base = Op.getOperand(0);
923 SDValue Sub = Op.getOperand(1);
924 EVT SubVT = Sub.getValueType();
925 unsigned NumSubElts = SubVT.getVectorNumElements();
926
927 // If index isn't constant, assume we need the original demanded base
928 // elements and ALL the inserted subvector elements.
929 APInt BaseElts = DemandedElts;
930 APInt SubElts = APInt::getAllOnesValue(NumSubElts);
931 if (isa<ConstantSDNode>(Op.getOperand(2))) {
932 const APInt &Idx = Op.getConstantOperandAPInt(2);
933 if (Idx.ule(NumElts - NumSubElts)) {
934 unsigned SubIdx = Idx.getZExtValue();
935 SubElts = DemandedElts.extractBits(NumSubElts, SubIdx);
936 BaseElts.insertBits(APInt::getNullValue(NumSubElts), SubIdx);
937 }
938 }
939
940 KnownBits KnownSub, KnownBase;
941 if (SimplifyDemandedBits(Sub, DemandedBits, SubElts, KnownSub, TLO,
942 Depth + 1))
943 return true;
944 if (SimplifyDemandedBits(Base, DemandedBits, BaseElts, KnownBase, TLO,
945 Depth + 1))
946 return true;
947
948 Known.Zero.setAllBits();
949 Known.One.setAllBits();
950 if (!!SubElts) {
951 Known.One &= KnownSub.One;
952 Known.Zero &= KnownSub.Zero;
953 }
954 if (!!BaseElts) {
955 Known.One &= KnownBase.One;
956 Known.Zero &= KnownBase.Zero;
957 }
958 break;
959 }
960 case ISD::EXTRACT_SUBVECTOR: {
961 // If index isn't constant, assume we need all the source vector elements.
962 SDValue Src = Op.getOperand(0);
963 ConstantSDNode *SubIdx = dyn_cast<ConstantSDNode>(Op.getOperand(1));
964 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
965 APInt SrcElts = APInt::getAllOnesValue(NumSrcElts);
966 if (SubIdx && SubIdx->getAPIntValue().ule(NumSrcElts - NumElts)) {
967 // Offset the demanded elts by the subvector index.
968 uint64_t Idx = SubIdx->getZExtValue();
969 SrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
970 }
971 if (SimplifyDemandedBits(Src, DemandedBits, SrcElts, Known, TLO, Depth + 1))
972 return true;
973 break;
974 }
975 case ISD::CONCAT_VECTORS: {
976 Known.Zero.setAllBits();
977 Known.One.setAllBits();
978 EVT SubVT = Op.getOperand(0).getValueType();
979 unsigned NumSubVecs = Op.getNumOperands();
980 unsigned NumSubElts = SubVT.getVectorNumElements();
981 for (unsigned i = 0; i != NumSubVecs; ++i) {
982 APInt DemandedSubElts =
983 DemandedElts.extractBits(NumSubElts, i * NumSubElts);
984 if (SimplifyDemandedBits(Op.getOperand(i), DemandedBits, DemandedSubElts,
985 Known2, TLO, Depth + 1))
986 return true;
987 // Known bits are shared by every demanded subvector element.
988 if (!!DemandedSubElts) {
989 Known.One &= Known2.One;
990 Known.Zero &= Known2.Zero;
991 }
992 }
993 break;
994 }
995 case ISD::VECTOR_SHUFFLE: {
996 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
997
998 // Collect demanded elements from shuffle operands..
999 APInt DemandedLHS(NumElts, 0);
1000 APInt DemandedRHS(NumElts, 0);
1001 for (unsigned i = 0; i != NumElts; ++i) {
1002 if (!DemandedElts[i])
1003 continue;
1004 int M = ShuffleMask[i];
1005 if (M < 0) {
1006 // For UNDEF elements, we don't know anything about the common state of
1007 // the shuffle result.
1008 DemandedLHS.clearAllBits();
1009 DemandedRHS.clearAllBits();
1010 break;
1011 }
1012 assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range")((0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range"
) ? static_cast<void> (0) : __assert_fail ("0 <= M && M < (int)(2 * NumElts) && \"Shuffle index out of range\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1012, __PRETTY_FUNCTION__))
;
1013 if (M < (int)NumElts)
1014 DemandedLHS.setBit(M);
1015 else
1016 DemandedRHS.setBit(M - NumElts);
1017 }
1018
1019 if (!!DemandedLHS || !!DemandedRHS) {
1020 SDValue Op0 = Op.getOperand(0);
1021 SDValue Op1 = Op.getOperand(1);
1022
1023 Known.Zero.setAllBits();
1024 Known.One.setAllBits();
1025 if (!!DemandedLHS) {
1026 if (SimplifyDemandedBits(Op0, DemandedBits, DemandedLHS, Known2, TLO,
1027 Depth + 1))
1028 return true;
1029 Known.One &= Known2.One;
1030 Known.Zero &= Known2.Zero;
1031 }
1032 if (!!DemandedRHS) {
1033 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedRHS, Known2, TLO,
1034 Depth + 1))
1035 return true;
1036 Known.One &= Known2.One;
1037 Known.Zero &= Known2.Zero;
1038 }
1039
1040 // Attempt to avoid multi-use ops if we don't need anything from them.
1041 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1042 Op0, DemandedBits, DemandedLHS, TLO.DAG, Depth + 1);
1043 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1044 Op1, DemandedBits, DemandedRHS, TLO.DAG, Depth + 1);
1045 if (DemandedOp0 || DemandedOp1) {
1046 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1047 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1048 SDValue NewOp = TLO.DAG.getVectorShuffle(VT, dl, Op0, Op1, ShuffleMask);
1049 return TLO.CombineTo(Op, NewOp);
1050 }
1051 }
1052 break;
1053 }
1054 case ISD::AND: {
1055 SDValue Op0 = Op.getOperand(0);
1056 SDValue Op1 = Op.getOperand(1);
1057
1058 // If the RHS is a constant, check to see if the LHS would be zero without
1059 // using the bits from the RHS. Below, we use knowledge about the RHS to
1060 // simplify the LHS, here we're using information from the LHS to simplify
1061 // the RHS.
1062 if (ConstantSDNode *RHSC = isConstOrConstSplat(Op1)) {
1063 // Do not increment Depth here; that can cause an infinite loop.
1064 KnownBits LHSKnown = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth);
1065 // If the LHS already has zeros where RHSC does, this 'and' is dead.
1066 if ((LHSKnown.Zero & DemandedBits) ==
1067 (~RHSC->getAPIntValue() & DemandedBits))
1068 return TLO.CombineTo(Op, Op0);
1069
1070 // If any of the set bits in the RHS are known zero on the LHS, shrink
1071 // the constant.
1072 if (ShrinkDemandedConstant(Op, ~LHSKnown.Zero & DemandedBits, TLO))
1073 return true;
1074
1075 // Bitwise-not (xor X, -1) is a special case: we don't usually shrink its
1076 // constant, but if this 'and' is only clearing bits that were just set by
1077 // the xor, then this 'and' can be eliminated by shrinking the mask of
1078 // the xor. For example, for a 32-bit X:
1079 // and (xor (srl X, 31), -1), 1 --> xor (srl X, 31), 1
1080 if (isBitwiseNot(Op0) && Op0.hasOneUse() &&
1081 LHSKnown.One == ~RHSC->getAPIntValue()) {
1082 SDValue Xor = TLO.DAG.getNode(ISD::XOR, dl, VT, Op0.getOperand(0), Op1);
1083 return TLO.CombineTo(Op, Xor);
1084 }
1085 }
1086
1087 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1088 Depth + 1))
1089 return true;
1090 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1090, __PRETTY_FUNCTION__))
;
1091 if (SimplifyDemandedBits(Op0, ~Known.Zero & DemandedBits, DemandedElts,
1092 Known2, TLO, Depth + 1))
1093 return true;
1094 assert(!Known2.hasConflict() && "Bits known to be one AND zero?")((!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1094, __PRETTY_FUNCTION__))
;
1095
1096 // Attempt to avoid multi-use ops if we don't need anything from them.
1097 if (!DemandedBits.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {
1098 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1099 Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1100 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1101 Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1102 if (DemandedOp0 || DemandedOp1) {
1103 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1104 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1105 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1106 return TLO.CombineTo(Op, NewOp);
1107 }
1108 }
1109
1110 // If all of the demanded bits are known one on one side, return the other.
1111 // These bits cannot contribute to the result of the 'and'.
1112 if (DemandedBits.isSubsetOf(Known2.Zero | Known.One))
1113 return TLO.CombineTo(Op, Op0);
1114 if (DemandedBits.isSubsetOf(Known.Zero | Known2.One))
1115 return TLO.CombineTo(Op, Op1);
1116 // If all of the demanded bits in the inputs are known zeros, return zero.
1117 if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
1118 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, dl, VT));
1119 // If the RHS is a constant, see if we can simplify it.
1120 if (ShrinkDemandedConstant(Op, ~Known2.Zero & DemandedBits, TLO))
1121 return true;
1122 // If the operation can be done in a smaller type, do so.
1123 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1124 return true;
1125
1126 // Output known-1 bits are only known if set in both the LHS & RHS.
1127 Known.One &= Known2.One;
1128 // Output known-0 are known to be clear if zero in either the LHS | RHS.
1129 Known.Zero |= Known2.Zero;
1130 break;
1131 }
1132 case ISD::OR: {
1133 SDValue Op0 = Op.getOperand(0);
1134 SDValue Op1 = Op.getOperand(1);
1135
1136 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1137 Depth + 1))
1138 return true;
1139 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1139, __PRETTY_FUNCTION__))
;
1140 if (SimplifyDemandedBits(Op0, ~Known.One & DemandedBits, DemandedElts,
1141 Known2, TLO, Depth + 1))
1142 return true;
1143 assert(!Known2.hasConflict() && "Bits known to be one AND zero?")((!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1143, __PRETTY_FUNCTION__))
;
1144
1145 // Attempt to avoid multi-use ops if we don't need anything from them.
1146 if (!DemandedBits.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {
1147 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1148 Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1149 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1150 Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1151 if (DemandedOp0 || DemandedOp1) {
1152 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1153 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1154 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1155 return TLO.CombineTo(Op, NewOp);
1156 }
1157 }
1158
1159 // If all of the demanded bits are known zero on one side, return the other.
1160 // These bits cannot contribute to the result of the 'or'.
1161 if (DemandedBits.isSubsetOf(Known2.One | Known.Zero))
1162 return TLO.CombineTo(Op, Op0);
1163 if (DemandedBits.isSubsetOf(Known.One | Known2.Zero))
1164 return TLO.CombineTo(Op, Op1);
1165 // If the RHS is a constant, see if we can simplify it.
1166 if (ShrinkDemandedConstant(Op, DemandedBits, TLO))
1167 return true;
1168 // If the operation can be done in a smaller type, do so.
1169 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1170 return true;
1171
1172 // Output known-0 bits are only known if clear in both the LHS & RHS.
1173 Known.Zero &= Known2.Zero;
1174 // Output known-1 are known to be set if set in either the LHS | RHS.
1175 Known.One |= Known2.One;
1176 break;
1177 }
1178 case ISD::XOR: {
1179 SDValue Op0 = Op.getOperand(0);
1180 SDValue Op1 = Op.getOperand(1);
1181
1182 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1183 Depth + 1))
1184 return true;
1185 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1185, __PRETTY_FUNCTION__))
;
1186 if (SimplifyDemandedBits(Op0, DemandedBits, DemandedElts, Known2, TLO,
1187 Depth + 1))
1188 return true;
1189 assert(!Known2.hasConflict() && "Bits known to be one AND zero?")((!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1189, __PRETTY_FUNCTION__))
;
1190
1191 // Attempt to avoid multi-use ops if we don't need anything from them.
1192 if (!DemandedBits.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {
1193 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1194 Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1195 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1196 Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1197 if (DemandedOp0 || DemandedOp1) {
1198 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1199 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1200 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1201 return TLO.CombineTo(Op, NewOp);
1202 }
1203 }
1204
1205 // If all of the demanded bits are known zero on one side, return the other.
1206 // These bits cannot contribute to the result of the 'xor'.
1207 if (DemandedBits.isSubsetOf(Known.Zero))
1208 return TLO.CombineTo(Op, Op0);
1209 if (DemandedBits.isSubsetOf(Known2.Zero))
1210 return TLO.CombineTo(Op, Op1);
1211 // If the operation can be done in a smaller type, do so.
1212 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1213 return true;
1214
1215 // If all of the unknown bits are known to be zero on one side or the other
1216 // (but not both) turn this into an *inclusive* or.
1217 // e.g. (A & C1)^(B & C2) -> (A & C1)|(B & C2) iff C1&C2 == 0
1218 if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
1219 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::OR, dl, VT, Op0, Op1));
1220
1221 // Output known-0 bits are known if clear or set in both the LHS & RHS.
1222 KnownOut.Zero = (Known.Zero & Known2.Zero) | (Known.One & Known2.One);
1223 // Output known-1 are known to be set if set in only one of the LHS, RHS.
1224 KnownOut.One = (Known.Zero & Known2.One) | (Known.One & Known2.Zero);
1225
1226 if (ConstantSDNode *C = isConstOrConstSplat(Op1)) {
1227 // If one side is a constant, and all of the known set bits on the other
1228 // side are also set in the constant, turn this into an AND, as we know
1229 // the bits will be cleared.
1230 // e.g. (X | C1) ^ C2 --> (X | C1) & ~C2 iff (C1&C2) == C2
1231 // NB: it is okay if more bits are known than are requested
1232 if (C->getAPIntValue() == Known2.One) {
1233 SDValue ANDC =
1234 TLO.DAG.getConstant(~C->getAPIntValue() & DemandedBits, dl, VT);
1235 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, dl, VT, Op0, ANDC));
1236 }
1237
1238 // If the RHS is a constant, see if we can change it. Don't alter a -1
1239 // constant because that's a 'not' op, and that is better for combining
1240 // and codegen.
1241 if (!C->isAllOnesValue()) {
1242 if (DemandedBits.isSubsetOf(C->getAPIntValue())) {
1243 // We're flipping all demanded bits. Flip the undemanded bits too.
1244 SDValue New = TLO.DAG.getNOT(dl, Op0, VT);
1245 return TLO.CombineTo(Op, New);
1246 }
1247 // If we can't turn this into a 'not', try to shrink the constant.
1248 if (ShrinkDemandedConstant(Op, DemandedBits, TLO))
1249 return true;
1250 }
1251 }
1252
1253 Known = std::move(KnownOut);
1254 break;
1255 }
1256 case ISD::SELECT:
1257 if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known, TLO,
1258 Depth + 1))
1259 return true;
1260 if (SimplifyDemandedBits(Op.getOperand(1), DemandedBits, Known2, TLO,
1261 Depth + 1))
1262 return true;
1263 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1263, __PRETTY_FUNCTION__))
;
1264 assert(!Known2.hasConflict() && "Bits known to be one AND zero?")((!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1264, __PRETTY_FUNCTION__))
;
1265
1266 // If the operands are constants, see if we can simplify them.
1267 if (ShrinkDemandedConstant(Op, DemandedBits, TLO))
1268 return true;
1269
1270 // Only known if known in both the LHS and RHS.
1271 Known.One &= Known2.One;
1272 Known.Zero &= Known2.Zero;
1273 break;
1274 case ISD::SELECT_CC:
1275 if (SimplifyDemandedBits(Op.getOperand(3), DemandedBits, Known, TLO,
1276 Depth + 1))
1277 return true;
1278 if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known2, TLO,
1279 Depth + 1))
1280 return true;
1281 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1281, __PRETTY_FUNCTION__))
;
1282 assert(!Known2.hasConflict() && "Bits known to be one AND zero?")((!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1282, __PRETTY_FUNCTION__))
;
1283
1284 // If the operands are constants, see if we can simplify them.
1285 if (ShrinkDemandedConstant(Op, DemandedBits, TLO))
1286 return true;
1287
1288 // Only known if known in both the LHS and RHS.
1289 Known.One &= Known2.One;
1290 Known.Zero &= Known2.Zero;
1291 break;
1292 case ISD::SETCC: {
1293 SDValue Op0 = Op.getOperand(0);
1294 SDValue Op1 = Op.getOperand(1);
1295 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
1296 // If (1) we only need the sign-bit, (2) the setcc operands are the same
1297 // width as the setcc result, and (3) the result of a setcc conforms to 0 or
1298 // -1, we may be able to bypass the setcc.
1299 if (DemandedBits.isSignMask() &&
1300 Op0.getScalarValueSizeInBits() == BitWidth &&
1301 getBooleanContents(Op0.getValueType()) ==
1302 BooleanContent::ZeroOrNegativeOneBooleanContent) {
1303 // If we're testing X < 0, then this compare isn't needed - just use X!
1304 // FIXME: We're limiting to integer types here, but this should also work
1305 // if we don't care about FP signed-zero. The use of SETLT with FP means
1306 // that we don't care about NaNs.
1307 if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
1308 (isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode())))
1309 return TLO.CombineTo(Op, Op0);
1310
1311 // TODO: Should we check for other forms of sign-bit comparisons?
1312 // Examples: X <= -1, X >= 0
1313 }
1314 if (getBooleanContents(Op0.getValueType()) ==
1315 TargetLowering::ZeroOrOneBooleanContent &&
1316 BitWidth > 1)
1317 Known.Zero.setBitsFrom(1);
1318 break;
1319 }
1320 case ISD::SHL: {
1321 SDValue Op0 = Op.getOperand(0);
1322 SDValue Op1 = Op.getOperand(1);
1323
1324 if (ConstantSDNode *SA = isConstOrConstSplat(Op1, DemandedElts)) {
1325 // If the shift count is an invalid immediate, don't do anything.
1326 if (SA->getAPIntValue().uge(BitWidth))
1327 break;
1328
1329 unsigned ShAmt = SA->getZExtValue();
1330 if (ShAmt == 0)
1331 return TLO.CombineTo(Op, Op0);
1332
1333 // If this is ((X >>u C1) << ShAmt), see if we can simplify this into a
1334 // single shift. We can do this if the bottom bits (which are shifted
1335 // out) are never demanded.
1336 // TODO - support non-uniform vector amounts.
1337 if (Op0.getOpcode() == ISD::SRL) {
1338 if (!DemandedBits.intersects(APInt::getLowBitsSet(BitWidth, ShAmt))) {
1339 if (ConstantSDNode *SA2 =
1340 isConstOrConstSplat(Op0.getOperand(1), DemandedElts)) {
1341 if (SA2->getAPIntValue().ult(BitWidth)) {
1342 unsigned C1 = SA2->getZExtValue();
1343 unsigned Opc = ISD::SHL;
1344 int Diff = ShAmt - C1;
1345 if (Diff < 0) {
1346 Diff = -Diff;
1347 Opc = ISD::SRL;
1348 }
1349
1350 SDValue NewSA = TLO.DAG.getConstant(Diff, dl, Op1.getValueType());
1351 return TLO.CombineTo(
1352 Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));
1353 }
1354 }
1355 }
1356 }
1357
1358 if (SimplifyDemandedBits(Op0, DemandedBits.lshr(ShAmt), DemandedElts,
1359 Known, TLO, Depth + 1))
1360 return true;
1361
1362 // Try shrinking the operation as long as the shift amount will still be
1363 // in range.
1364 if ((ShAmt < DemandedBits.getActiveBits()) &&
1365 ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1366 return true;
1367
1368 // Convert (shl (anyext x, c)) to (anyext (shl x, c)) if the high bits
1369 // are not demanded. This will likely allow the anyext to be folded away.
1370 if (Op0.getOpcode() == ISD::ANY_EXTEND) {
1371 SDValue InnerOp = Op0.getOperand(0);
1372 EVT InnerVT = InnerOp.getValueType();
1373 unsigned InnerBits = InnerVT.getScalarSizeInBits();
1374 if (ShAmt < InnerBits && DemandedBits.getActiveBits() <= InnerBits &&
1375 isTypeDesirableForOp(ISD::SHL, InnerVT)) {
1376 EVT ShTy = getShiftAmountTy(InnerVT, DL);
1377 if (!APInt(BitWidth, ShAmt).isIntN(ShTy.getSizeInBits()))
1378 ShTy = InnerVT;
1379 SDValue NarrowShl =
1380 TLO.DAG.getNode(ISD::SHL, dl, InnerVT, InnerOp,
1381 TLO.DAG.getConstant(ShAmt, dl, ShTy));
1382 return TLO.CombineTo(
1383 Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, NarrowShl));
1384 }
1385 // Repeat the SHL optimization above in cases where an extension
1386 // intervenes: (shl (anyext (shr x, c1)), c2) to
1387 // (shl (anyext x), c2-c1). This requires that the bottom c1 bits
1388 // aren't demanded (as above) and that the shifted upper c1 bits of
1389 // x aren't demanded.
1390 if (Op0.hasOneUse() && InnerOp.getOpcode() == ISD::SRL &&
1391 InnerOp.hasOneUse()) {
1392 if (ConstantSDNode *SA2 =
1393 isConstOrConstSplat(InnerOp.getOperand(1))) {
1394 unsigned InnerShAmt = SA2->getLimitedValue(InnerBits);
1395 if (InnerShAmt < ShAmt && InnerShAmt < InnerBits &&
1396 DemandedBits.getActiveBits() <=
1397 (InnerBits - InnerShAmt + ShAmt) &&
1398 DemandedBits.countTrailingZeros() >= ShAmt) {
1399 SDValue NewSA = TLO.DAG.getConstant(ShAmt - InnerShAmt, dl,
1400 Op1.getValueType());
1401 SDValue NewExt = TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT,
1402 InnerOp.getOperand(0));
1403 return TLO.CombineTo(
1404 Op, TLO.DAG.getNode(ISD::SHL, dl, VT, NewExt, NewSA));
1405 }
1406 }
1407 }
1408 }
1409
1410 Known.Zero <<= ShAmt;
1411 Known.One <<= ShAmt;
1412 // low bits known zero.
1413 Known.Zero.setLowBits(ShAmt);
1414 }
1415 break;
1416 }
1417 case ISD::SRL: {
1418 SDValue Op0 = Op.getOperand(0);
1419 SDValue Op1 = Op.getOperand(1);
1420
1421 if (ConstantSDNode *SA = isConstOrConstSplat(Op1, DemandedElts)) {
1422 // If the shift count is an invalid immediate, don't do anything.
1423 if (SA->getAPIntValue().uge(BitWidth))
1424 break;
1425
1426 unsigned ShAmt = SA->getZExtValue();
1427 if (ShAmt == 0)
1428 return TLO.CombineTo(Op, Op0);
1429
1430 EVT ShiftVT = Op1.getValueType();
1431 APInt InDemandedMask = (DemandedBits << ShAmt);
1432
1433 // If the shift is exact, then it does demand the low bits (and knows that
1434 // they are zero).
1435 if (Op->getFlags().hasExact())
1436 InDemandedMask.setLowBits(ShAmt);
1437
1438 // If this is ((X << C1) >>u ShAmt), see if we can simplify this into a
1439 // single shift. We can do this if the top bits (which are shifted out)
1440 // are never demanded.
1441 // TODO - support non-uniform vector amounts.
1442 if (Op0.getOpcode() == ISD::SHL) {
1443 if (ConstantSDNode *SA2 =
1444 isConstOrConstSplat(Op0.getOperand(1), DemandedElts)) {
1445 if (!DemandedBits.intersects(
1446 APInt::getHighBitsSet(BitWidth, ShAmt))) {
1447 if (SA2->getAPIntValue().ult(BitWidth)) {
1448 unsigned C1 = SA2->getZExtValue();
1449 unsigned Opc = ISD::SRL;
1450 int Diff = ShAmt - C1;
1451 if (Diff < 0) {
1452 Diff = -Diff;
1453 Opc = ISD::SHL;
1454 }
1455
1456 SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);
1457 return TLO.CombineTo(
1458 Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));
1459 }
1460 }
1461 }
1462 }
1463
1464 // Compute the new bits that are at the top now.
1465 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1466 Depth + 1))
1467 return true;
1468 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1468, __PRETTY_FUNCTION__))
;
1469 Known.Zero.lshrInPlace(ShAmt);
1470 Known.One.lshrInPlace(ShAmt);
1471
1472 Known.Zero.setHighBits(ShAmt); // High bits known zero.
1473 }
1474 break;
1475 }
1476 case ISD::SRA: {
1477 SDValue Op0 = Op.getOperand(0);
1478 SDValue Op1 = Op.getOperand(1);
1479
1480 // If this is an arithmetic shift right and only the low-bit is set, we can
1481 // always convert this into a logical shr, even if the shift amount is
1482 // variable. The low bit of the shift cannot be an input sign bit unless
1483 // the shift amount is >= the size of the datatype, which is undefined.
1484 if (DemandedBits.isOneValue())
1485 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1));
1486
1487 if (ConstantSDNode *SA = isConstOrConstSplat(Op1, DemandedElts)) {
1488 // If the shift count is an invalid immediate, don't do anything.
1489 if (SA->getAPIntValue().uge(BitWidth))
1490 break;
1491
1492 unsigned ShAmt = SA->getZExtValue();
1493 if (ShAmt == 0)
1494 return TLO.CombineTo(Op, Op0);
1495
1496 APInt InDemandedMask = (DemandedBits << ShAmt);
1497
1498 // If the shift is exact, then it does demand the low bits (and knows that
1499 // they are zero).
1500 if (Op->getFlags().hasExact())
1501 InDemandedMask.setLowBits(ShAmt);
1502
1503 // If any of the demanded bits are produced by the sign extension, we also
1504 // demand the input sign bit.
1505 if (DemandedBits.countLeadingZeros() < ShAmt)
1506 InDemandedMask.setSignBit();
1507
1508 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1509 Depth + 1))
1510 return true;
1511 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1511, __PRETTY_FUNCTION__))
;
1512 Known.Zero.lshrInPlace(ShAmt);
1513 Known.One.lshrInPlace(ShAmt);
1514
1515 // If the input sign bit is known to be zero, or if none of the top bits
1516 // are demanded, turn this into an unsigned shift right.
1517 if (Known.Zero[BitWidth - ShAmt - 1] ||
1518 DemandedBits.countLeadingZeros() >= ShAmt) {
1519 SDNodeFlags Flags;
1520 Flags.setExact(Op->getFlags().hasExact());
1521 return TLO.CombineTo(
1522 Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1, Flags));
1523 }
1524
1525 int Log2 = DemandedBits.exactLogBase2();
1526 if (Log2 >= 0) {
1527 // The bit must come from the sign.
1528 SDValue NewSA =
1529 TLO.DAG.getConstant(BitWidth - 1 - Log2, dl, Op1.getValueType());
1530 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, NewSA));
1531 }
1532
1533 if (Known.One[BitWidth - ShAmt - 1])
1534 // New bits are known one.
1535 Known.One.setHighBits(ShAmt);
1536 }
1537 break;
1538 }
1539 case ISD::FSHL:
1540 case ISD::FSHR: {
1541 SDValue Op0 = Op.getOperand(0);
1542 SDValue Op1 = Op.getOperand(1);
1543 SDValue Op2 = Op.getOperand(2);
1544 bool IsFSHL = (Op.getOpcode() == ISD::FSHL);
1545
1546 if (ConstantSDNode *SA = isConstOrConstSplat(Op2, DemandedElts)) {
1547 unsigned Amt = SA->getAPIntValue().urem(BitWidth);
1548
1549 // For fshl, 0-shift returns the 1st arg.
1550 // For fshr, 0-shift returns the 2nd arg.
1551 if (Amt == 0) {
1552 if (SimplifyDemandedBits(IsFSHL ? Op0 : Op1, DemandedBits, DemandedElts,
1553 Known, TLO, Depth + 1))
1554 return true;
1555 break;
1556 }
1557
1558 // fshl: (Op0 << Amt) | (Op1 >> (BW - Amt))
1559 // fshr: (Op0 << (BW - Amt)) | (Op1 >> Amt)
1560 APInt Demanded0 = DemandedBits.lshr(IsFSHL ? Amt : (BitWidth - Amt));
1561 APInt Demanded1 = DemandedBits << (IsFSHL ? (BitWidth - Amt) : Amt);
1562 if (SimplifyDemandedBits(Op0, Demanded0, DemandedElts, Known2, TLO,
1563 Depth + 1))
1564 return true;
1565 if (SimplifyDemandedBits(Op1, Demanded1, DemandedElts, Known, TLO,
1566 Depth + 1))
1567 return true;
1568
1569 Known2.One <<= (IsFSHL ? Amt : (BitWidth - Amt));
1570 Known2.Zero <<= (IsFSHL ? Amt : (BitWidth - Amt));
1571 Known.One.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);
1572 Known.Zero.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);
1573 Known.One |= Known2.One;
1574 Known.Zero |= Known2.Zero;
1575 }
1576 break;
1577 }
1578 case ISD::BITREVERSE: {
1579 SDValue Src = Op.getOperand(0);
1580 APInt DemandedSrcBits = DemandedBits.reverseBits();
1581 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
1582 Depth + 1))
1583 return true;
1584 Known.One = Known2.One.reverseBits();
1585 Known.Zero = Known2.Zero.reverseBits();
1586 break;
1587 }
1588 case ISD::BSWAP: {
1589 SDValue Src = Op.getOperand(0);
1590 APInt DemandedSrcBits = DemandedBits.byteSwap();
1591 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
1592 Depth + 1))
1593 return true;
1594 Known.One = Known2.One.byteSwap();
1595 Known.Zero = Known2.Zero.byteSwap();
1596 break;
1597 }
1598 case ISD::SIGN_EXTEND_INREG: {
1599 SDValue Op0 = Op.getOperand(0);
1600 EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
1601 unsigned ExVTBits = ExVT.getScalarSizeInBits();
1602
1603 // If we only care about the highest bit, don't bother shifting right.
1604 if (DemandedBits.isSignMask()) {
1605 unsigned NumSignBits = TLO.DAG.ComputeNumSignBits(Op0);
1606 bool AlreadySignExtended = NumSignBits >= BitWidth - ExVTBits + 1;
1607 // However if the input is already sign extended we expect the sign
1608 // extension to be dropped altogether later and do not simplify.
1609 if (!AlreadySignExtended) {
1610 // Compute the correct shift amount type, which must be getShiftAmountTy
1611 // for scalar types after legalization.
1612 EVT ShiftAmtTy = VT;
1613 if (TLO.LegalTypes() && !ShiftAmtTy.isVector())
1614 ShiftAmtTy = getShiftAmountTy(ShiftAmtTy, DL);
1615
1616 SDValue ShiftAmt =
1617 TLO.DAG.getConstant(BitWidth - ExVTBits, dl, ShiftAmtTy);
1618 return TLO.CombineTo(Op,
1619 TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, ShiftAmt));
1620 }
1621 }
1622
1623 // If none of the extended bits are demanded, eliminate the sextinreg.
1624 if (DemandedBits.getActiveBits() <= ExVTBits)
1625 return TLO.CombineTo(Op, Op0);
1626
1627 APInt InputDemandedBits = DemandedBits.getLoBits(ExVTBits);
1628
1629 // Since the sign extended bits are demanded, we know that the sign
1630 // bit is demanded.
1631 InputDemandedBits.setBit(ExVTBits - 1);
1632
1633 if (SimplifyDemandedBits(Op0, InputDemandedBits, Known, TLO, Depth + 1))
1634 return true;
1635 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1635, __PRETTY_FUNCTION__))
;
1636
1637 // If the sign bit of the input is known set or clear, then we know the
1638 // top bits of the result.
1639
1640 // If the input sign bit is known zero, convert this into a zero extension.
1641 if (Known.Zero[ExVTBits - 1])
1642 return TLO.CombineTo(
1643 Op, TLO.DAG.getZeroExtendInReg(Op0, dl, ExVT.getScalarType()));
1644
1645 APInt Mask = APInt::getLowBitsSet(BitWidth, ExVTBits);
1646 if (Known.One[ExVTBits - 1]) { // Input sign bit known set
1647 Known.One.setBitsFrom(ExVTBits);
1648 Known.Zero &= Mask;
1649 } else { // Input sign bit unknown
1650 Known.Zero &= Mask;
1651 Known.One &= Mask;
1652 }
1653 break;
1654 }
1655 case ISD::BUILD_PAIR: {
1656 EVT HalfVT = Op.getOperand(0).getValueType();
1657 unsigned HalfBitWidth = HalfVT.getScalarSizeInBits();
1658
1659 APInt MaskLo = DemandedBits.getLoBits(HalfBitWidth).trunc(HalfBitWidth);
1660 APInt MaskHi = DemandedBits.getHiBits(HalfBitWidth).trunc(HalfBitWidth);
1661
1662 KnownBits KnownLo, KnownHi;
1663
1664 if (SimplifyDemandedBits(Op.getOperand(0), MaskLo, KnownLo, TLO, Depth + 1))
1665 return true;
1666
1667 if (SimplifyDemandedBits(Op.getOperand(1), MaskHi, KnownHi, TLO, Depth + 1))
1668 return true;
1669
1670 Known.Zero = KnownLo.Zero.zext(BitWidth) |
1671 KnownHi.Zero.zext(BitWidth).shl(HalfBitWidth);
1672
1673 Known.One = KnownLo.One.zext(BitWidth) |
1674 KnownHi.One.zext(BitWidth).shl(HalfBitWidth);
1675 break;
1676 }
1677 case ISD::ZERO_EXTEND:
1678 case ISD::ZERO_EXTEND_VECTOR_INREG: {
1679 SDValue Src = Op.getOperand(0);
1680 EVT SrcVT = Src.getValueType();
1681 unsigned InBits = SrcVT.getScalarSizeInBits();
1682 unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
1683 bool IsVecInReg = Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG;
1684
1685 // If none of the top bits are demanded, convert this into an any_extend.
1686 if (DemandedBits.getActiveBits() <= InBits) {
1687 // If we only need the non-extended bits of the bottom element
1688 // then we can just bitcast to the result.
1689 if (IsVecInReg && DemandedElts == 1 &&
1690 VT.getSizeInBits() == SrcVT.getSizeInBits() &&
1691 TLO.DAG.getDataLayout().isLittleEndian())
1692 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
1693
1694 unsigned Opc =
1695 IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;
1696 if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
1697 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
1698 }
1699
1700 APInt InDemandedBits = DemandedBits.trunc(InBits);
1701 APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
1702 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
1703 Depth + 1))
1704 return true;
1705 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1705, __PRETTY_FUNCTION__))
;
1706 assert(Known.getBitWidth() == InBits && "Src width has changed?")((Known.getBitWidth() == InBits && "Src width has changed?"
) ? static_cast<void> (0) : __assert_fail ("Known.getBitWidth() == InBits && \"Src width has changed?\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1706, __PRETTY_FUNCTION__))
;
1707 Known = Known.zext(BitWidth, true /* ExtendedBitsAreKnownZero */);
1708 break;
1709 }
1710 case ISD::SIGN_EXTEND:
1711 case ISD::SIGN_EXTEND_VECTOR_INREG: {
1712 SDValue Src = Op.getOperand(0);
1713 EVT SrcVT = Src.getValueType();
1714 unsigned InBits = SrcVT.getScalarSizeInBits();
1715 unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
1716 bool IsVecInReg = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG;
1717
1718 // If none of the top bits are demanded, convert this into an any_extend.
1719 if (DemandedBits.getActiveBits() <= InBits) {
1720 // If we only need the non-extended bits of the bottom element
1721 // then we can just bitcast to the result.
1722 if (IsVecInReg && DemandedElts == 1 &&
1723 VT.getSizeInBits() == SrcVT.getSizeInBits() &&
1724 TLO.DAG.getDataLayout().isLittleEndian())
1725 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
1726
1727 unsigned Opc =
1728 IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;
1729 if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
1730 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
1731 }
1732
1733 APInt InDemandedBits = DemandedBits.trunc(InBits);
1734 APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
1735
1736 // Since some of the sign extended bits are demanded, we know that the sign
1737 // bit is demanded.
1738 InDemandedBits.setBit(InBits - 1);
1739
1740 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
1741 Depth + 1))
1742 return true;
1743 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1743, __PRETTY_FUNCTION__))
;
1744 assert(Known.getBitWidth() == InBits && "Src width has changed?")((Known.getBitWidth() == InBits && "Src width has changed?"
) ? static_cast<void> (0) : __assert_fail ("Known.getBitWidth() == InBits && \"Src width has changed?\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1744, __PRETTY_FUNCTION__))
;
1745
1746 // If the sign bit is known one, the top bits match.
1747 Known = Known.sext(BitWidth);
1748
1749 // If the sign bit is known zero, convert this to a zero extend.
1750 if (Known.isNonNegative()) {
1751 unsigned Opc =
1752 IsVecInReg ? ISD::ZERO_EXTEND_VECTOR_INREG : ISD::ZERO_EXTEND;
1753 if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
1754 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
1755 }
1756 break;
1757 }
1758 case ISD::ANY_EXTEND:
1759 case ISD::ANY_EXTEND_VECTOR_INREG: {
1760 SDValue Src = Op.getOperand(0);
1761 EVT SrcVT = Src.getValueType();
1762 unsigned InBits = SrcVT.getScalarSizeInBits();
1763 unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
1764 bool IsVecInReg = Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG;
1765
1766 // If we only need the bottom element then we can just bitcast.
1767 // TODO: Handle ANY_EXTEND?
1768 if (IsVecInReg && DemandedElts == 1 &&
1769 VT.getSizeInBits() == SrcVT.getSizeInBits() &&
1770 TLO.DAG.getDataLayout().isLittleEndian())
1771 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
1772
1773 APInt InDemandedBits = DemandedBits.trunc(InBits);
1774 APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
1775 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
1776 Depth + 1))
1777 return true;
1778 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1778, __PRETTY_FUNCTION__))
;
1779 assert(Known.getBitWidth() == InBits && "Src width has changed?")((Known.getBitWidth() == InBits && "Src width has changed?"
) ? static_cast<void> (0) : __assert_fail ("Known.getBitWidth() == InBits && \"Src width has changed?\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1779, __PRETTY_FUNCTION__))
;
1780 Known = Known.zext(BitWidth, false /* => any extend */);
1781 break;
1782 }
1783 case ISD::TRUNCATE: {
1784 SDValue Src = Op.getOperand(0);
1785
1786 // Simplify the input, using demanded bit information, and compute the known
1787 // zero/one bits live out.
1788 unsigned OperandBitWidth = Src.getScalarValueSizeInBits();
1789 APInt TruncMask = DemandedBits.zext(OperandBitWidth);
1790 if (SimplifyDemandedBits(Src, TruncMask, Known, TLO, Depth + 1))
1791 return true;
1792 Known = Known.trunc(BitWidth);
1793
1794 // Attempt to avoid multi-use ops if we don't need anything from them.
1795 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1796 Src, TruncMask, DemandedElts, TLO.DAG, Depth + 1))
1797 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, NewSrc));
1798
1799 // If the input is only used by this truncate, see if we can shrink it based
1800 // on the known demanded bits.
1801 if (Src.getNode()->hasOneUse()) {
1802 switch (Src.getOpcode()) {
1803 default:
1804 break;
1805 case ISD::SRL:
1806 // Shrink SRL by a constant if none of the high bits shifted in are
1807 // demanded.
1808 if (TLO.LegalTypes() && !isTypeDesirableForOp(ISD::SRL, VT))
1809 // Do not turn (vt1 truncate (vt2 srl)) into (vt1 srl) if vt1 is
1810 // undesirable.
1811 break;
1812
1813 SDValue ShAmt = Src.getOperand(1);
1814 auto *ShAmtC = dyn_cast<ConstantSDNode>(ShAmt);
1815 if (!ShAmtC || ShAmtC->getAPIntValue().uge(BitWidth))
1816 break;
1817 uint64_t ShVal = ShAmtC->getZExtValue();
1818
1819 APInt HighBits =
1820 APInt::getHighBitsSet(OperandBitWidth, OperandBitWidth - BitWidth);
1821 HighBits.lshrInPlace(ShVal);
1822 HighBits = HighBits.trunc(BitWidth);
1823
1824 if (!(HighBits & DemandedBits)) {
1825 // None of the shifted in bits are needed. Add a truncate of the
1826 // shift input, then shift it.
1827 if (TLO.LegalTypes())
1828 ShAmt = TLO.DAG.getConstant(ShVal, dl, getShiftAmountTy(VT, DL));
1829 SDValue NewTrunc =
1830 TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, Src.getOperand(0));
1831 return TLO.CombineTo(
1832 Op, TLO.DAG.getNode(ISD::SRL, dl, VT, NewTrunc, ShAmt));
1833 }
1834 break;
1835 }
1836 }
1837
1838 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1838, __PRETTY_FUNCTION__))
;
1839 break;
1840 }
1841 case ISD::AssertZext: {
1842 // AssertZext demands all of the high bits, plus any of the low bits
1843 // demanded by its users.
1844 EVT ZVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
1845 APInt InMask = APInt::getLowBitsSet(BitWidth, ZVT.getSizeInBits());
1846 if (SimplifyDemandedBits(Op.getOperand(0), ~InMask | DemandedBits, Known,
1847 TLO, Depth + 1))
1848 return true;
1849 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1849, __PRETTY_FUNCTION__))
;
1850
1851 Known.Zero |= ~InMask;
1852 break;
1853 }
1854 case ISD::EXTRACT_VECTOR_ELT: {
1855 SDValue Src = Op.getOperand(0);
1856 SDValue Idx = Op.getOperand(1);
1857 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
1858 unsigned EltBitWidth = Src.getScalarValueSizeInBits();
1859
1860 // Demand the bits from every vector element without a constant index.
1861 APInt DemandedSrcElts = APInt::getAllOnesValue(NumSrcElts);
1862 if (auto *CIdx = dyn_cast<ConstantSDNode>(Idx))
1863 if (CIdx->getAPIntValue().ult(NumSrcElts))
1864 DemandedSrcElts = APInt::getOneBitSet(NumSrcElts, CIdx->getZExtValue());
1865
1866 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
1867 // anything about the extended bits.
1868 APInt DemandedSrcBits = DemandedBits;
1869 if (BitWidth > EltBitWidth)
1870 DemandedSrcBits = DemandedSrcBits.trunc(EltBitWidth);
1871
1872 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts, Known2, TLO,
1873 Depth + 1))
1874 return true;
1875
1876 // Attempt to avoid multi-use ops if we don't need anything from them.
1877 if (!DemandedSrcBits.isAllOnesValue() ||
1878 !DemandedSrcElts.isAllOnesValue()) {
1879 if (SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
1880 Src, DemandedSrcBits, DemandedSrcElts, TLO.DAG, Depth + 1)) {
1881 SDValue NewOp =
1882 TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc, Idx);
1883 return TLO.CombineTo(Op, NewOp);
1884 }
1885 }
1886
1887 Known = Known2;
1888 if (BitWidth > EltBitWidth)
1889 Known = Known.zext(BitWidth, false /* => any extend */);
1890 break;
1891 }
1892 case ISD::BITCAST: {
1893 SDValue Src = Op.getOperand(0);
1894 EVT SrcVT = Src.getValueType();
1895 unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
1896
1897 // If this is an FP->Int bitcast and if the sign bit is the only
1898 // thing demanded, turn this into a FGETSIGN.
1899 if (!TLO.LegalOperations() && !VT.isVector() && !SrcVT.isVector() &&
1900 DemandedBits == APInt::getSignMask(Op.getValueSizeInBits()) &&
1901 SrcVT.isFloatingPoint()) {
1902 bool OpVTLegal = isOperationLegalOrCustom(ISD::FGETSIGN, VT);
1903 bool i32Legal = isOperationLegalOrCustom(ISD::FGETSIGN, MVT::i32);
1904 if ((OpVTLegal || i32Legal) && VT.isSimple() && SrcVT != MVT::f16 &&
1905 SrcVT != MVT::f128) {
1906 // Cannot eliminate/lower SHL for f128 yet.
1907 EVT Ty = OpVTLegal ? VT : MVT::i32;
1908 // Make a FGETSIGN + SHL to move the sign bit into the appropriate
1909 // place. We expect the SHL to be eliminated by other optimizations.
1910 SDValue Sign = TLO.DAG.getNode(ISD::FGETSIGN, dl, Ty, Src);
1911 unsigned OpVTSizeInBits = Op.getValueSizeInBits();
1912 if (!OpVTLegal && OpVTSizeInBits > 32)
1913 Sign = TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Sign);
1914 unsigned ShVal = Op.getValueSizeInBits() - 1;
1915 SDValue ShAmt = TLO.DAG.getConstant(ShVal, dl, VT);
1916 return TLO.CombineTo(Op,
1917 TLO.DAG.getNode(ISD::SHL, dl, VT, Sign, ShAmt));
1918 }
1919 }
1920
1921 // Bitcast from a vector using SimplifyDemanded Bits/VectorElts.
1922 // Demand the elt/bit if any of the original elts/bits are demanded.
1923 // TODO - bigendian once we have test coverage.
1924 if (SrcVT.isVector() && (BitWidth % NumSrcEltBits) == 0 &&
1925 TLO.DAG.getDataLayout().isLittleEndian()) {
1926 unsigned Scale = BitWidth / NumSrcEltBits;
1927 unsigned NumSrcElts = SrcVT.getVectorNumElements();
1928 APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);
1929 APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);
1930 for (unsigned i = 0; i != Scale; ++i) {
1931 unsigned Offset = i * NumSrcEltBits;
1932 APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset);
1933 if (!Sub.isNullValue()) {
1934 DemandedSrcBits |= Sub;
1935 for (unsigned j = 0; j != NumElts; ++j)
1936 if (DemandedElts[j])
1937 DemandedSrcElts.setBit((j * Scale) + i);
1938 }
1939 }
1940
1941 APInt KnownSrcUndef, KnownSrcZero;
1942 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
1943 KnownSrcZero, TLO, Depth + 1))
1944 return true;
1945
1946 KnownBits KnownSrcBits;
1947 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
1948 KnownSrcBits, TLO, Depth + 1))
1949 return true;
1950 } else if ((NumSrcEltBits % BitWidth) == 0 &&
1951 TLO.DAG.getDataLayout().isLittleEndian()) {
1952 unsigned Scale = NumSrcEltBits / BitWidth;
1953 unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
1954 APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);
1955 APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);
1956 for (unsigned i = 0; i != NumElts; ++i)
1957 if (DemandedElts[i]) {
1958 unsigned Offset = (i % Scale) * BitWidth;
1959 DemandedSrcBits.insertBits(DemandedBits, Offset);
1960 DemandedSrcElts.setBit(i / Scale);
1961 }
1962
1963 if (SrcVT.isVector()) {
1964 APInt KnownSrcUndef, KnownSrcZero;
1965 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
1966 KnownSrcZero, TLO, Depth + 1))
1967 return true;
1968 }
1969
1970 KnownBits KnownSrcBits;
1971 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
1972 KnownSrcBits, TLO, Depth + 1))
1973 return true;
1974 }
1975
1976 // If this is a bitcast, let computeKnownBits handle it. Only do this on a
1977 // recursive call where Known may be useful to the caller.
1978 if (Depth > 0) {
1979 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
1980 return false;
1981 }
1982 break;
1983 }
1984 case ISD::ADD:
1985 case ISD::MUL:
1986 case ISD::SUB: {
1987 // Add, Sub, and Mul don't demand any bits in positions beyond that
1988 // of the highest bit demanded of them.
1989 SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1);
1990 SDNodeFlags Flags = Op.getNode()->getFlags();
1991 unsigned DemandedBitsLZ = DemandedBits.countLeadingZeros();
1992 APInt LoMask = APInt::getLowBitsSet(BitWidth, BitWidth - DemandedBitsLZ);
1993 if (SimplifyDemandedBits(Op0, LoMask, DemandedElts, Known2, TLO,
1994 Depth + 1) ||
1995 SimplifyDemandedBits(Op1, LoMask, DemandedElts, Known2, TLO,
1996 Depth + 1) ||
1997 // See if the operation should be performed at a smaller bit width.
1998 ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO)) {
1999 if (Flags.hasNoSignedWrap() || Flags.hasNoUnsignedWrap()) {
2000 // Disable the nsw and nuw flags. We can no longer guarantee that we
2001 // won't wrap after simplification.
2002 Flags.setNoSignedWrap(false);
2003 Flags.setNoUnsignedWrap(false);
2004 SDValue NewOp =
2005 TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);
2006 return TLO.CombineTo(Op, NewOp);
2007 }
2008 return true;
2009 }
2010
2011 // Attempt to avoid multi-use ops if we don't need anything from them.
2012 if (!LoMask.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {
2013 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2014 Op0, LoMask, DemandedElts, TLO.DAG, Depth + 1);
2015 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
2016 Op1, LoMask, DemandedElts, TLO.DAG, Depth + 1);
2017 if (DemandedOp0 || DemandedOp1) {
2018 Flags.setNoSignedWrap(false);
2019 Flags.setNoUnsignedWrap(false);
2020 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
2021 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
2022 SDValue NewOp =
2023 TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);
2024 return TLO.CombineTo(Op, NewOp);
2025 }
2026 }
2027
2028 // If we have a constant operand, we may be able to turn it into -1 if we
2029 // do not demand the high bits. This can make the constant smaller to
2030 // encode, allow more general folding, or match specialized instruction
2031 // patterns (eg, 'blsr' on x86). Don't bother changing 1 to -1 because that
2032 // is probably not useful (and could be detrimental).
2033 ConstantSDNode *C = isConstOrConstSplat(Op1);
2034 APInt HighMask = APInt::getHighBitsSet(BitWidth, DemandedBitsLZ);
2035 if (C && !C->isAllOnesValue() && !C->isOne() &&
2036 (C->getAPIntValue() | HighMask).isAllOnesValue()) {
2037 SDValue Neg1 = TLO.DAG.getAllOnesConstant(dl, VT);
2038 // Disable the nsw and nuw flags. We can no longer guarantee that we
2039 // won't wrap after simplification.
2040 Flags.setNoSignedWrap(false);
2041 Flags.setNoUnsignedWrap(false);
2042 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Neg1, Flags);
2043 return TLO.CombineTo(Op, NewOp);
2044 }
2045
2046 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2047 }
2048 default:
2049 if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
2050 if (SimplifyDemandedBitsForTargetNode(Op, DemandedBits, DemandedElts,
2051 Known, TLO, Depth))
2052 return true;
2053 break;
2054 }
2055
2056 // Just use computeKnownBits to compute output bits.
2057 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2058 break;
2059 }
2060
2061 // If we know the value of all of the demanded bits, return this as a
2062 // constant.
2063 if (DemandedBits.isSubsetOf(Known.Zero | Known.One)) {
2064 // Avoid folding to a constant if any OpaqueConstant is involved.
2065 const SDNode *N = Op.getNode();
2066 for (SDNodeIterator I = SDNodeIterator::begin(N),
2067 E = SDNodeIterator::end(N);
2068 I != E; ++I) {
2069 SDNode *Op = *I;
2070 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op))
2071 if (C->isOpaque())
2072 return false;
2073 }
2074 // TODO: Handle float bits as well.
2075 if (VT.isInteger())
2076 return TLO.CombineTo(Op, TLO.DAG.getConstant(Known.One, dl, VT));
2077 }
2078
2079 return false;
2080}
2081
2082bool TargetLowering::SimplifyDemandedVectorElts(SDValue Op,
2083 const APInt &DemandedElts,
2084 APInt &KnownUndef,
2085 APInt &KnownZero,
2086 DAGCombinerInfo &DCI) const {
2087 SelectionDAG &DAG = DCI.DAG;
2088 TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
2089 !DCI.isBeforeLegalizeOps());
2090
2091 bool Simplified =
2092 SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero, TLO);
2093 if (Simplified) {
2094 DCI.AddToWorklist(Op.getNode());
2095 DCI.CommitTargetLoweringOpt(TLO);
2096 }
2097
2098 return Simplified;
2099}
2100
2101/// Given a vector binary operation and known undefined elements for each input
2102/// operand, compute whether each element of the output is undefined.
2103static APInt getKnownUndefForVectorBinop(SDValue BO, SelectionDAG &DAG,
2104 const APInt &UndefOp0,
2105 const APInt &UndefOp1) {
2106 EVT VT = BO.getValueType();
2107 assert(DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() &&((DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) &&
VT.isVector() && "Vector binop only") ? static_cast<
void> (0) : __assert_fail ("DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() && \"Vector binop only\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2108, __PRETTY_FUNCTION__))
2108 "Vector binop only")((DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) &&
VT.isVector() && "Vector binop only") ? static_cast<
void> (0) : __assert_fail ("DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() && \"Vector binop only\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2108, __PRETTY_FUNCTION__))
;
2109
2110 EVT EltVT = VT.getVectorElementType();
2111 unsigned NumElts = VT.getVectorNumElements();
2112 assert(UndefOp0.getBitWidth() == NumElts &&((UndefOp0.getBitWidth() == NumElts && UndefOp1.getBitWidth
() == NumElts && "Bad type for undef analysis") ? static_cast
<void> (0) : __assert_fail ("UndefOp0.getBitWidth() == NumElts && UndefOp1.getBitWidth() == NumElts && \"Bad type for undef analysis\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2113, __PRETTY_FUNCTION__))
2113 UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis")((UndefOp0.getBitWidth() == NumElts && UndefOp1.getBitWidth
() == NumElts && "Bad type for undef analysis") ? static_cast
<void> (0) : __assert_fail ("UndefOp0.getBitWidth() == NumElts && UndefOp1.getBitWidth() == NumElts && \"Bad type for undef analysis\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2113, __PRETTY_FUNCTION__))
;
2114
2115 auto getUndefOrConstantElt = [&](SDValue V, unsigned Index,
2116 const APInt &UndefVals) {
2117 if (UndefVals[Index])
2118 return DAG.getUNDEF(EltVT);
2119
2120 if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
2121 // Try hard to make sure that the getNode() call is not creating temporary
2122 // nodes. Ignore opaque integers because they do not constant fold.
2123 SDValue Elt = BV->getOperand(Index);
2124 auto *C = dyn_cast<ConstantSDNode>(Elt);
2125 if (isa<ConstantFPSDNode>(Elt) || Elt.isUndef() || (C && !C->isOpaque()))
2126 return Elt;
2127 }
2128
2129 return SDValue();
2130 };
2131
2132 APInt KnownUndef = APInt::getNullValue(NumElts);
2133 for (unsigned i = 0; i != NumElts; ++i) {
2134 // If both inputs for this element are either constant or undef and match
2135 // the element type, compute the constant/undef result for this element of
2136 // the vector.
2137 // TODO: Ideally we would use FoldConstantArithmetic() here, but that does
2138 // not handle FP constants. The code within getNode() should be refactored
2139 // to avoid the danger of creating a bogus temporary node here.
2140 SDValue C0 = getUndefOrConstantElt(BO.getOperand(0), i, UndefOp0);
2141 SDValue C1 = getUndefOrConstantElt(BO.getOperand(1), i, UndefOp1);
2142 if (C0 && C1 && C0.getValueType() == EltVT && C1.getValueType() == EltVT)
2143 if (DAG.getNode(BO.getOpcode(), SDLoc(BO), EltVT, C0, C1).isUndef())
2144 KnownUndef.setBit(i);
2145 }
2146 return KnownUndef;
2147}
2148
2149bool TargetLowering::SimplifyDemandedVectorElts(
2150 SDValue Op, const APInt &OriginalDemandedElts, APInt &KnownUndef,
2151 APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth,
2152 bool AssumeSingleUse) const {
2153 EVT VT = Op.getValueType();
2154 APInt DemandedElts = OriginalDemandedElts;
2155 unsigned NumElts = DemandedElts.getBitWidth();
2156 assert(VT.isVector() && "Expected vector op")((VT.isVector() && "Expected vector op") ? static_cast
<void> (0) : __assert_fail ("VT.isVector() && \"Expected vector op\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2156, __PRETTY_FUNCTION__))
;
2157 assert(VT.getVectorNumElements() == NumElts &&((VT.getVectorNumElements() == NumElts && "Mask size mismatches value type element count!"
) ? static_cast<void> (0) : __assert_fail ("VT.getVectorNumElements() == NumElts && \"Mask size mismatches value type element count!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2158, __PRETTY_FUNCTION__))
2158 "Mask size mismatches value type element count!")((VT.getVectorNumElements() == NumElts && "Mask size mismatches value type element count!"
) ? static_cast<void> (0) : __assert_fail ("VT.getVectorNumElements() == NumElts && \"Mask size mismatches value type element count!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2158, __PRETTY_FUNCTION__))
;
2159
2160 KnownUndef = KnownZero = APInt::getNullValue(NumElts);
2161
2162 // Undef operand.
2163 if (Op.isUndef()) {
2164 KnownUndef.setAllBits();
2165 return false;
2166 }
2167
2168 // If Op has other users, assume that all elements are needed.
2169 if (!Op.getNode()->hasOneUse() && !AssumeSingleUse)
2170 DemandedElts.setAllBits();
2171
2172 // Not demanding any elements from Op.
2173 if (DemandedElts == 0) {
2174 KnownUndef.setAllBits();
2175 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
2176 }
2177
2178 // Limit search depth.
2179 if (Depth >= SelectionDAG::MaxRecursionDepth)
2180 return false;
2181
2182 SDLoc DL(Op);
2183 unsigned EltSizeInBits = VT.getScalarSizeInBits();
2184
2185 switch (Op.getOpcode()) {
2186 case ISD::SCALAR_TO_VECTOR: {
2187 if (!DemandedElts[0]) {
2188 KnownUndef.setAllBits();
2189 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
2190 }
2191 KnownUndef.setHighBits(NumElts - 1);
2192 break;
2193 }
2194 case ISD::BITCAST: {
2195 SDValue Src = Op.getOperand(0);
2196 EVT SrcVT = Src.getValueType();
2197
2198 // We only handle vectors here.
2199 // TODO - investigate calling SimplifyDemandedBits/ComputeKnownBits?
2200 if (!SrcVT.isVector())
2201 break;
2202
2203 // Fast handling of 'identity' bitcasts.
2204 unsigned NumSrcElts = SrcVT.getVectorNumElements();
2205 if (NumSrcElts == NumElts)
2206 return SimplifyDemandedVectorElts(Src, DemandedElts, KnownUndef,
2207 KnownZero, TLO, Depth + 1);
2208
2209 APInt SrcZero, SrcUndef;
2210 APInt SrcDemandedElts = APInt::getNullValue(NumSrcElts);
2211
2212 // Bitcast from 'large element' src vector to 'small element' vector, we
2213 // must demand a source element if any DemandedElt maps to it.
2214 if ((NumElts % NumSrcElts) == 0) {
2215 unsigned Scale = NumElts / NumSrcElts;
2216 for (unsigned i = 0; i != NumElts; ++i)
2217 if (DemandedElts[i])
2218 SrcDemandedElts.setBit(i / Scale);
2219
2220 if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2221 TLO, Depth + 1))
2222 return true;
2223
2224 // Try calling SimplifyDemandedBits, converting demanded elts to the bits
2225 // of the large element.
2226 // TODO - bigendian once we have test coverage.
2227 if (TLO.DAG.getDataLayout().isLittleEndian()) {
2228 unsigned SrcEltSizeInBits = SrcVT.getScalarSizeInBits();
2229 APInt SrcDemandedBits = APInt::getNullValue(SrcEltSizeInBits);
2230 for (unsigned i = 0; i != NumElts; ++i)
2231 if (DemandedElts[i]) {
2232 unsigned Ofs = (i % Scale) * EltSizeInBits;
2233 SrcDemandedBits.setBits(Ofs, Ofs + EltSizeInBits);
2234 }
2235
2236 KnownBits Known;
2237 if (SimplifyDemandedBits(Src, SrcDemandedBits, Known, TLO, Depth + 1))
2238 return true;
2239 }
2240
2241 // If the src element is zero/undef then all the output elements will be -
2242 // only demanded elements are guaranteed to be correct.
2243 for (unsigned i = 0; i != NumSrcElts; ++i) {
2244 if (SrcDemandedElts[i]) {
2245 if (SrcZero[i])
2246 KnownZero.setBits(i * Scale, (i + 1) * Scale);
2247 if (SrcUndef[i])
2248 KnownUndef.setBits(i * Scale, (i + 1) * Scale);
2249 }
2250 }
2251 }
2252
2253 // Bitcast from 'small element' src vector to 'large element' vector, we
2254 // demand all smaller source elements covered by the larger demanded element
2255 // of this vector.
2256 if ((NumSrcElts % NumElts) == 0) {
2257 unsigned Scale = NumSrcElts / NumElts;
2258 for (unsigned i = 0; i != NumElts; ++i)
2259 if (DemandedElts[i])
2260 SrcDemandedElts.setBits(i * Scale, (i + 1) * Scale);
2261
2262 if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2263 TLO, Depth + 1))
2264 return true;
2265
2266 // If all the src elements covering an output element are zero/undef, then
2267 // the output element will be as well, assuming it was demanded.
2268 for (unsigned i = 0; i != NumElts; ++i) {
2269 if (DemandedElts[i]) {
2270 if (SrcZero.extractBits(Scale, i * Scale).isAllOnesValue())
2271 KnownZero.setBit(i);
2272 if (SrcUndef.extractBits(Scale, i * Scale).isAllOnesValue())
2273 KnownUndef.setBit(i);
2274 }
2275 }
2276 }
2277 break;
2278 }
2279 case ISD::BUILD_VECTOR: {
2280 // Check all elements and simplify any unused elements with UNDEF.
2281 if (!DemandedElts.isAllOnesValue()) {
2282 // Don't simplify BROADCASTS.
2283 if (llvm::any_of(Op->op_values(),
2284 [&](SDValue Elt) { return Op.getOperand(0) != Elt; })) {
2285 SmallVector<SDValue, 32> Ops(Op->op_begin(), Op->op_end());
2286 bool Updated = false;
2287 for (unsigned i = 0; i != NumElts; ++i) {
2288 if (!DemandedElts[i] && !Ops[i].isUndef()) {
2289 Ops[i] = TLO.DAG.getUNDEF(Ops[0].getValueType());
2290 KnownUndef.setBit(i);
2291 Updated = true;
2292 }
2293 }
2294 if (Updated)
2295 return TLO.CombineTo(Op, TLO.DAG.getBuildVector(VT, DL, Ops));
2296 }
2297 }
2298 for (unsigned i = 0; i != NumElts; ++i) {
2299 SDValue SrcOp = Op.getOperand(i);
2300 if (SrcOp.isUndef()) {
2301 KnownUndef.setBit(i);
2302 } else if (EltSizeInBits == SrcOp.getScalarValueSizeInBits() &&
2303 (isNullConstant(SrcOp) || isNullFPConstant(SrcOp))) {
2304 KnownZero.setBit(i);
2305 }
2306 }
2307 break;
2308 }
2309 case ISD::CONCAT_VECTORS: {
2310 EVT SubVT = Op.getOperand(0).getValueType();
2311 unsigned NumSubVecs = Op.getNumOperands();
2312 unsigned NumSubElts = SubVT.getVectorNumElements();
2313 for (unsigned i = 0; i != NumSubVecs; ++i) {
2314 SDValue SubOp = Op.getOperand(i);
2315 APInt SubElts = DemandedElts.extractBits(NumSubElts, i * NumSubElts);
2316 APInt SubUndef, SubZero;
2317 if (SimplifyDemandedVectorElts(SubOp, SubElts, SubUndef, SubZero, TLO,
2318 Depth + 1))
2319 return true;
2320 KnownUndef.insertBits(SubUndef, i * NumSubElts);
2321 KnownZero.insertBits(SubZero, i * NumSubElts);
2322 }
2323 break;
2324 }
2325 case ISD::INSERT_SUBVECTOR: {
2326 if (!isa<ConstantSDNode>(Op.getOperand(2)))
2327 break;
2328 SDValue Base = Op.getOperand(0);
2329 SDValue Sub = Op.getOperand(1);
2330 EVT SubVT = Sub.getValueType();
2331 unsigned NumSubElts = SubVT.getVectorNumElements();
2332 const APInt &Idx = Op.getConstantOperandAPInt(2);
2333 if (Idx.ugt(NumElts - NumSubElts))
2334 break;
2335 unsigned SubIdx = Idx.getZExtValue();
2336 APInt SubElts = DemandedElts.extractBits(NumSubElts, SubIdx);
2337 APInt SubUndef, SubZero;
2338 if (SimplifyDemandedVectorElts(Sub, SubElts, SubUndef, SubZero, TLO,
2339 Depth + 1))
2340 return true;
2341 APInt BaseElts = DemandedElts;
2342 BaseElts.insertBits(APInt::getNullValue(NumSubElts), SubIdx);
2343
2344 // If none of the base operand elements are demanded, replace it with undef.
2345 if (!BaseElts && !Base.isUndef())
2346 return TLO.CombineTo(Op,
2347 TLO.DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
2348 TLO.DAG.getUNDEF(VT),
2349 Op.getOperand(1),
2350 Op.getOperand(2)));
2351
2352 if (SimplifyDemandedVectorElts(Base, BaseElts, KnownUndef, KnownZero, TLO,
2353 Depth + 1))
2354 return true;
2355 KnownUndef.insertBits(SubUndef, SubIdx);
2356 KnownZero.insertBits(SubZero, SubIdx);
2357 break;
2358 }
2359 case ISD::EXTRACT_SUBVECTOR: {
2360 SDValue Src = Op.getOperand(0);
2361 ConstantSDNode *SubIdx = dyn_cast<ConstantSDNode>(Op.getOperand(1));
2362 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2363 if (SubIdx && SubIdx->getAPIntValue().ule(NumSrcElts - NumElts)) {
2364 // Offset the demanded elts by the subvector index.
2365 uint64_t Idx = SubIdx->getZExtValue();
2366 APInt SrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
2367 APInt SrcUndef, SrcZero;
2368 if (SimplifyDemandedVectorElts(Src, SrcElts, SrcUndef, SrcZero, TLO,
2369 Depth + 1))
2370 return true;
2371 KnownUndef = SrcUndef.extractBits(NumElts, Idx);
2372 KnownZero = SrcZero.extractBits(NumElts, Idx);
2373 }
2374 break;
2375 }
2376 case ISD::INSERT_VECTOR_ELT: {
2377 SDValue Vec = Op.getOperand(0);
2378 SDValue Scl = Op.getOperand(1);
2379 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
2380
2381 // For a legal, constant insertion index, if we don't need this insertion
2382 // then strip it, else remove it from the demanded elts.
2383 if (CIdx && CIdx->getAPIntValue().ult(NumElts)) {
2384 unsigned Idx = CIdx->getZExtValue();
2385 if (!DemandedElts[Idx])
2386 return TLO.CombineTo(Op, Vec);
2387
2388 APInt DemandedVecElts(DemandedElts);
2389 DemandedVecElts.clearBit(Idx);
2390 if (SimplifyDemandedVectorElts(Vec, DemandedVecElts, KnownUndef,
2391 KnownZero, TLO, Depth + 1))
2392 return true;
2393
2394 KnownUndef.clearBit(Idx);
2395 if (Scl.isUndef())
2396 KnownUndef.setBit(Idx);
2397
2398 KnownZero.clearBit(Idx);
2399 if (isNullConstant(Scl) || isNullFPConstant(Scl))
2400 KnownZero.setBit(Idx);
2401 break;
2402 }
2403
2404 APInt VecUndef, VecZero;
2405 if (SimplifyDemandedVectorElts(Vec, DemandedElts, VecUndef, VecZero, TLO,
2406 Depth + 1))
2407 return true;
2408 // Without knowing the insertion index we can't set KnownUndef/KnownZero.
2409 break;
2410 }
2411 case ISD::VSELECT: {
2412 // Try to transform the select condition based on the current demanded
2413 // elements.
2414 // TODO: If a condition element is undef, we can choose from one arm of the
2415 // select (and if one arm is undef, then we can propagate that to the
2416 // result).
2417 // TODO - add support for constant vselect masks (see IR version of this).
2418 APInt UnusedUndef, UnusedZero;
2419 if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, UnusedUndef,
2420 UnusedZero, TLO, Depth + 1))
2421 return true;
2422
2423 // See if we can simplify either vselect operand.
2424 APInt DemandedLHS(DemandedElts);
2425 APInt DemandedRHS(DemandedElts);
2426 APInt UndefLHS, ZeroLHS;
2427 APInt UndefRHS, ZeroRHS;
2428 if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedLHS, UndefLHS,
2429 ZeroLHS, TLO, Depth + 1))
2430 return true;
2431 if (SimplifyDemandedVectorElts(Op.getOperand(2), DemandedRHS, UndefRHS,
2432 ZeroRHS, TLO, Depth + 1))
2433 return true;
2434
2435 KnownUndef = UndefLHS & UndefRHS;
2436 KnownZero = ZeroLHS & ZeroRHS;
2437 break;
2438 }
2439 case ISD::VECTOR_SHUFFLE: {
2440 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
2441
2442 // Collect demanded elements from shuffle operands..
2443 APInt DemandedLHS(NumElts, 0);
2444 APInt DemandedRHS(NumElts, 0);
2445 for (unsigned i = 0; i != NumElts; ++i) {
2446 int M = ShuffleMask[i];
2447 if (M < 0 || !DemandedElts[i])
2448 continue;
2449 assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range")((0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range"
) ? static_cast<void> (0) : __assert_fail ("0 <= M && M < (int)(2 * NumElts) && \"Shuffle index out of range\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2449, __PRETTY_FUNCTION__))
;
2450 if (M < (int)NumElts)
2451 DemandedLHS.setBit(M);
2452 else
2453 DemandedRHS.setBit(M - NumElts);
2454 }
2455
2456 // See if we can simplify either shuffle operand.
2457 APInt UndefLHS, ZeroLHS;
2458 APInt UndefRHS, ZeroRHS;
2459 if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedLHS, UndefLHS,
2460 ZeroLHS, TLO, Depth + 1))
2461 return true;
2462 if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedRHS, UndefRHS,
2463 ZeroRHS, TLO, Depth + 1))
2464 return true;
2465
2466 // Simplify mask using undef elements from LHS/RHS.
2467 bool Updated = false;
2468 bool IdentityLHS = true, IdentityRHS = true;
2469 SmallVector<int, 32> NewMask(ShuffleMask.begin(), ShuffleMask.end());
2470 for (unsigned i = 0; i != NumElts; ++i) {
2471 int &M = NewMask[i];
2472 if (M < 0)
2473 continue;
2474 if (!DemandedElts[i] || (M < (int)NumElts && UndefLHS[M]) ||
2475 (M >= (int)NumElts && UndefRHS[M - NumElts])) {
2476 Updated = true;
2477 M = -1;
2478 }
2479 IdentityLHS &= (M < 0) || (M == (int)i);
2480 IdentityRHS &= (M < 0) || ((M - NumElts) == i);
2481 }
2482
2483 // Update legal shuffle masks based on demanded elements if it won't reduce
2484 // to Identity which can cause premature removal of the shuffle mask.
2485 if (Updated && !IdentityLHS && !IdentityRHS && !TLO.LegalOps) {
2486 SDValue LegalShuffle =
2487 buildLegalVectorShuffle(VT, DL, Op.getOperand(0), Op.getOperand(1),
2488 NewMask, TLO.DAG);
2489 if (LegalShuffle)
2490 return TLO.CombineTo(Op, LegalShuffle);
2491 }
2492
2493 // Propagate undef/zero elements from LHS/RHS.
2494 for (unsigned i = 0; i != NumElts; ++i) {
2495 int M = ShuffleMask[i];
2496 if (M < 0) {
2497 KnownUndef.setBit(i);
2498 } else if (M < (int)NumElts) {
2499 if (UndefLHS[M])
2500 KnownUndef.setBit(i);
2501 if (ZeroLHS[M])
2502 KnownZero.setBit(i);
2503 } else {
2504 if (UndefRHS[M - NumElts])
2505 KnownUndef.setBit(i);
2506 if (ZeroRHS[M - NumElts])
2507 KnownZero.setBit(i);
2508 }
2509 }
2510 break;
2511 }
2512 case ISD::ANY_EXTEND_VECTOR_INREG:
2513 case ISD::SIGN_EXTEND_VECTOR_INREG:
2514 case ISD::ZERO_EXTEND_VECTOR_INREG: {
2515 APInt SrcUndef, SrcZero;
2516 SDValue Src = Op.getOperand(0);
2517 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2518 APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts);
2519 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
2520 Depth + 1))
2521 return true;
2522 KnownZero = SrcZero.zextOrTrunc(NumElts);
2523 KnownUndef = SrcUndef.zextOrTrunc(NumElts);
2524
2525 if (Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG &&
2526 Op.getValueSizeInBits() == Src.getValueSizeInBits() &&
2527 DemandedSrcElts == 1 && TLO.DAG.getDataLayout().isLittleEndian()) {
2528 // aext - if we just need the bottom element then we can bitcast.
2529 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
2530 }
2531
2532 if (Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) {
2533 // zext(undef) upper bits are guaranteed to be zero.
2534 if (DemandedElts.isSubsetOf(KnownUndef))
2535 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
2536 KnownUndef.clearAllBits();
2537 }
2538 break;
2539 }
2540
2541 // TODO: There are more binop opcodes that could be handled here - MUL, MIN,
2542 // MAX, saturated math, etc.
2543 case ISD::OR:
2544 case ISD::XOR:
2545 case ISD::ADD:
2546 case ISD::SUB:
2547 case ISD::FADD:
2548 case ISD::FSUB:
2549 case ISD::FMUL:
2550 case ISD::FDIV:
2551 case ISD::FREM: {
2552 APInt UndefRHS, ZeroRHS;
2553 if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedElts, UndefRHS,
2554 ZeroRHS, TLO, Depth + 1))
2555 return true;
2556 APInt UndefLHS, ZeroLHS;
2557 if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, UndefLHS,
2558 ZeroLHS, TLO, Depth + 1))
2559 return true;
2560
2561 KnownZero = ZeroLHS & ZeroRHS;
2562 KnownUndef = getKnownUndefForVectorBinop(Op, TLO.DAG, UndefLHS, UndefRHS);
2563 break;
2564 }
2565 case ISD::SHL:
2566 case ISD::SRL:
2567 case ISD::SRA:
2568 case ISD::ROTL:
2569 case ISD::ROTR: {
2570 APInt UndefRHS, ZeroRHS;
2571 if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedElts, UndefRHS,
2572 ZeroRHS, TLO, Depth + 1))
2573 return true;
2574 APInt UndefLHS, ZeroLHS;
2575 if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, UndefLHS,
2576 ZeroLHS, TLO, Depth + 1))
2577 return true;
2578
2579 KnownZero = ZeroLHS;
2580 KnownUndef = UndefLHS & UndefRHS; // TODO: use getKnownUndefForVectorBinop?
2581 break;
2582 }
2583 case ISD::MUL:
2584 case ISD::AND: {
2585 APInt SrcUndef, SrcZero;
2586 if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedElts, SrcUndef,
2587 SrcZero, TLO, Depth + 1))
2588 return true;
2589 if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,
2590 KnownZero, TLO, Depth + 1))
2591 return true;
2592
2593 // If either side has a zero element, then the result element is zero, even
2594 // if the other is an UNDEF.
2595 // TODO: Extend getKnownUndefForVectorBinop to also deal with known zeros
2596 // and then handle 'and' nodes with the rest of the binop opcodes.
2597 KnownZero |= SrcZero;
2598 KnownUndef &= SrcUndef;
2599 KnownUndef &= ~KnownZero;
2600 break;
2601 }
2602 case ISD::TRUNCATE:
2603 case ISD::SIGN_EXTEND:
2604 case ISD::ZERO_EXTEND:
2605 if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,
2606 KnownZero, TLO, Depth + 1))
2607 return true;
2608
2609 if (Op.getOpcode() == ISD::ZERO_EXTEND) {
2610 // zext(undef) upper bits are guaranteed to be zero.
2611 if (DemandedElts.isSubsetOf(KnownUndef))
2612 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
2613 KnownUndef.clearAllBits();
2614 }
2615 break;
2616 default: {
2617 if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
2618 if (SimplifyDemandedVectorEltsForTargetNode(Op, DemandedElts, KnownUndef,
2619 KnownZero, TLO, Depth))
2620 return true;
2621 } else {
2622 KnownBits Known;
2623 APInt DemandedBits = APInt::getAllOnesValue(EltSizeInBits);
2624 if (SimplifyDemandedBits(Op, DemandedBits, OriginalDemandedElts, Known,
2625 TLO, Depth, AssumeSingleUse))
2626 return true;
2627 }
2628 break;
2629 }
2630 }
2631 assert((KnownUndef & KnownZero) == 0 && "Elements flagged as undef AND zero")(((KnownUndef & KnownZero) == 0 && "Elements flagged as undef AND zero"
) ? static_cast<void> (0) : __assert_fail ("(KnownUndef & KnownZero) == 0 && \"Elements flagged as undef AND zero\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2631, __PRETTY_FUNCTION__))
;
2632
2633 // Constant fold all undef cases.
2634 // TODO: Handle zero cases as well.
2635 if (DemandedElts.isSubsetOf(KnownUndef))
2636 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
2637
2638 return false;
2639}
2640
2641/// Determine which of the bits specified in Mask are known to be either zero or
2642/// one and return them in the Known.
2643void TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
2644 KnownBits &Known,
2645 const APInt &DemandedElts,
2646 const SelectionDAG &DAG,
2647 unsigned Depth) const {
2648 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2653, __PRETTY_FUNCTION__))
2649 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2653, __PRETTY_FUNCTION__))
2650 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2653, __PRETTY_FUNCTION__))
2651 Op.getOpcode() == ISD::INTRINSIC_VOID) &&(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2653, __PRETTY_FUNCTION__))
2652 "Should use MaskedValueIsZero if you don't know whether Op"(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2653, __PRETTY_FUNCTION__))
2653 " is a target node!")(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2653, __PRETTY_FUNCTION__))
;
2654 Known.resetAll();
2655}
2656
2657void TargetLowering::computeKnownBitsForTargetInstr(
2658 GISelKnownBits &Analysis, Register R, KnownBits &Known,
2659 const APInt &DemandedElts, const MachineRegisterInfo &MRI,
2660 unsigned Depth) const {
2661 Known.resetAll();
2662}
2663
2664void TargetLowering::computeKnownBitsForFrameIndex(const SDValue Op,
2665 KnownBits &Known,
2666 const APInt &DemandedElts,
2667 const SelectionDAG &DAG,
2668 unsigned Depth) const {
2669 assert(isa<FrameIndexSDNode>(Op) && "expected FrameIndex")((isa<FrameIndexSDNode>(Op) && "expected FrameIndex"
) ? static_cast<void> (0) : __assert_fail ("isa<FrameIndexSDNode>(Op) && \"expected FrameIndex\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2669, __PRETTY_FUNCTION__))
;
2670
2671 if (unsigned Align = DAG.InferPtrAlignment(Op)) {
2672 // The low bits are known zero if the pointer is aligned.
2673 Known.Zero.setLowBits(Log2_32(Align));
2674 }
2675}
2676
2677/// This method can be implemented by targets that want to expose additional
2678/// information about sign bits to the DAG Combiner.
2679unsigned TargetLowering::ComputeNumSignBitsForTargetNode(SDValue Op,
2680 const APInt &,
2681 const SelectionDAG &,
2682 unsigned Depth) const {
2683 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2688, __PRETTY_FUNCTION__))
2684 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2688, __PRETTY_FUNCTION__))
2685 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2688, __PRETTY_FUNCTION__))
2686 Op.getOpcode() == ISD::INTRINSIC_VOID) &&(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2688, __PRETTY_FUNCTION__))
2687 "Should use ComputeNumSignBits if you don't know whether Op"(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2688, __PRETTY_FUNCTION__))
2688 " is a target node!")(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2688, __PRETTY_FUNCTION__))
;
2689 return 1;
2690}
2691
2692bool TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
2693 SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,
2694 TargetLoweringOpt &TLO, unsigned Depth) const {
2695 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2700, __PRETTY_FUNCTION__))
2696 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2700, __PRETTY_FUNCTION__))
2697 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2700, __PRETTY_FUNCTION__))
2698 Op.getOpcode() == ISD::INTRINSIC_VOID) &&(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2700, __PRETTY_FUNCTION__))
2699 "Should use SimplifyDemandedVectorElts if you don't know whether Op"(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2700, __PRETTY_FUNCTION__))
2700 " is a target node!")(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2700, __PRETTY_FUNCTION__))
;
2701 return false;
2702}
2703
2704bool TargetLowering::SimplifyDemandedBitsForTargetNode(
2705 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
2706 KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const {
2707 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2712, __PRETTY_FUNCTION__))
2708 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2712, __PRETTY_FUNCTION__))
2709 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2712, __PRETTY_FUNCTION__))
2710 Op.getOpcode() == ISD::INTRINSIC_VOID) &&(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2712, __PRETTY_FUNCTION__))
2711 "Should use SimplifyDemandedBits if you don't know whether Op"(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2712, __PRETTY_FUNCTION__))
2712 " is a target node!")(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2712, __PRETTY_FUNCTION__))
;
2713 computeKnownBitsForTargetNode(Op, Known, DemandedElts, TLO.DAG, Depth);
2714 return false;
2715}
2716
2717SDValue TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(
2718 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
2719 SelectionDAG &DAG, unsigned Depth) const {
2720 assert((((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2726, __PRETTY_FUNCTION__))
2721 (Op.getOpcode() >= ISD::BUILTIN_OP_END ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2726, __PRETTY_FUNCTION__))
2722 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2726, __PRETTY_FUNCTION__))
2723 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2726, __PRETTY_FUNCTION__))
2724 Op.getOpcode() == ISD::INTRINSIC_VOID) &&(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2726, __PRETTY_FUNCTION__))
2725 "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2726, __PRETTY_FUNCTION__))
2726 " is a target node!")(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2726, __PRETTY_FUNCTION__))
;
2727 return SDValue();
2728}
2729
2730SDValue
2731TargetLowering::buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0,
2732 SDValue N1, MutableArrayRef<int> Mask,
2733 SelectionDAG &DAG) const {
2734 bool LegalMask = isShuffleMaskLegal(Mask, VT);
2735 if (!LegalMask) {
2736 std::swap(N0, N1);
2737 ShuffleVectorSDNode::commuteMask(Mask);
2738 LegalMask = isShuffleMaskLegal(Mask, VT);
2739 }
2740
2741 if (!LegalMask)
2742 return SDValue();
2743
2744 return DAG.getVectorShuffle(VT, DL, N0, N1, Mask);
2745}
2746
2747const Constant *TargetLowering::getTargetConstantFromLoad(LoadSDNode*) const {
2748 return nullptr;
2749}
2750
2751bool TargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
2752 const SelectionDAG &DAG,
2753 bool SNaN,
2754 unsigned Depth) const {
2755 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2760, __PRETTY_FUNCTION__))
2756 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2760, __PRETTY_FUNCTION__))
2757 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2760, __PRETTY_FUNCTION__))
2758 Op.getOpcode() == ISD::INTRINSIC_VOID) &&(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2760, __PRETTY_FUNCTION__))
2759 "Should use isKnownNeverNaN if you don't know whether Op"(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2760, __PRETTY_FUNCTION__))
2760 " is a target node!")(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
== ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
|| Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2760, __PRETTY_FUNCTION__))
;
2761 return false;
2762}
2763
2764// FIXME: Ideally, this would use ISD::isConstantSplatVector(), but that must
2765// work with truncating build vectors and vectors with elements of less than
2766// 8 bits.
2767bool TargetLowering::isConstTrueVal(const SDNode *N) const {
2768 if (!N)
2769 return false;
2770
2771 APInt CVal;
2772 if (auto *CN = dyn_cast<ConstantSDNode>(N)) {
2773 CVal = CN->getAPIntValue();
2774 } else if (auto *BV = dyn_cast<BuildVectorSDNode>(N)) {
2775 auto *CN = BV->getConstantSplatNode();
2776 if (!CN)
2777 return false;
2778
2779 // If this is a truncating build vector, truncate the splat value.
2780 // Otherwise, we may fail to match the expected values below.
2781 unsigned BVEltWidth = BV->getValueType(0).getScalarSizeInBits();
2782 CVal = CN->getAPIntValue();
2783 if (BVEltWidth < CVal.getBitWidth())
2784 CVal = CVal.trunc(BVEltWidth);
2785 } else {
2786 return false;
2787 }
2788
2789 switch (getBooleanContents(N->getValueType(0))) {
2790 case UndefinedBooleanContent:
2791 return CVal[0];
2792 case ZeroOrOneBooleanContent:
2793 return CVal.isOneValue();
2794 case ZeroOrNegativeOneBooleanContent:
2795 return CVal.isAllOnesValue();
2796 }
2797
2798 llvm_unreachable("Invalid boolean contents")::llvm::llvm_unreachable_internal("Invalid boolean contents",
"/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2798)
;
2799}
2800
2801bool TargetLowering::isConstFalseVal(const SDNode *N) const {
2802 if (!N)
2803 return false;
2804
2805 const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N);
2806 if (!CN) {
2807 const BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N);
2808 if (!BV)
2809 return false;
2810
2811 // Only interested in constant splats, we don't care about undef
2812 // elements in identifying boolean constants and getConstantSplatNode
2813 // returns NULL if all ops are undef;
2814 CN = BV->getConstantSplatNode();
2815 if (!CN)
2816 return false;
2817 }
2818
2819 if (getBooleanContents(N->getValueType(0)) == UndefinedBooleanContent)
2820 return !CN->getAPIntValue()[0];
2821
2822 return CN->isNullValue();
2823}
2824
2825bool TargetLowering::isExtendedTrueVal(const ConstantSDNode *N, EVT VT,
2826 bool SExt) const {
2827 if (VT == MVT::i1)
2828 return N->isOne();
2829
2830 TargetLowering::BooleanContent Cnt = getBooleanContents(VT);
2831 switch (Cnt) {
2832 case TargetLowering::ZeroOrOneBooleanContent:
2833 // An extended value of 1 is always true, unless its original type is i1,
2834 // in which case it will be sign extended to -1.
2835 return (N->isOne() && !SExt) || (SExt && (N->getValueType(0) != MVT::i1));
2836 case TargetLowering::UndefinedBooleanContent:
2837 case TargetLowering::ZeroOrNegativeOneBooleanContent:
2838 return N->isAllOnesValue() && SExt;
2839 }
2840 llvm_unreachable("Unexpected enumeration.")::llvm::llvm_unreachable_internal("Unexpected enumeration.", "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2840)
;
2841}
2842
2843/// This helper function of SimplifySetCC tries to optimize the comparison when
2844/// either operand of the SetCC node is a bitwise-and instruction.
2845SDValue TargetLowering::foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1,
2846 ISD::CondCode Cond, const SDLoc &DL,
2847 DAGCombinerInfo &DCI) const {
2848 // Match these patterns in any of their permutations:
2849 // (X & Y) == Y
2850 // (X & Y) != Y
2851 if (N1.getOpcode() == ISD::AND && N0.getOpcode() != ISD::AND)
2852 std::swap(N0, N1);
2853
2854 EVT OpVT = N0.getValueType();
2855 if (N0.getOpcode() != ISD::AND || !OpVT.isInteger() ||
2856 (Cond != ISD::SETEQ && Cond != ISD::SETNE))
2857 return SDValue();
2858
2859 SDValue X, Y;
2860 if (N0.getOperand(0) == N1) {
2861 X = N0.getOperand(1);
2862 Y = N0.getOperand(0);
2863 } else if (N0.getOperand(1) == N1) {
2864 X = N0.getOperand(0);
2865 Y = N0.getOperand(1);
2866 } else {
2867 return SDValue();
2868 }
2869
2870 SelectionDAG &DAG = DCI.DAG;
2871 SDValue Zero = DAG.getConstant(0, DL, OpVT);
2872 if (DAG.isKnownToBeAPowerOfTwo(Y)) {
2873 // Simplify X & Y == Y to X & Y != 0 if Y has exactly one bit set.
2874 // Note that where Y is variable and is known to have at most one bit set
2875 // (for example, if it is Z & 1) we cannot do this; the expressions are not
2876 // equivalent when Y == 0.
2877 assert(OpVT.isInteger())((OpVT.isInteger()) ? static_cast<void> (0) : __assert_fail
("OpVT.isInteger()", "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2877, __PRETTY_FUNCTION__))
;
2878 Cond = ISD::getSetCCInverse(Cond, OpVT);
2879 if (DCI.isBeforeLegalizeOps() ||
2880 isCondCodeLegal(Cond, N0.getSimpleValueType()))
2881 return DAG.getSetCC(DL, VT, N0, Zero, Cond);
2882 } else if (N0.hasOneUse() && hasAndNotCompare(Y)) {
2883 // If the target supports an 'and-not' or 'and-complement' logic operation,
2884 // try to use that to make a comparison operation more efficient.
2885 // But don't do this transform if the mask is a single bit because there are
2886 // more efficient ways to deal with that case (for example, 'bt' on x86 or
2887 // 'rlwinm' on PPC).
2888
2889 // Bail out if the compare operand that we want to turn into a zero is
2890 // already a zero (otherwise, infinite loop).
2891 auto *YConst = dyn_cast<ConstantSDNode>(Y);
2892 if (YConst && YConst->isNullValue())
2893 return SDValue();
2894
2895 // Transform this into: ~X & Y == 0.
2896 SDValue NotX = DAG.getNOT(SDLoc(X), X, OpVT);
2897 SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, NotX, Y);
2898 return DAG.getSetCC(DL, VT, NewAnd, Zero, Cond);
2899 }
2900
2901 return SDValue();
2902}
2903
2904/// There are multiple IR patterns that could be checking whether certain
2905/// truncation of a signed number would be lossy or not. The pattern which is
2906/// best at IR level, may not lower optimally. Thus, we want to unfold it.
2907/// We are looking for the following pattern: (KeptBits is a constant)
2908/// (add %x, (1 << (KeptBits-1))) srccond (1 << KeptBits)
2909/// KeptBits won't be bitwidth(x), that will be constant-folded to true/false.
2910/// KeptBits also can't be 1, that would have been folded to %x dstcond 0
2911/// We will unfold it into the natural trunc+sext pattern:
2912/// ((%x << C) a>> C) dstcond %x
2913/// Where C = bitwidth(x) - KeptBits and C u< bitwidth(x)
2914SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(
2915 EVT SCCVT, SDValue N0, SDValue N1, ISD::CondCode Cond, DAGCombinerInfo &DCI,
2916 const SDLoc &DL) const {
2917 // We must be comparing with a constant.
2918 ConstantSDNode *C1;
2919 if (!(C1 = dyn_cast<ConstantSDNode>(N1)))
2920 return SDValue();
2921
2922 // N0 should be: add %x, (1 << (KeptBits-1))
2923 if (N0->getOpcode() != ISD::ADD)
2924 return SDValue();
2925
2926 // And we must be 'add'ing a constant.
2927 ConstantSDNode *C01;
2928 if (!(C01 = dyn_cast<ConstantSDNode>(N0->getOperand(1))))
2929 return SDValue();
2930
2931 SDValue X = N0->getOperand(0);
2932 EVT XVT = X.getValueType();
2933
2934 // Validate constants ...
2935
2936 APInt I1 = C1->getAPIntValue();
2937
2938 ISD::CondCode NewCond;
2939 if (Cond == ISD::CondCode::SETULT) {
2940 NewCond = ISD::CondCode::SETEQ;
2941 } else if (Cond == ISD::CondCode::SETULE) {
2942 NewCond = ISD::CondCode::SETEQ;
2943 // But need to 'canonicalize' the constant.
2944 I1 += 1;
2945 } else if (Cond == ISD::CondCode::SETUGT) {
2946 NewCond = ISD::CondCode::SETNE;
2947 // But need to 'canonicalize' the constant.
2948 I1 += 1;
2949 } else if (Cond == ISD::CondCode::SETUGE) {
2950 NewCond = ISD::CondCode::SETNE;
2951 } else
2952 return SDValue();
2953
2954 APInt I01 = C01->getAPIntValue();
2955
2956 auto checkConstants = [&I1, &I01]() -> bool {
2957 // Both of them must be power-of-two, and the constant from setcc is bigger.
2958 return I1.ugt(I01) && I1.isPowerOf2() && I01.isPowerOf2();
2959 };
2960
2961 if (checkConstants()) {
2962 // Great, e.g. got icmp ult i16 (add i16 %x, 128), 256
2963 } else {
2964 // What if we invert constants? (and the target predicate)
2965 I1.negate();
2966 I01.negate();
2967 assert(XVT.isInteger())((XVT.isInteger()) ? static_cast<void> (0) : __assert_fail
("XVT.isInteger()", "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2967, __PRETTY_FUNCTION__))
;
2968 NewCond = getSetCCInverse(NewCond, XVT);
2969 if (!checkConstants())
2970 return SDValue();
2971 // Great, e.g. got icmp uge i16 (add i16 %x, -128), -256
2972 }
2973
2974 // They are power-of-two, so which bit is set?
2975 const unsigned KeptBits = I1.logBase2();
2976 const unsigned KeptBitsMinusOne = I01.logBase2();
2977
2978 // Magic!
2979 if (KeptBits != (KeptBitsMinusOne + 1))
2980 return SDValue();
2981 assert(KeptBits > 0 && KeptBits < XVT.getSizeInBits() && "unreachable")((KeptBits > 0 && KeptBits < XVT.getSizeInBits(
) && "unreachable") ? static_cast<void> (0) : __assert_fail
("KeptBits > 0 && KeptBits < XVT.getSizeInBits() && \"unreachable\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2981, __PRETTY_FUNCTION__))
;
2982
2983 // We don't want to do this in every single case.
2984 SelectionDAG &DAG = DCI.DAG;
2985 if (!DAG.getTargetLoweringInfo().shouldTransformSignedTruncationCheck(
2986 XVT, KeptBits))
2987 return SDValue();
2988
2989 const unsigned MaskedBits = XVT.getSizeInBits() - KeptBits;
2990 assert(MaskedBits > 0 && MaskedBits < XVT.getSizeInBits() && "unreachable")((MaskedBits > 0 && MaskedBits < XVT.getSizeInBits
() && "unreachable") ? static_cast<void> (0) : __assert_fail
("MaskedBits > 0 && MaskedBits < XVT.getSizeInBits() && \"unreachable\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2990, __PRETTY_FUNCTION__))
;
2991
2992 // Unfold into: ((%x << C) a>> C) cond %x
2993 // Where 'cond' will be either 'eq' or 'ne'.
2994 SDValue ShiftAmt = DAG.getConstant(MaskedBits, DL, XVT);
2995 SDValue T0 = DAG.getNode(ISD::SHL, DL, XVT, X, ShiftAmt);
2996 SDValue T1 = DAG.getNode(ISD::SRA, DL, XVT, T0, ShiftAmt);
2997 SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, X, NewCond);
2998
2999 return T2;
3000}
3001
3002// (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
3003SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(
3004 EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond,
3005 DAGCombinerInfo &DCI, const SDLoc &DL) const {
3006 assert(isConstOrConstSplat(N1C) &&((isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C
)->getAPIntValue().isNullValue() && "Should be a comparison with 0."
) ? static_cast<void> (0) : __assert_fail ("isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() && \"Should be a comparison with 0.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3008, __PRETTY_FUNCTION__))
3007 isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() &&((isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C
)->getAPIntValue().isNullValue() && "Should be a comparison with 0."
) ? static_cast<void> (0) : __assert_fail ("isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() && \"Should be a comparison with 0.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3008, __PRETTY_FUNCTION__))
3008 "Should be a comparison with 0.")((isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C
)->getAPIntValue().isNullValue() && "Should be a comparison with 0."
) ? static_cast<void> (0) : __assert_fail ("isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() && \"Should be a comparison with 0.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3008, __PRETTY_FUNCTION__))
;
3009 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&(((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Valid only for [in]equality comparisons."
) ? static_cast<void> (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Valid only for [in]equality comparisons.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3010, __PRETTY_FUNCTION__))
3010 "Valid only for [in]equality comparisons.")(((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Valid only for [in]equality comparisons."
) ? static_cast<void> (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Valid only for [in]equality comparisons.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3010, __PRETTY_FUNCTION__))
;
3011
3012 unsigned NewShiftOpcode;
3013 SDValue X, C, Y;
3014
3015 SelectionDAG &DAG = DCI.DAG;
3016 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3017
3018 // Look for '(C l>>/<< Y)'.
3019 auto Match = [&NewShiftOpcode, &X, &C, &Y, &TLI, &DAG](SDValue V) {
3020 // The shift should be one-use.
3021 if (!V.hasOneUse())
3022 return false;
3023 unsigned OldShiftOpcode = V.getOpcode();
3024 switch (OldShiftOpcode) {
3025 case ISD::SHL:
3026 NewShiftOpcode = ISD::SRL;
3027 break;
3028 case ISD::SRL:
3029 NewShiftOpcode = ISD::SHL;
3030 break;
3031 default:
3032 return false; // must be a logical shift.
3033 }
3034 // We should be shifting a constant.
3035 // FIXME: best to use isConstantOrConstantVector().
3036 C = V.getOperand(0);
3037 ConstantSDNode *CC =
3038 isConstOrConstSplat(C, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
3039 if (!CC)
3040 return false;
3041 Y = V.getOperand(1);
3042
3043 ConstantSDNode *XC =
3044 isConstOrConstSplat(X, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
3045 return TLI.shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
3046 X, XC, CC, Y, OldShiftOpcode, NewShiftOpcode, DAG);
3047 };
3048
3049 // LHS of comparison should be an one-use 'and'.
3050 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())
3051 return SDValue();
3052
3053 X = N0.getOperand(0);
3054 SDValue Mask = N0.getOperand(1);
3055
3056 // 'and' is commutative!
3057 if (!Match(Mask)) {
3058 std::swap(X, Mask);
3059 if (!Match(Mask))
3060 return SDValue();
3061 }
3062
3063 EVT VT = X.getValueType();
3064
3065 // Produce:
3066 // ((X 'OppositeShiftOpcode' Y) & C) Cond 0
3067 SDValue T0 = DAG.getNode(NewShiftOpcode, DL, VT, X, Y);
3068 SDValue T1 = DAG.getNode(ISD::AND, DL, VT, T0, C);
3069 SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, N1C, Cond);
3070 return T2;
3071}
3072
3073/// Try to fold an equality comparison with a {add/sub/xor} binary operation as
3074/// the 1st operand (N0). Callers are expected to swap the N0/N1 parameters to
3075/// handle the commuted versions of these patterns.
3076SDValue TargetLowering::foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1,
3077 ISD::CondCode Cond, const SDLoc &DL,
3078 DAGCombinerInfo &DCI) const {
3079 unsigned BOpcode = N0.getOpcode();
3080 assert((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) &&(((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD
::XOR) && "Unexpected binop") ? static_cast<void>
(0) : __assert_fail ("(BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) && \"Unexpected binop\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3081, __PRETTY_FUNCTION__))
3081 "Unexpected binop")(((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD
::XOR) && "Unexpected binop") ? static_cast<void>
(0) : __assert_fail ("(BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) && \"Unexpected binop\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3081, __PRETTY_FUNCTION__))
;
3082 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Unexpected condcode")(((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Unexpected condcode"
) ? static_cast<void> (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Unexpected condcode\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3082, __PRETTY_FUNCTION__))
;
3083
3084 // (X + Y) == X --> Y == 0
3085 // (X - Y) == X --> Y == 0
3086 // (X ^ Y) == X --> Y == 0
3087 SelectionDAG &DAG = DCI.DAG;
3088 EVT OpVT = N0.getValueType();
3089 SDValue X = N0.getOperand(0);
3090 SDValue Y = N0.getOperand(1);
3091 if (X == N1)
3092 return DAG.getSetCC(DL, VT, Y, DAG.getConstant(0, DL, OpVT), Cond);
3093
3094 if (Y != N1)
3095 return SDValue();
3096
3097 // (X + Y) == Y --> X == 0
3098 // (X ^ Y) == Y --> X == 0
3099 if (BOpcode == ISD::ADD || BOpcode == ISD::XOR)
3100 return DAG.getSetCC(DL, VT, X, DAG.getConstant(0, DL, OpVT), Cond);
3101
3102 // The shift would not be valid if the operands are boolean (i1).
3103 if (!N0.hasOneUse() || OpVT.getScalarSizeInBits() == 1)
3104 return SDValue();
3105
3106 // (X - Y) == Y --> X == Y << 1
3107 EVT ShiftVT = getShiftAmountTy(OpVT, DAG.getDataLayout(),
3108 !DCI.isBeforeLegalize());
3109 SDValue One = DAG.getConstant(1, DL, ShiftVT);
3110 SDValue YShl1 = DAG.getNode(ISD::SHL, DL, N1.getValueType(), Y, One);
3111 if (!DCI.isCalledByLegalizer())
3112 DCI.AddToWorklist(YShl1.getNode());
3113 return DAG.getSetCC(DL, VT, X, YShl1, Cond);
3114}
3115
3116/// Try to simplify a setcc built with the specified operands and cc. If it is
3117/// unable to simplify it, return a null SDValue.
3118SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
3119 ISD::CondCode Cond, bool foldBooleans,
3120 DAGCombinerInfo &DCI,
3121 const SDLoc &dl) const {
3122 SelectionDAG &DAG = DCI.DAG;
3123 const DataLayout &Layout = DAG.getDataLayout();
3124 EVT OpVT = N0.getValueType();
3125
3126 // Constant fold or commute setcc.
3127 if (SDValue Fold = DAG.FoldSetCC(VT, N0, N1, Cond, dl))
3128 return Fold;
3129
3130 // Ensure that the constant occurs on the RHS and fold constant comparisons.
3131 // TODO: Handle non-splat vector constants. All undef causes trouble.
3132 ISD::CondCode SwappedCC = ISD::getSetCCSwappedOperands(Cond);
3133 if (isConstOrConstSplat(N0) &&
3134 (DCI.isBeforeLegalizeOps() ||
3135 isCondCodeLegal(SwappedCC, N0.getSimpleValueType())))
3136 return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);
3137
3138 // If we have a subtract with the same 2 non-constant operands as this setcc
3139 // -- but in reverse order -- then try to commute the operands of this setcc
3140 // to match. A matching pair of setcc (cmp) and sub may be combined into 1
3141 // instruction on some targets.
3142 if (!isConstOrConstSplat(N0) && !isConstOrConstSplat(N1) &&
3143 (DCI.isBeforeLegalizeOps() ||
3144 isCondCodeLegal(SwappedCC, N0.getSimpleValueType())) &&
3145 DAG.getNodeIfExists(ISD::SUB, DAG.getVTList(OpVT), { N1, N0 } ) &&
3146 !DAG.getNodeIfExists(ISD::SUB, DAG.getVTList(OpVT), { N0, N1 } ))
3147 return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);
3148
3149 if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
3150 const APInt &C1 = N1C->getAPIntValue();
3151
3152 // If the LHS is '(srl (ctlz x), 5)', the RHS is 0/1, and this is an
3153 // equality comparison, then we're just comparing whether X itself is
3154 // zero.
3155 if (N0.getOpcode() == ISD::SRL && (C1.isNullValue() || C1.isOneValue()) &&
3156 N0.getOperand(0).getOpcode() == ISD::CTLZ &&
3157 N0.getOperand(1).getOpcode() == ISD::Constant) {
3158 const APInt &ShAmt = N0.getConstantOperandAPInt(1);
3159 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3160 ShAmt == Log2_32(N0.getValueSizeInBits())) {
3161 if ((C1 == 0) == (Cond == ISD::SETEQ)) {
3162 // (srl (ctlz x), 5) == 0 -> X != 0
3163 // (srl (ctlz x), 5) != 1 -> X != 0
3164 Cond = ISD::SETNE;
3165 } else {
3166 // (srl (ctlz x), 5) != 0 -> X == 0
3167 // (srl (ctlz x), 5) == 1 -> X == 0
3168 Cond = ISD::SETEQ;
3169 }
3170 SDValue Zero = DAG.getConstant(0, dl, N0.getValueType());
3171 return DAG.getSetCC(dl, VT, N0.getOperand(0).getOperand(0),
3172 Zero, Cond);
3173 }
3174 }
3175
3176 SDValue CTPOP = N0;
3177 // Look through truncs that don't change the value of a ctpop.
3178 if (N0.hasOneUse() && N0.getOpcode() == ISD::TRUNCATE)
3179 CTPOP = N0.getOperand(0);
3180
3181 if (CTPOP.hasOneUse() && CTPOP.getOpcode() == ISD::CTPOP &&
3182 (N0 == CTPOP ||
3183 N0.getValueSizeInBits() > Log2_32_Ceil(CTPOP.getValueSizeInBits()))) {
3184 EVT CTVT = CTPOP.getValueType();
3185 SDValue CTOp = CTPOP.getOperand(0);
3186
3187 // (ctpop x) u< 2 -> (x & x-1) == 0
3188 // (ctpop x) u> 1 -> (x & x-1) != 0
3189 if ((Cond == ISD::SETULT && C1 == 2) || (Cond == ISD::SETUGT && C1 == 1)){
3190 SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
3191 SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, CTOp, NegOne);
3192 SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Add);
3193 ISD::CondCode CC = Cond == ISD::SETULT ? ISD::SETEQ : ISD::SETNE;
3194 return DAG.getSetCC(dl, VT, And, DAG.getConstant(0, dl, CTVT), CC);
3195 }
3196
3197 // If ctpop is not supported, expand a power-of-2 comparison based on it.
3198 if (C1 == 1 && !isOperationLegalOrCustom(ISD::CTPOP, CTVT) &&
3199 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
3200 // (ctpop x) == 1 --> (x != 0) && ((x & x-1) == 0)
3201 // (ctpop x) != 1 --> (x == 0) || ((x & x-1) != 0)
3202 SDValue Zero = DAG.getConstant(0, dl, CTVT);
3203 SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
3204 assert(CTVT.isInteger())((CTVT.isInteger()) ? static_cast<void> (0) : __assert_fail
("CTVT.isInteger()", "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3204, __PRETTY_FUNCTION__))
;
3205 ISD::CondCode InvCond = ISD::getSetCCInverse(Cond, CTVT);
3206 SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, CTOp, NegOne);
3207 SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Add);
3208 SDValue LHS = DAG.getSetCC(dl, VT, CTOp, Zero, InvCond);
3209 SDValue RHS = DAG.getSetCC(dl, VT, And, Zero, Cond);
3210 unsigned LogicOpcode = Cond == ISD::SETEQ ? ISD::AND : ISD::OR;
3211 return DAG.getNode(LogicOpcode, dl, VT, LHS, RHS);
3212 }
3213 }
3214
3215 // (zext x) == C --> x == (trunc C)
3216 // (sext x) == C --> x == (trunc C)
3217 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3218 DCI.isBeforeLegalize() && N0->hasOneUse()) {
3219 unsigned MinBits = N0.getValueSizeInBits();
3220 SDValue PreExt;
3221 bool Signed = false;
3222 if (N0->getOpcode() == ISD::ZERO_EXTEND) {
3223 // ZExt
3224 MinBits = N0->getOperand(0).getValueSizeInBits();
3225 PreExt = N0->getOperand(0);
3226 } else if (N0->getOpcode() == ISD::AND) {
3227 // DAGCombine turns costly ZExts into ANDs
3228 if (auto *C = dyn_cast<ConstantSDNode>(N0->getOperand(1)))
3229 if ((C->getAPIntValue()+1).isPowerOf2()) {
3230 MinBits = C->getAPIntValue().countTrailingOnes();
3231 PreExt = N0->getOperand(0);
3232 }
3233 } else if (N0->getOpcode() == ISD::SIGN_EXTEND) {
3234 // SExt
3235 MinBits = N0->getOperand(0).getValueSizeInBits();
3236 PreExt = N0->getOperand(0);
3237 Signed = true;
3238 } else if (auto *LN0 = dyn_cast<LoadSDNode>(N0)) {
3239 // ZEXTLOAD / SEXTLOAD
3240 if (LN0->getExtensionType() == ISD::ZEXTLOAD) {
3241 MinBits = LN0->getMemoryVT().getSizeInBits();
3242 PreExt = N0;
3243 } else if (LN0->getExtensionType() == ISD::SEXTLOAD) {
3244 Signed = true;
3245 MinBits = LN0->getMemoryVT().getSizeInBits();
3246 PreExt = N0;
3247 }
3248 }
3249
3250 // Figure out how many bits we need to preserve this constant.
3251 unsigned ReqdBits = Signed ?
3252 C1.getBitWidth() - C1.getNumSignBits() + 1 :
3253 C1.getActiveBits();
3254
3255 // Make sure we're not losing bits from the constant.
3256 if (MinBits > 0 &&
3257 MinBits < C1.getBitWidth() &&
3258 MinBits >= ReqdBits) {
3259 EVT MinVT = EVT::getIntegerVT(*DAG.getContext(), MinBits);
3260 if (isTypeDesirableForOp(ISD::SETCC, MinVT)) {
3261 // Will get folded away.
3262 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MinVT, PreExt);
3263 if (MinBits == 1 && C1 == 1)
3264 // Invert the condition.
3265 return DAG.getSetCC(dl, VT, Trunc, DAG.getConstant(0, dl, MVT::i1),
3266 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
3267 SDValue C = DAG.getConstant(C1.trunc(MinBits), dl, MinVT);
3268 return DAG.getSetCC(dl, VT, Trunc, C, Cond);
3269 }
3270
3271 // If truncating the setcc operands is not desirable, we can still
3272 // simplify the expression in some cases:
3273 // setcc ([sz]ext (setcc x, y, cc)), 0, setne) -> setcc (x, y, cc)
3274 // setcc ([sz]ext (setcc x, y, cc)), 0, seteq) -> setcc (x, y, inv(cc))
3275 // setcc (zext (setcc x, y, cc)), 1, setne) -> setcc (x, y, inv(cc))
3276 // setcc (zext (setcc x, y, cc)), 1, seteq) -> setcc (x, y, cc)
3277 // setcc (sext (setcc x, y, cc)), -1, setne) -> setcc (x, y, inv(cc))
3278 // setcc (sext (setcc x, y, cc)), -1, seteq) -> setcc (x, y, cc)
3279 SDValue TopSetCC = N0->getOperand(0);
3280 unsigned N0Opc = N0->getOpcode();
3281 bool SExt = (N0Opc == ISD::SIGN_EXTEND);
3282 if (TopSetCC.getValueType() == MVT::i1 && VT == MVT::i1 &&
3283 TopSetCC.getOpcode() == ISD::SETCC &&
3284 (N0Opc == ISD::ZERO_EXTEND || N0Opc == ISD::SIGN_EXTEND) &&
3285 (isConstFalseVal(N1C) ||
3286 isExtendedTrueVal(N1C, N0->getValueType(0), SExt))) {
3287
3288 bool Inverse = (N1C->isNullValue() && Cond == ISD::SETEQ) ||
3289 (!N1C->isNullValue() && Cond == ISD::SETNE);
3290
3291 if (!Inverse)
3292 return TopSetCC;
3293
3294 ISD::CondCode InvCond = ISD::getSetCCInverse(
3295 cast<CondCodeSDNode>(TopSetCC.getOperand(2))->get(),
3296 TopSetCC.getOperand(0).getValueType());
3297 return DAG.getSetCC(dl, VT, TopSetCC.getOperand(0),
3298 TopSetCC.getOperand(1),
3299 InvCond);
3300 }
3301 }
3302 }
3303
3304 // If the LHS is '(and load, const)', the RHS is 0, the test is for
3305 // equality or unsigned, and all 1 bits of the const are in the same
3306 // partial word, see if we can shorten the load.
3307 if (DCI.isBeforeLegalize() &&
3308 !ISD::isSignedIntSetCC(Cond) &&
3309 N0.getOpcode() == ISD::AND && C1 == 0 &&
3310 N0.getNode()->hasOneUse() &&
3311 isa<LoadSDNode>(N0.getOperand(0)) &&
3312 N0.getOperand(0).getNode()->hasOneUse() &&
3313 isa<ConstantSDNode>(N0.getOperand(1))) {
3314 LoadSDNode *Lod = cast<LoadSDNode>(N0.getOperand(0));
3315 APInt bestMask;
3316 unsigned bestWidth = 0, bestOffset = 0;
3317 if (Lod->isSimple() && Lod->isUnindexed()) {
3318 unsigned origWidth = N0.getValueSizeInBits();
3319 unsigned maskWidth = origWidth;
3320 // We can narrow (e.g.) 16-bit extending loads on 32-bit target to
3321 // 8 bits, but have to be careful...
3322 if (Lod->getExtensionType() != ISD::NON_EXTLOAD)
3323 origWidth = Lod->getMemoryVT().getSizeInBits();
3324 const APInt &Mask = N0.getConstantOperandAPInt(1);
3325 for (unsigned width = origWidth / 2; width>=8; width /= 2) {
3326 APInt newMask = APInt::getLowBitsSet(maskWidth, width);
3327 for (unsigned offset=0; offset<origWidth/width; offset++) {
3328 if (Mask.isSubsetOf(newMask)) {
3329 if (Layout.isLittleEndian())
3330 bestOffset = (uint64_t)offset * (width/8);
3331 else
3332 bestOffset = (origWidth/width - offset - 1) * (width/8);
3333 bestMask = Mask.lshr(offset * (width/8) * 8);
3334 bestWidth = width;
3335 break;
3336 }
3337 newMask <<= width;
3338 }
3339 }
3340 }
3341 if (bestWidth) {
3342 EVT newVT = EVT::getIntegerVT(*DAG.getContext(), bestWidth);
3343 if (newVT.isRound() &&
3344 shouldReduceLoadWidth(Lod, ISD::NON_EXTLOAD, newVT)) {
3345 SDValue Ptr = Lod->getBasePtr();
3346 if (bestOffset != 0)
3347 Ptr = DAG.getMemBasePlusOffset(Ptr, bestOffset, dl);
3348 unsigned NewAlign = MinAlign(Lod->getAlignment(), bestOffset);
3349 SDValue NewLoad = DAG.getLoad(
3350 newVT, dl, Lod->getChain(), Ptr,
3351 Lod->getPointerInfo().getWithOffset(bestOffset), NewAlign);
3352 return DAG.getSetCC(dl, VT,
3353 DAG.getNode(ISD::AND, dl, newVT, NewLoad,
3354 DAG.getConstant(bestMask.trunc(bestWidth),
3355 dl, newVT)),
3356 DAG.getConstant(0LL, dl, newVT), Cond);
3357 }
3358 }
3359 }
3360
3361 // If the LHS is a ZERO_EXTEND, perform the comparison on the input.
3362 if (N0.getOpcode() == ISD::ZERO_EXTEND) {
3363 unsigned InSize = N0.getOperand(0).getValueSizeInBits();
3364
3365 // If the comparison constant has bits in the upper part, the
3366 // zero-extended value could never match.
3367 if (C1.intersects(APInt::getHighBitsSet(C1.getBitWidth(),
3368 C1.getBitWidth() - InSize))) {
3369 switch (Cond) {
3370 case ISD::SETUGT:
3371 case ISD::SETUGE:
3372 case ISD::SETEQ:
3373 return DAG.getConstant(0, dl, VT);
3374 case ISD::SETULT:
3375 case ISD::SETULE:
3376 case ISD::SETNE:
3377 return DAG.getConstant(1, dl, VT);
3378 case ISD::SETGT:
3379 case ISD::SETGE:
3380 // True if the sign bit of C1 is set.
3381 return DAG.getConstant(C1.isNegative(), dl, VT);
3382 case ISD::SETLT:
3383 case ISD::SETLE:
3384 // True if the sign bit of C1 isn't set.
3385 return DAG.getConstant(C1.isNonNegative(), dl, VT);
3386 default:
3387 break;
3388 }
3389 }
3390
3391 // Otherwise, we can perform the comparison with the low bits.
3392 switch (Cond) {
3393 case ISD::SETEQ:
3394 case ISD::SETNE:
3395 case ISD::SETUGT:
3396 case ISD::SETUGE:
3397 case ISD::SETULT:
3398 case ISD::SETULE: {
3399 EVT newVT = N0.getOperand(0).getValueType();
3400 if (DCI.isBeforeLegalizeOps() ||
3401 (isOperationLegal(ISD::SETCC, newVT) &&
3402 isCondCodeLegal(Cond, newVT.getSimpleVT()))) {
3403 EVT NewSetCCVT = getSetCCResultType(Layout, *DAG.getContext(), newVT);
3404 SDValue NewConst = DAG.getConstant(C1.trunc(InSize), dl, newVT);
3405
3406 SDValue NewSetCC = DAG.getSetCC(dl, NewSetCCVT, N0.getOperand(0),
3407 NewConst, Cond);
3408 return DAG.getBoolExtOrTrunc(NewSetCC, dl, VT, N0.getValueType());
3409 }
3410 break;
3411 }
3412 default:
3413 break; // todo, be more careful with signed comparisons
3414 }
3415 } else if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
3416 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
3417 EVT ExtSrcTy = cast<VTSDNode>(N0.getOperand(1))->getVT();
3418 unsigned ExtSrcTyBits = ExtSrcTy.getSizeInBits();
3419 EVT ExtDstTy = N0.getValueType();
3420 unsigned ExtDstTyBits = ExtDstTy.getSizeInBits();
3421
3422 // If the constant doesn't fit into the number of bits for the source of
3423 // the sign extension, it is impossible for both sides to be equal.
3424 if (C1.getMinSignedBits() > ExtSrcTyBits)
3425 return DAG.getConstant(Cond == ISD::SETNE, dl, VT);
3426
3427 SDValue ZextOp;
3428 EVT Op0Ty = N0.getOperand(0).getValueType();
3429 if (Op0Ty == ExtSrcTy) {
3430 ZextOp = N0.getOperand(0);
3431 } else {
3432 APInt Imm = APInt::getLowBitsSet(ExtDstTyBits, ExtSrcTyBits);
3433 ZextOp = DAG.getNode(ISD::AND, dl, Op0Ty, N0.getOperand(0),
3434 DAG.getConstant(Imm, dl, Op0Ty));
3435 }
3436 if (!DCI.isCalledByLegalizer())
3437 DCI.AddToWorklist(ZextOp.getNode());
3438 // Otherwise, make this a use of a zext.
3439 return DAG.getSetCC(dl, VT, ZextOp,
3440 DAG.getConstant(C1 & APInt::getLowBitsSet(
3441 ExtDstTyBits,
3442 ExtSrcTyBits),
3443 dl, ExtDstTy),
3444 Cond);
3445 } else if ((N1C->isNullValue() || N1C->isOne()) &&
3446 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
3447 // SETCC (SETCC), [0|1], [EQ|NE] -> SETCC
3448 if (N0.getOpcode() == ISD::SETCC &&
3449 isTypeLegal(VT) && VT.bitsLE(N0.getValueType()) &&
3450 (N0.getValueType() == MVT::i1 ||
3451 getBooleanContents(N0.getOperand(0).getValueType()) ==
3452 ZeroOrOneBooleanContent)) {
3453 bool TrueWhenTrue = (Cond == ISD::SETEQ) ^ (!N1C->isOne());
3454 if (TrueWhenTrue)
3455 return DAG.getNode(ISD::TRUNCATE, dl, VT, N0);
3456 // Invert the condition.
3457 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
3458 CC = ISD::getSetCCInverse(CC, N0.getOperand(0).getValueType());
3459 if (DCI.isBeforeLegalizeOps() ||
3460 isCondCodeLegal(CC, N0.getOperand(0).getSimpleValueType()))
3461 return DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC);
3462 }
3463
3464 if ((N0.getOpcode() == ISD::XOR ||
3465 (N0.getOpcode() == ISD::AND &&
3466 N0.getOperand(0).getOpcode() == ISD::XOR &&
3467 N0.getOperand(1) == N0.getOperand(0).getOperand(1))) &&
3468 isa<ConstantSDNode>(N0.getOperand(1)) &&
3469 cast<ConstantSDNode>(N0.getOperand(1))->isOne()) {
3470 // If this is (X^1) == 0/1, swap the RHS and eliminate the xor. We
3471 // can only do this if the top bits are known zero.
3472 unsigned BitWidth = N0.getValueSizeInBits();
3473 if (DAG.MaskedValueIsZero(N0,
3474 APInt::getHighBitsSet(BitWidth,
3475 BitWidth-1))) {
3476 // Okay, get the un-inverted input value.
3477 SDValue Val;
3478 if (N0.getOpcode() == ISD::XOR) {
3479 Val = N0.getOperand(0);
3480 } else {
3481 assert(N0.getOpcode() == ISD::AND &&((N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode
() == ISD::XOR) ? static_cast<void> (0) : __assert_fail
("N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::XOR"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3482, __PRETTY_FUNCTION__))
3482 N0.getOperand(0).getOpcode() == ISD::XOR)((N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode
() == ISD::XOR) ? static_cast<void> (0) : __assert_fail
("N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::XOR"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3482, __PRETTY_FUNCTION__))
;
3483 // ((X^1)&1)^1 -> X & 1
3484 Val = DAG.getNode(ISD::AND, dl, N0.getValueType(),
3485 N0.getOperand(0).getOperand(0),
3486 N0.getOperand(1));
3487 }
3488
3489 return DAG.getSetCC(dl, VT, Val, N1,
3490 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
3491 }
3492 } else if (N1C->isOne()) {
3493 SDValue Op0 = N0;
3494 if (Op0.getOpcode() == ISD::TRUNCATE)
3495 Op0 = Op0.getOperand(0);
3496
3497 if ((Op0.getOpcode() == ISD::XOR) &&
3498 Op0.getOperand(0).getOpcode() == ISD::SETCC &&
3499 Op0.getOperand(1).getOpcode() == ISD::SETCC) {
3500 SDValue XorLHS = Op0.getOperand(0);
3501 SDValue XorRHS = Op0.getOperand(1);
3502 // Ensure that the input setccs return an i1 type or 0/1 value.
3503 if (Op0.getValueType() == MVT::i1 ||
3504 (getBooleanContents(XorLHS.getOperand(0).getValueType()) ==
3505 ZeroOrOneBooleanContent &&
3506 getBooleanContents(XorRHS.getOperand(0).getValueType()) ==
3507 ZeroOrOneBooleanContent)) {
3508 // (xor (setcc), (setcc)) == / != 1 -> (setcc) != / == (setcc)
3509 Cond = (Cond == ISD::SETEQ) ? ISD::SETNE : ISD::SETEQ;
3510 return DAG.getSetCC(dl, VT, XorLHS, XorRHS, Cond);
3511 }
3512 }
3513 if (Op0.getOpcode() == ISD::AND &&
3514 isa<ConstantSDNode>(Op0.getOperand(1)) &&
3515 cast<ConstantSDNode>(Op0.getOperand(1))->isOne()) {
3516 // If this is (X&1) == / != 1, normalize it to (X&1) != / == 0.
3517 if (Op0.getValueType().bitsGT(VT))
3518 Op0 = DAG.getNode(ISD::AND, dl, VT,
3519 DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)),
3520 DAG.getConstant(1, dl, VT));
3521 else if (Op0.getValueType().bitsLT(VT))
3522 Op0 = DAG.getNode(ISD::AND, dl, VT,
3523 DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)),
3524 DAG.getConstant(1, dl, VT));
3525
3526 return DAG.getSetCC(dl, VT, Op0,
3527 DAG.getConstant(0, dl, Op0.getValueType()),
3528 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
3529 }
3530 if (Op0.getOpcode() == ISD::AssertZext &&
3531 cast<VTSDNode>(Op0.getOperand(1))->getVT() == MVT::i1)
3532 return DAG.getSetCC(dl, VT, Op0,
3533 DAG.getConstant(0, dl, Op0.getValueType()),
3534 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
3535 }
3536 }
3537
3538 // Given:
3539 // icmp eq/ne (urem %x, %y), 0
3540 // Iff %x has 0 or 1 bits set, and %y has at least 2 bits set, omit 'urem':
3541 // icmp eq/ne %x, 0
3542 if (N0.getOpcode() == ISD::UREM && N1C->isNullValue() &&
3543 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
3544 KnownBits XKnown = DAG.computeKnownBits(N0.getOperand(0));
3545 KnownBits YKnown = DAG.computeKnownBits(N0.getOperand(1));
3546 if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2)
3547 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1, Cond);
3548 }
3549
3550 if (SDValue V =
3551 optimizeSetCCOfSignedTruncationCheck(VT, N0, N1, Cond, DCI, dl))
3552 return V;
3553 }
3554
3555 // These simplifications apply to splat vectors as well.
3556 // TODO: Handle more splat vector cases.
3557 if (auto *N1C = isConstOrConstSplat(N1)) {
3558 const APInt &C1 = N1C->getAPIntValue();
3559
3560 APInt MinVal, MaxVal;
3561 unsigned OperandBitSize = N1C->getValueType(0).getScalarSizeInBits();
3562 if (ISD::isSignedIntSetCC(Cond)) {
3563 MinVal = APInt::getSignedMinValue(OperandBitSize);
3564 MaxVal = APInt::getSignedMaxValue(OperandBitSize);
3565 } else {
3566 MinVal = APInt::getMinValue(OperandBitSize);
3567 MaxVal = APInt::getMaxValue(OperandBitSize);
3568 }
3569
3570 // Canonicalize GE/LE comparisons to use GT/LT comparisons.
3571 if (Cond == ISD::SETGE || Cond == ISD::SETUGE) {
3572 // X >= MIN --> true
3573 if (C1 == MinVal)
3574 return DAG.getBoolConstant(true, dl, VT, OpVT);
3575
3576 if (!VT.isVector()) { // TODO: Support this for vectors.
3577 // X >= C0 --> X > (C0 - 1)
3578 APInt C = C1 - 1;
3579 ISD::CondCode NewCC = (Cond == ISD::SETGE) ? ISD::SETGT : ISD::SETUGT;
3580 if ((DCI.isBeforeLegalizeOps() ||
3581 isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
3582 (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
3583 isLegalICmpImmediate(C.getSExtValue())))) {
3584 return DAG.getSetCC(dl, VT, N0,
3585 DAG.getConstant(C, dl, N1.getValueType()),
3586 NewCC);
3587 }
3588 }
3589 }
3590
3591 if (Cond == ISD::SETLE || Cond == ISD::SETULE) {
3592 // X <= MAX --> true
3593 if (C1 == MaxVal)
3594 return DAG.getBoolConstant(true, dl, VT, OpVT);
3595
3596 // X <= C0 --> X < (C0 + 1)
3597 if (!VT.isVector()) { // TODO: Support this for vectors.
3598 APInt C = C1 + 1;
3599 ISD::CondCode NewCC = (Cond == ISD::SETLE) ? ISD::SETLT : ISD::SETULT;
3600 if ((DCI.isBeforeLegalizeOps() ||
3601 isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
3602 (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
3603 isLegalICmpImmediate(C.getSExtValue())))) {
3604 return DAG.getSetCC(dl, VT, N0,
3605 DAG.getConstant(C, dl, N1.getValueType()),
3606 NewCC);
3607 }
3608 }
3609 }
3610
3611 if (Cond == ISD::SETLT || Cond == ISD::SETULT) {
3612 if (C1 == MinVal)
3613 return DAG.getBoolConstant(false, dl, VT, OpVT); // X < MIN --> false
3614
3615 // TODO: Support this for vectors after legalize ops.
3616 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
3617 // Canonicalize setlt X, Max --> setne X, Max
3618 if (C1 == MaxVal)
3619 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
3620
3621 // If we have setult X, 1, turn it into seteq X, 0
3622 if (C1 == MinVal+1)
3623 return DAG.getSetCC(dl, VT, N0,
3624 DAG.getConstant(MinVal, dl, N0.getValueType()),
3625 ISD::SETEQ);
3626 }
3627 }
3628
3629 if (Cond == ISD::SETGT || Cond == ISD::SETUGT) {
3630 if (C1 == MaxVal)
3631 return DAG.getBoolConstant(false, dl, VT, OpVT); // X > MAX --> false
3632
3633 // TODO: Support this for vectors after legalize ops.
3634 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
3635 // Canonicalize setgt X, Min --> setne X, Min
3636 if (C1 == MinVal)
3637 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
3638
3639 // If we have setugt X, Max-1, turn it into seteq X, Max
3640 if (C1 == MaxVal-1)
3641 return DAG.getSetCC(dl, VT, N0,
3642 DAG.getConstant(MaxVal, dl, N0.getValueType()),
3643 ISD::SETEQ);
3644 }
3645 }
3646
3647 if (Cond == ISD::SETEQ || Cond == ISD::SETNE) {
3648 // (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
3649 if (C1.isNullValue())
3650 if (SDValue CC = optimizeSetCCByHoistingAndByConstFromLogicalShift(
3651 VT, N0, N1, Cond, DCI, dl))
3652 return CC;
3653 }
3654
3655 // If we have "setcc X, C0", check to see if we can shrink the immediate
3656 // by changing cc.
3657 // TODO: Support this for vectors after legalize ops.
3658 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
3659 // SETUGT X, SINTMAX -> SETLT X, 0
3660 if (Cond == ISD::SETUGT &&
3661 C1 == APInt::getSignedMaxValue(OperandBitSize))
3662 return DAG.getSetCC(dl, VT, N0,
3663 DAG.getConstant(0, dl, N1.getValueType()),
3664 ISD::SETLT);
3665
3666 // SETULT X, SINTMIN -> SETGT X, -1
3667 if (Cond == ISD::SETULT &&
3668 C1 == APInt::getSignedMinValue(OperandBitSize)) {
3669 SDValue ConstMinusOne =
3670 DAG.getConstant(APInt::getAllOnesValue(OperandBitSize), dl,
3671 N1.getValueType());
3672 return DAG.getSetCC(dl, VT, N0, ConstMinusOne, ISD::SETGT);
3673 }
3674 }
3675 }
3676
3677 // Back to non-vector simplifications.
3678 // TODO: Can we do these for vector splats?
3679 if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
3680 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3681 const APInt &C1 = N1C->getAPIntValue();
3682 EVT ShValTy = N0.getValueType();
3683
3684 // Fold bit comparisons when we can.
3685 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3686 (VT == ShValTy || (isTypeLegal(VT) && VT.bitsLE(ShValTy))) &&
3687 N0.getOpcode() == ISD::AND) {
3688 if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
3689 EVT ShiftTy =
3690 getShiftAmountTy(ShValTy, Layout, !DCI.isBeforeLegalize());
3691 if (Cond == ISD::SETNE && C1 == 0) {// (X & 8) != 0 --> (X & 8) >> 3
3692 // Perform the xform if the AND RHS is a single bit.
3693 unsigned ShCt = AndRHS->getAPIntValue().logBase2();
3694 if (AndRHS->getAPIntValue().isPowerOf2() &&
3695 !TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {
3696 return DAG.getNode(ISD::TRUNCATE, dl, VT,
3697 DAG.getNode(ISD::SRL, dl, ShValTy, N0,
3698 DAG.getConstant(ShCt, dl, ShiftTy)));
3699 }
3700 } else if (Cond == ISD::SETEQ && C1 == AndRHS->getAPIntValue()) {
3701 // (X & 8) == 8 --> (X & 8) >> 3
3702 // Perform the xform if C1 is a single bit.
3703 unsigned ShCt = C1.logBase2();
3704 if (C1.isPowerOf2() &&
3705 !TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {
3706 return DAG.getNode(ISD::TRUNCATE, dl, VT,
3707 DAG.getNode(ISD::SRL, dl, ShValTy, N0,
3708 DAG.getConstant(ShCt, dl, ShiftTy)));
3709 }
3710 }
3711 }
3712 }
3713
3714 if (C1.getMinSignedBits() <= 64 &&
3715 !isLegalICmpImmediate(C1.getSExtValue())) {
3716 EVT ShiftTy = getShiftAmountTy(ShValTy, Layout, !DCI.isBeforeLegalize());
3717 // (X & -256) == 256 -> (X >> 8) == 1
3718 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3719 N0.getOpcode() == ISD::AND && N0.hasOneUse()) {
3720 if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
3721 const APInt &AndRHSC = AndRHS->getAPIntValue();
3722 if ((-AndRHSC).isPowerOf2() && (AndRHSC & C1) == C1) {
3723 unsigned ShiftBits = AndRHSC.countTrailingZeros();
3724 if (!TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
3725 SDValue Shift =
3726 DAG.getNode(ISD::SRL, dl, ShValTy, N0.getOperand(0),
3727 DAG.getConstant(ShiftBits, dl, ShiftTy));
3728 SDValue CmpRHS = DAG.getConstant(C1.lshr(ShiftBits), dl, ShValTy);
3729 return DAG.getSetCC(dl, VT, Shift, CmpRHS, Cond);
3730 }
3731 }
3732 }
3733 } else if (Cond == ISD::SETULT || Cond == ISD::SETUGE ||
3734 Cond == ISD::SETULE || Cond == ISD::SETUGT) {
3735 bool AdjOne = (Cond == ISD::SETULE || Cond == ISD::SETUGT);
3736 // X < 0x100000000 -> (X >> 32) < 1
3737 // X >= 0x100000000 -> (X >> 32) >= 1
3738 // X <= 0x0ffffffff -> (X >> 32) < 1
3739 // X > 0x0ffffffff -> (X >> 32) >= 1
3740 unsigned ShiftBits;
3741 APInt NewC = C1;
3742 ISD::CondCode NewCond = Cond;
3743 if (AdjOne) {
3744 ShiftBits = C1.countTrailingOnes();
3745 NewC = NewC + 1;
3746 NewCond = (Cond == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
3747 } else {
3748 ShiftBits = C1.countTrailingZeros();
3749 }
3750 NewC.lshrInPlace(ShiftBits);
3751 if (ShiftBits && NewC.getMinSignedBits() <= 64 &&
3752 isLegalICmpImmediate(NewC.getSExtValue()) &&
3753 !TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
3754 SDValue Shift = DAG.getNode(ISD::SRL, dl, ShValTy, N0,
3755 DAG.getConstant(ShiftBits, dl, ShiftTy));
3756 SDValue CmpRHS = DAG.getConstant(NewC, dl, ShValTy);
3757 return DAG.getSetCC(dl, VT, Shift, CmpRHS, NewCond);
3758 }
3759 }
3760 }
3761 }
3762
3763 if (!isa<ConstantFPSDNode>(N0) && isa<ConstantFPSDNode>(N1)) {
3764 auto *CFP = cast<ConstantFPSDNode>(N1);
3765 assert(!CFP->getValueAPF().isNaN() && "Unexpected NaN value")((!CFP->getValueAPF().isNaN() && "Unexpected NaN value"
) ? static_cast<void> (0) : __assert_fail ("!CFP->getValueAPF().isNaN() && \"Unexpected NaN value\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3765, __PRETTY_FUNCTION__))
;
3766
3767 // Otherwise, we know the RHS is not a NaN. Simplify the node to drop the
3768 // constant if knowing that the operand is non-nan is enough. We prefer to
3769 // have SETO(x,x) instead of SETO(x, 0.0) because this avoids having to
3770 // materialize 0.0.
3771 if (Cond == ISD::SETO || Cond == ISD::SETUO)
3772 return DAG.getSetCC(dl, VT, N0, N0, Cond);
3773
3774 // setcc (fneg x), C -> setcc swap(pred) x, -C
3775 if (N0.getOpcode() == ISD::FNEG) {
3776 ISD::CondCode SwapCond = ISD::getSetCCSwappedOperands(Cond);
3777 if (DCI.isBeforeLegalizeOps() ||
3778 isCondCodeLegal(SwapCond, N0.getSimpleValueType())) {
3779 SDValue NegN1 = DAG.getNode(ISD::FNEG, dl, N0.getValueType(), N1);
3780 return DAG.getSetCC(dl, VT, N0.getOperand(0), NegN1, SwapCond);
3781 }
3782 }
3783
3784 // If the condition is not legal, see if we can find an equivalent one
3785 // which is legal.
3786 if (!isCondCodeLegal(Cond, N0.getSimpleValueType())) {
3787 // If the comparison was an awkward floating-point == or != and one of
3788 // the comparison operands is infinity or negative infinity, convert the
3789 // condition to a less-awkward <= or >=.
3790 if (CFP->getValueAPF().isInfinity()) {
3791 if (CFP->getValueAPF().isNegative()) {
3792 if (Cond == ISD::SETOEQ &&
3793 isCondCodeLegal(ISD::SETOLE, N0.getSimpleValueType()))
3794 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETOLE);
3795 if (Cond == ISD::SETUEQ &&
3796 isCondCodeLegal(ISD::SETOLE, N0.getSimpleValueType()))
3797 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETULE);
3798 if (Cond == ISD::SETUNE &&
3799 isCondCodeLegal(ISD::SETUGT, N0.getSimpleValueType()))
3800 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETUGT);
3801 if (Cond == ISD::SETONE &&
3802 isCondCodeLegal(ISD::SETUGT, N0.getSimpleValueType()))
3803 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETOGT);
3804 } else {
3805 if (Cond == ISD::SETOEQ &&
3806 isCondCodeLegal(ISD::SETOGE, N0.getSimpleValueType()))
3807 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETOGE);
3808 if (Cond == ISD::SETUEQ &&
3809 isCondCodeLegal(ISD::SETOGE, N0.getSimpleValueType()))
3810 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETUGE);
3811 if (Cond == ISD::SETUNE &&
3812 isCondCodeLegal(ISD::SETULT, N0.getSimpleValueType()))
3813 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETULT);
3814 if (Cond == ISD::SETONE &&
3815 isCondCodeLegal(ISD::SETULT, N0.getSimpleValueType()))
3816 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETOLT);
3817 }
3818 }
3819 }
3820 }
3821
3822 if (N0 == N1) {
3823 // The sext(setcc()) => setcc() optimization relies on the appropriate
3824 // constant being emitted.
3825 assert(!N0.getValueType().isInteger() &&((!N0.getValueType().isInteger() && "Integer types should be handled by FoldSetCC"
) ? static_cast<void> (0) : __assert_fail ("!N0.getValueType().isInteger() && \"Integer types should be handled by FoldSetCC\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3826, __PRETTY_FUNCTION__))
3826 "Integer types should be handled by FoldSetCC")((!N0.getValueType().isInteger() && "Integer types should be handled by FoldSetCC"
) ? static_cast<void> (0) : __assert_fail ("!N0.getValueType().isInteger() && \"Integer types should be handled by FoldSetCC\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3826, __PRETTY_FUNCTION__))
;
3827
3828 bool EqTrue = ISD::isTrueWhenEqual(Cond);
3829 unsigned UOF = ISD::getUnorderedFlavor(Cond);
3830 if (UOF == 2) // FP operators that are undefined on NaNs.
3831 return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
3832 if (UOF == unsigned(EqTrue))
3833 return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
3834 // Otherwise, we can't fold it. However, we can simplify it to SETUO/SETO
3835 // if it is not already.
3836 ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO;
3837 if (NewCond != Cond &&
3838 (DCI.isBeforeLegalizeOps() ||
3839 isCondCodeLegal(NewCond, N0.getSimpleValueType())))
3840 return DAG.getSetCC(dl, VT, N0, N1, NewCond);
3841 }
3842
3843 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3844 N0.getValueType().isInteger()) {
3845 if (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB ||
3846 N0.getOpcode() == ISD::XOR) {
3847 // Simplify (X+Y) == (X+Z) --> Y == Z
3848 if (N0.getOpcode() == N1.getOpcode()) {
3849 if (N0.getOperand(0) == N1.getOperand(0))
3850 return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(1), Cond);
3851 if (N0.getOperand(1) == N1.getOperand(1))
3852 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(0), Cond);
3853 if (isCommutativeBinOp(N0.getOpcode())) {
3854 // If X op Y == Y op X, try other combinations.
3855 if (N0.getOperand(0) == N1.getOperand(1))
3856 return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(0),
3857 Cond);
3858 if (N0.getOperand(1) == N1.getOperand(0))
3859 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(1),
3860 Cond);
3861 }
3862 }
3863
3864 // If RHS is a legal immediate value for a compare instruction, we need
3865 // to be careful about increasing register pressure needlessly.
3866 bool LegalRHSImm = false;
3867
3868 if (auto *RHSC = dyn_cast<ConstantSDNode>(N1)) {
3869 if (auto *LHSR = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
3870 // Turn (X+C1) == C2 --> X == C2-C1
3871 if (N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse()) {
3872 return DAG.getSetCC(dl, VT, N0.getOperand(0),
3873 DAG.getConstant(RHSC->getAPIntValue()-
3874 LHSR->getAPIntValue(),
3875 dl, N0.getValueType()), Cond);
3876 }
3877
3878 // Turn (X^C1) == C2 into X == C1^C2 iff X&~C1 = 0.
3879 if (N0.getOpcode() == ISD::XOR)
3880 // If we know that all of the inverted bits are zero, don't bother
3881 // performing the inversion.
3882 if (DAG.MaskedValueIsZero(N0.getOperand(0), ~LHSR->getAPIntValue()))
3883 return
3884 DAG.getSetCC(dl, VT, N0.getOperand(0),
3885 DAG.getConstant(LHSR->getAPIntValue() ^
3886 RHSC->getAPIntValue(),
3887 dl, N0.getValueType()),
3888 Cond);
3889 }
3890
3891 // Turn (C1-X) == C2 --> X == C1-C2
3892 if (auto *SUBC = dyn_cast<ConstantSDNode>(N0.getOperand(0))) {
3893 if (N0.getOpcode() == ISD::SUB && N0.getNode()->hasOneUse()) {
3894 return
3895 DAG.getSetCC(dl, VT, N0.getOperand(1),
3896 DAG.getConstant(SUBC->getAPIntValue() -
3897 RHSC->getAPIntValue(),
3898 dl, N0.getValueType()),
3899 Cond);
3900 }
3901 }
3902
3903 // Could RHSC fold directly into a compare?
3904 if (RHSC->getValueType(0).getSizeInBits() <= 64)
3905 LegalRHSImm = isLegalICmpImmediate(RHSC->getSExtValue());
3906 }
3907
3908 // (X+Y) == X --> Y == 0 and similar folds.
3909 // Don't do this if X is an immediate that can fold into a cmp
3910 // instruction and X+Y has other uses. It could be an induction variable
3911 // chain, and the transform would increase register pressure.
3912 if (!LegalRHSImm || N0.hasOneUse())
3913 if (SDValue V = foldSetCCWithBinOp(VT, N0, N1, Cond, dl, DCI))
3914 return V;
3915 }
3916
3917 if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB ||
3918 N1.getOpcode() == ISD::XOR)
3919 if (SDValue V = foldSetCCWithBinOp(VT, N1, N0, Cond, dl, DCI))
3920 return V;
3921
3922 if (SDValue V = foldSetCCWithAnd(VT, N0, N1, Cond, dl, DCI))
3923 return V;
3924 }
3925
3926 // Fold remainder of division by a constant.
3927 if ((N0.getOpcode() == ISD::UREM || N0.getOpcode() == ISD::SREM) &&
3928 N0.hasOneUse() && (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
3929 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
3930
3931 // When division is cheap or optimizing for minimum size,
3932 // fall through to DIVREM creation by skipping this fold.
3933 if (!isIntDivCheap(VT, Attr) && !Attr.hasFnAttribute(Attribute::MinSize)) {
3934 if (N0.getOpcode() == ISD::UREM) {
3935 if (SDValue Folded = buildUREMEqFold(VT, N0, N1, Cond, DCI, dl))
3936 return Folded;
3937 } else if (N0.getOpcode() == ISD::SREM) {
3938 if (SDValue Folded = buildSREMEqFold(VT, N0, N1, Cond, DCI, dl))
3939 return Folded;
3940 }
3941 }
3942 }
3943
3944 // Fold away ALL boolean setcc's.
3945 if (N0.getValueType().getScalarType() == MVT::i1 && foldBooleans) {
3946 SDValue Temp;
3947 switch (Cond) {
3948 default: llvm_unreachable("Unknown integer setcc!")::llvm::llvm_unreachable_internal("Unknown integer setcc!", "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3948)
;
3949 case ISD::SETEQ: // X == Y -> ~(X^Y)
3950 Temp = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
3951 N0 = DAG.getNOT(dl, Temp, OpVT);
3952 if (!DCI.isCalledByLegalizer())
3953 DCI.AddToWorklist(Temp.getNode());
3954 break;
3955 case ISD::SETNE: // X != Y --> (X^Y)
3956 N0 = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
3957 break;
3958 case ISD::SETGT: // X >s Y --> X == 0 & Y == 1 --> ~X & Y
3959 case ISD::SETULT: // X <u Y --> X == 0 & Y == 1 --> ~X & Y
3960 Temp = DAG.getNOT(dl, N0, OpVT);
3961 N0 = DAG.getNode(ISD::AND, dl, OpVT, N1, Temp);
3962 if (!DCI.isCalledByLegalizer())
3963 DCI.AddToWorklist(Temp.getNode());
3964 break;
3965 case ISD::SETLT: // X <s Y --> X == 1 & Y == 0 --> ~Y & X
3966 case ISD::SETUGT: // X >u Y --> X == 1 & Y == 0 --> ~Y & X
3967 Temp = DAG.getNOT(dl, N1, OpVT);
3968 N0 = DAG.getNode(ISD::AND, dl, OpVT, N0, Temp);
3969 if (!DCI.isCalledByLegalizer())
3970 DCI.AddToWorklist(Temp.getNode());
3971 break;
3972 case ISD::SETULE: // X <=u Y --> X == 0 | Y == 1 --> ~X | Y
3973 case ISD::SETGE: // X >=s Y --> X == 0 | Y == 1 --> ~X | Y
3974 Temp = DAG.getNOT(dl, N0, OpVT);
3975 N0 = DAG.getNode(ISD::OR, dl, OpVT, N1, Temp);
3976 if (!DCI.isCalledByLegalizer())
3977 DCI.AddToWorklist(Temp.getNode());
3978 break;
3979 case ISD::SETUGE: // X >=u Y --> X == 1 | Y == 0 --> ~Y | X
3980 case ISD::SETLE: // X <=s Y --> X == 1 | Y == 0 --> ~Y | X
3981 Temp = DAG.getNOT(dl, N1, OpVT);
3982 N0 = DAG.getNode(ISD::OR, dl, OpVT, N0, Temp);
3983 break;
3984 }
3985 if (VT.getScalarType() != MVT::i1) {
3986 if (!DCI.isCalledByLegalizer())
3987 DCI.AddToWorklist(N0.getNode());
3988 // FIXME: If running after legalize, we probably can't do this.
3989 ISD::NodeType ExtendCode = getExtendForContent(getBooleanContents(OpVT));
3990 N0 = DAG.getNode(ExtendCode, dl, VT, N0);
3991 }
3992 return N0;
3993 }
3994
3995 // Could not fold it.
3996 return SDValue();
3997}
3998
3999/// Returns true (and the GlobalValue and the offset) if the node is a
4000/// GlobalAddress + offset.
4001bool TargetLowering::isGAPlusOffset(SDNode *WN, const GlobalValue *&GA,
4002 int64_t &Offset) const {
4003
4004 SDNode *N = unwrapAddress(SDValue(WN, 0)).getNode();
4005
4006 if (auto *GASD = dyn_cast<GlobalAddressSDNode>(N)) {
4007 GA = GASD->getGlobal();
4008 Offset += GASD->getOffset();
4009 return true;
4010 }
4011
4012 if (N->getOpcode() == ISD::ADD) {
4013 SDValue N1 = N->getOperand(0);
4014 SDValue N2 = N->getOperand(1);
4015 if (isGAPlusOffset(N1.getNode(), GA, Offset)) {
4016 if (auto *V = dyn_cast<ConstantSDNode>(N2)) {
4017 Offset += V->getSExtValue();
4018 return true;
4019 }
4020 } else if (isGAPlusOffset(N2.getNode(), GA, Offset)) {
4021 if (auto *V = dyn_cast<ConstantSDNode>(N1)) {
4022 Offset += V->getSExtValue();
4023 return true;
4024 }
4025 }
4026 }
4027
4028 return false;
4029}
4030
4031SDValue TargetLowering::PerformDAGCombine(SDNode *N,
4032 DAGCombinerInfo &DCI) const {
4033 // Default implementation: no optimization.
4034 return SDValue();
4035}
4036
4037//===----------------------------------------------------------------------===//
4038// Inline Assembler Implementation Methods
4039//===----------------------------------------------------------------------===//
4040
4041TargetLowering::ConstraintType
4042TargetLowering::getConstraintType(StringRef Constraint) const {
4043 unsigned S = Constraint.size();
4044
4045 if (S == 1) {
4046 switch (Constraint[0]) {
4047 default: break;
4048 case 'r':
4049 return C_RegisterClass;
4050 case 'm': // memory
4051 case 'o': // offsetable
4052 case 'V': // not offsetable
4053 return C_Memory;
4054 case 'n': // Simple Integer
4055 case 'E': // Floating Point Constant
4056 case 'F': // Floating Point Constant
4057 return C_Immediate;
4058 case 'i': // Simple Integer or Relocatable Constant
4059 case 's': // Relocatable Constant
4060 case 'p': // Address.
4061 case 'X': // Allow ANY value.
4062 case 'I': // Target registers.
4063 case 'J':
4064 case 'K':
4065 case 'L':
4066 case 'M':
4067 case 'N':
4068 case 'O':
4069 case 'P':
4070 case '<':
4071 case '>':
4072 return C_Other;
4073 }
4074 }
4075
4076 if (S > 1 && Constraint[0] == '{' && Constraint[S - 1] == '}') {
4077 if (S == 8 && Constraint.substr(1, 6) == "memory") // "{memory}"
4078 return C_Memory;
4079 return C_Register;
4080 }
4081 return C_Unknown;
4082}
4083
4084/// Try to replace an X constraint, which matches anything, with another that
4085/// has more specific requirements based on the type of the corresponding
4086/// operand.
4087const char *TargetLowering::LowerXConstraint(EVT ConstraintVT) const {
4088 if (ConstraintVT.isInteger())
4089 return "r";
4090 if (ConstraintVT.isFloatingPoint())
4091 return "f"; // works for many targets
4092 return nullptr;
4093}
4094
4095SDValue TargetLowering::LowerAsmOutputForConstraint(
4096 SDValue &Chain, SDValue &Flag, SDLoc DL, const AsmOperandInfo &OpInfo,
4097 SelectionDAG &DAG) const {
4098 return SDValue();
4099}
4100
4101/// Lower the specified operand into the Ops vector.
4102/// If it is invalid, don't add anything to Ops.
4103void TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
4104 std::string &Constraint,
4105 std::vector<SDValue> &Ops,
4106 SelectionDAG &DAG) const {
4107
4108 if (Constraint.length() > 1) return;
4109
4110 char ConstraintLetter = Constraint[0];
4111 switch (ConstraintLetter) {
4112 default: break;
4113 case 'X': // Allows any operand; labels (basic block) use this.
4114 if (Op.getOpcode() == ISD::BasicBlock ||
4115 Op.getOpcode() == ISD::TargetBlockAddress) {
4116 Ops.push_back(Op);
4117 return;
4118 }
4119 LLVM_FALLTHROUGH[[gnu::fallthrough]];
4120 case 'i': // Simple Integer or Relocatable Constant
4121 case 'n': // Simple Integer
4122 case 's': { // Relocatable Constant
4123
4124 GlobalAddressSDNode *GA;
4125 ConstantSDNode *C;
4126 BlockAddressSDNode *BA;
4127 uint64_t Offset = 0;
4128
4129 // Match (GA) or (C) or (GA+C) or (GA-C) or ((GA+C)+C) or (((GA+C)+C)+C),
4130 // etc., since getelementpointer is variadic. We can't use
4131 // SelectionDAG::FoldSymbolOffset because it expects the GA to be accessible
4132 // while in this case the GA may be furthest from the root node which is
4133 // likely an ISD::ADD.
4134 while (1) {
4135 if ((GA = dyn_cast<GlobalAddressSDNode>(Op)) && ConstraintLetter != 'n') {
4136 Ops.push_back(DAG.getTargetGlobalAddress(GA->getGlobal(), SDLoc(Op),
4137 GA->getValueType(0),
4138 Offset + GA->getOffset()));
4139 return;
4140 } else if ((C = dyn_cast<ConstantSDNode>(Op)) &&
4141 ConstraintLetter != 's') {
4142 // gcc prints these as sign extended. Sign extend value to 64 bits
4143 // now; without this it would get ZExt'd later in
4144 // ScheduleDAGSDNodes::EmitNode, which is very generic.
4145 bool IsBool = C->getConstantIntValue()->getBitWidth() == 1;
4146 BooleanContent BCont = getBooleanContents(MVT::i64);
4147 ISD::NodeType ExtOpc = IsBool ? getExtendForContent(BCont)
4148 : ISD::SIGN_EXTEND;
4149 int64_t ExtVal = ExtOpc == ISD::ZERO_EXTEND ? C->getZExtValue()
4150 : C->getSExtValue();
4151 Ops.push_back(DAG.getTargetConstant(Offset + ExtVal,
4152 SDLoc(C), MVT::i64));
4153 return;
4154 } else if ((BA = dyn_cast<BlockAddressSDNode>(Op)) &&
4155 ConstraintLetter != 'n') {
4156 Ops.push_back(DAG.getTargetBlockAddress(
4157 BA->getBlockAddress(), BA->getValueType(0),
4158 Offset + BA->getOffset(), BA->getTargetFlags()));
4159 return;
4160 } else {
4161 const unsigned OpCode = Op.getOpcode();
4162 if (OpCode == ISD::ADD || OpCode == ISD::SUB) {
4163 if ((C = dyn_cast<ConstantSDNode>(Op.getOperand(0))))
4164 Op = Op.getOperand(1);
4165 // Subtraction is not commutative.
4166 else if (OpCode == ISD::ADD &&
4167 (C = dyn_cast<ConstantSDNode>(Op.getOperand(1))))
4168 Op = Op.getOperand(0);
4169 else
4170 return;
4171 Offset += (OpCode == ISD::ADD ? 1 : -1) * C->getSExtValue();
4172 continue;
4173 }
4174 }
4175 return;
4176 }
4177 break;
4178 }
4179 }
4180}
4181
4182std::pair<unsigned, const TargetRegisterClass *>
4183TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *RI,
4184 StringRef Constraint,
4185 MVT VT) const {
4186 if (Constraint.empty() || Constraint[0] != '{')
4187 return std::make_pair(0u, static_cast<TargetRegisterClass *>(nullptr));
4188 assert(*(Constraint.end() - 1) == '}' && "Not a brace enclosed constraint?")((*(Constraint.end() - 1) == '}' && "Not a brace enclosed constraint?"
) ? static_cast<void> (0) : __assert_fail ("*(Constraint.end() - 1) == '}' && \"Not a brace enclosed constraint?\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4188, __PRETTY_FUNCTION__))
;
4189
4190 // Remove the braces from around the name.
4191 StringRef RegName(Constraint.data() + 1, Constraint.size() - 2);
4192
4193 std::pair<unsigned, const TargetRegisterClass *> R =
4194 std::make_pair(0u, static_cast<const TargetRegisterClass *>(nullptr));
4195
4196 // Figure out which register class contains this reg.
4197 for (const TargetRegisterClass *RC : RI->regclasses()) {
4198 // If none of the value types for this register class are valid, we
4199 // can't use it. For example, 64-bit reg classes on 32-bit targets.
4200 if (!isLegalRC(*RI, *RC))
4201 continue;
4202
4203 for (TargetRegisterClass::iterator I = RC->begin(), E = RC->end();
4204 I != E; ++I) {
4205 if (RegName.equals_lower(RI->getRegAsmName(*I))) {
4206 std::pair<unsigned, const TargetRegisterClass *> S =
4207 std::make_pair(*I, RC);
4208
4209 // If this register class has the requested value type, return it,
4210 // otherwise keep searching and return the first class found
4211 // if no other is found which explicitly has the requested type.
4212 if (RI->isTypeLegalForClass(*RC, VT))
4213 return S;
4214 if (!R.second)
4215 R = S;
4216 }
4217 }
4218 }
4219
4220 return R;
4221}
4222
4223//===----------------------------------------------------------------------===//
4224// Constraint Selection.
4225
4226/// Return true of this is an input operand that is a matching constraint like
4227/// "4".
4228bool TargetLowering::AsmOperandInfo::isMatchingInputConstraint() const {
4229 assert(!ConstraintCode.empty() && "No known constraint!")((!ConstraintCode.empty() && "No known constraint!") ?
static_cast<void> (0) : __assert_fail ("!ConstraintCode.empty() && \"No known constraint!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4229, __PRETTY_FUNCTION__))
;
4230 return isdigit(static_cast<unsigned char>(ConstraintCode[0]));
4231}
4232
4233/// If this is an input matching constraint, this method returns the output
4234/// operand it matches.
4235unsigned TargetLowering::AsmOperandInfo::getMatchedOperand() const {
4236 assert(!ConstraintCode.empty() && "No known constraint!")((!ConstraintCode.empty() && "No known constraint!") ?
static_cast<void> (0) : __assert_fail ("!ConstraintCode.empty() && \"No known constraint!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4236, __PRETTY_FUNCTION__))
;
4237 return atoi(ConstraintCode.c_str());
4238}
4239
4240/// Split up the constraint string from the inline assembly value into the
4241/// specific constraints and their prefixes, and also tie in the associated
4242/// operand values.
4243/// If this returns an empty vector, and if the constraint string itself
4244/// isn't empty, there was an error parsing.
4245TargetLowering::AsmOperandInfoVector
4246TargetLowering::ParseConstraints(const DataLayout &DL,
4247 const TargetRegisterInfo *TRI,
4248 ImmutableCallSite CS) const {
4249 /// Information about all of the constraints.
4250 AsmOperandInfoVector ConstraintOperands;
4251 const InlineAsm *IA = cast<InlineAsm>(CS.getCalledValue());
4252 unsigned maCount = 0; // Largest number of multiple alternative constraints.
4253
4254 // Do a prepass over the constraints, canonicalizing them, and building up the
4255 // ConstraintOperands list.
4256 unsigned ArgNo = 0; // ArgNo - The argument of the CallInst.
4257 unsigned ResNo = 0; // ResNo - The result number of the next output.
4258
4259 for (InlineAsm::ConstraintInfo &CI : IA->ParseConstraints()) {
4260 ConstraintOperands.emplace_back(std::move(CI));
4261 AsmOperandInfo &OpInfo = ConstraintOperands.back();
4262
4263 // Update multiple alternative constraint count.
4264 if (OpInfo.multipleAlternatives.size() > maCount)
4265 maCount = OpInfo.multipleAlternatives.size();
4266
4267 OpInfo.ConstraintVT = MVT::Other;
4268
4269 // Compute the value type for each operand.
4270 switch (OpInfo.Type) {
4271 case InlineAsm::isOutput:
4272 // Indirect outputs just consume an argument.
4273 if (OpInfo.isIndirect) {
4274 OpInfo.CallOperandVal = const_cast<Value *>(CS.getArgument(ArgNo++));
4275 break;
4276 }
4277
4278 // The return value of the call is this value. As such, there is no
4279 // corresponding argument.
4280 assert(!CS.getType()->isVoidTy() &&((!CS.getType()->isVoidTy() && "Bad inline asm!") ?
static_cast<void> (0) : __assert_fail ("!CS.getType()->isVoidTy() && \"Bad inline asm!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4281, __PRETTY_FUNCTION__))
4281 "Bad inline asm!")((!CS.getType()->isVoidTy() && "Bad inline asm!") ?
static_cast<void> (0) : __assert_fail ("!CS.getType()->isVoidTy() && \"Bad inline asm!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4281, __PRETTY_FUNCTION__))
;
4282 if (StructType *STy = dyn_cast<StructType>(CS.getType())) {
4283 OpInfo.ConstraintVT =
4284 getSimpleValueType(DL, STy->getElementType(ResNo));
4285 } else {
4286 assert(ResNo == 0 && "Asm only has one result!")((ResNo == 0 && "Asm only has one result!") ? static_cast
<void> (0) : __assert_fail ("ResNo == 0 && \"Asm only has one result!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4286, __PRETTY_FUNCTION__))
;
4287 OpInfo.ConstraintVT = getSimpleValueType(DL, CS.getType());
4288 }
4289 ++ResNo;
4290 break;
4291 case InlineAsm::isInput:
4292 OpInfo.CallOperandVal = const_cast<Value *>(CS.getArgument(ArgNo++));
4293 break;
4294 case InlineAsm::isClobber:
4295 // Nothing to do.
4296 break;
4297 }
4298
4299 if (OpInfo.CallOperandVal) {
4300 llvm::Type *OpTy = OpInfo.CallOperandVal->getType();
4301 if (OpInfo.isIndirect) {
4302 llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy);
4303 if (!PtrTy)
4304 report_fatal_error("Indirect operand for inline asm not a pointer!");
4305 OpTy = PtrTy->getElementType();
4306 }
4307
4308 // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
4309 if (StructType *STy = dyn_cast<StructType>(OpTy))
4310 if (STy->getNumElements() == 1)
4311 OpTy = STy->getElementType(0);
4312
4313 // If OpTy is not a single value, it may be a struct/union that we
4314 // can tile with integers.
4315 if (!OpTy->isSingleValueType() && OpTy->isSized()) {
4316 unsigned BitSize = DL.getTypeSizeInBits(OpTy);
4317 switch (BitSize) {
4318 default: break;
4319 case 1:
4320 case 8:
4321 case 16:
4322 case 32:
4323 case 64:
4324 case 128:
4325 OpInfo.ConstraintVT =
4326 MVT::getVT(IntegerType::get(OpTy->getContext(), BitSize), true);
4327 break;
4328 }
4329 } else if (PointerType *PT = dyn_cast<PointerType>(OpTy)) {
4330 unsigned PtrSize = DL.getPointerSizeInBits(PT->getAddressSpace());
4331 OpInfo.ConstraintVT = MVT::getIntegerVT(PtrSize);
4332 } else {
4333 OpInfo.ConstraintVT = MVT::getVT(OpTy, true);
4334 }
4335 }
4336 }
4337
4338 // If we have multiple alternative constraints, select the best alternative.
4339 if (!ConstraintOperands.empty()) {
4340 if (maCount) {
4341 unsigned bestMAIndex = 0;
4342 int bestWeight = -1;
4343 // weight: -1 = invalid match, and 0 = so-so match to 5 = good match.
4344 int weight = -1;
4345 unsigned maIndex;
4346 // Compute the sums of the weights for each alternative, keeping track
4347 // of the best (highest weight) one so far.
4348 for (maIndex = 0; maIndex < maCount; ++maIndex) {
4349 int weightSum = 0;
4350 for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
4351 cIndex != eIndex; ++cIndex) {
4352 AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];
4353 if (OpInfo.Type == InlineAsm::isClobber)
4354 continue;
4355
4356 // If this is an output operand with a matching input operand,
4357 // look up the matching input. If their types mismatch, e.g. one
4358 // is an integer, the other is floating point, or their sizes are
4359 // different, flag it as an maCantMatch.
4360 if (OpInfo.hasMatchingInput()) {
4361 AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
4362 if (OpInfo.ConstraintVT != Input.ConstraintVT) {
4363 if ((OpInfo.ConstraintVT.isInteger() !=
4364 Input.ConstraintVT.isInteger()) ||
4365 (OpInfo.ConstraintVT.getSizeInBits() !=
4366 Input.ConstraintVT.getSizeInBits())) {
4367 weightSum = -1; // Can't match.
4368 break;
4369 }
4370 }
4371 }
4372 weight = getMultipleConstraintMatchWeight(OpInfo, maIndex);
4373 if (weight == -1) {
4374 weightSum = -1;
4375 break;
4376 }
4377 weightSum += weight;
4378 }
4379 // Update best.
4380 if (weightSum > bestWeight) {
4381 bestWeight = weightSum;
4382 bestMAIndex = maIndex;
4383 }
4384 }
4385
4386 // Now select chosen alternative in each constraint.
4387 for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
4388 cIndex != eIndex; ++cIndex) {
4389 AsmOperandInfo &cInfo = ConstraintOperands[cIndex];
4390 if (cInfo.Type == InlineAsm::isClobber)
4391 continue;
4392 cInfo.selectAlternative(bestMAIndex);
4393 }
4394 }
4395 }
4396
4397 // Check and hook up tied operands, choose constraint code to use.
4398 for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
4399 cIndex != eIndex; ++cIndex) {
4400 AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];
4401
4402 // If this is an output operand with a matching input operand, look up the
4403 // matching input. If their types mismatch, e.g. one is an integer, the
4404 // other is floating point, or their sizes are different, flag it as an
4405 // error.
4406 if (OpInfo.hasMatchingInput()) {
4407 AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
4408
4409 if (OpInfo.ConstraintVT != Input.ConstraintVT) {
4410 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
4411 getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,
4412 OpInfo.ConstraintVT);
4413 std::pair<unsigned, const TargetRegisterClass *> InputRC =
4414 getRegForInlineAsmConstraint(TRI, Input.ConstraintCode,
4415 Input.ConstraintVT);
4416 if ((OpInfo.ConstraintVT.isInteger() !=
4417 Input.ConstraintVT.isInteger()) ||
4418 (MatchRC.second != InputRC.second)) {
4419 report_fatal_error("Unsupported asm: input constraint"
4420 " with a matching output constraint of"
4421 " incompatible type!");
4422 }
4423 }
4424 }
4425 }
4426
4427 return ConstraintOperands;
4428}
4429
4430/// Return an integer indicating how general CT is.
4431static unsigned getConstraintGenerality(TargetLowering::ConstraintType CT) {
4432 switch (CT) {
4433 case TargetLowering::C_Immediate:
4434 case TargetLowering::C_Other:
4435 case TargetLowering::C_Unknown:
4436 return 0;
4437 case TargetLowering::C_Register:
4438 return 1;
4439 case TargetLowering::C_RegisterClass:
4440 return 2;
4441 case TargetLowering::C_Memory:
4442 return 3;
4443 }
4444 llvm_unreachable("Invalid constraint type")::llvm::llvm_unreachable_internal("Invalid constraint type", "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4444)
;
4445}
4446
4447/// Examine constraint type and operand type and determine a weight value.
4448/// This object must already have been set up with the operand type
4449/// and the current alternative constraint selected.
4450TargetLowering::ConstraintWeight
4451 TargetLowering::getMultipleConstraintMatchWeight(
4452 AsmOperandInfo &info, int maIndex) const {
4453 InlineAsm::ConstraintCodeVector *rCodes;
4454 if (maIndex >= (int)info.multipleAlternatives.size())
4455 rCodes = &info.Codes;
4456 else
4457 rCodes = &info.multipleAlternatives[maIndex].Codes;
4458 ConstraintWeight BestWeight = CW_Invalid;
4459
4460 // Loop over the options, keeping track of the most general one.
4461 for (unsigned i = 0, e = rCodes->size(); i != e; ++i) {
4462 ConstraintWeight weight =
4463 getSingleConstraintMatchWeight(info, (*rCodes)[i].c_str());
4464 if (weight > BestWeight)
4465 BestWeight = weight;
4466 }
4467
4468 return BestWeight;
4469}
4470
4471/// Examine constraint type and operand type and determine a weight value.
4472/// This object must already have been set up with the operand type
4473/// and the current alternative constraint selected.
4474TargetLowering::ConstraintWeight
4475 TargetLowering::getSingleConstraintMatchWeight(
4476 AsmOperandInfo &info, const char *constraint) const {
4477 ConstraintWeight weight = CW_Invalid;
4478 Value *CallOperandVal = info.CallOperandVal;
4479 // If we don't have a value, we can't do a match,
4480 // but allow it at the lowest weight.
4481 if (!CallOperandVal)
4482 return CW_Default;
4483 // Look at the constraint type.
4484 switch (*constraint) {
4485 case 'i': // immediate integer.
4486 case 'n': // immediate integer with a known value.
4487 if (isa<ConstantInt>(CallOperandVal))
4488 weight = CW_Constant;
4489 break;
4490 case 's': // non-explicit intregal immediate.
4491 if (isa<GlobalValue>(CallOperandVal))
4492 weight = CW_Constant;
4493 break;
4494 case 'E': // immediate float if host format.
4495 case 'F': // immediate float.
4496 if (isa<ConstantFP>(CallOperandVal))
4497 weight = CW_Constant;
4498 break;
4499 case '<': // memory operand with autodecrement.
4500 case '>': // memory operand with autoincrement.
4501 case 'm': // memory operand.
4502 case 'o': // offsettable memory operand
4503 case 'V': // non-offsettable memory operand
4504 weight = CW_Memory;
4505 break;
4506 case 'r': // general register.
4507 case 'g': // general register, memory operand or immediate integer.
4508 // note: Clang converts "g" to "imr".
4509 if (CallOperandVal->getType()->isIntegerTy())
4510 weight = CW_Register;
4511 break;
4512 case 'X': // any operand.
4513 default:
4514 weight = CW_Default;
4515 break;
4516 }
4517 return weight;
4518}
4519
4520/// If there are multiple different constraints that we could pick for this
4521/// operand (e.g. "imr") try to pick the 'best' one.
4522/// This is somewhat tricky: constraints fall into four classes:
4523/// Other -> immediates and magic values
4524/// Register -> one specific register
4525/// RegisterClass -> a group of regs
4526/// Memory -> memory
4527/// Ideally, we would pick the most specific constraint possible: if we have
4528/// something that fits into a register, we would pick it. The problem here
4529/// is that if we have something that could either be in a register or in
4530/// memory that use of the register could cause selection of *other*
4531/// operands to fail: they might only succeed if we pick memory. Because of
4532/// this the heuristic we use is:
4533///
4534/// 1) If there is an 'other' constraint, and if the operand is valid for
4535/// that constraint, use it. This makes us take advantage of 'i'
4536/// constraints when available.
4537/// 2) Otherwise, pick the most general constraint present. This prefers
4538/// 'm' over 'r', for example.
4539///
4540static void ChooseConstraint(TargetLowering::AsmOperandInfo &OpInfo,
4541 const TargetLowering &TLI,
4542 SDValue Op, SelectionDAG *DAG) {
4543 assert(OpInfo.Codes.size() > 1 && "Doesn't have multiple constraint options")((OpInfo.Codes.size() > 1 && "Doesn't have multiple constraint options"
) ? static_cast<void> (0) : __assert_fail ("OpInfo.Codes.size() > 1 && \"Doesn't have multiple constraint options\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4543, __PRETTY_FUNCTION__))
;
4544 unsigned BestIdx = 0;
4545 TargetLowering::ConstraintType BestType = TargetLowering::C_Unknown;
4546 int BestGenerality = -1;
4547
4548 // Loop over the options, keeping track of the most general one.
4549 for (unsigned i = 0, e = OpInfo.Codes.size(); i != e; ++i) {
4550 TargetLowering::ConstraintType CType =
4551 TLI.getConstraintType(OpInfo.Codes[i]);
4552
4553 // Indirect 'other' or 'immediate' constraints are not allowed.
4554 if (OpInfo.isIndirect && !(CType == TargetLowering::C_Memory ||
4555 CType == TargetLowering::C_Register ||
4556 CType == TargetLowering::C_RegisterClass))
4557 continue;
4558
4559 // If this is an 'other' or 'immediate' constraint, see if the operand is
4560 // valid for it. For example, on X86 we might have an 'rI' constraint. If
4561 // the operand is an integer in the range [0..31] we want to use I (saving a
4562 // load of a register), otherwise we must use 'r'.
4563 if ((CType == TargetLowering::C_Other ||
4564 CType == TargetLowering::C_Immediate) && Op.getNode()) {
4565 assert(OpInfo.Codes[i].size() == 1 &&((OpInfo.Codes[i].size() == 1 && "Unhandled multi-letter 'other' constraint"
) ? static_cast<void> (0) : __assert_fail ("OpInfo.Codes[i].size() == 1 && \"Unhandled multi-letter 'other' constraint\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4566, __PRETTY_FUNCTION__))
4566 "Unhandled multi-letter 'other' constraint")((OpInfo.Codes[i].size() == 1 && "Unhandled multi-letter 'other' constraint"
) ? static_cast<void> (0) : __assert_fail ("OpInfo.Codes[i].size() == 1 && \"Unhandled multi-letter 'other' constraint\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4566, __PRETTY_FUNCTION__))
;
4567 std::vector<SDValue> ResultOps;
4568 TLI.LowerAsmOperandForConstraint(Op, OpInfo.Codes[i],
4569 ResultOps, *DAG);
4570 if (!ResultOps.empty()) {
4571 BestType = CType;
4572 BestIdx = i;
4573 break;
4574 }
4575 }
4576
4577 // Things with matching constraints can only be registers, per gcc
4578 // documentation. This mainly affects "g" constraints.
4579 if (CType == TargetLowering::C_Memory && OpInfo.hasMatchingInput())
4580 continue;
4581
4582 // This constraint letter is more general than the previous one, use it.
4583 int Generality = getConstraintGenerality(CType);
4584 if (Generality > BestGenerality) {
4585 BestType = CType;
4586 BestIdx = i;
4587 BestGenerality = Generality;
4588 }
4589 }
4590
4591 OpInfo.ConstraintCode = OpInfo.Codes[BestIdx];
4592 OpInfo.ConstraintType = BestType;
4593}
4594
4595/// Determines the constraint code and constraint type to use for the specific
4596/// AsmOperandInfo, setting OpInfo.ConstraintCode and OpInfo.ConstraintType.
4597void TargetLowering::ComputeConstraintToUse(AsmOperandInfo &OpInfo,
4598 SDValue Op,
4599 SelectionDAG *DAG) const {
4600 assert(!OpInfo.Codes.empty() && "Must have at least one constraint")((!OpInfo.Codes.empty() && "Must have at least one constraint"
) ? static_cast<void> (0) : __assert_fail ("!OpInfo.Codes.empty() && \"Must have at least one constraint\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4600, __PRETTY_FUNCTION__))
;
4601
4602 // Single-letter constraints ('r') are very common.
4603 if (OpInfo.Codes.size() == 1) {
4604 OpInfo.ConstraintCode = OpInfo.Codes[0];
4605 OpInfo.ConstraintType = getConstraintType(OpInfo.ConstraintCode);
4606 } else {
4607 ChooseConstraint(OpInfo, *this, Op, DAG);
4608 }
4609
4610 // 'X' matches anything.
4611 if (OpInfo.ConstraintCode == "X" && OpInfo.CallOperandVal) {
4612 // Labels and constants are handled elsewhere ('X' is the only thing
4613 // that matches labels). For Functions, the type here is the type of
4614 // the result, which is not what we want to look at; leave them alone.
4615 Value *v = OpInfo.CallOperandVal;
4616 if (isa<BasicBlock>(v) || isa<ConstantInt>(v) || isa<Function>(v)) {
4617 OpInfo.CallOperandVal = v;
4618 return;
4619 }
4620
4621 if (Op.getNode() && Op.getOpcode() == ISD::TargetBlockAddress)
4622 return;
4623
4624 // Otherwise, try to resolve it to something we know about by looking at
4625 // the actual operand type.
4626 if (const char *Repl = LowerXConstraint(OpInfo.ConstraintVT)) {
4627 OpInfo.ConstraintCode = Repl;
4628 OpInfo.ConstraintType = getConstraintType(OpInfo.ConstraintCode);
4629 }
4630 }
4631}
4632
4633/// Given an exact SDIV by a constant, create a multiplication
4634/// with the multiplicative inverse of the constant.
4635static SDValue BuildExactSDIV(const TargetLowering &TLI, SDNode *N,
4636 const SDLoc &dl, SelectionDAG &DAG,
4637 SmallVectorImpl<SDNode *> &Created) {
4638 SDValue Op0 = N->getOperand(0);
4639 SDValue Op1 = N->getOperand(1);
4640 EVT VT = N->getValueType(0);
4641 EVT SVT = VT.getScalarType();
4642 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
4643 EVT ShSVT = ShVT.getScalarType();
4644
4645 bool UseSRA = false;
4646 SmallVector<SDValue, 16> Shifts, Factors;
4647
4648 auto BuildSDIVPattern = [&](ConstantSDNode *C) {
4649 if (C->isNullValue())
4650 return false;
4651 APInt Divisor = C->getAPIntValue();
4652 unsigned Shift = Divisor.countTrailingZeros();
4653 if (Shift) {
4654 Divisor.ashrInPlace(Shift);
4655 UseSRA = true;
4656 }
4657 // Calculate the multiplicative inverse, using Newton's method.
4658 APInt t;
4659 APInt Factor = Divisor;
4660 while ((t = Divisor * Factor) != 1)
4661 Factor *= APInt(Divisor.getBitWidth(), 2) - t;
4662 Shifts.push_back(DAG.getConstant(Shift, dl, ShSVT));
4663 Factors.push_back(DAG.getConstant(Factor, dl, SVT));
4664 return true;
4665 };
4666
4667 // Collect all magic values from the build vector.
4668 if (!ISD::matchUnaryPredicate(Op1, BuildSDIVPattern))
4669 return SDValue();
4670
4671 SDValue Shift, Factor;
4672 if (VT.isVector()) {
4673 Shift = DAG.getBuildVector(ShVT, dl, Shifts);
4674 Factor = DAG.getBuildVector(VT, dl, Factors);
4675 } else {
4676 Shift = Shifts[0];
4677 Factor = Factors[0];
4678 }
4679
4680 SDValue Res = Op0;
4681
4682 // Shift the value upfront if it is even, so the LSB is one.
4683 if (UseSRA) {
4684 // TODO: For UDIV use SRL instead of SRA.
4685 SDNodeFlags Flags;
4686 Flags.setExact(true);
4687 Res = DAG.getNode(ISD::SRA, dl, VT, Res, Shift, Flags);
4688 Created.push_back(Res.getNode());
4689 }
4690
4691 return DAG.getNode(ISD::MUL, dl, VT, Res, Factor);
4692}
4693
4694SDValue TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
4695 SelectionDAG &DAG,
4696 SmallVectorImpl<SDNode *> &Created) const {
4697 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
4698 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4699 if (TLI.isIntDivCheap(N->getValueType(0), Attr))
4700 return SDValue(N, 0); // Lower SDIV as SDIV
4701 return SDValue();
4702}
4703
4704/// Given an ISD::SDIV node expressing a divide by constant,
4705/// return a DAG expression to select that will generate the same value by
4706/// multiplying by a magic number.
4707/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
4708SDValue TargetLowering::BuildSDIV(SDNode *N, SelectionDAG &DAG,
4709 bool IsAfterLegalization,
4710 SmallVectorImpl<SDNode *> &Created) const {
4711 SDLoc dl(N);
4712 EVT VT = N->getValueType(0);
4713 EVT SVT = VT.getScalarType();
4714 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
4715 EVT ShSVT = ShVT.getScalarType();
4716 unsigned EltBits = VT.getScalarSizeInBits();
4717
4718 // Check to see if we can do this.
4719 // FIXME: We should be more aggressive here.
4720 if (!isTypeLegal(VT))
4721 return SDValue();
4722
4723 // If the sdiv has an 'exact' bit we can use a simpler lowering.
4724 if (N->getFlags().hasExact())
4725 return BuildExactSDIV(*this, N, dl, DAG, Created);
4726
4727 SmallVector<SDValue, 16> MagicFactors, Factors, Shifts, ShiftMasks;
4728
4729 auto BuildSDIVPattern = [&](ConstantSDNode *C) {
4730 if (C->isNullValue())
4731 return false;
4732
4733 const APInt &Divisor = C->getAPIntValue();
4734 APInt::ms magics = Divisor.magic();
4735 int NumeratorFactor = 0;
4736 int ShiftMask = -1;
4737
4738 if (Divisor.isOneValue() || Divisor.isAllOnesValue()) {
4739 // If d is +1/-1, we just multiply the numerator by +1/-1.
4740 NumeratorFactor = Divisor.getSExtValue();
4741 magics.m = 0;
4742 magics.s = 0;
4743 ShiftMask = 0;
4744 } else if (Divisor.isStrictlyPositive() && magics.m.isNegative()) {
4745 // If d > 0 and m < 0, add the numerator.
4746 NumeratorFactor = 1;
4747 } else if (Divisor.isNegative() && magics.m.isStrictlyPositive()) {
4748 // If d < 0 and m > 0, subtract the numerator.
4749 NumeratorFactor = -1;
4750 }
4751
4752 MagicFactors.push_back(DAG.getConstant(magics.m, dl, SVT));
4753 Factors.push_back(DAG.getConstant(NumeratorFactor, dl, SVT));
4754 Shifts.push_back(DAG.getConstant(magics.s, dl, ShSVT));
4755 ShiftMasks.push_back(DAG.getConstant(ShiftMask, dl, SVT));
4756 return true;
4757 };
4758
4759 SDValue N0 = N->getOperand(0);
4760 SDValue N1 = N->getOperand(1);
4761
4762 // Collect the shifts / magic values from each element.
4763 if (!ISD::matchUnaryPredicate(N1, BuildSDIVPattern))
4764 return SDValue();
4765
4766 SDValue MagicFactor, Factor, Shift, ShiftMask;
4767 if (VT.isVector()) {
4768 MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);
4769 Factor = DAG.getBuildVector(VT, dl, Factors);
4770 Shift = DAG.getBuildVector(ShVT, dl, Shifts);
4771 ShiftMask = DAG.getBuildVector(VT, dl, ShiftMasks);
4772 } else {
4773 MagicFactor = MagicFactors[0];
4774 Factor = Factors[0];
4775 Shift = Shifts[0];
4776 ShiftMask = ShiftMasks[0];
4777 }
4778
4779 // Multiply the numerator (operand 0) by the magic value.
4780 // FIXME: We should support doing a MUL in a wider type.
4781 SDValue Q;
4782 if (IsAfterLegalization ? isOperationLegal(ISD::MULHS, VT)
4783 : isOperationLegalOrCustom(ISD::MULHS, VT))
4784 Q = DAG.getNode(ISD::MULHS, dl, VT, N0, MagicFactor);
4785 else if (IsAfterLegalization ? isOperationLegal(ISD::SMUL_LOHI, VT)
4786 : isOperationLegalOrCustom(ISD::SMUL_LOHI, VT)) {
4787 SDValue LoHi =
4788 DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT), N0, MagicFactor);
4789 Q = SDValue(LoHi.getNode(), 1);
4790 } else
4791 return SDValue(); // No mulhs or equivalent.
4792 Created.push_back(Q.getNode());
4793
4794 // (Optionally) Add/subtract the numerator using Factor.
4795 Factor = DAG.getNode(ISD::MUL, dl, VT, N0, Factor);
4796 Created.push_back(Factor.getNode());
4797 Q = DAG.getNode(ISD::ADD, dl, VT, Q, Factor);
4798 Created.push_back(Q.getNode());
4799
4800 // Shift right algebraic by shift value.
4801 Q = DAG.getNode(ISD::SRA, dl, VT, Q, Shift);
4802 Created.push_back(Q.getNode());
4803
4804 // Extract the sign bit, mask it and add it to the quotient.
4805 SDValue SignShift = DAG.getConstant(EltBits - 1, dl, ShVT);
4806 SDValue T = DAG.getNode(ISD::SRL, dl, VT, Q, SignShift);
4807 Created.push_back(T.getNode());
4808 T = DAG.getNode(ISD::AND, dl, VT, T, ShiftMask);
4809 Created.push_back(T.getNode());
4810 return DAG.getNode(ISD::ADD, dl, VT, Q, T);
4811}
4812
4813/// Given an ISD::UDIV node expressing a divide by constant,
4814/// return a DAG expression to select that will generate the same value by
4815/// multiplying by a magic number.
4816/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
4817SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG,
4818 bool IsAfterLegalization,
4819 SmallVectorImpl<SDNode *> &Created) const {
4820 SDLoc dl(N);
4821 EVT VT = N->getValueType(0);
4822 EVT SVT = VT.getScalarType();
4823 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
4824 EVT ShSVT = ShVT.getScalarType();
4825 unsigned EltBits = VT.getScalarSizeInBits();
4826
4827 // Check to see if we can do this.
4828 // FIXME: We should be more aggressive here.
4829 if (!isTypeLegal(VT))
4830 return SDValue();
4831
4832 bool UseNPQ = false;
4833 SmallVector<SDValue, 16> PreShifts, PostShifts, MagicFactors, NPQFactors;
4834
4835 auto BuildUDIVPattern = [&](ConstantSDNode *C) {
4836 if (C->isNullValue())
4837 return false;
4838 // FIXME: We should use a narrower constant when the upper
4839 // bits are known to be zero.
4840 APInt Divisor = C->getAPIntValue();
4841 APInt::mu magics = Divisor.magicu();
4842 unsigned PreShift = 0, PostShift = 0;
4843
4844 // If the divisor is even, we can avoid using the expensive fixup by
4845 // shifting the divided value upfront.
4846 if (magics.a != 0 && !Divisor[0]) {
4847 PreShift = Divisor.countTrailingZeros();
4848 // Get magic number for the shifted divisor.
4849 magics = Divisor.lshr(PreShift).magicu(PreShift);
4850 assert(magics.a == 0 && "Should use cheap fixup now")((magics.a == 0 && "Should use cheap fixup now") ? static_cast
<void> (0) : __assert_fail ("magics.a == 0 && \"Should use cheap fixup now\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4850, __PRETTY_FUNCTION__))
;
4851 }
4852
4853 APInt Magic = magics.m;
4854
4855 unsigned SelNPQ;
4856 if (magics.a == 0 || Divisor.isOneValue()) {
4857 assert(magics.s < Divisor.getBitWidth() &&((magics.s < Divisor.getBitWidth() && "We shouldn't generate an undefined shift!"
) ? static_cast<void> (0) : __assert_fail ("magics.s < Divisor.getBitWidth() && \"We shouldn't generate an undefined shift!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4858, __PRETTY_FUNCTION__))
4858 "We shouldn't generate an undefined shift!")((magics.s < Divisor.getBitWidth() && "We shouldn't generate an undefined shift!"
) ? static_cast<void> (0) : __assert_fail ("magics.s < Divisor.getBitWidth() && \"We shouldn't generate an undefined shift!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4858, __PRETTY_FUNCTION__))
;
4859 PostShift = magics.s;
4860 SelNPQ = false;
4861 } else {
4862 PostShift = magics.s - 1;
4863 SelNPQ = true;
4864 }
4865
4866 PreShifts.push_back(DAG.getConstant(PreShift, dl, ShSVT));
4867 MagicFactors.push_back(DAG.getConstant(Magic, dl, SVT));
4868 NPQFactors.push_back(
4869 DAG.getConstant(SelNPQ ? APInt::getOneBitSet(EltBits, EltBits - 1)
4870 : APInt::getNullValue(EltBits),
4871 dl, SVT));
4872 PostShifts.push_back(DAG.getConstant(PostShift, dl, ShSVT));
4873 UseNPQ |= SelNPQ;
4874 return true;
4875 };
4876
4877 SDValue N0 = N->getOperand(0);
4878 SDValue N1 = N->getOperand(1);
4879
4880 // Collect the shifts/magic values from each element.
4881 if (!ISD::matchUnaryPredicate(N1, BuildUDIVPattern))
4882 return SDValue();
4883
4884 SDValue PreShift, PostShift, MagicFactor, NPQFactor;
4885 if (VT.isVector()) {
4886 PreShift = DAG.getBuildVector(ShVT, dl, PreShifts);
4887 MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);
4888 NPQFactor = DAG.getBuildVector(VT, dl, NPQFactors);
4889 PostShift = DAG.getBuildVector(ShVT, dl, PostShifts);
4890 } else {
4891 PreShift = PreShifts[0];
4892 MagicFactor = MagicFactors[0];
4893 PostShift = PostShifts[0];
4894 }
4895
4896 SDValue Q = N0;
4897 Q = DAG.getNode(ISD::SRL, dl, VT, Q, PreShift);
4898 Created.push_back(Q.getNode());
4899
4900 // FIXME: We should support doing a MUL in a wider type.
4901 auto GetMULHU = [&](SDValue X, SDValue Y) {
4902 if (IsAfterLegalization ? isOperationLegal(ISD::MULHU, VT)
4903 : isOperationLegalOrCustom(ISD::MULHU, VT))
4904 return DAG.getNode(ISD::MULHU, dl, VT, X, Y);
4905 if (IsAfterLegalization ? isOperationLegal(ISD::UMUL_LOHI, VT)
4906 : isOperationLegalOrCustom(ISD::UMUL_LOHI, VT)) {
4907 SDValue LoHi =
4908 DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);
4909 return SDValue(LoHi.getNode(), 1);
4910 }
4911 return SDValue(); // No mulhu or equivalent
4912 };
4913
4914 // Multiply the numerator (operand 0) by the magic value.
4915 Q = GetMULHU(Q, MagicFactor);
4916 if (!Q)
4917 return SDValue();
4918
4919 Created.push_back(Q.getNode());
4920
4921 if (UseNPQ) {
4922 SDValue NPQ = DAG.getNode(ISD::SUB, dl, VT, N0, Q);
4923 Created.push_back(NPQ.getNode());
4924
4925 // For vectors we might have a mix of non-NPQ/NPQ paths, so use
4926 // MULHU to act as a SRL-by-1 for NPQ, else multiply by zero.
4927 if (VT.isVector())
4928 NPQ = GetMULHU(NPQ, NPQFactor);
4929 else
4930 NPQ = DAG.getNode(ISD::SRL, dl, VT, NPQ, DAG.getConstant(1, dl, ShVT));
4931
4932 Created.push_back(NPQ.getNode());
4933
4934 Q = DAG.getNode(ISD::ADD, dl, VT, NPQ, Q);
4935 Created.push_back(Q.getNode());
4936 }
4937
4938 Q = DAG.getNode(ISD::SRL, dl, VT, Q, PostShift);
4939 Created.push_back(Q.getNode());
4940
4941 SDValue One = DAG.getConstant(1, dl, VT);
4942 SDValue IsOne = DAG.getSetCC(dl, VT, N1, One, ISD::SETEQ);
4943 return DAG.getSelect(dl, VT, IsOne, N0, Q);
4944}
4945
4946/// If all values in Values that *don't* match the predicate are same 'splat'
4947/// value, then replace all values with that splat value.
4948/// Else, if AlternativeReplacement was provided, then replace all values that
4949/// do match predicate with AlternativeReplacement value.
4950static void
4951turnVectorIntoSplatVector(MutableArrayRef<SDValue> Values,
4952 std::function<bool(SDValue)> Predicate,
4953 SDValue AlternativeReplacement = SDValue()) {
4954 SDValue Replacement;
4955 // Is there a value for which the Predicate does *NOT* match? What is it?
4956 auto SplatValue = llvm::find_if_not(Values, Predicate);
4957 if (SplatValue != Values.end()) {
4958 // Does Values consist only of SplatValue's and values matching Predicate?
4959 if (llvm::all_of(Values, [Predicate, SplatValue](SDValue Value) {
4960 return Value == *SplatValue || Predicate(Value);
4961 })) // Then we shall replace values matching predicate with SplatValue.
4962 Replacement = *SplatValue;
4963 }
4964 if (!Replacement) {
4965 // Oops, we did not find the "baseline" splat value.
4966 if (!AlternativeReplacement)
4967 return; // Nothing to do.
4968 // Let's replace with provided value then.
4969 Replacement = AlternativeReplacement;
4970 }
4971 std::replace_if(Values.begin(), Values.end(), Predicate, Replacement);
4972}
4973
4974/// Given an ISD::UREM used only by an ISD::SETEQ or ISD::SETNE
4975/// where the divisor is constant and the comparison target is zero,
4976/// return a DAG expression that will generate the same comparison result
4977/// using only multiplications, additions and shifts/rotations.
4978/// Ref: "Hacker's Delight" 10-17.
4979SDValue TargetLowering::buildUREMEqFold(EVT SETCCVT, SDValue REMNode,
4980 SDValue CompTargetNode,
4981 ISD::CondCode Cond,
4982 DAGCombinerInfo &DCI,
4983 const SDLoc &DL) const {
4984 SmallVector<SDNode *, 5> Built;
4985 if (SDValue Folded = prepareUREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
4986 DCI, DL, Built)) {
4987 for (SDNode *N : Built)
4988 DCI.AddToWorklist(N);
4989 return Folded;
4990 }
4991
4992 return SDValue();
4993}
4994
4995SDValue
4996TargetLowering::prepareUREMEqFold(EVT SETCCVT, SDValue REMNode,
4997 SDValue CompTargetNode, ISD::CondCode Cond,
4998 DAGCombinerInfo &DCI, const SDLoc &DL,
4999 SmallVectorImpl<SDNode *> &Created) const {
5000 // fold (seteq/ne (urem N, D), 0) -> (setule/ugt (rotr (mul N, P), K), Q)
5001 // - D must be constant, with D = D0 * 2^K where D0 is odd
5002 // - P is the multiplicative inverse of D0 modulo 2^W
5003 // - Q = floor(((2^W) - 1) / D)
5004 // where W is the width of the common type of N and D.
5005 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&(((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Only applicable for (in)equality comparisons."
) ? static_cast<void> (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5006, __PRETTY_FUNCTION__))
5006 "Only applicable for (in)equality comparisons.")(((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Only applicable for (in)equality comparisons."
) ? static_cast<void> (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5006, __PRETTY_FUNCTION__))
;
5007
5008 SelectionDAG &DAG = DCI.DAG;
5009
5010 EVT VT = REMNode.getValueType();
5011 EVT SVT = VT.getScalarType();
5012 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
5013 EVT ShSVT = ShVT.getScalarType();
5014
5015 // If MUL is unavailable, we cannot proceed in any case.
5016 if (!isOperationLegalOrCustom(ISD::MUL, VT))
5017 return SDValue();
5018
5019 bool ComparingWithAllZeros = true;
5020 bool AllComparisonsWithNonZerosAreTautological = true;
5021 bool HadTautologicalLanes = false;
5022 bool AllLanesAreTautological = true;
5023 bool HadEvenDivisor = false;
5024 bool AllDivisorsArePowerOfTwo = true;
5025 bool HadTautologicalInvertedLanes = false;
5026 SmallVector<SDValue, 16> PAmts, KAmts, QAmts, IAmts;
5027
5028 auto BuildUREMPattern = [&](ConstantSDNode *CDiv, ConstantSDNode *CCmp) {
5029 // Division by 0 is UB. Leave it to be constant-folded elsewhere.
5030 if (CDiv->isNullValue())
5031 return false;
5032
5033 const APInt &D = CDiv->getAPIntValue();
5034 const APInt &Cmp = CCmp->getAPIntValue();
5035
5036 ComparingWithAllZeros &= Cmp.isNullValue();
5037
5038 // x u% C1` is *always* less than C1. So given `x u% C1 == C2`,
5039 // if C2 is not less than C1, the comparison is always false.
5040 // But we will only be able to produce the comparison that will give the
5041 // opposive tautological answer. So this lane would need to be fixed up.
5042 bool TautologicalInvertedLane = D.ule(Cmp);
5043 HadTautologicalInvertedLanes |= TautologicalInvertedLane;
5044
5045 // If all lanes are tautological (either all divisors are ones, or divisor
5046 // is not greater than the constant we are comparing with),
5047 // we will prefer to avoid the fold.
5048 bool TautologicalLane = D.isOneValue() || TautologicalInvertedLane;
5049 HadTautologicalLanes |= TautologicalLane;
5050 AllLanesAreTautological &= TautologicalLane;
5051
5052 // If we are comparing with non-zero, we need'll need to subtract said
5053 // comparison value from the LHS. But there is no point in doing that if
5054 // every lane where we are comparing with non-zero is tautological..
5055 if (!Cmp.isNullValue())
5056 AllComparisonsWithNonZerosAreTautological &= TautologicalLane;
5057
5058 // Decompose D into D0 * 2^K
5059 unsigned K = D.countTrailingZeros();
5060 assert((!D.isOneValue() || (K == 0)) && "For divisor '1' we won't rotate.")(((!D.isOneValue() || (K == 0)) && "For divisor '1' we won't rotate."
) ? static_cast<void> (0) : __assert_fail ("(!D.isOneValue() || (K == 0)) && \"For divisor '1' we won't rotate.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5060, __PRETTY_FUNCTION__))
;
5061 APInt D0 = D.lshr(K);
5062
5063 // D is even if it has trailing zeros.
5064 HadEvenDivisor |= (K != 0);
5065 // D is a power-of-two if D0 is one.
5066 // If all divisors are power-of-two, we will prefer to avoid the fold.
5067 AllDivisorsArePowerOfTwo &= D0.isOneValue();
5068
5069 // P = inv(D0, 2^W)
5070 // 2^W requires W + 1 bits, so we have to extend and then truncate.
5071 unsigned W = D.getBitWidth();
5072 APInt P = D0.zext(W + 1)
5073 .multiplicativeInverse(APInt::getSignedMinValue(W + 1))
5074 .trunc(W);
5075 assert(!P.isNullValue() && "No multiplicative inverse!")((!P.isNullValue() && "No multiplicative inverse!") ?
static_cast<void> (0) : __assert_fail ("!P.isNullValue() && \"No multiplicative inverse!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5075, __PRETTY_FUNCTION__))
; // unreachable
5076 assert((D0 * P).isOneValue() && "Multiplicative inverse sanity check.")(((D0 * P).isOneValue() && "Multiplicative inverse sanity check."
) ? static_cast<void> (0) : __assert_fail ("(D0 * P).isOneValue() && \"Multiplicative inverse sanity check.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5076, __PRETTY_FUNCTION__))
;
5077
5078 // Q = floor((2^W - 1) u/ D)
5079 // R = ((2^W - 1) u% D)
5080 APInt Q, R;
5081 APInt::udivrem(APInt::getAllOnesValue(W), D, Q, R);
5082
5083 // If we are comparing with zero, then that comparison constant is okay,
5084 // else it may need to be one less than that.
5085 if (Cmp.ugt(R))
5086 Q -= 1;
5087
5088 assert(APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) &&((APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) &&
"We are expecting that K is always less than all-ones for ShSVT"
) ? static_cast<void> (0) : __assert_fail ("APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) && \"We are expecting that K is always less than all-ones for ShSVT\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5089, __PRETTY_FUNCTION__))
5089 "We are expecting that K is always less than all-ones for ShSVT")((APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) &&
"We are expecting that K is always less than all-ones for ShSVT"
) ? static_cast<void> (0) : __assert_fail ("APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) && \"We are expecting that K is always less than all-ones for ShSVT\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5089, __PRETTY_FUNCTION__))
;
5090
5091 // If the lane is tautological the result can be constant-folded.
5092 if (TautologicalLane) {
5093 // Set P and K amount to a bogus values so we can try to splat them.
5094 P = 0;
5095 K = -1;
5096 // And ensure that comparison constant is tautological,
5097 // it will always compare true/false.
5098 Q = -1;
5099 }
5100
5101 PAmts.push_back(DAG.getConstant(P, DL, SVT));
5102 KAmts.push_back(
5103 DAG.getConstant(APInt(ShSVT.getSizeInBits(), K), DL, ShSVT));
5104 QAmts.push_back(DAG.getConstant(Q, DL, SVT));
5105 return true;
5106 };
5107
5108 SDValue N = REMNode.getOperand(0);
5109 SDValue D = REMNode.getOperand(1);
5110
5111 // Collect the values from each element.
5112 if (!ISD::matchBinaryPredicate(D, CompTargetNode, BuildUREMPattern))
5113 return SDValue();
5114
5115 // If all lanes are tautological, the result can be constant-folded.
5116 if (AllLanesAreTautological)
5117 return SDValue();
5118
5119 // If this is a urem by a powers-of-two, avoid the fold since it can be
5120 // best implemented as a bit test.
5121 if (AllDivisorsArePowerOfTwo)
5122 return SDValue();
5123
5124 SDValue PVal, KVal, QVal;
5125 if (VT.isVector()) {
5126 if (HadTautologicalLanes) {
5127 // Try to turn PAmts into a splat, since we don't care about the values
5128 // that are currently '0'. If we can't, just keep '0'`s.
5129 turnVectorIntoSplatVector(PAmts, isNullConstant);
5130 // Try to turn KAmts into a splat, since we don't care about the values
5131 // that are currently '-1'. If we can't, change them to '0'`s.
5132 turnVectorIntoSplatVector(KAmts, isAllOnesConstant,
5133 DAG.getConstant(0, DL, ShSVT));
5134 }
5135
5136 PVal = DAG.getBuildVector(VT, DL, PAmts);
5137 KVal = DAG.getBuildVector(ShVT, DL, KAmts);
5138 QVal = DAG.getBuildVector(VT, DL, QAmts);
5139 } else {
5140 PVal = PAmts[0];
5141 KVal = KAmts[0];
5142 QVal = QAmts[0];
5143 }
5144
5145 if (!ComparingWithAllZeros && !AllComparisonsWithNonZerosAreTautological) {
5146 if (!isOperationLegalOrCustom(ISD::SUB, VT))
5147 return SDValue(); // FIXME: Could/should use `ISD::ADD`?
5148 assert(CompTargetNode.getValueType() == N.getValueType() &&((CompTargetNode.getValueType() == N.getValueType() &&
"Expecting that the types on LHS and RHS of comparisons match."
) ? static_cast<void> (0) : __assert_fail ("CompTargetNode.getValueType() == N.getValueType() && \"Expecting that the types on LHS and RHS of comparisons match.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5149, __PRETTY_FUNCTION__))
5149 "Expecting that the types on LHS and RHS of comparisons match.")((CompTargetNode.getValueType() == N.getValueType() &&
"Expecting that the types on LHS and RHS of comparisons match."
) ? static_cast<void> (0) : __assert_fail ("CompTargetNode.getValueType() == N.getValueType() && \"Expecting that the types on LHS and RHS of comparisons match.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5149, __PRETTY_FUNCTION__))
;
5150 N = DAG.getNode(ISD::SUB, DL, VT, N, CompTargetNode);
5151 }
5152
5153 // (mul N, P)
5154 SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal);
5155 Created.push_back(Op0.getNode());
5156
5157 // Rotate right only if any divisor was even. We avoid rotates for all-odd
5158 // divisors as a performance improvement, since rotating by 0 is a no-op.
5159 if (HadEvenDivisor) {
5160 // We need ROTR to do this.
5161 if (!isOperationLegalOrCustom(ISD::ROTR, VT))
5162 return SDValue();
5163 SDNodeFlags Flags;
5164 Flags.setExact(true);
5165 // UREM: (rotr (mul N, P), K)
5166 Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal, Flags);
5167 Created.push_back(Op0.getNode());
5168 }
5169
5170 // UREM: (setule/setugt (rotr (mul N, P), K), Q)
5171 SDValue NewCC =
5172 DAG.getSetCC(DL, SETCCVT, Op0, QVal,
5173 ((Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT));
5174 if (!HadTautologicalInvertedLanes)
5175 return NewCC;
5176
5177 // If any lanes previously compared always-false, the NewCC will give
5178 // always-true result for them, so we need to fixup those lanes.
5179 // Or the other way around for inequality predicate.
5180 assert(VT.isVector() && "Can/should only get here for vectors.")((VT.isVector() && "Can/should only get here for vectors."
) ? static_cast<void> (0) : __assert_fail ("VT.isVector() && \"Can/should only get here for vectors.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5180, __PRETTY_FUNCTION__))
;
5181 Created.push_back(NewCC.getNode());
5182
5183 // x u% C1` is *always* less than C1. So given `x u% C1 == C2`,
5184 // if C2 is not less than C1, the comparison is always false.
5185 // But we have produced the comparison that will give the
5186 // opposive tautological answer. So these lanes would need to be fixed up.
5187 SDValue TautologicalInvertedChannels =
5188 DAG.getSetCC(DL, SETCCVT, D, CompTargetNode, ISD::SETULE);
5189 Created.push_back(TautologicalInvertedChannels.getNode());
5190
5191 if (isOperationLegalOrCustom(ISD::VSELECT, SETCCVT)) {
5192 // If we have a vector select, let's replace the comparison results in the
5193 // affected lanes with the correct tautological result.
5194 SDValue Replacement = DAG.getBoolConstant(Cond == ISD::SETEQ ? false : true,
5195 DL, SETCCVT, SETCCVT);
5196 return DAG.getNode(ISD::VSELECT, DL, SETCCVT, TautologicalInvertedChannels,
5197 Replacement, NewCC);
5198 }
5199
5200 // Else, we can just invert the comparison result in the appropriate lanes.
5201 if (isOperationLegalOrCustom(ISD::XOR, SETCCVT))
5202 return DAG.getNode(ISD::XOR, DL, SETCCVT, NewCC,
5203 TautologicalInvertedChannels);
5204
5205 return SDValue(); // Don't know how to lower.
5206}
5207
5208/// Given an ISD::SREM used only by an ISD::SETEQ or ISD::SETNE
5209/// where the divisor is constant and the comparison target is zero,
5210/// return a DAG expression that will generate the same comparison result
5211/// using only multiplications, additions and shifts/rotations.
5212/// Ref: "Hacker's Delight" 10-17.
5213SDValue TargetLowering::buildSREMEqFold(EVT SETCCVT, SDValue REMNode,
5214 SDValue CompTargetNode,
5215 ISD::CondCode Cond,
5216 DAGCombinerInfo &DCI,
5217 const SDLoc &DL) const {
5218 SmallVector<SDNode *, 7> Built;
5219 if (SDValue Folded = prepareSREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
5220 DCI, DL, Built)) {
5221 assert(Built.size() <= 7 && "Max size prediction failed.")((Built.size() <= 7 && "Max size prediction failed."
) ? static_cast<void> (0) : __assert_fail ("Built.size() <= 7 && \"Max size prediction failed.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5221, __PRETTY_FUNCTION__))
;
5222 for (SDNode *N : Built)
5223 DCI.AddToWorklist(N);
5224 return Folded;
5225 }
5226
5227 return SDValue();
5228}
5229
5230SDValue
5231TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode,
5232 SDValue CompTargetNode, ISD::CondCode Cond,
5233 DAGCombinerInfo &DCI, const SDLoc &DL,
5234 SmallVectorImpl<SDNode *> &Created) const {
5235 // Fold:
5236 // (seteq/ne (srem N, D), 0)
5237 // To:
5238 // (setule/ugt (rotr (add (mul N, P), A), K), Q)
5239 //
5240 // - D must be constant, with D = D0 * 2^K where D0 is odd
5241 // - P is the multiplicative inverse of D0 modulo 2^W
5242 // - A = bitwiseand(floor((2^(W - 1) - 1) / D0), (-(2^k)))
5243 // - Q = floor((2 * A) / (2^K))
5244 // where W is the width of the common type of N and D.
5245 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&(((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Only applicable for (in)equality comparisons."
) ? static_cast<void> (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5246, __PRETTY_FUNCTION__))
5246 "Only applicable for (in)equality comparisons.")(((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Only applicable for (in)equality comparisons."
) ? static_cast<void> (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5246, __PRETTY_FUNCTION__))
;
5247
5248 SelectionDAG &DAG = DCI.DAG;
5249
5250 EVT VT = REMNode.getValueType();
5251 EVT SVT = VT.getScalarType();
5252 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
5253 EVT ShSVT = ShVT.getScalarType();
5254
5255 // If MUL is unavailable, we cannot proceed in any case.
5256 if (!isOperationLegalOrCustom(ISD::MUL, VT))
5257 return SDValue();
5258
5259 // TODO: Could support comparing with non-zero too.
5260 ConstantSDNode *CompTarget = isConstOrConstSplat(CompTargetNode);
5261 if (!CompTarget || !CompTarget->isNullValue())
5262 return SDValue();
5263
5264 bool HadIntMinDivisor = false;
5265 bool HadOneDivisor = false;
5266 bool AllDivisorsAreOnes = true;
5267 bool HadEvenDivisor = false;
5268 bool NeedToApplyOffset = false;
5269 bool AllDivisorsArePowerOfTwo = true;
5270 SmallVector<SDValue, 16> PAmts, AAmts, KAmts, QAmts;
5271
5272 auto BuildSREMPattern = [&](ConstantSDNode *C) {
5273 // Division by 0 is UB. Leave it to be constant-folded elsewhere.
5274 if (C->isNullValue())
1
Taking false branch
5275 return false;
5276
5277 // FIXME: we don't fold `rem %X, -C` to `rem %X, C` in DAGCombine.
5278
5279 // WARNING: this fold is only valid for positive divisors!
5280 APInt D = C->getAPIntValue();
5281 if (D.isNegative())
2
Taking false branch
5282 D.negate(); // `rem %X, -C` is equivalent to `rem %X, C`
5283
5284 HadIntMinDivisor |= D.isMinSignedValue();
5285
5286 // If all divisors are ones, we will prefer to avoid the fold.
5287 HadOneDivisor |= D.isOneValue();
5288 AllDivisorsAreOnes &= D.isOneValue();
5289
5290 // Decompose D into D0 * 2^K
5291 unsigned K = D.countTrailingZeros();
5292 assert((!D.isOneValue() || (K == 0)) && "For divisor '1' we won't rotate.")(((!D.isOneValue() || (K == 0)) && "For divisor '1' we won't rotate."
) ? static_cast<void> (0) : __assert_fail ("(!D.isOneValue() || (K == 0)) && \"For divisor '1' we won't rotate.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5292, __PRETTY_FUNCTION__))
;
3
Assuming 'K' is equal to 0
4
'?' condition is true
5293 APInt D0 = D.lshr(K);
5294
5295 if (!D.isMinSignedValue()) {
5
Calling 'APInt::isMinSignedValue'
5296 // D is even if it has trailing zeros; unless it's INT_MIN, in which case
5297 // we don't care about this lane in this fold, we'll special-handle it.
5298 HadEvenDivisor |= (K != 0);
5299 }
5300
5301 // D is a power-of-two if D0 is one. This includes INT_MIN.
5302 // If all divisors are power-of-two, we will prefer to avoid the fold.
5303 AllDivisorsArePowerOfTwo &= D0.isOneValue();
5304
5305 // P = inv(D0, 2^W)
5306 // 2^W requires W + 1 bits, so we have to extend and then truncate.
5307 unsigned W = D.getBitWidth();
5308 APInt P = D0.zext(W + 1)
5309 .multiplicativeInverse(APInt::getSignedMinValue(W + 1))
5310 .trunc(W);
5311 assert(!P.isNullValue() && "No multiplicative inverse!")((!P.isNullValue() && "No multiplicative inverse!") ?
static_cast<void> (0) : __assert_fail ("!P.isNullValue() && \"No multiplicative inverse!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5311, __PRETTY_FUNCTION__))
; // unreachable
5312 assert((D0 * P).isOneValue() && "Multiplicative inverse sanity check.")(((D0 * P).isOneValue() && "Multiplicative inverse sanity check."
) ? static_cast<void> (0) : __assert_fail ("(D0 * P).isOneValue() && \"Multiplicative inverse sanity check.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5312, __PRETTY_FUNCTION__))
;
5313
5314 // A = floor((2^(W - 1) - 1) / D0) & -2^K
5315 APInt A = APInt::getSignedMaxValue(W).udiv(D0);
5316 A.clearLowBits(K);
5317
5318 if (!D.isMinSignedValue()) {
5319 // If divisor INT_MIN, then we don't care about this lane in this fold,
5320 // we'll special-handle it.
5321 NeedToApplyOffset |= A != 0;
5322 }
5323
5324 // Q = floor((2 * A) / (2^K))
5325 APInt Q = (2 * A).udiv(APInt::getOneBitSet(W, K));
5326
5327 assert(APInt::getAllOnesValue(SVT.getSizeInBits()).ugt(A) &&((APInt::getAllOnesValue(SVT.getSizeInBits()).ugt(A) &&
"We are expecting that A is always less than all-ones for SVT"
) ? static_cast<void> (0) : __assert_fail ("APInt::getAllOnesValue(SVT.getSizeInBits()).ugt(A) && \"We are expecting that A is always less than all-ones for SVT\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5328, __PRETTY_FUNCTION__))
5328 "We are expecting that A is always less than all-ones for SVT")((APInt::getAllOnesValue(SVT.getSizeInBits()).ugt(A) &&
"We are expecting that A is always less than all-ones for SVT"
) ? static_cast<void> (0) : __assert_fail ("APInt::getAllOnesValue(SVT.getSizeInBits()).ugt(A) && \"We are expecting that A is always less than all-ones for SVT\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5328, __PRETTY_FUNCTION__))
;
5329 assert(APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) &&((APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) &&
"We are expecting that K is always less than all-ones for ShSVT"
) ? static_cast<void> (0) : __assert_fail ("APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) && \"We are expecting that K is always less than all-ones for ShSVT\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5330, __PRETTY_FUNCTION__))
5330 "We are expecting that K is always less than all-ones for ShSVT")((APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) &&
"We are expecting that K is always less than all-ones for ShSVT"
) ? static_cast<void> (0) : __assert_fail ("APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) && \"We are expecting that K is always less than all-ones for ShSVT\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5330, __PRETTY_FUNCTION__))
;
5331
5332 // If the divisor is 1 the result can be constant-folded. Likewise, we
5333 // don't care about INT_MIN lanes, those can be set to undef if appropriate.
5334 if (D.isOneValue()) {
5335 // Set P, A and K to a bogus values so we can try to splat them.
5336 P = 0;
5337 A = -1;
5338 K = -1;
5339
5340 // x ?% 1 == 0 <--> true <--> x u<= -1
5341 Q = -1;
5342 }
5343
5344 PAmts.push_back(DAG.getConstant(P, DL, SVT));
5345 AAmts.push_back(DAG.getConstant(A, DL, SVT));
5346 KAmts.push_back(
5347 DAG.getConstant(APInt(ShSVT.getSizeInBits(), K), DL, ShSVT));
5348 QAmts.push_back(DAG.getConstant(Q, DL, SVT));
5349 return true;
5350 };
5351
5352 SDValue N = REMNode.getOperand(0);
5353 SDValue D = REMNode.getOperand(1);
5354
5355 // Collect the values from each element.
5356 if (!ISD::matchUnaryPredicate(D, BuildSREMPattern))
5357 return SDValue();
5358
5359 // If this is a srem by a one, avoid the fold since it can be constant-folded.
5360 if (AllDivisorsAreOnes)
5361 return SDValue();
5362
5363 // If this is a srem by a powers-of-two (including INT_MIN), avoid the fold
5364 // since it can be best implemented as a bit test.
5365 if (AllDivisorsArePowerOfTwo)
5366 return SDValue();
5367
5368 SDValue PVal, AVal, KVal, QVal;
5369 if (VT.isVector()) {
5370 if (HadOneDivisor) {
5371 // Try to turn PAmts into a splat, since we don't care about the values
5372 // that are currently '0'. If we can't, just keep '0'`s.
5373 turnVectorIntoSplatVector(PAmts, isNullConstant);
5374 // Try to turn AAmts into a splat, since we don't care about the
5375 // values that are currently '-1'. If we can't, change them to '0'`s.
5376 turnVectorIntoSplatVector(AAmts, isAllOnesConstant,
5377 DAG.getConstant(0, DL, SVT));
5378 // Try to turn KAmts into a splat, since we don't care about the values
5379 // that are currently '-1'. If we can't, change them to '0'`s.
5380 turnVectorIntoSplatVector(KAmts, isAllOnesConstant,
5381 DAG.getConstant(0, DL, ShSVT));
5382 }
5383
5384 PVal = DAG.getBuildVector(VT, DL, PAmts);
5385 AVal = DAG.getBuildVector(VT, DL, AAmts);
5386 KVal = DAG.getBuildVector(ShVT, DL, KAmts);
5387 QVal = DAG.getBuildVector(VT, DL, QAmts);
5388 } else {
5389 PVal = PAmts[0];
5390 AVal = AAmts[0];
5391 KVal = KAmts[0];
5392 QVal = QAmts[0];
5393 }
5394
5395 // (mul N, P)
5396 SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal);
5397 Created.push_back(Op0.getNode());
5398
5399 if (NeedToApplyOffset) {
5400 // We need ADD to do this.
5401 if (!isOperationLegalOrCustom(ISD::ADD, VT))
5402 return SDValue();
5403
5404 // (add (mul N, P), A)
5405 Op0 = DAG.getNode(ISD::ADD, DL, VT, Op0, AVal);
5406 Created.push_back(Op0.getNode());
5407 }
5408
5409 // Rotate right only if any divisor was even. We avoid rotates for all-odd
5410 // divisors as a performance improvement, since rotating by 0 is a no-op.
5411 if (HadEvenDivisor) {
5412 // We need ROTR to do this.
5413 if (!isOperationLegalOrCustom(ISD::ROTR, VT))
5414 return SDValue();
5415 SDNodeFlags Flags;
5416 Flags.setExact(true);
5417 // SREM: (rotr (add (mul N, P), A), K)
5418 Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal, Flags);
5419 Created.push_back(Op0.getNode());
5420 }
5421
5422 // SREM: (setule/setugt (rotr (add (mul N, P), A), K), Q)
5423 SDValue Fold =
5424 DAG.getSetCC(DL, SETCCVT, Op0, QVal,
5425 ((Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT));
5426
5427 // If we didn't have lanes with INT_MIN divisor, then we're done.
5428 if (!HadIntMinDivisor)
5429 return Fold;
5430
5431 // That fold is only valid for positive divisors. Which effectively means,
5432 // it is invalid for INT_MIN divisors. So if we have such a lane,
5433 // we must fix-up results for said lanes.
5434 assert(VT.isVector() && "Can/should only get here for vectors.")((VT.isVector() && "Can/should only get here for vectors."
) ? static_cast<void> (0) : __assert_fail ("VT.isVector() && \"Can/should only get here for vectors.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5434, __PRETTY_FUNCTION__))
;
5435
5436 if (!isOperationLegalOrCustom(ISD::SETEQ, VT) ||
5437 !isOperationLegalOrCustom(ISD::AND, VT) ||
5438 !isOperationLegalOrCustom(Cond, VT) ||
5439 !isOperationLegalOrCustom(ISD::VSELECT, VT))
5440 return SDValue();
5441
5442 Created.push_back(Fold.getNode());
5443
5444 SDValue IntMin = DAG.getConstant(
5445 APInt::getSignedMinValue(SVT.getScalarSizeInBits()), DL, VT);
5446 SDValue IntMax = DAG.getConstant(
5447 APInt::getSignedMaxValue(SVT.getScalarSizeInBits()), DL, VT);
5448 SDValue Zero =
5449 DAG.getConstant(APInt::getNullValue(SVT.getScalarSizeInBits()), DL, VT);
5450
5451 // Which lanes had INT_MIN divisors? Divisor is constant, so const-folded.
5452 SDValue DivisorIsIntMin = DAG.getSetCC(DL, SETCCVT, D, IntMin, ISD::SETEQ);
5453 Created.push_back(DivisorIsIntMin.getNode());
5454
5455 // (N s% INT_MIN) ==/!= 0 <--> (N & INT_MAX) ==/!= 0
5456 SDValue Masked = DAG.getNode(ISD::AND, DL, VT, N, IntMax);
5457 Created.push_back(Masked.getNode());
5458 SDValue MaskedIsZero = DAG.getSetCC(DL, SETCCVT, Masked, Zero, Cond);
5459 Created.push_back(MaskedIsZero.getNode());
5460
5461 // To produce final result we need to blend 2 vectors: 'SetCC' and
5462 // 'MaskedIsZero'. If the divisor for channel was *NOT* INT_MIN, we pick
5463 // from 'Fold', else pick from 'MaskedIsZero'. Since 'DivisorIsIntMin' is
5464 // constant-folded, select can get lowered to a shuffle with constant mask.
5465 SDValue Blended =
5466 DAG.getNode(ISD::VSELECT, DL, VT, DivisorIsIntMin, MaskedIsZero, Fold);
5467
5468 return Blended;
5469}
5470
5471bool TargetLowering::
5472verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const {
5473 if (!isa<ConstantSDNode>(Op.getOperand(0))) {
5474 DAG.getContext()->emitError("argument to '__builtin_return_address' must "
5475 "be a constant integer");
5476 return true;
5477 }
5478
5479 return false;
5480}
5481
5482char TargetLowering::isNegatibleForFree(SDValue Op, SelectionDAG &DAG,
5483 bool LegalOperations, bool ForCodeSize,
5484 unsigned Depth) const {
5485 // fneg is removable even if it has multiple uses.
5486 if (Op.getOpcode() == ISD::FNEG)
5487 return 2;
5488
5489 // Don't allow anything with multiple uses unless we know it is free.
5490 EVT VT = Op.getValueType();
5491 const SDNodeFlags Flags = Op->getFlags();
5492 const TargetOptions &Options = DAG.getTarget().Options;
5493 if (!Op.hasOneUse() && !(Op.getOpcode() == ISD::FP_EXTEND &&
5494 isFPExtFree(VT, Op.getOperand(0).getValueType())))
5495 return 0;
5496
5497 // Don't recurse exponentially.
5498 if (Depth > SelectionDAG::MaxRecursionDepth)
5499 return 0;
5500
5501 switch (Op.getOpcode()) {
5502 case ISD::ConstantFP: {
5503 if (!LegalOperations)
5504 return 1;
5505
5506 // Don't invert constant FP values after legalization unless the target says
5507 // the negated constant is legal.
5508 return isOperationLegal(ISD::ConstantFP, VT) ||
5509 isFPImmLegal(neg(cast<ConstantFPSDNode>(Op)->getValueAPF()), VT,
5510 ForCodeSize);
5511 }
5512 case ISD::BUILD_VECTOR: {
5513 // Only permit BUILD_VECTOR of constants.
5514 if (llvm::any_of(Op->op_values(), [&](SDValue N) {
5515 return !N.isUndef() && !isa<ConstantFPSDNode>(N);
5516 }))
5517 return 0;
5518 if (!LegalOperations)
5519 return 1;
5520 if (isOperationLegal(ISD::ConstantFP, VT) &&
5521 isOperationLegal(ISD::BUILD_VECTOR, VT))
5522 return 1;
5523 return llvm::all_of(Op->op_values(), [&](SDValue N) {
5524 return N.isUndef() ||
5525 isFPImmLegal(neg(cast<ConstantFPSDNode>(N)->getValueAPF()), VT,
5526 ForCodeSize);
5527 });
5528 }
5529 case ISD::FADD:
5530 if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())
5531 return 0;
5532
5533 // After operation legalization, it might not be legal to create new FSUBs.
5534 if (LegalOperations && !isOperationLegalOrCustom(ISD::FSUB, VT))
5535 return 0;
5536
5537 // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
5538 if (char V = isNegatibleForFree(Op.getOperand(0), DAG, LegalOperations,
5539 ForCodeSize, Depth + 1))
5540 return V;
5541 // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
5542 return isNegatibleForFree(Op.getOperand(1), DAG, LegalOperations,
5543 ForCodeSize, Depth + 1);
5544 case ISD::FSUB:
5545 // We can't turn -(A-B) into B-A when we honor signed zeros.
5546 if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())
5547 return 0;
5548
5549 // fold (fneg (fsub A, B)) -> (fsub B, A)
5550 return 1;
5551
5552 case ISD::FMUL:
5553 case ISD::FDIV:
5554 // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y) or (fmul X, (fneg Y))
5555 if (char V = isNegatibleForFree(Op.getOperand(0), DAG, LegalOperations,
5556 ForCodeSize, Depth + 1))
5557 return V;
5558
5559 // Ignore X * 2.0 because that is expected to be canonicalized to X + X.
5560 if (auto *C = isConstOrConstSplatFP(Op.getOperand(1)))
5561 if (C->isExactlyValue(2.0) && Op.getOpcode() == ISD::FMUL)
5562 return 0;
5563
5564 return isNegatibleForFree(Op.getOperand(1), DAG, LegalOperations,
5565 ForCodeSize, Depth + 1);
5566
5567 case ISD::FMA:
5568 case ISD::FMAD: {
5569 if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())
5570 return 0;
5571
5572 // fold (fneg (fma X, Y, Z)) -> (fma (fneg X), Y, (fneg Z))
5573 // fold (fneg (fma X, Y, Z)) -> (fma X, (fneg Y), (fneg Z))
5574 char V2 = isNegatibleForFree(Op.getOperand(2), DAG, LegalOperations,
5575 ForCodeSize, Depth + 1);
5576 if (!V2)
5577 return 0;
5578
5579 // One of Op0/Op1 must be cheaply negatible, then select the cheapest.
5580 char V0 = isNegatibleForFree(Op.getOperand(0), DAG, LegalOperations,
5581 ForCodeSize, Depth + 1);
5582 char V1 = isNegatibleForFree(Op.getOperand(1), DAG, LegalOperations,
5583 ForCodeSize, Depth + 1);
5584 char V01 = std::max(V0, V1);
5585 return V01 ? std::max(V01, V2) : 0;
5586 }
5587
5588 case ISD::FP_EXTEND:
5589 case ISD::FP_ROUND:
5590 case ISD::FSIN:
5591 return isNegatibleForFree(Op.getOperand(0), DAG, LegalOperations,
5592 ForCodeSize, Depth + 1);
5593 }
5594
5595 return 0;
5596}
5597
5598SDValue TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,
5599 bool LegalOperations,
5600 bool ForCodeSize,
5601 unsigned Depth) const {
5602 // fneg is removable even if it has multiple uses.
5603 if (Op.getOpcode() == ISD::FNEG)
5604 return Op.getOperand(0);
5605
5606 assert(Depth <= SelectionDAG::MaxRecursionDepth &&((Depth <= SelectionDAG::MaxRecursionDepth && "getNegatedExpression doesn't match isNegatibleForFree"
) ? static_cast<void> (0) : __assert_fail ("Depth <= SelectionDAG::MaxRecursionDepth && \"getNegatedExpression doesn't match isNegatibleForFree\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5607, __PRETTY_FUNCTION__))
5607 "getNegatedExpression doesn't match isNegatibleForFree")((Depth <= SelectionDAG::MaxRecursionDepth && "getNegatedExpression doesn't match isNegatibleForFree"
) ? static_cast<void> (0) : __assert_fail ("Depth <= SelectionDAG::MaxRecursionDepth && \"getNegatedExpression doesn't match isNegatibleForFree\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5607, __PRETTY_FUNCTION__))
;
5608 const SDNodeFlags Flags = Op->getFlags();
5609
5610 switch (Op.getOpcode()) {
5611 case ISD::ConstantFP: {
5612 APFloat V = cast<ConstantFPSDNode>(Op)->getValueAPF();
5613 V.changeSign();
5614 return DAG.getConstantFP(V, SDLoc(Op), Op.getValueType());
5615 }
5616 case ISD::BUILD_VECTOR: {
5617 SmallVector<SDValue, 4> Ops;
5618 for (SDValue C : Op->op_values()) {
5619 if (C.isUndef()) {
5620 Ops.push_back(C);
5621 continue;
5622 }
5623 APFloat V = cast<ConstantFPSDNode>(C)->getValueAPF();
5624 V.changeSign();
5625 Ops.push_back(DAG.getConstantFP(V, SDLoc(Op), C.getValueType()));
5626 }
5627 return DAG.getBuildVector(Op.getValueType(), SDLoc(Op), Ops);
5628 }
5629 case ISD::FADD:
5630 assert((DAG.getTarget().Options.NoSignedZerosFPMath ||(((DAG.getTarget().Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros
()) && "Expected NSZ fp-flag") ? static_cast<void>
(0) : __assert_fail ("(DAG.getTarget().Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros()) && \"Expected NSZ fp-flag\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5632, __PRETTY_FUNCTION__))
5631 Flags.hasNoSignedZeros()) &&(((DAG.getTarget().Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros
()) && "Expected NSZ fp-flag") ? static_cast<void>
(0) : __assert_fail ("(DAG.getTarget().Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros()) && \"Expected NSZ fp-flag\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5632, __PRETTY_FUNCTION__))
5632 "Expected NSZ fp-flag")(((DAG.getTarget().Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros
()) && "Expected NSZ fp-flag") ? static_cast<void>
(0) : __assert_fail ("(DAG.getTarget().Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros()) && \"Expected NSZ fp-flag\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5632, __PRETTY_FUNCTION__))
;
5633
5634 // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
5635 if (isNegatibleForFree(Op.getOperand(0), DAG, LegalOperations, ForCodeSize,
5636 Depth + 1))
5637 return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
5638 getNegatedExpression(Op.getOperand(0), DAG,
5639 LegalOperations, ForCodeSize,
5640 Depth + 1),
5641 Op.getOperand(1), Flags);
5642 // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
5643 return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
5644 getNegatedExpression(Op.getOperand(1), DAG,
5645 LegalOperations, ForCodeSize,
5646 Depth + 1),
5647 Op.getOperand(0), Flags);
5648 case ISD::FSUB:
5649 // fold (fneg (fsub 0, B)) -> B
5650 if (ConstantFPSDNode *N0CFP =
5651 isConstOrConstSplatFP(Op.getOperand(0), /*AllowUndefs*/ true))
5652 if (N0CFP->isZero())
5653 return Op.getOperand(1);
5654
5655 // fold (fneg (fsub A, B)) -> (fsub B, A)
5656 return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
5657 Op.getOperand(1), Op.getOperand(0), Flags);
5658
5659 case ISD::FMUL:
5660 case ISD::FDIV:
5661 // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y)
5662 if (isNegatibleForFree(Op.getOperand(0), DAG, LegalOperations, ForCodeSize,
5663 Depth + 1))
5664 return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
5665 getNegatedExpression(Op.getOperand(0), DAG,
5666 LegalOperations, ForCodeSize,
5667 Depth + 1),
5668 Op.getOperand(1), Flags);
5669
5670 // fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y))
5671 return DAG.getNode(
5672 Op.getOpcode(), SDLoc(Op), Op.getValueType(), Op.getOperand(0),
5673 getNegatedExpression(Op.getOperand(1), DAG, LegalOperations,
5674 ForCodeSize, Depth + 1),
5675 Flags);
5676
5677 case ISD::FMA:
5678 case ISD::FMAD: {
5679 assert((DAG.getTarget().Options.NoSignedZerosFPMath ||(((DAG.getTarget().Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros
()) && "Expected NSZ fp-flag") ? static_cast<void>
(0) : __assert_fail ("(DAG.getTarget().Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros()) && \"Expected NSZ fp-flag\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5681, __PRETTY_FUNCTION__))
5680 Flags.hasNoSignedZeros()) &&(((DAG.getTarget().Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros
()) && "Expected NSZ fp-flag") ? static_cast<void>
(0) : __assert_fail ("(DAG.getTarget().Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros()) && \"Expected NSZ fp-flag\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5681, __PRETTY_FUNCTION__))
5681 "Expected NSZ fp-flag")(((DAG.getTarget().Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros
()) && "Expected NSZ fp-flag") ? static_cast<void>
(0) : __assert_fail ("(DAG.getTarget().Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros()) && \"Expected NSZ fp-flag\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5681, __PRETTY_FUNCTION__))
;
5682
5683 SDValue Neg2 = getNegatedExpression(Op.getOperand(2), DAG, LegalOperations,
5684 ForCodeSize, Depth + 1);
5685
5686 char V0 = isNegatibleForFree(Op.getOperand(0), DAG, LegalOperations,
5687 ForCodeSize, Depth + 1);
5688 char V1 = isNegatibleForFree(Op.getOperand(1), DAG, LegalOperations,
5689 ForCodeSize, Depth + 1);
5690 // TODO: This is a hack. It is possible that costs have changed between now
5691 // and the initial calls to isNegatibleForFree(). That is because we
5692 // are rewriting the expression, and that may change the number of
5693 // uses (and therefore the cost) of values. If the negation costs are
5694 // equal, only negate this value if it is a constant. Otherwise, try
5695 // operand 1. A better fix would eliminate uses as a cost factor or
5696 // track the change in uses as we rewrite the expression.
5697 if (V0 > V1 || (V0 == V1 && isa<ConstantFPSDNode>(Op.getOperand(0)))) {
5698 // fold (fneg (fma X, Y, Z)) -> (fma (fneg X), Y, (fneg Z))
5699 SDValue Neg0 = getNegatedExpression(
5700 Op.getOperand(0), DAG, LegalOperations, ForCodeSize, Depth + 1);
5701 return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(), Neg0,
5702 Op.getOperand(1), Neg2, Flags);
5703 }
5704
5705 // fold (fneg (fma X, Y, Z)) -> (fma X, (fneg Y), (fneg Z))
5706 SDValue Neg1 = getNegatedExpression(Op.getOperand(1), DAG, LegalOperations,
5707 ForCodeSize, Depth + 1);
5708 return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
5709 Op.getOperand(0), Neg1, Neg2, Flags);
5710 }
5711
5712 case ISD::FP_EXTEND:
5713 case ISD::FSIN:
5714 return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
5715 getNegatedExpression(Op.getOperand(0), DAG,
5716 LegalOperations, ForCodeSize,
5717 Depth + 1));
5718 case ISD::FP_ROUND:
5719 return DAG.getNode(ISD::FP_ROUND, SDLoc(Op), Op.getValueType(),
5720 getNegatedExpression(Op.getOperand(0), DAG,
5721 LegalOperations, ForCodeSize,
5722 Depth + 1),
5723 Op.getOperand(1));
5724 }
5725
5726 llvm_unreachable("Unknown code")::llvm::llvm_unreachable_internal("Unknown code", "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5726)
;
5727}
5728
5729//===----------------------------------------------------------------------===//
5730// Legalization Utilities
5731//===----------------------------------------------------------------------===//
5732
5733bool TargetLowering::expandMUL_LOHI(unsigned Opcode, EVT VT, SDLoc dl,
5734 SDValue LHS, SDValue RHS,
5735 SmallVectorImpl<SDValue> &Result,
5736 EVT HiLoVT, SelectionDAG &DAG,
5737 MulExpansionKind Kind, SDValue LL,
5738 SDValue LH, SDValue RL, SDValue RH) const {
5739 assert(Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI ||((Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI || Opcode ==
ISD::SMUL_LOHI) ? static_cast<void> (0) : __assert_fail
("Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI || Opcode == ISD::SMUL_LOHI"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5740, __PRETTY_FUNCTION__))
5740 Opcode == ISD::SMUL_LOHI)((Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI || Opcode ==
ISD::SMUL_LOHI) ? static_cast<void> (0) : __assert_fail
("Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI || Opcode == ISD::SMUL_LOHI"
, "/build