Bug Summary

File:llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
Warning:line 7915, column 5
Value stored to 'IsScaledIndex' is never read

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name TargetLowering.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/build-llvm/lib/CodeGen/SelectionDAG -resource-dir /usr/lib/llvm-13/lib/clang/13.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/build-llvm/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/build-llvm/include -I /build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-13/lib/clang/13.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/build-llvm/lib/CodeGen/SelectionDAG -fdebug-prefix-map=/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82=. -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2021-06-21-164211-33944-1 -x c++ /build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
1//===-- TargetLowering.cpp - Implement the TargetLowering class -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the TargetLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/CodeGen/TargetLowering.h"
14#include "llvm/ADT/STLExtras.h"
15#include "llvm/CodeGen/CallingConvLower.h"
16#include "llvm/CodeGen/MachineFrameInfo.h"
17#include "llvm/CodeGen/MachineFunction.h"
18#include "llvm/CodeGen/MachineJumpTableInfo.h"
19#include "llvm/CodeGen/MachineRegisterInfo.h"
20#include "llvm/CodeGen/SelectionDAG.h"
21#include "llvm/CodeGen/TargetRegisterInfo.h"
22#include "llvm/CodeGen/TargetSubtargetInfo.h"
23#include "llvm/IR/DataLayout.h"
24#include "llvm/IR/DerivedTypes.h"
25#include "llvm/IR/GlobalVariable.h"
26#include "llvm/IR/LLVMContext.h"
27#include "llvm/MC/MCAsmInfo.h"
28#include "llvm/MC/MCExpr.h"
29#include "llvm/Support/ErrorHandling.h"
30#include "llvm/Support/KnownBits.h"
31#include "llvm/Support/MathExtras.h"
32#include "llvm/Target/TargetLoweringObjectFile.h"
33#include "llvm/Target/TargetMachine.h"
34#include <cctype>
35using namespace llvm;
36
37/// NOTE: The TargetMachine owns TLOF.
38TargetLowering::TargetLowering(const TargetMachine &tm)
39 : TargetLoweringBase(tm) {}
40
41const char *TargetLowering::getTargetNodeName(unsigned Opcode) const {
42 return nullptr;
43}
44
45bool TargetLowering::isPositionIndependent() const {
46 return getTargetMachine().isPositionIndependent();
47}
48
49/// Check whether a given call node is in tail position within its function. If
50/// so, it sets Chain to the input chain of the tail call.
51bool TargetLowering::isInTailCallPosition(SelectionDAG &DAG, SDNode *Node,
52 SDValue &Chain) const {
53 const Function &F = DAG.getMachineFunction().getFunction();
54
55 // First, check if tail calls have been disabled in this function.
56 if (F.getFnAttribute("disable-tail-calls").getValueAsBool())
57 return false;
58
59 // Conservatively require the attributes of the call to match those of
60 // the return. Ignore following attributes because they don't affect the
61 // call sequence.
62 AttrBuilder CallerAttrs(F.getAttributes(), AttributeList::ReturnIndex);
63 for (const auto &Attr : {Attribute::Alignment, Attribute::Dereferenceable,
64 Attribute::DereferenceableOrNull, Attribute::NoAlias,
65 Attribute::NonNull})
66 CallerAttrs.removeAttribute(Attr);
67
68 if (CallerAttrs.hasAttributes())
69 return false;
70
71 // It's not safe to eliminate the sign / zero extension of the return value.
72 if (CallerAttrs.contains(Attribute::ZExt) ||
73 CallerAttrs.contains(Attribute::SExt))
74 return false;
75
76 // Check if the only use is a function return node.
77 return isUsedByReturnOnly(Node, Chain);
78}
79
80bool TargetLowering::parametersInCSRMatch(const MachineRegisterInfo &MRI,
81 const uint32_t *CallerPreservedMask,
82 const SmallVectorImpl<CCValAssign> &ArgLocs,
83 const SmallVectorImpl<SDValue> &OutVals) const {
84 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
85 const CCValAssign &ArgLoc = ArgLocs[I];
86 if (!ArgLoc.isRegLoc())
87 continue;
88 MCRegister Reg = ArgLoc.getLocReg();
89 // Only look at callee saved registers.
90 if (MachineOperand::clobbersPhysReg(CallerPreservedMask, Reg))
91 continue;
92 // Check that we pass the value used for the caller.
93 // (We look for a CopyFromReg reading a virtual register that is used
94 // for the function live-in value of register Reg)
95 SDValue Value = OutVals[I];
96 if (Value->getOpcode() != ISD::CopyFromReg)
97 return false;
98 Register ArgReg = cast<RegisterSDNode>(Value->getOperand(1))->getReg();
99 if (MRI.getLiveInPhysReg(ArgReg) != Reg)
100 return false;
101 }
102 return true;
103}
104
105/// Set CallLoweringInfo attribute flags based on a call instruction
106/// and called function attributes.
107void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call,
108 unsigned ArgIdx) {
109 IsSExt = Call->paramHasAttr(ArgIdx, Attribute::SExt);
110 IsZExt = Call->paramHasAttr(ArgIdx, Attribute::ZExt);
111 IsInReg = Call->paramHasAttr(ArgIdx, Attribute::InReg);
112 IsSRet = Call->paramHasAttr(ArgIdx, Attribute::StructRet);
113 IsNest = Call->paramHasAttr(ArgIdx, Attribute::Nest);
114 IsByVal = Call->paramHasAttr(ArgIdx, Attribute::ByVal);
115 IsPreallocated = Call->paramHasAttr(ArgIdx, Attribute::Preallocated);
116 IsInAlloca = Call->paramHasAttr(ArgIdx, Attribute::InAlloca);
117 IsReturned = Call->paramHasAttr(ArgIdx, Attribute::Returned);
118 IsSwiftSelf = Call->paramHasAttr(ArgIdx, Attribute::SwiftSelf);
119 IsSwiftAsync = Call->paramHasAttr(ArgIdx, Attribute::SwiftAsync);
120 IsSwiftError = Call->paramHasAttr(ArgIdx, Attribute::SwiftError);
121 Alignment = Call->getParamStackAlign(ArgIdx);
122 ByValType = nullptr;
123 if (IsByVal) {
124 ByValType = Call->getParamByValType(ArgIdx);
125 if (!Alignment)
126 Alignment = Call->getParamAlign(ArgIdx);
127 }
128 PreallocatedType = nullptr;
129 if (IsPreallocated)
130 PreallocatedType = Call->getParamPreallocatedType(ArgIdx);
131}
132
133/// Generate a libcall taking the given operands as arguments and returning a
134/// result of type RetVT.
135std::pair<SDValue, SDValue>
136TargetLowering::makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT,
137 ArrayRef<SDValue> Ops,
138 MakeLibCallOptions CallOptions,
139 const SDLoc &dl,
140 SDValue InChain) const {
141 if (!InChain)
142 InChain = DAG.getEntryNode();
143
144 TargetLowering::ArgListTy Args;
145 Args.reserve(Ops.size());
146
147 TargetLowering::ArgListEntry Entry;
148 for (unsigned i = 0; i < Ops.size(); ++i) {
149 SDValue NewOp = Ops[i];
150 Entry.Node = NewOp;
151 Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
152 Entry.IsSExt = shouldSignExtendTypeInLibCall(NewOp.getValueType(),
153 CallOptions.IsSExt);
154 Entry.IsZExt = !Entry.IsSExt;
155
156 if (CallOptions.IsSoften &&
157 !shouldExtendTypeInLibCall(CallOptions.OpsVTBeforeSoften[i])) {
158 Entry.IsSExt = Entry.IsZExt = false;
159 }
160 Args.push_back(Entry);
161 }
162
163 if (LC == RTLIB::UNKNOWN_LIBCALL)
164 report_fatal_error("Unsupported library call operation!");
165 SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
166 getPointerTy(DAG.getDataLayout()));
167
168 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
169 TargetLowering::CallLoweringInfo CLI(DAG);
170 bool signExtend = shouldSignExtendTypeInLibCall(RetVT, CallOptions.IsSExt);
171 bool zeroExtend = !signExtend;
172
173 if (CallOptions.IsSoften &&
174 !shouldExtendTypeInLibCall(CallOptions.RetVTBeforeSoften)) {
175 signExtend = zeroExtend = false;
176 }
177
178 CLI.setDebugLoc(dl)
179 .setChain(InChain)
180 .setLibCallee(getLibcallCallingConv(LC), RetTy, Callee, std::move(Args))
181 .setNoReturn(CallOptions.DoesNotReturn)
182 .setDiscardResult(!CallOptions.IsReturnValueUsed)
183 .setIsPostTypeLegalization(CallOptions.IsPostTypeLegalization)
184 .setSExtResult(signExtend)
185 .setZExtResult(zeroExtend);
186 return LowerCallTo(CLI);
187}
188
189bool TargetLowering::findOptimalMemOpLowering(
190 std::vector<EVT> &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS,
191 unsigned SrcAS, const AttributeList &FuncAttributes) const {
192 if (Op.isMemcpyWithFixedDstAlign() && Op.getSrcAlign() < Op.getDstAlign())
193 return false;
194
195 EVT VT = getOptimalMemOpType(Op, FuncAttributes);
196
197 if (VT == MVT::Other) {
198 // Use the largest integer type whose alignment constraints are satisfied.
199 // We only need to check DstAlign here as SrcAlign is always greater or
200 // equal to DstAlign (or zero).
201 VT = MVT::i64;
202 if (Op.isFixedDstAlign())
203 while (Op.getDstAlign() < (VT.getSizeInBits() / 8) &&
204 !allowsMisalignedMemoryAccesses(VT, DstAS, Op.getDstAlign()))
205 VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1);
206 assert(VT.isInteger())(static_cast <bool> (VT.isInteger()) ? void (0) : __assert_fail
("VT.isInteger()", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 206, __extension__ __PRETTY_FUNCTION__))
;
207
208 // Find the largest legal integer type.
209 MVT LVT = MVT::i64;
210 while (!isTypeLegal(LVT))
211 LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
212 assert(LVT.isInteger())(static_cast <bool> (LVT.isInteger()) ? void (0) : __assert_fail
("LVT.isInteger()", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 212, __extension__ __PRETTY_FUNCTION__))
;
213
214 // If the type we've chosen is larger than the largest legal integer type
215 // then use that instead.
216 if (VT.bitsGT(LVT))
217 VT = LVT;
218 }
219
220 unsigned NumMemOps = 0;
221 uint64_t Size = Op.size();
222 while (Size) {
223 unsigned VTSize = VT.getSizeInBits() / 8;
224 while (VTSize > Size) {
225 // For now, only use non-vector load / store's for the left-over pieces.
226 EVT NewVT = VT;
227 unsigned NewVTSize;
228
229 bool Found = false;
230 if (VT.isVector() || VT.isFloatingPoint()) {
231 NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;
232 if (isOperationLegalOrCustom(ISD::STORE, NewVT) &&
233 isSafeMemOpType(NewVT.getSimpleVT()))
234 Found = true;
235 else if (NewVT == MVT::i64 &&
236 isOperationLegalOrCustom(ISD::STORE, MVT::f64) &&
237 isSafeMemOpType(MVT::f64)) {
238 // i64 is usually not legal on 32-bit targets, but f64 may be.
239 NewVT = MVT::f64;
240 Found = true;
241 }
242 }
243
244 if (!Found) {
245 do {
246 NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
247 if (NewVT == MVT::i8)
248 break;
249 } while (!isSafeMemOpType(NewVT.getSimpleVT()));
250 }
251 NewVTSize = NewVT.getSizeInBits() / 8;
252
253 // If the new VT cannot cover all of the remaining bits, then consider
254 // issuing a (or a pair of) unaligned and overlapping load / store.
255 bool Fast;
256 if (NumMemOps && Op.allowOverlap() && NewVTSize < Size &&
257 allowsMisalignedMemoryAccesses(
258 VT, DstAS, Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1),
259 MachineMemOperand::MONone, &Fast) &&
260 Fast)
261 VTSize = Size;
262 else {
263 VT = NewVT;
264 VTSize = NewVTSize;
265 }
266 }
267
268 if (++NumMemOps > Limit)
269 return false;
270
271 MemOps.push_back(VT);
272 Size -= VTSize;
273 }
274
275 return true;
276}
277
278/// Soften the operands of a comparison. This code is shared among BR_CC,
279/// SELECT_CC, and SETCC handlers.
280void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,
281 SDValue &NewLHS, SDValue &NewRHS,
282 ISD::CondCode &CCCode,
283 const SDLoc &dl, const SDValue OldLHS,
284 const SDValue OldRHS) const {
285 SDValue Chain;
286 return softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, dl, OldLHS,
287 OldRHS, Chain);
288}
289
290void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,
291 SDValue &NewLHS, SDValue &NewRHS,
292 ISD::CondCode &CCCode,
293 const SDLoc &dl, const SDValue OldLHS,
294 const SDValue OldRHS,
295 SDValue &Chain,
296 bool IsSignaling) const {
297 // FIXME: Currently we cannot really respect all IEEE predicates due to libgcc
298 // not supporting it. We can update this code when libgcc provides such
299 // functions.
300
301 assert((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128)(static_cast <bool> ((VT == MVT::f32 || VT == MVT::f64 ||
VT == MVT::f128 || VT == MVT::ppcf128) && "Unsupported setcc type!"
) ? void (0) : __assert_fail ("(VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128) && \"Unsupported setcc type!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 302, __extension__ __PRETTY_FUNCTION__))
302 && "Unsupported setcc type!")(static_cast <bool> ((VT == MVT::f32 || VT == MVT::f64 ||
VT == MVT::f128 || VT == MVT::ppcf128) && "Unsupported setcc type!"
) ? void (0) : __assert_fail ("(VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128) && \"Unsupported setcc type!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 302, __extension__ __PRETTY_FUNCTION__))
;
303
304 // Expand into one or more soft-fp libcall(s).
305 RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL;
306 bool ShouldInvertCC = false;
307 switch (CCCode) {
308 case ISD::SETEQ:
309 case ISD::SETOEQ:
310 LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
311 (VT == MVT::f64) ? RTLIB::OEQ_F64 :
312 (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
313 break;
314 case ISD::SETNE:
315 case ISD::SETUNE:
316 LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 :
317 (VT == MVT::f64) ? RTLIB::UNE_F64 :
318 (VT == MVT::f128) ? RTLIB::UNE_F128 : RTLIB::UNE_PPCF128;
319 break;
320 case ISD::SETGE:
321 case ISD::SETOGE:
322 LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
323 (VT == MVT::f64) ? RTLIB::OGE_F64 :
324 (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
325 break;
326 case ISD::SETLT:
327 case ISD::SETOLT:
328 LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
329 (VT == MVT::f64) ? RTLIB::OLT_F64 :
330 (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
331 break;
332 case ISD::SETLE:
333 case ISD::SETOLE:
334 LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
335 (VT == MVT::f64) ? RTLIB::OLE_F64 :
336 (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
337 break;
338 case ISD::SETGT:
339 case ISD::SETOGT:
340 LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
341 (VT == MVT::f64) ? RTLIB::OGT_F64 :
342 (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
343 break;
344 case ISD::SETO:
345 ShouldInvertCC = true;
346 LLVM_FALLTHROUGH[[gnu::fallthrough]];
347 case ISD::SETUO:
348 LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
349 (VT == MVT::f64) ? RTLIB::UO_F64 :
350 (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
351 break;
352 case ISD::SETONE:
353 // SETONE = O && UNE
354 ShouldInvertCC = true;
355 LLVM_FALLTHROUGH[[gnu::fallthrough]];
356 case ISD::SETUEQ:
357 LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
358 (VT == MVT::f64) ? RTLIB::UO_F64 :
359 (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
360 LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
361 (VT == MVT::f64) ? RTLIB::OEQ_F64 :
362 (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
363 break;
364 default:
365 // Invert CC for unordered comparisons
366 ShouldInvertCC = true;
367 switch (CCCode) {
368 case ISD::SETULT:
369 LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
370 (VT == MVT::f64) ? RTLIB::OGE_F64 :
371 (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
372 break;
373 case ISD::SETULE:
374 LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
375 (VT == MVT::f64) ? RTLIB::OGT_F64 :
376 (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
377 break;
378 case ISD::SETUGT:
379 LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
380 (VT == MVT::f64) ? RTLIB::OLE_F64 :
381 (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
382 break;
383 case ISD::SETUGE:
384 LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
385 (VT == MVT::f64) ? RTLIB::OLT_F64 :
386 (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
387 break;
388 default: llvm_unreachable("Do not know how to soften this setcc!")::llvm::llvm_unreachable_internal("Do not know how to soften this setcc!"
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 388)
;
389 }
390 }
391
392 // Use the target specific return value for comparions lib calls.
393 EVT RetVT = getCmpLibcallReturnType();
394 SDValue Ops[2] = {NewLHS, NewRHS};
395 TargetLowering::MakeLibCallOptions CallOptions;
396 EVT OpsVT[2] = { OldLHS.getValueType(),
397 OldRHS.getValueType() };
398 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT, true);
399 auto Call = makeLibCall(DAG, LC1, RetVT, Ops, CallOptions, dl, Chain);
400 NewLHS = Call.first;
401 NewRHS = DAG.getConstant(0, dl, RetVT);
402
403 CCCode = getCmpLibcallCC(LC1);
404 if (ShouldInvertCC) {
405 assert(RetVT.isInteger())(static_cast <bool> (RetVT.isInteger()) ? void (0) : __assert_fail
("RetVT.isInteger()", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 405, __extension__ __PRETTY_FUNCTION__))
;
406 CCCode = getSetCCInverse(CCCode, RetVT);
407 }
408
409 if (LC2 == RTLIB::UNKNOWN_LIBCALL) {
410 // Update Chain.
411 Chain = Call.second;
412 } else {
413 EVT SetCCVT =
414 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), RetVT);
415 SDValue Tmp = DAG.getSetCC(dl, SetCCVT, NewLHS, NewRHS, CCCode);
416 auto Call2 = makeLibCall(DAG, LC2, RetVT, Ops, CallOptions, dl, Chain);
417 CCCode = getCmpLibcallCC(LC2);
418 if (ShouldInvertCC)
419 CCCode = getSetCCInverse(CCCode, RetVT);
420 NewLHS = DAG.getSetCC(dl, SetCCVT, Call2.first, NewRHS, CCCode);
421 if (Chain)
422 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Call.second,
423 Call2.second);
424 NewLHS = DAG.getNode(ShouldInvertCC ? ISD::AND : ISD::OR, dl,
425 Tmp.getValueType(), Tmp, NewLHS);
426 NewRHS = SDValue();
427 }
428}
429
430/// Return the entry encoding for a jump table in the current function. The
431/// returned value is a member of the MachineJumpTableInfo::JTEntryKind enum.
432unsigned TargetLowering::getJumpTableEncoding() const {
433 // In non-pic modes, just use the address of a block.
434 if (!isPositionIndependent())
435 return MachineJumpTableInfo::EK_BlockAddress;
436
437 // In PIC mode, if the target supports a GPRel32 directive, use it.
438 if (getTargetMachine().getMCAsmInfo()->getGPRel32Directive() != nullptr)
439 return MachineJumpTableInfo::EK_GPRel32BlockAddress;
440
441 // Otherwise, use a label difference.
442 return MachineJumpTableInfo::EK_LabelDifference32;
443}
444
445SDValue TargetLowering::getPICJumpTableRelocBase(SDValue Table,
446 SelectionDAG &DAG) const {
447 // If our PIC model is GP relative, use the global offset table as the base.
448 unsigned JTEncoding = getJumpTableEncoding();
449
450 if ((JTEncoding == MachineJumpTableInfo::EK_GPRel64BlockAddress) ||
451 (JTEncoding == MachineJumpTableInfo::EK_GPRel32BlockAddress))
452 return DAG.getGLOBAL_OFFSET_TABLE(getPointerTy(DAG.getDataLayout()));
453
454 return Table;
455}
456
457/// This returns the relocation base for the given PIC jumptable, the same as
458/// getPICJumpTableRelocBase, but as an MCExpr.
459const MCExpr *
460TargetLowering::getPICJumpTableRelocBaseExpr(const MachineFunction *MF,
461 unsigned JTI,MCContext &Ctx) const{
462 // The normal PIC reloc base is the label at the start of the jump table.
463 return MCSymbolRefExpr::create(MF->getJTISymbol(JTI, Ctx), Ctx);
464}
465
466bool
467TargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
468 const TargetMachine &TM = getTargetMachine();
469 const GlobalValue *GV = GA->getGlobal();
470
471 // If the address is not even local to this DSO we will have to load it from
472 // a got and then add the offset.
473 if (!TM.shouldAssumeDSOLocal(*GV->getParent(), GV))
474 return false;
475
476 // If the code is position independent we will have to add a base register.
477 if (isPositionIndependent())
478 return false;
479
480 // Otherwise we can do it.
481 return true;
482}
483
484//===----------------------------------------------------------------------===//
485// Optimization Methods
486//===----------------------------------------------------------------------===//
487
488/// If the specified instruction has a constant integer operand and there are
489/// bits set in that constant that are not demanded, then clear those bits and
490/// return true.
491bool TargetLowering::ShrinkDemandedConstant(SDValue Op,
492 const APInt &DemandedBits,
493 const APInt &DemandedElts,
494 TargetLoweringOpt &TLO) const {
495 SDLoc DL(Op);
496 unsigned Opcode = Op.getOpcode();
497
498 // Do target-specific constant optimization.
499 if (targetShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
500 return TLO.New.getNode();
501
502 // FIXME: ISD::SELECT, ISD::SELECT_CC
503 switch (Opcode) {
504 default:
505 break;
506 case ISD::XOR:
507 case ISD::AND:
508 case ISD::OR: {
509 auto *Op1C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
510 if (!Op1C)
511 return false;
512
513 // If this is a 'not' op, don't touch it because that's a canonical form.
514 const APInt &C = Op1C->getAPIntValue();
515 if (Opcode == ISD::XOR && DemandedBits.isSubsetOf(C))
516 return false;
517
518 if (!C.isSubsetOf(DemandedBits)) {
519 EVT VT = Op.getValueType();
520 SDValue NewC = TLO.DAG.getConstant(DemandedBits & C, DL, VT);
521 SDValue NewOp = TLO.DAG.getNode(Opcode, DL, VT, Op.getOperand(0), NewC);
522 return TLO.CombineTo(Op, NewOp);
523 }
524
525 break;
526 }
527 }
528
529 return false;
530}
531
532bool TargetLowering::ShrinkDemandedConstant(SDValue Op,
533 const APInt &DemandedBits,
534 TargetLoweringOpt &TLO) const {
535 EVT VT = Op.getValueType();
536 APInt DemandedElts = VT.isVector()
537 ? APInt::getAllOnesValue(VT.getVectorNumElements())
538 : APInt(1, 1);
539 return ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO);
540}
541
542/// Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.
543/// This uses isZExtFree and ZERO_EXTEND for the widening cast, but it could be
544/// generalized for targets with other types of implicit widening casts.
545bool TargetLowering::ShrinkDemandedOp(SDValue Op, unsigned BitWidth,
546 const APInt &Demanded,
547 TargetLoweringOpt &TLO) const {
548 assert(Op.getNumOperands() == 2 &&(static_cast <bool> (Op.getNumOperands() == 2 &&
"ShrinkDemandedOp only supports binary operators!") ? void (
0) : __assert_fail ("Op.getNumOperands() == 2 && \"ShrinkDemandedOp only supports binary operators!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 549, __extension__ __PRETTY_FUNCTION__))
549 "ShrinkDemandedOp only supports binary operators!")(static_cast <bool> (Op.getNumOperands() == 2 &&
"ShrinkDemandedOp only supports binary operators!") ? void (
0) : __assert_fail ("Op.getNumOperands() == 2 && \"ShrinkDemandedOp only supports binary operators!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 549, __extension__ __PRETTY_FUNCTION__))
;
550 assert(Op.getNode()->getNumValues() == 1 &&(static_cast <bool> (Op.getNode()->getNumValues() ==
1 && "ShrinkDemandedOp only supports nodes with one result!"
) ? void (0) : __assert_fail ("Op.getNode()->getNumValues() == 1 && \"ShrinkDemandedOp only supports nodes with one result!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 551, __extension__ __PRETTY_FUNCTION__))
551 "ShrinkDemandedOp only supports nodes with one result!")(static_cast <bool> (Op.getNode()->getNumValues() ==
1 && "ShrinkDemandedOp only supports nodes with one result!"
) ? void (0) : __assert_fail ("Op.getNode()->getNumValues() == 1 && \"ShrinkDemandedOp only supports nodes with one result!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 551, __extension__ __PRETTY_FUNCTION__))
;
552
553 SelectionDAG &DAG = TLO.DAG;
554 SDLoc dl(Op);
555
556 // Early return, as this function cannot handle vector types.
557 if (Op.getValueType().isVector())
558 return false;
559
560 // Don't do this if the node has another user, which may require the
561 // full value.
562 if (!Op.getNode()->hasOneUse())
563 return false;
564
565 // Search for the smallest integer type with free casts to and from
566 // Op's type. For expedience, just check power-of-2 integer types.
567 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
568 unsigned DemandedSize = Demanded.getActiveBits();
569 unsigned SmallVTBits = DemandedSize;
570 if (!isPowerOf2_32(SmallVTBits))
571 SmallVTBits = NextPowerOf2(SmallVTBits);
572 for (; SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(SmallVTBits)) {
573 EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), SmallVTBits);
574 if (TLI.isTruncateFree(Op.getValueType(), SmallVT) &&
575 TLI.isZExtFree(SmallVT, Op.getValueType())) {
576 // We found a type with free casts.
577 SDValue X = DAG.getNode(
578 Op.getOpcode(), dl, SmallVT,
579 DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(0)),
580 DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(1)));
581 assert(DemandedSize <= SmallVTBits && "Narrowed below demanded bits?")(static_cast <bool> (DemandedSize <= SmallVTBits &&
"Narrowed below demanded bits?") ? void (0) : __assert_fail (
"DemandedSize <= SmallVTBits && \"Narrowed below demanded bits?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 581, __extension__ __PRETTY_FUNCTION__))
;
582 SDValue Z = DAG.getNode(ISD::ANY_EXTEND, dl, Op.getValueType(), X);
583 return TLO.CombineTo(Op, Z);
584 }
585 }
586 return false;
587}
588
589bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
590 DAGCombinerInfo &DCI) const {
591 SelectionDAG &DAG = DCI.DAG;
592 TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
593 !DCI.isBeforeLegalizeOps());
594 KnownBits Known;
595
596 bool Simplified = SimplifyDemandedBits(Op, DemandedBits, Known, TLO);
597 if (Simplified) {
598 DCI.AddToWorklist(Op.getNode());
599 DCI.CommitTargetLoweringOpt(TLO);
600 }
601 return Simplified;
602}
603
604bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
605 KnownBits &Known,
606 TargetLoweringOpt &TLO,
607 unsigned Depth,
608 bool AssumeSingleUse) const {
609 EVT VT = Op.getValueType();
610
611 // TODO: We can probably do more work on calculating the known bits and
612 // simplifying the operations for scalable vectors, but for now we just
613 // bail out.
614 if (VT.isScalableVector()) {
615 // Pretend we don't know anything for now.
616 Known = KnownBits(DemandedBits.getBitWidth());
617 return false;
618 }
619
620 APInt DemandedElts = VT.isVector()
621 ? APInt::getAllOnesValue(VT.getVectorNumElements())
622 : APInt(1, 1);
623 return SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO, Depth,
624 AssumeSingleUse);
625}
626
627// TODO: Can we merge SelectionDAG::GetDemandedBits into this?
628// TODO: Under what circumstances can we create nodes? Constant folding?
629SDValue TargetLowering::SimplifyMultipleUseDemandedBits(
630 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
631 SelectionDAG &DAG, unsigned Depth) const {
632 // Limit search depth.
633 if (Depth >= SelectionDAG::MaxRecursionDepth)
634 return SDValue();
635
636 // Ignore UNDEFs.
637 if (Op.isUndef())
638 return SDValue();
639
640 // Not demanding any bits/elts from Op.
641 if (DemandedBits == 0 || DemandedElts == 0)
642 return DAG.getUNDEF(Op.getValueType());
643
644 unsigned NumElts = DemandedElts.getBitWidth();
645 unsigned BitWidth = DemandedBits.getBitWidth();
646 KnownBits LHSKnown, RHSKnown;
647 switch (Op.getOpcode()) {
648 case ISD::BITCAST: {
649 SDValue Src = peekThroughBitcasts(Op.getOperand(0));
650 EVT SrcVT = Src.getValueType();
651 EVT DstVT = Op.getValueType();
652 if (SrcVT == DstVT)
653 return Src;
654
655 unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
656 unsigned NumDstEltBits = DstVT.getScalarSizeInBits();
657 if (NumSrcEltBits == NumDstEltBits)
658 if (SDValue V = SimplifyMultipleUseDemandedBits(
659 Src, DemandedBits, DemandedElts, DAG, Depth + 1))
660 return DAG.getBitcast(DstVT, V);
661
662 // TODO - bigendian once we have test coverage.
663 if (SrcVT.isVector() && (NumDstEltBits % NumSrcEltBits) == 0 &&
664 DAG.getDataLayout().isLittleEndian()) {
665 unsigned Scale = NumDstEltBits / NumSrcEltBits;
666 unsigned NumSrcElts = SrcVT.getVectorNumElements();
667 APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);
668 APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);
669 for (unsigned i = 0; i != Scale; ++i) {
670 unsigned Offset = i * NumSrcEltBits;
671 APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset);
672 if (!Sub.isNullValue()) {
673 DemandedSrcBits |= Sub;
674 for (unsigned j = 0; j != NumElts; ++j)
675 if (DemandedElts[j])
676 DemandedSrcElts.setBit((j * Scale) + i);
677 }
678 }
679
680 if (SDValue V = SimplifyMultipleUseDemandedBits(
681 Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))
682 return DAG.getBitcast(DstVT, V);
683 }
684
685 // TODO - bigendian once we have test coverage.
686 if ((NumSrcEltBits % NumDstEltBits) == 0 &&
687 DAG.getDataLayout().isLittleEndian()) {
688 unsigned Scale = NumSrcEltBits / NumDstEltBits;
689 unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
690 APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);
691 APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);
692 for (unsigned i = 0; i != NumElts; ++i)
693 if (DemandedElts[i]) {
694 unsigned Offset = (i % Scale) * NumDstEltBits;
695 DemandedSrcBits.insertBits(DemandedBits, Offset);
696 DemandedSrcElts.setBit(i / Scale);
697 }
698
699 if (SDValue V = SimplifyMultipleUseDemandedBits(
700 Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))
701 return DAG.getBitcast(DstVT, V);
702 }
703
704 break;
705 }
706 case ISD::AND: {
707 LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
708 RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
709
710 // If all of the demanded bits are known 1 on one side, return the other.
711 // These bits cannot contribute to the result of the 'and' in this
712 // context.
713 if (DemandedBits.isSubsetOf(LHSKnown.Zero | RHSKnown.One))
714 return Op.getOperand(0);
715 if (DemandedBits.isSubsetOf(RHSKnown.Zero | LHSKnown.One))
716 return Op.getOperand(1);
717 break;
718 }
719 case ISD::OR: {
720 LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
721 RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
722
723 // If all of the demanded bits are known zero on one side, return the
724 // other. These bits cannot contribute to the result of the 'or' in this
725 // context.
726 if (DemandedBits.isSubsetOf(LHSKnown.One | RHSKnown.Zero))
727 return Op.getOperand(0);
728 if (DemandedBits.isSubsetOf(RHSKnown.One | LHSKnown.Zero))
729 return Op.getOperand(1);
730 break;
731 }
732 case ISD::XOR: {
733 LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
734 RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
735
736 // If all of the demanded bits are known zero on one side, return the
737 // other.
738 if (DemandedBits.isSubsetOf(RHSKnown.Zero))
739 return Op.getOperand(0);
740 if (DemandedBits.isSubsetOf(LHSKnown.Zero))
741 return Op.getOperand(1);
742 break;
743 }
744 case ISD::SHL: {
745 // If we are only demanding sign bits then we can use the shift source
746 // directly.
747 if (const APInt *MaxSA =
748 DAG.getValidMaximumShiftAmountConstant(Op, DemandedElts)) {
749 SDValue Op0 = Op.getOperand(0);
750 unsigned ShAmt = MaxSA->getZExtValue();
751 unsigned NumSignBits =
752 DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
753 unsigned UpperDemandedBits = BitWidth - DemandedBits.countTrailingZeros();
754 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
755 return Op0;
756 }
757 break;
758 }
759 case ISD::SETCC: {
760 SDValue Op0 = Op.getOperand(0);
761 SDValue Op1 = Op.getOperand(1);
762 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
763 // If (1) we only need the sign-bit, (2) the setcc operands are the same
764 // width as the setcc result, and (3) the result of a setcc conforms to 0 or
765 // -1, we may be able to bypass the setcc.
766 if (DemandedBits.isSignMask() &&
767 Op0.getScalarValueSizeInBits() == BitWidth &&
768 getBooleanContents(Op0.getValueType()) ==
769 BooleanContent::ZeroOrNegativeOneBooleanContent) {
770 // If we're testing X < 0, then this compare isn't needed - just use X!
771 // FIXME: We're limiting to integer types here, but this should also work
772 // if we don't care about FP signed-zero. The use of SETLT with FP means
773 // that we don't care about NaNs.
774 if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
775 (isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode())))
776 return Op0;
777 }
778 break;
779 }
780 case ISD::SIGN_EXTEND_INREG: {
781 // If none of the extended bits are demanded, eliminate the sextinreg.
782 SDValue Op0 = Op.getOperand(0);
783 EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
784 unsigned ExBits = ExVT.getScalarSizeInBits();
785 if (DemandedBits.getActiveBits() <= ExBits)
786 return Op0;
787 // If the input is already sign extended, just drop the extension.
788 unsigned NumSignBits = DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
789 if (NumSignBits >= (BitWidth - ExBits + 1))
790 return Op0;
791 break;
792 }
793 case ISD::ANY_EXTEND_VECTOR_INREG:
794 case ISD::SIGN_EXTEND_VECTOR_INREG:
795 case ISD::ZERO_EXTEND_VECTOR_INREG: {
796 // If we only want the lowest element and none of extended bits, then we can
797 // return the bitcasted source vector.
798 SDValue Src = Op.getOperand(0);
799 EVT SrcVT = Src.getValueType();
800 EVT DstVT = Op.getValueType();
801 if (DemandedElts == 1 && DstVT.getSizeInBits() == SrcVT.getSizeInBits() &&
802 DAG.getDataLayout().isLittleEndian() &&
803 DemandedBits.getActiveBits() <= SrcVT.getScalarSizeInBits()) {
804 return DAG.getBitcast(DstVT, Src);
805 }
806 break;
807 }
808 case ISD::INSERT_VECTOR_ELT: {
809 // If we don't demand the inserted element, return the base vector.
810 SDValue Vec = Op.getOperand(0);
811 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
812 EVT VecVT = Vec.getValueType();
813 if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements()) &&
814 !DemandedElts[CIdx->getZExtValue()])
815 return Vec;
816 break;
817 }
818 case ISD::INSERT_SUBVECTOR: {
819 // If we don't demand the inserted subvector, return the base vector.
820 SDValue Vec = Op.getOperand(0);
821 SDValue Sub = Op.getOperand(1);
822 uint64_t Idx = Op.getConstantOperandVal(2);
823 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
824 if (DemandedElts.extractBits(NumSubElts, Idx) == 0)
825 return Vec;
826 break;
827 }
828 case ISD::VECTOR_SHUFFLE: {
829 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
830
831 // If all the demanded elts are from one operand and are inline,
832 // then we can use the operand directly.
833 bool AllUndef = true, IdentityLHS = true, IdentityRHS = true;
834 for (unsigned i = 0; i != NumElts; ++i) {
835 int M = ShuffleMask[i];
836 if (M < 0 || !DemandedElts[i])
837 continue;
838 AllUndef = false;
839 IdentityLHS &= (M == (int)i);
840 IdentityRHS &= ((M - NumElts) == i);
841 }
842
843 if (AllUndef)
844 return DAG.getUNDEF(Op.getValueType());
845 if (IdentityLHS)
846 return Op.getOperand(0);
847 if (IdentityRHS)
848 return Op.getOperand(1);
849 break;
850 }
851 default:
852 if (Op.getOpcode() >= ISD::BUILTIN_OP_END)
853 if (SDValue V = SimplifyMultipleUseDemandedBitsForTargetNode(
854 Op, DemandedBits, DemandedElts, DAG, Depth))
855 return V;
856 break;
857 }
858 return SDValue();
859}
860
861SDValue TargetLowering::SimplifyMultipleUseDemandedBits(
862 SDValue Op, const APInt &DemandedBits, SelectionDAG &DAG,
863 unsigned Depth) const {
864 EVT VT = Op.getValueType();
865 APInt DemandedElts = VT.isVector()
866 ? APInt::getAllOnesValue(VT.getVectorNumElements())
867 : APInt(1, 1);
868 return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
869 Depth);
870}
871
872SDValue TargetLowering::SimplifyMultipleUseDemandedVectorElts(
873 SDValue Op, const APInt &DemandedElts, SelectionDAG &DAG,
874 unsigned Depth) const {
875 APInt DemandedBits = APInt::getAllOnesValue(Op.getScalarValueSizeInBits());
876 return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
877 Depth);
878}
879
880/// Look at Op. At this point, we know that only the OriginalDemandedBits of the
881/// result of Op are ever used downstream. If we can use this information to
882/// simplify Op, create a new simplified DAG node and return true, returning the
883/// original and new nodes in Old and New. Otherwise, analyze the expression and
884/// return a mask of Known bits for the expression (used to simplify the
885/// caller). The Known bits may only be accurate for those bits in the
886/// OriginalDemandedBits and OriginalDemandedElts.
887bool TargetLowering::SimplifyDemandedBits(
888 SDValue Op, const APInt &OriginalDemandedBits,
889 const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
890 unsigned Depth, bool AssumeSingleUse) const {
891 unsigned BitWidth = OriginalDemandedBits.getBitWidth();
892 assert(Op.getScalarValueSizeInBits() == BitWidth &&(static_cast <bool> (Op.getScalarValueSizeInBits() == BitWidth
&& "Mask size mismatches value type size!") ? void (
0) : __assert_fail ("Op.getScalarValueSizeInBits() == BitWidth && \"Mask size mismatches value type size!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 893, __extension__ __PRETTY_FUNCTION__))
893 "Mask size mismatches value type size!")(static_cast <bool> (Op.getScalarValueSizeInBits() == BitWidth
&& "Mask size mismatches value type size!") ? void (
0) : __assert_fail ("Op.getScalarValueSizeInBits() == BitWidth && \"Mask size mismatches value type size!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 893, __extension__ __PRETTY_FUNCTION__))
;
894
895 // Don't know anything.
896 Known = KnownBits(BitWidth);
897
898 // TODO: We can probably do more work on calculating the known bits and
899 // simplifying the operations for scalable vectors, but for now we just
900 // bail out.
901 if (Op.getValueType().isScalableVector())
902 return false;
903
904 unsigned NumElts = OriginalDemandedElts.getBitWidth();
905 assert((!Op.getValueType().isVector() ||(static_cast <bool> ((!Op.getValueType().isVector() || NumElts
== Op.getValueType().getVectorNumElements()) && "Unexpected vector size"
) ? void (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 907, __extension__ __PRETTY_FUNCTION__))
906 NumElts == Op.getValueType().getVectorNumElements()) &&(static_cast <bool> ((!Op.getValueType().isVector() || NumElts
== Op.getValueType().getVectorNumElements()) && "Unexpected vector size"
) ? void (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 907, __extension__ __PRETTY_FUNCTION__))
907 "Unexpected vector size")(static_cast <bool> ((!Op.getValueType().isVector() || NumElts
== Op.getValueType().getVectorNumElements()) && "Unexpected vector size"
) ? void (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 907, __extension__ __PRETTY_FUNCTION__))
;
908
909 APInt DemandedBits = OriginalDemandedBits;
910 APInt DemandedElts = OriginalDemandedElts;
911 SDLoc dl(Op);
912 auto &DL = TLO.DAG.getDataLayout();
913
914 // Undef operand.
915 if (Op.isUndef())
916 return false;
917
918 if (Op.getOpcode() == ISD::Constant) {
919 // We know all of the bits for a constant!
920 Known = KnownBits::makeConstant(cast<ConstantSDNode>(Op)->getAPIntValue());
921 return false;
922 }
923
924 if (Op.getOpcode() == ISD::ConstantFP) {
925 // We know all of the bits for a floating point constant!
926 Known = KnownBits::makeConstant(
927 cast<ConstantFPSDNode>(Op)->getValueAPF().bitcastToAPInt());
928 return false;
929 }
930
931 // Other users may use these bits.
932 EVT VT = Op.getValueType();
933 if (!Op.getNode()->hasOneUse() && !AssumeSingleUse) {
934 if (Depth != 0) {
935 // If not at the root, Just compute the Known bits to
936 // simplify things downstream.
937 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
938 return false;
939 }
940 // If this is the root being simplified, allow it to have multiple uses,
941 // just set the DemandedBits/Elts to all bits.
942 DemandedBits = APInt::getAllOnesValue(BitWidth);
943 DemandedElts = APInt::getAllOnesValue(NumElts);
944 } else if (OriginalDemandedBits == 0 || OriginalDemandedElts == 0) {
945 // Not demanding any bits/elts from Op.
946 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
947 } else if (Depth >= SelectionDAG::MaxRecursionDepth) {
948 // Limit search depth.
949 return false;
950 }
951
952 KnownBits Known2;
953 switch (Op.getOpcode()) {
954 case ISD::TargetConstant:
955 llvm_unreachable("Can't simplify this node")::llvm::llvm_unreachable_internal("Can't simplify this node",
"/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 955)
;
956 case ISD::SCALAR_TO_VECTOR: {
957 if (!DemandedElts[0])
958 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
959
960 KnownBits SrcKnown;
961 SDValue Src = Op.getOperand(0);
962 unsigned SrcBitWidth = Src.getScalarValueSizeInBits();
963 APInt SrcDemandedBits = DemandedBits.zextOrSelf(SrcBitWidth);
964 if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcKnown, TLO, Depth + 1))
965 return true;
966
967 // Upper elements are undef, so only get the knownbits if we just demand
968 // the bottom element.
969 if (DemandedElts == 1)
970 Known = SrcKnown.anyextOrTrunc(BitWidth);
971 break;
972 }
973 case ISD::BUILD_VECTOR:
974 // Collect the known bits that are shared by every demanded element.
975 // TODO: Call SimplifyDemandedBits for non-constant demanded elements.
976 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
977 return false; // Don't fall through, will infinitely loop.
978 case ISD::LOAD: {
979 auto *LD = cast<LoadSDNode>(Op);
980 if (getTargetConstantFromLoad(LD)) {
981 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
982 return false; // Don't fall through, will infinitely loop.
983 }
984 if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
985 // If this is a ZEXTLoad and we are looking at the loaded value.
986 EVT MemVT = LD->getMemoryVT();
987 unsigned MemBits = MemVT.getScalarSizeInBits();
988 Known.Zero.setBitsFrom(MemBits);
989 return false; // Don't fall through, will infinitely loop.
990 }
991 break;
992 }
993 case ISD::INSERT_VECTOR_ELT: {
994 SDValue Vec = Op.getOperand(0);
995 SDValue Scl = Op.getOperand(1);
996 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
997 EVT VecVT = Vec.getValueType();
998
999 // If index isn't constant, assume we need all vector elements AND the
1000 // inserted element.
1001 APInt DemandedVecElts(DemandedElts);
1002 if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements())) {
1003 unsigned Idx = CIdx->getZExtValue();
1004 DemandedVecElts.clearBit(Idx);
1005
1006 // Inserted element is not required.
1007 if (!DemandedElts[Idx])
1008 return TLO.CombineTo(Op, Vec);
1009 }
1010
1011 KnownBits KnownScl;
1012 unsigned NumSclBits = Scl.getScalarValueSizeInBits();
1013 APInt DemandedSclBits = DemandedBits.zextOrTrunc(NumSclBits);
1014 if (SimplifyDemandedBits(Scl, DemandedSclBits, KnownScl, TLO, Depth + 1))
1015 return true;
1016
1017 Known = KnownScl.anyextOrTrunc(BitWidth);
1018
1019 KnownBits KnownVec;
1020 if (SimplifyDemandedBits(Vec, DemandedBits, DemandedVecElts, KnownVec, TLO,
1021 Depth + 1))
1022 return true;
1023
1024 if (!!DemandedVecElts)
1025 Known = KnownBits::commonBits(Known, KnownVec);
1026
1027 return false;
1028 }
1029 case ISD::INSERT_SUBVECTOR: {
1030 // Demand any elements from the subvector and the remainder from the src its
1031 // inserted into.
1032 SDValue Src = Op.getOperand(0);
1033 SDValue Sub = Op.getOperand(1);
1034 uint64_t Idx = Op.getConstantOperandVal(2);
1035 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
1036 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
1037 APInt DemandedSrcElts = DemandedElts;
1038 DemandedSrcElts.insertBits(APInt::getNullValue(NumSubElts), Idx);
1039
1040 KnownBits KnownSub, KnownSrc;
1041 if (SimplifyDemandedBits(Sub, DemandedBits, DemandedSubElts, KnownSub, TLO,
1042 Depth + 1))
1043 return true;
1044 if (SimplifyDemandedBits(Src, DemandedBits, DemandedSrcElts, KnownSrc, TLO,
1045 Depth + 1))
1046 return true;
1047
1048 Known.Zero.setAllBits();
1049 Known.One.setAllBits();
1050 if (!!DemandedSubElts)
1051 Known = KnownBits::commonBits(Known, KnownSub);
1052 if (!!DemandedSrcElts)
1053 Known = KnownBits::commonBits(Known, KnownSrc);
1054
1055 // Attempt to avoid multi-use src if we don't need anything from it.
1056 if (!DemandedBits.isAllOnesValue() || !DemandedSubElts.isAllOnesValue() ||
1057 !DemandedSrcElts.isAllOnesValue()) {
1058 SDValue NewSub = SimplifyMultipleUseDemandedBits(
1059 Sub, DemandedBits, DemandedSubElts, TLO.DAG, Depth + 1);
1060 SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1061 Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);
1062 if (NewSub || NewSrc) {
1063 NewSub = NewSub ? NewSub : Sub;
1064 NewSrc = NewSrc ? NewSrc : Src;
1065 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc, NewSub,
1066 Op.getOperand(2));
1067 return TLO.CombineTo(Op, NewOp);
1068 }
1069 }
1070 break;
1071 }
1072 case ISD::EXTRACT_SUBVECTOR: {
1073 // Offset the demanded elts by the subvector index.
1074 SDValue Src = Op.getOperand(0);
1075 if (Src.getValueType().isScalableVector())
1076 break;
1077 uint64_t Idx = Op.getConstantOperandVal(1);
1078 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
1079 APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
1080
1081 if (SimplifyDemandedBits(Src, DemandedBits, DemandedSrcElts, Known, TLO,
1082 Depth + 1))
1083 return true;
1084
1085 // Attempt to avoid multi-use src if we don't need anything from it.
1086 if (!DemandedBits.isAllOnesValue() || !DemandedSrcElts.isAllOnesValue()) {
1087 SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
1088 Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);
1089 if (DemandedSrc) {
1090 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc,
1091 Op.getOperand(1));
1092 return TLO.CombineTo(Op, NewOp);
1093 }
1094 }
1095 break;
1096 }
1097 case ISD::CONCAT_VECTORS: {
1098 Known.Zero.setAllBits();
1099 Known.One.setAllBits();
1100 EVT SubVT = Op.getOperand(0).getValueType();
1101 unsigned NumSubVecs = Op.getNumOperands();
1102 unsigned NumSubElts = SubVT.getVectorNumElements();
1103 for (unsigned i = 0; i != NumSubVecs; ++i) {
1104 APInt DemandedSubElts =
1105 DemandedElts.extractBits(NumSubElts, i * NumSubElts);
1106 if (SimplifyDemandedBits(Op.getOperand(i), DemandedBits, DemandedSubElts,
1107 Known2, TLO, Depth + 1))
1108 return true;
1109 // Known bits are shared by every demanded subvector element.
1110 if (!!DemandedSubElts)
1111 Known = KnownBits::commonBits(Known, Known2);
1112 }
1113 break;
1114 }
1115 case ISD::VECTOR_SHUFFLE: {
1116 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
1117
1118 // Collect demanded elements from shuffle operands..
1119 APInt DemandedLHS(NumElts, 0);
1120 APInt DemandedRHS(NumElts, 0);
1121 for (unsigned i = 0; i != NumElts; ++i) {
1122 if (!DemandedElts[i])
1123 continue;
1124 int M = ShuffleMask[i];
1125 if (M < 0) {
1126 // For UNDEF elements, we don't know anything about the common state of
1127 // the shuffle result.
1128 DemandedLHS.clearAllBits();
1129 DemandedRHS.clearAllBits();
1130 break;
1131 }
1132 assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range")(static_cast <bool> (0 <= M && M < (int)(
2 * NumElts) && "Shuffle index out of range") ? void (
0) : __assert_fail ("0 <= M && M < (int)(2 * NumElts) && \"Shuffle index out of range\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1132, __extension__ __PRETTY_FUNCTION__))
;
1133 if (M < (int)NumElts)
1134 DemandedLHS.setBit(M);
1135 else
1136 DemandedRHS.setBit(M - NumElts);
1137 }
1138
1139 if (!!DemandedLHS || !!DemandedRHS) {
1140 SDValue Op0 = Op.getOperand(0);
1141 SDValue Op1 = Op.getOperand(1);
1142
1143 Known.Zero.setAllBits();
1144 Known.One.setAllBits();
1145 if (!!DemandedLHS) {
1146 if (SimplifyDemandedBits(Op0, DemandedBits, DemandedLHS, Known2, TLO,
1147 Depth + 1))
1148 return true;
1149 Known = KnownBits::commonBits(Known, Known2);
1150 }
1151 if (!!DemandedRHS) {
1152 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedRHS, Known2, TLO,
1153 Depth + 1))
1154 return true;
1155 Known = KnownBits::commonBits(Known, Known2);
1156 }
1157
1158 // Attempt to avoid multi-use ops if we don't need anything from them.
1159 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1160 Op0, DemandedBits, DemandedLHS, TLO.DAG, Depth + 1);
1161 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1162 Op1, DemandedBits, DemandedRHS, TLO.DAG, Depth + 1);
1163 if (DemandedOp0 || DemandedOp1) {
1164 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1165 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1166 SDValue NewOp = TLO.DAG.getVectorShuffle(VT, dl, Op0, Op1, ShuffleMask);
1167 return TLO.CombineTo(Op, NewOp);
1168 }
1169 }
1170 break;
1171 }
1172 case ISD::AND: {
1173 SDValue Op0 = Op.getOperand(0);
1174 SDValue Op1 = Op.getOperand(1);
1175
1176 // If the RHS is a constant, check to see if the LHS would be zero without
1177 // using the bits from the RHS. Below, we use knowledge about the RHS to
1178 // simplify the LHS, here we're using information from the LHS to simplify
1179 // the RHS.
1180 if (ConstantSDNode *RHSC = isConstOrConstSplat(Op1)) {
1181 // Do not increment Depth here; that can cause an infinite loop.
1182 KnownBits LHSKnown = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth);
1183 // If the LHS already has zeros where RHSC does, this 'and' is dead.
1184 if ((LHSKnown.Zero & DemandedBits) ==
1185 (~RHSC->getAPIntValue() & DemandedBits))
1186 return TLO.CombineTo(Op, Op0);
1187
1188 // If any of the set bits in the RHS are known zero on the LHS, shrink
1189 // the constant.
1190 if (ShrinkDemandedConstant(Op, ~LHSKnown.Zero & DemandedBits,
1191 DemandedElts, TLO))
1192 return true;
1193
1194 // Bitwise-not (xor X, -1) is a special case: we don't usually shrink its
1195 // constant, but if this 'and' is only clearing bits that were just set by
1196 // the xor, then this 'and' can be eliminated by shrinking the mask of
1197 // the xor. For example, for a 32-bit X:
1198 // and (xor (srl X, 31), -1), 1 --> xor (srl X, 31), 1
1199 if (isBitwiseNot(Op0) && Op0.hasOneUse() &&
1200 LHSKnown.One == ~RHSC->getAPIntValue()) {
1201 SDValue Xor = TLO.DAG.getNode(ISD::XOR, dl, VT, Op0.getOperand(0), Op1);
1202 return TLO.CombineTo(Op, Xor);
1203 }
1204 }
1205
1206 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1207 Depth + 1))
1208 return true;
1209 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1209, __extension__ __PRETTY_FUNCTION__))
;
1210 if (SimplifyDemandedBits(Op0, ~Known.Zero & DemandedBits, DemandedElts,
1211 Known2, TLO, Depth + 1))
1212 return true;
1213 assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1213, __extension__ __PRETTY_FUNCTION__))
;
1214
1215 // Attempt to avoid multi-use ops if we don't need anything from them.
1216 if (!DemandedBits.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {
1217 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1218 Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1219 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1220 Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1221 if (DemandedOp0 || DemandedOp1) {
1222 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1223 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1224 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1225 return TLO.CombineTo(Op, NewOp);
1226 }
1227 }
1228
1229 // If all of the demanded bits are known one on one side, return the other.
1230 // These bits cannot contribute to the result of the 'and'.
1231 if (DemandedBits.isSubsetOf(Known2.Zero | Known.One))
1232 return TLO.CombineTo(Op, Op0);
1233 if (DemandedBits.isSubsetOf(Known.Zero | Known2.One))
1234 return TLO.CombineTo(Op, Op1);
1235 // If all of the demanded bits in the inputs are known zeros, return zero.
1236 if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
1237 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, dl, VT));
1238 // If the RHS is a constant, see if we can simplify it.
1239 if (ShrinkDemandedConstant(Op, ~Known2.Zero & DemandedBits, DemandedElts,
1240 TLO))
1241 return true;
1242 // If the operation can be done in a smaller type, do so.
1243 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1244 return true;
1245
1246 Known &= Known2;
1247 break;
1248 }
1249 case ISD::OR: {
1250 SDValue Op0 = Op.getOperand(0);
1251 SDValue Op1 = Op.getOperand(1);
1252
1253 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1254 Depth + 1))
1255 return true;
1256 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1256, __extension__ __PRETTY_FUNCTION__))
;
1257 if (SimplifyDemandedBits(Op0, ~Known.One & DemandedBits, DemandedElts,
1258 Known2, TLO, Depth + 1))
1259 return true;
1260 assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1260, __extension__ __PRETTY_FUNCTION__))
;
1261
1262 // Attempt to avoid multi-use ops if we don't need anything from them.
1263 if (!DemandedBits.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {
1264 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1265 Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1266 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1267 Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1268 if (DemandedOp0 || DemandedOp1) {
1269 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1270 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1271 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1272 return TLO.CombineTo(Op, NewOp);
1273 }
1274 }
1275
1276 // If all of the demanded bits are known zero on one side, return the other.
1277 // These bits cannot contribute to the result of the 'or'.
1278 if (DemandedBits.isSubsetOf(Known2.One | Known.Zero))
1279 return TLO.CombineTo(Op, Op0);
1280 if (DemandedBits.isSubsetOf(Known.One | Known2.Zero))
1281 return TLO.CombineTo(Op, Op1);
1282 // If the RHS is a constant, see if we can simplify it.
1283 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1284 return true;
1285 // If the operation can be done in a smaller type, do so.
1286 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1287 return true;
1288
1289 Known |= Known2;
1290 break;
1291 }
1292 case ISD::XOR: {
1293 SDValue Op0 = Op.getOperand(0);
1294 SDValue Op1 = Op.getOperand(1);
1295
1296 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1297 Depth + 1))
1298 return true;
1299 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1299, __extension__ __PRETTY_FUNCTION__))
;
1300 if (SimplifyDemandedBits(Op0, DemandedBits, DemandedElts, Known2, TLO,
1301 Depth + 1))
1302 return true;
1303 assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1303, __extension__ __PRETTY_FUNCTION__))
;
1304
1305 // Attempt to avoid multi-use ops if we don't need anything from them.
1306 if (!DemandedBits.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {
1307 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1308 Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1309 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1310 Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1311 if (DemandedOp0 || DemandedOp1) {
1312 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1313 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1314 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1315 return TLO.CombineTo(Op, NewOp);
1316 }
1317 }
1318
1319 // If all of the demanded bits are known zero on one side, return the other.
1320 // These bits cannot contribute to the result of the 'xor'.
1321 if (DemandedBits.isSubsetOf(Known.Zero))
1322 return TLO.CombineTo(Op, Op0);
1323 if (DemandedBits.isSubsetOf(Known2.Zero))
1324 return TLO.CombineTo(Op, Op1);
1325 // If the operation can be done in a smaller type, do so.
1326 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1327 return true;
1328
1329 // If all of the unknown bits are known to be zero on one side or the other
1330 // turn this into an *inclusive* or.
1331 // e.g. (A & C1)^(B & C2) -> (A & C1)|(B & C2) iff C1&C2 == 0
1332 if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
1333 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::OR, dl, VT, Op0, Op1));
1334
1335 ConstantSDNode* C = isConstOrConstSplat(Op1, DemandedElts);
1336 if (C) {
1337 // If one side is a constant, and all of the set bits in the constant are
1338 // also known set on the other side, turn this into an AND, as we know
1339 // the bits will be cleared.
1340 // e.g. (X | C1) ^ C2 --> (X | C1) & ~C2 iff (C1&C2) == C2
1341 // NB: it is okay if more bits are known than are requested
1342 if (C->getAPIntValue() == Known2.One) {
1343 SDValue ANDC =
1344 TLO.DAG.getConstant(~C->getAPIntValue() & DemandedBits, dl, VT);
1345 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, dl, VT, Op0, ANDC));
1346 }
1347
1348 // If the RHS is a constant, see if we can change it. Don't alter a -1
1349 // constant because that's a 'not' op, and that is better for combining
1350 // and codegen.
1351 if (!C->isAllOnesValue() &&
1352 DemandedBits.isSubsetOf(C->getAPIntValue())) {
1353 // We're flipping all demanded bits. Flip the undemanded bits too.
1354 SDValue New = TLO.DAG.getNOT(dl, Op0, VT);
1355 return TLO.CombineTo(Op, New);
1356 }
1357 }
1358
1359 // If we can't turn this into a 'not', try to shrink the constant.
1360 if (!C || !C->isAllOnesValue())
1361 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1362 return true;
1363
1364 Known ^= Known2;
1365 break;
1366 }
1367 case ISD::SELECT:
1368 if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known, TLO,
1369 Depth + 1))
1370 return true;
1371 if (SimplifyDemandedBits(Op.getOperand(1), DemandedBits, Known2, TLO,
1372 Depth + 1))
1373 return true;
1374 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1374, __extension__ __PRETTY_FUNCTION__))
;
1375 assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1375, __extension__ __PRETTY_FUNCTION__))
;
1376
1377 // If the operands are constants, see if we can simplify them.
1378 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1379 return true;
1380
1381 // Only known if known in both the LHS and RHS.
1382 Known = KnownBits::commonBits(Known, Known2);
1383 break;
1384 case ISD::SELECT_CC:
1385 if (SimplifyDemandedBits(Op.getOperand(3), DemandedBits, Known, TLO,
1386 Depth + 1))
1387 return true;
1388 if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known2, TLO,
1389 Depth + 1))
1390 return true;
1391 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1391, __extension__ __PRETTY_FUNCTION__))
;
1392 assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1392, __extension__ __PRETTY_FUNCTION__))
;
1393
1394 // If the operands are constants, see if we can simplify them.
1395 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1396 return true;
1397
1398 // Only known if known in both the LHS and RHS.
1399 Known = KnownBits::commonBits(Known, Known2);
1400 break;
1401 case ISD::SETCC: {
1402 SDValue Op0 = Op.getOperand(0);
1403 SDValue Op1 = Op.getOperand(1);
1404 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
1405 // If (1) we only need the sign-bit, (2) the setcc operands are the same
1406 // width as the setcc result, and (3) the result of a setcc conforms to 0 or
1407 // -1, we may be able to bypass the setcc.
1408 if (DemandedBits.isSignMask() &&
1409 Op0.getScalarValueSizeInBits() == BitWidth &&
1410 getBooleanContents(Op0.getValueType()) ==
1411 BooleanContent::ZeroOrNegativeOneBooleanContent) {
1412 // If we're testing X < 0, then this compare isn't needed - just use X!
1413 // FIXME: We're limiting to integer types here, but this should also work
1414 // if we don't care about FP signed-zero. The use of SETLT with FP means
1415 // that we don't care about NaNs.
1416 if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
1417 (isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode())))
1418 return TLO.CombineTo(Op, Op0);
1419
1420 // TODO: Should we check for other forms of sign-bit comparisons?
1421 // Examples: X <= -1, X >= 0
1422 }
1423 if (getBooleanContents(Op0.getValueType()) ==
1424 TargetLowering::ZeroOrOneBooleanContent &&
1425 BitWidth > 1)
1426 Known.Zero.setBitsFrom(1);
1427 break;
1428 }
1429 case ISD::SHL: {
1430 SDValue Op0 = Op.getOperand(0);
1431 SDValue Op1 = Op.getOperand(1);
1432 EVT ShiftVT = Op1.getValueType();
1433
1434 if (const APInt *SA =
1435 TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
1436 unsigned ShAmt = SA->getZExtValue();
1437 if (ShAmt == 0)
1438 return TLO.CombineTo(Op, Op0);
1439
1440 // If this is ((X >>u C1) << ShAmt), see if we can simplify this into a
1441 // single shift. We can do this if the bottom bits (which are shifted
1442 // out) are never demanded.
1443 // TODO - support non-uniform vector amounts.
1444 if (Op0.getOpcode() == ISD::SRL) {
1445 if (!DemandedBits.intersects(APInt::getLowBitsSet(BitWidth, ShAmt))) {
1446 if (const APInt *SA2 =
1447 TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {
1448 unsigned C1 = SA2->getZExtValue();
1449 unsigned Opc = ISD::SHL;
1450 int Diff = ShAmt - C1;
1451 if (Diff < 0) {
1452 Diff = -Diff;
1453 Opc = ISD::SRL;
1454 }
1455 SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);
1456 return TLO.CombineTo(
1457 Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));
1458 }
1459 }
1460 }
1461
1462 // Convert (shl (anyext x, c)) to (anyext (shl x, c)) if the high bits
1463 // are not demanded. This will likely allow the anyext to be folded away.
1464 // TODO - support non-uniform vector amounts.
1465 if (Op0.getOpcode() == ISD::ANY_EXTEND) {
1466 SDValue InnerOp = Op0.getOperand(0);
1467 EVT InnerVT = InnerOp.getValueType();
1468 unsigned InnerBits = InnerVT.getScalarSizeInBits();
1469 if (ShAmt < InnerBits && DemandedBits.getActiveBits() <= InnerBits &&
1470 isTypeDesirableForOp(ISD::SHL, InnerVT)) {
1471 EVT ShTy = getShiftAmountTy(InnerVT, DL);
1472 if (!APInt(BitWidth, ShAmt).isIntN(ShTy.getSizeInBits()))
1473 ShTy = InnerVT;
1474 SDValue NarrowShl =
1475 TLO.DAG.getNode(ISD::SHL, dl, InnerVT, InnerOp,
1476 TLO.DAG.getConstant(ShAmt, dl, ShTy));
1477 return TLO.CombineTo(
1478 Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, NarrowShl));
1479 }
1480
1481 // Repeat the SHL optimization above in cases where an extension
1482 // intervenes: (shl (anyext (shr x, c1)), c2) to
1483 // (shl (anyext x), c2-c1). This requires that the bottom c1 bits
1484 // aren't demanded (as above) and that the shifted upper c1 bits of
1485 // x aren't demanded.
1486 // TODO - support non-uniform vector amounts.
1487 if (Op0.hasOneUse() && InnerOp.getOpcode() == ISD::SRL &&
1488 InnerOp.hasOneUse()) {
1489 if (const APInt *SA2 =
1490 TLO.DAG.getValidShiftAmountConstant(InnerOp, DemandedElts)) {
1491 unsigned InnerShAmt = SA2->getZExtValue();
1492 if (InnerShAmt < ShAmt && InnerShAmt < InnerBits &&
1493 DemandedBits.getActiveBits() <=
1494 (InnerBits - InnerShAmt + ShAmt) &&
1495 DemandedBits.countTrailingZeros() >= ShAmt) {
1496 SDValue NewSA =
1497 TLO.DAG.getConstant(ShAmt - InnerShAmt, dl, ShiftVT);
1498 SDValue NewExt = TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT,
1499 InnerOp.getOperand(0));
1500 return TLO.CombineTo(
1501 Op, TLO.DAG.getNode(ISD::SHL, dl, VT, NewExt, NewSA));
1502 }
1503 }
1504 }
1505 }
1506
1507 APInt InDemandedMask = DemandedBits.lshr(ShAmt);
1508 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1509 Depth + 1))
1510 return true;
1511 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1511, __extension__ __PRETTY_FUNCTION__))
;
1512 Known.Zero <<= ShAmt;
1513 Known.One <<= ShAmt;
1514 // low bits known zero.
1515 Known.Zero.setLowBits(ShAmt);
1516
1517 // Try shrinking the operation as long as the shift amount will still be
1518 // in range.
1519 if ((ShAmt < DemandedBits.getActiveBits()) &&
1520 ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1521 return true;
1522 }
1523
1524 // If we are only demanding sign bits then we can use the shift source
1525 // directly.
1526 if (const APInt *MaxSA =
1527 TLO.DAG.getValidMaximumShiftAmountConstant(Op, DemandedElts)) {
1528 unsigned ShAmt = MaxSA->getZExtValue();
1529 unsigned NumSignBits =
1530 TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
1531 unsigned UpperDemandedBits = BitWidth - DemandedBits.countTrailingZeros();
1532 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
1533 return TLO.CombineTo(Op, Op0);
1534 }
1535 break;
1536 }
1537 case ISD::SRL: {
1538 SDValue Op0 = Op.getOperand(0);
1539 SDValue Op1 = Op.getOperand(1);
1540 EVT ShiftVT = Op1.getValueType();
1541
1542 if (const APInt *SA =
1543 TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
1544 unsigned ShAmt = SA->getZExtValue();
1545 if (ShAmt == 0)
1546 return TLO.CombineTo(Op, Op0);
1547
1548 // If this is ((X << C1) >>u ShAmt), see if we can simplify this into a
1549 // single shift. We can do this if the top bits (which are shifted out)
1550 // are never demanded.
1551 // TODO - support non-uniform vector amounts.
1552 if (Op0.getOpcode() == ISD::SHL) {
1553 if (!DemandedBits.intersects(APInt::getHighBitsSet(BitWidth, ShAmt))) {
1554 if (const APInt *SA2 =
1555 TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {
1556 unsigned C1 = SA2->getZExtValue();
1557 unsigned Opc = ISD::SRL;
1558 int Diff = ShAmt - C1;
1559 if (Diff < 0) {
1560 Diff = -Diff;
1561 Opc = ISD::SHL;
1562 }
1563 SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);
1564 return TLO.CombineTo(
1565 Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));
1566 }
1567 }
1568 }
1569
1570 APInt InDemandedMask = (DemandedBits << ShAmt);
1571
1572 // If the shift is exact, then it does demand the low bits (and knows that
1573 // they are zero).
1574 if (Op->getFlags().hasExact())
1575 InDemandedMask.setLowBits(ShAmt);
1576
1577 // Compute the new bits that are at the top now.
1578 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1579 Depth + 1))
1580 return true;
1581 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1581, __extension__ __PRETTY_FUNCTION__))
;
1582 Known.Zero.lshrInPlace(ShAmt);
1583 Known.One.lshrInPlace(ShAmt);
1584 // High bits known zero.
1585 Known.Zero.setHighBits(ShAmt);
1586 }
1587 break;
1588 }
1589 case ISD::SRA: {
1590 SDValue Op0 = Op.getOperand(0);
1591 SDValue Op1 = Op.getOperand(1);
1592 EVT ShiftVT = Op1.getValueType();
1593
1594 // If we only want bits that already match the signbit then we don't need
1595 // to shift.
1596 unsigned NumHiDemandedBits = BitWidth - DemandedBits.countTrailingZeros();
1597 if (TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1) >=
1598 NumHiDemandedBits)
1599 return TLO.CombineTo(Op, Op0);
1600
1601 // If this is an arithmetic shift right and only the low-bit is set, we can
1602 // always convert this into a logical shr, even if the shift amount is
1603 // variable. The low bit of the shift cannot be an input sign bit unless
1604 // the shift amount is >= the size of the datatype, which is undefined.
1605 if (DemandedBits.isOneValue())
1606 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1));
1607
1608 if (const APInt *SA =
1609 TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
1610 unsigned ShAmt = SA->getZExtValue();
1611 if (ShAmt == 0)
1612 return TLO.CombineTo(Op, Op0);
1613
1614 APInt InDemandedMask = (DemandedBits << ShAmt);
1615
1616 // If the shift is exact, then it does demand the low bits (and knows that
1617 // they are zero).
1618 if (Op->getFlags().hasExact())
1619 InDemandedMask.setLowBits(ShAmt);
1620
1621 // If any of the demanded bits are produced by the sign extension, we also
1622 // demand the input sign bit.
1623 if (DemandedBits.countLeadingZeros() < ShAmt)
1624 InDemandedMask.setSignBit();
1625
1626 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1627 Depth + 1))
1628 return true;
1629 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1629, __extension__ __PRETTY_FUNCTION__))
;
1630 Known.Zero.lshrInPlace(ShAmt);
1631 Known.One.lshrInPlace(ShAmt);
1632
1633 // If the input sign bit is known to be zero, or if none of the top bits
1634 // are demanded, turn this into an unsigned shift right.
1635 if (Known.Zero[BitWidth - ShAmt - 1] ||
1636 DemandedBits.countLeadingZeros() >= ShAmt) {
1637 SDNodeFlags Flags;
1638 Flags.setExact(Op->getFlags().hasExact());
1639 return TLO.CombineTo(
1640 Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1, Flags));
1641 }
1642
1643 int Log2 = DemandedBits.exactLogBase2();
1644 if (Log2 >= 0) {
1645 // The bit must come from the sign.
1646 SDValue NewSA = TLO.DAG.getConstant(BitWidth - 1 - Log2, dl, ShiftVT);
1647 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, NewSA));
1648 }
1649
1650 if (Known.One[BitWidth - ShAmt - 1])
1651 // New bits are known one.
1652 Known.One.setHighBits(ShAmt);
1653
1654 // Attempt to avoid multi-use ops if we don't need anything from them.
1655 if (!InDemandedMask.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {
1656 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1657 Op0, InDemandedMask, DemandedElts, TLO.DAG, Depth + 1);
1658 if (DemandedOp0) {
1659 SDValue NewOp = TLO.DAG.getNode(ISD::SRA, dl, VT, DemandedOp0, Op1);
1660 return TLO.CombineTo(Op, NewOp);
1661 }
1662 }
1663 }
1664 break;
1665 }
1666 case ISD::FSHL:
1667 case ISD::FSHR: {
1668 SDValue Op0 = Op.getOperand(0);
1669 SDValue Op1 = Op.getOperand(1);
1670 SDValue Op2 = Op.getOperand(2);
1671 bool IsFSHL = (Op.getOpcode() == ISD::FSHL);
1672
1673 if (ConstantSDNode *SA = isConstOrConstSplat(Op2, DemandedElts)) {
1674 unsigned Amt = SA->getAPIntValue().urem(BitWidth);
1675
1676 // For fshl, 0-shift returns the 1st arg.
1677 // For fshr, 0-shift returns the 2nd arg.
1678 if (Amt == 0) {
1679 if (SimplifyDemandedBits(IsFSHL ? Op0 : Op1, DemandedBits, DemandedElts,
1680 Known, TLO, Depth + 1))
1681 return true;
1682 break;
1683 }
1684
1685 // fshl: (Op0 << Amt) | (Op1 >> (BW - Amt))
1686 // fshr: (Op0 << (BW - Amt)) | (Op1 >> Amt)
1687 APInt Demanded0 = DemandedBits.lshr(IsFSHL ? Amt : (BitWidth - Amt));
1688 APInt Demanded1 = DemandedBits << (IsFSHL ? (BitWidth - Amt) : Amt);
1689 if (SimplifyDemandedBits(Op0, Demanded0, DemandedElts, Known2, TLO,
1690 Depth + 1))
1691 return true;
1692 if (SimplifyDemandedBits(Op1, Demanded1, DemandedElts, Known, TLO,
1693 Depth + 1))
1694 return true;
1695
1696 Known2.One <<= (IsFSHL ? Amt : (BitWidth - Amt));
1697 Known2.Zero <<= (IsFSHL ? Amt : (BitWidth - Amt));
1698 Known.One.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);
1699 Known.Zero.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);
1700 Known.One |= Known2.One;
1701 Known.Zero |= Known2.Zero;
1702 }
1703
1704 // For pow-2 bitwidths we only demand the bottom modulo amt bits.
1705 if (isPowerOf2_32(BitWidth)) {
1706 APInt DemandedAmtBits(Op2.getScalarValueSizeInBits(), BitWidth - 1);
1707 if (SimplifyDemandedBits(Op2, DemandedAmtBits, DemandedElts,
1708 Known2, TLO, Depth + 1))
1709 return true;
1710 }
1711 break;
1712 }
1713 case ISD::ROTL:
1714 case ISD::ROTR: {
1715 SDValue Op0 = Op.getOperand(0);
1716 SDValue Op1 = Op.getOperand(1);
1717
1718 // If we're rotating an 0/-1 value, then it stays an 0/-1 value.
1719 if (BitWidth == TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1))
1720 return TLO.CombineTo(Op, Op0);
1721
1722 // For pow-2 bitwidths we only demand the bottom modulo amt bits.
1723 if (isPowerOf2_32(BitWidth)) {
1724 APInt DemandedAmtBits(Op1.getScalarValueSizeInBits(), BitWidth - 1);
1725 if (SimplifyDemandedBits(Op1, DemandedAmtBits, DemandedElts, Known2, TLO,
1726 Depth + 1))
1727 return true;
1728 }
1729 break;
1730 }
1731 case ISD::UMIN: {
1732 // Check if one arg is always less than (or equal) to the other arg.
1733 SDValue Op0 = Op.getOperand(0);
1734 SDValue Op1 = Op.getOperand(1);
1735 KnownBits Known0 = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth + 1);
1736 KnownBits Known1 = TLO.DAG.computeKnownBits(Op1, DemandedElts, Depth + 1);
1737 Known = KnownBits::umin(Known0, Known1);
1738 if (Optional<bool> IsULE = KnownBits::ule(Known0, Known1))
1739 return TLO.CombineTo(Op, IsULE.getValue() ? Op0 : Op1);
1740 if (Optional<bool> IsULT = KnownBits::ult(Known0, Known1))
1741 return TLO.CombineTo(Op, IsULT.getValue() ? Op0 : Op1);
1742 break;
1743 }
1744 case ISD::UMAX: {
1745 // Check if one arg is always greater than (or equal) to the other arg.
1746 SDValue Op0 = Op.getOperand(0);
1747 SDValue Op1 = Op.getOperand(1);
1748 KnownBits Known0 = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth + 1);
1749 KnownBits Known1 = TLO.DAG.computeKnownBits(Op1, DemandedElts, Depth + 1);
1750 Known = KnownBits::umax(Known0, Known1);
1751 if (Optional<bool> IsUGE = KnownBits::uge(Known0, Known1))
1752 return TLO.CombineTo(Op, IsUGE.getValue() ? Op0 : Op1);
1753 if (Optional<bool> IsUGT = KnownBits::ugt(Known0, Known1))
1754 return TLO.CombineTo(Op, IsUGT.getValue() ? Op0 : Op1);
1755 break;
1756 }
1757 case ISD::BITREVERSE: {
1758 SDValue Src = Op.getOperand(0);
1759 APInt DemandedSrcBits = DemandedBits.reverseBits();
1760 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
1761 Depth + 1))
1762 return true;
1763 Known.One = Known2.One.reverseBits();
1764 Known.Zero = Known2.Zero.reverseBits();
1765 break;
1766 }
1767 case ISD::BSWAP: {
1768 SDValue Src = Op.getOperand(0);
1769 APInt DemandedSrcBits = DemandedBits.byteSwap();
1770 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
1771 Depth + 1))
1772 return true;
1773 Known.One = Known2.One.byteSwap();
1774 Known.Zero = Known2.Zero.byteSwap();
1775 break;
1776 }
1777 case ISD::CTPOP: {
1778 // If only 1 bit is demanded, replace with PARITY as long as we're before
1779 // op legalization.
1780 // FIXME: Limit to scalars for now.
1781 if (DemandedBits.isOneValue() && !TLO.LegalOps && !VT.isVector())
1782 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::PARITY, dl, VT,
1783 Op.getOperand(0)));
1784
1785 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
1786 break;
1787 }
1788 case ISD::SIGN_EXTEND_INREG: {
1789 SDValue Op0 = Op.getOperand(0);
1790 EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
1791 unsigned ExVTBits = ExVT.getScalarSizeInBits();
1792
1793 // If we only care about the highest bit, don't bother shifting right.
1794 if (DemandedBits.isSignMask()) {
1795 unsigned NumSignBits =
1796 TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
1797 bool AlreadySignExtended = NumSignBits >= BitWidth - ExVTBits + 1;
1798 // However if the input is already sign extended we expect the sign
1799 // extension to be dropped altogether later and do not simplify.
1800 if (!AlreadySignExtended) {
1801 // Compute the correct shift amount type, which must be getShiftAmountTy
1802 // for scalar types after legalization.
1803 EVT ShiftAmtTy = VT;
1804 if (TLO.LegalTypes() && !ShiftAmtTy.isVector())
1805 ShiftAmtTy = getShiftAmountTy(ShiftAmtTy, DL);
1806
1807 SDValue ShiftAmt =
1808 TLO.DAG.getConstant(BitWidth - ExVTBits, dl, ShiftAmtTy);
1809 return TLO.CombineTo(Op,
1810 TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, ShiftAmt));
1811 }
1812 }
1813
1814 // If none of the extended bits are demanded, eliminate the sextinreg.
1815 if (DemandedBits.getActiveBits() <= ExVTBits)
1816 return TLO.CombineTo(Op, Op0);
1817
1818 APInt InputDemandedBits = DemandedBits.getLoBits(ExVTBits);
1819
1820 // Since the sign extended bits are demanded, we know that the sign
1821 // bit is demanded.
1822 InputDemandedBits.setBit(ExVTBits - 1);
1823
1824 if (SimplifyDemandedBits(Op0, InputDemandedBits, Known, TLO, Depth + 1))
1825 return true;
1826 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1826, __extension__ __PRETTY_FUNCTION__))
;
1827
1828 // If the sign bit of the input is known set or clear, then we know the
1829 // top bits of the result.
1830
1831 // If the input sign bit is known zero, convert this into a zero extension.
1832 if (Known.Zero[ExVTBits - 1])
1833 return TLO.CombineTo(Op, TLO.DAG.getZeroExtendInReg(Op0, dl, ExVT));
1834
1835 APInt Mask = APInt::getLowBitsSet(BitWidth, ExVTBits);
1836 if (Known.One[ExVTBits - 1]) { // Input sign bit known set
1837 Known.One.setBitsFrom(ExVTBits);
1838 Known.Zero &= Mask;
1839 } else { // Input sign bit unknown
1840 Known.Zero &= Mask;
1841 Known.One &= Mask;
1842 }
1843 break;
1844 }
1845 case ISD::BUILD_PAIR: {
1846 EVT HalfVT = Op.getOperand(0).getValueType();
1847 unsigned HalfBitWidth = HalfVT.getScalarSizeInBits();
1848
1849 APInt MaskLo = DemandedBits.getLoBits(HalfBitWidth).trunc(HalfBitWidth);
1850 APInt MaskHi = DemandedBits.getHiBits(HalfBitWidth).trunc(HalfBitWidth);
1851
1852 KnownBits KnownLo, KnownHi;
1853
1854 if (SimplifyDemandedBits(Op.getOperand(0), MaskLo, KnownLo, TLO, Depth + 1))
1855 return true;
1856
1857 if (SimplifyDemandedBits(Op.getOperand(1), MaskHi, KnownHi, TLO, Depth + 1))
1858 return true;
1859
1860 Known.Zero = KnownLo.Zero.zext(BitWidth) |
1861 KnownHi.Zero.zext(BitWidth).shl(HalfBitWidth);
1862
1863 Known.One = KnownLo.One.zext(BitWidth) |
1864 KnownHi.One.zext(BitWidth).shl(HalfBitWidth);
1865 break;
1866 }
1867 case ISD::ZERO_EXTEND:
1868 case ISD::ZERO_EXTEND_VECTOR_INREG: {
1869 SDValue Src = Op.getOperand(0);
1870 EVT SrcVT = Src.getValueType();
1871 unsigned InBits = SrcVT.getScalarSizeInBits();
1872 unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
1873 bool IsVecInReg = Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG;
1874
1875 // If none of the top bits are demanded, convert this into an any_extend.
1876 if (DemandedBits.getActiveBits() <= InBits) {
1877 // If we only need the non-extended bits of the bottom element
1878 // then we can just bitcast to the result.
1879 if (IsVecInReg && DemandedElts == 1 &&
1880 VT.getSizeInBits() == SrcVT.getSizeInBits() &&
1881 TLO.DAG.getDataLayout().isLittleEndian())
1882 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
1883
1884 unsigned Opc =
1885 IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;
1886 if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
1887 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
1888 }
1889
1890 APInt InDemandedBits = DemandedBits.trunc(InBits);
1891 APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
1892 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
1893 Depth + 1))
1894 return true;
1895 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1895, __extension__ __PRETTY_FUNCTION__))
;
1896 assert(Known.getBitWidth() == InBits && "Src width has changed?")(static_cast <bool> (Known.getBitWidth() == InBits &&
"Src width has changed?") ? void (0) : __assert_fail ("Known.getBitWidth() == InBits && \"Src width has changed?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1896, __extension__ __PRETTY_FUNCTION__))
;
1897 Known = Known.zext(BitWidth);
1898
1899 // Attempt to avoid multi-use ops if we don't need anything from them.
1900 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1901 Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
1902 return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
1903 break;
1904 }
1905 case ISD::SIGN_EXTEND:
1906 case ISD::SIGN_EXTEND_VECTOR_INREG: {
1907 SDValue Src = Op.getOperand(0);
1908 EVT SrcVT = Src.getValueType();
1909 unsigned InBits = SrcVT.getScalarSizeInBits();
1910 unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
1911 bool IsVecInReg = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG;
1912
1913 // If none of the top bits are demanded, convert this into an any_extend.
1914 if (DemandedBits.getActiveBits() <= InBits) {
1915 // If we only need the non-extended bits of the bottom element
1916 // then we can just bitcast to the result.
1917 if (IsVecInReg && DemandedElts == 1 &&
1918 VT.getSizeInBits() == SrcVT.getSizeInBits() &&
1919 TLO.DAG.getDataLayout().isLittleEndian())
1920 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
1921
1922 unsigned Opc =
1923 IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;
1924 if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
1925 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
1926 }
1927
1928 APInt InDemandedBits = DemandedBits.trunc(InBits);
1929 APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
1930
1931 // Since some of the sign extended bits are demanded, we know that the sign
1932 // bit is demanded.
1933 InDemandedBits.setBit(InBits - 1);
1934
1935 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
1936 Depth + 1))
1937 return true;
1938 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1938, __extension__ __PRETTY_FUNCTION__))
;
1939 assert(Known.getBitWidth() == InBits && "Src width has changed?")(static_cast <bool> (Known.getBitWidth() == InBits &&
"Src width has changed?") ? void (0) : __assert_fail ("Known.getBitWidth() == InBits && \"Src width has changed?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1939, __extension__ __PRETTY_FUNCTION__))
;
1940
1941 // If the sign bit is known one, the top bits match.
1942 Known = Known.sext(BitWidth);
1943
1944 // If the sign bit is known zero, convert this to a zero extend.
1945 if (Known.isNonNegative()) {
1946 unsigned Opc =
1947 IsVecInReg ? ISD::ZERO_EXTEND_VECTOR_INREG : ISD::ZERO_EXTEND;
1948 if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
1949 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
1950 }
1951
1952 // Attempt to avoid multi-use ops if we don't need anything from them.
1953 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1954 Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
1955 return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
1956 break;
1957 }
1958 case ISD::ANY_EXTEND:
1959 case ISD::ANY_EXTEND_VECTOR_INREG: {
1960 SDValue Src = Op.getOperand(0);
1961 EVT SrcVT = Src.getValueType();
1962 unsigned InBits = SrcVT.getScalarSizeInBits();
1963 unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
1964 bool IsVecInReg = Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG;
1965
1966 // If we only need the bottom element then we can just bitcast.
1967 // TODO: Handle ANY_EXTEND?
1968 if (IsVecInReg && DemandedElts == 1 &&
1969 VT.getSizeInBits() == SrcVT.getSizeInBits() &&
1970 TLO.DAG.getDataLayout().isLittleEndian())
1971 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
1972
1973 APInt InDemandedBits = DemandedBits.trunc(InBits);
1974 APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
1975 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
1976 Depth + 1))
1977 return true;
1978 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1978, __extension__ __PRETTY_FUNCTION__))
;
1979 assert(Known.getBitWidth() == InBits && "Src width has changed?")(static_cast <bool> (Known.getBitWidth() == InBits &&
"Src width has changed?") ? void (0) : __assert_fail ("Known.getBitWidth() == InBits && \"Src width has changed?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1979, __extension__ __PRETTY_FUNCTION__))
;
1980 Known = Known.anyext(BitWidth);
1981
1982 // Attempt to avoid multi-use ops if we don't need anything from them.
1983 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1984 Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
1985 return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
1986 break;
1987 }
1988 case ISD::TRUNCATE: {
1989 SDValue Src = Op.getOperand(0);
1990
1991 // Simplify the input, using demanded bit information, and compute the known
1992 // zero/one bits live out.
1993 unsigned OperandBitWidth = Src.getScalarValueSizeInBits();
1994 APInt TruncMask = DemandedBits.zext(OperandBitWidth);
1995 if (SimplifyDemandedBits(Src, TruncMask, DemandedElts, Known, TLO,
1996 Depth + 1))
1997 return true;
1998 Known = Known.trunc(BitWidth);
1999
2000 // Attempt to avoid multi-use ops if we don't need anything from them.
2001 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2002 Src, TruncMask, DemandedElts, TLO.DAG, Depth + 1))
2003 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, NewSrc));
2004
2005 // If the input is only used by this truncate, see if we can shrink it based
2006 // on the known demanded bits.
2007 if (Src.getNode()->hasOneUse()) {
2008 switch (Src.getOpcode()) {
2009 default:
2010 break;
2011 case ISD::SRL:
2012 // Shrink SRL by a constant if none of the high bits shifted in are
2013 // demanded.
2014 if (TLO.LegalTypes() && !isTypeDesirableForOp(ISD::SRL, VT))
2015 // Do not turn (vt1 truncate (vt2 srl)) into (vt1 srl) if vt1 is
2016 // undesirable.
2017 break;
2018
2019 const APInt *ShAmtC =
2020 TLO.DAG.getValidShiftAmountConstant(Src, DemandedElts);
2021 if (!ShAmtC || ShAmtC->uge(BitWidth))
2022 break;
2023 uint64_t ShVal = ShAmtC->getZExtValue();
2024
2025 APInt HighBits =
2026 APInt::getHighBitsSet(OperandBitWidth, OperandBitWidth - BitWidth);
2027 HighBits.lshrInPlace(ShVal);
2028 HighBits = HighBits.trunc(BitWidth);
2029
2030 if (!(HighBits & DemandedBits)) {
2031 // None of the shifted in bits are needed. Add a truncate of the
2032 // shift input, then shift it.
2033 SDValue NewShAmt = TLO.DAG.getConstant(
2034 ShVal, dl, getShiftAmountTy(VT, DL, TLO.LegalTypes()));
2035 SDValue NewTrunc =
2036 TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, Src.getOperand(0));
2037 return TLO.CombineTo(
2038 Op, TLO.DAG.getNode(ISD::SRL, dl, VT, NewTrunc, NewShAmt));
2039 }
2040 break;
2041 }
2042 }
2043
2044 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2044, __extension__ __PRETTY_FUNCTION__))
;
2045 break;
2046 }
2047 case ISD::AssertZext: {
2048 // AssertZext demands all of the high bits, plus any of the low bits
2049 // demanded by its users.
2050 EVT ZVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2051 APInt InMask = APInt::getLowBitsSet(BitWidth, ZVT.getSizeInBits());
2052 if (SimplifyDemandedBits(Op.getOperand(0), ~InMask | DemandedBits, Known,
2053 TLO, Depth + 1))
2054 return true;
2055 assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2055, __extension__ __PRETTY_FUNCTION__))
;
2056
2057 Known.Zero |= ~InMask;
2058 break;
2059 }
2060 case ISD::EXTRACT_VECTOR_ELT: {
2061 SDValue Src = Op.getOperand(0);
2062 SDValue Idx = Op.getOperand(1);
2063 ElementCount SrcEltCnt = Src.getValueType().getVectorElementCount();
2064 unsigned EltBitWidth = Src.getScalarValueSizeInBits();
2065
2066 if (SrcEltCnt.isScalable())
2067 return false;
2068
2069 // Demand the bits from every vector element without a constant index.
2070 unsigned NumSrcElts = SrcEltCnt.getFixedValue();
2071 APInt DemandedSrcElts = APInt::getAllOnesValue(NumSrcElts);
2072 if (auto *CIdx = dyn_cast<ConstantSDNode>(Idx))
2073 if (CIdx->getAPIntValue().ult(NumSrcElts))
2074 DemandedSrcElts = APInt::getOneBitSet(NumSrcElts, CIdx->getZExtValue());
2075
2076 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
2077 // anything about the extended bits.
2078 APInt DemandedSrcBits = DemandedBits;
2079 if (BitWidth > EltBitWidth)
2080 DemandedSrcBits = DemandedSrcBits.trunc(EltBitWidth);
2081
2082 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts, Known2, TLO,
2083 Depth + 1))
2084 return true;
2085
2086 // Attempt to avoid multi-use ops if we don't need anything from them.
2087 if (!DemandedSrcBits.isAllOnesValue() ||
2088 !DemandedSrcElts.isAllOnesValue()) {
2089 if (SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
2090 Src, DemandedSrcBits, DemandedSrcElts, TLO.DAG, Depth + 1)) {
2091 SDValue NewOp =
2092 TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc, Idx);
2093 return TLO.CombineTo(Op, NewOp);
2094 }
2095 }
2096
2097 Known = Known2;
2098 if (BitWidth > EltBitWidth)
2099 Known = Known.anyext(BitWidth);
2100 break;
2101 }
2102 case ISD::BITCAST: {
2103 SDValue Src = Op.getOperand(0);
2104 EVT SrcVT = Src.getValueType();
2105 unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
2106
2107 // If this is an FP->Int bitcast and if the sign bit is the only
2108 // thing demanded, turn this into a FGETSIGN.
2109 if (!TLO.LegalOperations() && !VT.isVector() && !SrcVT.isVector() &&
2110 DemandedBits == APInt::getSignMask(Op.getValueSizeInBits()) &&
2111 SrcVT.isFloatingPoint()) {
2112 bool OpVTLegal = isOperationLegalOrCustom(ISD::FGETSIGN, VT);
2113 bool i32Legal = isOperationLegalOrCustom(ISD::FGETSIGN, MVT::i32);
2114 if ((OpVTLegal || i32Legal) && VT.isSimple() && SrcVT != MVT::f16 &&
2115 SrcVT != MVT::f128) {
2116 // Cannot eliminate/lower SHL for f128 yet.
2117 EVT Ty = OpVTLegal ? VT : MVT::i32;
2118 // Make a FGETSIGN + SHL to move the sign bit into the appropriate
2119 // place. We expect the SHL to be eliminated by other optimizations.
2120 SDValue Sign = TLO.DAG.getNode(ISD::FGETSIGN, dl, Ty, Src);
2121 unsigned OpVTSizeInBits = Op.getValueSizeInBits();
2122 if (!OpVTLegal && OpVTSizeInBits > 32)
2123 Sign = TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Sign);
2124 unsigned ShVal = Op.getValueSizeInBits() - 1;
2125 SDValue ShAmt = TLO.DAG.getConstant(ShVal, dl, VT);
2126 return TLO.CombineTo(Op,
2127 TLO.DAG.getNode(ISD::SHL, dl, VT, Sign, ShAmt));
2128 }
2129 }
2130
2131 // Bitcast from a vector using SimplifyDemanded Bits/VectorElts.
2132 // Demand the elt/bit if any of the original elts/bits are demanded.
2133 // TODO - bigendian once we have test coverage.
2134 if (SrcVT.isVector() && (BitWidth % NumSrcEltBits) == 0 &&
2135 TLO.DAG.getDataLayout().isLittleEndian()) {
2136 unsigned Scale = BitWidth / NumSrcEltBits;
2137 unsigned NumSrcElts = SrcVT.getVectorNumElements();
2138 APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);
2139 APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);
2140 for (unsigned i = 0; i != Scale; ++i) {
2141 unsigned Offset = i * NumSrcEltBits;
2142 APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset);
2143 if (!Sub.isNullValue()) {
2144 DemandedSrcBits |= Sub;
2145 for (unsigned j = 0; j != NumElts; ++j)
2146 if (DemandedElts[j])
2147 DemandedSrcElts.setBit((j * Scale) + i);
2148 }
2149 }
2150
2151 APInt KnownSrcUndef, KnownSrcZero;
2152 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
2153 KnownSrcZero, TLO, Depth + 1))
2154 return true;
2155
2156 KnownBits KnownSrcBits;
2157 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
2158 KnownSrcBits, TLO, Depth + 1))
2159 return true;
2160 } else if ((NumSrcEltBits % BitWidth) == 0 &&
2161 TLO.DAG.getDataLayout().isLittleEndian()) {
2162 unsigned Scale = NumSrcEltBits / BitWidth;
2163 unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
2164 APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);
2165 APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);
2166 for (unsigned i = 0; i != NumElts; ++i)
2167 if (DemandedElts[i]) {
2168 unsigned Offset = (i % Scale) * BitWidth;
2169 DemandedSrcBits.insertBits(DemandedBits, Offset);
2170 DemandedSrcElts.setBit(i / Scale);
2171 }
2172
2173 if (SrcVT.isVector()) {
2174 APInt KnownSrcUndef, KnownSrcZero;
2175 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
2176 KnownSrcZero, TLO, Depth + 1))
2177 return true;
2178 }
2179
2180 KnownBits KnownSrcBits;
2181 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
2182 KnownSrcBits, TLO, Depth + 1))
2183 return true;
2184 }
2185
2186 // If this is a bitcast, let computeKnownBits handle it. Only do this on a
2187 // recursive call where Known may be useful to the caller.
2188 if (Depth > 0) {
2189 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2190 return false;
2191 }
2192 break;
2193 }
2194 case ISD::ADD:
2195 case ISD::MUL:
2196 case ISD::SUB: {
2197 // Add, Sub, and Mul don't demand any bits in positions beyond that
2198 // of the highest bit demanded of them.
2199 SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1);
2200 SDNodeFlags Flags = Op.getNode()->getFlags();
2201 unsigned DemandedBitsLZ = DemandedBits.countLeadingZeros();
2202 APInt LoMask = APInt::getLowBitsSet(BitWidth, BitWidth - DemandedBitsLZ);
2203 if (SimplifyDemandedBits(Op0, LoMask, DemandedElts, Known2, TLO,
2204 Depth + 1) ||
2205 SimplifyDemandedBits(Op1, LoMask, DemandedElts, Known2, TLO,
2206 Depth + 1) ||
2207 // See if the operation should be performed at a smaller bit width.
2208 ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO)) {
2209 if (Flags.hasNoSignedWrap() || Flags.hasNoUnsignedWrap()) {
2210 // Disable the nsw and nuw flags. We can no longer guarantee that we
2211 // won't wrap after simplification.
2212 Flags.setNoSignedWrap(false);
2213 Flags.setNoUnsignedWrap(false);
2214 SDValue NewOp =
2215 TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);
2216 return TLO.CombineTo(Op, NewOp);
2217 }
2218 return true;
2219 }
2220
2221 // Attempt to avoid multi-use ops if we don't need anything from them.
2222 if (!LoMask.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {
2223 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2224 Op0, LoMask, DemandedElts, TLO.DAG, Depth + 1);
2225 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
2226 Op1, LoMask, DemandedElts, TLO.DAG, Depth + 1);
2227 if (DemandedOp0 || DemandedOp1) {
2228 Flags.setNoSignedWrap(false);
2229 Flags.setNoUnsignedWrap(false);
2230 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
2231 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
2232 SDValue NewOp =
2233 TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);
2234 return TLO.CombineTo(Op, NewOp);
2235 }
2236 }
2237
2238 // If we have a constant operand, we may be able to turn it into -1 if we
2239 // do not demand the high bits. This can make the constant smaller to
2240 // encode, allow more general folding, or match specialized instruction
2241 // patterns (eg, 'blsr' on x86). Don't bother changing 1 to -1 because that
2242 // is probably not useful (and could be detrimental).
2243 ConstantSDNode *C = isConstOrConstSplat(Op1);
2244 APInt HighMask = APInt::getHighBitsSet(BitWidth, DemandedBitsLZ);
2245 if (C && !C->isAllOnesValue() && !C->isOne() &&
2246 (C->getAPIntValue() | HighMask).isAllOnesValue()) {
2247 SDValue Neg1 = TLO.DAG.getAllOnesConstant(dl, VT);
2248 // Disable the nsw and nuw flags. We can no longer guarantee that we
2249 // won't wrap after simplification.
2250 Flags.setNoSignedWrap(false);
2251 Flags.setNoUnsignedWrap(false);
2252 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Neg1, Flags);
2253 return TLO.CombineTo(Op, NewOp);
2254 }
2255
2256 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2257 }
2258 default:
2259 if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
2260 if (SimplifyDemandedBitsForTargetNode(Op, DemandedBits, DemandedElts,
2261 Known, TLO, Depth))
2262 return true;
2263 break;
2264 }
2265
2266 // Just use computeKnownBits to compute output bits.
2267 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2268 break;
2269 }
2270
2271 // If we know the value of all of the demanded bits, return this as a
2272 // constant.
2273 if (DemandedBits.isSubsetOf(Known.Zero | Known.One)) {
2274 // Avoid folding to a constant if any OpaqueConstant is involved.
2275 const SDNode *N = Op.getNode();
2276 for (SDNode *Op :
2277 llvm::make_range(SDNodeIterator::begin(N), SDNodeIterator::end(N))) {
2278 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op))
2279 if (C->isOpaque())
2280 return false;
2281 }
2282 if (VT.isInteger())
2283 return TLO.CombineTo(Op, TLO.DAG.getConstant(Known.One, dl, VT));
2284 if (VT.isFloatingPoint())
2285 return TLO.CombineTo(
2286 Op,
2287 TLO.DAG.getConstantFP(
2288 APFloat(TLO.DAG.EVTToAPFloatSemantics(VT), Known.One), dl, VT));
2289 }
2290
2291 return false;
2292}
2293
2294bool TargetLowering::SimplifyDemandedVectorElts(SDValue Op,
2295 const APInt &DemandedElts,
2296 APInt &KnownUndef,
2297 APInt &KnownZero,
2298 DAGCombinerInfo &DCI) const {
2299 SelectionDAG &DAG = DCI.DAG;
2300 TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
2301 !DCI.isBeforeLegalizeOps());
2302
2303 bool Simplified =
2304 SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero, TLO);
2305 if (Simplified) {
2306 DCI.AddToWorklist(Op.getNode());
2307 DCI.CommitTargetLoweringOpt(TLO);
2308 }
2309
2310 return Simplified;
2311}
2312
2313/// Given a vector binary operation and known undefined elements for each input
2314/// operand, compute whether each element of the output is undefined.
2315static APInt getKnownUndefForVectorBinop(SDValue BO, SelectionDAG &DAG,
2316 const APInt &UndefOp0,
2317 const APInt &UndefOp1) {
2318 EVT VT = BO.getValueType();
2319 assert(DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() &&(static_cast <bool> (DAG.getTargetLoweringInfo().isBinOp
(BO.getOpcode()) && VT.isVector() && "Vector binop only"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() && \"Vector binop only\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2320, __extension__ __PRETTY_FUNCTION__))
2320 "Vector binop only")(static_cast <bool> (DAG.getTargetLoweringInfo().isBinOp
(BO.getOpcode()) && VT.isVector() && "Vector binop only"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() && \"Vector binop only\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2320, __extension__ __PRETTY_FUNCTION__))
;
2321
2322 EVT EltVT = VT.getVectorElementType();
2323 unsigned NumElts = VT.getVectorNumElements();
2324 assert(UndefOp0.getBitWidth() == NumElts &&(static_cast <bool> (UndefOp0.getBitWidth() == NumElts &&
UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis"
) ? void (0) : __assert_fail ("UndefOp0.getBitWidth() == NumElts && UndefOp1.getBitWidth() == NumElts && \"Bad type for undef analysis\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2325, __extension__ __PRETTY_FUNCTION__))
2325 UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis")(static_cast <bool> (UndefOp0.getBitWidth() == NumElts &&
UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis"
) ? void (0) : __assert_fail ("UndefOp0.getBitWidth() == NumElts && UndefOp1.getBitWidth() == NumElts && \"Bad type for undef analysis\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2325, __extension__ __PRETTY_FUNCTION__))
;
2326
2327 auto getUndefOrConstantElt = [&](SDValue V, unsigned Index,
2328 const APInt &UndefVals) {
2329 if (UndefVals[Index])
2330 return DAG.getUNDEF(EltVT);
2331
2332 if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
2333 // Try hard to make sure that the getNode() call is not creating temporary
2334 // nodes. Ignore opaque integers because they do not constant fold.
2335 SDValue Elt = BV->getOperand(Index);
2336 auto *C = dyn_cast<ConstantSDNode>(Elt);
2337 if (isa<ConstantFPSDNode>(Elt) || Elt.isUndef() || (C && !C->isOpaque()))
2338 return Elt;
2339 }
2340
2341 return SDValue();
2342 };
2343
2344 APInt KnownUndef = APInt::getNullValue(NumElts);
2345 for (unsigned i = 0; i != NumElts; ++i) {
2346 // If both inputs for this element are either constant or undef and match
2347 // the element type, compute the constant/undef result for this element of
2348 // the vector.
2349 // TODO: Ideally we would use FoldConstantArithmetic() here, but that does
2350 // not handle FP constants. The code within getNode() should be refactored
2351 // to avoid the danger of creating a bogus temporary node here.
2352 SDValue C0 = getUndefOrConstantElt(BO.getOperand(0), i, UndefOp0);
2353 SDValue C1 = getUndefOrConstantElt(BO.getOperand(1), i, UndefOp1);
2354 if (C0 && C1 && C0.getValueType() == EltVT && C1.getValueType() == EltVT)
2355 if (DAG.getNode(BO.getOpcode(), SDLoc(BO), EltVT, C0, C1).isUndef())
2356 KnownUndef.setBit(i);
2357 }
2358 return KnownUndef;
2359}
2360
2361bool TargetLowering::SimplifyDemandedVectorElts(
2362 SDValue Op, const APInt &OriginalDemandedElts, APInt &KnownUndef,
2363 APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth,
2364 bool AssumeSingleUse) const {
2365 EVT VT = Op.getValueType();
2366 unsigned Opcode = Op.getOpcode();
2367 APInt DemandedElts = OriginalDemandedElts;
2368 unsigned NumElts = DemandedElts.getBitWidth();
2369 assert(VT.isVector() && "Expected vector op")(static_cast <bool> (VT.isVector() && "Expected vector op"
) ? void (0) : __assert_fail ("VT.isVector() && \"Expected vector op\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2369, __extension__ __PRETTY_FUNCTION__))
;
2370
2371 KnownUndef = KnownZero = APInt::getNullValue(NumElts);
2372
2373 // TODO: For now we assume we know nothing about scalable vectors.
2374 if (VT.isScalableVector())
2375 return false;
2376
2377 assert(VT.getVectorNumElements() == NumElts &&(static_cast <bool> (VT.getVectorNumElements() == NumElts
&& "Mask size mismatches value type element count!")
? void (0) : __assert_fail ("VT.getVectorNumElements() == NumElts && \"Mask size mismatches value type element count!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2378, __extension__ __PRETTY_FUNCTION__))
2378 "Mask size mismatches value type element count!")(static_cast <bool> (VT.getVectorNumElements() == NumElts
&& "Mask size mismatches value type element count!")
? void (0) : __assert_fail ("VT.getVectorNumElements() == NumElts && \"Mask size mismatches value type element count!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2378, __extension__ __PRETTY_FUNCTION__))
;
2379
2380 // Undef operand.
2381 if (Op.isUndef()) {
2382 KnownUndef.setAllBits();
2383 return false;
2384 }
2385
2386 // If Op has other users, assume that all elements are needed.
2387 if (!Op.getNode()->hasOneUse() && !AssumeSingleUse)
2388 DemandedElts.setAllBits();
2389
2390 // Not demanding any elements from Op.
2391 if (DemandedElts == 0) {
2392 KnownUndef.setAllBits();
2393 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
2394 }
2395
2396 // Limit search depth.
2397 if (Depth >= SelectionDAG::MaxRecursionDepth)
2398 return false;
2399
2400 SDLoc DL(Op);
2401 unsigned EltSizeInBits = VT.getScalarSizeInBits();
2402
2403 // Helper for demanding the specified elements and all the bits of both binary
2404 // operands.
2405 auto SimplifyDemandedVectorEltsBinOp = [&](SDValue Op0, SDValue Op1) {
2406 SDValue NewOp0 = SimplifyMultipleUseDemandedVectorElts(Op0, DemandedElts,
2407 TLO.DAG, Depth + 1);
2408 SDValue NewOp1 = SimplifyMultipleUseDemandedVectorElts(Op1, DemandedElts,
2409 TLO.DAG, Depth + 1);
2410 if (NewOp0 || NewOp1) {
2411 SDValue NewOp = TLO.DAG.getNode(
2412 Opcode, SDLoc(Op), VT, NewOp0 ? NewOp0 : Op0, NewOp1 ? NewOp1 : Op1);
2413 return TLO.CombineTo(Op, NewOp);
2414 }
2415 return false;
2416 };
2417
2418 switch (Opcode) {
2419 case ISD::SCALAR_TO_VECTOR: {
2420 if (!DemandedElts[0]) {
2421 KnownUndef.setAllBits();
2422 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
2423 }
2424 SDValue ScalarSrc = Op.getOperand(0);
2425 if (ScalarSrc.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
2426 SDValue Src = ScalarSrc.getOperand(0);
2427 SDValue Idx = ScalarSrc.getOperand(1);
2428 EVT SrcVT = Src.getValueType();
2429
2430 ElementCount SrcEltCnt = SrcVT.getVectorElementCount();
2431
2432 if (SrcEltCnt.isScalable())
2433 return false;
2434
2435 unsigned NumSrcElts = SrcEltCnt.getFixedValue();
2436 if (isNullConstant(Idx)) {
2437 APInt SrcDemandedElts = APInt::getOneBitSet(NumSrcElts, 0);
2438 APInt SrcUndef = KnownUndef.zextOrTrunc(NumSrcElts);
2439 APInt SrcZero = KnownZero.zextOrTrunc(NumSrcElts);
2440 if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2441 TLO, Depth + 1))
2442 return true;
2443 }
2444 }
2445 KnownUndef.setHighBits(NumElts - 1);
2446 break;
2447 }
2448 case ISD::BITCAST: {
2449 SDValue Src = Op.getOperand(0);
2450 EVT SrcVT = Src.getValueType();
2451
2452 // We only handle vectors here.
2453 // TODO - investigate calling SimplifyDemandedBits/ComputeKnownBits?
2454 if (!SrcVT.isVector())
2455 break;
2456
2457 // Fast handling of 'identity' bitcasts.
2458 unsigned NumSrcElts = SrcVT.getVectorNumElements();
2459 if (NumSrcElts == NumElts)
2460 return SimplifyDemandedVectorElts(Src, DemandedElts, KnownUndef,
2461 KnownZero, TLO, Depth + 1);
2462
2463 APInt SrcZero, SrcUndef;
2464 APInt SrcDemandedElts = APInt::getNullValue(NumSrcElts);
2465
2466 // Bitcast from 'large element' src vector to 'small element' vector, we
2467 // must demand a source element if any DemandedElt maps to it.
2468 if ((NumElts % NumSrcElts) == 0) {
2469 unsigned Scale = NumElts / NumSrcElts;
2470 for (unsigned i = 0; i != NumElts; ++i)
2471 if (DemandedElts[i])
2472 SrcDemandedElts.setBit(i / Scale);
2473
2474 if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2475 TLO, Depth + 1))
2476 return true;
2477
2478 // Try calling SimplifyDemandedBits, converting demanded elts to the bits
2479 // of the large element.
2480 // TODO - bigendian once we have test coverage.
2481 if (TLO.DAG.getDataLayout().isLittleEndian()) {
2482 unsigned SrcEltSizeInBits = SrcVT.getScalarSizeInBits();
2483 APInt SrcDemandedBits = APInt::getNullValue(SrcEltSizeInBits);
2484 for (unsigned i = 0; i != NumElts; ++i)
2485 if (DemandedElts[i]) {
2486 unsigned Ofs = (i % Scale) * EltSizeInBits;
2487 SrcDemandedBits.setBits(Ofs, Ofs + EltSizeInBits);
2488 }
2489
2490 KnownBits Known;
2491 if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcDemandedElts, Known,
2492 TLO, Depth + 1))
2493 return true;
2494 }
2495
2496 // If the src element is zero/undef then all the output elements will be -
2497 // only demanded elements are guaranteed to be correct.
2498 for (unsigned i = 0; i != NumSrcElts; ++i) {
2499 if (SrcDemandedElts[i]) {
2500 if (SrcZero[i])
2501 KnownZero.setBits(i * Scale, (i + 1) * Scale);
2502 if (SrcUndef[i])
2503 KnownUndef.setBits(i * Scale, (i + 1) * Scale);
2504 }
2505 }
2506 }
2507
2508 // Bitcast from 'small element' src vector to 'large element' vector, we
2509 // demand all smaller source elements covered by the larger demanded element
2510 // of this vector.
2511 if ((NumSrcElts % NumElts) == 0) {
2512 unsigned Scale = NumSrcElts / NumElts;
2513 for (unsigned i = 0; i != NumElts; ++i)
2514 if (DemandedElts[i])
2515 SrcDemandedElts.setBits(i * Scale, (i + 1) * Scale);
2516
2517 if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2518 TLO, Depth + 1))
2519 return true;
2520
2521 // If all the src elements covering an output element are zero/undef, then
2522 // the output element will be as well, assuming it was demanded.
2523 for (unsigned i = 0; i != NumElts; ++i) {
2524 if (DemandedElts[i]) {
2525 if (SrcZero.extractBits(Scale, i * Scale).isAllOnesValue())
2526 KnownZero.setBit(i);
2527 if (SrcUndef.extractBits(Scale, i * Scale).isAllOnesValue())
2528 KnownUndef.setBit(i);
2529 }
2530 }
2531 }
2532 break;
2533 }
2534 case ISD::BUILD_VECTOR: {
2535 // Check all elements and simplify any unused elements with UNDEF.
2536 if (!DemandedElts.isAllOnesValue()) {
2537 // Don't simplify BROADCASTS.
2538 if (llvm::any_of(Op->op_values(),
2539 [&](SDValue Elt) { return Op.getOperand(0) != Elt; })) {
2540 SmallVector<SDValue, 32> Ops(Op->op_begin(), Op->op_end());
2541 bool Updated = false;
2542 for (unsigned i = 0; i != NumElts; ++i) {
2543 if (!DemandedElts[i] && !Ops[i].isUndef()) {
2544 Ops[i] = TLO.DAG.getUNDEF(Ops[0].getValueType());
2545 KnownUndef.setBit(i);
2546 Updated = true;
2547 }
2548 }
2549 if (Updated)
2550 return TLO.CombineTo(Op, TLO.DAG.getBuildVector(VT, DL, Ops));
2551 }
2552 }
2553 for (unsigned i = 0; i != NumElts; ++i) {
2554 SDValue SrcOp = Op.getOperand(i);
2555 if (SrcOp.isUndef()) {
2556 KnownUndef.setBit(i);
2557 } else if (EltSizeInBits == SrcOp.getScalarValueSizeInBits() &&
2558 (isNullConstant(SrcOp) || isNullFPConstant(SrcOp))) {
2559 KnownZero.setBit(i);
2560 }
2561 }
2562 break;
2563 }
2564 case ISD::CONCAT_VECTORS: {
2565 EVT SubVT = Op.getOperand(0).getValueType();
2566 unsigned NumSubVecs = Op.getNumOperands();
2567 unsigned NumSubElts = SubVT.getVectorNumElements();
2568 for (unsigned i = 0; i != NumSubVecs; ++i) {
2569 SDValue SubOp = Op.getOperand(i);
2570 APInt SubElts = DemandedElts.extractBits(NumSubElts, i * NumSubElts);
2571 APInt SubUndef, SubZero;
2572 if (SimplifyDemandedVectorElts(SubOp, SubElts, SubUndef, SubZero, TLO,
2573 Depth + 1))
2574 return true;
2575 KnownUndef.insertBits(SubUndef, i * NumSubElts);
2576 KnownZero.insertBits(SubZero, i * NumSubElts);
2577 }
2578 break;
2579 }
2580 case ISD::INSERT_SUBVECTOR: {
2581 // Demand any elements from the subvector and the remainder from the src its
2582 // inserted into.
2583 SDValue Src = Op.getOperand(0);
2584 SDValue Sub = Op.getOperand(1);
2585 uint64_t Idx = Op.getConstantOperandVal(2);
2586 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
2587 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
2588 APInt DemandedSrcElts = DemandedElts;
2589 DemandedSrcElts.insertBits(APInt::getNullValue(NumSubElts), Idx);
2590
2591 APInt SubUndef, SubZero;
2592 if (SimplifyDemandedVectorElts(Sub, DemandedSubElts, SubUndef, SubZero, TLO,
2593 Depth + 1))
2594 return true;
2595
2596 // If none of the src operand elements are demanded, replace it with undef.
2597 if (!DemandedSrcElts && !Src.isUndef())
2598 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
2599 TLO.DAG.getUNDEF(VT), Sub,
2600 Op.getOperand(2)));
2601
2602 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownUndef, KnownZero,
2603 TLO, Depth + 1))
2604 return true;
2605 KnownUndef.insertBits(SubUndef, Idx);
2606 KnownZero.insertBits(SubZero, Idx);
2607
2608 // Attempt to avoid multi-use ops if we don't need anything from them.
2609 if (!DemandedSrcElts.isAllOnesValue() ||
2610 !DemandedSubElts.isAllOnesValue()) {
2611 SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
2612 Src, DemandedSrcElts, TLO.DAG, Depth + 1);
2613 SDValue NewSub = SimplifyMultipleUseDemandedVectorElts(
2614 Sub, DemandedSubElts, TLO.DAG, Depth + 1);
2615 if (NewSrc || NewSub) {
2616 NewSrc = NewSrc ? NewSrc : Src;
2617 NewSub = NewSub ? NewSub : Sub;
2618 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,
2619 NewSub, Op.getOperand(2));
2620 return TLO.CombineTo(Op, NewOp);
2621 }
2622 }
2623 break;
2624 }
2625 case ISD::EXTRACT_SUBVECTOR: {
2626 // Offset the demanded elts by the subvector index.
2627 SDValue Src = Op.getOperand(0);
2628 if (Src.getValueType().isScalableVector())
2629 break;
2630 uint64_t Idx = Op.getConstantOperandVal(1);
2631 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2632 APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
2633
2634 APInt SrcUndef, SrcZero;
2635 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
2636 Depth + 1))
2637 return true;
2638 KnownUndef = SrcUndef.extractBits(NumElts, Idx);
2639 KnownZero = SrcZero.extractBits(NumElts, Idx);
2640
2641 // Attempt to avoid multi-use ops if we don't need anything from them.
2642 if (!DemandedElts.isAllOnesValue()) {
2643 SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
2644 Src, DemandedSrcElts, TLO.DAG, Depth + 1);
2645 if (NewSrc) {
2646 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,
2647 Op.getOperand(1));
2648 return TLO.CombineTo(Op, NewOp);
2649 }
2650 }
2651 break;
2652 }
2653 case ISD::INSERT_VECTOR_ELT: {
2654 SDValue Vec = Op.getOperand(0);
2655 SDValue Scl = Op.getOperand(1);
2656 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
2657
2658 // For a legal, constant insertion index, if we don't need this insertion
2659 // then strip it, else remove it from the demanded elts.
2660 if (CIdx && CIdx->getAPIntValue().ult(NumElts)) {
2661 unsigned Idx = CIdx->getZExtValue();
2662 if (!DemandedElts[Idx])
2663 return TLO.CombineTo(Op, Vec);
2664
2665 APInt DemandedVecElts(DemandedElts);
2666 DemandedVecElts.clearBit(Idx);
2667 if (SimplifyDemandedVectorElts(Vec, DemandedVecElts, KnownUndef,
2668 KnownZero, TLO, Depth + 1))
2669 return true;
2670
2671 KnownUndef.setBitVal(Idx, Scl.isUndef());
2672
2673 KnownZero.setBitVal(Idx, isNullConstant(Scl) || isNullFPConstant(Scl));
2674 break;
2675 }
2676
2677 APInt VecUndef, VecZero;
2678 if (SimplifyDemandedVectorElts(Vec, DemandedElts, VecUndef, VecZero, TLO,
2679 Depth + 1))
2680 return true;
2681 // Without knowing the insertion index we can't set KnownUndef/KnownZero.
2682 break;
2683 }
2684 case ISD::VSELECT: {
2685 // Try to transform the select condition based on the current demanded
2686 // elements.
2687 // TODO: If a condition element is undef, we can choose from one arm of the
2688 // select (and if one arm is undef, then we can propagate that to the
2689 // result).
2690 // TODO - add support for constant vselect masks (see IR version of this).
2691 APInt UnusedUndef, UnusedZero;
2692 if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, UnusedUndef,
2693 UnusedZero, TLO, Depth + 1))
2694 return true;
2695
2696 // See if we can simplify either vselect operand.
2697 APInt DemandedLHS(DemandedElts);
2698 APInt DemandedRHS(DemandedElts);
2699 APInt UndefLHS, ZeroLHS;
2700 APInt UndefRHS, ZeroRHS;
2701 if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedLHS, UndefLHS,
2702 ZeroLHS, TLO, Depth + 1))
2703 return true;
2704 if (SimplifyDemandedVectorElts(Op.getOperand(2), DemandedRHS, UndefRHS,
2705 ZeroRHS, TLO, Depth + 1))
2706 return true;
2707
2708 KnownUndef = UndefLHS & UndefRHS;
2709 KnownZero = ZeroLHS & ZeroRHS;
2710 break;
2711 }
2712 case ISD::VECTOR_SHUFFLE: {
2713 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
2714
2715 // Collect demanded elements from shuffle operands..
2716 APInt DemandedLHS(NumElts, 0);
2717 APInt DemandedRHS(NumElts, 0);
2718 for (unsigned i = 0; i != NumElts; ++i) {
2719 int M = ShuffleMask[i];
2720 if (M < 0 || !DemandedElts[i])
2721 continue;
2722 assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range")(static_cast <bool> (0 <= M && M < (int)(
2 * NumElts) && "Shuffle index out of range") ? void (
0) : __assert_fail ("0 <= M && M < (int)(2 * NumElts) && \"Shuffle index out of range\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2722, __extension__ __PRETTY_FUNCTION__))
;
2723 if (M < (int)NumElts)
2724 DemandedLHS.setBit(M);
2725 else
2726 DemandedRHS.setBit(M - NumElts);
2727 }
2728
2729 // See if we can simplify either shuffle operand.
2730 APInt UndefLHS, ZeroLHS;
2731 APInt UndefRHS, ZeroRHS;
2732 if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedLHS, UndefLHS,
2733 ZeroLHS, TLO, Depth + 1))
2734 return true;
2735 if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedRHS, UndefRHS,
2736 ZeroRHS, TLO, Depth + 1))
2737 return true;
2738
2739 // Simplify mask using undef elements from LHS/RHS.
2740 bool Updated = false;
2741 bool IdentityLHS = true, IdentityRHS = true;
2742 SmallVector<int, 32> NewMask(ShuffleMask.begin(), ShuffleMask.end());
2743 for (unsigned i = 0; i != NumElts; ++i) {
2744 int &M = NewMask[i];
2745 if (M < 0)
2746 continue;
2747 if (!DemandedElts[i] || (M < (int)NumElts && UndefLHS[M]) ||
2748 (M >= (int)NumElts && UndefRHS[M - NumElts])) {
2749 Updated = true;
2750 M = -1;
2751 }
2752 IdentityLHS &= (M < 0) || (M == (int)i);
2753 IdentityRHS &= (M < 0) || ((M - NumElts) == i);
2754 }
2755
2756 // Update legal shuffle masks based on demanded elements if it won't reduce
2757 // to Identity which can cause premature removal of the shuffle mask.
2758 if (Updated && !IdentityLHS && !IdentityRHS && !TLO.LegalOps) {
2759 SDValue LegalShuffle =
2760 buildLegalVectorShuffle(VT, DL, Op.getOperand(0), Op.getOperand(1),
2761 NewMask, TLO.DAG);
2762 if (LegalShuffle)
2763 return TLO.CombineTo(Op, LegalShuffle);
2764 }
2765
2766 // Propagate undef/zero elements from LHS/RHS.
2767 for (unsigned i = 0; i != NumElts; ++i) {
2768 int M = ShuffleMask[i];
2769 if (M < 0) {
2770 KnownUndef.setBit(i);
2771 } else if (M < (int)NumElts) {
2772 if (UndefLHS[M])
2773 KnownUndef.setBit(i);
2774 if (ZeroLHS[M])
2775 KnownZero.setBit(i);
2776 } else {
2777 if (UndefRHS[M - NumElts])
2778 KnownUndef.setBit(i);
2779 if (ZeroRHS[M - NumElts])
2780 KnownZero.setBit(i);
2781 }
2782 }
2783 break;
2784 }
2785 case ISD::ANY_EXTEND_VECTOR_INREG:
2786 case ISD::SIGN_EXTEND_VECTOR_INREG:
2787 case ISD::ZERO_EXTEND_VECTOR_INREG: {
2788 APInt SrcUndef, SrcZero;
2789 SDValue Src = Op.getOperand(0);
2790 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2791 APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts);
2792 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
2793 Depth + 1))
2794 return true;
2795 KnownZero = SrcZero.zextOrTrunc(NumElts);
2796 KnownUndef = SrcUndef.zextOrTrunc(NumElts);
2797
2798 if (Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG &&
2799 Op.getValueSizeInBits() == Src.getValueSizeInBits() &&
2800 DemandedSrcElts == 1 && TLO.DAG.getDataLayout().isLittleEndian()) {
2801 // aext - if we just need the bottom element then we can bitcast.
2802 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
2803 }
2804
2805 if (Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) {
2806 // zext(undef) upper bits are guaranteed to be zero.
2807 if (DemandedElts.isSubsetOf(KnownUndef))
2808 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
2809 KnownUndef.clearAllBits();
2810 }
2811 break;
2812 }
2813
2814 // TODO: There are more binop opcodes that could be handled here - MIN,
2815 // MAX, saturated math, etc.
2816 case ISD::OR:
2817 case ISD::XOR:
2818 case ISD::ADD:
2819 case ISD::SUB:
2820 case ISD::FADD:
2821 case ISD::FSUB:
2822 case ISD::FMUL:
2823 case ISD::FDIV:
2824 case ISD::FREM: {
2825 SDValue Op0 = Op.getOperand(0);
2826 SDValue Op1 = Op.getOperand(1);
2827
2828 APInt UndefRHS, ZeroRHS;
2829 if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,
2830 Depth + 1))
2831 return true;
2832 APInt UndefLHS, ZeroLHS;
2833 if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
2834 Depth + 1))
2835 return true;
2836
2837 KnownZero = ZeroLHS & ZeroRHS;
2838 KnownUndef = getKnownUndefForVectorBinop(Op, TLO.DAG, UndefLHS, UndefRHS);
2839
2840 // Attempt to avoid multi-use ops if we don't need anything from them.
2841 // TODO - use KnownUndef to relax the demandedelts?
2842 if (!DemandedElts.isAllOnesValue())
2843 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
2844 return true;
2845 break;
2846 }
2847 case ISD::SHL:
2848 case ISD::SRL:
2849 case ISD::SRA:
2850 case ISD::ROTL:
2851 case ISD::ROTR: {
2852 SDValue Op0 = Op.getOperand(0);
2853 SDValue Op1 = Op.getOperand(1);
2854
2855 APInt UndefRHS, ZeroRHS;
2856 if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,
2857 Depth + 1))
2858 return true;
2859 APInt UndefLHS, ZeroLHS;
2860 if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
2861 Depth + 1))
2862 return true;
2863
2864 KnownZero = ZeroLHS;
2865 KnownUndef = UndefLHS & UndefRHS; // TODO: use getKnownUndefForVectorBinop?
2866
2867 // Attempt to avoid multi-use ops if we don't need anything from them.
2868 // TODO - use KnownUndef to relax the demandedelts?
2869 if (!DemandedElts.isAllOnesValue())
2870 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
2871 return true;
2872 break;
2873 }
2874 case ISD::MUL:
2875 case ISD::AND: {
2876 SDValue Op0 = Op.getOperand(0);
2877 SDValue Op1 = Op.getOperand(1);
2878
2879 APInt SrcUndef, SrcZero;
2880 if (SimplifyDemandedVectorElts(Op1, DemandedElts, SrcUndef, SrcZero, TLO,
2881 Depth + 1))
2882 return true;
2883 if (SimplifyDemandedVectorElts(Op0, DemandedElts, KnownUndef, KnownZero,
2884 TLO, Depth + 1))
2885 return true;
2886
2887 // If either side has a zero element, then the result element is zero, even
2888 // if the other is an UNDEF.
2889 // TODO: Extend getKnownUndefForVectorBinop to also deal with known zeros
2890 // and then handle 'and' nodes with the rest of the binop opcodes.
2891 KnownZero |= SrcZero;
2892 KnownUndef &= SrcUndef;
2893 KnownUndef &= ~KnownZero;
2894
2895 // Attempt to avoid multi-use ops if we don't need anything from them.
2896 // TODO - use KnownUndef to relax the demandedelts?
2897 if (!DemandedElts.isAllOnesValue())
2898 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
2899 return true;
2900 break;
2901 }
2902 case ISD::TRUNCATE:
2903 case ISD::SIGN_EXTEND:
2904 case ISD::ZERO_EXTEND:
2905 if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,
2906 KnownZero, TLO, Depth + 1))
2907 return true;
2908
2909 if (Op.getOpcode() == ISD::ZERO_EXTEND) {
2910 // zext(undef) upper bits are guaranteed to be zero.
2911 if (DemandedElts.isSubsetOf(KnownUndef))
2912 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
2913 KnownUndef.clearAllBits();
2914 }
2915 break;
2916 default: {
2917 if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
2918 if (SimplifyDemandedVectorEltsForTargetNode(Op, DemandedElts, KnownUndef,
2919 KnownZero, TLO, Depth))
2920 return true;
2921 } else {
2922 KnownBits Known;
2923 APInt DemandedBits = APInt::getAllOnesValue(EltSizeInBits);
2924 if (SimplifyDemandedBits(Op, DemandedBits, OriginalDemandedElts, Known,
2925 TLO, Depth, AssumeSingleUse))
2926 return true;
2927 }
2928 break;
2929 }
2930 }
2931 assert((KnownUndef & KnownZero) == 0 && "Elements flagged as undef AND zero")(static_cast <bool> ((KnownUndef & KnownZero) == 0 &&
"Elements flagged as undef AND zero") ? void (0) : __assert_fail
("(KnownUndef & KnownZero) == 0 && \"Elements flagged as undef AND zero\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2931, __extension__ __PRETTY_FUNCTION__))
;
2932
2933 // Constant fold all undef cases.
2934 // TODO: Handle zero cases as well.
2935 if (DemandedElts.isSubsetOf(KnownUndef))
2936 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
2937
2938 return false;
2939}
2940
2941/// Determine which of the bits specified in Mask are known to be either zero or
2942/// one and return them in the Known.
2943void TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
2944 KnownBits &Known,
2945 const APInt &DemandedElts,
2946 const SelectionDAG &DAG,
2947 unsigned Depth) const {
2948 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2953, __extension__ __PRETTY_FUNCTION__))
2949 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2953, __extension__ __PRETTY_FUNCTION__))
2950 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2953, __extension__ __PRETTY_FUNCTION__))
2951 Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2953, __extension__ __PRETTY_FUNCTION__))
2952 "Should use MaskedValueIsZero if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2953, __extension__ __PRETTY_FUNCTION__))
2953 " is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2953, __extension__ __PRETTY_FUNCTION__))
;
2954 Known.resetAll();
2955}
2956
2957void TargetLowering::computeKnownBitsForTargetInstr(
2958 GISelKnownBits &Analysis, Register R, KnownBits &Known,
2959 const APInt &DemandedElts, const MachineRegisterInfo &MRI,
2960 unsigned Depth) const {
2961 Known.resetAll();
2962}
2963
2964void TargetLowering::computeKnownBitsForFrameIndex(
2965 const int FrameIdx, KnownBits &Known, const MachineFunction &MF) const {
2966 // The low bits are known zero if the pointer is aligned.
2967 Known.Zero.setLowBits(Log2(MF.getFrameInfo().getObjectAlign(FrameIdx)));
2968}
2969
2970Align TargetLowering::computeKnownAlignForTargetInstr(
2971 GISelKnownBits &Analysis, Register R, const MachineRegisterInfo &MRI,
2972 unsigned Depth) const {
2973 return Align(1);
2974}
2975
2976/// This method can be implemented by targets that want to expose additional
2977/// information about sign bits to the DAG Combiner.
2978unsigned TargetLowering::ComputeNumSignBitsForTargetNode(SDValue Op,
2979 const APInt &,
2980 const SelectionDAG &,
2981 unsigned Depth) const {
2982 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2987, __extension__ __PRETTY_FUNCTION__))
2983 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2987, __extension__ __PRETTY_FUNCTION__))
2984 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2987, __extension__ __PRETTY_FUNCTION__))
2985 Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2987, __extension__ __PRETTY_FUNCTION__))
2986 "Should use ComputeNumSignBits if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2987, __extension__ __PRETTY_FUNCTION__))
2987 " is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2987, __extension__ __PRETTY_FUNCTION__))
;
2988 return 1;
2989}
2990
2991unsigned TargetLowering::computeNumSignBitsForTargetInstr(
2992 GISelKnownBits &Analysis, Register R, const APInt &DemandedElts,
2993 const MachineRegisterInfo &MRI, unsigned Depth) const {
2994 return 1;
2995}
2996
2997bool TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
2998 SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,
2999 TargetLoweringOpt &TLO, unsigned Depth) const {
3000 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3005, __extension__ __PRETTY_FUNCTION__))
3001 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3005, __extension__ __PRETTY_FUNCTION__))
3002 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3005, __extension__ __PRETTY_FUNCTION__))
3003 Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3005, __extension__ __PRETTY_FUNCTION__))
3004 "Should use SimplifyDemandedVectorElts if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3005, __extension__ __PRETTY_FUNCTION__))
3005 " is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3005, __extension__ __PRETTY_FUNCTION__))
;
3006 return false;
3007}
3008
3009bool TargetLowering::SimplifyDemandedBitsForTargetNode(
3010 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
3011 KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const {
3012 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3017, __extension__ __PRETTY_FUNCTION__))
3013 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3017, __extension__ __PRETTY_FUNCTION__))
3014 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3017, __extension__ __PRETTY_FUNCTION__))
3015 Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3017, __extension__ __PRETTY_FUNCTION__))
3016 "Should use SimplifyDemandedBits if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3017, __extension__ __PRETTY_FUNCTION__))
3017 " is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3017, __extension__ __PRETTY_FUNCTION__))
;
3018 computeKnownBitsForTargetNode(Op, Known, DemandedElts, TLO.DAG, Depth);
3019 return false;
3020}
3021
3022SDValue TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(
3023 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
3024 SelectionDAG &DAG, unsigned Depth) const {
3025 assert((static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3031, __extension__ __PRETTY_FUNCTION__))
3026 (Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3031, __extension__ __PRETTY_FUNCTION__))
3027 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3031, __extension__ __PRETTY_FUNCTION__))
3028 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3031, __extension__ __PRETTY_FUNCTION__))
3029 Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3031, __extension__ __PRETTY_FUNCTION__))
3030 "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3031, __extension__ __PRETTY_FUNCTION__))
3031 " is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3031, __extension__ __PRETTY_FUNCTION__))
;
3032 return SDValue();
3033}
3034
3035SDValue
3036TargetLowering::buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0,
3037 SDValue N1, MutableArrayRef<int> Mask,
3038 SelectionDAG &DAG) const {
3039 bool LegalMask = isShuffleMaskLegal(Mask, VT);
3040 if (!LegalMask) {
3041 std::swap(N0, N1);
3042 ShuffleVectorSDNode::commuteMask(Mask);
3043 LegalMask = isShuffleMaskLegal(Mask, VT);
3044 }
3045
3046 if (!LegalMask)
3047 return SDValue();
3048
3049 return DAG.getVectorShuffle(VT, DL, N0, N1, Mask);
3050}
3051
3052const Constant *TargetLowering::getTargetConstantFromLoad(LoadSDNode*) const {
3053 return nullptr;
3054}
3055
3056bool TargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
3057 const SelectionDAG &DAG,
3058 bool SNaN,
3059 unsigned Depth) const {
3060 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3065, __extension__ __PRETTY_FUNCTION__))
3061 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3065, __extension__ __PRETTY_FUNCTION__))
3062 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3065, __extension__ __PRETTY_FUNCTION__))
3063 Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3065, __extension__ __PRETTY_FUNCTION__))
3064 "Should use isKnownNeverNaN if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3065, __extension__ __PRETTY_FUNCTION__))
3065 " is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
|| Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
" is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3065, __extension__ __PRETTY_FUNCTION__))
;
3066 return false;
3067}
3068
3069// FIXME: Ideally, this would use ISD::isConstantSplatVector(), but that must
3070// work with truncating build vectors and vectors with elements of less than
3071// 8 bits.
3072bool TargetLowering::isConstTrueVal(const SDNode *N) const {
3073 if (!N)
3074 return false;
3075
3076 APInt CVal;
3077 if (auto *CN = dyn_cast<ConstantSDNode>(N)) {
3078 CVal = CN->getAPIntValue();
3079 } else if (auto *BV = dyn_cast<BuildVectorSDNode>(N)) {
3080 auto *CN = BV->getConstantSplatNode();
3081 if (!CN)
3082 return false;
3083
3084 // If this is a truncating build vector, truncate the splat value.
3085 // Otherwise, we may fail to match the expected values below.
3086 unsigned BVEltWidth = BV->getValueType(0).getScalarSizeInBits();
3087 CVal = CN->getAPIntValue();
3088 if (BVEltWidth < CVal.getBitWidth())
3089 CVal = CVal.trunc(BVEltWidth);
3090 } else {
3091 return false;
3092 }
3093
3094 switch (getBooleanContents(N->getValueType(0))) {
3095 case UndefinedBooleanContent:
3096 return CVal[0];
3097 case ZeroOrOneBooleanContent:
3098 return CVal.isOneValue();
3099 case ZeroOrNegativeOneBooleanContent:
3100 return CVal.isAllOnesValue();
3101 }
3102
3103 llvm_unreachable("Invalid boolean contents")::llvm::llvm_unreachable_internal("Invalid boolean contents",
"/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3103)
;
3104}
3105
3106bool TargetLowering::isConstFalseVal(const SDNode *N) const {
3107 if (!N)
3108 return false;
3109
3110 const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N);
3111 if (!CN) {
3112 const BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N);
3113 if (!BV)
3114 return false;
3115
3116 // Only interested in constant splats, we don't care about undef
3117 // elements in identifying boolean constants and getConstantSplatNode
3118 // returns NULL if all ops are undef;
3119 CN = BV->getConstantSplatNode();
3120 if (!CN)
3121 return false;
3122 }
3123
3124 if (getBooleanContents(N->getValueType(0)) == UndefinedBooleanContent)
3125 return !CN->getAPIntValue()[0];
3126
3127 return CN->isNullValue();
3128}
3129
3130bool TargetLowering::isExtendedTrueVal(const ConstantSDNode *N, EVT VT,
3131 bool SExt) const {
3132 if (VT == MVT::i1)
3133 return N->isOne();
3134
3135 TargetLowering::BooleanContent Cnt = getBooleanContents(VT);
3136 switch (Cnt) {
3137 case TargetLowering::ZeroOrOneBooleanContent:
3138 // An extended value of 1 is always true, unless its original type is i1,
3139 // in which case it will be sign extended to -1.
3140 return (N->isOne() && !SExt) || (SExt && (N->getValueType(0) != MVT::i1));
3141 case TargetLowering::UndefinedBooleanContent:
3142 case TargetLowering::ZeroOrNegativeOneBooleanContent:
3143 return N->isAllOnesValue() && SExt;
3144 }
3145 llvm_unreachable("Unexpected enumeration.")::llvm::llvm_unreachable_internal("Unexpected enumeration.", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3145)
;
3146}
3147
3148/// This helper function of SimplifySetCC tries to optimize the comparison when
3149/// either operand of the SetCC node is a bitwise-and instruction.
3150SDValue TargetLowering::foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1,
3151 ISD::CondCode Cond, const SDLoc &DL,
3152 DAGCombinerInfo &DCI) const {
3153 // Match these patterns in any of their permutations:
3154 // (X & Y) == Y
3155 // (X & Y) != Y
3156 if (N1.getOpcode() == ISD::AND && N0.getOpcode() != ISD::AND)
3157 std::swap(N0, N1);
3158
3159 EVT OpVT = N0.getValueType();
3160 if (N0.getOpcode() != ISD::AND || !OpVT.isInteger() ||
3161 (Cond != ISD::SETEQ && Cond != ISD::SETNE))
3162 return SDValue();
3163
3164 SDValue X, Y;
3165 if (N0.getOperand(0) == N1) {
3166 X = N0.getOperand(1);
3167 Y = N0.getOperand(0);
3168 } else if (N0.getOperand(1) == N1) {
3169 X = N0.getOperand(0);
3170 Y = N0.getOperand(1);
3171 } else {
3172 return SDValue();
3173 }
3174
3175 SelectionDAG &DAG = DCI.DAG;
3176 SDValue Zero = DAG.getConstant(0, DL, OpVT);
3177 if (DAG.isKnownToBeAPowerOfTwo(Y)) {
3178 // Simplify X & Y == Y to X & Y != 0 if Y has exactly one bit set.
3179 // Note that where Y is variable and is known to have at most one bit set
3180 // (for example, if it is Z & 1) we cannot do this; the expressions are not
3181 // equivalent when Y == 0.
3182 assert(OpVT.isInteger())(static_cast <bool> (OpVT.isInteger()) ? void (0) : __assert_fail
("OpVT.isInteger()", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3182, __extension__ __PRETTY_FUNCTION__))
;
3183 Cond = ISD::getSetCCInverse(Cond, OpVT);
3184 if (DCI.isBeforeLegalizeOps() ||
3185 isCondCodeLegal(Cond, N0.getSimpleValueType()))
3186 return DAG.getSetCC(DL, VT, N0, Zero, Cond);
3187 } else if (N0.hasOneUse() && hasAndNotCompare(Y)) {
3188 // If the target supports an 'and-not' or 'and-complement' logic operation,
3189 // try to use that to make a comparison operation more efficient.
3190 // But don't do this transform if the mask is a single bit because there are
3191 // more efficient ways to deal with that case (for example, 'bt' on x86 or
3192 // 'rlwinm' on PPC).
3193
3194 // Bail out if the compare operand that we want to turn into a zero is
3195 // already a zero (otherwise, infinite loop).
3196 auto *YConst = dyn_cast<ConstantSDNode>(Y);
3197 if (YConst && YConst->isNullValue())
3198 return SDValue();
3199
3200 // Transform this into: ~X & Y == 0.
3201 SDValue NotX = DAG.getNOT(SDLoc(X), X, OpVT);
3202 SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, NotX, Y);
3203 return DAG.getSetCC(DL, VT, NewAnd, Zero, Cond);
3204 }
3205
3206 return SDValue();
3207}
3208
3209/// There are multiple IR patterns that could be checking whether certain
3210/// truncation of a signed number would be lossy or not. The pattern which is
3211/// best at IR level, may not lower optimally. Thus, we want to unfold it.
3212/// We are looking for the following pattern: (KeptBits is a constant)
3213/// (add %x, (1 << (KeptBits-1))) srccond (1 << KeptBits)
3214/// KeptBits won't be bitwidth(x), that will be constant-folded to true/false.
3215/// KeptBits also can't be 1, that would have been folded to %x dstcond 0
3216/// We will unfold it into the natural trunc+sext pattern:
3217/// ((%x << C) a>> C) dstcond %x
3218/// Where C = bitwidth(x) - KeptBits and C u< bitwidth(x)
3219SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(
3220 EVT SCCVT, SDValue N0, SDValue N1, ISD::CondCode Cond, DAGCombinerInfo &DCI,
3221 const SDLoc &DL) const {
3222 // We must be comparing with a constant.
3223 ConstantSDNode *C1;
3224 if (!(C1 = dyn_cast<ConstantSDNode>(N1)))
3225 return SDValue();
3226
3227 // N0 should be: add %x, (1 << (KeptBits-1))
3228 if (N0->getOpcode() != ISD::ADD)
3229 return SDValue();
3230
3231 // And we must be 'add'ing a constant.
3232 ConstantSDNode *C01;
3233 if (!(C01 = dyn_cast<ConstantSDNode>(N0->getOperand(1))))
3234 return SDValue();
3235
3236 SDValue X = N0->getOperand(0);
3237 EVT XVT = X.getValueType();
3238
3239 // Validate constants ...
3240
3241 APInt I1 = C1->getAPIntValue();
3242
3243 ISD::CondCode NewCond;
3244 if (Cond == ISD::CondCode::SETULT) {
3245 NewCond = ISD::CondCode::SETEQ;
3246 } else if (Cond == ISD::CondCode::SETULE) {
3247 NewCond = ISD::CondCode::SETEQ;
3248 // But need to 'canonicalize' the constant.
3249 I1 += 1;
3250 } else if (Cond == ISD::CondCode::SETUGT) {
3251 NewCond = ISD::CondCode::SETNE;
3252 // But need to 'canonicalize' the constant.
3253 I1 += 1;
3254 } else if (Cond == ISD::CondCode::SETUGE) {
3255 NewCond = ISD::CondCode::SETNE;
3256 } else
3257 return SDValue();
3258
3259 APInt I01 = C01->getAPIntValue();
3260
3261 auto checkConstants = [&I1, &I01]() -> bool {
3262 // Both of them must be power-of-two, and the constant from setcc is bigger.
3263 return I1.ugt(I01) && I1.isPowerOf2() && I01.isPowerOf2();
3264 };
3265
3266 if (checkConstants()) {
3267 // Great, e.g. got icmp ult i16 (add i16 %x, 128), 256
3268 } else {
3269 // What if we invert constants? (and the target predicate)
3270 I1.negate();
3271 I01.negate();
3272 assert(XVT.isInteger())(static_cast <bool> (XVT.isInteger()) ? void (0) : __assert_fail
("XVT.isInteger()", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3272, __extension__ __PRETTY_FUNCTION__))
;
3273 NewCond = getSetCCInverse(NewCond, XVT);
3274 if (!checkConstants())
3275 return SDValue();
3276 // Great, e.g. got icmp uge i16 (add i16 %x, -128), -256
3277 }
3278
3279 // They are power-of-two, so which bit is set?
3280 const unsigned KeptBits = I1.logBase2();
3281 const unsigned KeptBitsMinusOne = I01.logBase2();
3282
3283 // Magic!
3284 if (KeptBits != (KeptBitsMinusOne + 1))
3285 return SDValue();
3286 assert(KeptBits > 0 && KeptBits < XVT.getSizeInBits() && "unreachable")(static_cast <bool> (KeptBits > 0 && KeptBits
< XVT.getSizeInBits() && "unreachable") ? void (0
) : __assert_fail ("KeptBits > 0 && KeptBits < XVT.getSizeInBits() && \"unreachable\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3286, __extension__ __PRETTY_FUNCTION__))
;
3287
3288 // We don't want to do this in every single case.
3289 SelectionDAG &DAG = DCI.DAG;
3290 if (!DAG.getTargetLoweringInfo().shouldTransformSignedTruncationCheck(
3291 XVT, KeptBits))
3292 return SDValue();
3293
3294 const unsigned MaskedBits = XVT.getSizeInBits() - KeptBits;
3295 assert(MaskedBits > 0 && MaskedBits < XVT.getSizeInBits() && "unreachable")(static_cast <bool> (MaskedBits > 0 && MaskedBits
< XVT.getSizeInBits() && "unreachable") ? void (0
) : __assert_fail ("MaskedBits > 0 && MaskedBits < XVT.getSizeInBits() && \"unreachable\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3295, __extension__ __PRETTY_FUNCTION__))
;
3296
3297 // Unfold into: ((%x << C) a>> C) cond %x
3298 // Where 'cond' will be either 'eq' or 'ne'.
3299 SDValue ShiftAmt = DAG.getConstant(MaskedBits, DL, XVT);
3300 SDValue T0 = DAG.getNode(ISD::SHL, DL, XVT, X, ShiftAmt);
3301 SDValue T1 = DAG.getNode(ISD::SRA, DL, XVT, T0, ShiftAmt);
3302 SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, X, NewCond);
3303
3304 return T2;
3305}
3306
3307// (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
3308SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(
3309 EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond,
3310 DAGCombinerInfo &DCI, const SDLoc &DL) const {
3311 assert(isConstOrConstSplat(N1C) &&(static_cast <bool> (isConstOrConstSplat(N1C) &&
isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() &&
"Should be a comparison with 0.") ? void (0) : __assert_fail
("isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() && \"Should be a comparison with 0.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3313, __extension__ __PRETTY_FUNCTION__))
3312 isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() &&(static_cast <bool> (isConstOrConstSplat(N1C) &&
isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() &&
"Should be a comparison with 0.") ? void (0) : __assert_fail
("isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() && \"Should be a comparison with 0.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3313, __extension__ __PRETTY_FUNCTION__))
3313 "Should be a comparison with 0.")(static_cast <bool> (isConstOrConstSplat(N1C) &&
isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() &&
"Should be a comparison with 0.") ? void (0) : __assert_fail
("isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() && \"Should be a comparison with 0.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3313, __extension__ __PRETTY_FUNCTION__))
;
3314 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Valid only for [in]equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Valid only for [in]equality comparisons.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3315, __extension__ __PRETTY_FUNCTION__))
3315 "Valid only for [in]equality comparisons.")(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Valid only for [in]equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Valid only for [in]equality comparisons.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3315, __extension__ __PRETTY_FUNCTION__))
;
3316
3317 unsigned NewShiftOpcode;
3318 SDValue X, C, Y;
3319
3320 SelectionDAG &DAG = DCI.DAG;
3321 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3322
3323 // Look for '(C l>>/<< Y)'.
3324 auto Match = [&NewShiftOpcode, &X, &C, &Y, &TLI, &DAG](SDValue V) {
3325 // The shift should be one-use.
3326 if (!V.hasOneUse())
3327 return false;
3328 unsigned OldShiftOpcode = V.getOpcode();
3329 switch (OldShiftOpcode) {
3330 case ISD::SHL:
3331 NewShiftOpcode = ISD::SRL;
3332 break;
3333 case ISD::SRL:
3334 NewShiftOpcode = ISD::SHL;
3335 break;
3336 default:
3337 return false; // must be a logical shift.
3338 }
3339 // We should be shifting a constant.
3340 // FIXME: best to use isConstantOrConstantVector().
3341 C = V.getOperand(0);
3342 ConstantSDNode *CC =
3343 isConstOrConstSplat(C, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
3344 if (!CC)
3345 return false;
3346 Y = V.getOperand(1);
3347
3348 ConstantSDNode *XC =
3349 isConstOrConstSplat(X, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
3350 return TLI.shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
3351 X, XC, CC, Y, OldShiftOpcode, NewShiftOpcode, DAG);
3352 };
3353
3354 // LHS of comparison should be an one-use 'and'.
3355 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())
3356 return SDValue();
3357
3358 X = N0.getOperand(0);
3359 SDValue Mask = N0.getOperand(1);
3360
3361 // 'and' is commutative!
3362 if (!Match(Mask)) {
3363 std::swap(X, Mask);
3364 if (!Match(Mask))
3365 return SDValue();
3366 }
3367
3368 EVT VT = X.getValueType();
3369
3370 // Produce:
3371 // ((X 'OppositeShiftOpcode' Y) & C) Cond 0
3372 SDValue T0 = DAG.getNode(NewShiftOpcode, DL, VT, X, Y);
3373 SDValue T1 = DAG.getNode(ISD::AND, DL, VT, T0, C);
3374 SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, N1C, Cond);
3375 return T2;
3376}
3377
3378/// Try to fold an equality comparison with a {add/sub/xor} binary operation as
3379/// the 1st operand (N0). Callers are expected to swap the N0/N1 parameters to
3380/// handle the commuted versions of these patterns.
3381SDValue TargetLowering::foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1,
3382 ISD::CondCode Cond, const SDLoc &DL,
3383 DAGCombinerInfo &DCI) const {
3384 unsigned BOpcode = N0.getOpcode();
3385 assert((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) &&(static_cast <bool> ((BOpcode == ISD::ADD || BOpcode ==
ISD::SUB || BOpcode == ISD::XOR) && "Unexpected binop"
) ? void (0) : __assert_fail ("(BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) && \"Unexpected binop\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3386, __extension__ __PRETTY_FUNCTION__))
3386 "Unexpected binop")(static_cast <bool> ((BOpcode == ISD::ADD || BOpcode ==
ISD::SUB || BOpcode == ISD::XOR) && "Unexpected binop"
) ? void (0) : __assert_fail ("(BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) && \"Unexpected binop\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3386, __extension__ __PRETTY_FUNCTION__))
;
3387 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Unexpected condcode")(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Unexpected condcode") ? void (0) : __assert_fail
("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Unexpected condcode\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3387, __extension__ __PRETTY_FUNCTION__))
;
3388
3389 // (X + Y) == X --> Y == 0
3390 // (X - Y) == X --> Y == 0
3391 // (X ^ Y) == X --> Y == 0
3392 SelectionDAG &DAG = DCI.DAG;
3393 EVT OpVT = N0.getValueType();
3394 SDValue X = N0.getOperand(0);
3395 SDValue Y = N0.getOperand(1);
3396 if (X == N1)
3397 return DAG.getSetCC(DL, VT, Y, DAG.getConstant(0, DL, OpVT), Cond);
3398
3399 if (Y != N1)
3400 return SDValue();
3401
3402 // (X + Y) == Y --> X == 0
3403 // (X ^ Y) == Y --> X == 0
3404 if (BOpcode == ISD::ADD || BOpcode == ISD::XOR)
3405 return DAG.getSetCC(DL, VT, X, DAG.getConstant(0, DL, OpVT), Cond);
3406
3407 // The shift would not be valid if the operands are boolean (i1).
3408 if (!N0.hasOneUse() || OpVT.getScalarSizeInBits() == 1)
3409 return SDValue();
3410
3411 // (X - Y) == Y --> X == Y << 1
3412 EVT ShiftVT = getShiftAmountTy(OpVT, DAG.getDataLayout(),
3413 !DCI.isBeforeLegalize());
3414 SDValue One = DAG.getConstant(1, DL, ShiftVT);
3415 SDValue YShl1 = DAG.getNode(ISD::SHL, DL, N1.getValueType(), Y, One);
3416 if (!DCI.isCalledByLegalizer())
3417 DCI.AddToWorklist(YShl1.getNode());
3418 return DAG.getSetCC(DL, VT, X, YShl1, Cond);
3419}
3420
3421static SDValue simplifySetCCWithCTPOP(const TargetLowering &TLI, EVT VT,
3422 SDValue N0, const APInt &C1,
3423 ISD::CondCode Cond, const SDLoc &dl,
3424 SelectionDAG &DAG) {
3425 // Look through truncs that don't change the value of a ctpop.
3426 // FIXME: Add vector support? Need to be careful with setcc result type below.
3427 SDValue CTPOP = N0;
3428 if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && !VT.isVector() &&
3429 N0.getScalarValueSizeInBits() > Log2_32(N0.getOperand(0).getScalarValueSizeInBits()))
3430 CTPOP = N0.getOperand(0);
3431
3432 if (CTPOP.getOpcode() != ISD::CTPOP || !CTPOP.hasOneUse())
3433 return SDValue();
3434
3435 EVT CTVT = CTPOP.getValueType();
3436 SDValue CTOp = CTPOP.getOperand(0);
3437
3438 // If this is a vector CTPOP, keep the CTPOP if it is legal.
3439 // TODO: Should we check if CTPOP is legal(or custom) for scalars?
3440 if (VT.isVector() && TLI.isOperationLegal(ISD::CTPOP, CTVT))
3441 return SDValue();
3442
3443 // (ctpop x) u< 2 -> (x & x-1) == 0
3444 // (ctpop x) u> 1 -> (x & x-1) != 0
3445 if (Cond == ISD::SETULT || Cond == ISD::SETUGT) {
3446 unsigned CostLimit = TLI.getCustomCtpopCost(CTVT, Cond);
3447 if (C1.ugt(CostLimit + (Cond == ISD::SETULT)))
3448 return SDValue();
3449 if (C1 == 0 && (Cond == ISD::SETULT))
3450 return SDValue(); // This is handled elsewhere.
3451
3452 unsigned Passes = C1.getLimitedValue() - (Cond == ISD::SETULT);
3453
3454 SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
3455 SDValue Result = CTOp;
3456 for (unsigned i = 0; i < Passes; i++) {
3457 SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, Result, NegOne);
3458 Result = DAG.getNode(ISD::AND, dl, CTVT, Result, Add);
3459 }
3460 ISD::CondCode CC = Cond == ISD::SETULT ? ISD::SETEQ : ISD::SETNE;
3461 return DAG.getSetCC(dl, VT, Result, DAG.getConstant(0, dl, CTVT), CC);
3462 }
3463
3464 // If ctpop is not supported, expand a power-of-2 comparison based on it.
3465 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && C1 == 1) {
3466 // For scalars, keep CTPOP if it is legal or custom.
3467 if (!VT.isVector() && TLI.isOperationLegalOrCustom(ISD::CTPOP, CTVT))
3468 return SDValue();
3469 // This is based on X86's custom lowering for CTPOP which produces more
3470 // instructions than the expansion here.
3471
3472 // (ctpop x) == 1 --> (x != 0) && ((x & x-1) == 0)
3473 // (ctpop x) != 1 --> (x == 0) || ((x & x-1) != 0)
3474 SDValue Zero = DAG.getConstant(0, dl, CTVT);
3475 SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
3476 assert(CTVT.isInteger())(static_cast <bool> (CTVT.isInteger()) ? void (0) : __assert_fail
("CTVT.isInteger()", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3476, __extension__ __PRETTY_FUNCTION__))
;
3477 ISD::CondCode InvCond = ISD::getSetCCInverse(Cond, CTVT);
3478 SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, CTOp, NegOne);
3479 SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Add);
3480 SDValue LHS = DAG.getSetCC(dl, VT, CTOp, Zero, InvCond);
3481 SDValue RHS = DAG.getSetCC(dl, VT, And, Zero, Cond);
3482 unsigned LogicOpcode = Cond == ISD::SETEQ ? ISD::AND : ISD::OR;
3483 return DAG.getNode(LogicOpcode, dl, VT, LHS, RHS);
3484 }
3485
3486 return SDValue();
3487}
3488
3489/// Try to simplify a setcc built with the specified operands and cc. If it is
3490/// unable to simplify it, return a null SDValue.
3491SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
3492 ISD::CondCode Cond, bool foldBooleans,
3493 DAGCombinerInfo &DCI,
3494 const SDLoc &dl) const {
3495 SelectionDAG &DAG = DCI.DAG;
3496 const DataLayout &Layout = DAG.getDataLayout();
3497 EVT OpVT = N0.getValueType();
3498
3499 // Constant fold or commute setcc.
3500 if (SDValue Fold = DAG.FoldSetCC(VT, N0, N1, Cond, dl))
3501 return Fold;
3502
3503 // Ensure that the constant occurs on the RHS and fold constant comparisons.
3504 // TODO: Handle non-splat vector constants. All undef causes trouble.
3505 // FIXME: We can't yet fold constant scalable vector splats, so avoid an
3506 // infinite loop here when we encounter one.
3507 ISD::CondCode SwappedCC = ISD::getSetCCSwappedOperands(Cond);
3508 if (isConstOrConstSplat(N0) &&
3509 (!OpVT.isScalableVector() || !isConstOrConstSplat(N1)) &&
3510 (DCI.isBeforeLegalizeOps() ||
3511 isCondCodeLegal(SwappedCC, N0.getSimpleValueType())))
3512 return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);
3513
3514 // If we have a subtract with the same 2 non-constant operands as this setcc
3515 // -- but in reverse order -- then try to commute the operands of this setcc
3516 // to match. A matching pair of setcc (cmp) and sub may be combined into 1
3517 // instruction on some targets.
3518 if (!isConstOrConstSplat(N0) && !isConstOrConstSplat(N1) &&
3519 (DCI.isBeforeLegalizeOps() ||
3520 isCondCodeLegal(SwappedCC, N0.getSimpleValueType())) &&
3521 DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N1, N0}) &&
3522 !DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N0, N1}))
3523 return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);
3524
3525 if (auto *N1C = isConstOrConstSplat(N1)) {
3526 const APInt &C1 = N1C->getAPIntValue();
3527
3528 // Optimize some CTPOP cases.
3529 if (SDValue V = simplifySetCCWithCTPOP(*this, VT, N0, C1, Cond, dl, DAG))
3530 return V;
3531
3532 // If the LHS is '(srl (ctlz x), 5)', the RHS is 0/1, and this is an
3533 // equality comparison, then we're just comparing whether X itself is
3534 // zero.
3535 if (N0.getOpcode() == ISD::SRL && (C1.isNullValue() || C1.isOneValue()) &&
3536 N0.getOperand(0).getOpcode() == ISD::CTLZ &&
3537 isPowerOf2_32(N0.getScalarValueSizeInBits())) {
3538 if (ConstantSDNode *ShAmt = isConstOrConstSplat(N0.getOperand(1))) {
3539 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3540 ShAmt->getAPIntValue() == Log2_32(N0.getScalarValueSizeInBits())) {
3541 if ((C1 == 0) == (Cond == ISD::SETEQ)) {
3542 // (srl (ctlz x), 5) == 0 -> X != 0
3543 // (srl (ctlz x), 5) != 1 -> X != 0
3544 Cond = ISD::SETNE;
3545 } else {
3546 // (srl (ctlz x), 5) != 0 -> X == 0
3547 // (srl (ctlz x), 5) == 1 -> X == 0
3548 Cond = ISD::SETEQ;
3549 }
3550 SDValue Zero = DAG.getConstant(0, dl, N0.getValueType());
3551 return DAG.getSetCC(dl, VT, N0.getOperand(0).getOperand(0), Zero,
3552 Cond);
3553 }
3554 }
3555 }
3556 }
3557
3558 // FIXME: Support vectors.
3559 if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
3560 const APInt &C1 = N1C->getAPIntValue();
3561
3562 // (zext x) == C --> x == (trunc C)
3563 // (sext x) == C --> x == (trunc C)
3564 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3565 DCI.isBeforeLegalize() && N0->hasOneUse()) {
3566 unsigned MinBits = N0.getValueSizeInBits();
3567 SDValue PreExt;
3568 bool Signed = false;
3569 if (N0->getOpcode() == ISD::ZERO_EXTEND) {
3570 // ZExt
3571 MinBits = N0->getOperand(0).getValueSizeInBits();
3572 PreExt = N0->getOperand(0);
3573 } else if (N0->getOpcode() == ISD::AND) {
3574 // DAGCombine turns costly ZExts into ANDs
3575 if (auto *C = dyn_cast<ConstantSDNode>(N0->getOperand(1)))
3576 if ((C->getAPIntValue()+1).isPowerOf2()) {
3577 MinBits = C->getAPIntValue().countTrailingOnes();
3578 PreExt = N0->getOperand(0);
3579 }
3580 } else if (N0->getOpcode() == ISD::SIGN_EXTEND) {
3581 // SExt
3582 MinBits = N0->getOperand(0).getValueSizeInBits();
3583 PreExt = N0->getOperand(0);
3584 Signed = true;
3585 } else if (auto *LN0 = dyn_cast<LoadSDNode>(N0)) {
3586 // ZEXTLOAD / SEXTLOAD
3587 if (LN0->getExtensionType() == ISD::ZEXTLOAD) {
3588 MinBits = LN0->getMemoryVT().getSizeInBits();
3589 PreExt = N0;
3590 } else if (LN0->getExtensionType() == ISD::SEXTLOAD) {
3591 Signed = true;
3592 MinBits = LN0->getMemoryVT().getSizeInBits();
3593 PreExt = N0;
3594 }
3595 }
3596
3597 // Figure out how many bits we need to preserve this constant.
3598 unsigned ReqdBits = Signed ?
3599 C1.getBitWidth() - C1.getNumSignBits() + 1 :
3600 C1.getActiveBits();
3601
3602 // Make sure we're not losing bits from the constant.
3603 if (MinBits > 0 &&
3604 MinBits < C1.getBitWidth() &&
3605 MinBits >= ReqdBits) {
3606 EVT MinVT = EVT::getIntegerVT(*DAG.getContext(), MinBits);
3607 if (isTypeDesirableForOp(ISD::SETCC, MinVT)) {
3608 // Will get folded away.
3609 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MinVT, PreExt);
3610 if (MinBits == 1 && C1 == 1)
3611 // Invert the condition.
3612 return DAG.getSetCC(dl, VT, Trunc, DAG.getConstant(0, dl, MVT::i1),
3613 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
3614 SDValue C = DAG.getConstant(C1.trunc(MinBits), dl, MinVT);
3615 return DAG.getSetCC(dl, VT, Trunc, C, Cond);
3616 }
3617
3618 // If truncating the setcc operands is not desirable, we can still
3619 // simplify the expression in some cases:
3620 // setcc ([sz]ext (setcc x, y, cc)), 0, setne) -> setcc (x, y, cc)
3621 // setcc ([sz]ext (setcc x, y, cc)), 0, seteq) -> setcc (x, y, inv(cc))
3622 // setcc (zext (setcc x, y, cc)), 1, setne) -> setcc (x, y, inv(cc))
3623 // setcc (zext (setcc x, y, cc)), 1, seteq) -> setcc (x, y, cc)
3624 // setcc (sext (setcc x, y, cc)), -1, setne) -> setcc (x, y, inv(cc))
3625 // setcc (sext (setcc x, y, cc)), -1, seteq) -> setcc (x, y, cc)
3626 SDValue TopSetCC = N0->getOperand(0);
3627 unsigned N0Opc = N0->getOpcode();
3628 bool SExt = (N0Opc == ISD::SIGN_EXTEND);
3629 if (TopSetCC.getValueType() == MVT::i1 && VT == MVT::i1 &&
3630 TopSetCC.getOpcode() == ISD::SETCC &&
3631 (N0Opc == ISD::ZERO_EXTEND || N0Opc == ISD::SIGN_EXTEND) &&
3632 (isConstFalseVal(N1C) ||
3633 isExtendedTrueVal(N1C, N0->getValueType(0), SExt))) {
3634
3635 bool Inverse = (N1C->isNullValue() && Cond == ISD::SETEQ) ||
3636 (!N1C->isNullValue() && Cond == ISD::SETNE);
3637
3638 if (!Inverse)
3639 return TopSetCC;
3640
3641 ISD::CondCode InvCond = ISD::getSetCCInverse(
3642 cast<CondCodeSDNode>(TopSetCC.getOperand(2))->get(),
3643 TopSetCC.getOperand(0).getValueType());
3644 return DAG.getSetCC(dl, VT, TopSetCC.getOperand(0),
3645 TopSetCC.getOperand(1),
3646 InvCond);
3647 }
3648 }
3649 }
3650
3651 // If the LHS is '(and load, const)', the RHS is 0, the test is for
3652 // equality or unsigned, and all 1 bits of the const are in the same
3653 // partial word, see if we can shorten the load.
3654 if (DCI.isBeforeLegalize() &&
3655 !ISD::isSignedIntSetCC(Cond) &&
3656 N0.getOpcode() == ISD::AND && C1 == 0 &&
3657 N0.getNode()->hasOneUse() &&
3658 isa<LoadSDNode>(N0.getOperand(0)) &&
3659 N0.getOperand(0).getNode()->hasOneUse() &&
3660 isa<ConstantSDNode>(N0.getOperand(1))) {
3661 LoadSDNode *Lod = cast<LoadSDNode>(N0.getOperand(0));
3662 APInt bestMask;
3663 unsigned bestWidth = 0, bestOffset = 0;
3664 if (Lod->isSimple() && Lod->isUnindexed()) {
3665 unsigned origWidth = N0.getValueSizeInBits();
3666 unsigned maskWidth = origWidth;
3667 // We can narrow (e.g.) 16-bit extending loads on 32-bit target to
3668 // 8 bits, but have to be careful...
3669 if (Lod->getExtensionType() != ISD::NON_EXTLOAD)
3670 origWidth = Lod->getMemoryVT().getSizeInBits();
3671 const APInt &Mask = N0.getConstantOperandAPInt(1);
3672 for (unsigned width = origWidth / 2; width>=8; width /= 2) {
3673 APInt newMask = APInt::getLowBitsSet(maskWidth, width);
3674 for (unsigned offset=0; offset<origWidth/width; offset++) {
3675 if (Mask.isSubsetOf(newMask)) {
3676 if (Layout.isLittleEndian())
3677 bestOffset = (uint64_t)offset * (width/8);
3678 else
3679 bestOffset = (origWidth/width - offset - 1) * (width/8);
3680 bestMask = Mask.lshr(offset * (width/8) * 8);
3681 bestWidth = width;
3682 break;
3683 }
3684 newMask <<= width;
3685 }
3686 }
3687 }
3688 if (bestWidth) {
3689 EVT newVT = EVT::getIntegerVT(*DAG.getContext(), bestWidth);
3690 if (newVT.isRound() &&
3691 shouldReduceLoadWidth(Lod, ISD::NON_EXTLOAD, newVT)) {
3692 SDValue Ptr = Lod->getBasePtr();
3693 if (bestOffset != 0)
3694 Ptr =
3695 DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(bestOffset), dl);
3696 SDValue NewLoad =
3697 DAG.getLoad(newVT, dl, Lod->getChain(), Ptr,
3698 Lod->getPointerInfo().getWithOffset(bestOffset),
3699 Lod->getOriginalAlign());
3700 return DAG.getSetCC(dl, VT,
3701 DAG.getNode(ISD::AND, dl, newVT, NewLoad,
3702 DAG.getConstant(bestMask.trunc(bestWidth),
3703 dl, newVT)),
3704 DAG.getConstant(0LL, dl, newVT), Cond);
3705 }
3706 }
3707 }
3708
3709 // If the LHS is a ZERO_EXTEND, perform the comparison on the input.
3710 if (N0.getOpcode() == ISD::ZERO_EXTEND) {
3711 unsigned InSize = N0.getOperand(0).getValueSizeInBits();
3712
3713 // If the comparison constant has bits in the upper part, the
3714 // zero-extended value could never match.
3715 if (C1.intersects(APInt::getHighBitsSet(C1.getBitWidth(),
3716 C1.getBitWidth() - InSize))) {
3717 switch (Cond) {
3718 case ISD::SETUGT:
3719 case ISD::SETUGE:
3720 case ISD::SETEQ:
3721 return DAG.getConstant(0, dl, VT);
3722 case ISD::SETULT:
3723 case ISD::SETULE:
3724 case ISD::SETNE:
3725 return DAG.getConstant(1, dl, VT);
3726 case ISD::SETGT:
3727 case ISD::SETGE:
3728 // True if the sign bit of C1 is set.
3729 return DAG.getConstant(C1.isNegative(), dl, VT);
3730 case ISD::SETLT:
3731 case ISD::SETLE:
3732 // True if the sign bit of C1 isn't set.
3733 return DAG.getConstant(C1.isNonNegative(), dl, VT);
3734 default:
3735 break;
3736 }
3737 }
3738
3739 // Otherwise, we can perform the comparison with the low bits.
3740 switch (Cond) {
3741 case ISD::SETEQ:
3742 case ISD::SETNE:
3743 case ISD::SETUGT:
3744 case ISD::SETUGE:
3745 case ISD::SETULT:
3746 case ISD::SETULE: {
3747 EVT newVT = N0.getOperand(0).getValueType();
3748 if (DCI.isBeforeLegalizeOps() ||
3749 (isOperationLegal(ISD::SETCC, newVT) &&
3750 isCondCodeLegal(Cond, newVT.getSimpleVT()))) {
3751 EVT NewSetCCVT = getSetCCResultType(Layout, *DAG.getContext(), newVT);
3752 SDValue NewConst = DAG.getConstant(C1.trunc(InSize), dl, newVT);
3753
3754 SDValue NewSetCC = DAG.getSetCC(dl, NewSetCCVT, N0.getOperand(0),
3755 NewConst, Cond);
3756 return DAG.getBoolExtOrTrunc(NewSetCC, dl, VT, N0.getValueType());
3757 }
3758 break;
3759 }
3760 default:
3761 break; // todo, be more careful with signed comparisons
3762 }
3763 } else if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
3764 (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3765 !isSExtCheaperThanZExt(cast<VTSDNode>(N0.getOperand(1))->getVT(),
3766 OpVT)) {
3767 EVT ExtSrcTy = cast<VTSDNode>(N0.getOperand(1))->getVT();
3768 unsigned ExtSrcTyBits = ExtSrcTy.getSizeInBits();
3769 EVT ExtDstTy = N0.getValueType();
3770 unsigned ExtDstTyBits = ExtDstTy.getSizeInBits();
3771
3772 // If the constant doesn't fit into the number of bits for the source of
3773 // the sign extension, it is impossible for both sides to be equal.
3774 if (C1.getMinSignedBits() > ExtSrcTyBits)
3775 return DAG.getBoolConstant(Cond == ISD::SETNE, dl, VT, OpVT);
3776
3777 assert(ExtDstTy == N0.getOperand(0).getValueType() &&(static_cast <bool> (ExtDstTy == N0.getOperand(0).getValueType
() && ExtDstTy != ExtSrcTy && "Unexpected types!"
) ? void (0) : __assert_fail ("ExtDstTy == N0.getOperand(0).getValueType() && ExtDstTy != ExtSrcTy && \"Unexpected types!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3778, __extension__ __PRETTY_FUNCTION__))
3778 ExtDstTy != ExtSrcTy && "Unexpected types!")(static_cast <bool> (ExtDstTy == N0.getOperand(0).getValueType
() && ExtDstTy != ExtSrcTy && "Unexpected types!"
) ? void (0) : __assert_fail ("ExtDstTy == N0.getOperand(0).getValueType() && ExtDstTy != ExtSrcTy && \"Unexpected types!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3778, __extension__ __PRETTY_FUNCTION__))
;
3779 APInt Imm = APInt::getLowBitsSet(ExtDstTyBits, ExtSrcTyBits);
3780 SDValue ZextOp = DAG.getNode(ISD::AND, dl, ExtDstTy, N0.getOperand(0),
3781 DAG.getConstant(Imm, dl, ExtDstTy));
3782 if (!DCI.isCalledByLegalizer())
3783 DCI.AddToWorklist(ZextOp.getNode());
3784 // Otherwise, make this a use of a zext.
3785 return DAG.getSetCC(dl, VT, ZextOp,
3786 DAG.getConstant(C1 & Imm, dl, ExtDstTy), Cond);
3787 } else if ((N1C->isNullValue() || N1C->isOne()) &&
3788 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
3789 // SETCC (SETCC), [0|1], [EQ|NE] -> SETCC
3790 if (N0.getOpcode() == ISD::SETCC &&
3791 isTypeLegal(VT) && VT.bitsLE(N0.getValueType()) &&
3792 (N0.getValueType() == MVT::i1 ||
3793 getBooleanContents(N0.getOperand(0).getValueType()) ==
3794 ZeroOrOneBooleanContent)) {
3795 bool TrueWhenTrue = (Cond == ISD::SETEQ) ^ (!N1C->isOne());
3796 if (TrueWhenTrue)
3797 return DAG.getNode(ISD::TRUNCATE, dl, VT, N0);
3798 // Invert the condition.
3799 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
3800 CC = ISD::getSetCCInverse(CC, N0.getOperand(0).getValueType());
3801 if (DCI.isBeforeLegalizeOps() ||
3802 isCondCodeLegal(CC, N0.getOperand(0).getSimpleValueType()))
3803 return DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC);
3804 }
3805
3806 if ((N0.getOpcode() == ISD::XOR ||
3807 (N0.getOpcode() == ISD::AND &&
3808 N0.getOperand(0).getOpcode() == ISD::XOR &&
3809 N0.getOperand(1) == N0.getOperand(0).getOperand(1))) &&
3810 isOneConstant(N0.getOperand(1))) {
3811 // If this is (X^1) == 0/1, swap the RHS and eliminate the xor. We
3812 // can only do this if the top bits are known zero.
3813 unsigned BitWidth = N0.getValueSizeInBits();
3814 if (DAG.MaskedValueIsZero(N0,
3815 APInt::getHighBitsSet(BitWidth,
3816 BitWidth-1))) {
3817 // Okay, get the un-inverted input value.
3818 SDValue Val;
3819 if (N0.getOpcode() == ISD::XOR) {
3820 Val = N0.getOperand(0);
3821 } else {
3822 assert(N0.getOpcode() == ISD::AND &&(static_cast <bool> (N0.getOpcode() == ISD::AND &&
N0.getOperand(0).getOpcode() == ISD::XOR) ? void (0) : __assert_fail
("N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::XOR"
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3823, __extension__ __PRETTY_FUNCTION__))
3823 N0.getOperand(0).getOpcode() == ISD::XOR)(static_cast <bool> (N0.getOpcode() == ISD::AND &&
N0.getOperand(0).getOpcode() == ISD::XOR) ? void (0) : __assert_fail
("N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::XOR"
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3823, __extension__ __PRETTY_FUNCTION__))
;
3824 // ((X^1)&1)^1 -> X & 1
3825 Val = DAG.getNode(ISD::AND, dl, N0.getValueType(),
3826 N0.getOperand(0).getOperand(0),
3827 N0.getOperand(1));
3828 }
3829
3830 return DAG.getSetCC(dl, VT, Val, N1,
3831 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
3832 }
3833 } else if (N1C->isOne()) {
3834 SDValue Op0 = N0;
3835 if (Op0.getOpcode() == ISD::TRUNCATE)
3836 Op0 = Op0.getOperand(0);
3837
3838 if ((Op0.getOpcode() == ISD::XOR) &&
3839 Op0.getOperand(0).getOpcode() == ISD::SETCC &&
3840 Op0.getOperand(1).getOpcode() == ISD::SETCC) {
3841 SDValue XorLHS = Op0.getOperand(0);
3842 SDValue XorRHS = Op0.getOperand(1);
3843 // Ensure that the input setccs return an i1 type or 0/1 value.
3844 if (Op0.getValueType() == MVT::i1 ||
3845 (getBooleanContents(XorLHS.getOperand(0).getValueType()) ==
3846 ZeroOrOneBooleanContent &&
3847 getBooleanContents(XorRHS.getOperand(0).getValueType()) ==
3848 ZeroOrOneBooleanContent)) {
3849 // (xor (setcc), (setcc)) == / != 1 -> (setcc) != / == (setcc)
3850 Cond = (Cond == ISD::SETEQ) ? ISD::SETNE : ISD::SETEQ;
3851 return DAG.getSetCC(dl, VT, XorLHS, XorRHS, Cond);
3852 }
3853 }
3854 if (Op0.getOpcode() == ISD::AND && isOneConstant(Op0.getOperand(1))) {
3855 // If this is (X&1) == / != 1, normalize it to (X&1) != / == 0.
3856 if (Op0.getValueType().bitsGT(VT))
3857 Op0 = DAG.getNode(ISD::AND, dl, VT,
3858 DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)),
3859 DAG.getConstant(1, dl, VT));
3860 else if (Op0.getValueType().bitsLT(VT))
3861 Op0 = DAG.getNode(ISD::AND, dl, VT,
3862 DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)),
3863 DAG.getConstant(1, dl, VT));
3864
3865 return DAG.getSetCC(dl, VT, Op0,
3866 DAG.getConstant(0, dl, Op0.getValueType()),
3867 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
3868 }
3869 if (Op0.getOpcode() == ISD::AssertZext &&
3870 cast<VTSDNode>(Op0.getOperand(1))->getVT() == MVT::i1)
3871 return DAG.getSetCC(dl, VT, Op0,
3872 DAG.getConstant(0, dl, Op0.getValueType()),
3873 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
3874 }
3875 }
3876
3877 // Given:
3878 // icmp eq/ne (urem %x, %y), 0
3879 // Iff %x has 0 or 1 bits set, and %y has at least 2 bits set, omit 'urem':
3880 // icmp eq/ne %x, 0
3881 if (N0.getOpcode() == ISD::UREM && N1C->isNullValue() &&
3882 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
3883 KnownBits XKnown = DAG.computeKnownBits(N0.getOperand(0));
3884 KnownBits YKnown = DAG.computeKnownBits(N0.getOperand(1));
3885 if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2)
3886 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1, Cond);
3887 }
3888
3889 if (SDValue V =
3890 optimizeSetCCOfSignedTruncationCheck(VT, N0, N1, Cond, DCI, dl))
3891 return V;
3892 }
3893
3894 // These simplifications apply to splat vectors as well.
3895 // TODO: Handle more splat vector cases.
3896 if (auto *N1C = isConstOrConstSplat(N1)) {
3897 const APInt &C1 = N1C->getAPIntValue();
3898
3899 APInt MinVal, MaxVal;
3900 unsigned OperandBitSize = N1C->getValueType(0).getScalarSizeInBits();
3901 if (ISD::isSignedIntSetCC(Cond)) {
3902 MinVal = APInt::getSignedMinValue(OperandBitSize);
3903 MaxVal = APInt::getSignedMaxValue(OperandBitSize);
3904 } else {
3905 MinVal = APInt::getMinValue(OperandBitSize);
3906 MaxVal = APInt::getMaxValue(OperandBitSize);
3907 }
3908
3909 // Canonicalize GE/LE comparisons to use GT/LT comparisons.
3910 if (Cond == ISD::SETGE || Cond == ISD::SETUGE) {
3911 // X >= MIN --> true
3912 if (C1 == MinVal)
3913 return DAG.getBoolConstant(true, dl, VT, OpVT);
3914
3915 if (!VT.isVector()) { // TODO: Support this for vectors.
3916 // X >= C0 --> X > (C0 - 1)
3917 APInt C = C1 - 1;
3918 ISD::CondCode NewCC = (Cond == ISD::SETGE) ? ISD::SETGT : ISD::SETUGT;
3919 if ((DCI.isBeforeLegalizeOps() ||
3920 isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
3921 (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
3922 isLegalICmpImmediate(C.getSExtValue())))) {
3923 return DAG.getSetCC(dl, VT, N0,
3924 DAG.getConstant(C, dl, N1.getValueType()),
3925 NewCC);
3926 }
3927 }
3928 }
3929
3930 if (Cond == ISD::SETLE || Cond == ISD::SETULE) {
3931 // X <= MAX --> true
3932 if (C1 == MaxVal)
3933 return DAG.getBoolConstant(true, dl, VT, OpVT);
3934
3935 // X <= C0 --> X < (C0 + 1)
3936 if (!VT.isVector()) { // TODO: Support this for vectors.
3937 APInt C = C1 + 1;
3938 ISD::CondCode NewCC = (Cond == ISD::SETLE) ? ISD::SETLT : ISD::SETULT;
3939 if ((DCI.isBeforeLegalizeOps() ||
3940 isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
3941 (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
3942 isLegalICmpImmediate(C.getSExtValue())))) {
3943 return DAG.getSetCC(dl, VT, N0,
3944 DAG.getConstant(C, dl, N1.getValueType()),
3945 NewCC);
3946 }
3947 }
3948 }
3949
3950 if (Cond == ISD::SETLT || Cond == ISD::SETULT) {
3951 if (C1 == MinVal)
3952 return DAG.getBoolConstant(false, dl, VT, OpVT); // X < MIN --> false
3953
3954 // TODO: Support this for vectors after legalize ops.
3955 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
3956 // Canonicalize setlt X, Max --> setne X, Max
3957 if (C1 == MaxVal)
3958 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
3959
3960 // If we have setult X, 1, turn it into seteq X, 0
3961 if (C1 == MinVal+1)
3962 return DAG.getSetCC(dl, VT, N0,
3963 DAG.getConstant(MinVal, dl, N0.getValueType()),
3964 ISD::SETEQ);
3965 }
3966 }
3967
3968 if (Cond == ISD::SETGT || Cond == ISD::SETUGT) {
3969 if (C1 == MaxVal)
3970 return DAG.getBoolConstant(false, dl, VT, OpVT); // X > MAX --> false
3971
3972 // TODO: Support this for vectors after legalize ops.
3973 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
3974 // Canonicalize setgt X, Min --> setne X, Min
3975 if (C1 == MinVal)
3976 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
3977
3978 // If we have setugt X, Max-1, turn it into seteq X, Max
3979 if (C1 == MaxVal-1)
3980 return DAG.getSetCC(dl, VT, N0,
3981 DAG.getConstant(MaxVal, dl, N0.getValueType()),
3982 ISD::SETEQ);
3983 }
3984 }
3985
3986 if (Cond == ISD::SETEQ || Cond == ISD::SETNE) {
3987 // (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
3988 if (C1.isNullValue())
3989 if (SDValue CC = optimizeSetCCByHoistingAndByConstFromLogicalShift(
3990 VT, N0, N1, Cond, DCI, dl))
3991 return CC;
3992
3993 // For all/any comparisons, replace or(x,shl(y,bw/2)) with and/or(x,y).
3994 // For example, when high 32-bits of i64 X are known clear:
3995 // all bits clear: (X | (Y<<32)) == 0 --> (X | Y) == 0
3996 // all bits set: (X | (Y<<32)) == -1 --> (X & Y) == -1
3997 bool CmpZero = N1C->getAPIntValue().isNullValue();
3998 bool CmpNegOne = N1C->getAPIntValue().isAllOnesValue();
3999 if ((CmpZero || CmpNegOne) && N0.hasOneUse()) {
4000 // Match or(lo,shl(hi,bw/2)) pattern.
4001 auto IsConcat = [&](SDValue V, SDValue &Lo, SDValue &Hi) {
4002 unsigned EltBits = V.getScalarValueSizeInBits();
4003 if (V.getOpcode() != ISD::OR || (EltBits % 2) != 0)
4004 return false;
4005 SDValue LHS = V.getOperand(0);
4006 SDValue RHS = V.getOperand(1);
4007 APInt HiBits = APInt::getHighBitsSet(EltBits, EltBits / 2);
4008 // Unshifted element must have zero upperbits.
4009 if (RHS.getOpcode() == ISD::SHL &&
4010 isa<ConstantSDNode>(RHS.getOperand(1)) &&
4011 RHS.getConstantOperandAPInt(1) == (EltBits / 2) &&
4012 DAG.MaskedValueIsZero(LHS, HiBits)) {
4013 Lo = LHS;
4014 Hi = RHS.getOperand(0);
4015 return true;
4016 }
4017 if (LHS.getOpcode() == ISD::SHL &&
4018 isa<ConstantSDNode>(LHS.getOperand(1)) &&
4019 LHS.getConstantOperandAPInt(1) == (EltBits / 2) &&
4020 DAG.MaskedValueIsZero(RHS, HiBits)) {
4021 Lo = RHS;
4022 Hi = LHS.getOperand(0);
4023 return true;
4024 }
4025 return false;
4026 };
4027
4028 auto MergeConcat = [&](SDValue Lo, SDValue Hi) {
4029 unsigned EltBits = N0.getScalarValueSizeInBits();
4030 unsigned HalfBits = EltBits / 2;
4031 APInt HiBits = APInt::getHighBitsSet(EltBits, HalfBits);
4032 SDValue LoBits = DAG.getConstant(~HiBits, dl, OpVT);
4033 SDValue HiMask = DAG.getNode(ISD::AND, dl, OpVT, Hi, LoBits);
4034 SDValue NewN0 =
4035 DAG.getNode(CmpZero ? ISD::OR : ISD::AND, dl, OpVT, Lo, HiMask);
4036 SDValue NewN1 = CmpZero ? DAG.getConstant(0, dl, OpVT) : LoBits;
4037 return DAG.getSetCC(dl, VT, NewN0, NewN1, Cond);
4038 };
4039
4040 SDValue Lo, Hi;
4041 if (IsConcat(N0, Lo, Hi))
4042 return MergeConcat(Lo, Hi);
4043
4044 if (N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR) {
4045 SDValue Lo0, Lo1, Hi0, Hi1;
4046 if (IsConcat(N0.getOperand(0), Lo0, Hi0) &&
4047 IsConcat(N0.getOperand(1), Lo1, Hi1)) {
4048 return MergeConcat(DAG.getNode(N0.getOpcode(), dl, OpVT, Lo0, Lo1),
4049 DAG.getNode(N0.getOpcode(), dl, OpVT, Hi0, Hi1));
4050 }
4051 }
4052 }
4053 }
4054
4055 // If we have "setcc X, C0", check to see if we can shrink the immediate
4056 // by changing cc.
4057 // TODO: Support this for vectors after legalize ops.
4058 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
4059 // SETUGT X, SINTMAX -> SETLT X, 0
4060 // SETUGE X, SINTMIN -> SETLT X, 0
4061 if ((Cond == ISD::SETUGT && C1.isMaxSignedValue()) ||
4062 (Cond == ISD::SETUGE && C1.isMinSignedValue()))
4063 return DAG.getSetCC(dl, VT, N0,
4064 DAG.getConstant(0, dl, N1.getValueType()),
4065 ISD::SETLT);
4066
4067 // SETULT X, SINTMIN -> SETGT X, -1
4068 // SETULE X, SINTMAX -> SETGT X, -1
4069 if ((Cond == ISD::SETULT && C1.isMinSignedValue()) ||
4070 (Cond == ISD::SETULE && C1.isMaxSignedValue()))
4071 return DAG.getSetCC(dl, VT, N0,
4072 DAG.getAllOnesConstant(dl, N1.getValueType()),
4073 ISD::SETGT);
4074 }
4075 }
4076
4077 // Back to non-vector simplifications.
4078 // TODO: Can we do these for vector splats?
4079 if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
4080 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4081 const APInt &C1 = N1C->getAPIntValue();
4082 EVT ShValTy = N0.getValueType();
4083
4084 // Fold bit comparisons when we can. This will result in an
4085 // incorrect value when boolean false is negative one, unless
4086 // the bitsize is 1 in which case the false value is the same
4087 // in practice regardless of the representation.
4088 if ((VT.getSizeInBits() == 1 ||
4089 getBooleanContents(N0.getValueType()) == ZeroOrOneBooleanContent) &&
4090 (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4091 (VT == ShValTy || (isTypeLegal(VT) && VT.bitsLE(ShValTy))) &&
4092 N0.getOpcode() == ISD::AND) {
4093 if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
4094 EVT ShiftTy =
4095 getShiftAmountTy(ShValTy, Layout, !DCI.isBeforeLegalize());
4096 if (Cond == ISD::SETNE && C1 == 0) {// (X & 8) != 0 --> (X & 8) >> 3
4097 // Perform the xform if the AND RHS is a single bit.
4098 unsigned ShCt = AndRHS->getAPIntValue().logBase2();
4099 if (AndRHS->getAPIntValue().isPowerOf2() &&
4100 !TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {
4101 return DAG.getNode(ISD::TRUNCATE, dl, VT,
4102 DAG.getNode(ISD::SRL, dl, ShValTy, N0,
4103 DAG.getConstant(ShCt, dl, ShiftTy)));
4104 }
4105 } else if (Cond == ISD::SETEQ && C1 == AndRHS->getAPIntValue()) {
4106 // (X & 8) == 8 --> (X & 8) >> 3
4107 // Perform the xform if C1 is a single bit.
4108 unsigned ShCt = C1.logBase2();
4109 if (C1.isPowerOf2() &&
4110 !TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {
4111 return DAG.getNode(ISD::TRUNCATE, dl, VT,
4112 DAG.getNode(ISD::SRL, dl, ShValTy, N0,
4113 DAG.getConstant(ShCt, dl, ShiftTy)));
4114 }
4115 }
4116 }
4117 }
4118
4119 if (C1.getMinSignedBits() <= 64 &&
4120 !isLegalICmpImmediate(C1.getSExtValue())) {
4121 EVT ShiftTy = getShiftAmountTy(ShValTy, Layout, !DCI.isBeforeLegalize());
4122 // (X & -256) == 256 -> (X >> 8) == 1
4123 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4124 N0.getOpcode() == ISD::AND && N0.hasOneUse()) {
4125 if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
4126 const APInt &AndRHSC = AndRHS->getAPIntValue();
4127 if ((-AndRHSC).isPowerOf2() && (AndRHSC & C1) == C1) {
4128 unsigned ShiftBits = AndRHSC.countTrailingZeros();
4129 if (!TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
4130 SDValue Shift =
4131 DAG.getNode(ISD::SRL, dl, ShValTy, N0.getOperand(0),
4132 DAG.getConstant(ShiftBits, dl, ShiftTy));
4133 SDValue CmpRHS = DAG.getConstant(C1.lshr(ShiftBits), dl, ShValTy);
4134 return DAG.getSetCC(dl, VT, Shift, CmpRHS, Cond);
4135 }
4136 }
4137 }
4138 } else if (Cond == ISD::SETULT || Cond == ISD::SETUGE ||
4139 Cond == ISD::SETULE || Cond == ISD::SETUGT) {
4140 bool AdjOne = (Cond == ISD::SETULE || Cond == ISD::SETUGT);
4141 // X < 0x100000000 -> (X >> 32) < 1
4142 // X >= 0x100000000 -> (X >> 32) >= 1
4143 // X <= 0x0ffffffff -> (X >> 32) < 1
4144 // X > 0x0ffffffff -> (X >> 32) >= 1
4145 unsigned ShiftBits;
4146 APInt NewC = C1;
4147 ISD::CondCode NewCond = Cond;
4148 if (AdjOne) {
4149 ShiftBits = C1.countTrailingOnes();
4150 NewC = NewC + 1;
4151 NewCond = (Cond == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
4152 } else {
4153 ShiftBits = C1.countTrailingZeros();
4154 }
4155 NewC.lshrInPlace(ShiftBits);
4156 if (ShiftBits && NewC.getMinSignedBits() <= 64 &&
4157 isLegalICmpImmediate(NewC.getSExtValue()) &&
4158 !TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
4159 SDValue Shift = DAG.getNode(ISD::SRL, dl, ShValTy, N0,
4160 DAG.getConstant(ShiftBits, dl, ShiftTy));
4161 SDValue CmpRHS = DAG.getConstant(NewC, dl, ShValTy);
4162 return DAG.getSetCC(dl, VT, Shift, CmpRHS, NewCond);
4163 }
4164 }
4165 }
4166 }
4167
4168 if (!isa<ConstantFPSDNode>(N0) && isa<ConstantFPSDNode>(N1)) {
4169 auto *CFP = cast<ConstantFPSDNode>(N1);
4170 assert(!CFP->getValueAPF().isNaN() && "Unexpected NaN value")(static_cast <bool> (!CFP->getValueAPF().isNaN() &&
"Unexpected NaN value") ? void (0) : __assert_fail ("!CFP->getValueAPF().isNaN() && \"Unexpected NaN value\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4170, __extension__ __PRETTY_FUNCTION__))
;
4171
4172 // Otherwise, we know the RHS is not a NaN. Simplify the node to drop the
4173 // constant if knowing that the operand is non-nan is enough. We prefer to
4174 // have SETO(x,x) instead of SETO(x, 0.0) because this avoids having to
4175 // materialize 0.0.
4176 if (Cond == ISD::SETO || Cond == ISD::SETUO)
4177 return DAG.getSetCC(dl, VT, N0, N0, Cond);
4178
4179 // setcc (fneg x), C -> setcc swap(pred) x, -C
4180 if (N0.getOpcode() == ISD::FNEG) {
4181 ISD::CondCode SwapCond = ISD::getSetCCSwappedOperands(Cond);
4182 if (DCI.isBeforeLegalizeOps() ||
4183 isCondCodeLegal(SwapCond, N0.getSimpleValueType())) {
4184 SDValue NegN1 = DAG.getNode(ISD::FNEG, dl, N0.getValueType(), N1);
4185 return DAG.getSetCC(dl, VT, N0.getOperand(0), NegN1, SwapCond);
4186 }
4187 }
4188
4189 // If the condition is not legal, see if we can find an equivalent one
4190 // which is legal.
4191 if (!isCondCodeLegal(Cond, N0.getSimpleValueType())) {
4192 // If the comparison was an awkward floating-point == or != and one of
4193 // the comparison operands is infinity or negative infinity, convert the
4194 // condition to a less-awkward <= or >=.
4195 if (CFP->getValueAPF().isInfinity()) {
4196 bool IsNegInf = CFP->getValueAPF().isNegative();
4197 ISD::CondCode NewCond = ISD::SETCC_INVALID;
4198 switch (Cond) {
4199 case ISD::SETOEQ: NewCond = IsNegInf ? ISD::SETOLE : ISD::SETOGE; break;
4200 case ISD::SETUEQ: NewCond = IsNegInf ? ISD::SETULE : ISD::SETUGE; break;
4201 case ISD::SETUNE: NewCond = IsNegInf ? ISD::SETUGT : ISD::SETULT; break;
4202 case ISD::SETONE: NewCond = IsNegInf ? ISD::SETOGT : ISD::SETOLT; break;
4203 default: break;
4204 }
4205 if (NewCond != ISD::SETCC_INVALID &&
4206 isCondCodeLegal(NewCond, N0.getSimpleValueType()))
4207 return DAG.getSetCC(dl, VT, N0, N1, NewCond);
4208 }
4209 }
4210 }
4211
4212 if (N0 == N1) {
4213 // The sext(setcc()) => setcc() optimization relies on the appropriate
4214 // constant being emitted.
4215 assert(!N0.getValueType().isInteger() &&(static_cast <bool> (!N0.getValueType().isInteger() &&
"Integer types should be handled by FoldSetCC") ? void (0) :
__assert_fail ("!N0.getValueType().isInteger() && \"Integer types should be handled by FoldSetCC\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4216, __extension__ __PRETTY_FUNCTION__))
4216 "Integer types should be handled by FoldSetCC")(static_cast <bool> (!N0.getValueType().isInteger() &&
"Integer types should be handled by FoldSetCC") ? void (0) :
__assert_fail ("!N0.getValueType().isInteger() && \"Integer types should be handled by FoldSetCC\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4216, __extension__ __PRETTY_FUNCTION__))
;
4217
4218 bool EqTrue = ISD::isTrueWhenEqual(Cond);
4219 unsigned UOF = ISD::getUnorderedFlavor(Cond);
4220 if (UOF == 2) // FP operators that are undefined on NaNs.
4221 return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
4222 if (UOF == unsigned(EqTrue))
4223 return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
4224 // Otherwise, we can't fold it. However, we can simplify it to SETUO/SETO
4225 // if it is not already.
4226 ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO;
4227 if (NewCond != Cond &&
4228 (DCI.isBeforeLegalizeOps() ||
4229 isCondCodeLegal(NewCond, N0.getSimpleValueType())))
4230 return DAG.getSetCC(dl, VT, N0, N1, NewCond);
4231 }
4232
4233 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4234 N0.getValueType().isInteger()) {
4235 if (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB ||
4236 N0.getOpcode() == ISD::XOR) {
4237 // Simplify (X+Y) == (X+Z) --> Y == Z
4238 if (N0.getOpcode() == N1.getOpcode()) {
4239 if (N0.getOperand(0) == N1.getOperand(0))
4240 return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(1), Cond);
4241 if (N0.getOperand(1) == N1.getOperand(1))
4242 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(0), Cond);
4243 if (isCommutativeBinOp(N0.getOpcode())) {
4244 // If X op Y == Y op X, try other combinations.
4245 if (N0.getOperand(0) == N1.getOperand(1))
4246 return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(0),
4247 Cond);
4248 if (N0.getOperand(1) == N1.getOperand(0))
4249 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(1),
4250 Cond);
4251 }
4252 }
4253
4254 // If RHS is a legal immediate value for a compare instruction, we need
4255 // to be careful about increasing register pressure needlessly.
4256 bool LegalRHSImm = false;
4257
4258 if (auto *RHSC = dyn_cast<ConstantSDNode>(N1)) {
4259 if (auto *LHSR = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
4260 // Turn (X+C1) == C2 --> X == C2-C1
4261 if (N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse()) {
4262 return DAG.getSetCC(dl, VT, N0.getOperand(0),
4263 DAG.getConstant(RHSC->getAPIntValue()-
4264 LHSR->getAPIntValue(),
4265 dl, N0.getValueType()), Cond);
4266 }
4267
4268 // Turn (X^C1) == C2 into X == C1^C2 iff X&~C1 = 0.
4269 if (N0.getOpcode() == ISD::XOR)
4270 // If we know that all of the inverted bits are zero, don't bother
4271 // performing the inversion.
4272 if (DAG.MaskedValueIsZero(N0.getOperand(0), ~LHSR->getAPIntValue()))
4273 return
4274 DAG.getSetCC(dl, VT, N0.getOperand(0),
4275 DAG.getConstant(LHSR->getAPIntValue() ^
4276 RHSC->getAPIntValue(),
4277 dl, N0.getValueType()),
4278 Cond);
4279 }
4280
4281 // Turn (C1-X) == C2 --> X == C1-C2
4282 if (auto *SUBC = dyn_cast<ConstantSDNode>(N0.getOperand(0))) {
4283 if (N0.getOpcode() == ISD::SUB && N0.getNode()->hasOneUse()) {
4284 return
4285 DAG.getSetCC(dl, VT, N0.getOperand(1),
4286 DAG.getConstant(SUBC->getAPIntValue() -
4287 RHSC->getAPIntValue(),
4288 dl, N0.getValueType()),
4289 Cond);
4290 }
4291 }
4292
4293 // Could RHSC fold directly into a compare?
4294 if (RHSC->getValueType(0).getSizeInBits() <= 64)
4295 LegalRHSImm = isLegalICmpImmediate(RHSC->getSExtValue());
4296 }
4297
4298 // (X+Y) == X --> Y == 0 and similar folds.
4299 // Don't do this if X is an immediate that can fold into a cmp
4300 // instruction and X+Y has other uses. It could be an induction variable
4301 // chain, and the transform would increase register pressure.
4302 if (!LegalRHSImm || N0.hasOneUse())
4303 if (SDValue V = foldSetCCWithBinOp(VT, N0, N1, Cond, dl, DCI))
4304 return V;
4305 }
4306
4307 if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB ||
4308 N1.getOpcode() == ISD::XOR)
4309 if (SDValue V = foldSetCCWithBinOp(VT, N1, N0, Cond, dl, DCI))
4310 return V;
4311
4312 if (SDValue V = foldSetCCWithAnd(VT, N0, N1, Cond, dl, DCI))
4313 return V;
4314 }
4315
4316 // Fold remainder of division by a constant.
4317 if ((N0.getOpcode() == ISD::UREM || N0.getOpcode() == ISD::SREM) &&
4318 N0.hasOneUse() && (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
4319 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
4320
4321 // When division is cheap or optimizing for minimum size,
4322 // fall through to DIVREM creation by skipping this fold.
4323 if (!isIntDivCheap(VT, Attr) && !Attr.hasFnAttribute(Attribute::MinSize)) {
4324 if (N0.getOpcode() == ISD::UREM) {
4325 if (SDValue Folded = buildUREMEqFold(VT, N0, N1, Cond, DCI, dl))
4326 return Folded;
4327 } else if (N0.getOpcode() == ISD::SREM) {
4328 if (SDValue Folded = buildSREMEqFold(VT, N0, N1, Cond, DCI, dl))
4329 return Folded;
4330 }
4331 }
4332 }
4333
4334 // Fold away ALL boolean setcc's.
4335 if (N0.getValueType().getScalarType() == MVT::i1 && foldBooleans) {
4336 SDValue Temp;
4337 switch (Cond) {
4338 default: llvm_unreachable("Unknown integer setcc!")::llvm::llvm_unreachable_internal("Unknown integer setcc!", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4338)
;
4339 case ISD::SETEQ: // X == Y -> ~(X^Y)
4340 Temp = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
4341 N0 = DAG.getNOT(dl, Temp, OpVT);
4342 if (!DCI.isCalledByLegalizer())
4343 DCI.AddToWorklist(Temp.getNode());
4344 break;
4345 case ISD::SETNE: // X != Y --> (X^Y)
4346 N0 = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
4347 break;
4348 case ISD::SETGT: // X >s Y --> X == 0 & Y == 1 --> ~X & Y
4349 case ISD::SETULT: // X <u Y --> X == 0 & Y == 1 --> ~X & Y
4350 Temp = DAG.getNOT(dl, N0, OpVT);
4351 N0 = DAG.getNode(ISD::AND, dl, OpVT, N1, Temp);
4352 if (!DCI.isCalledByLegalizer())
4353 DCI.AddToWorklist(Temp.getNode());
4354 break;
4355 case ISD::SETLT: // X <s Y --> X == 1 & Y == 0 --> ~Y & X
4356 case ISD::SETUGT: // X >u Y --> X == 1 & Y == 0 --> ~Y & X
4357 Temp = DAG.getNOT(dl, N1, OpVT);
4358 N0 = DAG.getNode(ISD::AND, dl, OpVT, N0, Temp);
4359 if (!DCI.isCalledByLegalizer())
4360 DCI.AddToWorklist(Temp.getNode());
4361 break;
4362 case ISD::SETULE: // X <=u Y --> X == 0 | Y == 1 --> ~X | Y
4363 case ISD::SETGE: // X >=s Y --> X == 0 | Y == 1 --> ~X | Y
4364 Temp = DAG.getNOT(dl, N0, OpVT);
4365 N0 = DAG.getNode(ISD::OR, dl, OpVT, N1, Temp);
4366 if (!DCI.isCalledByLegalizer())
4367 DCI.AddToWorklist(Temp.getNode());
4368 break;
4369 case ISD::SETUGE: // X >=u Y --> X == 1 | Y == 0 --> ~Y | X
4370 case ISD::SETLE: // X <=s Y --> X == 1 | Y == 0 --> ~Y | X
4371 Temp = DAG.getNOT(dl, N1, OpVT);
4372 N0 = DAG.getNode(ISD::OR, dl, OpVT, N0, Temp);
4373 break;
4374 }
4375 if (VT.getScalarType() != MVT::i1) {
4376 if (!DCI.isCalledByLegalizer())
4377 DCI.AddToWorklist(N0.getNode());
4378 // FIXME: If running after legalize, we probably can't do this.
4379 ISD::NodeType ExtendCode = getExtendForContent(getBooleanContents(OpVT));
4380 N0 = DAG.getNode(ExtendCode, dl, VT, N0);
4381 }
4382 return N0;
4383 }
4384
4385 // Could not fold it.
4386 return SDValue();
4387}
4388
4389/// Returns true (and the GlobalValue and the offset) if the node is a
4390/// GlobalAddress + offset.
4391bool TargetLowering::isGAPlusOffset(SDNode *WN, const GlobalValue *&GA,
4392 int64_t &Offset) const {
4393
4394 SDNode *N = unwrapAddress(SDValue(WN, 0)).getNode();
4395
4396 if (auto *GASD = dyn_cast<GlobalAddressSDNode>(N)) {
4397 GA = GASD->getGlobal();
4398 Offset += GASD->getOffset();
4399 return true;
4400 }
4401
4402 if (N->getOpcode() == ISD::ADD) {
4403 SDValue N1 = N->getOperand(0);
4404 SDValue N2 = N->getOperand(1);
4405 if (isGAPlusOffset(N1.getNode(), GA, Offset)) {
4406 if (auto *V = dyn_cast<ConstantSDNode>(N2)) {
4407 Offset += V->getSExtValue();
4408 return true;
4409 }
4410 } else if (isGAPlusOffset(N2.getNode(), GA, Offset)) {
4411 if (auto *V = dyn_cast<ConstantSDNode>(N1)) {
4412 Offset += V->getSExtValue();
4413 return true;
4414 }
4415 }
4416 }
4417
4418 return false;
4419}
4420
4421SDValue TargetLowering::PerformDAGCombine(SDNode *N,
4422 DAGCombinerInfo &DCI) const {
4423 // Default implementation: no optimization.
4424 return SDValue();
4425}
4426
4427//===----------------------------------------------------------------------===//
4428// Inline Assembler Implementation Methods
4429//===----------------------------------------------------------------------===//
4430
4431TargetLowering::ConstraintType
4432TargetLowering::getConstraintType(StringRef Constraint) const {
4433 unsigned S = Constraint.size();
4434
4435 if (S == 1) {
4436 switch (Constraint[0]) {
4437 default: break;
4438 case 'r':
4439 return C_RegisterClass;
4440 case 'm': // memory
4441 case 'o': // offsetable
4442 case 'V': // not offsetable
4443 return C_Memory;
4444 case 'n': // Simple Integer
4445 case 'E': // Floating Point Constant
4446 case 'F': // Floating Point Constant
4447 return C_Immediate;
4448 case 'i': // Simple Integer or Relocatable Constant
4449 case 's': // Relocatable Constant
4450 case 'p': // Address.
4451 case 'X': // Allow ANY value.
4452 case 'I': // Target registers.
4453 case 'J':
4454 case 'K':
4455 case 'L':
4456 case 'M':
4457 case 'N':
4458 case 'O':
4459 case 'P':
4460 case '<':
4461 case '>':
4462 return C_Other;
4463 }
4464 }
4465
4466 if (S > 1 && Constraint[0] == '{' && Constraint[S - 1] == '}') {
4467 if (S == 8 && Constraint.substr(1, 6) == "memory") // "{memory}"
4468 return C_Memory;
4469 return C_Register;
4470 }
4471 return C_Unknown;
4472}
4473
4474/// Try to replace an X constraint, which matches anything, with another that
4475/// has more specific requirements based on the type of the corresponding
4476/// operand.
4477const char *TargetLowering::LowerXConstraint(EVT ConstraintVT) const {
4478 if (ConstraintVT.isInteger())
4479 return "r";
4480 if (ConstraintVT.isFloatingPoint())
4481 return "f"; // works for many targets
4482 return nullptr;
4483}
4484
4485SDValue TargetLowering::LowerAsmOutputForConstraint(
4486 SDValue &Chain, SDValue &Flag, const SDLoc &DL,
4487 const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const {
4488 return SDValue();
4489}
4490
4491/// Lower the specified operand into the Ops vector.
4492/// If it is invalid, don't add anything to Ops.
4493void TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
4494 std::string &Constraint,
4495 std::vector<SDValue> &Ops,
4496 SelectionDAG &DAG) const {
4497
4498 if (Constraint.length() > 1) return;
4499
4500 char ConstraintLetter = Constraint[0];
4501 switch (ConstraintLetter) {
4502 default: break;
4503 case 'X': // Allows any operand; labels (basic block) use this.
4504 if (Op.getOpcode() == ISD::BasicBlock ||
4505 Op.getOpcode() == ISD::TargetBlockAddress) {
4506 Ops.push_back(Op);
4507 return;
4508 }
4509 LLVM_FALLTHROUGH[[gnu::fallthrough]];
4510 case 'i': // Simple Integer or Relocatable Constant
4511 case 'n': // Simple Integer
4512 case 's': { // Relocatable Constant
4513
4514 GlobalAddressSDNode *GA;
4515 ConstantSDNode *C;
4516 BlockAddressSDNode *BA;
4517 uint64_t Offset = 0;
4518
4519 // Match (GA) or (C) or (GA+C) or (GA-C) or ((GA+C)+C) or (((GA+C)+C)+C),
4520 // etc., since getelementpointer is variadic. We can't use
4521 // SelectionDAG::FoldSymbolOffset because it expects the GA to be accessible
4522 // while in this case the GA may be furthest from the root node which is
4523 // likely an ISD::ADD.
4524 while (1) {
4525 if ((GA = dyn_cast<GlobalAddressSDNode>(Op)) && ConstraintLetter != 'n') {
4526 Ops.push_back(DAG.getTargetGlobalAddress(GA->getGlobal(), SDLoc(Op),
4527 GA->getValueType(0),
4528 Offset + GA->getOffset()));
4529 return;
4530 }
4531 if ((C = dyn_cast<ConstantSDNode>(Op)) && ConstraintLetter != 's') {
4532 // gcc prints these as sign extended. Sign extend value to 64 bits
4533 // now; without this it would get ZExt'd later in
4534 // ScheduleDAGSDNodes::EmitNode, which is very generic.
4535 bool IsBool = C->getConstantIntValue()->getBitWidth() == 1;
4536 BooleanContent BCont = getBooleanContents(MVT::i64);
4537 ISD::NodeType ExtOpc =
4538 IsBool ? getExtendForContent(BCont) : ISD::SIGN_EXTEND;
4539 int64_t ExtVal =
4540 ExtOpc == ISD::ZERO_EXTEND ? C->getZExtValue() : C->getSExtValue();
4541 Ops.push_back(
4542 DAG.getTargetConstant(Offset + ExtVal, SDLoc(C), MVT::i64));
4543 return;
4544 }
4545 if ((BA = dyn_cast<BlockAddressSDNode>(Op)) && ConstraintLetter != 'n') {
4546 Ops.push_back(DAG.getTargetBlockAddress(
4547 BA->getBlockAddress(), BA->getValueType(0),
4548 Offset + BA->getOffset(), BA->getTargetFlags()));
4549 return;
4550 }
4551 const unsigned OpCode = Op.getOpcode();
4552 if (OpCode == ISD::ADD || OpCode == ISD::SUB) {
4553 if ((C = dyn_cast<ConstantSDNode>(Op.getOperand(0))))
4554 Op = Op.getOperand(1);
4555 // Subtraction is not commutative.
4556 else if (OpCode == ISD::ADD &&
4557 (C = dyn_cast<ConstantSDNode>(Op.getOperand(1))))
4558 Op = Op.getOperand(0);
4559 else
4560 return;
4561 Offset += (OpCode == ISD::ADD ? 1 : -1) * C->getSExtValue();
4562 continue;
4563 }
4564 return;
4565 }
4566 break;
4567 }
4568 }
4569}
4570
4571std::pair<unsigned, const TargetRegisterClass *>
4572TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *RI,
4573 StringRef Constraint,
4574 MVT VT) const {
4575 if (Constraint.empty() || Constraint[0] != '{')
4576 return std::make_pair(0u, static_cast<TargetRegisterClass *>(nullptr));
4577 assert(*(Constraint.end() - 1) == '}' && "Not a brace enclosed constraint?")(static_cast <bool> (*(Constraint.end() - 1) == '}' &&
"Not a brace enclosed constraint?") ? void (0) : __assert_fail
("*(Constraint.end() - 1) == '}' && \"Not a brace enclosed constraint?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4577, __extension__ __PRETTY_FUNCTION__))
;
4578
4579 // Remove the braces from around the name.
4580 StringRef RegName(Constraint.data() + 1, Constraint.size() - 2);
4581
4582 std::pair<unsigned, const TargetRegisterClass *> R =
4583 std::make_pair(0u, static_cast<const TargetRegisterClass *>(nullptr));
4584
4585 // Figure out which register class contains this reg.
4586 for (const TargetRegisterClass *RC : RI->regclasses()) {
4587 // If none of the value types for this register class are valid, we
4588 // can't use it. For example, 64-bit reg classes on 32-bit targets.
4589 if (!isLegalRC(*RI, *RC))
4590 continue;
4591
4592 for (const MCPhysReg &PR : *RC) {
4593 if (RegName.equals_lower(RI->getRegAsmName(PR))) {
4594 std::pair<unsigned, const TargetRegisterClass *> S =
4595 std::make_pair(PR, RC);
4596
4597 // If this register class has the requested value type, return it,
4598 // otherwise keep searching and return the first class found
4599 // if no other is found which explicitly has the requested type.
4600 if (RI->isTypeLegalForClass(*RC, VT))
4601 return S;
4602 if (!R.second)
4603 R = S;
4604 }
4605 }
4606 }
4607
4608 return R;
4609}
4610
4611//===----------------------------------------------------------------------===//
4612// Constraint Selection.
4613
4614/// Return true of this is an input operand that is a matching constraint like
4615/// "4".
4616bool TargetLowering::AsmOperandInfo::isMatchingInputConstraint() const {
4617 assert(!ConstraintCode.empty() && "No known constraint!")(static_cast <bool> (!ConstraintCode.empty() &&
"No known constraint!") ? void (0) : __assert_fail ("!ConstraintCode.empty() && \"No known constraint!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4617, __extension__ __PRETTY_FUNCTION__))
;
4618 return isdigit(static_cast<unsigned char>(ConstraintCode[0]));
4619}
4620
4621/// If this is an input matching constraint, this method returns the output
4622/// operand it matches.
4623unsigned TargetLowering::AsmOperandInfo::getMatchedOperand() const {
4624 assert(!ConstraintCode.empty() && "No known constraint!")(static_cast <bool> (!ConstraintCode.empty() &&
"No known constraint!") ? void (0) : __assert_fail ("!ConstraintCode.empty() && \"No known constraint!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4624, __extension__ __PRETTY_FUNCTION__))
;
4625 return atoi(ConstraintCode.c_str());
4626}
4627
4628/// Split up the constraint string from the inline assembly value into the
4629/// specific constraints and their prefixes, and also tie in the associated
4630/// operand values.
4631/// If this returns an empty vector, and if the constraint string itself
4632/// isn't empty, there was an error parsing.
4633TargetLowering::AsmOperandInfoVector
4634TargetLowering::ParseConstraints(const DataLayout &DL,
4635 const TargetRegisterInfo *TRI,
4636 const CallBase &Call) const {
4637 /// Information about all of the constraints.
4638 AsmOperandInfoVector ConstraintOperands;
4639 const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
4640 unsigned maCount = 0; // Largest number of multiple alternative constraints.
4641
4642 // Do a prepass over the constraints, canonicalizing them, and building up the
4643 // ConstraintOperands list.
4644 unsigned ArgNo = 0; // ArgNo - The argument of the CallInst.
4645 unsigned ResNo = 0; // ResNo - The result number of the next output.
4646
4647 for (InlineAsm::ConstraintInfo &CI : IA->ParseConstraints()) {
4648 ConstraintOperands.emplace_back(std::move(CI));
4649 AsmOperandInfo &OpInfo = ConstraintOperands.back();
4650
4651 // Update multiple alternative constraint count.
4652 if (OpInfo.multipleAlternatives.size() > maCount)
4653 maCount = OpInfo.multipleAlternatives.size();
4654
4655 OpInfo.ConstraintVT = MVT::Other;
4656
4657 // Compute the value type for each operand.
4658 switch (OpInfo.Type) {
4659 case InlineAsm::isOutput:
4660 // Indirect outputs just consume an argument.
4661 if (OpInfo.isIndirect) {
4662 OpInfo.CallOperandVal = Call.getArgOperand(ArgNo++);
4663 break;
4664 }
4665
4666 // The return value of the call is this value. As such, there is no
4667 // corresponding argument.
4668 assert(!Call.getType()->isVoidTy() && "Bad inline asm!")(static_cast <bool> (!Call.getType()->isVoidTy() &&
"Bad inline asm!") ? void (0) : __assert_fail ("!Call.getType()->isVoidTy() && \"Bad inline asm!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4668, __extension__ __PRETTY_FUNCTION__))
;
4669 if (StructType *STy = dyn_cast<StructType>(Call.getType())) {
4670 OpInfo.ConstraintVT =
4671 getSimpleValueType(DL, STy->getElementType(ResNo));
4672 } else {
4673 assert(ResNo == 0 && "Asm only has one result!")(static_cast <bool> (ResNo == 0 && "Asm only has one result!"
) ? void (0) : __assert_fail ("ResNo == 0 && \"Asm only has one result!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4673, __extension__ __PRETTY_FUNCTION__))
;
4674 OpInfo.ConstraintVT = getSimpleValueType(DL, Call.getType());
4675 }
4676 ++ResNo;
4677 break;
4678 case InlineAsm::isInput:
4679 OpInfo.CallOperandVal = Call.getArgOperand(ArgNo++);
4680 break;
4681 case InlineAsm::isClobber:
4682 // Nothing to do.
4683 break;
4684 }
4685
4686 if (OpInfo.CallOperandVal) {
4687 llvm::Type *OpTy = OpInfo.CallOperandVal->getType();
4688 if (OpInfo.isIndirect) {
4689 llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy);
4690 if (!PtrTy)
4691 report_fatal_error("Indirect operand for inline asm not a pointer!");
4692 OpTy = PtrTy->getElementType();
4693 }
4694
4695 // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
4696 if (StructType *STy = dyn_cast<StructType>(OpTy))
4697 if (STy->getNumElements() == 1)
4698 OpTy = STy->getElementType(0);
4699
4700 // If OpTy is not a single value, it may be a struct/union that we
4701 // can tile with integers.
4702 if (!OpTy->isSingleValueType() && OpTy->isSized()) {
4703 unsigned BitSize = DL.getTypeSizeInBits(OpTy);
4704 switch (BitSize) {
4705 default: break;
4706 case 1:
4707 case 8:
4708 case 16:
4709 case 32:
4710 case 64:
4711 case 128:
4712 OpInfo.ConstraintVT =
4713 MVT::getVT(IntegerType::get(OpTy->getContext(), BitSize), true);
4714 break;
4715 }
4716 } else if (PointerType *PT = dyn_cast<PointerType>(OpTy)) {
4717 unsigned PtrSize = DL.getPointerSizeInBits(PT->getAddressSpace());
4718 OpInfo.ConstraintVT = MVT::getIntegerVT(PtrSize);
4719 } else {
4720 OpInfo.ConstraintVT = MVT::getVT(OpTy, true);
4721 }
4722 }
4723 }
4724
4725 // If we have multiple alternative constraints, select the best alternative.
4726 if (!ConstraintOperands.empty()) {
4727 if (maCount) {
4728 unsigned bestMAIndex = 0;
4729 int bestWeight = -1;
4730 // weight: -1 = invalid match, and 0 = so-so match to 5 = good match.
4731 int weight = -1;
4732 unsigned maIndex;
4733 // Compute the sums of the weights for each alternative, keeping track
4734 // of the best (highest weight) one so far.
4735 for (maIndex = 0; maIndex < maCount; ++maIndex) {
4736 int weightSum = 0;
4737 for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
4738 cIndex != eIndex; ++cIndex) {
4739 AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];
4740 if (OpInfo.Type == InlineAsm::isClobber)
4741 continue;
4742
4743 // If this is an output operand with a matching input operand,
4744 // look up the matching input. If their types mismatch, e.g. one
4745 // is an integer, the other is floating point, or their sizes are
4746 // different, flag it as an maCantMatch.
4747 if (OpInfo.hasMatchingInput()) {
4748 AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
4749 if (OpInfo.ConstraintVT != Input.ConstraintVT) {
4750 if ((OpInfo.ConstraintVT.isInteger() !=
4751 Input.ConstraintVT.isInteger()) ||
4752 (OpInfo.ConstraintVT.getSizeInBits() !=
4753 Input.ConstraintVT.getSizeInBits())) {
4754 weightSum = -1; // Can't match.
4755 break;
4756 }
4757 }
4758 }
4759 weight = getMultipleConstraintMatchWeight(OpInfo, maIndex);
4760 if (weight == -1) {
4761 weightSum = -1;
4762 break;
4763 }
4764 weightSum += weight;
4765 }
4766 // Update best.
4767 if (weightSum > bestWeight) {
4768 bestWeight = weightSum;
4769 bestMAIndex = maIndex;
4770 }
4771 }
4772
4773 // Now select chosen alternative in each constraint.
4774 for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
4775 cIndex != eIndex; ++cIndex) {
4776 AsmOperandInfo &cInfo = ConstraintOperands[cIndex];
4777 if (cInfo.Type == InlineAsm::isClobber)
4778 continue;
4779 cInfo.selectAlternative(bestMAIndex);
4780 }
4781 }
4782 }
4783
4784 // Check and hook up tied operands, choose constraint code to use.
4785 for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
4786 cIndex != eIndex; ++cIndex) {
4787 AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];
4788
4789 // If this is an output operand with a matching input operand, look up the
4790 // matching input. If their types mismatch, e.g. one is an integer, the
4791 // other is floating point, or their sizes are different, flag it as an
4792 // error.
4793 if (OpInfo.hasMatchingInput()) {
4794 AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
4795
4796 if (OpInfo.ConstraintVT != Input.ConstraintVT) {
4797 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
4798 getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,
4799 OpInfo.ConstraintVT);
4800 std::pair<unsigned, const TargetRegisterClass *> InputRC =
4801 getRegForInlineAsmConstraint(TRI, Input.ConstraintCode,
4802 Input.ConstraintVT);
4803 if ((OpInfo.ConstraintVT.isInteger() !=
4804 Input.ConstraintVT.isInteger()) ||
4805 (MatchRC.second != InputRC.second)) {
4806 report_fatal_error("Unsupported asm: input constraint"
4807 " with a matching output constraint of"
4808 " incompatible type!");
4809 }
4810 }
4811 }
4812 }
4813
4814 return ConstraintOperands;
4815}
4816
4817/// Return an integer indicating how general CT is.
4818static unsigned getConstraintGenerality(TargetLowering::ConstraintType CT) {
4819 switch (CT) {
4820 case TargetLowering::C_Immediate:
4821 case TargetLowering::C_Other:
4822 case TargetLowering::C_Unknown:
4823 return 0;
4824 case TargetLowering::C_Register:
4825 return 1;
4826 case TargetLowering::C_RegisterClass:
4827 return 2;
4828 case TargetLowering::C_Memory:
4829 return 3;
4830 }
4831 llvm_unreachable("Invalid constraint type")::llvm::llvm_unreachable_internal("Invalid constraint type", "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4831)
;
4832}
4833
4834/// Examine constraint type and operand type and determine a weight value.
4835/// This object must already have been set up with the operand type
4836/// and the current alternative constraint selected.
4837TargetLowering::ConstraintWeight
4838 TargetLowering::getMultipleConstraintMatchWeight(
4839 AsmOperandInfo &info, int maIndex) const {
4840 InlineAsm::ConstraintCodeVector *rCodes;
4841 if (maIndex >= (int)info.multipleAlternatives.size())
4842 rCodes = &info.Codes;
4843 else
4844 rCodes = &info.multipleAlternatives[maIndex].Codes;
4845 ConstraintWeight BestWeight = CW_Invalid;
4846
4847 // Loop over the options, keeping track of the most general one.
4848 for (unsigned i = 0, e = rCodes->size(); i != e; ++i) {
4849 ConstraintWeight weight =
4850 getSingleConstraintMatchWeight(info, (*rCodes)[i].c_str());
4851 if (weight > BestWeight)
4852 BestWeight = weight;
4853 }
4854
4855 return BestWeight;
4856}
4857
4858/// Examine constraint type and operand type and determine a weight value.
4859/// This object must already have been set up with the operand type
4860/// and the current alternative constraint selected.
4861TargetLowering::ConstraintWeight
4862 TargetLowering::getSingleConstraintMatchWeight(
4863 AsmOperandInfo &info, const char *constraint) const {
4864 ConstraintWeight weight = CW_Invalid;
4865 Value *CallOperandVal = info.CallOperandVal;
4866 // If we don't have a value, we can't do a match,
4867 // but allow it at the lowest weight.
4868 if (!CallOperandVal)
4869 return CW_Default;
4870 // Look at the constraint type.
4871 switch (*constraint) {
4872 case 'i': // immediate integer.
4873 case 'n': // immediate integer with a known value.
4874 if (isa<ConstantInt>(CallOperandVal))
4875 weight = CW_Constant;
4876 break;
4877 case 's': // non-explicit intregal immediate.
4878 if (isa<GlobalValue>(CallOperandVal))
4879 weight = CW_Constant;
4880 break;
4881 case 'E': // immediate float if host format.
4882 case 'F': // immediate float.
4883 if (isa<ConstantFP>(CallOperandVal))
4884 weight = CW_Constant;
4885 break;
4886 case '<': // memory operand with autodecrement.
4887 case '>': // memory operand with autoincrement.
4888 case 'm': // memory operand.
4889 case 'o': // offsettable memory operand
4890 case 'V': // non-offsettable memory operand
4891 weight = CW_Memory;
4892 break;
4893 case 'r': // general register.
4894 case 'g': // general register, memory operand or immediate integer.
4895 // note: Clang converts "g" to "imr".
4896 if (CallOperandVal->getType()->isIntegerTy())
4897 weight = CW_Register;
4898 break;
4899 case 'X': // any operand.
4900 default:
4901 weight = CW_Default;
4902 break;
4903 }
4904 return weight;
4905}
4906
4907/// If there are multiple different constraints that we could pick for this
4908/// operand (e.g. "imr") try to pick the 'best' one.
4909/// This is somewhat tricky: constraints fall into four classes:
4910/// Other -> immediates and magic values
4911/// Register -> one specific register
4912/// RegisterClass -> a group of regs
4913/// Memory -> memory
4914/// Ideally, we would pick the most specific constraint possible: if we have
4915/// something that fits into a register, we would pick it. The problem here
4916/// is that if we have something that could either be in a register or in
4917/// memory that use of the register could cause selection of *other*
4918/// operands to fail: they might only succeed if we pick memory. Because of
4919/// this the heuristic we use is:
4920///
4921/// 1) If there is an 'other' constraint, and if the operand is valid for
4922/// that constraint, use it. This makes us take advantage of 'i'
4923/// constraints when available.
4924/// 2) Otherwise, pick the most general constraint present. This prefers
4925/// 'm' over 'r', for example.
4926///
4927static void ChooseConstraint(TargetLowering::AsmOperandInfo &OpInfo,
4928 const TargetLowering &TLI,
4929 SDValue Op, SelectionDAG *DAG) {
4930 assert(OpInfo.Codes.size() > 1 && "Doesn't have multiple constraint options")(static_cast <bool> (OpInfo.Codes.size() > 1 &&
"Doesn't have multiple constraint options") ? void (0) : __assert_fail
("OpInfo.Codes.size() > 1 && \"Doesn't have multiple constraint options\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4930, __extension__ __PRETTY_FUNCTION__))
;
4931 unsigned BestIdx = 0;
4932 TargetLowering::ConstraintType BestType = TargetLowering::C_Unknown;
4933 int BestGenerality = -1;
4934
4935 // Loop over the options, keeping track of the most general one.
4936 for (unsigned i = 0, e = OpInfo.Codes.size(); i != e; ++i) {
4937 TargetLowering::ConstraintType CType =
4938 TLI.getConstraintType(OpInfo.Codes[i]);
4939
4940 // Indirect 'other' or 'immediate' constraints are not allowed.
4941 if (OpInfo.isIndirect && !(CType == TargetLowering::C_Memory ||
4942 CType == TargetLowering::C_Register ||
4943 CType == TargetLowering::C_RegisterClass))
4944 continue;
4945
4946 // If this is an 'other' or 'immediate' constraint, see if the operand is
4947 // valid for it. For example, on X86 we might have an 'rI' constraint. If
4948 // the operand is an integer in the range [0..31] we want to use I (saving a
4949 // load of a register), otherwise we must use 'r'.
4950 if ((CType == TargetLowering::C_Other ||
4951 CType == TargetLowering::C_Immediate) && Op.getNode()) {
4952 assert(OpInfo.Codes[i].size() == 1 &&(static_cast <bool> (OpInfo.Codes[i].size() == 1 &&
"Unhandled multi-letter 'other' constraint") ? void (0) : __assert_fail
("OpInfo.Codes[i].size() == 1 && \"Unhandled multi-letter 'other' constraint\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4953, __extension__ __PRETTY_FUNCTION__))
4953 "Unhandled multi-letter 'other' constraint")(static_cast <bool> (OpInfo.Codes[i].size() == 1 &&
"Unhandled multi-letter 'other' constraint") ? void (0) : __assert_fail
("OpInfo.Codes[i].size() == 1 && \"Unhandled multi-letter 'other' constraint\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4953, __extension__ __PRETTY_FUNCTION__))
;
4954 std::vector<SDValue> ResultOps;
4955 TLI.LowerAsmOperandForConstraint(Op, OpInfo.Codes[i],
4956 ResultOps, *DAG);
4957 if (!ResultOps.empty()) {
4958 BestType = CType;
4959 BestIdx = i;
4960 break;
4961 }
4962 }
4963
4964 // Things with matching constraints can only be registers, per gcc
4965 // documentation. This mainly affects "g" constraints.
4966 if (CType == TargetLowering::C_Memory && OpInfo.hasMatchingInput())
4967 continue;
4968
4969 // This constraint letter is more general than the previous one, use it.
4970 int Generality = getConstraintGenerality(CType);
4971 if (Generality > BestGenerality) {
4972 BestType = CType;
4973 BestIdx = i;
4974 BestGenerality = Generality;
4975 }
4976 }
4977
4978 OpInfo.ConstraintCode = OpInfo.Codes[BestIdx];
4979 OpInfo.ConstraintType = BestType;
4980}
4981
4982/// Determines the constraint code and constraint type to use for the specific
4983/// AsmOperandInfo, setting OpInfo.ConstraintCode and OpInfo.ConstraintType.
4984void TargetLowering::ComputeConstraintToUse(AsmOperandInfo &OpInfo,
4985 SDValue Op,
4986 SelectionDAG *DAG) const {
4987 assert(!OpInfo.Codes.empty() && "Must have at least one constraint")(static_cast <bool> (!OpInfo.Codes.empty() && "Must have at least one constraint"
) ? void (0) : __assert_fail ("!OpInfo.Codes.empty() && \"Must have at least one constraint\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4987, __extension__ __PRETTY_FUNCTION__))
;
4988
4989 // Single-letter constraints ('r') are very common.
4990 if (OpInfo.Codes.size() == 1) {
4991 OpInfo.ConstraintCode = OpInfo.Codes[0];
4992 OpInfo.ConstraintType = getConstraintType(OpInfo.ConstraintCode);
4993 } else {
4994 ChooseConstraint(OpInfo, *this, Op, DAG);
4995 }
4996
4997 // 'X' matches anything.
4998 if (OpInfo.ConstraintCode == "X" && OpInfo.CallOperandVal) {
4999 // Labels and constants are handled elsewhere ('X' is the only thing
5000 // that matches labels). For Functions, the type here is the type of
5001 // the result, which is not what we want to look at; leave them alone.
5002 Value *v = OpInfo.CallOperandVal;
5003 if (isa<BasicBlock>(v) || isa<ConstantInt>(v) || isa<Function>(v)) {
5004 OpInfo.CallOperandVal = v;
5005 return;
5006 }
5007
5008 if (Op.getNode() && Op.getOpcode() == ISD::TargetBlockAddress)
5009 return;
5010
5011 // Otherwise, try to resolve it to something we know about by looking at
5012 // the actual operand type.
5013 if (const char *Repl = LowerXConstraint(OpInfo.ConstraintVT)) {
5014 OpInfo.ConstraintCode = Repl;
5015 OpInfo.ConstraintType = getConstraintType(OpInfo.ConstraintCode);
5016 }
5017 }
5018}
5019
5020/// Given an exact SDIV by a constant, create a multiplication
5021/// with the multiplicative inverse of the constant.
5022static SDValue BuildExactSDIV(const TargetLowering &TLI, SDNode *N,
5023 const SDLoc &dl, SelectionDAG &DAG,
5024 SmallVectorImpl<SDNode *> &Created) {
5025 SDValue Op0 = N->getOperand(0);
5026 SDValue Op1 = N->getOperand(1);
5027 EVT VT = N->getValueType(0);
5028 EVT SVT = VT.getScalarType();
5029 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
5030 EVT ShSVT = ShVT.getScalarType();
5031
5032 bool UseSRA = false;
5033 SmallVector<SDValue, 16> Shifts, Factors;
5034
5035 auto BuildSDIVPattern = [&](ConstantSDNode *C) {
5036 if (C->isNullValue())
5037 return false;
5038 APInt Divisor = C->getAPIntValue();
5039 unsigned Shift = Divisor.countTrailingZeros();
5040 if (Shift) {
5041 Divisor.ashrInPlace(Shift);
5042 UseSRA = true;
5043 }
5044 // Calculate the multiplicative inverse, using Newton's method.
5045 APInt t;
5046 APInt Factor = Divisor;
5047 while ((t = Divisor * Factor) != 1)
5048 Factor *= APInt(Divisor.getBitWidth(), 2) - t;
5049 Shifts.push_back(DAG.getConstant(Shift, dl, ShSVT));
5050 Factors.push_back(DAG.getConstant(Factor, dl, SVT));
5051 return true;
5052 };
5053
5054 // Collect all magic values from the build vector.
5055 if (!ISD::matchUnaryPredicate(Op1, BuildSDIVPattern))
5056 return SDValue();
5057
5058 SDValue Shift, Factor;
5059 if (Op1.getOpcode() == ISD::BUILD_VECTOR) {
5060 Shift = DAG.getBuildVector(ShVT, dl, Shifts);
5061 Factor = DAG.getBuildVector(VT, dl, Factors);
5062 } else if (Op1.getOpcode() == ISD::SPLAT_VECTOR) {
5063 assert(Shifts.size() == 1 && Factors.size() == 1 &&(static_cast <bool> (Shifts.size() == 1 && Factors
.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("Shifts.size() == 1 && Factors.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5065, __extension__ __PRETTY_FUNCTION__))
5064 "Expected matchUnaryPredicate to return one element for scalable "(static_cast <bool> (Shifts.size() == 1 && Factors
.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("Shifts.size() == 1 && Factors.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5065, __extension__ __PRETTY_FUNCTION__))
5065 "vectors")(static_cast <bool> (Shifts.size() == 1 && Factors
.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("Shifts.size() == 1 && Factors.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5065, __extension__ __PRETTY_FUNCTION__))
;
5066 Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]);
5067 Factor = DAG.getSplatVector(VT, dl, Factors[0]);
5068 } else {
5069 assert(isa<ConstantSDNode>(Op1) && "Expected a constant")(static_cast <bool> (isa<ConstantSDNode>(Op1) &&
"Expected a constant") ? void (0) : __assert_fail ("isa<ConstantSDNode>(Op1) && \"Expected a constant\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5069, __extension__ __PRETTY_FUNCTION__))
;
5070 Shift = Shifts[0];
5071 Factor = Factors[0];
5072 }
5073
5074 SDValue Res = Op0;
5075
5076 // Shift the value upfront if it is even, so the LSB is one.
5077 if (UseSRA) {
5078 // TODO: For UDIV use SRL instead of SRA.
5079 SDNodeFlags Flags;
5080 Flags.setExact(true);
5081 Res = DAG.getNode(ISD::SRA, dl, VT, Res, Shift, Flags);
5082 Created.push_back(Res.getNode());
5083 }
5084
5085 return DAG.getNode(ISD::MUL, dl, VT, Res, Factor);
5086}
5087
5088SDValue TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
5089 SelectionDAG &DAG,
5090 SmallVectorImpl<SDNode *> &Created) const {
5091 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
5092 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5093 if (TLI.isIntDivCheap(N->getValueType(0), Attr))
5094 return SDValue(N, 0); // Lower SDIV as SDIV
5095 return SDValue();
5096}
5097
5098/// Given an ISD::SDIV node expressing a divide by constant,
5099/// return a DAG expression to select that will generate the same value by
5100/// multiplying by a magic number.
5101/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
5102SDValue TargetLowering::BuildSDIV(SDNode *N, SelectionDAG &DAG,
5103 bool IsAfterLegalization,
5104 SmallVectorImpl<SDNode *> &Created) const {
5105 SDLoc dl(N);
5106 EVT VT = N->getValueType(0);
5107 EVT SVT = VT.getScalarType();
5108 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
5109 EVT ShSVT = ShVT.getScalarType();
5110 unsigned EltBits = VT.getScalarSizeInBits();
5111 EVT MulVT;
5112
5113 // Check to see if we can do this.
5114 // FIXME: We should be more aggressive here.
5115 if (!isTypeLegal(VT)) {
5116 // Limit this to simple scalars for now.
5117 if (VT.isVector() || !VT.isSimple())
5118 return SDValue();
5119
5120 // If this type will be promoted to a large enough type with a legal
5121 // multiply operation, we can go ahead and do this transform.
5122 if (getTypeAction(VT.getSimpleVT()) != TypePromoteInteger)
5123 return SDValue();
5124
5125 MulVT = getTypeToTransformTo(*DAG.getContext(), VT);
5126 if (MulVT.getSizeInBits() < (2 * EltBits) ||
5127 !isOperationLegal(ISD::MUL, MulVT))
5128 return SDValue();
5129 }
5130
5131 // If the sdiv has an 'exact' bit we can use a simpler lowering.
5132 if (N->getFlags().hasExact())
5133 return BuildExactSDIV(*this, N, dl, DAG, Created);
5134
5135 SmallVector<SDValue, 16> MagicFactors, Factors, Shifts, ShiftMasks;
5136
5137 auto BuildSDIVPattern = [&](ConstantSDNode *C) {
5138 if (C->isNullValue())
5139 return false;
5140
5141 const APInt &Divisor = C->getAPIntValue();
5142 APInt::ms magics = Divisor.magic();
5143 int NumeratorFactor = 0;
5144 int ShiftMask = -1;
5145
5146 if (Divisor.isOneValue() || Divisor.isAllOnesValue()) {
5147 // If d is +1/-1, we just multiply the numerator by +1/-1.
5148 NumeratorFactor = Divisor.getSExtValue();
5149 magics.m = 0;
5150 magics.s = 0;
5151 ShiftMask = 0;
5152 } else if (Divisor.isStrictlyPositive() && magics.m.isNegative()) {
5153 // If d > 0 and m < 0, add the numerator.
5154 NumeratorFactor = 1;
5155 } else if (Divisor.isNegative() && magics.m.isStrictlyPositive()) {
5156 // If d < 0 and m > 0, subtract the numerator.
5157 NumeratorFactor = -1;
5158 }
5159
5160 MagicFactors.push_back(DAG.getConstant(magics.m, dl, SVT));
5161 Factors.push_back(DAG.getConstant(NumeratorFactor, dl, SVT));
5162 Shifts.push_back(DAG.getConstant(magics.s, dl, ShSVT));
5163 ShiftMasks.push_back(DAG.getConstant(ShiftMask, dl, SVT));
5164 return true;
5165 };
5166
5167 SDValue N0 = N->getOperand(0);
5168 SDValue N1 = N->getOperand(1);
5169
5170 // Collect the shifts / magic values from each element.
5171 if (!ISD::matchUnaryPredicate(N1, BuildSDIVPattern))
5172 return SDValue();
5173
5174 SDValue MagicFactor, Factor, Shift, ShiftMask;
5175 if (N1.getOpcode() == ISD::BUILD_VECTOR) {
5176 MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);
5177 Factor = DAG.getBuildVector(VT, dl, Factors);
5178 Shift = DAG.getBuildVector(ShVT, dl, Shifts);
5179 ShiftMask = DAG.getBuildVector(VT, dl, ShiftMasks);
5180 } else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {
5181 assert(MagicFactors.size() == 1 && Factors.size() == 1 &&(static_cast <bool> (MagicFactors.size() == 1 &&
Factors.size() == 1 && Shifts.size() == 1 &&
ShiftMasks.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("MagicFactors.size() == 1 && Factors.size() == 1 && Shifts.size() == 1 && ShiftMasks.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5184, __extension__ __PRETTY_FUNCTION__))
5182 Shifts.size() == 1 && ShiftMasks.size() == 1 &&(static_cast <bool> (MagicFactors.size() == 1 &&
Factors.size() == 1 && Shifts.size() == 1 &&
ShiftMasks.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("MagicFactors.size() == 1 && Factors.size() == 1 && Shifts.size() == 1 && ShiftMasks.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5184, __extension__ __PRETTY_FUNCTION__))
5183 "Expected matchUnaryPredicate to return one element for scalable "(static_cast <bool> (MagicFactors.size() == 1 &&
Factors.size() == 1 && Shifts.size() == 1 &&
ShiftMasks.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("MagicFactors.size() == 1 && Factors.size() == 1 && Shifts.size() == 1 && ShiftMasks.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5184, __extension__ __PRETTY_FUNCTION__))
5184 "vectors")(static_cast <bool> (MagicFactors.size() == 1 &&
Factors.size() == 1 && Shifts.size() == 1 &&
ShiftMasks.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
"vectors") ? void (0) : __assert_fail ("MagicFactors.size() == 1 && Factors.size() == 1 && Shifts.size() == 1 && ShiftMasks.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5184, __extension__ __PRETTY_FUNCTION__))
;
5185 MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]);
5186 Factor = DAG.getSplatVector(VT, dl, Factors[0]);
5187 Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]);
5188 ShiftMask = DAG.getSplatVector(VT, dl, ShiftMasks[0]);
5189 } else {
5190 assert(isa<ConstantSDNode>(N1) && "Expected a constant")(static_cast <bool> (isa<ConstantSDNode>(N1) &&
"Expected a constant") ? void (0) : __assert_fail ("isa<ConstantSDNode>(N1) && \"Expected a constant\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5190, __extension__ __PRETTY_FUNCTION__))
;
5191 MagicFactor = MagicFactors[0];
5192 Factor = Factors[0];
5193 Shift = Shifts[0];
5194 ShiftMask = ShiftMasks[0];
5195 }
5196
5197 // Multiply the numerator (operand 0) by the magic value.
5198 // FIXME: We should support doing a MUL in a wider type.
5199 auto GetMULHS = [&](SDValue X, SDValue Y) {
5200 // If the type isn't legal, use a wider mul of the the type calculated
5201 // earlier.
5202 if (!isTypeLegal(VT)) {
5203 X = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, X);
5204 Y = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, Y);
5205 Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y);
5206 Y = DAG.getNode(ISD::SRL, dl, MulVT, Y,
5207 DAG.getShiftAmountConstant(EltBits, MulVT, dl));
5208 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);
5209 }
5210
5211 if (isOperationLegalOrCustom(ISD::MULHS, VT, IsAfterLegalization))
5212 return DAG.getNode(ISD::MULHS, dl, VT, X, Y);
5213 if (isOperationLegalOrCustom(ISD::SMUL_LOHI, VT, IsAfterLegalization)) {
5214 SDValue LoHi =
5215 DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);
5216 return SDValue(LoHi.getNode(), 1);
5217 }
5218 return SDValue();
5219 };
5220
5221 SDValue Q = GetMULHS(N0, MagicFactor);
5222 if (!Q)
5223 return SDValue();
5224
5225 Created.push_back(Q.getNode());
5226
5227 // (Optionally) Add/subtract the numerator using Factor.
5228 Factor = DAG.getNode(ISD::MUL, dl, VT, N0, Factor);
5229 Created.push_back(Factor.getNode());
5230 Q = DAG.getNode(ISD::ADD, dl, VT, Q, Factor);
5231 Created.push_back(Q.getNode());
5232
5233 // Shift right algebraic by shift value.
5234 Q = DAG.getNode(ISD::SRA, dl, VT, Q, Shift);
5235 Created.push_back(Q.getNode());
5236
5237 // Extract the sign bit, mask it and add it to the quotient.
5238 SDValue SignShift = DAG.getConstant(EltBits - 1, dl, ShVT);
5239 SDValue T = DAG.getNode(ISD::SRL, dl, VT, Q, SignShift);
5240 Created.push_back(T.getNode());
5241 T = DAG.getNode(ISD::AND, dl, VT, T, ShiftMask);
5242 Created.push_back(T.getNode());
5243 return DAG.getNode(ISD::ADD, dl, VT, Q, T);
5244}
5245
5246/// Given an ISD::UDIV node expressing a divide by constant,
5247/// return a DAG expression to select that will generate the same value by
5248/// multiplying by a magic number.
5249/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
5250SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG,
5251 bool IsAfterLegalization,
5252 SmallVectorImpl<SDNode *> &Created) const {
5253 SDLoc dl(N);
5254 EVT VT = N->getValueType(0);
5255 EVT SVT = VT.getScalarType();
5256 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
5257 EVT ShSVT = ShVT.getScalarType();
5258 unsigned EltBits = VT.getScalarSizeInBits();
5259 EVT MulVT;
5260
5261 // Check to see if we can do this.
5262 // FIXME: We should be more aggressive here.
5263 if (!isTypeLegal(VT)) {
5264 // Limit this to simple scalars for now.
5265 if (VT.isVector() || !VT.isSimple())
5266 return SDValue();
5267
5268 // If this type will be promoted to a large enough type with a legal
5269 // multiply operation, we can go ahead and do this transform.
5270 if (getTypeAction(VT.getSimpleVT()) != TypePromoteInteger)
5271 return SDValue();
5272
5273 MulVT = getTypeToTransformTo(*DAG.getContext(), VT);
5274 if (MulVT.getSizeInBits() < (2 * EltBits) ||
5275 !isOperationLegal(ISD::MUL, MulVT))
5276 return SDValue();
5277 }
5278
5279 bool UseNPQ = false;
5280 SmallVector<SDValue, 16> PreShifts, PostShifts, MagicFactors, NPQFactors;
5281
5282 auto BuildUDIVPattern = [&](ConstantSDNode *C) {
5283 if (C->isNullValue())
5284 return false;
5285 // FIXME: We should use a narrower constant when the upper
5286 // bits are known to be zero.
5287 const APInt& Divisor = C->getAPIntValue();
5288 APInt::mu magics = Divisor.magicu();
5289 unsigned PreShift = 0, PostShift = 0;
5290
5291 // If the divisor is even, we can avoid using the expensive fixup by
5292 // shifting the divided value upfront.
5293 if (magics.a != 0 && !Divisor[0]) {
5294 PreShift = Divisor.countTrailingZeros();
5295 // Get magic number for the shifted divisor.
5296 magics = Divisor.lshr(PreShift).magicu(PreShift);
5297 assert(magics.a == 0 && "Should use cheap fixup now")(static_cast <bool> (magics.a == 0 && "Should use cheap fixup now"
) ? void (0) : __assert_fail ("magics.a == 0 && \"Should use cheap fixup now\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5297, __extension__ __PRETTY_FUNCTION__))
;
5298 }
5299
5300 APInt Magic = magics.m;
5301
5302 unsigned SelNPQ;
5303 if (magics.a == 0 || Divisor.isOneValue()) {
5304 assert(magics.s < Divisor.getBitWidth() &&(static_cast <bool> (magics.s < Divisor.getBitWidth(
) && "We shouldn't generate an undefined shift!") ? void
(0) : __assert_fail ("magics.s < Divisor.getBitWidth() && \"We shouldn't generate an undefined shift!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5305, __extension__ __PRETTY_FUNCTION__))
5305 "We shouldn't generate an undefined shift!")(static_cast <bool> (magics.s < Divisor.getBitWidth(
) && "We shouldn't generate an undefined shift!") ? void
(0) : __assert_fail ("magics.s < Divisor.getBitWidth() && \"We shouldn't generate an undefined shift!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5305, __extension__ __PRETTY_FUNCTION__))
;
5306 PostShift = magics.s;
5307 SelNPQ = false;
5308 } else {
5309 PostShift = magics.s - 1;
5310 SelNPQ = true;
5311 }
5312
5313 PreShifts.push_back(DAG.getConstant(PreShift, dl, ShSVT));
5314 MagicFactors.push_back(DAG.getConstant(Magic, dl, SVT));
5315 NPQFactors.push_back(
5316 DAG.getConstant(SelNPQ ? APInt::getOneBitSet(EltBits, EltBits - 1)
5317 : APInt::getNullValue(EltBits),
5318 dl, SVT));
5319 PostShifts.push_back(DAG.getConstant(PostShift, dl, ShSVT));
5320 UseNPQ |= SelNPQ;
5321 return true;
5322 };
5323
5324 SDValue N0 = N->getOperand(0);
5325 SDValue N1 = N->getOperand(1);
5326
5327 // Collect the shifts/magic values from each element.
5328 if (!ISD::matchUnaryPredicate(N1, BuildUDIVPattern))
5329 return SDValue();
5330
5331 SDValue PreShift, PostShift, MagicFactor, NPQFactor;
5332 if (N1.getOpcode() == ISD::BUILD_VECTOR) {
5333 PreShift = DAG.getBuildVector(ShVT, dl, PreShifts);
5334 MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);
5335 NPQFactor = DAG.getBuildVector(VT, dl, NPQFactors);
5336 PostShift = DAG.getBuildVector(ShVT, dl, PostShifts);
5337 } else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {
5338 assert(PreShifts.size() == 1 && MagicFactors.size() == 1 &&(static_cast <bool> (PreShifts.size() == 1 && MagicFactors
.size() == 1 && NPQFactors.size() == 1 && PostShifts
.size() == 1 && "Expected matchUnaryPredicate to return one for scalable vectors"
) ? void (0) : __assert_fail ("PreShifts.size() == 1 && MagicFactors.size() == 1 && NPQFactors.size() == 1 && PostShifts.size() == 1 && \"Expected matchUnaryPredicate to return one for scalable vectors\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5340, __extension__ __PRETTY_FUNCTION__))
5339 NPQFactors.size() == 1 && PostShifts.size() == 1 &&(static_cast <bool> (PreShifts.size() == 1 && MagicFactors
.size() == 1 && NPQFactors.size() == 1 && PostShifts
.size() == 1 && "Expected matchUnaryPredicate to return one for scalable vectors"
) ? void (0) : __assert_fail ("PreShifts.size() == 1 && MagicFactors.size() == 1 && NPQFactors.size() == 1 && PostShifts.size() == 1 && \"Expected matchUnaryPredicate to return one for scalable vectors\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5340, __extension__ __PRETTY_FUNCTION__))
5340 "Expected matchUnaryPredicate to return one for scalable vectors")(static_cast <bool> (PreShifts.size() == 1 && MagicFactors
.size() == 1 && NPQFactors.size() == 1 && PostShifts
.size() == 1 && "Expected matchUnaryPredicate to return one for scalable vectors"
) ? void (0) : __assert_fail ("PreShifts.size() == 1 && MagicFactors.size() == 1 && NPQFactors.size() == 1 && PostShifts.size() == 1 && \"Expected matchUnaryPredicate to return one for scalable vectors\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5340, __extension__ __PRETTY_FUNCTION__))
;
5341 PreShift = DAG.getSplatVector(ShVT, dl, PreShifts[0]);
5342 MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]);
5343 NPQFactor = DAG.getSplatVector(VT, dl, NPQFactors[0]);
5344 PostShift = DAG.getSplatVector(ShVT, dl, PostShifts[0]);
5345 } else {
5346 assert(isa<ConstantSDNode>(N1) && "Expected a constant")(static_cast <bool> (isa<ConstantSDNode>(N1) &&
"Expected a constant") ? void (0) : __assert_fail ("isa<ConstantSDNode>(N1) && \"Expected a constant\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5346, __extension__ __PRETTY_FUNCTION__))
;
5347 PreShift = PreShifts[0];
5348 MagicFactor = MagicFactors[0];
5349 PostShift = PostShifts[0];
5350 }
5351
5352 SDValue Q = N0;
5353 Q = DAG.getNode(ISD::SRL, dl, VT, Q, PreShift);
5354 Created.push_back(Q.getNode());
5355
5356 // FIXME: We should support doing a MUL in a wider type.
5357 auto GetMULHU = [&](SDValue X, SDValue Y) {
5358 // If the type isn't legal, use a wider mul of the the type calculated
5359 // earlier.
5360 if (!isTypeLegal(VT)) {
5361 X = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, X);
5362 Y = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, Y);
5363 Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y);
5364 Y = DAG.getNode(ISD::SRL, dl, MulVT, Y,
5365 DAG.getShiftAmountConstant(EltBits, MulVT, dl));
5366 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);
5367 }
5368
5369 if (isOperationLegalOrCustom(ISD::MULHU, VT, IsAfterLegalization))
5370 return DAG.getNode(ISD::MULHU, dl, VT, X, Y);
5371 if (isOperationLegalOrCustom(ISD::UMUL_LOHI, VT, IsAfterLegalization)) {
5372 SDValue LoHi =
5373 DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);
5374 return SDValue(LoHi.getNode(), 1);
5375 }
5376 return SDValue(); // No mulhu or equivalent
5377 };
5378
5379 // Multiply the numerator (operand 0) by the magic value.
5380 Q = GetMULHU(Q, MagicFactor);
5381 if (!Q)
5382 return SDValue();
5383
5384 Created.push_back(Q.getNode());
5385
5386 if (UseNPQ) {
5387 SDValue NPQ = DAG.getNode(ISD::SUB, dl, VT, N0, Q);
5388 Created.push_back(NPQ.getNode());
5389
5390 // For vectors we might have a mix of non-NPQ/NPQ paths, so use
5391 // MULHU to act as a SRL-by-1 for NPQ, else multiply by zero.
5392 if (VT.isVector())
5393 NPQ = GetMULHU(NPQ, NPQFactor);
5394 else
5395 NPQ = DAG.getNode(ISD::SRL, dl, VT, NPQ, DAG.getConstant(1, dl, ShVT));
5396
5397 Created.push_back(NPQ.getNode());
5398
5399 Q = DAG.getNode(ISD::ADD, dl, VT, NPQ, Q);
5400 Created.push_back(Q.getNode());
5401 }
5402
5403 Q = DAG.getNode(ISD::SRL, dl, VT, Q, PostShift);
5404 Created.push_back(Q.getNode());
5405
5406 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
5407
5408 SDValue One = DAG.getConstant(1, dl, VT);
5409 SDValue IsOne = DAG.getSetCC(dl, SetCCVT, N1, One, ISD::SETEQ);
5410 return DAG.getSelect(dl, VT, IsOne, N0, Q);
5411}
5412
5413/// If all values in Values that *don't* match the predicate are same 'splat'
5414/// value, then replace all values with that splat value.
5415/// Else, if AlternativeReplacement was provided, then replace all values that
5416/// do match predicate with AlternativeReplacement value.
5417static void
5418turnVectorIntoSplatVector(MutableArrayRef<SDValue> Values,
5419 std::function<bool(SDValue)> Predicate,
5420 SDValue AlternativeReplacement = SDValue()) {
5421 SDValue Replacement;
5422 // Is there a value for which the Predicate does *NOT* match? What is it?
5423 auto SplatValue = llvm::find_if_not(Values, Predicate);
5424 if (SplatValue != Values.end()) {
5425 // Does Values consist only of SplatValue's and values matching Predicate?
5426 if (llvm::all_of(Values, [Predicate, SplatValue](SDValue Value) {
5427 return Value == *SplatValue || Predicate(Value);
5428 })) // Then we shall replace values matching predicate with SplatValue.
5429 Replacement = *SplatValue;
5430 }
5431 if (!Replacement) {
5432 // Oops, we did not find the "baseline" splat value.
5433 if (!AlternativeReplacement)
5434 return; // Nothing to do.
5435 // Let's replace with provided value then.
5436 Replacement = AlternativeReplacement;
5437 }
5438 std::replace_if(Values.begin(), Values.end(), Predicate, Replacement);
5439}
5440
5441/// Given an ISD::UREM used only by an ISD::SETEQ or ISD::SETNE
5442/// where the divisor is constant and the comparison target is zero,
5443/// return a DAG expression that will generate the same comparison result
5444/// using only multiplications, additions and shifts/rotations.
5445/// Ref: "Hacker's Delight" 10-17.
5446SDValue TargetLowering::buildUREMEqFold(EVT SETCCVT, SDValue REMNode,
5447 SDValue CompTargetNode,
5448 ISD::CondCode Cond,
5449 DAGCombinerInfo &DCI,
5450 const SDLoc &DL) const {
5451 SmallVector<SDNode *, 5> Built;
5452 if (SDValue Folded = prepareUREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
5453 DCI, DL, Built)) {
5454 for (SDNode *N : Built)
5455 DCI.AddToWorklist(N);
5456 return Folded;
5457 }
5458
5459 return SDValue();
5460}
5461
5462SDValue
5463TargetLowering::prepareUREMEqFold(EVT SETCCVT, SDValue REMNode,
5464 SDValue CompTargetNode, ISD::CondCode Cond,
5465 DAGCombinerInfo &DCI, const SDLoc &DL,
5466 SmallVectorImpl<SDNode *> &Created) const {
5467 // fold (seteq/ne (urem N, D), 0) -> (setule/ugt (rotr (mul N, P), K), Q)
5468 // - D must be constant, with D = D0 * 2^K where D0 is odd
5469 // - P is the multiplicative inverse of D0 modulo 2^W
5470 // - Q = floor(((2^W) - 1) / D)
5471 // where W is the width of the common type of N and D.
5472 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Only applicable for (in)equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5473, __extension__ __PRETTY_FUNCTION__))
5473 "Only applicable for (in)equality comparisons.")(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Only applicable for (in)equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5473, __extension__ __PRETTY_FUNCTION__))
;
5474
5475 SelectionDAG &DAG = DCI.DAG;
5476
5477 EVT VT = REMNode.getValueType();
5478 EVT SVT = VT.getScalarType();
5479 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout(), !DCI.isBeforeLegalize());
5480 EVT ShSVT = ShVT.getScalarType();
5481
5482 // If MUL is unavailable, we cannot proceed in any case.
5483 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT))
5484 return SDValue();
5485
5486 bool ComparingWithAllZeros = true;
5487 bool AllComparisonsWithNonZerosAreTautological = true;
5488 bool HadTautologicalLanes = false;
5489 bool AllLanesAreTautological = true;
5490 bool HadEvenDivisor = false;
5491 bool AllDivisorsArePowerOfTwo = true;
5492 bool HadTautologicalInvertedLanes = false;
5493 SmallVector<SDValue, 16> PAmts, KAmts, QAmts, IAmts;
5494
5495 auto BuildUREMPattern = [&](ConstantSDNode *CDiv, ConstantSDNode *CCmp) {
5496 // Division by 0 is UB. Leave it to be constant-folded elsewhere.
5497 if (CDiv->isNullValue())
5498 return false;
5499
5500 const APInt &D = CDiv->getAPIntValue();
5501 const APInt &Cmp = CCmp->getAPIntValue();
5502
5503 ComparingWithAllZeros &= Cmp.isNullValue();
5504
5505 // x u% C1` is *always* less than C1. So given `x u% C1 == C2`,
5506 // if C2 is not less than C1, the comparison is always false.
5507 // But we will only be able to produce the comparison that will give the
5508 // opposive tautological answer. So this lane would need to be fixed up.
5509 bool TautologicalInvertedLane = D.ule(Cmp);
5510 HadTautologicalInvertedLanes |= TautologicalInvertedLane;
5511
5512 // If all lanes are tautological (either all divisors are ones, or divisor
5513 // is not greater than the constant we are comparing with),
5514 // we will prefer to avoid the fold.
5515 bool TautologicalLane = D.isOneValue() || TautologicalInvertedLane;
5516 HadTautologicalLanes |= TautologicalLane;
5517 AllLanesAreTautological &= TautologicalLane;
5518
5519 // If we are comparing with non-zero, we need'll need to subtract said
5520 // comparison value from the LHS. But there is no point in doing that if
5521 // every lane where we are comparing with non-zero is tautological..
5522 if (!Cmp.isNullValue())
5523 AllComparisonsWithNonZerosAreTautological &= TautologicalLane;
5524
5525 // Decompose D into D0 * 2^K
5526 unsigned K = D.countTrailingZeros();
5527 assert((!D.isOneValue() || (K == 0)) && "For divisor '1' we won't rotate.")(static_cast <bool> ((!D.isOneValue() || (K == 0)) &&
"For divisor '1' we won't rotate.") ? void (0) : __assert_fail
("(!D.isOneValue() || (K == 0)) && \"For divisor '1' we won't rotate.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5527, __extension__ __PRETTY_FUNCTION__))
;
5528 APInt D0 = D.lshr(K);
5529
5530 // D is even if it has trailing zeros.
5531 HadEvenDivisor |= (K != 0);
5532 // D is a power-of-two if D0 is one.
5533 // If all divisors are power-of-two, we will prefer to avoid the fold.
5534 AllDivisorsArePowerOfTwo &= D0.isOneValue();
5535
5536 // P = inv(D0, 2^W)
5537 // 2^W requires W + 1 bits, so we have to extend and then truncate.
5538 unsigned W = D.getBitWidth();
5539 APInt P = D0.zext(W + 1)
5540 .multiplicativeInverse(APInt::getSignedMinValue(W + 1))
5541 .trunc(W);
5542 assert(!P.isNullValue() && "No multiplicative inverse!")(static_cast <bool> (!P.isNullValue() && "No multiplicative inverse!"
) ? void (0) : __assert_fail ("!P.isNullValue() && \"No multiplicative inverse!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5542, __extension__ __PRETTY_FUNCTION__))
; // unreachable
5543 assert((D0 * P).isOneValue() && "Multiplicative inverse sanity check.")(static_cast <bool> ((D0 * P).isOneValue() && "Multiplicative inverse sanity check."
) ? void (0) : __assert_fail ("(D0 * P).isOneValue() && \"Multiplicative inverse sanity check.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5543, __extension__ __PRETTY_FUNCTION__))
;
5544
5545 // Q = floor((2^W - 1) u/ D)
5546 // R = ((2^W - 1) u% D)
5547 APInt Q, R;
5548 APInt::udivrem(APInt::getAllOnesValue(W), D, Q, R);
5549
5550 // If we are comparing with zero, then that comparison constant is okay,
5551 // else it may need to be one less than that.
5552 if (Cmp.ugt(R))
5553 Q -= 1;
5554
5555 assert(APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) &&(static_cast <bool> (APInt::getAllOnesValue(ShSVT.getSizeInBits
()).ugt(K) && "We are expecting that K is always less than all-ones for ShSVT"
) ? void (0) : __assert_fail ("APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) && \"We are expecting that K is always less than all-ones for ShSVT\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5556, __extension__ __PRETTY_FUNCTION__))
5556 "We are expecting that K is always less than all-ones for ShSVT")(static_cast <bool> (APInt::getAllOnesValue(ShSVT.getSizeInBits
()).ugt(K) && "We are expecting that K is always less than all-ones for ShSVT"
) ? void (0) : __assert_fail ("APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) && \"We are expecting that K is always less than all-ones for ShSVT\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5556, __extension__ __PRETTY_FUNCTION__))
;
5557
5558 // If the lane is tautological the result can be constant-folded.
5559 if (TautologicalLane) {
5560 // Set P and K amount to a bogus values so we can try to splat them.
5561 P = 0;
5562 K = -1;
5563 // And ensure that comparison constant is tautological,
5564 // it will always compare true/false.
5565 Q = -1;
5566 }
5567
5568 PAmts.push_back(DAG.getConstant(P, DL, SVT));
5569 KAmts.push_back(
5570 DAG.getConstant(APInt(ShSVT.getSizeInBits(), K), DL, ShSVT));
5571 QAmts.push_back(DAG.getConstant(Q, DL, SVT));
5572 return true;
5573 };
5574
5575 SDValue N = REMNode.getOperand(0);
5576 SDValue D = REMNode.getOperand(1);
5577
5578 // Collect the values from each element.
5579 if (!ISD::matchBinaryPredicate(D, CompTargetNode, BuildUREMPattern))
5580 return SDValue();
5581
5582 // If all lanes are tautological, the result can be constant-folded.
5583 if (AllLanesAreTautological)
5584 return SDValue();
5585
5586 // If this is a urem by a powers-of-two, avoid the fold since it can be
5587 // best implemented as a bit test.
5588 if (AllDivisorsArePowerOfTwo)
5589 return SDValue();
5590
5591 SDValue PVal, KVal, QVal;
5592 if (VT.isVector()) {
5593 if (HadTautologicalLanes) {
5594 // Try to turn PAmts into a splat, since we don't care about the values
5595 // that are currently '0'. If we can't, just keep '0'`s.
5596 turnVectorIntoSplatVector(PAmts, isNullConstant);
5597 // Try to turn KAmts into a splat, since we don't care about the values
5598 // that are currently '-1'. If we can't, change them to '0'`s.
5599 turnVectorIntoSplatVector(KAmts, isAllOnesConstant,
5600 DAG.getConstant(0, DL, ShSVT));
5601 }
5602
5603 PVal = DAG.getBuildVector(VT, DL, PAmts);
5604 KVal = DAG.getBuildVector(ShVT, DL, KAmts);
5605 QVal = DAG.getBuildVector(VT, DL, QAmts);
5606 } else {
5607 PVal = PAmts[0];
5608 KVal = KAmts[0];
5609 QVal = QAmts[0];
5610 }
5611
5612 if (!ComparingWithAllZeros && !AllComparisonsWithNonZerosAreTautological) {
5613 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::SUB, VT))
5614 return SDValue(); // FIXME: Could/should use `ISD::ADD`?
5615 assert(CompTargetNode.getValueType() == N.getValueType() &&(static_cast <bool> (CompTargetNode.getValueType() == N
.getValueType() && "Expecting that the types on LHS and RHS of comparisons match."
) ? void (0) : __assert_fail ("CompTargetNode.getValueType() == N.getValueType() && \"Expecting that the types on LHS and RHS of comparisons match.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5616, __extension__ __PRETTY_FUNCTION__))
5616 "Expecting that the types on LHS and RHS of comparisons match.")(static_cast <bool> (CompTargetNode.getValueType() == N
.getValueType() && "Expecting that the types on LHS and RHS of comparisons match."
) ? void (0) : __assert_fail ("CompTargetNode.getValueType() == N.getValueType() && \"Expecting that the types on LHS and RHS of comparisons match.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5616, __extension__ __PRETTY_FUNCTION__))
;
5617 N = DAG.getNode(ISD::SUB, DL, VT, N, CompTargetNode);
5618 }
5619
5620 // (mul N, P)
5621 SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal);
5622 Created.push_back(Op0.getNode());
5623
5624 // Rotate right only if any divisor was even. We avoid rotates for all-odd
5625 // divisors as a performance improvement, since rotating by 0 is a no-op.
5626 if (HadEvenDivisor) {
5627 // We need ROTR to do this.
5628 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ROTR, VT))
5629 return SDValue();
5630 SDNodeFlags Flags;
5631 Flags.setExact(true);
5632 // UREM: (rotr (mul N, P), K)
5633 Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal, Flags);
5634 Created.push_back(Op0.getNode());
5635 }
5636
5637 // UREM: (setule/setugt (rotr (mul N, P), K), Q)
5638 SDValue NewCC =
5639 DAG.getSetCC(DL, SETCCVT, Op0, QVal,
5640 ((Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT));
5641 if (!HadTautologicalInvertedLanes)
5642 return NewCC;
5643
5644 // If any lanes previously compared always-false, the NewCC will give
5645 // always-true result for them, so we need to fixup those lanes.
5646 // Or the other way around for inequality predicate.
5647 assert(VT.isVector() && "Can/should only get here for vectors.")(static_cast <bool> (VT.isVector() && "Can/should only get here for vectors."
) ? void (0) : __assert_fail ("VT.isVector() && \"Can/should only get here for vectors.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5647, __extension__ __PRETTY_FUNCTION__))
;
5648 Created.push_back(NewCC.getNode());
5649
5650 // x u% C1` is *always* less than C1. So given `x u% C1 == C2`,
5651 // if C2 is not less than C1, the comparison is always false.
5652 // But we have produced the comparison that will give the
5653 // opposive tautological answer. So these lanes would need to be fixed up.
5654 SDValue TautologicalInvertedChannels =
5655 DAG.getSetCC(DL, SETCCVT, D, CompTargetNode, ISD::SETULE);
5656 Created.push_back(TautologicalInvertedChannels.getNode());
5657
5658 // NOTE: we avoid letting illegal types through even if we're before legalize
5659 // ops – legalization has a hard time producing good code for this.
5660 if (isOperationLegalOrCustom(ISD::VSELECT, SETCCVT)) {
5661 // If we have a vector select, let's replace the comparison results in the
5662 // affected lanes with the correct tautological result.
5663 SDValue Replacement = DAG.getBoolConstant(Cond == ISD::SETEQ ? false : true,
5664 DL, SETCCVT, SETCCVT);
5665 return DAG.getNode(ISD::VSELECT, DL, SETCCVT, TautologicalInvertedChannels,
5666 Replacement, NewCC);
5667 }
5668
5669 // Else, we can just invert the comparison result in the appropriate lanes.
5670 //
5671 // NOTE: see the note above VSELECT above.
5672 if (isOperationLegalOrCustom(ISD::XOR, SETCCVT))
5673 return DAG.getNode(ISD::XOR, DL, SETCCVT, NewCC,
5674 TautologicalInvertedChannels);
5675
5676 return SDValue(); // Don't know how to lower.
5677}
5678
5679/// Given an ISD::SREM used only by an ISD::SETEQ or ISD::SETNE
5680/// where the divisor is constant and the comparison target is zero,
5681/// return a DAG expression that will generate the same comparison result
5682/// using only multiplications, additions and shifts/rotations.
5683/// Ref: "Hacker's Delight" 10-17.
5684SDValue TargetLowering::buildSREMEqFold(EVT SETCCVT, SDValue REMNode,
5685 SDValue CompTargetNode,
5686 ISD::CondCode Cond,
5687 DAGCombinerInfo &DCI,
5688 const SDLoc &DL) const {
5689 SmallVector<SDNode *, 7> Built;
5690 if (SDValue Folded = prepareSREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
5691 DCI, DL, Built)) {
5692 assert(Built.size() <= 7 && "Max size prediction failed.")(static_cast <bool> (Built.size() <= 7 && "Max size prediction failed."
) ? void (0) : __assert_fail ("Built.size() <= 7 && \"Max size prediction failed.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5692, __extension__ __PRETTY_FUNCTION__))
;
5693 for (SDNode *N : Built)
5694 DCI.AddToWorklist(N);
5695 return Folded;
5696 }
5697
5698 return SDValue();
5699}
5700
5701SDValue
5702TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode,
5703 SDValue CompTargetNode, ISD::CondCode Cond,
5704 DAGCombinerInfo &DCI, const SDLoc &DL,
5705 SmallVectorImpl<SDNode *> &Created) const {
5706 // Fold:
5707 // (seteq/ne (srem N, D), 0)
5708 // To:
5709 // (setule/ugt (rotr (add (mul N, P), A), K), Q)
5710 //
5711 // - D must be constant, with D = D0 * 2^K where D0 is odd
5712 // - P is the multiplicative inverse of D0 modulo 2^W
5713 // - A = bitwiseand(floor((2^(W - 1) - 1) / D0), (-(2^k)))
5714 // - Q = floor((2 * A) / (2^K))
5715 // where W is the width of the common type of N and D.
5716 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Only applicable for (in)equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5717, __extension__ __PRETTY_FUNCTION__))
5717 "Only applicable for (in)equality comparisons.")(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Only applicable for (in)equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5717, __extension__ __PRETTY_FUNCTION__))
;
5718
5719 SelectionDAG &DAG = DCI.DAG;
5720
5721 EVT VT = REMNode.getValueType();
5722 EVT SVT = VT.getScalarType();
5723 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout(), !DCI.isBeforeLegalize());
5724 EVT ShSVT = ShVT.getScalarType();
5725
5726 // If we are after ops legalization, and MUL is unavailable, we can not
5727 // proceed.
5728 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT))
5729 return SDValue();
5730
5731 // TODO: Could support comparing with non-zero too.
5732 ConstantSDNode *CompTarget = isConstOrConstSplat(CompTargetNode);
5733 if (!CompTarget || !CompTarget->isNullValue())