Bug Summary

File:llvm/include/llvm/CodeGen/SelectionDAGNodes.h
Warning:line 1154, column 10
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name DAGCombiner.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -mframe-pointer=none -fmath-errno -fno-rounding-math -masm-verbose -mconstructor-aliases -munwind-tables -target-cpu x86-64 -dwarf-column-info -fno-split-dwarf-inlining -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-10/lib/clang/10.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/build-llvm/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/build-llvm/include -I /build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-10/lib/clang/10.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/build-llvm/lib/CodeGen/SelectionDAG -fdebug-prefix-map=/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd=. -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2020-01-13-084841-49055-1 -x c++ /build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

1//===- DAGCombiner.cpp - Implement a DAG node combiner --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass combines dag nodes to form fewer, simpler DAG nodes. It can be run
10// both before and after the DAG is legalized.
11//
12// This pass is not a substitute for the LLVM IR instcombine pass. This pass is
13// primarily intended to handle simplification opportunities that are implicit
14// in the LLVM IR and exposed by the various codegen lowering phases.
15//
16//===----------------------------------------------------------------------===//
17
18#include "llvm/ADT/APFloat.h"
19#include "llvm/ADT/APInt.h"
20#include "llvm/ADT/ArrayRef.h"
21#include "llvm/ADT/DenseMap.h"
22#include "llvm/ADT/IntervalMap.h"
23#include "llvm/ADT/None.h"
24#include "llvm/ADT/Optional.h"
25#include "llvm/ADT/STLExtras.h"
26#include "llvm/ADT/SetVector.h"
27#include "llvm/ADT/SmallPtrSet.h"
28#include "llvm/ADT/SmallSet.h"
29#include "llvm/ADT/SmallVector.h"
30#include "llvm/ADT/Statistic.h"
31#include "llvm/Analysis/AliasAnalysis.h"
32#include "llvm/Analysis/MemoryLocation.h"
33#include "llvm/CodeGen/DAGCombine.h"
34#include "llvm/CodeGen/ISDOpcodes.h"
35#include "llvm/CodeGen/MachineFrameInfo.h"
36#include "llvm/CodeGen/MachineFunction.h"
37#include "llvm/CodeGen/MachineMemOperand.h"
38#include "llvm/CodeGen/RuntimeLibcalls.h"
39#include "llvm/CodeGen/SelectionDAG.h"
40#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
41#include "llvm/CodeGen/SelectionDAGNodes.h"
42#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
43#include "llvm/CodeGen/TargetLowering.h"
44#include "llvm/CodeGen/TargetRegisterInfo.h"
45#include "llvm/CodeGen/TargetSubtargetInfo.h"
46#include "llvm/CodeGen/ValueTypes.h"
47#include "llvm/IR/Attributes.h"
48#include "llvm/IR/Constant.h"
49#include "llvm/IR/DataLayout.h"
50#include "llvm/IR/DerivedTypes.h"
51#include "llvm/IR/Function.h"
52#include "llvm/IR/LLVMContext.h"
53#include "llvm/IR/Metadata.h"
54#include "llvm/Support/Casting.h"
55#include "llvm/Support/CodeGen.h"
56#include "llvm/Support/CommandLine.h"
57#include "llvm/Support/Compiler.h"
58#include "llvm/Support/Debug.h"
59#include "llvm/Support/ErrorHandling.h"
60#include "llvm/Support/KnownBits.h"
61#include "llvm/Support/MachineValueType.h"
62#include "llvm/Support/MathExtras.h"
63#include "llvm/Support/raw_ostream.h"
64#include "llvm/Target/TargetMachine.h"
65#include "llvm/Target/TargetOptions.h"
66#include <algorithm>
67#include <cassert>
68#include <cstdint>
69#include <functional>
70#include <iterator>
71#include <string>
72#include <tuple>
73#include <utility>
74
75using namespace llvm;
76
77#define DEBUG_TYPE"dagcombine" "dagcombine"
78
79STATISTIC(NodesCombined , "Number of dag nodes combined")static llvm::Statistic NodesCombined = {"dagcombine", "NodesCombined"
, "Number of dag nodes combined"}
;
80STATISTIC(PreIndexedNodes , "Number of pre-indexed nodes created")static llvm::Statistic PreIndexedNodes = {"dagcombine", "PreIndexedNodes"
, "Number of pre-indexed nodes created"}
;
81STATISTIC(PostIndexedNodes, "Number of post-indexed nodes created")static llvm::Statistic PostIndexedNodes = {"dagcombine", "PostIndexedNodes"
, "Number of post-indexed nodes created"}
;
82STATISTIC(OpsNarrowed , "Number of load/op/store narrowed")static llvm::Statistic OpsNarrowed = {"dagcombine", "OpsNarrowed"
, "Number of load/op/store narrowed"}
;
83STATISTIC(LdStFP2Int , "Number of fp load/store pairs transformed to int")static llvm::Statistic LdStFP2Int = {"dagcombine", "LdStFP2Int"
, "Number of fp load/store pairs transformed to int"}
;
84STATISTIC(SlicedLoads, "Number of load sliced")static llvm::Statistic SlicedLoads = {"dagcombine", "SlicedLoads"
, "Number of load sliced"}
;
85STATISTIC(NumFPLogicOpsConv, "Number of logic ops converted to fp ops")static llvm::Statistic NumFPLogicOpsConv = {"dagcombine", "NumFPLogicOpsConv"
, "Number of logic ops converted to fp ops"}
;
86
87static cl::opt<bool>
88CombinerGlobalAA("combiner-global-alias-analysis", cl::Hidden,
89 cl::desc("Enable DAG combiner's use of IR alias analysis"));
90
91static cl::opt<bool>
92UseTBAA("combiner-use-tbaa", cl::Hidden, cl::init(true),
93 cl::desc("Enable DAG combiner's use of TBAA"));
94
95#ifndef NDEBUG
96static cl::opt<std::string>
97CombinerAAOnlyFunc("combiner-aa-only-func", cl::Hidden,
98 cl::desc("Only use DAG-combiner alias analysis in this"
99 " function"));
100#endif
101
102/// Hidden option to stress test load slicing, i.e., when this option
103/// is enabled, load slicing bypasses most of its profitability guards.
104static cl::opt<bool>
105StressLoadSlicing("combiner-stress-load-slicing", cl::Hidden,
106 cl::desc("Bypass the profitability model of load slicing"),
107 cl::init(false));
108
109static cl::opt<bool>
110 MaySplitLoadIndex("combiner-split-load-index", cl::Hidden, cl::init(true),
111 cl::desc("DAG combiner may split indexing from loads"));
112
113static cl::opt<bool>
114 EnableStoreMerging("combiner-store-merging", cl::Hidden, cl::init(true),
115 cl::desc("DAG combiner enable merging multiple stores "
116 "into a wider store"));
117
118static cl::opt<unsigned> TokenFactorInlineLimit(
119 "combiner-tokenfactor-inline-limit", cl::Hidden, cl::init(2048),
120 cl::desc("Limit the number of operands to inline for Token Factors"));
121
122static cl::opt<unsigned> StoreMergeDependenceLimit(
123 "combiner-store-merge-dependence-limit", cl::Hidden, cl::init(10),
124 cl::desc("Limit the number of times for the same StoreNode and RootNode "
125 "to bail out in store merging dependence check"));
126
127namespace {
128
129 class DAGCombiner {
130 SelectionDAG &DAG;
131 const TargetLowering &TLI;
132 CombineLevel Level;
133 CodeGenOpt::Level OptLevel;
134 bool LegalDAG = false;
135 bool LegalOperations = false;
136 bool LegalTypes = false;
137 bool ForCodeSize;
138
139 /// Worklist of all of the nodes that need to be simplified.
140 ///
141 /// This must behave as a stack -- new nodes to process are pushed onto the
142 /// back and when processing we pop off of the back.
143 ///
144 /// The worklist will not contain duplicates but may contain null entries
145 /// due to nodes being deleted from the underlying DAG.
146 SmallVector<SDNode *, 64> Worklist;
147
148 /// Mapping from an SDNode to its position on the worklist.
149 ///
150 /// This is used to find and remove nodes from the worklist (by nulling
151 /// them) when they are deleted from the underlying DAG. It relies on
152 /// stable indices of nodes within the worklist.
153 DenseMap<SDNode *, unsigned> WorklistMap;
154 /// This records all nodes attempted to add to the worklist since we
155 /// considered a new worklist entry. As we keep do not add duplicate nodes
156 /// in the worklist, this is different from the tail of the worklist.
157 SmallSetVector<SDNode *, 32> PruningList;
158
159 /// Set of nodes which have been combined (at least once).
160 ///
161 /// This is used to allow us to reliably add any operands of a DAG node
162 /// which have not yet been combined to the worklist.
163 SmallPtrSet<SDNode *, 32> CombinedNodes;
164
165 /// Map from candidate StoreNode to the pair of RootNode and count.
166 /// The count is used to track how many times we have seen the StoreNode
167 /// with the same RootNode bail out in dependence check. If we have seen
168 /// the bail out for the same pair many times over a limit, we won't
169 /// consider the StoreNode with the same RootNode as store merging
170 /// candidate again.
171 DenseMap<SDNode *, std::pair<SDNode *, unsigned>> StoreRootCountMap;
172
173 // AA - Used for DAG load/store alias analysis.
174 AliasAnalysis *AA;
175
176 /// When an instruction is simplified, add all users of the instruction to
177 /// the work lists because they might get more simplified now.
178 void AddUsersToWorklist(SDNode *N) {
179 for (SDNode *Node : N->uses())
180 AddToWorklist(Node);
181 }
182
183 /// Convenient shorthand to add a node and all of its user to the worklist.
184 void AddToWorklistWithUsers(SDNode *N) {
185 AddUsersToWorklist(N);
186 AddToWorklist(N);
187 }
188
189 // Prune potentially dangling nodes. This is called after
190 // any visit to a node, but should also be called during a visit after any
191 // failed combine which may have created a DAG node.
192 void clearAddedDanglingWorklistEntries() {
193 // Check any nodes added to the worklist to see if they are prunable.
194 while (!PruningList.empty()) {
195 auto *N = PruningList.pop_back_val();
196 if (N->use_empty())
197 recursivelyDeleteUnusedNodes(N);
198 }
199 }
200
201 SDNode *getNextWorklistEntry() {
202 // Before we do any work, remove nodes that are not in use.
203 clearAddedDanglingWorklistEntries();
204 SDNode *N = nullptr;
205 // The Worklist holds the SDNodes in order, but it may contain null
206 // entries.
207 while (!N && !Worklist.empty()) {
208 N = Worklist.pop_back_val();
209 }
210
211 if (N) {
212 bool GoodWorklistEntry = WorklistMap.erase(N);
213 (void)GoodWorklistEntry;
214 assert(GoodWorklistEntry &&((GoodWorklistEntry && "Found a worklist entry without a corresponding map entry!"
) ? static_cast<void> (0) : __assert_fail ("GoodWorklistEntry && \"Found a worklist entry without a corresponding map entry!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 215, __PRETTY_FUNCTION__))
215 "Found a worklist entry without a corresponding map entry!")((GoodWorklistEntry && "Found a worklist entry without a corresponding map entry!"
) ? static_cast<void> (0) : __assert_fail ("GoodWorklistEntry && \"Found a worklist entry without a corresponding map entry!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 215, __PRETTY_FUNCTION__))
;
216 }
217 return N;
218 }
219
220 /// Call the node-specific routine that folds each particular type of node.
221 SDValue visit(SDNode *N);
222
223 public:
224 DAGCombiner(SelectionDAG &D, AliasAnalysis *AA, CodeGenOpt::Level OL)
225 : DAG(D), TLI(D.getTargetLoweringInfo()), Level(BeforeLegalizeTypes),
226 OptLevel(OL), AA(AA) {
227 ForCodeSize = DAG.shouldOptForSize();
228
229 MaximumLegalStoreInBits = 0;
230 // We use the minimum store size here, since that's all we can guarantee
231 // for the scalable vector types.
232 for (MVT VT : MVT::all_valuetypes())
233 if (EVT(VT).isSimple() && VT != MVT::Other &&
234 TLI.isTypeLegal(EVT(VT)) &&
235 VT.getSizeInBits().getKnownMinSize() >= MaximumLegalStoreInBits)
236 MaximumLegalStoreInBits = VT.getSizeInBits().getKnownMinSize();
237 }
238
239 void ConsiderForPruning(SDNode *N) {
240 // Mark this for potential pruning.
241 PruningList.insert(N);
242 }
243
244 /// Add to the worklist making sure its instance is at the back (next to be
245 /// processed.)
246 void AddToWorklist(SDNode *N) {
247 assert(N->getOpcode() != ISD::DELETED_NODE &&((N->getOpcode() != ISD::DELETED_NODE && "Deleted Node added to Worklist"
) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && \"Deleted Node added to Worklist\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 248, __PRETTY_FUNCTION__))
248 "Deleted Node added to Worklist")((N->getOpcode() != ISD::DELETED_NODE && "Deleted Node added to Worklist"
) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && \"Deleted Node added to Worklist\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 248, __PRETTY_FUNCTION__))
;
249
250 // Skip handle nodes as they can't usefully be combined and confuse the
251 // zero-use deletion strategy.
252 if (N->getOpcode() == ISD::HANDLENODE)
253 return;
254
255 ConsiderForPruning(N);
256
257 if (WorklistMap.insert(std::make_pair(N, Worklist.size())).second)
258 Worklist.push_back(N);
259 }
260
261 /// Remove all instances of N from the worklist.
262 void removeFromWorklist(SDNode *N) {
263 CombinedNodes.erase(N);
264 PruningList.remove(N);
265 StoreRootCountMap.erase(N);
266
267 auto It = WorklistMap.find(N);
268 if (It == WorklistMap.end())
269 return; // Not in the worklist.
270
271 // Null out the entry rather than erasing it to avoid a linear operation.
272 Worklist[It->second] = nullptr;
273 WorklistMap.erase(It);
274 }
275
276 void deleteAndRecombine(SDNode *N);
277 bool recursivelyDeleteUnusedNodes(SDNode *N);
278
279 /// Replaces all uses of the results of one DAG node with new values.
280 SDValue CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
281 bool AddTo = true);
282
283 /// Replaces all uses of the results of one DAG node with new values.
284 SDValue CombineTo(SDNode *N, SDValue Res, bool AddTo = true) {
285 return CombineTo(N, &Res, 1, AddTo);
286 }
287
288 /// Replaces all uses of the results of one DAG node with new values.
289 SDValue CombineTo(SDNode *N, SDValue Res0, SDValue Res1,
290 bool AddTo = true) {
291 SDValue To[] = { Res0, Res1 };
292 return CombineTo(N, To, 2, AddTo);
293 }
294
295 void CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO);
296
297 private:
298 unsigned MaximumLegalStoreInBits;
299
300 /// Check the specified integer node value to see if it can be simplified or
301 /// if things it uses can be simplified by bit propagation.
302 /// If so, return true.
303 bool SimplifyDemandedBits(SDValue Op) {
304 unsigned BitWidth = Op.getScalarValueSizeInBits();
305 APInt DemandedBits = APInt::getAllOnesValue(BitWidth);
306 return SimplifyDemandedBits(Op, DemandedBits);
307 }
308
309 bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits) {
310 EVT VT = Op.getValueType();
311 unsigned NumElts = VT.isVector() ? VT.getVectorNumElements() : 1;
312 APInt DemandedElts = APInt::getAllOnesValue(NumElts);
313 return SimplifyDemandedBits(Op, DemandedBits, DemandedElts);
314 }
315
316 /// Check the specified vector node value to see if it can be simplified or
317 /// if things it uses can be simplified as it only uses some of the
318 /// elements. If so, return true.
319 bool SimplifyDemandedVectorElts(SDValue Op) {
320 unsigned NumElts = Op.getValueType().getVectorNumElements();
321 APInt DemandedElts = APInt::getAllOnesValue(NumElts);
322 return SimplifyDemandedVectorElts(Op, DemandedElts);
323 }
324
325 bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
326 const APInt &DemandedElts);
327 bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedElts,
328 bool AssumeSingleUse = false);
329
330 bool CombineToPreIndexedLoadStore(SDNode *N);
331 bool CombineToPostIndexedLoadStore(SDNode *N);
332 SDValue SplitIndexingFromLoad(LoadSDNode *LD);
333 bool SliceUpLoad(SDNode *N);
334
335 // Scalars have size 0 to distinguish from singleton vectors.
336 SDValue ForwardStoreValueToDirectLoad(LoadSDNode *LD);
337 bool getTruncatedStoreValue(StoreSDNode *ST, SDValue &Val);
338 bool extendLoadedValueToExtension(LoadSDNode *LD, SDValue &Val);
339
340 /// Replace an ISD::EXTRACT_VECTOR_ELT of a load with a narrowed
341 /// load.
342 ///
343 /// \param EVE ISD::EXTRACT_VECTOR_ELT to be replaced.
344 /// \param InVecVT type of the input vector to EVE with bitcasts resolved.
345 /// \param EltNo index of the vector element to load.
346 /// \param OriginalLoad load that EVE came from to be replaced.
347 /// \returns EVE on success SDValue() on failure.
348 SDValue scalarizeExtractedVectorLoad(SDNode *EVE, EVT InVecVT,
349 SDValue EltNo,
350 LoadSDNode *OriginalLoad);
351 void ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad);
352 SDValue PromoteOperand(SDValue Op, EVT PVT, bool &Replace);
353 SDValue SExtPromoteOperand(SDValue Op, EVT PVT);
354 SDValue ZExtPromoteOperand(SDValue Op, EVT PVT);
355 SDValue PromoteIntBinOp(SDValue Op);
356 SDValue PromoteIntShiftOp(SDValue Op);
357 SDValue PromoteExtend(SDValue Op);
358 bool PromoteLoad(SDValue Op);
359
360 /// Call the node-specific routine that knows how to fold each
361 /// particular type of node. If that doesn't do anything, try the
362 /// target-specific DAG combines.
363 SDValue combine(SDNode *N);
364
365 // Visitation implementation - Implement dag node combining for different
366 // node types. The semantics are as follows:
367 // Return Value:
368 // SDValue.getNode() == 0 - No change was made
369 // SDValue.getNode() == N - N was replaced, is dead and has been handled.
370 // otherwise - N should be replaced by the returned Operand.
371 //
372 SDValue visitTokenFactor(SDNode *N);
373 SDValue visitMERGE_VALUES(SDNode *N);
374 SDValue visitADD(SDNode *N);
375 SDValue visitADDLike(SDNode *N);
376 SDValue visitADDLikeCommutative(SDValue N0, SDValue N1, SDNode *LocReference);
377 SDValue visitSUB(SDNode *N);
378 SDValue visitADDSAT(SDNode *N);
379 SDValue visitSUBSAT(SDNode *N);
380 SDValue visitADDC(SDNode *N);
381 SDValue visitADDO(SDNode *N);
382 SDValue visitUADDOLike(SDValue N0, SDValue N1, SDNode *N);
383 SDValue visitSUBC(SDNode *N);
384 SDValue visitSUBO(SDNode *N);
385 SDValue visitADDE(SDNode *N);
386 SDValue visitADDCARRY(SDNode *N);
387 SDValue visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn, SDNode *N);
388 SDValue visitSUBE(SDNode *N);
389 SDValue visitSUBCARRY(SDNode *N);
390 SDValue visitMUL(SDNode *N);
391 SDValue visitMULFIX(SDNode *N);
392 SDValue useDivRem(SDNode *N);
393 SDValue visitSDIV(SDNode *N);
394 SDValue visitSDIVLike(SDValue N0, SDValue N1, SDNode *N);
395 SDValue visitUDIV(SDNode *N);
396 SDValue visitUDIVLike(SDValue N0, SDValue N1, SDNode *N);
397 SDValue visitREM(SDNode *N);
398 SDValue visitMULHU(SDNode *N);
399 SDValue visitMULHS(SDNode *N);
400 SDValue visitSMUL_LOHI(SDNode *N);
401 SDValue visitUMUL_LOHI(SDNode *N);
402 SDValue visitMULO(SDNode *N);
403 SDValue visitIMINMAX(SDNode *N);
404 SDValue visitAND(SDNode *N);
405 SDValue visitANDLike(SDValue N0, SDValue N1, SDNode *N);
406 SDValue visitOR(SDNode *N);
407 SDValue visitORLike(SDValue N0, SDValue N1, SDNode *N);
408 SDValue visitXOR(SDNode *N);
409 SDValue SimplifyVBinOp(SDNode *N);
410 SDValue visitSHL(SDNode *N);
411 SDValue visitSRA(SDNode *N);
412 SDValue visitSRL(SDNode *N);
413 SDValue visitFunnelShift(SDNode *N);
414 SDValue visitRotate(SDNode *N);
415 SDValue visitABS(SDNode *N);
416 SDValue visitBSWAP(SDNode *N);
417 SDValue visitBITREVERSE(SDNode *N);
418 SDValue visitCTLZ(SDNode *N);
419 SDValue visitCTLZ_ZERO_UNDEF(SDNode *N);
420 SDValue visitCTTZ(SDNode *N);
421 SDValue visitCTTZ_ZERO_UNDEF(SDNode *N);
422 SDValue visitCTPOP(SDNode *N);
423 SDValue visitSELECT(SDNode *N);
424 SDValue visitVSELECT(SDNode *N);
425 SDValue visitSELECT_CC(SDNode *N);
426 SDValue visitSETCC(SDNode *N);
427 SDValue visitSETCCCARRY(SDNode *N);
428 SDValue visitSIGN_EXTEND(SDNode *N);
429 SDValue visitZERO_EXTEND(SDNode *N);
430 SDValue visitANY_EXTEND(SDNode *N);
431 SDValue visitAssertExt(SDNode *N);
432 SDValue visitSIGN_EXTEND_INREG(SDNode *N);
433 SDValue visitSIGN_EXTEND_VECTOR_INREG(SDNode *N);
434 SDValue visitZERO_EXTEND_VECTOR_INREG(SDNode *N);
435 SDValue visitTRUNCATE(SDNode *N);
436 SDValue visitBITCAST(SDNode *N);
437 SDValue visitBUILD_PAIR(SDNode *N);
438 SDValue visitFADD(SDNode *N);
439 SDValue visitFSUB(SDNode *N);
440 SDValue visitFMUL(SDNode *N);
441 SDValue visitFMA(SDNode *N);
442 SDValue visitFDIV(SDNode *N);
443 SDValue visitFREM(SDNode *N);
444 SDValue visitFSQRT(SDNode *N);
445 SDValue visitFCOPYSIGN(SDNode *N);
446 SDValue visitFPOW(SDNode *N);
447 SDValue visitSINT_TO_FP(SDNode *N);
448 SDValue visitUINT_TO_FP(SDNode *N);
449 SDValue visitFP_TO_SINT(SDNode *N);
450 SDValue visitFP_TO_UINT(SDNode *N);
451 SDValue visitFP_ROUND(SDNode *N);
452 SDValue visitFP_EXTEND(SDNode *N);
453 SDValue visitFNEG(SDNode *N);
454 SDValue visitFABS(SDNode *N);
455 SDValue visitFCEIL(SDNode *N);
456 SDValue visitFTRUNC(SDNode *N);
457 SDValue visitFFLOOR(SDNode *N);
458 SDValue visitFMINNUM(SDNode *N);
459 SDValue visitFMAXNUM(SDNode *N);
460 SDValue visitFMINIMUM(SDNode *N);
461 SDValue visitFMAXIMUM(SDNode *N);
462 SDValue visitBRCOND(SDNode *N);
463 SDValue visitBR_CC(SDNode *N);
464 SDValue visitLOAD(SDNode *N);
465
466 SDValue replaceStoreChain(StoreSDNode *ST, SDValue BetterChain);
467 SDValue replaceStoreOfFPConstant(StoreSDNode *ST);
468
469 SDValue visitSTORE(SDNode *N);
470 SDValue visitLIFETIME_END(SDNode *N);
471 SDValue visitINSERT_VECTOR_ELT(SDNode *N);
472 SDValue visitEXTRACT_VECTOR_ELT(SDNode *N);
473 SDValue visitBUILD_VECTOR(SDNode *N);
474 SDValue visitCONCAT_VECTORS(SDNode *N);
475 SDValue visitEXTRACT_SUBVECTOR(SDNode *N);
476 SDValue visitVECTOR_SHUFFLE(SDNode *N);
477 SDValue visitSCALAR_TO_VECTOR(SDNode *N);
478 SDValue visitINSERT_SUBVECTOR(SDNode *N);
479 SDValue visitMLOAD(SDNode *N);
480 SDValue visitMSTORE(SDNode *N);
481 SDValue visitMGATHER(SDNode *N);
482 SDValue visitMSCATTER(SDNode *N);
483 SDValue visitFP_TO_FP16(SDNode *N);
484 SDValue visitFP16_TO_FP(SDNode *N);
485 SDValue visitVECREDUCE(SDNode *N);
486
487 SDValue visitFADDForFMACombine(SDNode *N);
488 SDValue visitFSUBForFMACombine(SDNode *N);
489 SDValue visitFMULForFMADistributiveCombine(SDNode *N);
490
491 SDValue XformToShuffleWithZero(SDNode *N);
492 bool reassociationCanBreakAddressingModePattern(unsigned Opc,
493 const SDLoc &DL, SDValue N0,
494 SDValue N1);
495 SDValue reassociateOpsCommutative(unsigned Opc, const SDLoc &DL, SDValue N0,
496 SDValue N1);
497 SDValue reassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0,
498 SDValue N1, SDNodeFlags Flags);
499
500 SDValue visitShiftByConstant(SDNode *N);
501
502 SDValue foldSelectOfConstants(SDNode *N);
503 SDValue foldVSelectOfConstants(SDNode *N);
504 SDValue foldBinOpIntoSelect(SDNode *BO);
505 bool SimplifySelectOps(SDNode *SELECT, SDValue LHS, SDValue RHS);
506 SDValue hoistLogicOpWithSameOpcodeHands(SDNode *N);
507 SDValue SimplifySelect(const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2);
508 SDValue SimplifySelectCC(const SDLoc &DL, SDValue N0, SDValue N1,
509 SDValue N2, SDValue N3, ISD::CondCode CC,
510 bool NotExtCompare = false);
511 SDValue convertSelectOfFPConstantsToLoadOffset(
512 const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2, SDValue N3,
513 ISD::CondCode CC);
514 SDValue foldSelectCCToShiftAnd(const SDLoc &DL, SDValue N0, SDValue N1,
515 SDValue N2, SDValue N3, ISD::CondCode CC);
516 SDValue foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1,
517 const SDLoc &DL);
518 SDValue unfoldMaskedMerge(SDNode *N);
519 SDValue unfoldExtremeBitClearingToShifts(SDNode *N);
520 SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
521 const SDLoc &DL, bool foldBooleans);
522 SDValue rebuildSetCC(SDValue N);
523
524 bool isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
525 SDValue &CC) const;
526 bool isOneUseSetCC(SDValue N) const;
527 bool isCheaperToUseNegatedFPOps(SDValue X, SDValue Y);
528
529 SDValue SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
530 unsigned HiOp);
531 SDValue CombineConsecutiveLoads(SDNode *N, EVT VT);
532 SDValue CombineExtLoad(SDNode *N);
533 SDValue CombineZExtLogicopShiftLoad(SDNode *N);
534 SDValue combineRepeatedFPDivisors(SDNode *N);
535 SDValue combineInsertEltToShuffle(SDNode *N, unsigned InsIndex);
536 SDValue ConstantFoldBITCASTofBUILD_VECTOR(SDNode *, EVT);
537 SDValue BuildSDIV(SDNode *N);
538 SDValue BuildSDIVPow2(SDNode *N);
539 SDValue BuildUDIV(SDNode *N);
540 SDValue BuildLogBase2(SDValue V, const SDLoc &DL);
541 SDValue BuildDivEstimate(SDValue N, SDValue Op, SDNodeFlags Flags);
542 SDValue buildRsqrtEstimate(SDValue Op, SDNodeFlags Flags);
543 SDValue buildSqrtEstimate(SDValue Op, SDNodeFlags Flags);
544 SDValue buildSqrtEstimateImpl(SDValue Op, SDNodeFlags Flags, bool Recip);
545 SDValue buildSqrtNROneConst(SDValue Arg, SDValue Est, unsigned Iterations,
546 SDNodeFlags Flags, bool Reciprocal);
547 SDValue buildSqrtNRTwoConst(SDValue Arg, SDValue Est, unsigned Iterations,
548 SDNodeFlags Flags, bool Reciprocal);
549 SDValue MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
550 bool DemandHighBits = true);
551 SDValue MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1);
552 SDValue MatchRotatePosNeg(SDValue Shifted, SDValue Pos, SDValue Neg,
553 SDValue InnerPos, SDValue InnerNeg,
554 unsigned PosOpcode, unsigned NegOpcode,
555 const SDLoc &DL);
556 SDValue MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL);
557 SDValue MatchLoadCombine(SDNode *N);
558 SDValue MatchStoreCombine(StoreSDNode *N);
559 SDValue ReduceLoadWidth(SDNode *N);
560 SDValue ReduceLoadOpStoreWidth(SDNode *N);
561 SDValue splitMergedValStore(StoreSDNode *ST);
562 SDValue TransformFPLoadStorePair(SDNode *N);
563 SDValue convertBuildVecZextToZext(SDNode *N);
564 SDValue reduceBuildVecExtToExtBuildVec(SDNode *N);
565 SDValue reduceBuildVecToShuffle(SDNode *N);
566 SDValue createBuildVecShuffle(const SDLoc &DL, SDNode *N,
567 ArrayRef<int> VectorMask, SDValue VecIn1,
568 SDValue VecIn2, unsigned LeftIdx,
569 bool DidSplitVec);
570 SDValue matchVSelectOpSizesWithSetCC(SDNode *Cast);
571
572 /// Walk up chain skipping non-aliasing memory nodes,
573 /// looking for aliasing nodes and adding them to the Aliases vector.
574 void GatherAllAliases(SDNode *N, SDValue OriginalChain,
575 SmallVectorImpl<SDValue> &Aliases);
576
577 /// Return true if there is any possibility that the two addresses overlap.
578 bool isAlias(SDNode *Op0, SDNode *Op1) const;
579
580 /// Walk up chain skipping non-aliasing memory nodes, looking for a better
581 /// chain (aliasing node.)
582 SDValue FindBetterChain(SDNode *N, SDValue Chain);
583
584 /// Try to replace a store and any possibly adjacent stores on
585 /// consecutive chains with better chains. Return true only if St is
586 /// replaced.
587 ///
588 /// Notice that other chains may still be replaced even if the function
589 /// returns false.
590 bool findBetterNeighborChains(StoreSDNode *St);
591
592 // Helper for findBetterNeighborChains. Walk up store chain add additional
593 // chained stores that do not overlap and can be parallelized.
594 bool parallelizeChainedStores(StoreSDNode *St);
595
596 /// Holds a pointer to an LSBaseSDNode as well as information on where it
597 /// is located in a sequence of memory operations connected by a chain.
598 struct MemOpLink {
599 // Ptr to the mem node.
600 LSBaseSDNode *MemNode;
601
602 // Offset from the base ptr.
603 int64_t OffsetFromBase;
604
605 MemOpLink(LSBaseSDNode *N, int64_t Offset)
606 : MemNode(N), OffsetFromBase(Offset) {}
607 };
608
609 /// This is a helper function for visitMUL to check the profitability
610 /// of folding (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2).
611 /// MulNode is the original multiply, AddNode is (add x, c1),
612 /// and ConstNode is c2.
613 bool isMulAddWithConstProfitable(SDNode *MulNode,
614 SDValue &AddNode,
615 SDValue &ConstNode);
616
617 /// This is a helper function for visitAND and visitZERO_EXTEND. Returns
618 /// true if the (and (load x) c) pattern matches an extload. ExtVT returns
619 /// the type of the loaded value to be extended.
620 bool isAndLoadExtLoad(ConstantSDNode *AndC, LoadSDNode *LoadN,
621 EVT LoadResultTy, EVT &ExtVT);
622
623 /// Helper function to calculate whether the given Load/Store can have its
624 /// width reduced to ExtVT.
625 bool isLegalNarrowLdSt(LSBaseSDNode *LDSTN, ISD::LoadExtType ExtType,
626 EVT &MemVT, unsigned ShAmt = 0);
627
628 /// Used by BackwardsPropagateMask to find suitable loads.
629 bool SearchForAndLoads(SDNode *N, SmallVectorImpl<LoadSDNode*> &Loads,
630 SmallPtrSetImpl<SDNode*> &NodesWithConsts,
631 ConstantSDNode *Mask, SDNode *&NodeToMask);
632 /// Attempt to propagate a given AND node back to load leaves so that they
633 /// can be combined into narrow loads.
634 bool BackwardsPropagateMask(SDNode *N);
635
636 /// Helper function for MergeConsecutiveStores which merges the
637 /// component store chains.
638 SDValue getMergeStoreChains(SmallVectorImpl<MemOpLink> &StoreNodes,
639 unsigned NumStores);
640
641 /// This is a helper function for MergeConsecutiveStores. When the
642 /// source elements of the consecutive stores are all constants or
643 /// all extracted vector elements, try to merge them into one
644 /// larger store introducing bitcasts if necessary. \return True
645 /// if a merged store was created.
646 bool MergeStoresOfConstantsOrVecElts(SmallVectorImpl<MemOpLink> &StoreNodes,
647 EVT MemVT, unsigned NumStores,
648 bool IsConstantSrc, bool UseVector,
649 bool UseTrunc);
650
651 /// This is a helper function for MergeConsecutiveStores. Stores
652 /// that potentially may be merged with St are placed in
653 /// StoreNodes. RootNode is a chain predecessor to all store
654 /// candidates.
655 void getStoreMergeCandidates(StoreSDNode *St,
656 SmallVectorImpl<MemOpLink> &StoreNodes,
657 SDNode *&Root);
658
659 /// Helper function for MergeConsecutiveStores. Checks if
660 /// candidate stores have indirect dependency through their
661 /// operands. RootNode is the predecessor to all stores calculated
662 /// by getStoreMergeCandidates and is used to prune the dependency check.
663 /// \return True if safe to merge.
664 bool checkMergeStoreCandidatesForDependencies(
665 SmallVectorImpl<MemOpLink> &StoreNodes, unsigned NumStores,
666 SDNode *RootNode);
667
668 /// Merge consecutive store operations into a wide store.
669 /// This optimization uses wide integers or vectors when possible.
670 /// \return number of stores that were merged into a merged store (the
671 /// affected nodes are stored as a prefix in \p StoreNodes).
672 bool MergeConsecutiveStores(StoreSDNode *St);
673
674 /// Try to transform a truncation where C is a constant:
675 /// (trunc (and X, C)) -> (and (trunc X), (trunc C))
676 ///
677 /// \p N needs to be a truncation and its first operand an AND. Other
678 /// requirements are checked by the function (e.g. that trunc is
679 /// single-use) and if missed an empty SDValue is returned.
680 SDValue distributeTruncateThroughAnd(SDNode *N);
681
682 /// Helper function to determine whether the target supports operation
683 /// given by \p Opcode for type \p VT, that is, whether the operation
684 /// is legal or custom before legalizing operations, and whether is
685 /// legal (but not custom) after legalization.
686 bool hasOperation(unsigned Opcode, EVT VT) {
687 if (LegalOperations)
688 return TLI.isOperationLegal(Opcode, VT);
689 return TLI.isOperationLegalOrCustom(Opcode, VT);
690 }
691
692 public:
693 /// Runs the dag combiner on all nodes in the work list
694 void Run(CombineLevel AtLevel);
695
696 SelectionDAG &getDAG() const { return DAG; }
697
698 /// Returns a type large enough to hold any valid shift amount - before type
699 /// legalization these can be huge.
700 EVT getShiftAmountTy(EVT LHSTy) {
701 assert(LHSTy.isInteger() && "Shift amount is not an integer type!")((LHSTy.isInteger() && "Shift amount is not an integer type!"
) ? static_cast<void> (0) : __assert_fail ("LHSTy.isInteger() && \"Shift amount is not an integer type!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 701, __PRETTY_FUNCTION__))
;
702 return TLI.getShiftAmountTy(LHSTy, DAG.getDataLayout(), LegalTypes);
703 }
704
705 /// This method returns true if we are running before type legalization or
706 /// if the specified VT is legal.
707 bool isTypeLegal(const EVT &VT) {
708 if (!LegalTypes) return true;
709 return TLI.isTypeLegal(VT);
710 }
711
712 /// Convenience wrapper around TargetLowering::getSetCCResultType
713 EVT getSetCCResultType(EVT VT) const {
714 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
715 }
716
717 void ExtendSetCCUses(const SmallVectorImpl<SDNode *> &SetCCs,
718 SDValue OrigLoad, SDValue ExtLoad,
719 ISD::NodeType ExtType);
720 };
721
722/// This class is a DAGUpdateListener that removes any deleted
723/// nodes from the worklist.
724class WorklistRemover : public SelectionDAG::DAGUpdateListener {
725 DAGCombiner &DC;
726
727public:
728 explicit WorklistRemover(DAGCombiner &dc)
729 : SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
730
731 void NodeDeleted(SDNode *N, SDNode *E) override {
732 DC.removeFromWorklist(N);
733 }
734};
735
736class WorklistInserter : public SelectionDAG::DAGUpdateListener {
737 DAGCombiner &DC;
738
739public:
740 explicit WorklistInserter(DAGCombiner &dc)
741 : SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
742
743 // FIXME: Ideally we could add N to the worklist, but this causes exponential
744 // compile time costs in large DAGs, e.g. Halide.
745 void NodeInserted(SDNode *N) override { DC.ConsiderForPruning(N); }
746};
747
748} // end anonymous namespace
749
750//===----------------------------------------------------------------------===//
751// TargetLowering::DAGCombinerInfo implementation
752//===----------------------------------------------------------------------===//
753
754void TargetLowering::DAGCombinerInfo::AddToWorklist(SDNode *N) {
755 ((DAGCombiner*)DC)->AddToWorklist(N);
756}
757
758SDValue TargetLowering::DAGCombinerInfo::
759CombineTo(SDNode *N, ArrayRef<SDValue> To, bool AddTo) {
760 return ((DAGCombiner*)DC)->CombineTo(N, &To[0], To.size(), AddTo);
761}
762
763SDValue TargetLowering::DAGCombinerInfo::
764CombineTo(SDNode *N, SDValue Res, bool AddTo) {
765 return ((DAGCombiner*)DC)->CombineTo(N, Res, AddTo);
766}
767
768SDValue TargetLowering::DAGCombinerInfo::
769CombineTo(SDNode *N, SDValue Res0, SDValue Res1, bool AddTo) {
770 return ((DAGCombiner*)DC)->CombineTo(N, Res0, Res1, AddTo);
771}
772
773bool TargetLowering::DAGCombinerInfo::
774recursivelyDeleteUnusedNodes(SDNode *N) {
775 return ((DAGCombiner*)DC)->recursivelyDeleteUnusedNodes(N);
776}
777
778void TargetLowering::DAGCombinerInfo::
779CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
780 return ((DAGCombiner*)DC)->CommitTargetLoweringOpt(TLO);
781}
782
783//===----------------------------------------------------------------------===//
784// Helper Functions
785//===----------------------------------------------------------------------===//
786
787void DAGCombiner::deleteAndRecombine(SDNode *N) {
788 removeFromWorklist(N);
789
790 // If the operands of this node are only used by the node, they will now be
791 // dead. Make sure to re-visit them and recursively delete dead nodes.
792 for (const SDValue &Op : N->ops())
793 // For an operand generating multiple values, one of the values may
794 // become dead allowing further simplification (e.g. split index
795 // arithmetic from an indexed load).
796 if (Op->hasOneUse() || Op->getNumValues() > 1)
797 AddToWorklist(Op.getNode());
798
799 DAG.DeleteNode(N);
800}
801
802// APInts must be the same size for most operations, this helper
803// function zero extends the shorter of the pair so that they match.
804// We provide an Offset so that we can create bitwidths that won't overflow.
805static void zeroExtendToMatch(APInt &LHS, APInt &RHS, unsigned Offset = 0) {
806 unsigned Bits = Offset + std::max(LHS.getBitWidth(), RHS.getBitWidth());
807 LHS = LHS.zextOrSelf(Bits);
808 RHS = RHS.zextOrSelf(Bits);
809}
810
811// Return true if this node is a setcc, or is a select_cc
812// that selects between the target values used for true and false, making it
813// equivalent to a setcc. Also, set the incoming LHS, RHS, and CC references to
814// the appropriate nodes based on the type of node we are checking. This
815// simplifies life a bit for the callers.
816bool DAGCombiner::isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
817 SDValue &CC) const {
818 if (N.getOpcode() == ISD::SETCC) {
819 LHS = N.getOperand(0);
820 RHS = N.getOperand(1);
821 CC = N.getOperand(2);
822 return true;
823 }
824
825 if (N.getOpcode() != ISD::SELECT_CC ||
826 !TLI.isConstTrueVal(N.getOperand(2).getNode()) ||
827 !TLI.isConstFalseVal(N.getOperand(3).getNode()))
828 return false;
829
830 if (TLI.getBooleanContents(N.getValueType()) ==
831 TargetLowering::UndefinedBooleanContent)
832 return false;
833
834 LHS = N.getOperand(0);
835 RHS = N.getOperand(1);
836 CC = N.getOperand(4);
837 return true;
838}
839
840/// Return true if this is a SetCC-equivalent operation with only one use.
841/// If this is true, it allows the users to invert the operation for free when
842/// it is profitable to do so.
843bool DAGCombiner::isOneUseSetCC(SDValue N) const {
844 SDValue N0, N1, N2;
845 if (isSetCCEquivalent(N, N0, N1, N2) && N.getNode()->hasOneUse())
846 return true;
847 return false;
848}
849
850// Returns the SDNode if it is a constant float BuildVector
851// or constant float.
852static SDNode *isConstantFPBuildVectorOrConstantFP(SDValue N) {
853 if (isa<ConstantFPSDNode>(N))
854 return N.getNode();
855 if (ISD::isBuildVectorOfConstantFPSDNodes(N.getNode()))
856 return N.getNode();
857 return nullptr;
858}
859
860// Determines if it is a constant integer or a build vector of constant
861// integers (and undefs).
862// Do not permit build vector implicit truncation.
863static bool isConstantOrConstantVector(SDValue N, bool NoOpaques = false) {
864 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N))
865 return !(Const->isOpaque() && NoOpaques);
866 if (N.getOpcode() != ISD::BUILD_VECTOR)
867 return false;
868 unsigned BitWidth = N.getScalarValueSizeInBits();
869 for (const SDValue &Op : N->op_values()) {
870 if (Op.isUndef())
871 continue;
872 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Op);
873 if (!Const || Const->getAPIntValue().getBitWidth() != BitWidth ||
874 (Const->isOpaque() && NoOpaques))
875 return false;
876 }
877 return true;
878}
879
880// Determines if a BUILD_VECTOR is composed of all-constants possibly mixed with
881// undef's.
882static bool isAnyConstantBuildVector(SDValue V, bool NoOpaques = false) {
883 if (V.getOpcode() != ISD::BUILD_VECTOR)
884 return false;
885 return isConstantOrConstantVector(V, NoOpaques) ||
886 ISD::isBuildVectorOfConstantFPSDNodes(V.getNode());
887}
888
889bool DAGCombiner::reassociationCanBreakAddressingModePattern(unsigned Opc,
890 const SDLoc &DL,
891 SDValue N0,
892 SDValue N1) {
893 // Currently this only tries to ensure we don't undo the GEP splits done by
894 // CodeGenPrepare when shouldConsiderGEPOffsetSplit is true. To ensure this,
895 // we check if the following transformation would be problematic:
896 // (load/store (add, (add, x, offset1), offset2)) ->
897 // (load/store (add, x, offset1+offset2)).
898
899 if (Opc != ISD::ADD || N0.getOpcode() != ISD::ADD)
900 return false;
901
902 if (N0.hasOneUse())
903 return false;
904
905 auto *C1 = dyn_cast<ConstantSDNode>(N0.getOperand(1));
906 auto *C2 = dyn_cast<ConstantSDNode>(N1);
907 if (!C1 || !C2)
908 return false;
909
910 const APInt &C1APIntVal = C1->getAPIntValue();
911 const APInt &C2APIntVal = C2->getAPIntValue();
912 if (C1APIntVal.getBitWidth() > 64 || C2APIntVal.getBitWidth() > 64)
913 return false;
914
915 const APInt CombinedValueIntVal = C1APIntVal + C2APIntVal;
916 if (CombinedValueIntVal.getBitWidth() > 64)
917 return false;
918 const int64_t CombinedValue = CombinedValueIntVal.getSExtValue();
919
920 for (SDNode *Node : N0->uses()) {
921 auto LoadStore = dyn_cast<MemSDNode>(Node);
922 if (LoadStore) {
923 // Is x[offset2] already not a legal addressing mode? If so then
924 // reassociating the constants breaks nothing (we test offset2 because
925 // that's the one we hope to fold into the load or store).
926 TargetLoweringBase::AddrMode AM;
927 AM.HasBaseReg = true;
928 AM.BaseOffs = C2APIntVal.getSExtValue();
929 EVT VT = LoadStore->getMemoryVT();
930 unsigned AS = LoadStore->getAddressSpace();
931 Type *AccessTy = VT.getTypeForEVT(*DAG.getContext());
932 if (!TLI.isLegalAddressingMode(DAG.getDataLayout(), AM, AccessTy, AS))
933 continue;
934
935 // Would x[offset1+offset2] still be a legal addressing mode?
936 AM.BaseOffs = CombinedValue;
937 if (!TLI.isLegalAddressingMode(DAG.getDataLayout(), AM, AccessTy, AS))
938 return true;
939 }
940 }
941
942 return false;
943}
944
945// Helper for DAGCombiner::reassociateOps. Try to reassociate an expression
946// such as (Opc N0, N1), if \p N0 is the same kind of operation as \p Opc.
947SDValue DAGCombiner::reassociateOpsCommutative(unsigned Opc, const SDLoc &DL,
948 SDValue N0, SDValue N1) {
949 EVT VT = N0.getValueType();
950
951 if (N0.getOpcode() != Opc)
952 return SDValue();
953
954 // Don't reassociate reductions.
955 if (N0->getFlags().hasVectorReduction())
956 return SDValue();
957
958 if (SDNode *C1 = DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1))) {
959 if (SDNode *C2 = DAG.isConstantIntBuildVectorOrConstantInt(N1)) {
960 // Reassociate: (op (op x, c1), c2) -> (op x, (op c1, c2))
961 if (SDValue OpNode = DAG.FoldConstantArithmetic(Opc, DL, VT, C1, C2))
962 return DAG.getNode(Opc, DL, VT, N0.getOperand(0), OpNode);
963 return SDValue();
964 }
965 if (N0.hasOneUse()) {
966 // Reassociate: (op (op x, c1), y) -> (op (op x, y), c1)
967 // iff (op x, c1) has one use
968 SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N0.getOperand(0), N1);
969 if (!OpNode.getNode())
970 return SDValue();
971 return DAG.getNode(Opc, DL, VT, OpNode, N0.getOperand(1));
972 }
973 }
974 return SDValue();
975}
976
977// Try to reassociate commutative binops.
978SDValue DAGCombiner::reassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0,
979 SDValue N1, SDNodeFlags Flags) {
980 assert(TLI.isCommutativeBinOp(Opc) && "Operation not commutative.")((TLI.isCommutativeBinOp(Opc) && "Operation not commutative."
) ? static_cast<void> (0) : __assert_fail ("TLI.isCommutativeBinOp(Opc) && \"Operation not commutative.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 980, __PRETTY_FUNCTION__))
;
981 // Don't reassociate reductions.
982 if (Flags.hasVectorReduction())
983 return SDValue();
984
985 // Floating-point reassociation is not allowed without loose FP math.
986 if (N0.getValueType().isFloatingPoint() ||
987 N1.getValueType().isFloatingPoint())
988 if (!Flags.hasAllowReassociation() || !Flags.hasNoSignedZeros())
989 return SDValue();
990
991 if (SDValue Combined = reassociateOpsCommutative(Opc, DL, N0, N1))
992 return Combined;
993 if (SDValue Combined = reassociateOpsCommutative(Opc, DL, N1, N0))
994 return Combined;
995 return SDValue();
996}
997
998SDValue DAGCombiner::CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
999 bool AddTo) {
1000 assert(N->getNumValues() == NumTo && "Broken CombineTo call!")((N->getNumValues() == NumTo && "Broken CombineTo call!"
) ? static_cast<void> (0) : __assert_fail ("N->getNumValues() == NumTo && \"Broken CombineTo call!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1000, __PRETTY_FUNCTION__))
;
1001 ++NodesCombined;
1002 LLVM_DEBUG(dbgs() << "\nReplacing.1 "; N->dump(&DAG); dbgs() << "\nWith: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.1 "; N->dump
(&DAG); dbgs() << "\nWith: "; To[0].getNode()->dump
(&DAG); dbgs() << " and " << NumTo - 1 <<
" other values\n"; } } while (false)
1003 To[0].getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.1 "; N->dump
(&DAG); dbgs() << "\nWith: "; To[0].getNode()->dump
(&DAG); dbgs() << " and " << NumTo - 1 <<
" other values\n"; } } while (false)
1004 dbgs() << " and " << NumTo - 1 << " other values\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.1 "; N->dump
(&DAG); dbgs() << "\nWith: "; To[0].getNode()->dump
(&DAG); dbgs() << " and " << NumTo - 1 <<
" other values\n"; } } while (false)
;
1005 for (unsigned i = 0, e = NumTo; i != e; ++i)
1006 assert((!To[i].getNode() ||(((!To[i].getNode() || N->getValueType(i) == To[i].getValueType
()) && "Cannot combine value to value of different type!"
) ? static_cast<void> (0) : __assert_fail ("(!To[i].getNode() || N->getValueType(i) == To[i].getValueType()) && \"Cannot combine value to value of different type!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1008, __PRETTY_FUNCTION__))
1007 N->getValueType(i) == To[i].getValueType()) &&(((!To[i].getNode() || N->getValueType(i) == To[i].getValueType
()) && "Cannot combine value to value of different type!"
) ? static_cast<void> (0) : __assert_fail ("(!To[i].getNode() || N->getValueType(i) == To[i].getValueType()) && \"Cannot combine value to value of different type!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1008, __PRETTY_FUNCTION__))
1008 "Cannot combine value to value of different type!")(((!To[i].getNode() || N->getValueType(i) == To[i].getValueType
()) && "Cannot combine value to value of different type!"
) ? static_cast<void> (0) : __assert_fail ("(!To[i].getNode() || N->getValueType(i) == To[i].getValueType()) && \"Cannot combine value to value of different type!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1008, __PRETTY_FUNCTION__))
;
1009
1010 WorklistRemover DeadNodes(*this);
1011 DAG.ReplaceAllUsesWith(N, To);
1012 if (AddTo) {
1013 // Push the new nodes and any users onto the worklist
1014 for (unsigned i = 0, e = NumTo; i != e; ++i) {
1015 if (To[i].getNode()) {
1016 AddToWorklist(To[i].getNode());
1017 AddUsersToWorklist(To[i].getNode());
1018 }
1019 }
1020 }
1021
1022 // Finally, if the node is now dead, remove it from the graph. The node
1023 // may not be dead if the replacement process recursively simplified to
1024 // something else needing this node.
1025 if (N->use_empty())
1026 deleteAndRecombine(N);
1027 return SDValue(N, 0);
1028}
1029
1030void DAGCombiner::
1031CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
1032 // Replace all uses. If any nodes become isomorphic to other nodes and
1033 // are deleted, make sure to remove them from our worklist.
1034 WorklistRemover DeadNodes(*this);
1035 DAG.ReplaceAllUsesOfValueWith(TLO.Old, TLO.New);
1036
1037 // Push the new node and any (possibly new) users onto the worklist.
1038 AddToWorklistWithUsers(TLO.New.getNode());
1039
1040 // Finally, if the node is now dead, remove it from the graph. The node
1041 // may not be dead if the replacement process recursively simplified to
1042 // something else needing this node.
1043 if (TLO.Old.getNode()->use_empty())
1044 deleteAndRecombine(TLO.Old.getNode());
1045}
1046
1047/// Check the specified integer node value to see if it can be simplified or if
1048/// things it uses can be simplified by bit propagation. If so, return true.
1049bool DAGCombiner::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
1050 const APInt &DemandedElts) {
1051 TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations);
1052 KnownBits Known;
1053 if (!TLI.SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO))
1054 return false;
1055
1056 // Revisit the node.
1057 AddToWorklist(Op.getNode());
1058
1059 // Replace the old value with the new one.
1060 ++NodesCombined;
1061 LLVM_DEBUG(dbgs() << "\nReplacing.2 "; TLO.Old.getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.getNode
()->dump(&DAG); dbgs() << "\nWith: "; TLO.New.getNode
()->dump(&DAG); dbgs() << '\n'; } } while (false
)
1062 dbgs() << "\nWith: "; TLO.New.getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.getNode
()->dump(&DAG); dbgs() << "\nWith: "; TLO.New.getNode
()->dump(&DAG); dbgs() << '\n'; } } while (false
)
1063 dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.getNode
()->dump(&DAG); dbgs() << "\nWith: "; TLO.New.getNode
()->dump(&DAG); dbgs() << '\n'; } } while (false
)
;
1064
1065 CommitTargetLoweringOpt(TLO);
1066 return true;
1067}
1068
1069/// Check the specified vector node value to see if it can be simplified or
1070/// if things it uses can be simplified as it only uses some of the elements.
1071/// If so, return true.
1072bool DAGCombiner::SimplifyDemandedVectorElts(SDValue Op,
1073 const APInt &DemandedElts,
1074 bool AssumeSingleUse) {
1075 TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations);
1076 APInt KnownUndef, KnownZero;
1077 if (!TLI.SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero,
1078 TLO, 0, AssumeSingleUse))
1079 return false;
1080
1081 // Revisit the node.
1082 AddToWorklist(Op.getNode());
1083
1084 // Replace the old value with the new one.
1085 ++NodesCombined;
1086 LLVM_DEBUG(dbgs() << "\nReplacing.2 "; TLO.Old.getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.getNode
()->dump(&DAG); dbgs() << "\nWith: "; TLO.New.getNode
()->dump(&DAG); dbgs() << '\n'; } } while (false
)
1087 dbgs() << "\nWith: "; TLO.New.getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.getNode
()->dump(&DAG); dbgs() << "\nWith: "; TLO.New.getNode
()->dump(&DAG); dbgs() << '\n'; } } while (false
)
1088 dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.getNode
()->dump(&DAG); dbgs() << "\nWith: "; TLO.New.getNode
()->dump(&DAG); dbgs() << '\n'; } } while (false
)
;
1089
1090 CommitTargetLoweringOpt(TLO);
1091 return true;
1092}
1093
1094void DAGCombiner::ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad) {
1095 SDLoc DL(Load);
1096 EVT VT = Load->getValueType(0);
1097 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, VT, SDValue(ExtLoad, 0));
1098
1099 LLVM_DEBUG(dbgs() << "\nReplacing.9 "; Load->dump(&DAG); dbgs() << "\nWith: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.9 "; Load->
dump(&DAG); dbgs() << "\nWith: "; Trunc.getNode()->
dump(&DAG); dbgs() << '\n'; } } while (false)
1100 Trunc.getNode()->dump(&DAG); dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.9 "; Load->
dump(&DAG); dbgs() << "\nWith: "; Trunc.getNode()->
dump(&DAG); dbgs() << '\n'; } } while (false)
;
1101 WorklistRemover DeadNodes(*this);
1102 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), Trunc);
1103 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), SDValue(ExtLoad, 1));
1104 deleteAndRecombine(Load);
1105 AddToWorklist(Trunc.getNode());
1106}
1107
1108SDValue DAGCombiner::PromoteOperand(SDValue Op, EVT PVT, bool &Replace) {
1109 Replace = false;
1110 SDLoc DL(Op);
1111 if (ISD::isUNINDEXEDLoad(Op.getNode())) {
1112 LoadSDNode *LD = cast<LoadSDNode>(Op);
1113 EVT MemVT = LD->getMemoryVT();
1114 ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD) ? ISD::EXTLOAD
1115 : LD->getExtensionType();
1116 Replace = true;
1117 return DAG.getExtLoad(ExtType, DL, PVT,
1118 LD->getChain(), LD->getBasePtr(),
1119 MemVT, LD->getMemOperand());
1120 }
1121
1122 unsigned Opc = Op.getOpcode();
1123 switch (Opc) {
1124 default: break;
1125 case ISD::AssertSext:
1126 if (SDValue Op0 = SExtPromoteOperand(Op.getOperand(0), PVT))
1127 return DAG.getNode(ISD::AssertSext, DL, PVT, Op0, Op.getOperand(1));
1128 break;
1129 case ISD::AssertZext:
1130 if (SDValue Op0 = ZExtPromoteOperand(Op.getOperand(0), PVT))
1131 return DAG.getNode(ISD::AssertZext, DL, PVT, Op0, Op.getOperand(1));
1132 break;
1133 case ISD::Constant: {
1134 unsigned ExtOpc =
1135 Op.getValueType().isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
1136 return DAG.getNode(ExtOpc, DL, PVT, Op);
1137 }
1138 }
1139
1140 if (!TLI.isOperationLegal(ISD::ANY_EXTEND, PVT))
1141 return SDValue();
1142 return DAG.getNode(ISD::ANY_EXTEND, DL, PVT, Op);
1143}
1144
1145SDValue DAGCombiner::SExtPromoteOperand(SDValue Op, EVT PVT) {
1146 if (!TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, PVT))
1147 return SDValue();
1148 EVT OldVT = Op.getValueType();
1149 SDLoc DL(Op);
1150 bool Replace = false;
1151 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
1152 if (!NewOp.getNode())
1153 return SDValue();
1154 AddToWorklist(NewOp.getNode());
1155
1156 if (Replace)
1157 ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
1158 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, NewOp.getValueType(), NewOp,
1159 DAG.getValueType(OldVT));
1160}
1161
1162SDValue DAGCombiner::ZExtPromoteOperand(SDValue Op, EVT PVT) {
1163 EVT OldVT = Op.getValueType();
1164 SDLoc DL(Op);
1165 bool Replace = false;
1166 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
1167 if (!NewOp.getNode())
1168 return SDValue();
1169 AddToWorklist(NewOp.getNode());
1170
1171 if (Replace)
1172 ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
1173 return DAG.getZeroExtendInReg(NewOp, DL, OldVT);
1174}
1175
1176/// Promote the specified integer binary operation if the target indicates it is
1177/// beneficial. e.g. On x86, it's usually better to promote i16 operations to
1178/// i32 since i16 instructions are longer.
1179SDValue DAGCombiner::PromoteIntBinOp(SDValue Op) {
1180 if (!LegalOperations)
1181 return SDValue();
1182
1183 EVT VT = Op.getValueType();
1184 if (VT.isVector() || !VT.isInteger())
1185 return SDValue();
1186
1187 // If operation type is 'undesirable', e.g. i16 on x86, consider
1188 // promoting it.
1189 unsigned Opc = Op.getOpcode();
1190 if (TLI.isTypeDesirableForOp(Opc, VT))
1191 return SDValue();
1192
1193 EVT PVT = VT;
1194 // Consult target whether it is a good idea to promote this operation and
1195 // what's the right type to promote it to.
1196 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1197 assert(PVT != VT && "Don't know what type to promote to!")((PVT != VT && "Don't know what type to promote to!")
? static_cast<void> (0) : __assert_fail ("PVT != VT && \"Don't know what type to promote to!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1197, __PRETTY_FUNCTION__))
;
1198
1199 LLVM_DEBUG(dbgs() << "\nPromoting "; Op.getNode()->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; Op.getNode(
)->dump(&DAG); } } while (false)
;
1200
1201 bool Replace0 = false;
1202 SDValue N0 = Op.getOperand(0);
1203 SDValue NN0 = PromoteOperand(N0, PVT, Replace0);
1204
1205 bool Replace1 = false;
1206 SDValue N1 = Op.getOperand(1);
1207 SDValue NN1 = PromoteOperand(N1, PVT, Replace1);
1208 SDLoc DL(Op);
1209
1210 SDValue RV =
1211 DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, NN0, NN1));
1212
1213 // We are always replacing N0/N1's use in N and only need
1214 // additional replacements if there are additional uses.
1215 Replace0 &= !N0->hasOneUse();
1216 Replace1 &= (N0 != N1) && !N1->hasOneUse();
1217
1218 // Combine Op here so it is preserved past replacements.
1219 CombineTo(Op.getNode(), RV);
1220
1221 // If operands have a use ordering, make sure we deal with
1222 // predecessor first.
1223 if (Replace0 && Replace1 && N0.getNode()->isPredecessorOf(N1.getNode())) {
1224 std::swap(N0, N1);
1225 std::swap(NN0, NN1);
1226 }
1227
1228 if (Replace0) {
1229 AddToWorklist(NN0.getNode());
1230 ReplaceLoadWithPromotedLoad(N0.getNode(), NN0.getNode());
1231 }
1232 if (Replace1) {
1233 AddToWorklist(NN1.getNode());
1234 ReplaceLoadWithPromotedLoad(N1.getNode(), NN1.getNode());
1235 }
1236 return Op;
1237 }
1238 return SDValue();
1239}
1240
1241/// Promote the specified integer shift operation if the target indicates it is
1242/// beneficial. e.g. On x86, it's usually better to promote i16 operations to
1243/// i32 since i16 instructions are longer.
1244SDValue DAGCombiner::PromoteIntShiftOp(SDValue Op) {
1245 if (!LegalOperations)
1246 return SDValue();
1247
1248 EVT VT = Op.getValueType();
1249 if (VT.isVector() || !VT.isInteger())
1250 return SDValue();
1251
1252 // If operation type is 'undesirable', e.g. i16 on x86, consider
1253 // promoting it.
1254 unsigned Opc = Op.getOpcode();
1255 if (TLI.isTypeDesirableForOp(Opc, VT))
1256 return SDValue();
1257
1258 EVT PVT = VT;
1259 // Consult target whether it is a good idea to promote this operation and
1260 // what's the right type to promote it to.
1261 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1262 assert(PVT != VT && "Don't know what type to promote to!")((PVT != VT && "Don't know what type to promote to!")
? static_cast<void> (0) : __assert_fail ("PVT != VT && \"Don't know what type to promote to!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1262, __PRETTY_FUNCTION__))
;
1263
1264 LLVM_DEBUG(dbgs() << "\nPromoting "; Op.getNode()->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; Op.getNode(
)->dump(&DAG); } } while (false)
;
1265
1266 bool Replace = false;
1267 SDValue N0 = Op.getOperand(0);
1268 SDValue N1 = Op.getOperand(1);
1269 if (Opc == ISD::SRA)
1270 N0 = SExtPromoteOperand(N0, PVT);
1271 else if (Opc == ISD::SRL)
1272 N0 = ZExtPromoteOperand(N0, PVT);
1273 else
1274 N0 = PromoteOperand(N0, PVT, Replace);
1275
1276 if (!N0.getNode())
1277 return SDValue();
1278
1279 SDLoc DL(Op);
1280 SDValue RV =
1281 DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, N0, N1));
1282
1283 if (Replace)
1284 ReplaceLoadWithPromotedLoad(Op.getOperand(0).getNode(), N0.getNode());
1285
1286 // Deal with Op being deleted.
1287 if (Op && Op.getOpcode() != ISD::DELETED_NODE)
1288 return RV;
1289 }
1290 return SDValue();
1291}
1292
1293SDValue DAGCombiner::PromoteExtend(SDValue Op) {
1294 if (!LegalOperations)
1295 return SDValue();
1296
1297 EVT VT = Op.getValueType();
1298 if (VT.isVector() || !VT.isInteger())
1299 return SDValue();
1300
1301 // If operation type is 'undesirable', e.g. i16 on x86, consider
1302 // promoting it.
1303 unsigned Opc = Op.getOpcode();
1304 if (TLI.isTypeDesirableForOp(Opc, VT))
1305 return SDValue();
1306
1307 EVT PVT = VT;
1308 // Consult target whether it is a good idea to promote this operation and
1309 // what's the right type to promote it to.
1310 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1311 assert(PVT != VT && "Don't know what type to promote to!")((PVT != VT && "Don't know what type to promote to!")
? static_cast<void> (0) : __assert_fail ("PVT != VT && \"Don't know what type to promote to!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1311, __PRETTY_FUNCTION__))
;
1312 // fold (aext (aext x)) -> (aext x)
1313 // fold (aext (zext x)) -> (zext x)
1314 // fold (aext (sext x)) -> (sext x)
1315 LLVM_DEBUG(dbgs() << "\nPromoting "; Op.getNode()->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; Op.getNode(
)->dump(&DAG); } } while (false)
;
1316 return DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, Op.getOperand(0));
1317 }
1318 return SDValue();
1319}
1320
1321bool DAGCombiner::PromoteLoad(SDValue Op) {
1322 if (!LegalOperations)
1323 return false;
1324
1325 if (!ISD::isUNINDEXEDLoad(Op.getNode()))
1326 return false;
1327
1328 EVT VT = Op.getValueType();
1329 if (VT.isVector() || !VT.isInteger())
1330 return false;
1331
1332 // If operation type is 'undesirable', e.g. i16 on x86, consider
1333 // promoting it.
1334 unsigned Opc = Op.getOpcode();
1335 if (TLI.isTypeDesirableForOp(Opc, VT))
1336 return false;
1337
1338 EVT PVT = VT;
1339 // Consult target whether it is a good idea to promote this operation and
1340 // what's the right type to promote it to.
1341 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1342 assert(PVT != VT && "Don't know what type to promote to!")((PVT != VT && "Don't know what type to promote to!")
? static_cast<void> (0) : __assert_fail ("PVT != VT && \"Don't know what type to promote to!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1342, __PRETTY_FUNCTION__))
;
1343
1344 SDLoc DL(Op);
1345 SDNode *N = Op.getNode();
1346 LoadSDNode *LD = cast<LoadSDNode>(N);
1347 EVT MemVT = LD->getMemoryVT();
1348 ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD) ? ISD::EXTLOAD
1349 : LD->getExtensionType();
1350 SDValue NewLD = DAG.getExtLoad(ExtType, DL, PVT,
1351 LD->getChain(), LD->getBasePtr(),
1352 MemVT, LD->getMemOperand());
1353 SDValue Result = DAG.getNode(ISD::TRUNCATE, DL, VT, NewLD);
1354
1355 LLVM_DEBUG(dbgs() << "\nPromoting "; N->dump(&DAG); dbgs() << "\nTo: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; N->dump(
&DAG); dbgs() << "\nTo: "; Result.getNode()->dump
(&DAG); dbgs() << '\n'; } } while (false)
1356 Result.getNode()->dump(&DAG); dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; N->dump(
&DAG); dbgs() << "\nTo: "; Result.getNode()->dump
(&DAG); dbgs() << '\n'; } } while (false)
;
1357 WorklistRemover DeadNodes(*this);
1358 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result);
1359 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), NewLD.getValue(1));
1360 deleteAndRecombine(N);
1361 AddToWorklist(Result.getNode());
1362 return true;
1363 }
1364 return false;
1365}
1366
1367/// Recursively delete a node which has no uses and any operands for
1368/// which it is the only use.
1369///
1370/// Note that this both deletes the nodes and removes them from the worklist.
1371/// It also adds any nodes who have had a user deleted to the worklist as they
1372/// may now have only one use and subject to other combines.
1373bool DAGCombiner::recursivelyDeleteUnusedNodes(SDNode *N) {
1374 if (!N->use_empty())
1375 return false;
1376
1377 SmallSetVector<SDNode *, 16> Nodes;
1378 Nodes.insert(N);
1379 do {
1380 N = Nodes.pop_back_val();
1381 if (!N)
1382 continue;
1383
1384 if (N->use_empty()) {
1385 for (const SDValue &ChildN : N->op_values())
1386 Nodes.insert(ChildN.getNode());
1387
1388 removeFromWorklist(N);
1389 DAG.DeleteNode(N);
1390 } else {
1391 AddToWorklist(N);
1392 }
1393 } while (!Nodes.empty());
1394 return true;
1395}
1396
1397//===----------------------------------------------------------------------===//
1398// Main DAG Combiner implementation
1399//===----------------------------------------------------------------------===//
1400
1401void DAGCombiner::Run(CombineLevel AtLevel) {
1402 // set the instance variables, so that the various visit routines may use it.
1403 Level = AtLevel;
1404 LegalDAG = Level >= AfterLegalizeDAG;
1405 LegalOperations = Level >= AfterLegalizeVectorOps;
1406 LegalTypes = Level >= AfterLegalizeTypes;
1407
1408 WorklistInserter AddNodes(*this);
1409
1410 // Add all the dag nodes to the worklist.
1411 for (SDNode &Node : DAG.allnodes())
1412 AddToWorklist(&Node);
1413
1414 // Create a dummy node (which is not added to allnodes), that adds a reference
1415 // to the root node, preventing it from being deleted, and tracking any
1416 // changes of the root.
1417 HandleSDNode Dummy(DAG.getRoot());
1418
1419 // While we have a valid worklist entry node, try to combine it.
1420 while (SDNode *N = getNextWorklistEntry()) {
1421 // If N has no uses, it is dead. Make sure to revisit all N's operands once
1422 // N is deleted from the DAG, since they too may now be dead or may have a
1423 // reduced number of uses, allowing other xforms.
1424 if (recursivelyDeleteUnusedNodes(N))
1425 continue;
1426
1427 WorklistRemover DeadNodes(*this);
1428
1429 // If this combine is running after legalizing the DAG, re-legalize any
1430 // nodes pulled off the worklist.
1431 if (LegalDAG) {
1432 SmallSetVector<SDNode *, 16> UpdatedNodes;
1433 bool NIsValid = DAG.LegalizeOp(N, UpdatedNodes);
1434
1435 for (SDNode *LN : UpdatedNodes)
1436 AddToWorklistWithUsers(LN);
1437
1438 if (!NIsValid)
1439 continue;
1440 }
1441
1442 LLVM_DEBUG(dbgs() << "\nCombining: "; N->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nCombining: "; N->dump
(&DAG); } } while (false)
;
1443
1444 // Add any operands of the new node which have not yet been combined to the
1445 // worklist as well. Because the worklist uniques things already, this
1446 // won't repeatedly process the same operand.
1447 CombinedNodes.insert(N);
1448 for (const SDValue &ChildN : N->op_values())
1449 if (!CombinedNodes.count(ChildN.getNode()))
1450 AddToWorklist(ChildN.getNode());
1451
1452 SDValue RV = combine(N);
1453
1454 if (!RV.getNode())
1455 continue;
1456
1457 ++NodesCombined;
1458
1459 // If we get back the same node we passed in, rather than a new node or
1460 // zero, we know that the node must have defined multiple values and
1461 // CombineTo was used. Since CombineTo takes care of the worklist
1462 // mechanics for us, we have no work to do in this case.
1463 if (RV.getNode() == N)
1464 continue;
1465
1466 assert(N->getOpcode() != ISD::DELETED_NODE &&((N->getOpcode() != ISD::DELETED_NODE && RV.getOpcode
() != ISD::DELETED_NODE && "Node was deleted but visit returned new node!"
) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && RV.getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned new node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1468, __PRETTY_FUNCTION__))
1467 RV.getOpcode() != ISD::DELETED_NODE &&((N->getOpcode() != ISD::DELETED_NODE && RV.getOpcode
() != ISD::DELETED_NODE && "Node was deleted but visit returned new node!"
) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && RV.getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned new node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1468, __PRETTY_FUNCTION__))
1468 "Node was deleted but visit returned new node!")((N->getOpcode() != ISD::DELETED_NODE && RV.getOpcode
() != ISD::DELETED_NODE && "Node was deleted but visit returned new node!"
) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && RV.getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned new node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1468, __PRETTY_FUNCTION__))
;
1469
1470 LLVM_DEBUG(dbgs() << " ... into: "; RV.getNode()->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << " ... into: "; RV.getNode()
->dump(&DAG); } } while (false)
;
1471
1472 if (N->getNumValues() == RV.getNode()->getNumValues())
1473 DAG.ReplaceAllUsesWith(N, RV.getNode());
1474 else {
1475 assert(N->getValueType(0) == RV.getValueType() &&((N->getValueType(0) == RV.getValueType() && N->
getNumValues() == 1 && "Type mismatch") ? static_cast
<void> (0) : __assert_fail ("N->getValueType(0) == RV.getValueType() && N->getNumValues() == 1 && \"Type mismatch\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1476, __PRETTY_FUNCTION__))
1476 N->getNumValues() == 1 && "Type mismatch")((N->getValueType(0) == RV.getValueType() && N->
getNumValues() == 1 && "Type mismatch") ? static_cast
<void> (0) : __assert_fail ("N->getValueType(0) == RV.getValueType() && N->getNumValues() == 1 && \"Type mismatch\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1476, __PRETTY_FUNCTION__))
;
1477 DAG.ReplaceAllUsesWith(N, &RV);
1478 }
1479
1480 // Push the new node and any users onto the worklist
1481 AddToWorklist(RV.getNode());
1482 AddUsersToWorklist(RV.getNode());
1483
1484 // Finally, if the node is now dead, remove it from the graph. The node
1485 // may not be dead if the replacement process recursively simplified to
1486 // something else needing this node. This will also take care of adding any
1487 // operands which have lost a user to the worklist.
1488 recursivelyDeleteUnusedNodes(N);
1489 }
1490
1491 // If the root changed (e.g. it was a dead load, update the root).
1492 DAG.setRoot(Dummy.getValue());
1493 DAG.RemoveDeadNodes();
1494}
1495
1496SDValue DAGCombiner::visit(SDNode *N) {
1497 switch (N->getOpcode()) {
1498 default: break;
1499 case ISD::TokenFactor: return visitTokenFactor(N);
1500 case ISD::MERGE_VALUES: return visitMERGE_VALUES(N);
1501 case ISD::ADD: return visitADD(N);
1502 case ISD::SUB: return visitSUB(N);
1503 case ISD::SADDSAT:
1504 case ISD::UADDSAT: return visitADDSAT(N);
1505 case ISD::SSUBSAT:
1506 case ISD::USUBSAT: return visitSUBSAT(N);
1507 case ISD::ADDC: return visitADDC(N);
1508 case ISD::SADDO:
1509 case ISD::UADDO: return visitADDO(N);
1510 case ISD::SUBC: return visitSUBC(N);
1511 case ISD::SSUBO:
1512 case ISD::USUBO: return visitSUBO(N);
1513 case ISD::ADDE: return visitADDE(N);
1514 case ISD::ADDCARRY: return visitADDCARRY(N);
1515 case ISD::SUBE: return visitSUBE(N);
1516 case ISD::SUBCARRY: return visitSUBCARRY(N);
1517 case ISD::SMULFIX:
1518 case ISD::SMULFIXSAT:
1519 case ISD::UMULFIX:
1520 case ISD::UMULFIXSAT: return visitMULFIX(N);
1521 case ISD::MUL: return visitMUL(N);
1522 case ISD::SDIV: return visitSDIV(N);
1523 case ISD::UDIV: return visitUDIV(N);
1524 case ISD::SREM:
1525 case ISD::UREM: return visitREM(N);
1526 case ISD::MULHU: return visitMULHU(N);
1527 case ISD::MULHS: return visitMULHS(N);
1528 case ISD::SMUL_LOHI: return visitSMUL_LOHI(N);
1529 case ISD::UMUL_LOHI: return visitUMUL_LOHI(N);
1530 case ISD::SMULO:
1531 case ISD::UMULO: return visitMULO(N);
1532 case ISD::SMIN:
1533 case ISD::SMAX:
1534 case ISD::UMIN:
1535 case ISD::UMAX: return visitIMINMAX(N);
1536 case ISD::AND: return visitAND(N);
1537 case ISD::OR: return visitOR(N);
1538 case ISD::XOR: return visitXOR(N);
1539 case ISD::SHL: return visitSHL(N);
1540 case ISD::SRA: return visitSRA(N);
1541 case ISD::SRL: return visitSRL(N);
1542 case ISD::ROTR:
1543 case ISD::ROTL: return visitRotate(N);
1544 case ISD::FSHL:
1545 case ISD::FSHR: return visitFunnelShift(N);
1546 case ISD::ABS: return visitABS(N);
1547 case ISD::BSWAP: return visitBSWAP(N);
1548 case ISD::BITREVERSE: return visitBITREVERSE(N);
1549 case ISD::CTLZ: return visitCTLZ(N);
1550 case ISD::CTLZ_ZERO_UNDEF: return visitCTLZ_ZERO_UNDEF(N);
1551 case ISD::CTTZ: return visitCTTZ(N);
1552 case ISD::CTTZ_ZERO_UNDEF: return visitCTTZ_ZERO_UNDEF(N);
1553 case ISD::CTPOP: return visitCTPOP(N);
1554 case ISD::SELECT: return visitSELECT(N);
1555 case ISD::VSELECT: return visitVSELECT(N);
1556 case ISD::SELECT_CC: return visitSELECT_CC(N);
1557 case ISD::SETCC: return visitSETCC(N);
1558 case ISD::SETCCCARRY: return visitSETCCCARRY(N);
1559 case ISD::SIGN_EXTEND: return visitSIGN_EXTEND(N);
1560 case ISD::ZERO_EXTEND: return visitZERO_EXTEND(N);
1561 case ISD::ANY_EXTEND: return visitANY_EXTEND(N);
1562 case ISD::AssertSext:
1563 case ISD::AssertZext: return visitAssertExt(N);
1564 case ISD::SIGN_EXTEND_INREG: return visitSIGN_EXTEND_INREG(N);
1565 case ISD::SIGN_EXTEND_VECTOR_INREG: return visitSIGN_EXTEND_VECTOR_INREG(N);
1566 case ISD::ZERO_EXTEND_VECTOR_INREG: return visitZERO_EXTEND_VECTOR_INREG(N);
1567 case ISD::TRUNCATE: return visitTRUNCATE(N);
1568 case ISD::BITCAST: return visitBITCAST(N);
1569 case ISD::BUILD_PAIR: return visitBUILD_PAIR(N);
1570 case ISD::FADD: return visitFADD(N);
1571 case ISD::FSUB: return visitFSUB(N);
1572 case ISD::FMUL: return visitFMUL(N);
1573 case ISD::FMA: return visitFMA(N);
1574 case ISD::FDIV: return visitFDIV(N);
1575 case ISD::FREM: return visitFREM(N);
1576 case ISD::FSQRT: return visitFSQRT(N);
1577 case ISD::FCOPYSIGN: return visitFCOPYSIGN(N);
1578 case ISD::FPOW: return visitFPOW(N);
1579 case ISD::SINT_TO_FP: return visitSINT_TO_FP(N);
1580 case ISD::UINT_TO_FP: return visitUINT_TO_FP(N);
1581 case ISD::FP_TO_SINT: return visitFP_TO_SINT(N);
1582 case ISD::FP_TO_UINT: return visitFP_TO_UINT(N);
1583 case ISD::FP_ROUND: return visitFP_ROUND(N);
1584 case ISD::FP_EXTEND: return visitFP_EXTEND(N);
1585 case ISD::FNEG: return visitFNEG(N);
1586 case ISD::FABS: return visitFABS(N);
1587 case ISD::FFLOOR: return visitFFLOOR(N);
1588 case ISD::FMINNUM: return visitFMINNUM(N);
1589 case ISD::FMAXNUM: return visitFMAXNUM(N);
1590 case ISD::FMINIMUM: return visitFMINIMUM(N);
1591 case ISD::FMAXIMUM: return visitFMAXIMUM(N);
1592 case ISD::FCEIL: return visitFCEIL(N);
1593 case ISD::FTRUNC: return visitFTRUNC(N);
1594 case ISD::BRCOND: return visitBRCOND(N);
1595 case ISD::BR_CC: return visitBR_CC(N);
1596 case ISD::LOAD: return visitLOAD(N);
1597 case ISD::STORE: return visitSTORE(N);
1598 case ISD::INSERT_VECTOR_ELT: return visitINSERT_VECTOR_ELT(N);
1599 case ISD::EXTRACT_VECTOR_ELT: return visitEXTRACT_VECTOR_ELT(N);
1600 case ISD::BUILD_VECTOR: return visitBUILD_VECTOR(N);
1601 case ISD::CONCAT_VECTORS: return visitCONCAT_VECTORS(N);
1602 case ISD::EXTRACT_SUBVECTOR: return visitEXTRACT_SUBVECTOR(N);
1603 case ISD::VECTOR_SHUFFLE: return visitVECTOR_SHUFFLE(N);
1604 case ISD::SCALAR_TO_VECTOR: return visitSCALAR_TO_VECTOR(N);
1605 case ISD::INSERT_SUBVECTOR: return visitINSERT_SUBVECTOR(N);
1606 case ISD::MGATHER: return visitMGATHER(N);
1607 case ISD::MLOAD: return visitMLOAD(N);
1608 case ISD::MSCATTER: return visitMSCATTER(N);
1609 case ISD::MSTORE: return visitMSTORE(N);
1610 case ISD::LIFETIME_END: return visitLIFETIME_END(N);
1611 case ISD::FP_TO_FP16: return visitFP_TO_FP16(N);
1612 case ISD::FP16_TO_FP: return visitFP16_TO_FP(N);
1613 case ISD::VECREDUCE_FADD:
1614 case ISD::VECREDUCE_FMUL:
1615 case ISD::VECREDUCE_ADD:
1616 case ISD::VECREDUCE_MUL:
1617 case ISD::VECREDUCE_AND:
1618 case ISD::VECREDUCE_OR:
1619 case ISD::VECREDUCE_XOR:
1620 case ISD::VECREDUCE_SMAX:
1621 case ISD::VECREDUCE_SMIN:
1622 case ISD::VECREDUCE_UMAX:
1623 case ISD::VECREDUCE_UMIN:
1624 case ISD::VECREDUCE_FMAX:
1625 case ISD::VECREDUCE_FMIN: return visitVECREDUCE(N);
1626 }
1627 return SDValue();
1628}
1629
1630SDValue DAGCombiner::combine(SDNode *N) {
1631 SDValue RV = visit(N);
1632
1633 // If nothing happened, try a target-specific DAG combine.
1634 if (!RV.getNode()) {
1635 assert(N->getOpcode() != ISD::DELETED_NODE &&((N->getOpcode() != ISD::DELETED_NODE && "Node was deleted but visit returned NULL!"
) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned NULL!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1636, __PRETTY_FUNCTION__))
1636 "Node was deleted but visit returned NULL!")((N->getOpcode() != ISD::DELETED_NODE && "Node was deleted but visit returned NULL!"
) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned NULL!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1636, __PRETTY_FUNCTION__))
;
1637
1638 if (N->getOpcode() >= ISD::BUILTIN_OP_END ||
1639 TLI.hasTargetDAGCombine((ISD::NodeType)N->getOpcode())) {
1640
1641 // Expose the DAG combiner to the target combiner impls.
1642 TargetLowering::DAGCombinerInfo
1643 DagCombineInfo(DAG, Level, false, this);
1644
1645 RV = TLI.PerformDAGCombine(N, DagCombineInfo);
1646 }
1647 }
1648
1649 // If nothing happened still, try promoting the operation.
1650 if (!RV.getNode()) {
1651 switch (N->getOpcode()) {
1652 default: break;
1653 case ISD::ADD:
1654 case ISD::SUB:
1655 case ISD::MUL:
1656 case ISD::AND:
1657 case ISD::OR:
1658 case ISD::XOR:
1659 RV = PromoteIntBinOp(SDValue(N, 0));
1660 break;
1661 case ISD::SHL:
1662 case ISD::SRA:
1663 case ISD::SRL:
1664 RV = PromoteIntShiftOp(SDValue(N, 0));
1665 break;
1666 case ISD::SIGN_EXTEND:
1667 case ISD::ZERO_EXTEND:
1668 case ISD::ANY_EXTEND:
1669 RV = PromoteExtend(SDValue(N, 0));
1670 break;
1671 case ISD::LOAD:
1672 if (PromoteLoad(SDValue(N, 0)))
1673 RV = SDValue(N, 0);
1674 break;
1675 }
1676 }
1677
1678 // If N is a commutative binary node, try to eliminate it if the commuted
1679 // version is already present in the DAG.
1680 if (!RV.getNode() && TLI.isCommutativeBinOp(N->getOpcode()) &&
1681 N->getNumValues() == 1) {
1682 SDValue N0 = N->getOperand(0);
1683 SDValue N1 = N->getOperand(1);
1684
1685 // Constant operands are canonicalized to RHS.
1686 if (N0 != N1 && (isa<ConstantSDNode>(N0) || !isa<ConstantSDNode>(N1))) {
1687 SDValue Ops[] = {N1, N0};
1688 SDNode *CSENode = DAG.getNodeIfExists(N->getOpcode(), N->getVTList(), Ops,
1689 N->getFlags());
1690 if (CSENode)
1691 return SDValue(CSENode, 0);
1692 }
1693 }
1694
1695 return RV;
1696}
1697
1698/// Given a node, return its input chain if it has one, otherwise return a null
1699/// sd operand.
1700static SDValue getInputChainForNode(SDNode *N) {
1701 if (unsigned NumOps = N->getNumOperands()) {
1702 if (N->getOperand(0).getValueType() == MVT::Other)
1703 return N->getOperand(0);
1704 if (N->getOperand(NumOps-1).getValueType() == MVT::Other)
1705 return N->getOperand(NumOps-1);
1706 for (unsigned i = 1; i < NumOps-1; ++i)
1707 if (N->getOperand(i).getValueType() == MVT::Other)
1708 return N->getOperand(i);
1709 }
1710 return SDValue();
1711}
1712
1713SDValue DAGCombiner::visitTokenFactor(SDNode *N) {
1714 // If N has two operands, where one has an input chain equal to the other,
1715 // the 'other' chain is redundant.
1716 if (N->getNumOperands() == 2) {
1717 if (getInputChainForNode(N->getOperand(0).getNode()) == N->getOperand(1))
1718 return N->getOperand(0);
1719 if (getInputChainForNode(N->getOperand(1).getNode()) == N->getOperand(0))
1720 return N->getOperand(1);
1721 }
1722
1723 // Don't simplify token factors if optnone.
1724 if (OptLevel == CodeGenOpt::None)
1725 return SDValue();
1726
1727 // If the sole user is a token factor, we should make sure we have a
1728 // chance to merge them together. This prevents TF chains from inhibiting
1729 // optimizations.
1730 if (N->hasOneUse() && N->use_begin()->getOpcode() == ISD::TokenFactor)
1731 AddToWorklist(*(N->use_begin()));
1732
1733 SmallVector<SDNode *, 8> TFs; // List of token factors to visit.
1734 SmallVector<SDValue, 8> Ops; // Ops for replacing token factor.
1735 SmallPtrSet<SDNode*, 16> SeenOps;
1736 bool Changed = false; // If we should replace this token factor.
1737
1738 // Start out with this token factor.
1739 TFs.push_back(N);
1740
1741 // Iterate through token factors. The TFs grows when new token factors are
1742 // encountered.
1743 for (unsigned i = 0; i < TFs.size(); ++i) {
1744 // Limit number of nodes to inline, to avoid quadratic compile times.
1745 // We have to add the outstanding Token Factors to Ops, otherwise we might
1746 // drop Ops from the resulting Token Factors.
1747 if (Ops.size() > TokenFactorInlineLimit) {
1748 for (unsigned j = i; j < TFs.size(); j++)
1749 Ops.emplace_back(TFs[j], 0);
1750 // Drop unprocessed Token Factors from TFs, so we do not add them to the
1751 // combiner worklist later.
1752 TFs.resize(i);
1753 break;
1754 }
1755
1756 SDNode *TF = TFs[i];
1757 // Check each of the operands.
1758 for (const SDValue &Op : TF->op_values()) {
1759 switch (Op.getOpcode()) {
1760 case ISD::EntryToken:
1761 // Entry tokens don't need to be added to the list. They are
1762 // redundant.
1763 Changed = true;
1764 break;
1765
1766 case ISD::TokenFactor:
1767 if (Op.hasOneUse() && !is_contained(TFs, Op.getNode())) {
1768 // Queue up for processing.
1769 TFs.push_back(Op.getNode());
1770 Changed = true;
1771 break;
1772 }
1773 LLVM_FALLTHROUGH[[gnu::fallthrough]];
1774
1775 default:
1776 // Only add if it isn't already in the list.
1777 if (SeenOps.insert(Op.getNode()).second)
1778 Ops.push_back(Op);
1779 else
1780 Changed = true;
1781 break;
1782 }
1783 }
1784 }
1785
1786 // Re-visit inlined Token Factors, to clean them up in case they have been
1787 // removed. Skip the first Token Factor, as this is the current node.
1788 for (unsigned i = 1, e = TFs.size(); i < e; i++)
1789 AddToWorklist(TFs[i]);
1790
1791 // Remove Nodes that are chained to another node in the list. Do so
1792 // by walking up chains breath-first stopping when we've seen
1793 // another operand. In general we must climb to the EntryNode, but we can exit
1794 // early if we find all remaining work is associated with just one operand as
1795 // no further pruning is possible.
1796
1797 // List of nodes to search through and original Ops from which they originate.
1798 SmallVector<std::pair<SDNode *, unsigned>, 8> Worklist;
1799 SmallVector<unsigned, 8> OpWorkCount; // Count of work for each Op.
1800 SmallPtrSet<SDNode *, 16> SeenChains;
1801 bool DidPruneOps = false;
1802
1803 unsigned NumLeftToConsider = 0;
1804 for (const SDValue &Op : Ops) {
1805 Worklist.push_back(std::make_pair(Op.getNode(), NumLeftToConsider++));
1806 OpWorkCount.push_back(1);
1807 }
1808
1809 auto AddToWorklist = [&](unsigned CurIdx, SDNode *Op, unsigned OpNumber) {
1810 // If this is an Op, we can remove the op from the list. Remark any
1811 // search associated with it as from the current OpNumber.
1812 if (SeenOps.count(Op) != 0) {
1813 Changed = true;
1814 DidPruneOps = true;
1815 unsigned OrigOpNumber = 0;
1816 while (OrigOpNumber < Ops.size() && Ops[OrigOpNumber].getNode() != Op)
1817 OrigOpNumber++;
1818 assert((OrigOpNumber != Ops.size()) &&(((OrigOpNumber != Ops.size()) && "expected to find TokenFactor Operand"
) ? static_cast<void> (0) : __assert_fail ("(OrigOpNumber != Ops.size()) && \"expected to find TokenFactor Operand\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1819, __PRETTY_FUNCTION__))
1819 "expected to find TokenFactor Operand")(((OrigOpNumber != Ops.size()) && "expected to find TokenFactor Operand"
) ? static_cast<void> (0) : __assert_fail ("(OrigOpNumber != Ops.size()) && \"expected to find TokenFactor Operand\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1819, __PRETTY_FUNCTION__))
;
1820 // Re-mark worklist from OrigOpNumber to OpNumber
1821 for (unsigned i = CurIdx + 1; i < Worklist.size(); ++i) {
1822 if (Worklist[i].second == OrigOpNumber) {
1823 Worklist[i].second = OpNumber;
1824 }
1825 }
1826 OpWorkCount[OpNumber] += OpWorkCount[OrigOpNumber];
1827 OpWorkCount[OrigOpNumber] = 0;
1828 NumLeftToConsider--;
1829 }
1830 // Add if it's a new chain
1831 if (SeenChains.insert(Op).second) {
1832 OpWorkCount[OpNumber]++;
1833 Worklist.push_back(std::make_pair(Op, OpNumber));
1834 }
1835 };
1836
1837 for (unsigned i = 0; i < Worklist.size() && i < 1024; ++i) {
1838 // We need at least be consider at least 2 Ops to prune.
1839 if (NumLeftToConsider <= 1)
1840 break;
1841 auto CurNode = Worklist[i].first;
1842 auto CurOpNumber = Worklist[i].second;
1843 assert((OpWorkCount[CurOpNumber] > 0) &&(((OpWorkCount[CurOpNumber] > 0) && "Node should not appear in worklist"
) ? static_cast<void> (0) : __assert_fail ("(OpWorkCount[CurOpNumber] > 0) && \"Node should not appear in worklist\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1844, __PRETTY_FUNCTION__))
1844 "Node should not appear in worklist")(((OpWorkCount[CurOpNumber] > 0) && "Node should not appear in worklist"
) ? static_cast<void> (0) : __assert_fail ("(OpWorkCount[CurOpNumber] > 0) && \"Node should not appear in worklist\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1844, __PRETTY_FUNCTION__))
;
1845 switch (CurNode->getOpcode()) {
1846 case ISD::EntryToken:
1847 // Hitting EntryToken is the only way for the search to terminate without
1848 // hitting
1849 // another operand's search. Prevent us from marking this operand
1850 // considered.
1851 NumLeftToConsider++;
1852 break;
1853 case ISD::TokenFactor:
1854 for (const SDValue &Op : CurNode->op_values())
1855 AddToWorklist(i, Op.getNode(), CurOpNumber);
1856 break;
1857 case ISD::LIFETIME_START:
1858 case ISD::LIFETIME_END:
1859 case ISD::CopyFromReg:
1860 case ISD::CopyToReg:
1861 AddToWorklist(i, CurNode->getOperand(0).getNode(), CurOpNumber);
1862 break;
1863 default:
1864 if (auto *MemNode = dyn_cast<MemSDNode>(CurNode))
1865 AddToWorklist(i, MemNode->getChain().getNode(), CurOpNumber);
1866 break;
1867 }
1868 OpWorkCount[CurOpNumber]--;
1869 if (OpWorkCount[CurOpNumber] == 0)
1870 NumLeftToConsider--;
1871 }
1872
1873 // If we've changed things around then replace token factor.
1874 if (Changed) {
1875 SDValue Result;
1876 if (Ops.empty()) {
1877 // The entry token is the only possible outcome.
1878 Result = DAG.getEntryNode();
1879 } else {
1880 if (DidPruneOps) {
1881 SmallVector<SDValue, 8> PrunedOps;
1882 //
1883 for (const SDValue &Op : Ops) {
1884 if (SeenChains.count(Op.getNode()) == 0)
1885 PrunedOps.push_back(Op);
1886 }
1887 Result = DAG.getTokenFactor(SDLoc(N), PrunedOps);
1888 } else {
1889 Result = DAG.getTokenFactor(SDLoc(N), Ops);
1890 }
1891 }
1892 return Result;
1893 }
1894 return SDValue();
1895}
1896
1897/// MERGE_VALUES can always be eliminated.
1898SDValue DAGCombiner::visitMERGE_VALUES(SDNode *N) {
1899 WorklistRemover DeadNodes(*this);
1900 // Replacing results may cause a different MERGE_VALUES to suddenly
1901 // be CSE'd with N, and carry its uses with it. Iterate until no
1902 // uses remain, to ensure that the node can be safely deleted.
1903 // First add the users of this node to the work list so that they
1904 // can be tried again once they have new operands.
1905 AddUsersToWorklist(N);
1906 do {
1907 // Do as a single replacement to avoid rewalking use lists.
1908 SmallVector<SDValue, 8> Ops;
1909 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
1910 Ops.push_back(N->getOperand(i));
1911 DAG.ReplaceAllUsesWith(N, Ops.data());
1912 } while (!N->use_empty());
1913 deleteAndRecombine(N);
1914 return SDValue(N, 0); // Return N so it doesn't get rechecked!
1915}
1916
1917/// If \p N is a ConstantSDNode with isOpaque() == false return it casted to a
1918/// ConstantSDNode pointer else nullptr.
1919static ConstantSDNode *getAsNonOpaqueConstant(SDValue N) {
1920 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N);
1921 return Const != nullptr && !Const->isOpaque() ? Const : nullptr;
1922}
1923
1924SDValue DAGCombiner::foldBinOpIntoSelect(SDNode *BO) {
1925 assert(TLI.isBinOp(BO->getOpcode()) && BO->getNumValues() == 1 &&((TLI.isBinOp(BO->getOpcode()) && BO->getNumValues
() == 1 && "Unexpected binary operator") ? static_cast
<void> (0) : __assert_fail ("TLI.isBinOp(BO->getOpcode()) && BO->getNumValues() == 1 && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1926, __PRETTY_FUNCTION__))
1926 "Unexpected binary operator")((TLI.isBinOp(BO->getOpcode()) && BO->getNumValues
() == 1 && "Unexpected binary operator") ? static_cast
<void> (0) : __assert_fail ("TLI.isBinOp(BO->getOpcode()) && BO->getNumValues() == 1 && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1926, __PRETTY_FUNCTION__))
;
1927
1928 // Don't do this unless the old select is going away. We want to eliminate the
1929 // binary operator, not replace a binop with a select.
1930 // TODO: Handle ISD::SELECT_CC.
1931 unsigned SelOpNo = 0;
1932 SDValue Sel = BO->getOperand(0);
1933 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse()) {
1934 SelOpNo = 1;
1935 Sel = BO->getOperand(1);
1936 }
1937
1938 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse())
1939 return SDValue();
1940
1941 SDValue CT = Sel.getOperand(1);
1942 if (!isConstantOrConstantVector(CT, true) &&
1943 !isConstantFPBuildVectorOrConstantFP(CT))
1944 return SDValue();
1945
1946 SDValue CF = Sel.getOperand(2);
1947 if (!isConstantOrConstantVector(CF, true) &&
1948 !isConstantFPBuildVectorOrConstantFP(CF))
1949 return SDValue();
1950
1951 // Bail out if any constants are opaque because we can't constant fold those.
1952 // The exception is "and" and "or" with either 0 or -1 in which case we can
1953 // propagate non constant operands into select. I.e.:
1954 // and (select Cond, 0, -1), X --> select Cond, 0, X
1955 // or X, (select Cond, -1, 0) --> select Cond, -1, X
1956 auto BinOpcode = BO->getOpcode();
1957 bool CanFoldNonConst =
1958 (BinOpcode == ISD::AND || BinOpcode == ISD::OR) &&
1959 (isNullOrNullSplat(CT) || isAllOnesOrAllOnesSplat(CT)) &&
1960 (isNullOrNullSplat(CF) || isAllOnesOrAllOnesSplat(CF));
1961
1962 SDValue CBO = BO->getOperand(SelOpNo ^ 1);
1963 if (!CanFoldNonConst &&
1964 !isConstantOrConstantVector(CBO, true) &&
1965 !isConstantFPBuildVectorOrConstantFP(CBO))
1966 return SDValue();
1967
1968 EVT VT = Sel.getValueType();
1969
1970 // In case of shift value and shift amount may have different VT. For instance
1971 // on x86 shift amount is i8 regardles of LHS type. Bail out if we have
1972 // swapped operands and value types do not match. NB: x86 is fine if operands
1973 // are not swapped with shift amount VT being not bigger than shifted value.
1974 // TODO: that is possible to check for a shift operation, correct VTs and
1975 // still perform optimization on x86 if needed.
1976 if (SelOpNo && VT != CBO.getValueType())
1977 return SDValue();
1978
1979 // We have a select-of-constants followed by a binary operator with a
1980 // constant. Eliminate the binop by pulling the constant math into the select.
1981 // Example: add (select Cond, CT, CF), CBO --> select Cond, CT + CBO, CF + CBO
1982 SDLoc DL(Sel);
1983 SDValue NewCT = SelOpNo ? DAG.getNode(BinOpcode, DL, VT, CBO, CT)
1984 : DAG.getNode(BinOpcode, DL, VT, CT, CBO);
1985 if (!CanFoldNonConst && !NewCT.isUndef() &&
1986 !isConstantOrConstantVector(NewCT, true) &&
1987 !isConstantFPBuildVectorOrConstantFP(NewCT))
1988 return SDValue();
1989
1990 SDValue NewCF = SelOpNo ? DAG.getNode(BinOpcode, DL, VT, CBO, CF)
1991 : DAG.getNode(BinOpcode, DL, VT, CF, CBO);
1992 if (!CanFoldNonConst && !NewCF.isUndef() &&
1993 !isConstantOrConstantVector(NewCF, true) &&
1994 !isConstantFPBuildVectorOrConstantFP(NewCF))
1995 return SDValue();
1996
1997 SDValue SelectOp = DAG.getSelect(DL, VT, Sel.getOperand(0), NewCT, NewCF);
1998 SelectOp->setFlags(BO->getFlags());
1999 return SelectOp;
2000}
2001
2002static SDValue foldAddSubBoolOfMaskedVal(SDNode *N, SelectionDAG &DAG) {
2003 assert((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) &&(((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::
SUB) && "Expecting add or sub") ? static_cast<void
> (0) : __assert_fail ("(N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) && \"Expecting add or sub\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2004, __PRETTY_FUNCTION__))
2004 "Expecting add or sub")(((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::
SUB) && "Expecting add or sub") ? static_cast<void
> (0) : __assert_fail ("(N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) && \"Expecting add or sub\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2004, __PRETTY_FUNCTION__))
;
2005
2006 // Match a constant operand and a zext operand for the math instruction:
2007 // add Z, C
2008 // sub C, Z
2009 bool IsAdd = N->getOpcode() == ISD::ADD;
2010 SDValue C = IsAdd ? N->getOperand(1) : N->getOperand(0);
2011 SDValue Z = IsAdd ? N->getOperand(0) : N->getOperand(1);
2012 auto *CN = dyn_cast<ConstantSDNode>(C);
2013 if (!CN || Z.getOpcode() != ISD::ZERO_EXTEND)
2014 return SDValue();
2015
2016 // Match the zext operand as a setcc of a boolean.
2017 if (Z.getOperand(0).getOpcode() != ISD::SETCC ||
2018 Z.getOperand(0).getValueType() != MVT::i1)
2019 return SDValue();
2020
2021 // Match the compare as: setcc (X & 1), 0, eq.
2022 SDValue SetCC = Z.getOperand(0);
2023 ISD::CondCode CC = cast<CondCodeSDNode>(SetCC->getOperand(2))->get();
2024 if (CC != ISD::SETEQ || !isNullConstant(SetCC.getOperand(1)) ||
2025 SetCC.getOperand(0).getOpcode() != ISD::AND ||
2026 !isOneConstant(SetCC.getOperand(0).getOperand(1)))
2027 return SDValue();
2028
2029 // We are adding/subtracting a constant and an inverted low bit. Turn that
2030 // into a subtract/add of the low bit with incremented/decremented constant:
2031 // add (zext i1 (seteq (X & 1), 0)), C --> sub C+1, (zext (X & 1))
2032 // sub C, (zext i1 (seteq (X & 1), 0)) --> add C-1, (zext (X & 1))
2033 EVT VT = C.getValueType();
2034 SDLoc DL(N);
2035 SDValue LowBit = DAG.getZExtOrTrunc(SetCC.getOperand(0), DL, VT);
2036 SDValue C1 = IsAdd ? DAG.getConstant(CN->getAPIntValue() + 1, DL, VT) :
2037 DAG.getConstant(CN->getAPIntValue() - 1, DL, VT);
2038 return DAG.getNode(IsAdd ? ISD::SUB : ISD::ADD, DL, VT, C1, LowBit);
2039}
2040
2041/// Try to fold a 'not' shifted sign-bit with add/sub with constant operand into
2042/// a shift and add with a different constant.
2043static SDValue foldAddSubOfSignBit(SDNode *N, SelectionDAG &DAG) {
2044 assert((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) &&(((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::
SUB) && "Expecting add or sub") ? static_cast<void
> (0) : __assert_fail ("(N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) && \"Expecting add or sub\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2045, __PRETTY_FUNCTION__))
2045 "Expecting add or sub")(((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::
SUB) && "Expecting add or sub") ? static_cast<void
> (0) : __assert_fail ("(N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) && \"Expecting add or sub\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2045, __PRETTY_FUNCTION__))
;
2046
2047 // We need a constant operand for the add/sub, and the other operand is a
2048 // logical shift right: add (srl), C or sub C, (srl).
2049 // TODO - support non-uniform vector amounts.
2050 bool IsAdd = N->getOpcode() == ISD::ADD;
2051 SDValue ConstantOp = IsAdd ? N->getOperand(1) : N->getOperand(0);
2052 SDValue ShiftOp = IsAdd ? N->getOperand(0) : N->getOperand(1);
2053 ConstantSDNode *C = isConstOrConstSplat(ConstantOp);
2054 if (!C || ShiftOp.getOpcode() != ISD::SRL)
2055 return SDValue();
2056
2057 // The shift must be of a 'not' value.
2058 SDValue Not = ShiftOp.getOperand(0);
2059 if (!Not.hasOneUse() || !isBitwiseNot(Not))
2060 return SDValue();
2061
2062 // The shift must be moving the sign bit to the least-significant-bit.
2063 EVT VT = ShiftOp.getValueType();
2064 SDValue ShAmt = ShiftOp.getOperand(1);
2065 ConstantSDNode *ShAmtC = isConstOrConstSplat(ShAmt);
2066 if (!ShAmtC || ShAmtC->getAPIntValue() != (VT.getScalarSizeInBits() - 1))
2067 return SDValue();
2068
2069 // Eliminate the 'not' by adjusting the shift and add/sub constant:
2070 // add (srl (not X), 31), C --> add (sra X, 31), (C + 1)
2071 // sub C, (srl (not X), 31) --> add (srl X, 31), (C - 1)
2072 SDLoc DL(N);
2073 auto ShOpcode = IsAdd ? ISD::SRA : ISD::SRL;
2074 SDValue NewShift = DAG.getNode(ShOpcode, DL, VT, Not.getOperand(0), ShAmt);
2075 APInt NewC = IsAdd ? C->getAPIntValue() + 1 : C->getAPIntValue() - 1;
2076 return DAG.getNode(ISD::ADD, DL, VT, NewShift, DAG.getConstant(NewC, DL, VT));
2077}
2078
2079/// Try to fold a node that behaves like an ADD (note that N isn't necessarily
2080/// an ISD::ADD here, it could for example be an ISD::OR if we know that there
2081/// are no common bits set in the operands).
2082SDValue DAGCombiner::visitADDLike(SDNode *N) {
2083 SDValue N0 = N->getOperand(0);
2084 SDValue N1 = N->getOperand(1);
2085 EVT VT = N0.getValueType();
2086 SDLoc DL(N);
2087
2088 // fold vector ops
2089 if (VT.isVector()) {
2090 if (SDValue FoldedVOp = SimplifyVBinOp(N))
2091 return FoldedVOp;
2092
2093 // fold (add x, 0) -> x, vector edition
2094 if (ISD::isBuildVectorAllZeros(N1.getNode()))
2095 return N0;
2096 if (ISD::isBuildVectorAllZeros(N0.getNode()))
2097 return N1;
2098 }
2099
2100 // fold (add x, undef) -> undef
2101 if (N0.isUndef())
2102 return N0;
2103
2104 if (N1.isUndef())
2105 return N1;
2106
2107 if (DAG.isConstantIntBuildVectorOrConstantInt(N0)) {
2108 // canonicalize constant to RHS
2109 if (!DAG.isConstantIntBuildVectorOrConstantInt(N1))
2110 return DAG.getNode(ISD::ADD, DL, VT, N1, N0);
2111 // fold (add c1, c2) -> c1+c2
2112 return DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, N0.getNode(),
2113 N1.getNode());
2114 }
2115
2116 // fold (add x, 0) -> x
2117 if (isNullConstant(N1))
2118 return N0;
2119
2120 if (isConstantOrConstantVector(N1, /* NoOpaque */ true)) {
2121 // fold ((A-c1)+c2) -> (A+(c2-c1))
2122 if (N0.getOpcode() == ISD::SUB &&
2123 isConstantOrConstantVector(N0.getOperand(1), /* NoOpaque */ true)) {
2124 SDValue Sub = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, N1.getNode(),
2125 N0.getOperand(1).getNode());
2126 assert(Sub && "Constant folding failed")((Sub && "Constant folding failed") ? static_cast<
void> (0) : __assert_fail ("Sub && \"Constant folding failed\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2126, __PRETTY_FUNCTION__))
;
2127 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), Sub);
2128 }
2129
2130 // fold ((c1-A)+c2) -> (c1+c2)-A
2131 if (N0.getOpcode() == ISD::SUB &&
2132 isConstantOrConstantVector(N0.getOperand(0), /* NoOpaque */ true)) {
2133 SDValue Add = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, N1.getNode(),
2134 N0.getOperand(0).getNode());
2135 assert(Add && "Constant folding failed")((Add && "Constant folding failed") ? static_cast<
void> (0) : __assert_fail ("Add && \"Constant folding failed\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2135, __PRETTY_FUNCTION__))
;
2136 return DAG.getNode(ISD::SUB, DL, VT, Add, N0.getOperand(1));
2137 }
2138
2139 // add (sext i1 X), 1 -> zext (not i1 X)
2140 // We don't transform this pattern:
2141 // add (zext i1 X), -1 -> sext (not i1 X)
2142 // because most (?) targets generate better code for the zext form.
2143 if (N0.getOpcode() == ISD::SIGN_EXTEND && N0.hasOneUse() &&
2144 isOneOrOneSplat(N1)) {
2145 SDValue X = N0.getOperand(0);
2146 if ((!LegalOperations ||
2147 (TLI.isOperationLegal(ISD::XOR, X.getValueType()) &&
2148 TLI.isOperationLegal(ISD::ZERO_EXTEND, VT))) &&
2149 X.getScalarValueSizeInBits() == 1) {
2150 SDValue Not = DAG.getNOT(DL, X, X.getValueType());
2151 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Not);
2152 }
2153 }
2154
2155 // Undo the add -> or combine to merge constant offsets from a frame index.
2156 if (N0.getOpcode() == ISD::OR &&
2157 isa<FrameIndexSDNode>(N0.getOperand(0)) &&
2158 isa<ConstantSDNode>(N0.getOperand(1)) &&
2159 DAG.haveNoCommonBitsSet(N0.getOperand(0), N0.getOperand(1))) {
2160 SDValue Add0 = DAG.getNode(ISD::ADD, DL, VT, N1, N0.getOperand(1));
2161 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), Add0);
2162 }
2163 }
2164
2165 if (SDValue NewSel = foldBinOpIntoSelect(N))
2166 return NewSel;
2167
2168 // reassociate add
2169 if (!reassociationCanBreakAddressingModePattern(ISD::ADD, DL, N0, N1)) {
2170 if (SDValue RADD = reassociateOps(ISD::ADD, DL, N0, N1, N->getFlags()))
2171 return RADD;
2172 }
2173 // fold ((0-A) + B) -> B-A
2174 if (N0.getOpcode() == ISD::SUB && isNullOrNullSplat(N0.getOperand(0)))
2175 return DAG.getNode(ISD::SUB, DL, VT, N1, N0.getOperand(1));
2176
2177 // fold (A + (0-B)) -> A-B
2178 if (N1.getOpcode() == ISD::SUB && isNullOrNullSplat(N1.getOperand(0)))
2179 return DAG.getNode(ISD::SUB, DL, VT, N0, N1.getOperand(1));
2180
2181 // fold (A+(B-A)) -> B
2182 if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(1))
2183 return N1.getOperand(0);
2184
2185 // fold ((B-A)+A) -> B
2186 if (N0.getOpcode() == ISD::SUB && N1 == N0.getOperand(1))
2187 return N0.getOperand(0);
2188
2189 // fold ((A-B)+(C-A)) -> (C-B)
2190 if (N0.getOpcode() == ISD::SUB && N1.getOpcode() == ISD::SUB &&
2191 N0.getOperand(0) == N1.getOperand(1))
2192 return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0),
2193 N0.getOperand(1));
2194
2195 // fold ((A-B)+(B-C)) -> (A-C)
2196 if (N0.getOpcode() == ISD::SUB && N1.getOpcode() == ISD::SUB &&
2197 N0.getOperand(1) == N1.getOperand(0))
2198 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0),
2199 N1.getOperand(1));
2200
2201 // fold (A+(B-(A+C))) to (B-C)
2202 if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD &&
2203 N0 == N1.getOperand(1).getOperand(0))
2204 return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0),
2205 N1.getOperand(1).getOperand(1));
2206
2207 // fold (A+(B-(C+A))) to (B-C)
2208 if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD &&
2209 N0 == N1.getOperand(1).getOperand(1))
2210 return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0),
2211 N1.getOperand(1).getOperand(0));
2212
2213 // fold (A+((B-A)+or-C)) to (B+or-C)
2214 if ((N1.getOpcode() == ISD::SUB || N1.getOpcode() == ISD::ADD) &&
2215 N1.getOperand(0).getOpcode() == ISD::SUB &&
2216 N0 == N1.getOperand(0).getOperand(1))
2217 return DAG.getNode(N1.getOpcode(), DL, VT, N1.getOperand(0).getOperand(0),
2218 N1.getOperand(1));
2219
2220 // fold (A-B)+(C-D) to (A+C)-(B+D) when A or C is constant
2221 if (N0.getOpcode() == ISD::SUB && N1.getOpcode() == ISD::SUB) {
2222 SDValue N00 = N0.getOperand(0);
2223 SDValue N01 = N0.getOperand(1);
2224 SDValue N10 = N1.getOperand(0);
2225 SDValue N11 = N1.getOperand(1);
2226
2227 if (isConstantOrConstantVector(N00) || isConstantOrConstantVector(N10))
2228 return DAG.getNode(ISD::SUB, DL, VT,
2229 DAG.getNode(ISD::ADD, SDLoc(N0), VT, N00, N10),
2230 DAG.getNode(ISD::ADD, SDLoc(N1), VT, N01, N11));
2231 }
2232
2233 // fold (add (umax X, C), -C) --> (usubsat X, C)
2234 if (N0.getOpcode() == ISD::UMAX && hasOperation(ISD::USUBSAT, VT)) {
2235 auto MatchUSUBSAT = [](ConstantSDNode *Max, ConstantSDNode *Op) {
2236 return (!Max && !Op) ||
2237 (Max && Op && Max->getAPIntValue() == (-Op->getAPIntValue()));
2238 };
2239 if (ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchUSUBSAT,
2240 /*AllowUndefs*/ true))
2241 return DAG.getNode(ISD::USUBSAT, DL, VT, N0.getOperand(0),
2242 N0.getOperand(1));
2243 }
2244
2245 if (SimplifyDemandedBits(SDValue(N, 0)))
2246 return SDValue(N, 0);
2247
2248 if (isOneOrOneSplat(N1)) {
2249 // fold (add (xor a, -1), 1) -> (sub 0, a)
2250 if (isBitwiseNot(N0))
2251 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
2252 N0.getOperand(0));
2253
2254 // fold (add (add (xor a, -1), b), 1) -> (sub b, a)
2255 if (N0.getOpcode() == ISD::ADD ||
2256 N0.getOpcode() == ISD::UADDO ||
2257 N0.getOpcode() == ISD::SADDO) {
2258 SDValue A, Xor;
2259
2260 if (isBitwiseNot(N0.getOperand(0))) {
2261 A = N0.getOperand(1);
2262 Xor = N0.getOperand(0);
2263 } else if (isBitwiseNot(N0.getOperand(1))) {
2264 A = N0.getOperand(0);
2265 Xor = N0.getOperand(1);
2266 }
2267
2268 if (Xor)
2269 return DAG.getNode(ISD::SUB, DL, VT, A, Xor.getOperand(0));
2270 }
2271
2272 // Look for:
2273 // add (add x, y), 1
2274 // And if the target does not like this form then turn into:
2275 // sub y, (xor x, -1)
2276 if (!TLI.preferIncOfAddToSubOfNot(VT) && N0.hasOneUse() &&
2277 N0.getOpcode() == ISD::ADD) {
2278 SDValue Not = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(0),
2279 DAG.getAllOnesConstant(DL, VT));
2280 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(1), Not);
2281 }
2282 }
2283
2284 // (x - y) + -1 -> add (xor y, -1), x
2285 if (N0.hasOneUse() && N0.getOpcode() == ISD::SUB &&
2286 isAllOnesOrAllOnesSplat(N1)) {
2287 SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(1), N1);
2288 return DAG.getNode(ISD::ADD, DL, VT, Xor, N0.getOperand(0));
2289 }
2290
2291 if (SDValue Combined = visitADDLikeCommutative(N0, N1, N))
2292 return Combined;
2293
2294 if (SDValue Combined = visitADDLikeCommutative(N1, N0, N))
2295 return Combined;
2296
2297 return SDValue();
2298}
2299
2300SDValue DAGCombiner::visitADD(SDNode *N) {
2301 SDValue N0 = N->getOperand(0);
2302 SDValue N1 = N->getOperand(1);
2303 EVT VT = N0.getValueType();
2304 SDLoc DL(N);
2305
2306 if (SDValue Combined = visitADDLike(N))
2307 return Combined;
2308
2309 if (SDValue V = foldAddSubBoolOfMaskedVal(N, DAG))
2310 return V;
2311
2312 if (SDValue V = foldAddSubOfSignBit(N, DAG))
2313 return V;
2314
2315 // fold (a+b) -> (a|b) iff a and b share no bits.
2316 if ((!LegalOperations || TLI.isOperationLegal(ISD::OR, VT)) &&
2317 DAG.haveNoCommonBitsSet(N0, N1))
2318 return DAG.getNode(ISD::OR, DL, VT, N0, N1);
2319
2320 return SDValue();
2321}
2322
2323SDValue DAGCombiner::visitADDSAT(SDNode *N) {
2324 unsigned Opcode = N->getOpcode();
2325 SDValue N0 = N->getOperand(0);
2326 SDValue N1 = N->getOperand(1);
2327 EVT VT = N0.getValueType();
2328 SDLoc DL(N);
2329
2330 // fold vector ops
2331 if (VT.isVector()) {
2332 // TODO SimplifyVBinOp
2333
2334 // fold (add_sat x, 0) -> x, vector edition
2335 if (ISD::isBuildVectorAllZeros(N1.getNode()))
2336 return N0;
2337 if (ISD::isBuildVectorAllZeros(N0.getNode()))
2338 return N1;
2339 }
2340
2341 // fold (add_sat x, undef) -> -1
2342 if (N0.isUndef() || N1.isUndef())
2343 return DAG.getAllOnesConstant(DL, VT);
2344
2345 if (DAG.isConstantIntBuildVectorOrConstantInt(N0)) {
2346 // canonicalize constant to RHS
2347 if (!DAG.isConstantIntBuildVectorOrConstantInt(N1))
2348 return DAG.getNode(Opcode, DL, VT, N1, N0);
2349 // fold (add_sat c1, c2) -> c3
2350 return DAG.FoldConstantArithmetic(Opcode, DL, VT, N0.getNode(),
2351 N1.getNode());
2352 }
2353
2354 // fold (add_sat x, 0) -> x
2355 if (isNullConstant(N1))
2356 return N0;
2357
2358 // If it cannot overflow, transform into an add.
2359 if (Opcode == ISD::UADDSAT)
2360 if (DAG.computeOverflowKind(N0, N1) == SelectionDAG::OFK_Never)
2361 return DAG.getNode(ISD::ADD, DL, VT, N0, N1);
2362
2363 return SDValue();
2364}
2365
2366static SDValue getAsCarry(const TargetLowering &TLI, SDValue V) {
2367 bool Masked = false;
2368
2369 // First, peel away TRUNCATE/ZERO_EXTEND/AND nodes due to legalization.
2370 while (true) {
2371 if (V.getOpcode() == ISD::TRUNCATE || V.getOpcode() == ISD::ZERO_EXTEND) {
2372 V = V.getOperand(0);
2373 continue;
2374 }
2375
2376 if (V.getOpcode() == ISD::AND && isOneConstant(V.getOperand(1))) {
2377 Masked = true;
2378 V = V.getOperand(0);
2379 continue;
2380 }
2381
2382 break;
2383 }
2384
2385 // If this is not a carry, return.
2386 if (V.getResNo() != 1)
2387 return SDValue();
2388
2389 if (V.getOpcode() != ISD::ADDCARRY && V.getOpcode() != ISD::SUBCARRY &&
2390 V.getOpcode() != ISD::UADDO && V.getOpcode() != ISD::USUBO)
2391 return SDValue();
2392
2393 EVT VT = V.getNode()->getValueType(0);
2394 if (!TLI.isOperationLegalOrCustom(V.getOpcode(), VT))
2395 return SDValue();
2396
2397 // If the result is masked, then no matter what kind of bool it is we can
2398 // return. If it isn't, then we need to make sure the bool type is either 0 or
2399 // 1 and not other values.
2400 if (Masked ||
2401 TLI.getBooleanContents(V.getValueType()) ==
2402 TargetLoweringBase::ZeroOrOneBooleanContent)
2403 return V;
2404
2405 return SDValue();
2406}
2407
2408/// Given the operands of an add/sub operation, see if the 2nd operand is a
2409/// masked 0/1 whose source operand is actually known to be 0/-1. If so, invert
2410/// the opcode and bypass the mask operation.
2411static SDValue foldAddSubMasked1(bool IsAdd, SDValue N0, SDValue N1,
2412 SelectionDAG &DAG, const SDLoc &DL) {
2413 if (N1.getOpcode() != ISD::AND || !isOneOrOneSplat(N1->getOperand(1)))
2414 return SDValue();
2415
2416 EVT VT = N0.getValueType();
2417 if (DAG.ComputeNumSignBits(N1.getOperand(0)) != VT.getScalarSizeInBits())
2418 return SDValue();
2419
2420 // add N0, (and (AssertSext X, i1), 1) --> sub N0, X
2421 // sub N0, (and (AssertSext X, i1), 1) --> add N0, X
2422 return DAG.getNode(IsAdd ? ISD::SUB : ISD::ADD, DL, VT, N0, N1.getOperand(0));
2423}
2424
2425/// Helper for doing combines based on N0 and N1 being added to each other.
2426SDValue DAGCombiner::visitADDLikeCommutative(SDValue N0, SDValue N1,
2427 SDNode *LocReference) {
2428 EVT VT = N0.getValueType();
2429 SDLoc DL(LocReference);
2430
2431 // fold (add x, shl(0 - y, n)) -> sub(x, shl(y, n))
2432 if (N1.getOpcode() == ISD::SHL && N1.getOperand(0).getOpcode() == ISD::SUB &&
2433 isNullOrNullSplat(N1.getOperand(0).getOperand(0)))
2434 return DAG.getNode(ISD::SUB, DL, VT, N0,
2435 DAG.getNode(ISD::SHL, DL, VT,
2436 N1.getOperand(0).getOperand(1),
2437 N1.getOperand(1)));
2438
2439 if (SDValue V = foldAddSubMasked1(true, N0, N1, DAG, DL))
2440 return V;
2441
2442 // Look for:
2443 // add (add x, 1), y
2444 // And if the target does not like this form then turn into:
2445 // sub y, (xor x, -1)
2446 if (!TLI.preferIncOfAddToSubOfNot(VT) && N0.hasOneUse() &&
2447 N0.getOpcode() == ISD::ADD && isOneOrOneSplat(N0.getOperand(1))) {
2448 SDValue Not = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(0),
2449 DAG.getAllOnesConstant(DL, VT));
2450 return DAG.getNode(ISD::SUB, DL, VT, N1, Not);
2451 }
2452
2453 // Hoist one-use subtraction by non-opaque constant:
2454 // (x - C) + y -> (x + y) - C
2455 // This is necessary because SUB(X,C) -> ADD(X,-C) doesn't work for vectors.
2456 if (N0.hasOneUse() && N0.getOpcode() == ISD::SUB &&
2457 isConstantOrConstantVector(N0.getOperand(1), /*NoOpaques=*/true)) {
2458 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), N1);
2459 return DAG.getNode(ISD::SUB, DL, VT, Add, N0.getOperand(1));
2460 }
2461 // Hoist one-use subtraction from non-opaque constant:
2462 // (C - x) + y -> (y - x) + C
2463 if (N0.hasOneUse() && N0.getOpcode() == ISD::SUB &&
2464 isConstantOrConstantVector(N0.getOperand(0), /*NoOpaques=*/true)) {
2465 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N1, N0.getOperand(1));
2466 return DAG.getNode(ISD::ADD, DL, VT, Sub, N0.getOperand(0));
2467 }
2468
2469 // If the target's bool is represented as 0/1, prefer to make this 'sub 0/1'
2470 // rather than 'add 0/-1' (the zext should get folded).
2471 // add (sext i1 Y), X --> sub X, (zext i1 Y)
2472 if (N0.getOpcode() == ISD::SIGN_EXTEND &&
2473 N0.getOperand(0).getScalarValueSizeInBits() == 1 &&
2474 TLI.getBooleanContents(VT) == TargetLowering::ZeroOrOneBooleanContent) {
2475 SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0));
2476 return DAG.getNode(ISD::SUB, DL, VT, N1, ZExt);
2477 }
2478
2479 // add X, (sextinreg Y i1) -> sub X, (and Y 1)
2480 if (N1.getOpcode() == ISD::SIGN_EXTEND_INREG) {
2481 VTSDNode *TN = cast<VTSDNode>(N1.getOperand(1));
2482 if (TN->getVT() == MVT::i1) {
2483 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0),
2484 DAG.getConstant(1, DL, VT));
2485 return DAG.getNode(ISD::SUB, DL, VT, N0, ZExt);
2486 }
2487 }
2488
2489 // (add X, (addcarry Y, 0, Carry)) -> (addcarry X, Y, Carry)
2490 if (N1.getOpcode() == ISD::ADDCARRY && isNullConstant(N1.getOperand(1)) &&
2491 N1.getResNo() == 0)
2492 return DAG.getNode(ISD::ADDCARRY, DL, N1->getVTList(),
2493 N0, N1.getOperand(0), N1.getOperand(2));
2494
2495 // (add X, Carry) -> (addcarry X, 0, Carry)
2496 if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT))
2497 if (SDValue Carry = getAsCarry(TLI, N1))
2498 return DAG.getNode(ISD::ADDCARRY, DL,
2499 DAG.getVTList(VT, Carry.getValueType()), N0,
2500 DAG.getConstant(0, DL, VT), Carry);
2501
2502 return SDValue();
2503}
2504
2505SDValue DAGCombiner::visitADDC(SDNode *N) {
2506 SDValue N0 = N->getOperand(0);
2507 SDValue N1 = N->getOperand(1);
2508 EVT VT = N0.getValueType();
2509 SDLoc DL(N);
2510
2511 // If the flag result is dead, turn this into an ADD.
2512 if (!N->hasAnyUseOfValue(1))
2513 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2514 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2515
2516 // canonicalize constant to RHS.
2517 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
2518 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2519 if (N0C && !N1C)
2520 return DAG.getNode(ISD::ADDC, DL, N->getVTList(), N1, N0);
2521
2522 // fold (addc x, 0) -> x + no carry out
2523 if (isNullConstant(N1))
2524 return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE,
2525 DL, MVT::Glue));
2526
2527 // If it cannot overflow, transform into an add.
2528 if (DAG.computeOverflowKind(N0, N1) == SelectionDAG::OFK_Never)
2529 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2530 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2531
2532 return SDValue();
2533}
2534
2535static SDValue flipBoolean(SDValue V, const SDLoc &DL,
2536 SelectionDAG &DAG, const TargetLowering &TLI) {
2537 EVT VT = V.getValueType();
2538
2539 SDValue Cst;
2540 switch (TLI.getBooleanContents(VT)) {
2541 case TargetLowering::ZeroOrOneBooleanContent:
2542 case TargetLowering::UndefinedBooleanContent:
2543 Cst = DAG.getConstant(1, DL, VT);
2544 break;
2545 case TargetLowering::ZeroOrNegativeOneBooleanContent:
2546 Cst = DAG.getAllOnesConstant(DL, VT);
2547 break;
2548 }
2549
2550 return DAG.getNode(ISD::XOR, DL, VT, V, Cst);
2551}
2552
2553/**
2554 * Flips a boolean if it is cheaper to compute. If the Force parameters is set,
2555 * then the flip also occurs if computing the inverse is the same cost.
2556 * This function returns an empty SDValue in case it cannot flip the boolean
2557 * without increasing the cost of the computation. If you want to flip a boolean
2558 * no matter what, use flipBoolean.
2559 */
2560static SDValue extractBooleanFlip(SDValue V, SelectionDAG &DAG,
2561 const TargetLowering &TLI,
2562 bool Force) {
2563 if (Force && isa<ConstantSDNode>(V))
2564 return flipBoolean(V, SDLoc(V), DAG, TLI);
2565
2566 if (V.getOpcode() != ISD::XOR)
2567 return SDValue();
2568
2569 ConstantSDNode *Const = isConstOrConstSplat(V.getOperand(1), false);
2570 if (!Const)
2571 return SDValue();
2572
2573 EVT VT = V.getValueType();
2574
2575 bool IsFlip = false;
2576 switch(TLI.getBooleanContents(VT)) {
2577 case TargetLowering::ZeroOrOneBooleanContent:
2578 IsFlip = Const->isOne();
2579 break;
2580 case TargetLowering::ZeroOrNegativeOneBooleanContent:
2581 IsFlip = Const->isAllOnesValue();
2582 break;
2583 case TargetLowering::UndefinedBooleanContent:
2584 IsFlip = (Const->getAPIntValue() & 0x01) == 1;
2585 break;
2586 }
2587
2588 if (IsFlip)
2589 return V.getOperand(0);
2590 if (Force)
2591 return flipBoolean(V, SDLoc(V), DAG, TLI);
2592 return SDValue();
2593}
2594
2595SDValue DAGCombiner::visitADDO(SDNode *N) {
2596 SDValue N0 = N->getOperand(0);
2597 SDValue N1 = N->getOperand(1);
2598 EVT VT = N0.getValueType();
2599 bool IsSigned = (ISD::SADDO == N->getOpcode());
2600
2601 EVT CarryVT = N->getValueType(1);
2602 SDLoc DL(N);
2603
2604 // If the flag result is dead, turn this into an ADD.
2605 if (!N->hasAnyUseOfValue(1))
2606 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2607 DAG.getUNDEF(CarryVT));
2608
2609 // canonicalize constant to RHS.
2610 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
2611 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
2612 return DAG.getNode(N->getOpcode(), DL, N->getVTList(), N1, N0);
2613
2614 // fold (addo x, 0) -> x + no carry out
2615 if (isNullOrNullSplat(N1))
2616 return CombineTo(N, N0, DAG.getConstant(0, DL, CarryVT));
2617
2618 if (!IsSigned) {
2619 // If it cannot overflow, transform into an add.
2620 if (DAG.computeOverflowKind(N0, N1) == SelectionDAG::OFK_Never)
2621 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2622 DAG.getConstant(0, DL, CarryVT));
2623
2624 // fold (uaddo (xor a, -1), 1) -> (usub 0, a) and flip carry.
2625 if (isBitwiseNot(N0) && isOneOrOneSplat(N1)) {
2626 SDValue Sub = DAG.getNode(ISD::USUBO, DL, N->getVTList(),
2627 DAG.getConstant(0, DL, VT), N0.getOperand(0));
2628 return CombineTo(N, Sub,
2629 flipBoolean(Sub.getValue(1), DL, DAG, TLI));
2630 }
2631
2632 if (SDValue Combined = visitUADDOLike(N0, N1, N))
2633 return Combined;
2634
2635 if (SDValue Combined = visitUADDOLike(N1, N0, N))
2636 return Combined;
2637 }
2638
2639 return SDValue();
2640}
2641
2642SDValue DAGCombiner::visitUADDOLike(SDValue N0, SDValue N1, SDNode *N) {
2643 EVT VT = N0.getValueType();
2644 if (VT.isVector())
2645 return SDValue();
2646
2647 // (uaddo X, (addcarry Y, 0, Carry)) -> (addcarry X, Y, Carry)
2648 // If Y + 1 cannot overflow.
2649 if (N1.getOpcode() == ISD::ADDCARRY && isNullConstant(N1.getOperand(1))) {
2650 SDValue Y = N1.getOperand(0);
2651 SDValue One = DAG.getConstant(1, SDLoc(N), Y.getValueType());
2652 if (DAG.computeOverflowKind(Y, One) == SelectionDAG::OFK_Never)
2653 return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0, Y,
2654 N1.getOperand(2));
2655 }
2656
2657 // (uaddo X, Carry) -> (addcarry X, 0, Carry)
2658 if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT))
2659 if (SDValue Carry = getAsCarry(TLI, N1))
2660 return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0,
2661 DAG.getConstant(0, SDLoc(N), VT), Carry);
2662
2663 return SDValue();
2664}
2665
2666SDValue DAGCombiner::visitADDE(SDNode *N) {
2667 SDValue N0 = N->getOperand(0);
2668 SDValue N1 = N->getOperand(1);
2669 SDValue CarryIn = N->getOperand(2);
2670
2671 // canonicalize constant to RHS
2672 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
2673 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2674 if (N0C && !N1C)
2675 return DAG.getNode(ISD::ADDE, SDLoc(N), N->getVTList(),
2676 N1, N0, CarryIn);
2677
2678 // fold (adde x, y, false) -> (addc x, y)
2679 if (CarryIn.getOpcode() == ISD::CARRY_FALSE)
2680 return DAG.getNode(ISD::ADDC, SDLoc(N), N->getVTList(), N0, N1);
2681
2682 return SDValue();
2683}
2684
2685SDValue DAGCombiner::visitADDCARRY(SDNode *N) {
2686 SDValue N0 = N->getOperand(0);
1
Value assigned to 'N0.Node'
2687 SDValue N1 = N->getOperand(1);
2688 SDValue CarryIn = N->getOperand(2);
2689 SDLoc DL(N);
2690
2691 // canonicalize constant to RHS
2692 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
2
Calling 'dyn_cast<llvm::ConstantSDNode, llvm::SDValue>'
17
Returning from 'dyn_cast<llvm::ConstantSDNode, llvm::SDValue>'
2693 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2694 if (N0C && !N1C)
18
Assuming 'N0C' is null
2695 return DAG.getNode(ISD::ADDCARRY, DL, N->getVTList(), N1, N0, CarryIn);
2696
2697 // fold (addcarry x, y, false) -> (uaddo x, y)
2698 if (isNullConstant(CarryIn)) {
19
Assuming the condition is false
20
Taking false branch
2699 if (!LegalOperations ||
2700 TLI.isOperationLegalOrCustom(ISD::UADDO, N->getValueType(0)))
2701 return DAG.getNode(ISD::UADDO, DL, N->getVTList(), N0, N1);
2702 }
2703
2704 // fold (addcarry 0, 0, X) -> (and (ext/trunc X), 1) and no carry.
2705 if (isNullConstant(N0) && isNullConstant(N1)) {
21
Assuming the condition is true
22
Assuming the condition is true
23
Taking true branch
2706 EVT VT = N0.getValueType();
24
Calling 'SDValue::getValueType'
2707 EVT CarryVT = CarryIn.getValueType();
2708 SDValue CarryExt = DAG.getBoolExtOrTrunc(CarryIn, DL, VT, CarryVT);
2709 AddToWorklist(CarryExt.getNode());
2710 return CombineTo(N, DAG.getNode(ISD::AND, DL, VT, CarryExt,
2711 DAG.getConstant(1, DL, VT)),
2712 DAG.getConstant(0, DL, CarryVT));
2713 }
2714
2715 if (SDValue Combined = visitADDCARRYLike(N0, N1, CarryIn, N))
2716 return Combined;
2717
2718 if (SDValue Combined = visitADDCARRYLike(N1, N0, CarryIn, N))
2719 return Combined;
2720
2721 return SDValue();
2722}
2723
2724/**
2725 * If we are facing some sort of diamond carry propapagtion pattern try to
2726 * break it up to generate something like:
2727 * (addcarry X, 0, (addcarry A, B, Z):Carry)
2728 *
2729 * The end result is usually an increase in operation required, but because the
2730 * carry is now linearized, other tranforms can kick in and optimize the DAG.
2731 *
2732 * Patterns typically look something like
2733 * (uaddo A, B)
2734 * / \
2735 * Carry Sum
2736 * | \
2737 * | (addcarry *, 0, Z)
2738 * | /
2739 * \ Carry
2740 * | /
2741 * (addcarry X, *, *)
2742 *
2743 * But numerous variation exist. Our goal is to identify A, B, X and Z and
2744 * produce a combine with a single path for carry propagation.
2745 */
2746static SDValue combineADDCARRYDiamond(DAGCombiner &Combiner, SelectionDAG &DAG,
2747 SDValue X, SDValue Carry0, SDValue Carry1,
2748 SDNode *N) {
2749 if (Carry1.getResNo() != 1 || Carry0.getResNo() != 1)
2750 return SDValue();
2751 if (Carry1.getOpcode() != ISD::UADDO)
2752 return SDValue();
2753
2754 SDValue Z;
2755
2756 /**
2757 * First look for a suitable Z. It will present itself in the form of
2758 * (addcarry Y, 0, Z) or its equivalent (uaddo Y, 1) for Z=true
2759 */
2760 if (Carry0.getOpcode() == ISD::ADDCARRY &&
2761 isNullConstant(Carry0.getOperand(1))) {
2762 Z = Carry0.getOperand(2);
2763 } else if (Carry0.getOpcode() == ISD::UADDO &&
2764 isOneConstant(Carry0.getOperand(1))) {
2765 EVT VT = Combiner.getSetCCResultType(Carry0.getValueType());
2766 Z = DAG.getConstant(1, SDLoc(Carry0.getOperand(1)), VT);
2767 } else {
2768 // We couldn't find a suitable Z.
2769 return SDValue();
2770 }
2771
2772
2773 auto cancelDiamond = [&](SDValue A,SDValue B) {
2774 SDLoc DL(N);
2775 SDValue NewY = DAG.getNode(ISD::ADDCARRY, DL, Carry0->getVTList(), A, B, Z);
2776 Combiner.AddToWorklist(NewY.getNode());
2777 return DAG.getNode(ISD::ADDCARRY, DL, N->getVTList(), X,
2778 DAG.getConstant(0, DL, X.getValueType()),
2779 NewY.getValue(1));
2780 };
2781
2782 /**
2783 * (uaddo A, B)
2784 * |
2785 * Sum
2786 * |
2787 * (addcarry *, 0, Z)
2788 */
2789 if (Carry0.getOperand(0) == Carry1.getValue(0)) {
2790 return cancelDiamond(Carry1.getOperand(0), Carry1.getOperand(1));
2791 }
2792
2793 /**
2794 * (addcarry A, 0, Z)
2795 * |
2796 * Sum
2797 * |
2798 * (uaddo *, B)
2799 */
2800 if (Carry1.getOperand(0) == Carry0.getValue(0)) {
2801 return cancelDiamond(Carry0.getOperand(0), Carry1.getOperand(1));
2802 }
2803
2804 if (Carry1.getOperand(1) == Carry0.getValue(0)) {
2805 return cancelDiamond(Carry1.getOperand(0), Carry0.getOperand(0));
2806 }
2807
2808 return SDValue();
2809}
2810
2811// If we are facing some sort of diamond carry/borrow in/out pattern try to
2812// match patterns like:
2813//
2814// (uaddo A, B) CarryIn
2815// | \ |
2816// | \ |
2817// PartialSum PartialCarryOutX /
2818// | | /
2819// | ____|____________/
2820// | / |
2821// (uaddo *, *) \________
2822// | \ \
2823// | \ |
2824// | PartialCarryOutY |
2825// | \ |
2826// | \ /
2827// AddCarrySum | ______/
2828// | /
2829// CarryOut = (or *, *)
2830//
2831// And generate ADDCARRY (or SUBCARRY) with two result values:
2832//
2833// {AddCarrySum, CarryOut} = (addcarry A, B, CarryIn)
2834//
2835// Our goal is to identify A, B, and CarryIn and produce ADDCARRY/SUBCARRY with
2836// a single path for carry/borrow out propagation:
2837static SDValue combineCarryDiamond(DAGCombiner &Combiner, SelectionDAG &DAG,
2838 const TargetLowering &TLI, SDValue Carry0,
2839 SDValue Carry1, SDNode *N) {
2840 if (Carry0.getResNo() != 1 || Carry1.getResNo() != 1)
2841 return SDValue();
2842 unsigned Opcode = Carry0.getOpcode();
2843 if (Opcode != Carry1.getOpcode())
2844 return SDValue();
2845 if (Opcode != ISD::UADDO && Opcode != ISD::USUBO)
2846 return SDValue();
2847
2848 // Canonicalize the add/sub of A and B as Carry0 and the add/sub of the
2849 // carry/borrow in as Carry1. (The top and middle uaddo nodes respectively in
2850 // the above ASCII art.)
2851 if (Carry1.getOperand(0) != Carry0.getValue(0) &&
2852 Carry1.getOperand(1) != Carry0.getValue(0))
2853 std::swap(Carry0, Carry1);
2854 if (Carry1.getOperand(0) != Carry0.getValue(0) &&
2855 Carry1.getOperand(1) != Carry0.getValue(0))
2856 return SDValue();
2857
2858 // The carry in value must be on the righthand side for subtraction.
2859 unsigned CarryInOperandNum =
2860 Carry1.getOperand(0) == Carry0.getValue(0) ? 1 : 0;
2861 if (Opcode == ISD::USUBO && CarryInOperandNum != 1)
2862 return SDValue();
2863 SDValue CarryIn = Carry1.getOperand(CarryInOperandNum);
2864
2865 unsigned NewOp = Opcode == ISD::UADDO ? ISD::ADDCARRY : ISD::SUBCARRY;
2866 if (!TLI.isOperationLegalOrCustom(NewOp, Carry0.getValue(0).getValueType()))
2867 return SDValue();
2868
2869 // Verify that the carry/borrow in is plausibly a carry/borrow bit.
2870 // TODO: make getAsCarry() aware of how partial carries are merged.
2871 if (CarryIn.getOpcode() != ISD::ZERO_EXTEND)
2872 return SDValue();
2873 CarryIn = CarryIn.getOperand(0);
2874 if (CarryIn.getValueType() != MVT::i1)
2875 return SDValue();
2876
2877 SDLoc DL(N);
2878 SDValue Merged =
2879 DAG.getNode(NewOp, DL, Carry1->getVTList(), Carry0.getOperand(0),
2880 Carry0.getOperand(1), CarryIn);
2881
2882 // Please note that because we have proven that the result of the UADDO/USUBO
2883 // of A and B feeds into the UADDO/USUBO that does the carry/borrow in, we can
2884 // therefore prove that if the first UADDO/USUBO overflows, the second
2885 // UADDO/USUBO cannot. For example consider 8-bit numbers where 0xFF is the
2886 // maximum value.
2887 //
2888 // 0xFF + 0xFF == 0xFE with carry but 0xFE + 1 does not carry
2889 // 0x00 - 0xFF == 1 with a carry/borrow but 1 - 1 == 0 (no carry/borrow)
2890 //
2891 // This is important because it means that OR and XOR can be used to merge
2892 // carry flags; and that AND can return a constant zero.
2893 //
2894 // TODO: match other operations that can merge flags (ADD, etc)
2895 DAG.ReplaceAllUsesOfValueWith(Carry1.getValue(0), Merged.getValue(0));
2896 if (N->getOpcode() == ISD::AND)
2897 return DAG.getConstant(0, DL, MVT::i1);
2898 return Merged.getValue(1);
2899}
2900
2901SDValue DAGCombiner::visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn,
2902 SDNode *N) {
2903 // fold (addcarry (xor a, -1), b, c) -> (subcarry b, a, !c) and flip carry.
2904 if (isBitwiseNot(N0))
2905 if (SDValue NotC = extractBooleanFlip(CarryIn, DAG, TLI, true)) {
2906 SDLoc DL(N);
2907 SDValue Sub = DAG.getNode(ISD::SUBCARRY, DL, N->getVTList(), N1,
2908 N0.getOperand(0), NotC);
2909 return CombineTo(N, Sub,
2910 flipBoolean(Sub.getValue(1), DL, DAG, TLI));
2911 }
2912
2913 // Iff the flag result is dead:
2914 // (addcarry (add|uaddo X, Y), 0, Carry) -> (addcarry X, Y, Carry)
2915 // Don't do this if the Carry comes from the uaddo. It won't remove the uaddo
2916 // or the dependency between the instructions.
2917 if ((N0.getOpcode() == ISD::ADD ||
2918 (N0.getOpcode() == ISD::UADDO && N0.getResNo() == 0 &&
2919 N0.getValue(1) != CarryIn)) &&
2920 isNullConstant(N1) && !N->hasAnyUseOfValue(1))
2921 return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(),
2922 N0.getOperand(0), N0.getOperand(1), CarryIn);
2923
2924 /**
2925 * When one of the addcarry argument is itself a carry, we may be facing
2926 * a diamond carry propagation. In which case we try to transform the DAG
2927 * to ensure linear carry propagation if that is possible.
2928 */
2929 if (auto Y = getAsCarry(TLI, N1)) {
2930 // Because both are carries, Y and Z can be swapped.
2931 if (auto R = combineADDCARRYDiamond(*this, DAG, N0, Y, CarryIn, N))
2932 return R;
2933 if (auto R = combineADDCARRYDiamond(*this, DAG, N0, CarryIn, Y, N))
2934 return R;
2935 }
2936
2937 return SDValue();
2938}
2939
2940// Since it may not be valid to emit a fold to zero for vector initializers
2941// check if we can before folding.
2942static SDValue tryFoldToZero(const SDLoc &DL, const TargetLowering &TLI, EVT VT,
2943 SelectionDAG &DAG, bool LegalOperations) {
2944 if (!VT.isVector())
2945 return DAG.getConstant(0, DL, VT);
2946 if (!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT))
2947 return DAG.getConstant(0, DL, VT);
2948 return SDValue();
2949}
2950
2951SDValue DAGCombiner::visitSUB(SDNode *N) {
2952 SDValue N0 = N->getOperand(0);
2953 SDValue N1 = N->getOperand(1);
2954 EVT VT = N0.getValueType();
2955 SDLoc DL(N);
2956
2957 // fold vector ops
2958 if (VT.isVector()) {
2959 if (SDValue FoldedVOp = SimplifyVBinOp(N))
2960 return FoldedVOp;
2961
2962 // fold (sub x, 0) -> x, vector edition
2963 if (ISD::isBuildVectorAllZeros(N1.getNode()))
2964 return N0;
2965 }
2966
2967 // fold (sub x, x) -> 0
2968 // FIXME: Refactor this and xor and other similar operations together.
2969 if (N0 == N1)
2970 return tryFoldToZero(DL, TLI, VT, DAG, LegalOperations);
2971 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
2972 DAG.isConstantIntBuildVectorOrConstantInt(N1)) {
2973 // fold (sub c1, c2) -> c1-c2
2974 return DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, N0.getNode(),
2975 N1.getNode());
2976 }
2977
2978 if (SDValue NewSel = foldBinOpIntoSelect(N))
2979 return NewSel;
2980
2981 ConstantSDNode *N1C = getAsNonOpaqueConstant(N1);
2982
2983 // fold (sub x, c) -> (add x, -c)
2984 if (N1C) {
2985 return DAG.getNode(ISD::ADD, DL, VT, N0,
2986 DAG.getConstant(-N1C->getAPIntValue(), DL, VT));
2987 }
2988
2989 if (isNullOrNullSplat(N0)) {
2990 unsigned BitWidth = VT.getScalarSizeInBits();
2991 // Right-shifting everything out but the sign bit followed by negation is
2992 // the same as flipping arithmetic/logical shift type without the negation:
2993 // -(X >>u 31) -> (X >>s 31)
2994 // -(X >>s 31) -> (X >>u 31)
2995 if (N1->getOpcode() == ISD::SRA || N1->getOpcode() == ISD::SRL) {
2996 ConstantSDNode *ShiftAmt = isConstOrConstSplat(N1.getOperand(1));
2997 if (ShiftAmt && ShiftAmt->getAPIntValue() == (BitWidth - 1)) {
2998 auto NewSh = N1->getOpcode() == ISD::SRA ? ISD::SRL : ISD::SRA;
2999 if (!LegalOperations || TLI.isOperationLegal(NewSh, VT))
3000 return DAG.getNode(NewSh, DL, VT, N1.getOperand(0), N1.getOperand(1));
3001 }
3002 }
3003
3004 // 0 - X --> 0 if the sub is NUW.
3005 if (N->getFlags().hasNoUnsignedWrap())
3006 return N0;
3007
3008 if (DAG.MaskedValueIsZero(N1, ~APInt::getSignMask(BitWidth))) {
3009 // N1 is either 0 or the minimum signed value. If the sub is NSW, then
3010 // N1 must be 0 because negating the minimum signed value is undefined.
3011 if (N->getFlags().hasNoSignedWrap())
3012 return N0;
3013
3014 // 0 - X --> X if X is 0 or the minimum signed value.
3015 return N1;
3016 }
3017 }
3018
3019 // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1)
3020 if (isAllOnesOrAllOnesSplat(N0))
3021 return DAG.getNode(ISD::XOR, DL, VT, N1, N0);
3022
3023 // fold (A - (0-B)) -> A+B
3024 if (N1.getOpcode() == ISD::SUB && isNullOrNullSplat(N1.getOperand(0)))
3025 return DAG.getNode(ISD::ADD, DL, VT, N0, N1.getOperand(1));
3026
3027 // fold A-(A-B) -> B
3028 if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(0))
3029 return N1.getOperand(1);
3030
3031 // fold (A+B)-A -> B
3032 if (N0.getOpcode() == ISD::ADD && N0.getOperand(0) == N1)
3033 return N0.getOperand(1);
3034
3035 // fold (A+B)-B -> A
3036 if (N0.getOpcode() == ISD::ADD && N0.getOperand(1) == N1)
3037 return N0.getOperand(0);
3038
3039 // fold (A+C1)-C2 -> A+(C1-C2)
3040 if (N0.getOpcode() == ISD::ADD &&
3041 isConstantOrConstantVector(N1, /* NoOpaques */ true) &&
3042 isConstantOrConstantVector(N0.getOperand(1), /* NoOpaques */ true)) {
3043 SDValue NewC = DAG.FoldConstantArithmetic(
3044 ISD::SUB, DL, VT, N0.getOperand(1).getNode(), N1.getNode());
3045 assert(NewC && "Constant folding failed")((NewC && "Constant folding failed") ? static_cast<
void> (0) : __assert_fail ("NewC && \"Constant folding failed\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3045, __PRETTY_FUNCTION__))
;
3046 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), NewC);
3047 }
3048
3049 // fold C2-(A+C1) -> (C2-C1)-A
3050 if (N1.getOpcode() == ISD::ADD) {
3051 SDValue N11 = N1.getOperand(1);
3052 if (isConstantOrConstantVector(N0, /* NoOpaques */ true) &&
3053 isConstantOrConstantVector(N11, /* NoOpaques */ true)) {
3054 SDValue NewC = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, N0.getNode(),
3055 N11.getNode());
3056 assert(NewC && "Constant folding failed")((NewC && "Constant folding failed") ? static_cast<
void> (0) : __assert_fail ("NewC && \"Constant folding failed\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3056, __PRETTY_FUNCTION__))
;
3057 return DAG.getNode(ISD::SUB, DL, VT, NewC, N1.getOperand(0));
3058 }
3059 }
3060
3061 // fold (A-C1)-C2 -> A-(C1+C2)
3062 if (N0.getOpcode() == ISD::SUB &&
3063 isConstantOrConstantVector(N1, /* NoOpaques */ true) &&
3064 isConstantOrConstantVector(N0.getOperand(1), /* NoOpaques */ true)) {
3065 SDValue NewC = DAG.FoldConstantArithmetic(
3066 ISD::ADD, DL, VT, N0.getOperand(1).getNode(), N1.getNode());
3067 assert(NewC && "Constant folding failed")((NewC && "Constant folding failed") ? static_cast<
void> (0) : __assert_fail ("NewC && \"Constant folding failed\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3067, __PRETTY_FUNCTION__))
;
3068 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), NewC);
3069 }
3070
3071 // fold (c1-A)-c2 -> (c1-c2)-A
3072 if (N0.getOpcode() == ISD::SUB &&
3073 isConstantOrConstantVector(N1, /* NoOpaques */ true) &&
3074 isConstantOrConstantVector(N0.getOperand(0), /* NoOpaques */ true)) {
3075 SDValue NewC = DAG.FoldConstantArithmetic(
3076 ISD::SUB, DL, VT, N0.getOperand(0).getNode(), N1.getNode());
3077 assert(NewC && "Constant folding failed")((NewC && "Constant folding failed") ? static_cast<
void> (0) : __assert_fail ("NewC && \"Constant folding failed\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3077, __PRETTY_FUNCTION__))
;
3078 return DAG.getNode(ISD::SUB, DL, VT, NewC, N0.getOperand(1));
3079 }
3080
3081 // fold ((A+(B+or-C))-B) -> A+or-C
3082 if (N0.getOpcode() == ISD::ADD &&
3083 (N0.getOperand(1).getOpcode() == ISD::SUB ||
3084 N0.getOperand(1).getOpcode() == ISD::ADD) &&
3085 N0.getOperand(1).getOperand(0) == N1)
3086 return DAG.getNode(N0.getOperand(1).getOpcode(), DL, VT, N0.getOperand(0),
3087 N0.getOperand(1).getOperand(1));
3088
3089 // fold ((A+(C+B))-B) -> A+C
3090 if (N0.getOpcode() == ISD::ADD && N0.getOperand(1).getOpcode() == ISD::ADD &&
3091 N0.getOperand(1).getOperand(1) == N1)
3092 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0),
3093 N0.getOperand(1).getOperand(0));
3094
3095 // fold ((A-(B-C))-C) -> A-B
3096 if (N0.getOpcode() == ISD::SUB && N0.getOperand(1).getOpcode() == ISD::SUB &&
3097 N0.getOperand(1).getOperand(1) == N1)
3098 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0),
3099 N0.getOperand(1).getOperand(0));
3100
3101 // fold (A-(B-C)) -> A+(C-B)
3102 if (N1.getOpcode() == ISD::SUB && N1.hasOneUse())
3103 return DAG.getNode(ISD::ADD, DL, VT, N0,
3104 DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(1),
3105 N1.getOperand(0)));
3106
3107 // A - (A & B) -> A & (~B)
3108 if (N1.getOpcode() == ISD::AND) {
3109 SDValue A = N1.getOperand(0);
3110 SDValue B = N1.getOperand(1);
3111 if (A != N0)
3112 std::swap(A, B);
3113 if (A == N0 &&
3114 (N1.hasOneUse() || isConstantOrConstantVector(B, /*NoOpaques=*/true))) {
3115 SDValue InvB =
3116 DAG.getNode(ISD::XOR, DL, VT, B, DAG.getAllOnesConstant(DL, VT));
3117 return DAG.getNode(ISD::AND, DL, VT, A, InvB);
3118 }
3119 }
3120
3121 // fold (X - (-Y * Z)) -> (X + (Y * Z))
3122 if (N1.getOpcode() == ISD::MUL && N1.hasOneUse()) {
3123 if (N1.getOperand(0).getOpcode() == ISD::SUB &&
3124 isNullOrNullSplat(N1.getOperand(0).getOperand(0))) {
3125 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT,
3126 N1.getOperand(0).getOperand(1),
3127 N1.getOperand(1));
3128 return DAG.getNode(ISD::ADD, DL, VT, N0, Mul);
3129 }
3130 if (N1.getOperand(1).getOpcode() == ISD::SUB &&
3131 isNullOrNullSplat(N1.getOperand(1).getOperand(0))) {
3132 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT,
3133 N1.getOperand(0),
3134 N1.getOperand(1).getOperand(1));
3135 return DAG.getNode(ISD::ADD, DL, VT, N0, Mul);
3136 }
3137 }
3138
3139 // If either operand of a sub is undef, the result is undef
3140 if (N0.isUndef())
3141 return N0;
3142 if (N1.isUndef())
3143 return N1;
3144
3145 if (SDValue V = foldAddSubBoolOfMaskedVal(N, DAG))
3146 return V;
3147
3148 if (SDValue V = foldAddSubOfSignBit(N, DAG))
3149 return V;
3150
3151 if (SDValue V = foldAddSubMasked1(false, N0, N1, DAG, SDLoc(N)))
3152 return V;
3153
3154 // (x - y) - 1 -> add (xor y, -1), x
3155 if (N0.hasOneUse() && N0.getOpcode() == ISD::SUB && isOneOrOneSplat(N1)) {
3156 SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(1),
3157 DAG.getAllOnesConstant(DL, VT));
3158 return DAG.getNode(ISD::ADD, DL, VT, Xor, N0.getOperand(0));
3159 }
3160
3161 // Look for:
3162 // sub y, (xor x, -1)
3163 // And if the target does not like this form then turn into:
3164 // add (add x, y), 1
3165 if (TLI.preferIncOfAddToSubOfNot(VT) && N1.hasOneUse() && isBitwiseNot(N1)) {
3166 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, N1.getOperand(0));
3167 return DAG.getNode(ISD::ADD, DL, VT, Add, DAG.getConstant(1, DL, VT));
3168 }
3169
3170 // Hoist one-use addition by non-opaque constant:
3171 // (x + C) - y -> (x - y) + C
3172 if (N0.hasOneUse() && N0.getOpcode() == ISD::ADD &&
3173 isConstantOrConstantVector(N0.getOperand(1), /*NoOpaques=*/true)) {
3174 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), N1);
3175 return DAG.getNode(ISD::ADD, DL, VT, Sub, N0.getOperand(1));
3176 }
3177 // y - (x + C) -> (y - x) - C
3178 if (N1.hasOneUse() && N1.getOpcode() == ISD::ADD &&
3179 isConstantOrConstantVector(N1.getOperand(1), /*NoOpaques=*/true)) {
3180 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, N1.getOperand(0));
3181 return DAG.getNode(ISD::SUB, DL, VT, Sub, N1.getOperand(1));
3182 }
3183 // (x - C) - y -> (x - y) - C
3184 // This is necessary because SUB(X,C) -> ADD(X,-C) doesn't work for vectors.
3185 if (N0.hasOneUse() && N0.getOpcode() == ISD::SUB &&
3186 isConstantOrConstantVector(N0.getOperand(1), /*NoOpaques=*/true)) {
3187 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), N1);
3188 return DAG.getNode(ISD::SUB, DL, VT, Sub, N0.getOperand(1));
3189 }
3190 // (C - x) - y -> C - (x + y)
3191 if (N0.hasOneUse() && N0.getOpcode() == ISD::SUB &&
3192 isConstantOrConstantVector(N0.getOperand(0), /*NoOpaques=*/true)) {
3193 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(1), N1);
3194 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), Add);
3195 }
3196
3197 // If the target's bool is represented as 0/-1, prefer to make this 'add 0/-1'
3198 // rather than 'sub 0/1' (the sext should get folded).
3199 // sub X, (zext i1 Y) --> add X, (sext i1 Y)
3200 if (N1.getOpcode() == ISD::ZERO_EXTEND &&
3201 N1.getOperand(0).getScalarValueSizeInBits() == 1 &&
3202 TLI.getBooleanContents(VT) ==
3203 TargetLowering::ZeroOrNegativeOneBooleanContent) {
3204 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, N1.getOperand(0));
3205 return DAG.getNode(ISD::ADD, DL, VT, N0, SExt);
3206 }
3207
3208 // fold Y = sra (X, size(X)-1); sub (xor (X, Y), Y) -> (abs X)
3209 if (TLI.isOperationLegalOrCustom(ISD::ABS, VT)) {
3210 if (N0.getOpcode() == ISD::XOR && N1.getOpcode() == ISD::SRA) {
3211 SDValue X0 = N0.getOperand(0), X1 = N0.getOperand(1);
3212 SDValue S0 = N1.getOperand(0);
3213 if ((X0 == S0 && X1 == N1) || (X0 == N1 && X1 == S0)) {
3214 unsigned OpSizeInBits = VT.getScalarSizeInBits();
3215 if (ConstantSDNode *C = isConstOrConstSplat(N1.getOperand(1)))
3216 if (C->getAPIntValue() == (OpSizeInBits - 1))
3217 return DAG.getNode(ISD::ABS, SDLoc(N), VT, S0);
3218 }
3219 }
3220 }
3221
3222 // If the relocation model supports it, consider symbol offsets.
3223 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N0))
3224 if (!LegalOperations && TLI.isOffsetFoldingLegal(GA)) {
3225 // fold (sub Sym, c) -> Sym-c
3226 if (N1C && GA->getOpcode() == ISD::GlobalAddress)
3227 return DAG.getGlobalAddress(GA->getGlobal(), SDLoc(N1C), VT,
3228 GA->getOffset() -
3229 (uint64_t)N1C->getSExtValue());
3230 // fold (sub Sym+c1, Sym+c2) -> c1-c2
3231 if (GlobalAddressSDNode *GB = dyn_cast<GlobalAddressSDNode>(N1))
3232 if (GA->getGlobal() == GB->getGlobal())
3233 return DAG.getConstant((uint64_t)GA->getOffset() - GB->getOffset(),
3234 DL, VT);
3235 }
3236
3237 // sub X, (sextinreg Y i1) -> add X, (and Y 1)
3238 if (N1.getOpcode() == ISD::SIGN_EXTEND_INREG) {
3239 VTSDNode *TN = cast<VTSDNode>(N1.getOperand(1));
3240 if (TN->getVT() == MVT::i1) {
3241 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0),
3242 DAG.getConstant(1, DL, VT));
3243 return DAG.getNode(ISD::ADD, DL, VT, N0, ZExt);
3244 }
3245 }
3246
3247 // Prefer an add for more folding potential and possibly better codegen:
3248 // sub N0, (lshr N10, width-1) --> add N0, (ashr N10, width-1)
3249 if (!LegalOperations && N1.getOpcode() == ISD::SRL && N1.hasOneUse()) {
3250 SDValue ShAmt = N1.getOperand(1);
3251 ConstantSDNode *ShAmtC = isConstOrConstSplat(ShAmt);
3252 if (ShAmtC &&
3253 ShAmtC->getAPIntValue() == (N1.getScalarValueSizeInBits() - 1)) {
3254 SDValue SRA = DAG.getNode(ISD::SRA, DL, VT, N1.getOperand(0), ShAmt);
3255 return DAG.getNode(ISD::ADD, DL, VT, N0, SRA);
3256 }
3257 }
3258
3259 if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT)) {
3260 // (sub Carry, X) -> (addcarry (sub 0, X), 0, Carry)
3261 if (SDValue Carry = getAsCarry(TLI, N0)) {
3262 SDValue X = N1;
3263 SDValue Zero = DAG.getConstant(0, DL, VT);
3264 SDValue NegX = DAG.getNode(ISD::SUB, DL, VT, Zero, X);
3265 return DAG.getNode(ISD::ADDCARRY, DL,
3266 DAG.getVTList(VT, Carry.getValueType()), NegX, Zero,
3267 Carry);
3268 }
3269 }
3270
3271 return SDValue();
3272}
3273
3274SDValue DAGCombiner::visitSUBSAT(SDNode *N) {
3275 SDValue N0 = N->getOperand(0);
3276 SDValue N1 = N->getOperand(1);
3277 EVT VT = N0.getValueType();
3278 SDLoc DL(N);
3279
3280 // fold vector ops
3281 if (VT.isVector()) {
3282 // TODO SimplifyVBinOp
3283
3284 // fold (sub_sat x, 0) -> x, vector edition
3285 if (ISD::isBuildVectorAllZeros(N1.getNode()))
3286 return N0;
3287 }
3288
3289 // fold (sub_sat x, undef) -> 0
3290 if (N0.isUndef() || N1.isUndef())
3291 return DAG.getConstant(0, DL, VT);
3292
3293 // fold (sub_sat x, x) -> 0
3294 if (N0 == N1)
3295 return DAG.getConstant(0, DL, VT);
3296
3297 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
3298 DAG.isConstantIntBuildVectorOrConstantInt(N1)) {
3299 // fold (sub_sat c1, c2) -> c3
3300 return DAG.FoldConstantArithmetic(N->getOpcode(), DL, VT, N0.getNode(),
3301 N1.getNode());
3302 }
3303
3304 // fold (sub_sat x, 0) -> x
3305 if (isNullConstant(N1))
3306 return N0;
3307
3308 return SDValue();
3309}
3310
3311SDValue DAGCombiner::visitSUBC(SDNode *N) {
3312 SDValue N0 = N->getOperand(0);
3313 SDValue N1 = N->getOperand(1);
3314 EVT VT = N0.getValueType();
3315 SDLoc DL(N);
3316
3317 // If the flag result is dead, turn this into an SUB.
3318 if (!N->hasAnyUseOfValue(1))
3319 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1),
3320 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
3321
3322 // fold (subc x, x) -> 0 + no borrow
3323 if (N0 == N1)
3324 return CombineTo(N, DAG.getConstant(0, DL, VT),
3325 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
3326
3327 // fold (subc x, 0) -> x + no borrow
3328 if (isNullConstant(N1))
3329 return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
3330
3331 // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1) + no borrow
3332 if (isAllOnesConstant(N0))
3333 return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0),
3334 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
3335
3336 return SDValue();
3337}
3338
3339SDValue DAGCombiner::visitSUBO(SDNode *N) {
3340 SDValue N0 = N->getOperand(0);
3341 SDValue N1 = N->getOperand(1);
3342 EVT VT = N0.getValueType();
3343 bool IsSigned = (ISD::SSUBO == N->getOpcode());
3344
3345 EVT CarryVT = N->getValueType(1);
3346 SDLoc DL(N);
3347
3348 // If the flag result is dead, turn this into an SUB.
3349 if (!N->hasAnyUseOfValue(1))
3350 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1),
3351 DAG.getUNDEF(CarryVT));
3352
3353 // fold (subo x, x) -> 0 + no borrow
3354 if (N0 == N1)
3355 return CombineTo(N, DAG.getConstant(0, DL, VT),
3356 DAG.getConstant(0, DL, CarryVT));
3357
3358 ConstantSDNode *N1C = getAsNonOpaqueConstant(N1);
3359
3360 // fold (subox, c) -> (addo x, -c)
3361 if (IsSigned && N1C && !N1C->getAPIntValue().isMinSignedValue()) {
3362 return DAG.getNode(ISD::SADDO, DL, N->getVTList(), N0,
3363 DAG.getConstant(-N1C->getAPIntValue(), DL, VT));
3364 }
3365
3366 // fold (subo x, 0) -> x + no borrow
3367 if (isNullOrNullSplat(N1))
3368 return CombineTo(N, N0, DAG.getConstant(0, DL, CarryVT));
3369
3370 // Canonicalize (usubo -1, x) -> ~x, i.e. (xor x, -1) + no borrow
3371 if (!IsSigned && isAllOnesOrAllOnesSplat(N0))
3372 return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0),
3373 DAG.getConstant(0, DL, CarryVT));
3374
3375 return SDValue();
3376}
3377
3378SDValue DAGCombiner::visitSUBE(SDNode *N) {
3379 SDValue N0 = N->getOperand(0);
3380 SDValue N1 = N->getOperand(1);
3381 SDValue CarryIn = N->getOperand(2);
3382
3383 // fold (sube x, y, false) -> (subc x, y)
3384 if (CarryIn.getOpcode() == ISD::CARRY_FALSE)
3385 return DAG.getNode(ISD::SUBC, SDLoc(N), N->getVTList(), N0, N1);
3386
3387 return SDValue();
3388}
3389
3390SDValue DAGCombiner::visitSUBCARRY(SDNode *N) {
3391 SDValue N0 = N->getOperand(0);
3392 SDValue N1 = N->getOperand(1);
3393 SDValue CarryIn = N->getOperand(2);
3394
3395 // fold (subcarry x, y, false) -> (usubo x, y)
3396 if (isNullConstant(CarryIn)) {
3397 if (!LegalOperations ||
3398 TLI.isOperationLegalOrCustom(ISD::USUBO, N->getValueType(0)))
3399 return DAG.getNode(ISD::USUBO, SDLoc(N), N->getVTList(), N0, N1);
3400 }
3401
3402 return SDValue();
3403}
3404
3405// Notice that "mulfix" can be any of SMULFIX, SMULFIXSAT, UMULFIX and
3406// UMULFIXSAT here.
3407SDValue DAGCombiner::visitMULFIX(SDNode *N) {
3408 SDValue N0 = N->getOperand(0);
3409 SDValue N1 = N->getOperand(1);
3410 SDValue Scale = N->getOperand(2);
3411 EVT VT = N0.getValueType();
3412
3413 // fold (mulfix x, undef, scale) -> 0
3414 if (N0.isUndef() || N1.isUndef())
3415 return DAG.getConstant(0, SDLoc(N), VT);
3416
3417 // Canonicalize constant to RHS (vector doesn't have to splat)
3418 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
3419 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
3420 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N1, N0, Scale);
3421
3422 // fold (mulfix x, 0, scale) -> 0
3423 if (isNullConstant(N1))
3424 return DAG.getConstant(0, SDLoc(N), VT);
3425
3426 return SDValue();
3427}
3428
3429SDValue DAGCombiner::visitMUL(SDNode *N) {
3430 SDValue N0 = N->getOperand(0);
3431 SDValue N1 = N->getOperand(1);
3432 EVT VT = N0.getValueType();
3433
3434 // fold (mul x, undef) -> 0
3435 if (N0.isUndef() || N1.isUndef())
3436 return DAG.getConstant(0, SDLoc(N), VT);
3437
3438 bool N0IsConst = false;
3439 bool N1IsConst = false;
3440 bool N1IsOpaqueConst = false;
3441 bool N0IsOpaqueConst = false;
3442 APInt ConstValue0, ConstValue1;
3443 // fold vector ops
3444 if (VT.isVector()) {
3445 if (SDValue FoldedVOp = SimplifyVBinOp(N))
3446 return FoldedVOp;
3447
3448 N0IsConst = ISD::isConstantSplatVector(N0.getNode(), ConstValue0);
3449 N1IsConst = ISD::isConstantSplatVector(N1.getNode(), ConstValue1);
3450 assert((!N0IsConst ||(((!N0IsConst || ConstValue0.getBitWidth() == VT.getScalarSizeInBits
()) && "Splat APInt should be element width") ? static_cast
<void> (0) : __assert_fail ("(!N0IsConst || ConstValue0.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3452, __PRETTY_FUNCTION__))
3451 ConstValue0.getBitWidth() == VT.getScalarSizeInBits()) &&(((!N0IsConst || ConstValue0.getBitWidth() == VT.getScalarSizeInBits
()) && "Splat APInt should be element width") ? static_cast
<void> (0) : __assert_fail ("(!N0IsConst || ConstValue0.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3452, __PRETTY_FUNCTION__))
3452 "Splat APInt should be element width")(((!N0IsConst || ConstValue0.getBitWidth() == VT.getScalarSizeInBits
()) && "Splat APInt should be element width") ? static_cast
<void> (0) : __assert_fail ("(!N0IsConst || ConstValue0.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3452, __PRETTY_FUNCTION__))
;
3453 assert((!N1IsConst ||(((!N1IsConst || ConstValue1.getBitWidth() == VT.getScalarSizeInBits
()) && "Splat APInt should be element width") ? static_cast
<void> (0) : __assert_fail ("(!N1IsConst || ConstValue1.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3455, __PRETTY_FUNCTION__))
3454 ConstValue1.getBitWidth() == VT.getScalarSizeInBits()) &&(((!N1IsConst || ConstValue1.getBitWidth() == VT.getScalarSizeInBits
()) && "Splat APInt should be element width") ? static_cast
<void> (0) : __assert_fail ("(!N1IsConst || ConstValue1.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3455, __PRETTY_FUNCTION__))
3455 "Splat APInt should be element width")(((!N1IsConst || ConstValue1.getBitWidth() == VT.getScalarSizeInBits
()) && "Splat APInt should be element width") ? static_cast
<void> (0) : __assert_fail ("(!N1IsConst || ConstValue1.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3455, __PRETTY_FUNCTION__))
;
3456 } else {
3457 N0IsConst = isa<ConstantSDNode>(N0);
3458 if (N0IsConst) {
3459 ConstValue0 = cast<ConstantSDNode>(N0)->getAPIntValue();
3460 N0IsOpaqueConst = cast<ConstantSDNode>(N0)->isOpaque();
3461 }
3462 N1IsConst = isa<ConstantSDNode>(N1);
3463 if (N1IsConst) {
3464 ConstValue1 = cast<ConstantSDNode>(N1)->getAPIntValue();
3465 N1IsOpaqueConst = cast<ConstantSDNode>(N1)->isOpaque();
3466 }
3467 }
3468
3469 // fold (mul c1, c2) -> c1*c2
3470 if (N0IsConst && N1IsConst && !N0IsOpaqueConst && !N1IsOpaqueConst)
3471 return DAG.FoldConstantArithmetic(ISD::MUL, SDLoc(N), VT,
3472 N0.getNode(), N1.getNode());
3473
3474 // canonicalize constant to RHS (vector doesn't have to splat)
3475 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
3476 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
3477 return DAG.getNode(ISD::MUL, SDLoc(N), VT, N1, N0);
3478 // fold (mul x, 0) -> 0
3479 if (N1IsConst && ConstValue1.isNullValue())
3480 return N1;
3481 // fold (mul x, 1) -> x
3482 if (N1IsConst && ConstValue1.isOneValue())
3483 return N0;
3484
3485 if (SDValue NewSel = foldBinOpIntoSelect(N))
3486 return NewSel;
3487
3488 // fold (mul x, -1) -> 0-x
3489 if (N1IsConst && ConstValue1.isAllOnesValue()) {
3490 SDLoc DL(N);
3491 return DAG.getNode(ISD::SUB, DL, VT,
3492 DAG.getConstant(0, DL, VT), N0);
3493 }
3494 // fold (mul x, (1 << c)) -> x << c
3495 if (isConstantOrConstantVector(N1, /*NoOpaques*/ true) &&
3496 DAG.isKnownToBeAPowerOfTwo(N1) &&
3497 (!VT.isVector() || Level <= AfterLegalizeVectorOps)) {
3498 SDLoc DL(N);
3499 SDValue LogBase2 = BuildLogBase2(N1, DL);
3500 EVT ShiftVT = getShiftAmountTy(N0.getValueType());
3501 SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ShiftVT);
3502 return DAG.getNode(ISD::SHL, DL, VT, N0, Trunc);
3503 }
3504 // fold (mul x, -(1 << c)) -> -(x << c) or (-x) << c
3505 if (N1IsConst && !N1IsOpaqueConst && (-ConstValue1).isPowerOf2()) {
3506 unsigned Log2Val = (-ConstValue1).logBase2();
3507 SDLoc DL(N);
3508 // FIXME: If the input is something that is easily negated (e.g. a
3509 // single-use add), we should put the negate there.
3510 return DAG.getNode(ISD::SUB, DL, VT,
3511 DAG.getConstant(0, DL, VT),
3512 DAG.getNode(ISD::SHL, DL, VT, N0,
3513 DAG.getConstant(Log2Val, DL,
3514 getShiftAmountTy(N0.getValueType()))));
3515 }
3516
3517 // Try to transform multiply-by-(power-of-2 +/- 1) into shift and add/sub.
3518 // mul x, (2^N + 1) --> add (shl x, N), x
3519 // mul x, (2^N - 1) --> sub (shl x, N), x
3520 // Examples: x * 33 --> (x << 5) + x
3521 // x * 15 --> (x << 4) - x
3522 // x * -33 --> -((x << 5) + x)
3523 // x * -15 --> -((x << 4) - x) ; this reduces --> x - (x << 4)
3524 if (N1IsConst && TLI.decomposeMulByConstant(*DAG.getContext(), VT, N1)) {
3525 // TODO: We could handle more general decomposition of any constant by
3526 // having the target set a limit on number of ops and making a
3527 // callback to determine that sequence (similar to sqrt expansion).
3528 unsigned MathOp = ISD::DELETED_NODE;
3529 APInt MulC = ConstValue1.abs();
3530 if ((MulC - 1).isPowerOf2())
3531 MathOp = ISD::ADD;
3532 else if ((MulC + 1).isPowerOf2())
3533 MathOp = ISD::SUB;
3534
3535 if (MathOp != ISD::DELETED_NODE) {
3536 unsigned ShAmt =
3537 MathOp == ISD::ADD ? (MulC - 1).logBase2() : (MulC + 1).logBase2();
3538 assert(ShAmt < VT.getScalarSizeInBits() &&((ShAmt < VT.getScalarSizeInBits() && "multiply-by-constant generated out of bounds shift"
) ? static_cast<void> (0) : __assert_fail ("ShAmt < VT.getScalarSizeInBits() && \"multiply-by-constant generated out of bounds shift\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3539, __PRETTY_FUNCTION__))
3539 "multiply-by-constant generated out of bounds shift")((ShAmt < VT.getScalarSizeInBits() && "multiply-by-constant generated out of bounds shift"
) ? static_cast<void> (0) : __assert_fail ("ShAmt < VT.getScalarSizeInBits() && \"multiply-by-constant generated out of bounds shift\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3539, __PRETTY_FUNCTION__))
;
3540 SDLoc DL(N);
3541 SDValue Shl =
3542 DAG.getNode(ISD::SHL, DL, VT, N0, DAG.getConstant(ShAmt, DL, VT));
3543 SDValue R = DAG.getNode(MathOp, DL, VT, Shl, N0);
3544 if (ConstValue1.isNegative())
3545 R = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), R);
3546 return R;
3547 }
3548 }
3549
3550 // (mul (shl X, c1), c2) -> (mul X, c2 << c1)
3551 if (N0.getOpcode() == ISD::SHL &&
3552 isConstantOrConstantVector(N1, /* NoOpaques */ true) &&
3553 isConstantOrConstantVector(N0.getOperand(1), /* NoOpaques */ true)) {
3554 SDValue C3 = DAG.getNode(ISD::SHL, SDLoc(N), VT, N1, N0.getOperand(1));
3555 if (isConstantOrConstantVector(C3))
3556 return DAG.getNode(ISD::MUL, SDLoc(N), VT, N0.getOperand(0), C3);
3557 }
3558
3559 // Change (mul (shl X, C), Y) -> (shl (mul X, Y), C) when the shift has one
3560 // use.
3561 {
3562 SDValue Sh(nullptr, 0), Y(nullptr, 0);
3563
3564 // Check for both (mul (shl X, C), Y) and (mul Y, (shl X, C)).
3565 if (N0.getOpcode() == ISD::SHL &&
3566 isConstantOrConstantVector(N0.getOperand(1)) &&
3567 N0.getNode()->hasOneUse()) {
3568 Sh = N0; Y = N1;
3569 } else if (N1.getOpcode() == ISD::SHL &&
3570 isConstantOrConstantVector(N1.getOperand(1)) &&
3571 N1.getNode()->hasOneUse()) {
3572 Sh = N1; Y = N0;
3573 }
3574
3575 if (Sh.getNode()) {
3576 SDValue Mul = DAG.getNode(ISD::MUL, SDLoc(N), VT, Sh.getOperand(0), Y);
3577 return DAG.getNode(ISD::SHL, SDLoc(N), VT, Mul, Sh.getOperand(1));
3578 }
3579 }
3580
3581 // fold (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2)
3582 if (DAG.isConstantIntBuildVectorOrConstantInt(N1) &&
3583 N0.getOpcode() == ISD::ADD &&
3584 DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1)) &&
3585 isMulAddWithConstProfitable(N, N0, N1))
3586 return DAG.getNode(ISD::ADD, SDLoc(N), VT,
3587 DAG.getNode(ISD::MUL, SDLoc(N0), VT,
3588 N0.getOperand(0), N1),
3589 DAG.getNode(ISD::MUL, SDLoc(N1), VT,
3590 N0.getOperand(1), N1));
3591
3592 // reassociate mul
3593 if (SDValue RMUL = reassociateOps(ISD::MUL, SDLoc(N), N0, N1, N->getFlags()))
3594 return RMUL;
3595
3596 return SDValue();
3597}
3598
3599/// Return true if divmod libcall is available.
3600static bool isDivRemLibcallAvailable(SDNode *Node, bool isSigned,
3601 const TargetLowering &TLI) {
3602 RTLIB::Libcall LC;
3603 EVT NodeType = Node->getValueType(0);
3604 if (!NodeType.isSimple())
3605 return false;
3606 switch (NodeType.getSimpleVT().SimpleTy) {
3607 default: return false; // No libcall for vector types.
3608 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
3609 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
3610 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
3611 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
3612 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
3613 }
3614
3615 return TLI.getLibcallName(LC) != nullptr;
3616}
3617
3618/// Issue divrem if both quotient and remainder are needed.
3619SDValue DAGCombiner::useDivRem(SDNode *Node) {
3620 if (Node->use_empty())
3621 return SDValue(); // This is a dead node, leave it alone.
3622
3623 unsigned Opcode = Node->getOpcode();
3624 bool isSigned = (Opcode == ISD::SDIV) || (Opcode == ISD::SREM);
3625 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3626
3627 // DivMod lib calls can still work on non-legal types if using lib-calls.
3628 EVT VT = Node->getValueType(0);
3629 if (VT.isVector() || !VT.isInteger())
3630 return SDValue();
3631
3632 if (!TLI.isTypeLegal(VT) && !TLI.isOperationCustom(DivRemOpc, VT))
3633 return SDValue();
3634
3635 // If DIVREM is going to get expanded into a libcall,
3636 // but there is no libcall available, then don't combine.
3637 if (!TLI.isOperationLegalOrCustom(DivRemOpc, VT) &&
3638 !isDivRemLibcallAvailable(Node, isSigned, TLI))
3639 return SDValue();
3640
3641 // If div is legal, it's better to do the normal expansion
3642 unsigned OtherOpcode = 0;
3643 if ((Opcode == ISD::SDIV) || (Opcode == ISD::UDIV)) {
3644 OtherOpcode = isSigned ? ISD::SREM : ISD::UREM;
3645 if (TLI.isOperationLegalOrCustom(Opcode, VT))
3646 return SDValue();
3647 } else {
3648 OtherOpcode = isSigned ? ISD::SDIV : ISD::UDIV;
3649 if (TLI.isOperationLegalOrCustom(OtherOpcode, VT))
3650 return SDValue();
3651 }
3652
3653 SDValue Op0 = Node->getOperand(0);
3654 SDValue Op1 = Node->getOperand(1);
3655 SDValue combined;
3656 for (SDNode::use_iterator UI = Op0.getNode()->use_begin(),
3657 UE = Op0.getNode()->use_end(); UI != UE; ++UI) {
3658 SDNode *User = *UI;
3659 if (User == Node || User->getOpcode() == ISD::DELETED_NODE ||
3660 User->use_empty())
3661 continue;
3662 // Convert the other matching node(s), too;
3663 // otherwise, the DIVREM may get target-legalized into something
3664 // target-specific that we won't be able to recognize.
3665 unsigned UserOpc = User->getOpcode();
3666 if ((UserOpc == Opcode || UserOpc == OtherOpcode || UserOpc == DivRemOpc) &&
3667 User->getOperand(0) == Op0 &&
3668 User->getOperand(1) == Op1) {
3669 if (!combined) {
3670 if (UserOpc == OtherOpcode) {
3671 SDVTList VTs = DAG.getVTList(VT, VT);
3672 combined = DAG.getNode(DivRemOpc, SDLoc(Node), VTs, Op0, Op1);
3673 } else if (UserOpc == DivRemOpc) {
3674 combined = SDValue(User, 0);
3675 } else {
3676 assert(UserOpc == Opcode)((UserOpc == Opcode) ? static_cast<void> (0) : __assert_fail
("UserOpc == Opcode", "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3676, __PRETTY_FUNCTION__))
;
3677 continue;
3678 }
3679 }
3680 if (UserOpc == ISD::SDIV || UserOpc == ISD::UDIV)
3681 CombineTo(User, combined);
3682 else if (UserOpc == ISD::SREM || UserOpc == ISD::UREM)
3683 CombineTo(User, combined.getValue(1));
3684 }
3685 }
3686 return combined;
3687}
3688
3689static SDValue simplifyDivRem(SDNode *N, SelectionDAG &DAG) {
3690 SDValue N0 = N->getOperand(0);
3691 SDValue N1 = N->getOperand(1);
3692 EVT VT = N->getValueType(0);
3693 SDLoc DL(N);
3694
3695 unsigned Opc = N->getOpcode();
3696 bool IsDiv = (ISD::SDIV == Opc) || (ISD::UDIV == Opc);
3697 ConstantSDNode *N1C = isConstOrConstSplat(N1);
3698
3699 // X / undef -> undef
3700 // X % undef -> undef
3701 // X / 0 -> undef
3702 // X % 0 -> undef
3703 // NOTE: This includes vectors where any divisor element is zero/undef.
3704 if (DAG.isUndef(Opc, {N0, N1}))
3705 return DAG.getUNDEF(VT);
3706
3707 // undef / X -> 0
3708 // undef % X -> 0
3709 if (N0.isUndef())
3710 return DAG.getConstant(0, DL, VT);
3711
3712 // 0 / X -> 0
3713 // 0 % X -> 0
3714 ConstantSDNode *N0C = isConstOrConstSplat(N0);
3715 if (N0C && N0C->isNullValue())
3716 return N0;
3717
3718 // X / X -> 1
3719 // X % X -> 0
3720 if (N0 == N1)
3721 return DAG.getConstant(IsDiv ? 1 : 0, DL, VT);
3722
3723 // X / 1 -> X
3724 // X % 1 -> 0
3725 // If this is a boolean op (single-bit element type), we can't have
3726 // division-by-zero or remainder-by-zero, so assume the divisor is 1.
3727 // TODO: Similarly, if we're zero-extending a boolean divisor, then assume
3728 // it's a 1.
3729 if ((N1C && N1C->isOne()) || (VT.getScalarType() == MVT::i1))
3730 return IsDiv ? N0 : DAG.getConstant(0, DL, VT);
3731
3732 return SDValue();
3733}
3734
3735SDValue DAGCombiner::visitSDIV(SDNode *N) {
3736 SDValue N0 = N->getOperand(0);
3737 SDValue N1 = N->getOperand(1);
3738 EVT VT = N->getValueType(0);
3739 EVT CCVT = getSetCCResultType(VT);
3740
3741 // fold vector ops
3742 if (VT.isVector())
3743 if (SDValue FoldedVOp = SimplifyVBinOp(N))
3744 return FoldedVOp;
3745
3746 SDLoc DL(N);
3747
3748 // fold (sdiv c1, c2) -> c1/c2
3749 ConstantSDNode *N0C = isConstOrConstSplat(N0);
3750 ConstantSDNode *N1C = isConstOrConstSplat(N1);
3751 if (N0C && N1C && !N0C->isOpaque() && !N1C->isOpaque())
3752 return DAG.FoldConstantArithmetic(ISD::SDIV, DL, VT, N0C, N1C);
3753 // fold (sdiv X, -1) -> 0-X
3754 if (N1C && N1C->isAllOnesValue())
3755 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), N0);
3756 // fold (sdiv X, MIN_SIGNED) -> select(X == MIN_SIGNED, 1, 0)
3757 if (N1C && N1C->getAPIntValue().isMinSignedValue())
3758 return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ),
3759 DAG.getConstant(1, DL, VT),
3760 DAG.getConstant(0, DL, VT));
3761
3762 if (SDValue V = simplifyDivRem(N, DAG))
3763 return V;
3764
3765 if (SDValue NewSel = foldBinOpIntoSelect(N))
3766 return NewSel;
3767
3768 // If we know the sign bits of both operands are zero, strength reduce to a
3769 // udiv instead. Handles (X&15) /s 4 -> X&15 >> 2
3770 if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
3771 return DAG.getNode(ISD::UDIV, DL, N1.getValueType(), N0, N1);
3772
3773 if (SDValue V = visitSDIVLike(N0, N1, N)) {
3774 // If the corresponding remainder node exists, update its users with
3775 // (Dividend - (Quotient * Divisor).
3776 if (SDNode *RemNode = DAG.getNodeIfExists(ISD::SREM, N->getVTList(),
3777 { N0, N1 })) {
3778 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, V, N1);
3779 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
3780 AddToWorklist(Mul.getNode());
3781 AddToWorklist(Sub.getNode());
3782 CombineTo(RemNode, Sub);
3783 }
3784 return V;
3785 }
3786
3787 // sdiv, srem -> sdivrem
3788 // If the divisor is constant, then return DIVREM only if isIntDivCheap() is
3789 // true. Otherwise, we break the simplification logic in visitREM().
3790 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
3791 if (!N1C || TLI.isIntDivCheap(N->getValueType(0), Attr))
3792 if (SDValue DivRem = useDivRem(N))
3793 return DivRem;
3794
3795 return SDValue();
3796}
3797
3798SDValue DAGCombiner::visitSDIVLike(SDValue N0, SDValue N1, SDNode *N) {
3799 SDLoc DL(N);
3800 EVT VT = N->getValueType(0);
3801 EVT CCVT = getSetCCResultType(VT);
3802 unsigned BitWidth = VT.getScalarSizeInBits();
3803
3804 // Helper for determining whether a value is a power-2 constant scalar or a
3805 // vector of such elements.
3806 auto IsPowerOfTwo = [](ConstantSDNode *C) {
3807 if (C->isNullValue() || C->isOpaque())
3808 return false;
3809 if (C->getAPIntValue().isPowerOf2())
3810 return true;
3811 if ((-C->getAPIntValue()).isPowerOf2())
3812 return true;
3813 return false;
3814 };
3815
3816 // fold (sdiv X, pow2) -> simple ops after legalize
3817 // FIXME: We check for the exact bit here because the generic lowering gives
3818 // better results in that case. The target-specific lowering should learn how
3819 // to handle exact sdivs efficiently.
3820 if (!N->getFlags().hasExact() && ISD::matchUnaryPredicate(N1, IsPowerOfTwo)) {
3821 // Target-specific implementation of sdiv x, pow2.
3822 if (SDValue Res = BuildSDIVPow2(N))
3823 return Res;
3824
3825 // Create constants that are functions of the shift amount value.
3826 EVT ShiftAmtTy = getShiftAmountTy(N0.getValueType());
3827 SDValue Bits = DAG.getConstant(BitWidth, DL, ShiftAmtTy);
3828 SDValue C1 = DAG.getNode(ISD::CTTZ, DL, VT, N1);
3829 C1 = DAG.getZExtOrTrunc(C1, DL, ShiftAmtTy);
3830 SDValue Inexact = DAG.getNode(ISD::SUB, DL, ShiftAmtTy, Bits, C1);
3831 if (!isConstantOrConstantVector(Inexact))
3832 return SDValue();
3833
3834 // Splat the sign bit into the register
3835 SDValue Sign = DAG.getNode(ISD::SRA, DL, VT, N0,
3836 DAG.getConstant(BitWidth - 1, DL, ShiftAmtTy));
3837 AddToWorklist(Sign.getNode());
3838
3839 // Add (N0 < 0) ? abs2 - 1 : 0;
3840 SDValue Srl = DAG.getNode(ISD::SRL, DL, VT, Sign, Inexact);
3841 AddToWorklist(Srl.getNode());
3842 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Srl);
3843 AddToWorklist(Add.getNode());
3844 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Add, C1);
3845 AddToWorklist(Sra.getNode());
3846
3847 // Special case: (sdiv X, 1) -> X
3848 // Special Case: (sdiv X, -1) -> 0-X
3849 SDValue One = DAG.getConstant(1, DL, VT);
3850 SDValue AllOnes = DAG.getAllOnesConstant(DL, VT);
3851 SDValue IsOne = DAG.getSetCC(DL, CCVT, N1, One, ISD::SETEQ);
3852 SDValue IsAllOnes = DAG.getSetCC(DL, CCVT, N1, AllOnes, ISD::SETEQ);
3853 SDValue IsOneOrAllOnes = DAG.getNode(ISD::OR, DL, CCVT, IsOne, IsAllOnes);
3854 Sra = DAG.getSelect(DL, VT, IsOneOrAllOnes, N0, Sra);
3855
3856 // If dividing by a positive value, we're done. Otherwise, the result must
3857 // be negated.
3858 SDValue Zero = DAG.getConstant(0, DL, VT);
3859 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, Zero, Sra);
3860
3861 // FIXME: Use SELECT_CC once we improve SELECT_CC constant-folding.
3862 SDValue IsNeg = DAG.getSetCC(DL, CCVT, N1, Zero, ISD::SETLT);
3863 SDValue Res = DAG.getSelect(DL, VT, IsNeg, Sub, Sra);
3864 return Res;
3865 }
3866
3867 // If integer divide is expensive and we satisfy the requirements, emit an
3868 // alternate sequence. Targets may check function attributes for size/speed
3869 // trade-offs.
3870 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
3871 if (isConstantOrConstantVector(N1) &&
3872 !TLI.isIntDivCheap(N->getValueType(0), Attr))
3873 if (SDValue Op = BuildSDIV(N))
3874 return Op;
3875
3876 return SDValue();
3877}
3878
3879SDValue DAGCombiner::visitUDIV(SDNode *N) {
3880 SDValue N0 = N->getOperand(0);
3881 SDValue N1 = N->getOperand(1);
3882 EVT VT = N->getValueType(0);
3883 EVT CCVT = getSetCCResultType(VT);
3884
3885 // fold vector ops
3886 if (VT.isVector())
3887 if (SDValue FoldedVOp = SimplifyVBinOp(N))
3888 return FoldedVOp;
3889
3890 SDLoc DL(N);
3891
3892 // fold (udiv c1, c2) -> c1/c2
3893 ConstantSDNode *N0C = isConstOrConstSplat(N0);
3894 ConstantSDNode *N1C = isConstOrConstSplat(N1);
3895 if (N0C && N1C)
3896 if (SDValue Folded = DAG.FoldConstantArithmetic(ISD::UDIV, DL, VT,
3897 N0C, N1C))
3898 return Folded;
3899 // fold (udiv X, -1) -> select(X == -1, 1, 0)
3900 if (N1C && N1C->getAPIntValue().isAllOnesValue())
3901 return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ),
3902 DAG.getConstant(1, DL, VT),
3903 DAG.getConstant(0, DL, VT));
3904
3905 if (SDValue V = simplifyDivRem(N, DAG))
3906 return V;
3907
3908 if (SDValue NewSel = foldBinOpIntoSelect(N))
3909 return NewSel;
3910
3911 if (SDValue V = visitUDIVLike(N0, N1, N)) {
3912 // If the corresponding remainder node exists, update its users with
3913 // (Dividend - (Quotient * Divisor).
3914 if (SDNode *RemNode = DAG.getNodeIfExists(ISD::UREM, N->getVTList(),
3915 { N0, N1 })) {
3916 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, V, N1);
3917 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
3918 AddToWorklist(Mul.getNode());
3919 AddToWorklist(Sub.getNode());
3920 CombineTo(RemNode, Sub);
3921 }
3922 return V;
3923 }
3924
3925 // sdiv, srem -> sdivrem
3926 // If the divisor is constant, then return DIVREM only if isIntDivCheap() is
3927 // true. Otherwise, we break the simplification logic in visitREM().
3928 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
3929 if (!N1C || TLI.isIntDivCheap(N->getValueType(0), Attr))
3930 if (SDValue DivRem = useDivRem(N))
3931 return DivRem;
3932
3933 return SDValue();
3934}
3935
3936SDValue DAGCombiner::visitUDIVLike(SDValue N0, SDValue N1, SDNode *N) {
3937 SDLoc DL(N);
3938 EVT VT = N->getValueType(0);
3939
3940 // fold (udiv x, (1 << c)) -> x >>u c
3941 if (isConstantOrConstantVector(N1, /*NoOpaques*/ true) &&
3942 DAG.isKnownToBeAPowerOfTwo(N1)) {
3943 SDValue LogBase2 = BuildLogBase2(N1, DL);
3944 AddToWorklist(LogBase2.getNode());
3945
3946 EVT ShiftVT = getShiftAmountTy(N0.getValueType());
3947 SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ShiftVT);
3948 AddToWorklist(Trunc.getNode());
3949 return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc);
3950 }
3951
3952 // fold (udiv x, (shl c, y)) -> x >>u (log2(c)+y) iff c is power of 2
3953 if (N1.getOpcode() == ISD::SHL) {
3954 SDValue N10 = N1.getOperand(0);
3955 if (isConstantOrConstantVector(N10, /*NoOpaques*/ true) &&
3956 DAG.isKnownToBeAPowerOfTwo(N10)) {
3957 SDValue LogBase2 = BuildLogBase2(N10, DL);
3958 AddToWorklist(LogBase2.getNode());
3959
3960 EVT ADDVT = N1.getOperand(1).getValueType();
3961 SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ADDVT);
3962 AddToWorklist(Trunc.getNode());
3963 SDValue Add = DAG.getNode(ISD::ADD, DL, ADDVT, N1.getOperand(1), Trunc);
3964 AddToWorklist(Add.getNode());
3965 return DAG.getNode(ISD::SRL, DL, VT, N0, Add);
3966 }
3967 }
3968
3969 // fold (udiv x, c) -> alternate
3970 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
3971 if (isConstantOrConstantVector(N1) &&
3972 !TLI.isIntDivCheap(N->getValueType(0), Attr))
3973 if (SDValue Op = BuildUDIV(N))
3974 return Op;
3975
3976 return SDValue();
3977}
3978
3979// handles ISD::SREM and ISD::UREM
3980SDValue DAGCombiner::visitREM(SDNode *N) {
3981 unsigned Opcode = N->getOpcode();
3982 SDValue N0 = N->getOperand(0);
3983 SDValue N1 = N->getOperand(1);
3984 EVT VT = N->getValueType(0);
3985 EVT CCVT = getSetCCResultType(VT);
3986
3987 bool isSigned = (Opcode == ISD::SREM);
3988 SDLoc DL(N);
3989
3990 // fold (rem c1, c2) -> c1%c2
3991 ConstantSDNode *N0C = isConstOrConstSplat(N0);
3992 ConstantSDNode *N1C = isConstOrConstSplat(N1);
3993 if (N0C && N1C)
3994 if (SDValue Folded = DAG.FoldConstantArithmetic(Opcode, DL, VT, N0C, N1C))
3995 return Folded;
3996 // fold (urem X, -1) -> select(X == -1, 0, x)
3997 if (!isSigned && N1C && N1C->getAPIntValue().isAllOnesValue())
3998 return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ),
3999 DAG.getConstant(0, DL, VT), N0);
4000
4001 if (SDValue V = simplifyDivRem(N, DAG))
4002 return V;
4003
4004 if (SDValue NewSel = foldBinOpIntoSelect(N))
4005 return NewSel;
4006
4007 if (isSigned) {
4008 // If we know the sign bits of both operands are zero, strength reduce to a
4009 // urem instead. Handles (X & 0x0FFFFFFF) %s 16 -> X&15
4010 if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
4011 return DAG.getNode(ISD::UREM, DL, VT, N0, N1);
4012 } else {
4013 SDValue NegOne = DAG.getAllOnesConstant(DL, VT);
4014 if (DAG.isKnownToBeAPowerOfTwo(N1)) {
4015 // fold (urem x, pow2) -> (and x, pow2-1)
4016 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne);
4017 AddToWorklist(Add.getNode());
4018 return DAG.getNode(ISD::AND, DL, VT, N0, Add);
4019 }
4020 if (N1.getOpcode() == ISD::SHL &&
4021 DAG.isKnownToBeAPowerOfTwo(N1.getOperand(0))) {
4022 // fold (urem x, (shl pow2, y)) -> (and x, (add (shl pow2, y), -1))
4023 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne);
4024 AddToWorklist(Add.getNode());
4025 return DAG.getNode(ISD::AND, DL, VT, N0, Add);
4026 }
4027 }
4028
4029 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
4030
4031 // If X/C can be simplified by the division-by-constant logic, lower
4032 // X%C to the equivalent of X-X/C*C.
4033 // Reuse the SDIVLike/UDIVLike combines - to avoid mangling nodes, the
4034 // speculative DIV must not cause a DIVREM conversion. We guard against this
4035 // by skipping the simplification if isIntDivCheap(). When div is not cheap,
4036 // combine will not return a DIVREM. Regardless, checking cheapness here
4037 // makes sense since the simplification results in fatter code.
4038 if (DAG.isKnownNeverZero(N1) && !TLI.isIntDivCheap(VT, Attr)) {
4039 SDValue OptimizedDiv =
4040 isSigned ? visitSDIVLike(N0, N1, N) : visitUDIVLike(N0, N1, N);
4041 if (OptimizedDiv.getNode()) {
4042 // If the equivalent Div node also exists, update its users.
4043 unsigned DivOpcode = isSigned ? ISD::SDIV : ISD::UDIV;
4044 if (SDNode *DivNode = DAG.getNodeIfExists(DivOpcode, N->getVTList(),
4045 { N0, N1 }))
4046 CombineTo(DivNode, OptimizedDiv);
4047 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, OptimizedDiv, N1);
4048 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
4049 AddToWorklist(OptimizedDiv.getNode());
4050 AddToWorklist(Mul.getNode());
4051 return Sub;
4052 }
4053 }
4054
4055 // sdiv, srem -> sdivrem
4056 if (SDValue DivRem = useDivRem(N))
4057 return DivRem.getValue(1);
4058
4059 return SDValue();
4060}
4061
4062SDValue DAGCombiner::visitMULHS(SDNode *N) {
4063 SDValue N0 = N->getOperand(0);
4064 SDValue N1 = N->getOperand(1);
4065 EVT VT = N->getValueType(0);
4066 SDLoc DL(N);
4067
4068 if (VT.isVector()) {
4069 // fold (mulhs x, 0) -> 0
4070 // do not return N0/N1, because undef node may exist.
4071 if (ISD::isBuildVectorAllZeros(N0.getNode()) ||
4072 ISD::isBuildVectorAllZeros(N1.getNode()))
4073 return DAG.getConstant(0, DL, VT);
4074 }
4075
4076 // fold (mulhs x, 0) -> 0
4077 if (isNullConstant(N1))
4078 return N1;
4079 // fold (mulhs x, 1) -> (sra x, size(x)-1)
4080 if (isOneConstant(N1))
4081 return DAG.getNode(ISD::SRA, DL, N0.getValueType(), N0,
4082 DAG.getConstant(N0.getScalarValueSizeInBits() - 1, DL,
4083 getShiftAmountTy(N0.getValueType())));
4084
4085 // fold (mulhs x, undef) -> 0
4086 if (N0.isUndef() || N1.isUndef())
4087 return DAG.getConstant(0, DL, VT);
4088
4089 // If the type twice as wide is legal, transform the mulhs to a wider multiply
4090 // plus a shift.
4091 if (VT.isSimple() && !VT.isVector()) {
4092 MVT Simple = VT.getSimpleVT();
4093 unsigned SimpleSize = Simple.getSizeInBits();
4094 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
4095 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
4096 N0 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N0);
4097 N1 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N1);
4098 N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
4099 N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
4100 DAG.getConstant(SimpleSize, DL,
4101 getShiftAmountTy(N1.getValueType())));
4102 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
4103 }
4104 }
4105
4106 return SDValue();
4107}
4108
4109SDValue DAGCombiner::visitMULHU(SDNode *N) {
4110 SDValue N0 = N->getOperand(0);
4111 SDValue N1 = N->getOperand(1);
4112 EVT VT = N->getValueType(0);
4113 SDLoc DL(N);
4114
4115 if (VT.isVector()) {
4116 // fold (mulhu x, 0) -> 0
4117 // do not return N0/N1, because undef node may exist.
4118 if (ISD::isBuildVectorAllZeros(N0.getNode()) ||
4119 ISD::isBuildVectorAllZeros(N1.getNode()))
4120 return DAG.getConstant(0, DL, VT);
4121 }
4122
4123 // fold (mulhu x, 0) -> 0
4124 if (isNullConstant(N1))
4125 return N1;
4126 // fold (mulhu x, 1) -> 0
4127 if (isOneConstant(N1))
4128 return DAG.getConstant(0, DL, N0.getValueType());
4129 // fold (mulhu x, undef) -> 0
4130 if (N0.isUndef() || N1.isUndef())
4131 return DAG.getConstant(0, DL, VT);
4132
4133 // fold (mulhu x, (1 << c)) -> x >> (bitwidth - c)
4134 if (isConstantOrConstantVector(N1, /*NoOpaques*/ true) &&
4135 DAG.isKnownToBeAPowerOfTwo(N1) && hasOperation(ISD::SRL, VT)) {
4136 unsigned NumEltBits = VT.getScalarSizeInBits();
4137 SDValue LogBase2 = BuildLogBase2(N1, DL);
4138 SDValue SRLAmt = DAG.getNode(
4139 ISD::SUB, DL, VT, DAG.getConstant(NumEltBits, DL, VT), LogBase2);
4140 EVT ShiftVT = getShiftAmountTy(N0.getValueType());
4141 SDValue Trunc = DAG.getZExtOrTrunc(SRLAmt, DL, ShiftVT);
4142 return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc);
4143 }
4144
4145 // If the type twice as wide is legal, transform the mulhu to a wider multiply
4146 // plus a shift.
4147 if (VT.isSimple() && !VT.isVector()) {
4148 MVT Simple = VT.getSimpleVT();
4149 unsigned SimpleSize = Simple.getSizeInBits();
4150 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
4151 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
4152 N0 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N0);
4153 N1 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N1);
4154 N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
4155 N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
4156 DAG.getConstant(SimpleSize, DL,
4157 getShiftAmountTy(N1.getValueType())));
4158 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
4159 }
4160 }
4161
4162 return SDValue();
4163}
4164
4165/// Perform optimizations common to nodes that compute two values. LoOp and HiOp
4166/// give the opcodes for the two computations that are being performed. Return
4167/// true if a simplification was made.
4168SDValue DAGCombiner::SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
4169 unsigned HiOp) {
4170 // If the high half is not needed, just compute the low half.
4171 bool HiExists = N->hasAnyUseOfValue(1);
4172 if (!HiExists && (!LegalOperations ||
4173 TLI.isOperationLegalOrCustom(LoOp, N->getValueType(0)))) {
4174 SDValue Res = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0), N->ops());
4175 return CombineTo(N, Res, Res);
4176 }
4177
4178 // If the low half is not needed, just compute the high half.
4179 bool LoExists = N->hasAnyUseOfValue(0);
4180 if (!LoExists && (!LegalOperations ||
4181 TLI.isOperationLegalOrCustom(HiOp, N->getValueType(1)))) {
4182 SDValue Res = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1), N->ops());
4183 return CombineTo(N, Res, Res);
4184 }
4185
4186 // If both halves are used, return as it is.
4187 if (LoExists && HiExists)
4188 return SDValue();
4189
4190 // If the two computed results can be simplified separately, separate them.
4191 if (LoExists) {
4192 SDValue Lo = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0), N->ops());
4193 AddToWorklist(Lo.getNode());
4194 SDValue LoOpt = combine(Lo.getNode());
4195 if (LoOpt.getNode() && LoOpt.getNode() != Lo.getNode() &&
4196 (!LegalOperations ||
4197 TLI.isOperationLegalOrCustom(LoOpt.getOpcode(), LoOpt.getValueType())))
4198 return CombineTo(N, LoOpt, LoOpt);
4199 }
4200
4201 if (HiExists) {
4202 SDValue Hi = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1), N->ops());
4203 AddToWorklist(Hi.getNode());
4204 SDValue HiOpt = combine(Hi.getNode());
4205 if (HiOpt.getNode() && HiOpt != Hi &&
4206 (!LegalOperations ||
4207 TLI.isOperationLegalOrCustom(HiOpt.getOpcode(), HiOpt.getValueType())))
4208 return CombineTo(N, HiOpt, HiOpt);
4209 }
4210
4211 return SDValue();
4212}
4213
4214SDValue DAGCombiner::visitSMUL_LOHI(SDNode *N) {
4215 if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHS))
4216 return Res;
4217
4218 EVT VT = N->getValueType(0);
4219 SDLoc DL(N);
4220
4221 // If the type is twice as wide is legal, transform the mulhu to a wider
4222 // multiply plus a shift.
4223 if (VT.isSimple() && !VT.isVector()) {
4224 MVT Simple = VT.getSimpleVT();
4225 unsigned SimpleSize = Simple.getSizeInBits();
4226 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
4227 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
4228 SDValue Lo = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(0));
4229 SDValue Hi = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(1));
4230 Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
4231 // Compute the high part as N1.
4232 Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
4233 DAG.getConstant(SimpleSize, DL,
4234 getShiftAmountTy(Lo.getValueType())));
4235 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
4236 // Compute the low part as N0.
4237 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
4238 return CombineTo(N, Lo, Hi);
4239 }
4240 }
4241
4242 return SDValue();
4243}
4244
4245SDValue DAGCombiner::visitUMUL_LOHI(SDNode *N) {
4246 if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHU))
4247 return Res;
4248
4249 EVT VT = N->getValueType(0);
4250 SDLoc DL(N);
4251
4252 // (umul_lohi N0, 0) -> (0, 0)
4253 if (isNullConstant(N->getOperand(1))) {
4254 SDValue Zero = DAG.getConstant(0, DL, VT);
4255 return CombineTo(N, Zero, Zero);
4256 }
4257
4258 // (umul_lohi N0, 1) -> (N0, 0)
4259 if (isOneConstant(N->getOperand(1))) {
4260 SDValue Zero = DAG.getConstant(0, DL, VT);
4261 return CombineTo(N, N->getOperand(0), Zero);
4262 }
4263
4264 // If the type is twice as wide is legal, transform the mulhu to a wider
4265 // multiply plus a shift.
4266 if (VT.isSimple() && !VT.isVector()) {
4267 MVT Simple = VT.getSimpleVT();
4268 unsigned SimpleSize = Simple.getSizeInBits();
4269 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
4270 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
4271 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(0));
4272 SDValue Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(1));
4273 Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
4274 // Compute the high part as N1.
4275 Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
4276 DAG.getConstant(SimpleSize, DL,
4277 getShiftAmountTy(Lo.getValueType())));
4278 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
4279 // Compute the low part as N0.
4280 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
4281 return CombineTo(N, Lo, Hi);
4282 }
4283 }
4284
4285 return SDValue();
4286}
4287
4288SDValue DAGCombiner::visitMULO(SDNode *N) {
4289 SDValue N0 = N->getOperand(0);
4290 SDValue N1 = N->getOperand(1);
4291 EVT VT = N0.getValueType();
4292 bool IsSigned = (ISD::SMULO == N->getOpcode());
4293
4294 EVT CarryVT = N->getValueType(1);
4295 SDLoc DL(N);
4296
4297 // canonicalize constant to RHS.
4298 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
4299 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
4300 return DAG.getNode(N->getOpcode(), DL, N->getVTList(), N1, N0);
4301
4302 // fold (mulo x, 0) -> 0 + no carry out
4303 if (isNullOrNullSplat(N1))
4304 return CombineTo(N, DAG.getConstant(0, DL, VT),
4305 DAG.getConstant(0, DL, CarryVT));
4306
4307 // (mulo x, 2) -> (addo x, x)
4308 if (ConstantSDNode *C2 = isConstOrConstSplat(N1))
4309 if (C2->getAPIntValue() == 2)
4310 return DAG.getNode(IsSigned ? ISD::SADDO : ISD::UADDO, DL,
4311 N->getVTList(), N0, N0);
4312
4313 return SDValue();
4314}
4315
4316SDValue DAGCombiner::visitIMINMAX(SDNode *N) {
4317 SDValue N0 = N->getOperand(0);
4318 SDValue N1 = N->getOperand(1);
4319 EVT VT = N0.getValueType();
4320
4321 // fold vector ops
4322 if (VT.isVector())
4323 if (SDValue FoldedVOp = SimplifyVBinOp(N))
4324 return FoldedVOp;
4325
4326 // fold operation with constant operands.
4327 ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
4328 ConstantSDNode *N1C = getAsNonOpaqueConstant(N1);
4329 if (N0C && N1C)
4330 return DAG.FoldConstantArithmetic(N->getOpcode(), SDLoc(N), VT, N0C, N1C);
4331
4332 // canonicalize constant to RHS
4333 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
4334 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
4335 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N1, N0);
4336
4337 // Is sign bits are zero, flip between UMIN/UMAX and SMIN/SMAX.
4338 // Only do this if the current op isn't legal and the flipped is.
4339 unsigned Opcode = N->getOpcode();
4340 if (!TLI.isOperationLegal(Opcode, VT) &&
4341 (N0.isUndef() || DAG.SignBitIsZero(N0)) &&
4342 (N1.isUndef() || DAG.SignBitIsZero(N1))) {
4343 unsigned AltOpcode;
4344 switch (Opcode) {
4345 case ISD::SMIN: AltOpcode = ISD::UMIN; break;
4346 case ISD::SMAX: AltOpcode = ISD::UMAX; break;
4347 case ISD::UMIN: AltOpcode = ISD::SMIN; break;
4348 case ISD::UMAX: AltOpcode = ISD::SMAX; break;
4349 default: llvm_unreachable("Unknown MINMAX opcode")::llvm::llvm_unreachable_internal("Unknown MINMAX opcode", "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4349)
;
4350 }
4351 if (TLI.isOperationLegal(AltOpcode, VT))
4352 return DAG.getNode(AltOpcode, SDLoc(N), VT, N0, N1);
4353 }
4354
4355 return SDValue();
4356}
4357
4358/// If this is a bitwise logic instruction and both operands have the same
4359/// opcode, try to sink the other opcode after the logic instruction.
4360SDValue DAGCombiner::hoistLogicOpWithSameOpcodeHands(SDNode *N) {
4361 SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
4362 EVT VT = N0.getValueType();
4363 unsigned LogicOpcode = N->getOpcode();
4364 unsigned HandOpcode = N0.getOpcode();
4365 assert((LogicOpcode == ISD::AND || LogicOpcode == ISD::OR ||(((LogicOpcode == ISD::AND || LogicOpcode == ISD::OR || LogicOpcode
== ISD::XOR) && "Expected logic opcode") ? static_cast
<void> (0) : __assert_fail ("(LogicOpcode == ISD::AND || LogicOpcode == ISD::OR || LogicOpcode == ISD::XOR) && \"Expected logic opcode\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4366, __PRETTY_FUNCTION__))
4366 LogicOpcode == ISD::XOR) && "Expected logic opcode")(((LogicOpcode == ISD::AND || LogicOpcode == ISD::OR || LogicOpcode
== ISD::XOR) && "Expected logic opcode") ? static_cast
<void> (0) : __assert_fail ("(LogicOpcode == ISD::AND || LogicOpcode == ISD::OR || LogicOpcode == ISD::XOR) && \"Expected logic opcode\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4366, __PRETTY_FUNCTION__))
;
4367 assert(HandOpcode == N1.getOpcode() && "Bad input!")((HandOpcode == N1.getOpcode() && "Bad input!") ? static_cast
<void> (0) : __assert_fail ("HandOpcode == N1.getOpcode() && \"Bad input!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4367, __PRETTY_FUNCTION__))
;
4368
4369 // Bail early if none of these transforms apply.
4370 if (N0.getNumOperands() == 0)
4371 return SDValue();
4372
4373 // FIXME: We should check number of uses of the operands to not increase
4374 // the instruction count for all transforms.
4375
4376 // Handle size-changing casts.
4377 SDValue X = N0.getOperand(0);
4378 SDValue Y = N1.getOperand(0);
4379 EVT XVT = X.getValueType();
4380 SDLoc DL(N);
4381 if (HandOpcode == ISD::ANY_EXTEND || HandOpcode == ISD::ZERO_EXTEND ||
4382 HandOpcode == ISD::SIGN_EXTEND) {
4383 // If both operands have other uses, this transform would create extra
4384 // instructions without eliminating anything.
4385 if (!N0.hasOneUse() && !N1.hasOneUse())
4386 return SDValue();
4387 // We need matching integer source types.
4388 if (XVT != Y.getValueType())
4389 return SDValue();
4390 // Don't create an illegal op during or after legalization. Don't ever
4391 // create an unsupported vector op.
4392 if ((VT.isVector() || LegalOperations) &&
4393 !TLI.isOperationLegalOrCustom(LogicOpcode, XVT))
4394 return SDValue();
4395 // Avoid infinite looping with PromoteIntBinOp.
4396 // TODO: Should we apply desirable/legal constraints to all opcodes?
4397 if (HandOpcode == ISD::ANY_EXTEND && LegalTypes &&
4398 !TLI.isTypeDesirableForOp(LogicOpcode, XVT))
4399 return SDValue();
4400 // logic_op (hand_op X), (hand_op Y) --> hand_op (logic_op X, Y)
4401 SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
4402 return DAG.getNode(HandOpcode, DL, VT, Logic);
4403 }
4404
4405 // logic_op (truncate x), (truncate y) --> truncate (logic_op x, y)
4406 if (HandOpcode == ISD::TRUNCATE) {
4407 // If both operands have other uses, this transform would create extra
4408 // instructions without eliminating anything.
4409 if (!N0.hasOneUse() && !N1.hasOneUse())
4410 return SDValue();
4411 // We need matching source types.
4412 if (XVT != Y.getValueType())
4413 return SDValue();
4414 // Don't create an illegal op during or after legalization.
4415 if (LegalOperations && !TLI.isOperationLegal(LogicOpcode, XVT))
4416 return SDValue();
4417 // Be extra careful sinking truncate. If it's free, there's no benefit in
4418 // widening a binop. Also, don't create a logic op on an illegal type.
4419 if (TLI.isZExtFree(VT, XVT) && TLI.isTruncateFree(XVT, VT))
4420 return SDValue();
4421 if (!TLI.isTypeLegal(XVT))
4422 return SDValue();
4423 SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
4424 return DAG.getNode(HandOpcode, DL, VT, Logic);
4425 }
4426
4427 // For binops SHL/SRL/SRA/AND:
4428 // logic_op (OP x, z), (OP y, z) --> OP (logic_op x, y), z
4429 if ((HandOpcode == ISD::SHL || HandOpcode == ISD::SRL ||
4430 HandOpcode == ISD::SRA || HandOpcode == ISD::AND) &&
4431 N0.getOperand(1) == N1.getOperand(1)) {
4432 // If either operand has other uses, this transform is not an improvement.
4433 if (!N0.hasOneUse() || !N1.hasOneUse())
4434 return SDValue();
4435 SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
4436 return DAG.getNode(HandOpcode, DL, VT, Logic, N0.getOperand(1));
4437 }
4438
4439 // Unary ops: logic_op (bswap x), (bswap y) --> bswap (logic_op x, y)
4440 if (HandOpcode == ISD::BSWAP) {
4441 // If either operand has other uses, this transform is not an improvement.
4442 if (!N0.hasOneUse() || !N1.hasOneUse())
4443 return SDValue();
4444 SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
4445 return DAG.getNode(HandOpcode, DL, VT, Logic);
4446 }
4447
4448 // Simplify xor/and/or (bitcast(A), bitcast(B)) -> bitcast(op (A,B))
4449 // Only perform this optimization up until type legalization, before
4450 // LegalizeVectorOprs. LegalizeVectorOprs promotes vector operations by
4451 // adding bitcasts. For example (xor v4i32) is promoted to (v2i64), and
4452 // we don't want to undo this promotion.
4453 // We also handle SCALAR_TO_VECTOR because xor/or/and operations are cheaper
4454 // on scalars.
4455 if ((HandOpcode == ISD::BITCAST || HandOpcode == ISD::SCALAR_TO_VECTOR) &&
4456 Level <= AfterLegalizeTypes) {
4457 // Input types must be integer and the same.
4458 if (XVT.isInteger() && XVT == Y.getValueType() &&
4459 !(VT.isVector() && TLI.isTypeLegal(VT) &&
4460 !XVT.isVector() && !TLI.isTypeLegal(XVT))) {
4461 SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
4462 return DAG.getNode(HandOpcode, DL, VT, Logic);
4463 }
4464 }
4465
4466 // Xor/and/or are indifferent to the swizzle operation (shuffle of one value).
4467 // Simplify xor/and/or (shuff(A), shuff(B)) -> shuff(op (A,B))
4468 // If both shuffles use the same mask, and both shuffle within a single
4469 // vector, then it is worthwhile to move the swizzle after the operation.
4470 // The type-legalizer generates this pattern when loading illegal
4471 // vector types from memory. In many cases this allows additional shuffle
4472 // optimizations.
4473 // There are other cases where moving the shuffle after the xor/and/or
4474 // is profitable even if shuffles don't perform a swizzle.
4475 // If both shuffles use the same mask, and both shuffles have the same first
4476 // or second operand, then it might still be profitable to move the shuffle
4477 // after the xor/and/or operation.
4478 if (HandOpcode == ISD::VECTOR_SHUFFLE && Level < AfterLegalizeDAG) {
4479 auto *SVN0 = cast<ShuffleVectorSDNode>(N0);
4480 auto *SVN1 = cast<ShuffleVectorSDNode>(N1);
4481 assert(X.getValueType() == Y.getValueType() &&((X.getValueType() == Y.getValueType() && "Inputs to shuffles are not the same type"
) ? static_cast<void> (0) : __assert_fail ("X.getValueType() == Y.getValueType() && \"Inputs to shuffles are not the same type\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4482, __PRETTY_FUNCTION__))
4482 "Inputs to shuffles are not the same type")((X.getValueType() == Y.getValueType() && "Inputs to shuffles are not the same type"
) ? static_cast<void> (0) : __assert_fail ("X.getValueType() == Y.getValueType() && \"Inputs to shuffles are not the same type\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4482, __PRETTY_FUNCTION__))
;
4483
4484 // Check that both shuffles use the same mask. The masks are known to be of
4485 // the same length because the result vector type is the same.
4486 // Check also that shuffles have only one use to avoid introducing extra
4487 // instructions.
4488 if (!SVN0->hasOneUse() || !SVN1->hasOneUse() ||
4489 !SVN0->getMask().equals(SVN1->getMask()))
4490 return SDValue();
4491
4492 // Don't try to fold this node if it requires introducing a
4493 // build vector of all zeros that might be illegal at this stage.
4494 SDValue ShOp = N0.getOperand(1);
4495 if (LogicOpcode == ISD::XOR && !ShOp.isUndef())
4496 ShOp = tryFoldToZero(DL, TLI, VT, DAG, LegalOperations);
4497
4498 // (logic_op (shuf (A, C), shuf (B, C))) --> shuf (logic_op (A, B), C)
4499 if (N0.getOperand(1) == N1.getOperand(1) && ShOp.getNode()) {
4500 SDValue Logic = DAG.getNode(LogicOpcode, DL, VT,
4501 N0.getOperand(0), N1.getOperand(0));
4502 return DAG.getVectorShuffle(VT, DL, Logic, ShOp, SVN0->getMask());
4503 }
4504
4505 // Don't try to fold this node if it requires introducing a
4506 // build vector of all zeros that might be illegal at this stage.
4507 ShOp = N0.getOperand(0);
4508 if (LogicOpcode == ISD::XOR && !ShOp.isUndef())
4509 ShOp = tryFoldToZero(DL, TLI, VT, DAG, LegalOperations);
4510
4511 // (logic_op (shuf (C, A), shuf (C, B))) --> shuf (C, logic_op (A, B))
4512 if (N0.getOperand(0) == N1.getOperand(0) && ShOp.getNode()) {
4513 SDValue Logic = DAG.getNode(LogicOpcode, DL, VT, N0.getOperand(1),
4514 N1.getOperand(1));
4515 return DAG.getVectorShuffle(VT, DL, ShOp, Logic, SVN0->getMask());
4516 }
4517 }
4518
4519 return SDValue();
4520}
4521
4522/// Try to make (and/or setcc (LL, LR), setcc (RL, RR)) more efficient.
4523SDValue DAGCombiner::foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1,
4524 const SDLoc &DL) {
4525 SDValue LL, LR, RL, RR, N0CC, N1CC;
4526 if (!isSetCCEquivalent(N0, LL, LR, N0CC) ||
4527 !isSetCCEquivalent(N1, RL, RR, N1CC))
4528 return SDValue();
4529
4530 assert(N0.getValueType() == N1.getValueType() &&((N0.getValueType() == N1.getValueType() && "Unexpected operand types for bitwise logic op"
) ? static_cast<void> (0) : __assert_fail ("N0.getValueType() == N1.getValueType() && \"Unexpected operand types for bitwise logic op\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4531, __PRETTY_FUNCTION__))
4531 "Unexpected operand types for bitwise logic op")((N0.getValueType() == N1.getValueType() && "Unexpected operand types for bitwise logic op"
) ? static_cast<void> (0) : __assert_fail ("N0.getValueType() == N1.getValueType() && \"Unexpected operand types for bitwise logic op\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4531, __PRETTY_FUNCTION__))
;
4532 assert(LL.getValueType() == LR.getValueType() &&((LL.getValueType() == LR.getValueType() && RL.getValueType
() == RR.getValueType() && "Unexpected operand types for setcc"
) ? static_cast<void> (0) : __assert_fail ("LL.getValueType() == LR.getValueType() && RL.getValueType() == RR.getValueType() && \"Unexpected operand types for setcc\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4534, __PRETTY_FUNCTION__))
4533 RL.getValueType() == RR.getValueType() &&((LL.getValueType() == LR.getValueType() && RL.getValueType
() == RR.getValueType() && "Unexpected operand types for setcc"
) ? static_cast<void> (0) : __assert_fail ("LL.getValueType() == LR.getValueType() && RL.getValueType() == RR.getValueType() && \"Unexpected operand types for setcc\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4534, __PRETTY_FUNCTION__))
4534 "Unexpected operand types for setcc")((LL.getValueType() == LR.getValueType() && RL.getValueType
() == RR.getValueType() && "Unexpected operand types for setcc"
) ? static_cast<void> (0) : __assert_fail ("LL.getValueType() == LR.getValueType() && RL.getValueType() == RR.getValueType() && \"Unexpected operand types for setcc\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4534, __PRETTY_FUNCTION__))
;
4535
4536 // If we're here post-legalization or the logic op type is not i1, the logic
4537 // op type must match a setcc result type. Also, all folds require new
4538 // operations on the left and right operands, so those types must match.
4539 EVT VT = N0.getValueType();
4540 EVT OpVT = LL.getValueType();
4541 if (LegalOperations || VT.getScalarType() != MVT::i1)
4542 if (VT != getSetCCResultType(OpVT))
4543 return SDValue();
4544 if (OpVT != RL.getValueType())
4545 return SDValue();
4546
4547 ISD::CondCode CC0 = cast<CondCodeSDNode>(N0CC)->get();
4548 ISD::CondCode CC1 = cast<CondCodeSDNode>(N1CC)->get();
4549 bool IsInteger = OpVT.isInteger();
4550 if (LR == RR && CC0 == CC1 && IsInteger) {
4551 bool IsZero = isNullOrNullSplat(LR);
4552 bool IsNeg1 = isAllOnesOrAllOnesSplat(LR);
4553
4554 // All bits clear?
4555 bool AndEqZero = IsAnd && CC1 == ISD::SETEQ && IsZero;
4556 // All sign bits clear?
4557 bool AndGtNeg1 = IsAnd && CC1 == ISD::SETGT && IsNeg1;
4558 // Any bits set?
4559 bool OrNeZero = !IsAnd && CC1 == ISD::SETNE && IsZero;
4560 // Any sign bits set?
4561 bool OrLtZero = !IsAnd && CC1 == ISD::SETLT && IsZero;
4562
4563 // (and (seteq X, 0), (seteq Y, 0)) --> (seteq (or X, Y), 0)
4564 // (and (setgt X, -1), (setgt Y, -1)) --> (setgt (or X, Y), -1)
4565 // (or (setne X, 0), (setne Y, 0)) --> (setne (or X, Y), 0)
4566 // (or (setlt X, 0), (setlt Y, 0)) --> (setlt (or X, Y), 0)
4567 if (AndEqZero || AndGtNeg1 || OrNeZero || OrLtZero) {
4568 SDValue Or = DAG.getNode(ISD::OR, SDLoc(N0), OpVT, LL, RL);
4569 AddToWorklist(Or.getNode());
4570 return DAG.getSetCC(DL, VT, Or, LR, CC1);
4571 }
4572
4573 // All bits set?
4574 bool AndEqNeg1 = IsAnd && CC1 == ISD::SETEQ && IsNeg1;
4575 // All sign bits set?
4576 bool AndLtZero = IsAnd && CC1 == ISD::SETLT && IsZero;
4577 // Any bits clear?
4578 bool OrNeNeg1 = !IsAnd && CC1 == ISD::SETNE && IsNeg1;
4579 // Any sign bits clear?
4580 bool OrGtNeg1 = !IsAnd && CC1 == ISD::SETGT && IsNeg1;
4581
4582 // (and (seteq X, -1), (seteq Y, -1)) --> (seteq (and X, Y), -1)
4583 // (and (setlt X, 0), (setlt Y, 0)) --> (setlt (and X, Y), 0)
4584 // (or (setne X, -1), (setne Y, -1)) --> (setne (and X, Y), -1)
4585 // (or (setgt X, -1), (setgt Y -1)) --> (setgt (and X, Y), -1)
4586 if (AndEqNeg1 || AndLtZero || OrNeNeg1 || OrGtNeg1) {
4587 SDValue And = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, LL, RL);
4588 AddToWorklist(And.getNode());
4589 return DAG.getSetCC(DL, VT, And, LR, CC1);
4590 }
4591 }
4592
4593 // TODO: What is the 'or' equivalent of this fold?
4594 // (and (setne X, 0), (setne X, -1)) --> (setuge (add X, 1), 2)
4595 if (IsAnd && LL == RL && CC0 == CC1 && OpVT.getScalarSizeInBits() > 1 &&
4596 IsInteger && CC0 == ISD::SETNE &&
4597 ((isNullConstant(LR) && isAllOnesConstant(RR)) ||
4598 (isAllOnesConstant(LR) && isNullConstant(RR)))) {
4599 SDValue One = DAG.getConstant(1, DL, OpVT);
4600 SDValue Two = DAG.getConstant(2, DL, OpVT);
4601 SDValue Add = DAG.getNode(ISD::ADD, SDLoc(N0), OpVT, LL, One);
4602 AddToWorklist(Add.getNode());
4603 return DAG.getSetCC(DL, VT, Add, Two, ISD::SETUGE);
4604 }
4605
4606 // Try more general transforms if the predicates match and the only user of
4607 // the compares is the 'and' or 'or'.
4608 if (IsInteger && TLI.convertSetCCLogicToBitwiseLogic(OpVT) && CC0 == CC1 &&
4609 N0.hasOneUse() && N1.hasOneUse()) {
4610 // and (seteq A, B), (seteq C, D) --> seteq (or (xor A, B), (xor C, D)), 0
4611 // or (setne A, B), (setne C, D) --> setne (or (xor A, B), (xor C, D)), 0
4612 if ((IsAnd && CC1 == ISD::SETEQ) || (!IsAnd && CC1 == ISD::SETNE)) {
4613 SDValue XorL = DAG.getNode(ISD::XOR, SDLoc(N0), OpVT, LL, LR);
4614 SDValue XorR = DAG.getNode(ISD::XOR, SDLoc(N1), OpVT, RL, RR);
4615 SDValue Or = DAG.getNode(ISD::OR, DL, OpVT, XorL, XorR);
4616 SDValue Zero = DAG.getConstant(0, DL, OpVT);
4617 return DAG.getSetCC(DL, VT, Or, Zero, CC1);
4618 }
4619
4620 // Turn compare of constants whose difference is 1 bit into add+and+setcc.
4621 // TODO - support non-uniform vector amounts.
4622 if ((IsAnd && CC1 == ISD::SETNE) || (!IsAnd && CC1 == ISD::SETEQ)) {
4623 // Match a shared variable operand and 2 non-opaque constant operands.
4624 ConstantSDNode *C0 = isConstOrConstSplat(LR);
4625 ConstantSDNode *C1 = isConstOrConstSplat(RR);
4626 if (LL == RL && C0 && C1 && !C0->isOpaque() && !C1->isOpaque()) {
4627 // Canonicalize larger constant as C0.
4628 if (C1->getAPIntValue().ugt(C0->getAPIntValue()))
4629 std::swap(C0, C1);
4630
4631 // The difference of the constants must be a single bit.
4632 const APInt &C0Val = C0->getAPIntValue();
4633 const APInt &C1Val = C1->getAPIntValue();
4634 if ((C0Val - C1Val).isPowerOf2()) {
4635 // and/or (setcc X, C0, ne), (setcc X, C1, ne/eq) -->
4636 // setcc ((add X, -C1), ~(C0 - C1)), 0, ne/eq
4637 SDValue OffsetC = DAG.getConstant(-C1Val, DL, OpVT);
4638 SDValue Add = DAG.getNode(ISD::ADD, DL, OpVT, LL, OffsetC);
4639 SDValue MaskC = DAG.getConstant(~(C0Val - C1Val), DL, OpVT);
4640 SDValue And = DAG.getNode(ISD::AND, DL, OpVT, Add, MaskC);
4641 SDValue Zero = DAG.getConstant(0, DL, OpVT);
4642 return DAG.getSetCC(DL, VT, And, Zero, CC0);
4643 }
4644 }
4645 }
4646 }
4647
4648 // Canonicalize equivalent operands to LL == RL.
4649 if (LL == RR && LR == RL) {
4650 CC1 = ISD::getSetCCSwappedOperands(CC1);
4651 std::swap(RL, RR);
4652 }
4653
4654 // (and (setcc X, Y, CC0), (setcc X, Y, CC1)) --> (setcc X, Y, NewCC)
4655 // (or (setcc X, Y, CC0), (setcc X, Y, CC1)) --> (setcc X, Y, NewCC)
4656 if (LL == RL && LR == RR) {
4657 ISD::CondCode NewCC = IsAnd ? ISD::getSetCCAndOperation(CC0, CC1, OpVT)
4658 : ISD::getSetCCOrOperation(CC0, CC1, OpVT);
4659 if (NewCC != ISD::SETCC_INVALID &&
4660 (!LegalOperations ||
4661 (TLI.isCondCodeLegal(NewCC, LL.getSimpleValueType()) &&
4662 TLI.isOperationLegal(ISD::SETCC, OpVT))))
4663 return DAG.getSetCC(DL, VT, LL, LR, NewCC);
4664 }
4665
4666 return SDValue();
4667}
4668
4669/// This contains all DAGCombine rules which reduce two values combined by
4670/// an And operation to a single value. This makes them reusable in the context
4671/// of visitSELECT(). Rules involving constants are not included as
4672/// visitSELECT() already handles those cases.
4673SDValue DAGCombiner::visitANDLike(SDValue N0, SDValue N1, SDNode *N) {
4674 EVT VT = N1.getValueType();
4675 SDLoc DL(N);
4676
4677 // fold (and x, undef) -> 0
4678 if (N0.isUndef() || N1.isUndef())
4679 return DAG.getConstant(0, DL, VT);
4680
4681 if (SDValue V = foldLogicOfSetCCs(true, N0, N1, DL))
4682 return V;
4683
4684 if (N0.getOpcode() == ISD::ADD && N1.getOpcode() == ISD::SRL &&
4685 VT.getSizeInBits() <= 64) {
4686 if (ConstantSDNode *ADDI = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
4687 if (ConstantSDNode *SRLI = dyn_cast<ConstantSDNode>(N1.getOperand(1))) {
4688 // Look for (and (add x, c1), (lshr y, c2)). If C1 wasn't a legal
4689 // immediate for an add, but it is legal if its top c2 bits are set,
4690 // transform the ADD so the immediate doesn't need to be materialized
4691 // in a register.
4692 APInt ADDC = ADDI->getAPIntValue();
4693 APInt SRLC = SRLI->getAPIntValue();
4694 if (ADDC.getMinSignedBits() <= 64 &&
4695 SRLC.ult(VT.getSizeInBits()) &&
4696 !TLI.isLegalAddImmediate(ADDC.getSExtValue())) {
4697 APInt Mask = APInt::getHighBitsSet(VT.getSizeInBits(),
4698 SRLC.getZExtValue());
4699 if (DAG.MaskedValueIsZero(N0.getOperand(1), Mask)) {
4700 ADDC |= Mask;
4701 if (TLI.isLegalAddImmediate(ADDC.getSExtValue())) {
4702 SDLoc DL0(N0);
4703 SDValue NewAdd =
4704 DAG.getNode(ISD::ADD, DL0, VT,
4705 N0.getOperand(0), DAG.getConstant(ADDC, DL, VT));
4706 CombineTo(N0.getNode(), NewAdd);
4707 // Return N so it doesn't get rechecked!
4708 return SDValue(N, 0);
4709 }
4710 }
4711 }
4712 }
4713 }
4714 }
4715
4716 // Reduce bit extract of low half of an integer to the narrower type.
4717 // (and (srl i64:x, K), KMask) ->
4718 // (i64 zero_extend (and (srl (i32 (trunc i64:x)), K)), KMask)
4719 if (N0.getOpcode() == ISD::SRL && N0.hasOneUse()) {
4720 if (ConstantSDNode *CAnd = dyn_cast<ConstantSDNode>(N1)) {
4721 if (ConstantSDNode *CShift = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
4722 unsigned Size = VT.getSizeInBits();
4723 const APInt &AndMask = CAnd->getAPIntValue();
4724 unsigned ShiftBits = CShift->getZExtValue();
4725
4726 // Bail out, this node will probably disappear anyway.
4727 if (ShiftBits == 0)
4728 return SDValue();
4729
4730 unsigned MaskBits = AndMask.countTrailingOnes();
4731 EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), Size / 2);
4732
4733 if (AndMask.isMask() &&
4734 // Required bits must not span the two halves of the integer and
4735 // must fit in the half size type.
4736 (ShiftBits + MaskBits <= Size / 2) &&
4737 TLI.isNarrowingProfitable(VT, HalfVT) &&
4738 TLI.isTypeDesirableForOp(ISD::AND, HalfVT) &&
4739 TLI.isTypeDesirableForOp(ISD::SRL, HalfVT) &&
4740 TLI.isTruncateFree(VT, HalfVT) &&
4741 TLI.isZExtFree(HalfVT, VT)) {
4742 // The isNarrowingProfitable is to avoid regressions on PPC and
4743 // AArch64 which match a few 64-bit bit insert / bit extract patterns
4744 // on downstream users of this. Those patterns could probably be
4745 // extended to handle extensions mixed in.
4746
4747 SDValue SL(N0);
4748 assert(MaskBits <= Size)((MaskBits <= Size) ? static_cast<void> (0) : __assert_fail
("MaskBits <= Size", "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4748, __PRETTY_FUNCTION__))
;
4749
4750 // Extracting the highest bit of the low half.
4751 EVT ShiftVT = TLI.getShiftAmountTy(HalfVT, DAG.getDataLayout());
4752 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, HalfVT,
4753 N0.getOperand(0));
4754
4755 SDValue NewMask = DAG.getConstant(AndMask.trunc(Size / 2), SL, HalfVT);
4756 SDValue ShiftK = DAG.getConstant(ShiftBits, SL, ShiftVT);
4757 SDValue Shift = DAG.getNode(ISD::SRL, SL, HalfVT, Trunc, ShiftK);
4758 SDValue And = DAG.getNode(ISD::AND, SL, HalfVT, Shift, NewMask);
4759 return DAG.getNode(ISD::ZERO_EXTEND, SL, VT, And);
4760 }
4761 }
4762 }
4763 }
4764
4765 return SDValue();
4766}
4767
4768bool DAGCombiner::isAndLoadExtLoad(ConstantSDNode *AndC, LoadSDNode *LoadN,
4769 EVT LoadResultTy, EVT &ExtVT) {
4770 if (!AndC->getAPIntValue().isMask())
4771 return false;
4772
4773 unsigned ActiveBits = AndC->getAPIntValue().countTrailingOnes();
4774
4775 ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
4776 EVT LoadedVT = LoadN->getMemoryVT();
4777
4778 if (ExtVT == LoadedVT &&
4779 (!LegalOperations ||
4780 TLI.isLoadExtLegal(ISD::ZEXTLOAD, LoadResultTy, ExtVT))) {
4781 // ZEXTLOAD will match without needing to change the size of the value being
4782 // loaded.
4783 return true;
4784 }
4785
4786 // Do not change the width of a volatile or atomic loads.
4787 if (!LoadN->isSimple())
4788 return false;
4789
4790 // Do not generate loads of non-round integer types since these can
4791 // be expensive (and would be wrong if the type is not byte sized).
4792 if (!LoadedVT.bitsGT(ExtVT) || !ExtVT.isRound())
4793 return false;
4794
4795 if (LegalOperations &&
4796 !TLI.isLoadExtLegal(ISD::ZEXTLOAD, LoadResultTy, ExtVT))
4797 return false;
4798
4799 if (!TLI.shouldReduceLoadWidth(LoadN, ISD::ZEXTLOAD, ExtVT))
4800 return false;
4801
4802 return true;
4803}
4804
4805bool DAGCombiner::isLegalNarrowLdSt(LSBaseSDNode *LDST,
4806 ISD::LoadExtType ExtType, EVT &MemVT,
4807 unsigned ShAmt) {
4808 if (!LDST)
4809 return false;
4810 // Only allow byte offsets.
4811 if (ShAmt % 8)
4812 return false;
4813
4814 // Do not generate loads of non-round integer types since these can
4815 // be expensive (and would be wrong if the type is not byte sized).
4816 if (!MemVT.isRound())
4817 return false;
4818
4819 // Don't change the width of a volatile or atomic loads.
4820 if (!LDST->isSimple())
4821 return false;
4822
4823 // Verify that we are actually reducing a load width here.
4824 if (LDST->getMemoryVT().getSizeInBits() < MemVT.getSizeInBits())
4825 return false;
4826
4827 // Ensure that this isn't going to produce an unsupported memory access.
4828 if (ShAmt &&
4829 !TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), MemVT,
4830 LDST->getAddressSpace(), ShAmt / 8,
4831 LDST->getMemOperand()->getFlags()))
4832 return false;
4833
4834 // It's not possible to generate a constant of extended or untyped type.
4835 EVT PtrType = LDST->getBasePtr().getValueType();
4836 if (PtrType == MVT::Untyped || PtrType.isExtended())
4837 return false;
4838
4839 if (isa<LoadSDNode>(LDST)) {
4840 LoadSDNode *Load = cast<LoadSDNode>(LDST);
4841 // Don't transform one with multiple uses, this would require adding a new
4842 // load.
4843 if (!SDValue(Load, 0).hasOneUse())
4844 return false;
4845
4846 if (LegalOperations &&
4847 !TLI.isLoadExtLegal(ExtType, Load->getValueType(0), MemVT))
4848 return false;
4849
4850 // For the transform to be legal, the load must produce only two values
4851 // (the value loaded and the chain). Don't transform a pre-increment
4852 // load, for example, which produces an extra value. Otherwise the
4853 // transformation is not equivalent, and the downstream logic to replace
4854 // uses gets things wrong.
4855 if (Load->getNumValues() > 2)
4856 return false;
4857
4858 // If the load that we're shrinking is an extload and we're not just
4859 // discarding the extension we can't simply shrink the load. Bail.
4860 // TODO: It would be possible to merge the extensions in some cases.
4861 if (Load->getExtensionType() != ISD::NON_EXTLOAD &&
4862 Load->getMemoryVT().getSizeInBits() < MemVT.getSizeInBits() + ShAmt)
4863 return false;
4864
4865 if (!TLI.shouldReduceLoadWidth(Load, ExtType, MemVT))
4866 return false;
4867 } else {
4868 assert(isa<StoreSDNode>(LDST) && "It is not a Load nor a Store SDNode")((isa<StoreSDNode>(LDST) && "It is not a Load nor a Store SDNode"
) ? static_cast<void> (0) : __assert_fail ("isa<StoreSDNode>(LDST) && \"It is not a Load nor a Store SDNode\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4868, __PRETTY_FUNCTION__))
;
4869 StoreSDNode *Store = cast<StoreSDNode>(LDST);
4870 // Can't write outside the original store
4871 if (Store->getMemoryVT().getSizeInBits() < MemVT.getSizeInBits() + ShAmt)
4872 return false;
4873
4874 if (LegalOperations &&
4875 !TLI.isTruncStoreLegal(Store->getValue().getValueType(), MemVT))
4876 return false;
4877 }
4878 return true;
4879}
4880
4881bool DAGCombiner::SearchForAndLoads(SDNode *N,
4882 SmallVectorImpl<LoadSDNode*> &Loads,
4883 SmallPtrSetImpl<SDNode*> &NodesWithConsts,
4884 ConstantSDNode *Mask,
4885 SDNode *&NodeToMask) {
4886 // Recursively search for the operands, looking for loads which can be
4887 // narrowed.
4888 for (SDValue Op : N->op_values()) {
4889 if (Op.getValueType().isVector())
4890 return false;
4891
4892 // Some constants may need fixing up later if they are too large.
4893 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
4894 if ((N->getOpcode() == ISD::OR || N->getOpcode() == ISD::XOR) &&
4895 (Mask->getAPIntValue() & C->getAPIntValue()) != C->getAPIntValue())
4896 NodesWithConsts.insert(N);
4897 continue;
4898 }
4899
4900 if (!Op.hasOneUse())
4901 return false;
4902
4903 switch(Op.getOpcode()) {
4904 case ISD::LOAD: {
4905 auto *Load = cast<LoadSDNode>(Op);
4906 EVT ExtVT;
4907 if (isAndLoadExtLoad(Mask, Load, Load->getValueType(0), ExtVT) &&
4908 isLegalNarrowLdSt(Load, ISD::ZEXTLOAD, ExtVT)) {
4909
4910 // ZEXTLOAD is already small enough.
4911 if (Load->getExtensionType() == ISD::ZEXTLOAD &&
4912 ExtVT.bitsGE(Load->getMemoryVT()))
4913 continue;
4914
4915 // Use LE to convert equal sized loads to zext.
4916 if (ExtVT.bitsLE(Load->getMemoryVT()))
4917 Loads.push_back(Load);
4918
4919 continue;
4920 }
4921 return false;
4922 }
4923 case ISD::ZERO_EXTEND:
4924 case ISD::AssertZext: {
4925 unsigned ActiveBits = Mask->getAPIntValue().countTrailingOnes();
4926 EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
4927 EVT VT = Op.getOpcode() == ISD::AssertZext ?
4928 cast<VTSDNode>(Op.getOperand(1))->getVT() :
4929 Op.getOperand(0).getValueType();
4930
4931 // We can accept extending nodes if the mask is wider or an equal
4932 // width to the original type.
4933 if (ExtVT.bitsGE(VT))
4934 continue;
4935 break;
4936 }
4937 case ISD::OR:
4938 case ISD::XOR:
4939 case ISD::AND:
4940 if (!SearchForAndLoads(Op.getNode(), Loads, NodesWithConsts, Mask,
4941 NodeToMask))
4942 return false;
4943 continue;
4944 }
4945
4946 // Allow one node which will masked along with any loads found.
4947 if (NodeToMask)
4948 return false;
4949
4950 // Also ensure that the node to be masked only produces one data result.
4951 NodeToMask = Op.getNode();
4952 if (NodeToMask->getNumValues() > 1) {
4953 bool HasValue = false;
4954 for (unsigned i = 0, e = NodeToMask->getNumValues(); i < e; ++i) {
4955 MVT VT = SDValue(NodeToMask, i).getSimpleValueType();
4956 if (VT != MVT::Glue && VT != MVT::Other) {
4957 if (HasValue) {
4958 NodeToMask = nullptr;
4959 return false;
4960 }
4961 HasValue = true;
4962 }
4963 }
4964 assert(HasValue && "Node to be masked has no data result?")((HasValue && "Node to be masked has no data result?"
) ? static_cast<void> (0) : __assert_fail ("HasValue && \"Node to be masked has no data result?\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4964, __PRETTY_FUNCTION__))
;
4965 }
4966 }
4967 return true;
4968}
4969
4970bool DAGCombiner::BackwardsPropagateMask(SDNode *N) {
4971 auto *Mask = dyn_cast<ConstantSDNode>(N->getOperand(1));
4972 if (!Mask)
4973 return false;
4974
4975 if (!Mask->getAPIntValue().isMask())
4976 return false;
4977
4978 // No need to do anything if the and directly uses a load.
4979 if (isa<LoadSDNode>(N->getOperand(0)))
4980 return false;
4981
4982 SmallVector<LoadSDNode*, 8> Loads;
4983 SmallPtrSet<SDNode*, 2> NodesWithConsts;
4984 SDNode *FixupNode = nullptr;
4985 if (SearchForAndLoads(N, Loads, NodesWithConsts, Mask, FixupNode)) {
4986 if (Loads.size() == 0)
4987 return false;
4988
4989 LLVM_DEBUG(dbgs() << "Backwards propagate AND: "; N->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "Backwards propagate AND: "
; N->dump(); } } while (false)
;
4990 SDValue MaskOp = N->getOperand(1);
4991
4992 // If it exists, fixup the single node we allow in the tree that needs
4993 // masking.
4994 if (FixupNode) {
4995 LLVM_DEBUG(dbgs() << "First, need to fix up: "; FixupNode->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "First, need to fix up: "; FixupNode
->dump(); } } while (false)
;
4996 SDValue And = DAG.getNode(ISD::AND, SDLoc(FixupNode),
4997 FixupNode->getValueType(0),
4998 SDValue(FixupNode, 0), MaskOp);
4999 DAG.ReplaceAllUsesOfValueWith(SDValue(FixupNode, 0), And);
5000 if (And.getOpcode() == ISD ::AND)
5001 DAG.UpdateNodeOperands(And.getNode(), SDValue(FixupNode, 0), MaskOp);
5002 }
5003
5004 // Narrow any constants that need it.
5005 for (auto *LogicN : NodesWithConsts) {
5006 SDValue Op0 = LogicN->getOperand(0);
5007 SDValue Op1 = LogicN->getOperand(1);
5008
5009 if (isa<ConstantSDNode>(Op0))
5010 std::swap(Op0, Op1);
5011
5012 SDValue And = DAG.getNode(ISD::AND, SDLoc(Op1), Op1.getValueType(),
5013 Op1, MaskOp);
5014
5015 DAG.UpdateNodeOperands(LogicN, Op0, And);
5016 }
5017
5018 // Create narrow loads.
5019 for (auto *Load : Loads) {
5020 LLVM_DEBUG(dbgs() << "Propagate AND back to: "; Load->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "Propagate AND back to: "; Load
->dump(); } } while (false)
;
5021 SDValue And = DAG.getNode(ISD::AND, SDLoc(Load), Load->getValueType(0),
5022 SDValue(Load, 0), MaskOp);
5023 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), And);
5024 if (And.getOpcode() == ISD ::AND)
5025 And = SDValue(
5026 DAG.UpdateNodeOperands(And.getNode(), SDValue(Load, 0), MaskOp), 0);
5027 SDValue NewLoad = ReduceLoadWidth(And.getNode());
5028 assert(NewLoad &&((NewLoad && "Shouldn't be masking the load if it can't be narrowed"
) ? static_cast<void> (0) : __assert_fail ("NewLoad && \"Shouldn't be masking the load if it can't be narrowed\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5029, __PRETTY_FUNCTION__))
5029 "Shouldn't be masking the load if it can't be narrowed")((NewLoad && "Shouldn't be masking the load if it can't be narrowed"
) ? static_cast<void> (0) : __assert_fail ("NewLoad && \"Shouldn't be masking the load if it can't be narrowed\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5029, __PRETTY_FUNCTION__))
;
5030 CombineTo(Load, NewLoad, NewLoad.getValue(1));
5031 }
5032 DAG.ReplaceAllUsesWith(N, N->getOperand(0).getNode());
5033 return true;
5034 }
5035 return false;
5036}
5037
5038// Unfold
5039// x & (-1 'logical shift' y)
5040// To
5041// (x 'opposite logical shift' y) 'logical shift' y
5042// if it is better for performance.
5043SDValue DAGCombiner::unfoldExtremeBitClearingToShifts(SDNode *N) {
5044 assert(N->getOpcode() == ISD::AND)((N->getOpcode() == ISD::AND) ? static_cast<void> (0
) : __assert_fail ("N->getOpcode() == ISD::AND", "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5044, __PRETTY_FUNCTION__))
;
5045
5046 SDValue N0 = N->getOperand(0);
5047 SDValue N1 = N->getOperand(1);
5048
5049 // Do we actually prefer shifts over mask?
5050 if (!TLI.shouldFoldMaskToVariableShiftPair(N0))
5051 return SDValue();
5052
5053 // Try to match (-1 '[outer] logical shift' y)
5054 unsigned OuterShift;
5055 unsigned InnerShift; // The opposite direction to the OuterShift.
5056 SDValue Y; // Shift amount.
5057 auto matchMask = [&OuterShift, &InnerShift, &Y](SDValue M) -> bool {
5058 if (!M.hasOneUse())
5059 return false;
5060 OuterShift = M->getOpcode();
5061 if (OuterShift == ISD::SHL)
5062 InnerShift = ISD::SRL;
5063 else if (OuterShift == ISD::SRL)
5064 InnerShift = ISD::SHL;
5065 else
5066 return false;
5067 if (!isAllOnesConstant(M->getOperand(0)))
5068 return false;
5069 Y = M->getOperand(1);
5070 return true;
5071 };
5072
5073 SDValue X;
5074 if (matchMask(N1))
5075 X = N0;
5076 else if (matchMask(N0))
5077 X = N1;
5078 else
5079 return SDValue();
5080
5081 SDLoc DL(N);
5082 EVT VT = N->getValueType(0);
5083
5084 // tmp = x 'opposite logical shift' y
5085 SDValue T0 = DAG.getNode(InnerShift, DL, VT, X, Y);
5086 // ret = tmp 'logical shift' y
5087 SDValue T1 = DAG.getNode(OuterShift, DL, VT, T0, Y);
5088
5089 return T1;
5090}
5091
5092/// Try to replace shift/logic that tests if a bit is clear with mask + setcc.
5093/// For a target with a bit test, this is expected to become test + set and save
5094/// at least 1 instruction.
5095static SDValue combineShiftAnd1ToBitTest(SDNode *And, SelectionDAG &DAG) {
5096 assert(And->getOpcode() == ISD::AND && "Expected an 'and' op")((And->getOpcode() == ISD::AND && "Expected an 'and' op"
) ? static_cast<void> (0) : __assert_fail ("And->getOpcode() == ISD::AND && \"Expected an 'and' op\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5096, __PRETTY_FUNCTION__))
;
5097
5098 // This is probably not worthwhile without a supported type.
5099 EVT VT = And->getValueType(0);
5100 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5101 if (!TLI.isTypeLegal(VT))
5102 return SDValue();
5103
5104 // Look through an optional extension and find a 'not'.
5105 // TODO: Should we favor test+set even without the 'not' op?
5106 SDValue Not = And->getOperand(0), And1 = And->getOperand(1);
5107 if (Not.getOpcode() == ISD::ANY_EXTEND)
5108 Not = Not.getOperand(0);
5109 if (!isBitwiseNot(Not) || !Not.hasOneUse() || !isOneConstant(And1))
5110 return SDValue();
5111
5112 // Look though an optional truncation. The source operand may not be the same
5113 // type as the original 'and', but that is ok because we are masking off
5114 // everything but the low bit.
5115 SDValue Srl = Not.getOperand(0);
5116 if (Srl.getOpcode() == ISD::TRUNCATE)
5117 Srl = Srl.getOperand(0);
5118
5119 // Match a shift-right by constant.
5120 if (Srl.getOpcode() != ISD::SRL || !Srl.hasOneUse() ||
5121 !isa<ConstantSDNode>(Srl.getOperand(1)))
5122 return SDValue();
5123
5124 // We might have looked through casts that make this transform invalid.
5125 // TODO: If the source type is wider than the result type, do the mask and
5126 // compare in the source type.
5127 const APInt &ShiftAmt = Srl.getConstantOperandAPInt(1);
5128 unsigned VTBitWidth = VT.getSizeInBits();
5129 if (ShiftAmt.uge(VTBitWidth))
5130 return SDValue();
5131
5132 // Turn this into a bit-test pattern using mask op + setcc:
5133 // and (not (srl X, C)), 1 --> (and X, 1<<C) == 0
5134 SDLoc DL(And);
5135 SDValue X = DAG.getZExtOrTrunc(Srl.getOperand(0), DL, VT);
5136 EVT CCVT = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
5137 SDValue Mask = DAG.getConstant(
5138 APInt::getOneBitSet(VTBitWidth, ShiftAmt.getZExtValue()), DL, VT);
5139 SDValue NewAnd = DAG.getNode(ISD::AND, DL, VT, X, Mask);
5140 SDValue Zero = DAG.getConstant(0, DL, VT);
5141 SDValue Setcc = DAG.getSetCC(DL, CCVT, NewAnd, Zero, ISD::SETEQ);
5142 return DAG.getZExtOrTrunc(Setcc, DL, VT);
5143}
5144
5145SDValue DAGCombiner::visitAND(SDNode *N) {
5146 SDValue N0 = N->getOperand(0);
5147 SDValue N1 = N->getOperand(1);
5148 EVT VT = N1.getValueType();
5149
5150 // x & x --> x
5151 if (N0 == N1)
5152 return N0;
5153
5154 // fold vector ops
5155 if (VT.isVector()) {
5156 if (SDValue FoldedVOp = SimplifyVBinOp(N))
5157 return FoldedVOp;
5158
5159 // fold (and x, 0) -> 0, vector edition
5160 if (ISD::isBuildVectorAllZeros(N0.getNode()))
5161 // do not return N0, because undef node may exist in N0
5162 return DAG.getConstant(APInt::getNullValue(N0.getScalarValueSizeInBits()),
5163 SDLoc(N), N0.getValueType());
5164 if (ISD::isBuildVectorAllZeros(N1.getNode()))
5165 // do not return N1, because undef node may exist in N1
5166 return DAG.getConstant(APInt::getNullValue(N1.getScalarValueSizeInBits()),
5167 SDLoc(N), N1.getValueType());
5168
5169 // fold (and x, -1) -> x, vector edition
5170 if (ISD::isBuildVectorAllOnes(N0.getNode()))
5171 return N1;
5172 if (ISD::isBuildVectorAllOnes(N1.getNode()))
5173 return N0;
5174 }
5175
5176 // fold (and c1, c2) -> c1&c2
5177 ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
5178 ConstantSDNode *N1C = isConstOrConstSplat(N1);
5179 if (N0C && N1C && !N1C->isOpaque())
5180 return DAG.FoldConstantArithmetic(ISD::AND, SDLoc(N), VT, N0C, N1C);
5181 // canonicalize constant to RHS
5182 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
5183 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
5184 return DAG.getNode(ISD::AND, SDLoc(N), VT, N1, N0);
5185 // fold (and x, -1) -> x
5186 if (isAllOnesConstant(N1))
5187 return N0;
5188 // if (and x, c) is known to be zero, return 0
5189 unsigned BitWidth = VT.getScalarSizeInBits();
5190 if (N1C && DAG.MaskedValueIsZero(SDValue(N, 0),
5191 APInt::getAllOnesValue(BitWidth)))
5192 return DAG.getConstant(0, SDLoc(N), VT);
5193
5194 if (SDValue NewSel = foldBinOpIntoSelect(N))
5195 return NewSel;
5196
5197 // reassociate and
5198 if (SDValue RAND = reassociateOps(ISD::AND, SDLoc(N), N0, N1, N->getFlags()))
5199 return RAND;
5200
5201 // Try to convert a constant mask AND into a shuffle clear mask.
5202 if (VT.isVector())
5203 if (SDValue Shuffle = XformToShuffleWithZero(N))
5204 return Shuffle;
5205
5206 if (SDValue Combined = combineCarryDiamond(*this, DAG, TLI, N0, N1, N))
5207 return Combined;
5208
5209 // fold (and (or x, C), D) -> D if (C & D) == D
5210 auto MatchSubset = [](ConstantSDNode *LHS, ConstantSDNode *RHS) {
5211 return RHS->getAPIntValue().isSubsetOf(LHS->getAPIntValue());
5212 };
5213 if (N0.getOpcode() == ISD::OR &&
5214 ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchSubset))
5215 return N1;
5216 // fold (and (any_ext V), c) -> (zero_ext V) if 'and' only clears top bits.
5217 if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) {
5218 SDValue N0Op0 = N0.getOperand(0);
5219 APInt Mask = ~N1C->getAPIntValue();
5220 Mask = Mask.trunc(N0Op0.getScalarValueSizeInBits());
5221 if (DAG.MaskedValueIsZero(N0Op0, Mask)) {
5222 SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N),
5223 N0.getValueType(), N0Op0);
5224
5225 // Replace uses of the AND with uses of the Zero extend node.
5226 CombineTo(N, Zext);
5227
5228 // We actually want to replace all uses of the any_extend with the
5229 // zero_extend, to avoid duplicating things. This will later cause this
5230 // AND to be folded.
5231 CombineTo(N0.getNode(), Zext);
5232 return SDValue(N, 0); // Return N so it doesn't get rechecked!
5233 }
5234 }
5235
5236 // similarly fold (and (X (load ([non_ext|any_ext|zero_ext] V))), c) ->
5237 // (X (load ([non_ext|zero_ext] V))) if 'and' only clears top bits which must
5238 // already be zero by virtue of the width of the base type of the load.
5239 //
5240 // the 'X' node here can either be nothing or an extract_vector_elt to catch
5241 // more cases.
5242 if ((N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
5243 N0.getValueSizeInBits() == N0.getOperand(0).getScalarValueSizeInBits() &&
5244 N0.getOperand(0).getOpcode() == ISD::LOAD &&
5245 N0.getOperand(0).getResNo() == 0) ||
5246 (N0.getOpcode() == ISD::LOAD && N0.getResNo() == 0)) {
5247 LoadSDNode *Load = cast<LoadSDNode>( (N0.getOpcode() == ISD::LOAD) ?
5248 N0 : N0.getOperand(0) );
5249
5250 // Get the constant (if applicable) the zero'th operand is being ANDed with.
5251 // This can be a pure constant or a vector splat, in which case we treat the
5252 // vector as a scalar and use the splat value.
5253 APInt Constant = APInt::getNullValue(1);
5254 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) {
5255 Constant = C->getAPIntValue();
5256 } else if (BuildVectorSDNode *Vector = dyn_cast<BuildVectorSDNode>(N1)) {
5257 APInt SplatValue, SplatUndef;
5258 unsigned SplatBitSize;
5259 bool HasAnyUndefs;
5260 bool IsSplat = Vector->isConstantSplat(SplatValue, SplatUndef,
5261 SplatBitSize, HasAnyUndefs);
5262 if (IsSplat) {
5263 // Undef bits can contribute to a possible optimisation if set, so
5264 // set them.
5265 SplatValue |= SplatUndef;
5266
5267 // The splat value may be something like "0x00FFFFFF", which means 0 for
5268 // the first vector value and FF for the rest, repeating. We need a mask
5269 // that will apply equally to all members of the vector, so AND all the
5270 // lanes of the constant together.
5271 unsigned EltBitWidth = Vector->getValueType(0).getScalarSizeInBits();
5272
5273 // If the splat value has been compressed to a bitlength lower
5274 // than the size of the vector lane, we need to re-expand it to
5275 // the lane size.
5276 if (EltBitWidth > SplatBitSize)
5277 for (SplatValue = SplatValue.zextOrTrunc(EltBitWidth);
5278 SplatBitSize < EltBitWidth; SplatBitSize = SplatBitSize * 2)
5279 SplatValue |= SplatValue.shl(SplatBitSize);
5280
5281 // Make sure that variable 'Constant' is only set if 'SplatBitSize' is a
5282 // multiple of 'BitWidth'. Otherwise, we could propagate a wrong value.
5283 if ((SplatBitSize % EltBitWidth) == 0) {
5284 Constant = APInt::getAllOnesValue(EltBitWidth);
5285 for (unsigned i = 0, n = (SplatBitSize / EltBitWidth); i < n; ++i)
5286 Constant &= SplatValue.extractBits(EltBitWidth, i * EltBitWidth);
5287 }
5288 }
5289 }
5290
5291 // If we want to change an EXTLOAD to a ZEXTLOAD, ensure a ZEXTLOAD is
5292 // actually legal and isn't going to get expanded, else this is a false
5293 // optimisation.
5294 bool CanZextLoadProfitably = TLI.isLoadExtLegal(ISD::ZEXTLOAD,
5295 Load->getValueType(0),
5296 Load->getMemoryVT());
5297
5298 // Resize the constant to the same size as the original memory access before
5299 // extension. If it is still the AllOnesValue then this AND is completely
5300 // unneeded.
5301 Constant = Constant.zextOrTrunc(Load->getMemoryVT().getScalarSizeInBits());
5302
5303 bool B;
5304 switch (Load->getExtensionType()) {
5305 default: B = false; break;
5306 case ISD::EXTLOAD: B = CanZextLoadProfitably; break;
5307 case ISD::ZEXTLOAD:
5308 case ISD::NON_EXTLOAD: B = true; break;
5309 }
5310
5311 if (B && Constant.isAllOnesValue()) {
5312 // If the load type was an EXTLOAD, convert to ZEXTLOAD in order to
5313 // preserve semantics once we get rid of the AND.
5314 SDValue NewLoad(Load, 0);
5315
5316 // Fold the AND away. NewLoad may get replaced immediately.
5317 CombineTo(N, (N0.getNode() == Load) ? NewLoad : N0);
5318
5319 if (Load->getExtensionType() == ISD::EXTLOAD) {
5320 NewLoad = DAG.getLoad(Load->getAddressingMode(), ISD::ZEXTLOAD,
5321 Load->getValueType(0), SDLoc(Load),
5322 Load->getChain(), Load->getBasePtr(),
5323 Load->getOffset(), Load->getMemoryVT(),
5324 Load->getMemOperand());
5325 // Replace uses of the EXTLOAD with the new ZEXTLOAD.
5326 if (Load->getNumValues() == 3) {
5327 // PRE/POST_INC loads have 3 values.
5328 SDValue To[] = { NewLoad.getValue(0), NewLoad.getValue(1),
5329 NewLoad.getValue(2) };
5330 CombineTo(Load, To, 3, true);
5331 } else {
5332 CombineTo(Load, NewLoad.getValue(0), NewLoad.getValue(1));
5333 }
5334 }
5335
5336 return SDValue(N, 0); // Return N so it doesn't get rechecked!
5337 }
5338 }
5339
5340 // fold (and (load x), 255) -> (zextload x, i8)
5341 // fold (and (extload x, i16), 255) -> (zextload x, i8)
5342 // fold (and (any_ext (extload x, i16)), 255) -> (zextload x, i8)
5343 if (!VT.isVector() && N1C && (N0.getOpcode() == ISD::LOAD ||
5344 (N0.getOpcode() == ISD::ANY_EXTEND &&
5345 N0.getOperand(0).getOpcode() == ISD::LOAD))) {
5346 if (SDValue Res = ReduceLoadWidth(N)) {
5347 LoadSDNode *LN0 = N0->getOpcode() == ISD::ANY_EXTEND
5348 ? cast<LoadSDNode>(N0.getOperand(0)) : cast<LoadSDNode>(N0);
5349 AddToWorklist(N);
5350 DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 0), Res);
5351 return SDValue(N, 0);
5352 }
5353 }
5354
5355 if (LegalTypes) {
5356 // Attempt to propagate the AND back up to the leaves which, if they're
5357 // loads, can be combined to narrow loads and the AND node can be removed.
5358 // Perform after legalization so that extend nodes will already be
5359 // combined into the loads.
5360 if (BackwardsPropagateMask(N))
5361 return SDValue(N, 0);
5362 }
5363
5364 if (SDValue Combined = visitANDLike(N0, N1, N))
5365 return Combined;
5366
5367 // Simplify: (and (op x...), (op y...)) -> (op (and x, y))
5368 if (N0.getOpcode() == N1.getOpcode())
5369 if (SDValue V = hoistLogicOpWithSameOpcodeHands(N))
5370 return V;
5371
5372 // Masking the negated extension of a boolean is just the zero-extended
5373 // boolean:
5374 // and (sub 0, zext(bool X)), 1 --> zext(bool X)
5375 // and (sub 0, sext(bool X)), 1 --> zext(bool X)
5376 //
5377 // Note: the SimplifyDemandedBits fold below can make an information-losing
5378 // transform, and then we have no way to find this better fold.
5379 if (N1C && N1C->isOne() && N0.getOpcode() == ISD::SUB) {
5380 if (isNullOrNullSplat(N0.getOperand(0))) {
5381 SDValue SubRHS = N0.getOperand(1);
5382 if (SubRHS.getOpcode() == ISD::ZERO_EXTEND &&
5383 SubRHS.getOperand(0).getScalarValueSizeInBits() == 1)
5384 return SubRHS;
5385 if (SubRHS.getOpcode() == ISD::SIGN_EXTEND &&
5386 SubRHS.getOperand(0).getScalarValueSizeInBits() == 1)
5387 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, SubRHS.getOperand(0));
5388 }
5389 }
5390
5391 // fold (and (sign_extend_inreg x, i16 to i32), 1) -> (and x, 1)
5392 // fold (and (sra)) -> (and (srl)) when possible.
5393 if (SimplifyDemandedBits(SDValue(N, 0)))
5394 return SDValue(N, 0);
5395
5396 // fold (zext_inreg (extload x)) -> (zextload x)
5397 // fold (zext_inreg (sextload x)) -> (zextload x) iff load has one use
5398 if (ISD::isUNINDEXEDLoad(N0.getNode()) &&
5399 (ISD::isEXTLoad(N0.getNode()) ||
5400 (ISD::isSEXTLoad(N0.getNode()) && N0.hasOneUse()))) {
5401 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
5402 EVT MemVT = LN0->getMemoryVT();
5403 // If we zero all the possible extended bits, then we can turn this into
5404 // a zextload if we are running before legalize or the operation is legal.
5405 unsigned ExtBitSize = N1.getScalarValueSizeInBits();
5406 unsigned MemBitSize = MemVT.getScalarSizeInBits();
5407 APInt ExtBits = APInt::getHighBitsSet(ExtBitSize, ExtBitSize - MemBitSize);
5408 if (DAG.MaskedValueIsZero(N1, ExtBits) &&
5409 ((!LegalOperations && LN0->isSimple()) ||
5410 TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT))) {
5411 SDValue ExtLoad =
5412 DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT, LN0->getChain(),
5413 LN0->getBasePtr(), MemVT, LN0->getMemOperand());
5414 AddToWorklist(N);
5415 CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
5416 return SDValue(N, 0); // Return N so it doesn't get rechecked!
5417 }
5418 }
5419
5420 // fold (and (or (srl N, 8), (shl N, 8)), 0xffff) -> (srl (bswap N), const)
5421 if (N1C && N1C->getAPIntValue() == 0xffff && N0.getOpcode() == ISD::OR) {
5422 if (SDValue BSwap = MatchBSwapHWordLow(N0.getNode(), N0.getOperand(0),
5423 N0.getOperand(1), false))
5424 return BSwap;
5425 }
5426
5427 if (SDValue Shifts = unfoldExtremeBitClearingToShifts(N))
5428 return Shifts;
5429
5430 if (TLI.hasBitTest(N0, N1))
5431 if (SDValue V = combineShiftAnd1ToBitTest(N, DAG))
5432 return V;
5433
5434 return SDValue();
5435}
5436
5437/// Match (a >> 8) | (a << 8) as (bswap a) >> 16.
5438SDValue DAGCombiner::MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
5439 bool DemandHighBits) {
5440 if (!LegalOperations)
5441 return SDValue();
5442
5443 EVT VT = N->getValueType(0);
5444 if (VT != MVT::i64 && VT != MVT::i32 && VT != MVT::i16)
5445 return SDValue();
5446 if (!TLI.isOperationLegalOrCustom(ISD::BSWAP, VT))
5447 return SDValue();
5448
5449 // Recognize (and (shl a, 8), 0xff00), (and (srl a, 8), 0xff)
5450 bool LookPassAnd0 = false;
5451 bool LookPassAnd1 = false;
5452 if (N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::SRL)
5453 std::swap(N0, N1);
5454 if (N1.getOpcode() == ISD::AND && N1.getOperand(0).getOpcode() == ISD::SHL)
5455 std::swap(N0, N1);
5456 if (N0.getOpcode() == ISD::AND) {
5457 if (!N0.getNode()->hasOneUse())
5458 return SDValue();
5459 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
5460 // Also handle 0xffff since the LHS is guaranteed to have zeros there.
5461 // This is needed for X86.
5462 if (!N01C || (N01C->getZExtValue() != 0xFF00 &&
5463 N01C->getZExtValue() != 0xFFFF))
5464 return SDValue();
5465 N0 = N0.getOperand(0);
5466 LookPassAnd0 = true;
5467 }
5468
5469 if (N1.getOpcode() == ISD::AND) {
5470 if (!N1.getNode()->hasOneUse())
5471 return SDValue();
5472 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
5473 if (!N11C || N11C->getZExtValue() != 0xFF)
5474 return SDValue();
5475 N1 = N1.getOperand(0);
5476 LookPassAnd1 = true;
5477 }
5478
5479 if (N0.getOpcode() == ISD::SRL && N1.getOpcode() == ISD::SHL)
5480 std::swap(N0, N1);
5481 if (N0.getOpcode() != ISD::SHL || N1.getOpcode() != ISD::SRL)
5482 return SDValue();
5483 if (!N0.getNode()->hasOneUse() || !N1.getNode()->hasOneUse())
5484 return SDValue();
5485
5486 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
5487 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
5488 if (!N01C || !N11C)
5489 return SDValue();
5490 if (N01C->getZExtValue() != 8 || N11C->getZExtValue() != 8)
5491 return SDValue();
5492
5493 // Look for (shl (and a, 0xff), 8), (srl (and a, 0xff00), 8)
5494 SDValue N00 = N0->getOperand(0);
5495 if (!LookPassAnd0 && N00.getOpcode() == ISD::AND) {
5496 if (!N00.getNode()->hasOneUse())
5497 return SDValue();
5498 ConstantSDNode *N001C = dyn_cast<ConstantSDNode>(N00.getOperand(1));
5499 if (!N001C || N001C->getZExtValue() != 0xFF)
5500 return SDValue();
5501 N00 = N00.getOperand(0);
5502 LookPassAnd0 = true;
5503 }
5504
5505 SDValue N10 = N1->getOperand(0);
5506 if (!LookPassAnd1 && N10.getOpcode() == ISD::AND) {
5507 if (!N10.getNode()->hasOneUse())
5508 return SDValue();
5509 ConstantSDNode *N101C = dyn_cast<ConstantSDNode>(N10.getOperand(1));
5510 // Also allow 0xFFFF since the bits will be shifted out. This is needed
5511 // for X86.
5512 if (!N101C || (N101C->getZExtValue() != 0xFF00 &&
5513 N101C->getZExtValue() != 0xFFFF))
5514 return SDValue();
5515 N10 = N10.getOperand(0);
5516 LookPassAnd1 = true;
5517 }
5518
5519 if (N00 != N10)
5520 return SDValue();
5521
5522 // Make sure everything beyond the low halfword gets set to zero since the SRL
5523 // 16 will clear the top bits.
5524 unsigned OpSizeInBits = VT.getSizeInBits();
5525 if (DemandHighBits && OpSizeInBits > 16) {
5526 // If the left-shift isn't masked out then the only way this is a bswap is
5527 // if all bits beyond the low 8 are 0. In that case the entire pattern
5528 // reduces to a left shift anyway: leave it for other parts of the combiner.
5529 if (!LookPassAnd0)
5530 return SDValue();
5531
5532 // However, if the right shift isn't masked out then it might be because
5533 // it's not needed. See if we can spot that too.
5534 if (!LookPassAnd1 &&
5535 !DAG.MaskedValueIsZero(
5536 N10, APInt::getHighBitsSet(OpSizeInBits, OpSizeInBits - 16)))
5537 return SDValue();
5538 }
5539
5540 SDValue Res = DAG.getNode(ISD::BSWAP, SDLoc(N), VT, N00);
5541 if (OpSizeInBits > 16) {
5542 SDLoc DL(N);
5543 Res = DAG.getNode(ISD::SRL, DL, VT, Res,
5544 DAG.getConstant(OpSizeInBits - 16, DL,
5545 getShiftAmountTy(VT)));
5546 }
5547 return Res;
5548}
5549
5550/// Return true if the specified node is an element that makes up a 32-bit
5551/// packed halfword byteswap.
5552/// ((x & 0x000000ff) << 8) |
5553/// ((x & 0x0000ff00) >> 8) |
5554/// ((x & 0x00ff0000) << 8) |
5555/// ((x & 0xff000000) >> 8)
5556static bool isBSwapHWordElement(SDValue N, MutableArrayRef<SDNode *> Parts) {
5557 if (!N.getNode()->hasOneUse())
5558 return false;
5559
5560 unsigned Opc = N.getOpcode();
5561 if (Opc != ISD::AND && Opc != ISD::SHL && Opc != ISD::SRL)
5562 return false;
5563
5564 SDValue N0 = N.getOperand(0);
5565 unsigned Opc0 = N0.getOpcode();
5566 if (Opc0 != ISD::AND && Opc0 != ISD::SHL && Opc0 != ISD::SRL)
5567 return false;
5568
5569 ConstantSDNode *N1C = nullptr;
5570 // SHL or SRL: look upstream for AND mask operand
5571 if (Opc == ISD::AND)
5572 N1C = dyn_cast<ConstantSDNode>(N.getOperand(1));
5573 else if (Opc0 == ISD::AND)
5574 N1C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
5575 if (!N1C)
5576 return false;
5577
5578 unsigned MaskByteOffset;
5579 switch (N1C->getZExtValue()) {
5580 default:
5581 return false;
5582 case 0xFF: MaskByteOffset = 0; break;
5583 case 0xFF00: MaskByteOffset = 1; break;
5584 case 0xFFFF:
5585 // In case demanded bits didn't clear the bits that will be shifted out.
5586 // This is needed for X86.
5587 if (Opc == ISD::SRL || (Opc == ISD::AND && Opc0 == ISD::SHL)) {
5588 MaskByteOffset = 1;
5589 break;
5590 }
5591 return false;
5592 case 0xFF0000: MaskByteOffset = 2; break;
5593 case 0xFF000000: MaskByteOffset = 3; break;
5594 }
5595
5596 // Look for (x & 0xff) << 8 as well as ((x << 8) & 0xff00).
5597 if (Opc == ISD::AND) {
5598 if (MaskByteOffset == 0 || MaskByteOffset == 2) {
5599 // (x >> 8) & 0xff
5600 // (x >> 8) & 0xff0000
5601 if (Opc0 != ISD::SRL)
5602 return false;
5603 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
5604 if (!C || C->getZExtValue() != 8)
5605 return false;
5606 } else {
5607 // (x << 8) & 0xff00
5608 // (x << 8) & 0xff000000
5609 if (Opc0 != ISD::SHL)
5610 return false;
5611 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
5612 if (!C || C->getZExtValue() != 8)
5613 return false;
5614 }
5615 } else if (Opc == ISD::SHL) {
5616 // (x & 0xff) << 8
5617 // (x & 0xff0000) << 8
5618 if (MaskByteOffset != 0 && MaskByteOffset != 2)
5619 return false;
5620 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
5621 if (!C || C->getZExtValue() != 8)
5622 return false;
5623 } else { // Opc == ISD::SRL
5624 // (x & 0xff00) >> 8
5625 // (x & 0xff000000) >> 8
5626 if (MaskByteOffset != 1 && MaskByteOffset != 3)
5627 return false;
5628 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
5629 if (!C || C->getZExtValue() != 8)
5630 return false;
5631 }
5632
5633 if (Parts[MaskByteOffset])
5634 return false;
5635
5636 Parts[MaskByteOffset] = N0.getOperand(0).getNode();
5637 return true;
5638}
5639
5640// Match 2 elements of a packed halfword bswap.
5641static bool isBSwapHWordPair(SDValue N, MutableArrayRef<SDNode *> Parts) {
5642 if (N.getOpcode() == ISD::OR)
5643 return isBSwapHWordElement(N.getOperand(0), Parts) &&
5644 isBSwapHWordElement(N.getOperand(1), Parts);
5645
5646 if (N.getOpcode() == ISD::SRL && N.getOperand(0).getOpcode() == ISD::BSWAP) {
5647 ConstantSDNode *C = isConstOrConstSplat(N.getOperand(1));
5648 if (!C || C->getAPIntValue() != 16)
5649 return false;
5650 Parts[0] = Parts[1] = N.getOperand(0).getOperand(0).getNode();
5651 return true;
5652 }
5653
5654 return false;
5655}
5656
5657/// Match a 32-bit packed halfword bswap. That is
5658/// ((x & 0x000000ff) << 8) |
5659/// ((x & 0x0000ff00) >> 8) |
5660/// ((x & 0x00ff0000) << 8) |
5661/// ((x & 0xff000000) >> 8)
5662/// => (rotl (bswap x), 16)
5663SDValue DAGCombiner::MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1) {
5664 if (!LegalOperations)
5665 return SDValue();
5666
5667 EVT VT = N->getValueType(0);
5668 if (VT != MVT::i32)
5669 return SDValue();
5670 if (!TLI.isOperationLegalOrCustom(ISD::BSWAP, VT))
5671 return SDValue();
5672
5673 // Look for either
5674 // (or (bswaphpair), (bswaphpair))
5675 // (or (or (bswaphpair), (and)), (and))
5676 // (or (or (and), (bswaphpair)), (and))
5677 SDNode *Parts[4] = {};
5678
5679 if (isBSwapHWordPair(N0, Parts)) {
5680 // (or (or (and), (and)), (or (and), (and)))
5681 if (!isBSwapHWordPair(N1, Parts))
5682 return SDValue();
5683 } else if (N0.getOpcode() == ISD::OR) {
5684 // (or (or (or (and), (and)), (and)), (and))
5685 if (!isBSwapHWordElement(N1, Parts))
5686 return SDValue();
5687 SDValue N00 = N0.getOperand(0);
5688 SDValue N01 = N0.getOperand(1);
5689 if (!(isBSwapHWordElement(N01, Parts) && isBSwapHWordPair(N00, Parts)) &&
5690 !(isBSwapHWordElement(N00, Parts) && isBSwapHWordPair(N01, Parts)))
5691 return SDValue();
5692 } else
5693 return SDValue();
5694
5695 // Make sure the parts are all coming from the same node.
5696 if (Parts[0] != Parts[1] || Parts[0] != Parts[2] || Parts[0] != Parts[3])
5697 return SDValue();
5698
5699 SDLoc DL(N);
5700 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT,
5701 SDValue(Parts[0], 0));
5702
5703 // Result of the bswap should be rotated by 16. If it's not legal, then
5704 // do (x << 16) | (x >> 16).
5705 SDValue ShAmt = DAG.getConstant(16, DL, getShiftAmountTy(VT));
5706 if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT))
5707 return DAG.getNode(ISD::ROTL, DL, VT, BSwap, ShAmt);
5708 if (TLI.isOperationLegalOrCustom(ISD::ROTR, VT))
5709 return DAG.getNode(ISD::ROTR, DL, VT, BSwap, ShAmt);
5710 return DAG.getNode(ISD::OR, DL, VT,
5711 DAG.getNode(ISD::SHL, DL, VT, BSwap, ShAmt),
5712 DAG.getNode(ISD::SRL, DL, VT, BSwap, ShAmt));
5713}
5714
5715/// This contains all DAGCombine rules which reduce two values combined by
5716/// an Or operation to a single value \see visitANDLike().
5717SDValue DAGCombiner::visitORLike(SDValue N0, SDValue N1, SDNode *N) {
5718 EVT VT = N1.getValueType();
5719 SDLoc DL(N);
5720
5721 // fold (or x, undef) -> -1
5722 if (!LegalOperations && (N0.isUndef() || N1.isUndef()))
5723 return DAG.getAllOnesConstant(DL, VT);
5724
5725 if (SDValue V = foldLogicOfSetCCs(false, N0, N1, DL))
5726 return V;
5727
5728 // (or (and X, C1), (and Y, C2)) -> (and (or X, Y), C3) if possible.
5729 if (N0.getOpcode() == ISD::AND && N1.getOpcode() == ISD::AND &&
5730 // Don't increase # computations.
5731 (N0.getNode()->hasOneUse() || N1.getNode()->hasOneUse())) {
5732 // We can only do this xform if we know that bits from X that are set in C2
5733 // but not in C1 are already zero. Likewise for Y.
5734 if (const ConstantSDNode *N0O1C =
5735 getAsNonOpaqueConstant(N0.getOperand(1))) {
5736 if (const ConstantSDNode *N1O1C =
5737 getAsNonOpaqueConstant(N1.getOperand(1))) {
5738 // We can only do this xform if we know that bits from X that are set in
5739 // C2 but not in C1 are already zero. Likewise for Y.
5740 const APInt &LHSMask = N0O1C->getAPIntValue();
5741 const APInt &RHSMask = N1O1C->getAPIntValue();
5742
5743 if (DAG.MaskedValueIsZero(N0.getOperand(0), RHSMask&~LHSMask) &&
5744 DAG.MaskedValueIsZero(N1.getOperand(0), LHSMask&~RHSMask)) {
5745 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT,
5746 N0.getOperand(0), N1.getOperand(0));
5747 return DAG.getNode(ISD::AND, DL, VT, X,
5748 DAG.getConstant(LHSMask | RHSMask, DL, VT));
5749 }
5750 }
5751 }
5752 }
5753
5754 // (or (and X, M), (and X, N)) -> (and X, (or M, N))
5755 if (N0.getOpcode() == ISD::AND &&
5756 N1.getOpcode() == ISD::AND &&
5757 N0.getOperand(0) == N1.getOperand(0) &&
5758 // Don't increase # computations.
5759 (N0.getNode()->hasOneUse() || N1.getNode()->hasOneUse())) {
5760 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT,
5761 N0.getOperand(1), N1.getOperand(1));
5762 return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), X);
5763 }
5764
5765 return SDValue();
5766}
5767
5768/// OR combines for which the commuted variant will be tried as well.
5769static SDValue visitORCommutative(
5770 SelectionDAG &DAG, SDValue N0, SDValue N1, SDNode *N) {
5771 EVT VT = N0.getValueType();
5772 if (N0.getOpcode() == ISD::AND) {
5773 // fold (or (and X, (xor Y, -1)), Y) -> (or X, Y)
5774 if (isBitwiseNot(N0.getOperand(1)) && N0.getOperand(1).getOperand(0) == N1)
5775 return DAG.getNode(ISD::OR, SDLoc(N), VT, N0.getOperand(0), N1);
5776
5777 // fold (or (and (xor Y, -1), X), Y) -> (or X, Y)
5778 if (isBitwiseNot(N0.getOperand(0)) && N0.getOperand(0).getOperand(0) == N1)
5779 return DAG.getNode(ISD::OR, SDLoc(N), VT, N0.getOperand(1), N1);
5780 }
5781
5782 return SDValue();
5783}
5784
5785SDValue DAGCombiner::visitOR(SDNode *N) {
5786 SDValue N0 = N->getOperand(0);
5787 SDValue N1 = N->getOperand(1);
5788 EVT VT = N1.getValueType();
5789
5790 // x | x --> x
5791 if (N0 == N1)
5792 return N0;
5793
5794 // fold vector ops
5795 if (VT.isVector()) {
5796 if (SDValue FoldedVOp = SimplifyVBinOp(N))
5797 return FoldedVOp;
5798
5799 // fold (or x, 0) -> x, vector edition
5800 if (ISD::isBuildVectorAllZeros(N0.getNode()))
5801 return N1;
5802 if (ISD::isBuildVectorAllZeros(N1.getNode()))
5803 return N0;
5804
5805 // fold (or x, -1) -> -1, vector edition
5806 if (ISD::isBuildVectorAllOnes(N0.getNode()))
5807 // do not return N0, because undef node may exist in N0
5808 return DAG.getAllOnesConstant(SDLoc(N), N0.getValueType());
5809 if (ISD::isBuildVectorAllOnes(N1.getNode()))
5810 // do not return N1, because undef node may exist in N1
5811 return DAG.getAllOnesConstant(SDLoc(N), N1.getValueType());
5812
5813 // fold (or (shuf A, V_0, MA), (shuf B, V_0, MB)) -> (shuf A, B, Mask)
5814 // Do this only if the resulting shuffle is legal.
5815 if (isa<ShuffleVectorSDNode>(N0) &&
5816 isa<ShuffleVectorSDNode>(N1) &&
5817 // Avoid folding a node with illegal type.
5818 TLI.isTypeLegal(VT)) {
5819 bool ZeroN00 = ISD::isBuildVectorAllZeros(N0.getOperand(0).getNode());
5820 bool ZeroN01 = ISD::isBuildVectorAllZeros(N0.getOperand(1).getNode());
5821 bool ZeroN10 = ISD::isBuildVectorAllZeros(N1.getOperand(0).getNode());
5822 bool ZeroN11 = ISD::isBuildVectorAllZeros(N1.getOperand(1).getNode());
5823 // Ensure both shuffles have a zero input.
5824 if ((ZeroN00 != ZeroN01) && (ZeroN10 != ZeroN11)) {
5825 assert((!ZeroN00 || !ZeroN01) && "Both inputs zero!")(((!ZeroN00 || !ZeroN01) && "Both inputs zero!") ? static_cast
<void> (0) : __assert_fail ("(!ZeroN00 || !ZeroN01) && \"Both inputs zero!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5825, __PRETTY_FUNCTION__))
;
5826 assert((!ZeroN10 || !ZeroN11) && "Both inputs zero!")(((!ZeroN10 || !ZeroN11) && "Both inputs zero!") ? static_cast
<void> (0) : __assert_fail ("(!ZeroN10 || !ZeroN11) && \"Both inputs zero!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5826, __PRETTY_FUNCTION__))
;
5827 const ShuffleVectorSDNode *SV0 = cast<ShuffleVectorSDNode>(N0);
5828 const ShuffleVectorSDNode *SV1 = cast<ShuffleVectorSDNode>(N1);
5829 bool CanFold = true;
5830 int NumElts = VT.getVectorNumElements();
5831 SmallVector<int, 4> Mask(NumElts);
5832
5833 for (int i = 0; i != NumElts; ++i) {
5834 int M0 = SV0->getMaskElt(i);
5835 int M1 = SV1->getMaskElt(i);
5836
5837 // Determine if either index is pointing to a zero vector.
5838 bool M0Zero = M0 < 0 || (ZeroN00 == (M0 < NumElts));
5839 bool M1Zero = M1 < 0 || (ZeroN10 == (M1 < NumElts));
5840
5841 // If one element is zero and the otherside is undef, keep undef.
5842 // This also handles the case that both are undef.
5843 if ((M0Zero && M1 < 0) || (M1Zero && M0 < 0)) {
5844 Mask[i] = -1;
5845 continue;
5846 }
5847
5848 // Make sure only one of the elements is zero.
5849 if (M0Zero == M1Zero) {
5850 CanFold = false;
5851 break;
5852 }
5853
5854 assert((M0 >= 0 || M1 >= 0) && "Undef index!")(((M0 >= 0 || M1 >= 0) && "Undef index!") ? static_cast
<void> (0) : __assert_fail ("(M0 >= 0 || M1 >= 0) && \"Undef index!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5854, __PRETTY_FUNCTION__))
;
5855
5856 // We have a zero and non-zero element. If the non-zero came from
5857 // SV0 make the index a LHS index. If it came from SV1, make it
5858 // a RHS index. We need to mod by NumElts because we don't care
5859 // which operand it came from in the original shuffles.
5860 Mask[i] = M1Zero ? M0 % NumElts : (M1 % NumElts) + NumElts;
5861 }
5862
5863 if (CanFold) {
5864 SDValue NewLHS = ZeroN00 ? N0.getOperand(1) : N0.getOperand(0);
5865 SDValue NewRHS = ZeroN10 ? N1.getOperand(1) : N1.getOperand(0);
5866
5867 SDValue LegalShuffle =
5868 TLI.buildLegalVectorShuffle(VT, SDLoc(N), NewLHS, NewRHS,
5869 Mask, DAG);
5870 if (LegalShuffle)
5871 return LegalShuffle;
5872 }
5873 }
5874 }
5875 }
5876
5877 // fold (or c1, c2) -> c1|c2
5878 ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
5879 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
5880 if (N0C && N1C && !N1C->isOpaque())
5881 return DAG.FoldConstantArithmetic(ISD::OR, SDLoc(N), VT, N0C, N1C);
5882 // canonicalize constant to RHS
5883 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
5884 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
5885 return DAG.getNode(ISD::OR, SDLoc(N), VT, N1, N0);
5886 // fold (or x, 0) -> x
5887 if (isNullConstant(N1))
5888 return N0;
5889 // fold (or x, -1) -> -1
5890 if (isAllOnesConstant(N1))
5891 return N1;
5892
5893 if (SDValue NewSel = foldBinOpIntoSelect(N))
5894 return NewSel;
5895
5896 // fold (or x, c) -> c iff (x & ~c) == 0
5897 if (N1C && DAG.MaskedValueIsZero(N0, ~N1C->getAPIntValue()))
5898 return N1;
5899
5900 if (SDValue Combined = visitORLike(N0, N1, N))
5901 return Combined;
5902
5903 if (SDValue Combined = combineCarryDiamond(*this, DAG, TLI, N0, N1, N))
5904 return Combined;
5905
5906 // Recognize halfword bswaps as (bswap + rotl 16) or (bswap + shl 16)
5907 if (SDValue BSwap = MatchBSwapHWord(N, N0, N1))
5908 return BSwap;
5909 if (SDValue BSwap = MatchBSwapHWordLow(N, N0, N1))
5910 return BSwap;
5911
5912 // reassociate or
5913 if (SDValue ROR = reassociateOps(ISD::OR, SDLoc(N), N0, N1, N->getFlags()))
5914 return ROR;
5915
5916 // Canonicalize (or (and X, c1), c2) -> (and (or X, c2), c1|c2)
5917 // iff (c1 & c2) != 0 or c1/c2 are undef.
5918 auto MatchIntersect = [](ConstantSDNode *C1, ConstantSDNode *C2) {
5919 return !C1 || !C2 || C1->getAPIntValue().intersects(C2->getAPIntValue());
5920 };
5921 if (N0.getOpcode() == ISD::AND && N0.getNode()->hasOneUse() &&
5922 ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchIntersect, true)) {
5923 if (SDValue COR = DAG.FoldConstantArithmetic(
5924 ISD::OR, SDLoc(N1), VT, N1.getNode(), N0.getOperand(1).getNode())) {
5925 SDValue IOR = DAG.getNode(ISD::OR, SDLoc(N0), VT, N0.getOperand(0), N1);
5926 AddToWorklist(IOR.getNode());
5927 return DAG.getNode(ISD::AND, SDLoc(N), VT, COR, IOR);
5928 }
5929 }
5930
5931 if (SDValue Combined = visitORCommutative(DAG, N0, N1, N))
5932 return Combined;
5933 if (SDValue Combined = visitORCommutative(DAG, N1, N0, N))
5934 return Combined;
5935
5936 // Simplify: (or (op x...), (op y...)) -> (op (or x, y))
5937 if (N0.getOpcode() == N1.getOpcode())
5938 if (SDValue V = hoistLogicOpWithSameOpcodeHands(N))
5939 return V;
5940
5941 // See if this is some rotate idiom.
5942 if (SDValue Rot = MatchRotate(N0, N1, SDLoc(N)))
5943 return Rot;
5944
5945 if (SDValue Load = MatchLoadCombine(N))
5946 return Load;
5947
5948 // Simplify the operands using demanded-bits information.
5949 if (SimplifyDemandedBits(SDValue(N, 0)))
5950 return SDValue(N, 0);
5951
5952 // If OR can be rewritten into ADD, try combines based on ADD.
5953 if ((!LegalOperations || TLI.isOperationLegal(ISD::ADD, VT)) &&
5954 DAG.haveNoCommonBitsSet(N0, N1))
5955 if (SDValue Combined = visitADDLike(N))
5956 return Combined;
5957
5958 return SDValue();
5959}
5960
5961static SDValue stripConstantMask(SelectionDAG &DAG, SDValue Op, SDValue &Mask) {
5962 if (Op.getOpcode() == ISD::AND &&
5963 DAG.isConstantIntBuildVectorOrConstantInt(Op.getOperand(1))) {
5964 Mask = Op.getOperand(1);
5965 return Op.getOperand(0);
5966 }
5967 return Op;
5968}
5969
5970/// Match "(X shl/srl V1) & V2" where V2 may not be present.
5971static bool matchRotateHalf(SelectionDAG &DAG, SDValue Op, SDValue &Shift,
5972 SDValue &Mask) {
5973 Op = stripConstantMask(DAG, Op, Mask);
5974 if (Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SHL) {
5975 Shift = Op;
5976 return true;
5977 }
5978 return false;
5979}
5980
5981/// Helper function for visitOR to extract the needed side of a rotate idiom
5982/// from a shl/srl/mul/udiv. This is meant to handle cases where
5983/// InstCombine merged some outside op with one of the shifts from
5984/// the rotate pattern.
5985/// \returns An empty \c SDValue if the needed shift couldn't be extracted.
5986/// Otherwise, returns an expansion of \p ExtractFrom based on the following
5987/// patterns:
5988///
5989/// (or (add v v) (shrl v bitwidth-1)):
5990/// expands (add v v) -> (shl v 1)
5991///
5992/// (or (mul v c0) (shrl (mul v c1) c2)):
5993/// expands (mul v c0) -> (shl (mul v c1) c3)
5994///
5995/// (or (udiv v c0) (shl (udiv v c1) c2)):
5996/// expands (udiv v c0) -> (shrl (udiv v c1) c3)
5997///
5998/// (or (shl v c0) (shrl (shl v c1) c2)):
5999/// expands (shl v c0) -> (shl (shl v c1) c3)
6000///
6001/// (or (shrl v c0) (shl (shrl v c1) c2)):
6002/// expands (shrl v c0) -> (shrl (shrl v c1) c3)
6003///
6004/// Such that in all cases, c3+c2==bitwidth(op v c1).
6005static SDValue extractShiftForRotate(SelectionDAG &DAG, SDValue OppShift,
6006 SDValue ExtractFrom, SDValue &Mask,
6007 const SDLoc &DL) {
6008 assert(OppShift && ExtractFrom && "Empty SDValue")((OppShift && ExtractFrom && "Empty SDValue")
? static_cast<void> (0) : __assert_fail ("OppShift && ExtractFrom && \"Empty SDValue\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 6008, __PRETTY_FUNCTION__))
;
6009 assert((((OppShift.getOpcode() == ISD::SHL || OppShift.getOpcode() ==
ISD::SRL) && "Existing shift must be valid as a rotate half"
) ? static_cast<void> (0) : __assert_fail ("(OppShift.getOpcode() == ISD::SHL || OppShift.getOpcode() == ISD::SRL) && \"Existing shift must be valid as a rotate half\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 6011, __PRETTY_FUNCTION__))
6010 (OppShift.getOpcode() == ISD::SHL || Opp