Bug Summary

File:build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
Warning:line 1361, column 12
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name DAGCombiner.cpp -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/build-llvm -resource-dir /usr/lib/llvm-16/lib/clang/16.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/llvm/lib/CodeGen/SelectionDAG -I include -I /build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/llvm/include -D _FORTIFY_SOURCE=2 -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-16/lib/clang/16.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -fmacro-prefix-map=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/build-llvm=build-llvm -fmacro-prefix-map=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/= -fcoverage-prefix-map=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/build-llvm=build-llvm -fcoverage-prefix-map=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/= -O3 -Wno-unused-command-line-argument -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -Wno-misleading-indentation -std=c++17 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/build-llvm -fdebug-prefix-map=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/build-llvm=build-llvm -fdebug-prefix-map=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/= -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2022-10-03-140002-15933-1 -x c++ /build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

1//===- DAGCombiner.cpp - Implement a DAG node combiner --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass combines dag nodes to form fewer, simpler DAG nodes. It can be run
10// both before and after the DAG is legalized.
11//
12// This pass is not a substitute for the LLVM IR instcombine pass. This pass is
13// primarily intended to handle simplification opportunities that are implicit
14// in the LLVM IR and exposed by the various codegen lowering phases.
15//
16//===----------------------------------------------------------------------===//
17
18#include "llvm/ADT/APFloat.h"
19#include "llvm/ADT/APInt.h"
20#include "llvm/ADT/ArrayRef.h"
21#include "llvm/ADT/DenseMap.h"
22#include "llvm/ADT/IntervalMap.h"
23#include "llvm/ADT/None.h"
24#include "llvm/ADT/Optional.h"
25#include "llvm/ADT/STLExtras.h"
26#include "llvm/ADT/SetVector.h"
27#include "llvm/ADT/SmallBitVector.h"
28#include "llvm/ADT/SmallPtrSet.h"
29#include "llvm/ADT/SmallSet.h"
30#include "llvm/ADT/SmallVector.h"
31#include "llvm/ADT/Statistic.h"
32#include "llvm/Analysis/AliasAnalysis.h"
33#include "llvm/Analysis/MemoryLocation.h"
34#include "llvm/Analysis/TargetLibraryInfo.h"
35#include "llvm/Analysis/VectorUtils.h"
36#include "llvm/CodeGen/DAGCombine.h"
37#include "llvm/CodeGen/ISDOpcodes.h"
38#include "llvm/CodeGen/MachineFunction.h"
39#include "llvm/CodeGen/MachineMemOperand.h"
40#include "llvm/CodeGen/RuntimeLibcalls.h"
41#include "llvm/CodeGen/SelectionDAG.h"
42#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
43#include "llvm/CodeGen/SelectionDAGNodes.h"
44#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
45#include "llvm/CodeGen/TargetLowering.h"
46#include "llvm/CodeGen/TargetRegisterInfo.h"
47#include "llvm/CodeGen/TargetSubtargetInfo.h"
48#include "llvm/CodeGen/ValueTypes.h"
49#include "llvm/IR/Attributes.h"
50#include "llvm/IR/Constant.h"
51#include "llvm/IR/DataLayout.h"
52#include "llvm/IR/DerivedTypes.h"
53#include "llvm/IR/Function.h"
54#include "llvm/IR/Metadata.h"
55#include "llvm/Support/Casting.h"
56#include "llvm/Support/CodeGen.h"
57#include "llvm/Support/CommandLine.h"
58#include "llvm/Support/Compiler.h"
59#include "llvm/Support/Debug.h"
60#include "llvm/Support/ErrorHandling.h"
61#include "llvm/Support/KnownBits.h"
62#include "llvm/Support/MachineValueType.h"
63#include "llvm/Support/MathExtras.h"
64#include "llvm/Support/raw_ostream.h"
65#include "llvm/Target/TargetMachine.h"
66#include "llvm/Target/TargetOptions.h"
67#include <algorithm>
68#include <cassert>
69#include <cstdint>
70#include <functional>
71#include <iterator>
72#include <string>
73#include <tuple>
74#include <utility>
75#include <variant>
76
77using namespace llvm;
78
79#define DEBUG_TYPE"dagcombine" "dagcombine"
80
81STATISTIC(NodesCombined , "Number of dag nodes combined")static llvm::Statistic NodesCombined = {"dagcombine", "NodesCombined"
, "Number of dag nodes combined"}
;
82STATISTIC(PreIndexedNodes , "Number of pre-indexed nodes created")static llvm::Statistic PreIndexedNodes = {"dagcombine", "PreIndexedNodes"
, "Number of pre-indexed nodes created"}
;
83STATISTIC(PostIndexedNodes, "Number of post-indexed nodes created")static llvm::Statistic PostIndexedNodes = {"dagcombine", "PostIndexedNodes"
, "Number of post-indexed nodes created"}
;
84STATISTIC(OpsNarrowed , "Number of load/op/store narrowed")static llvm::Statistic OpsNarrowed = {"dagcombine", "OpsNarrowed"
, "Number of load/op/store narrowed"}
;
85STATISTIC(LdStFP2Int , "Number of fp load/store pairs transformed to int")static llvm::Statistic LdStFP2Int = {"dagcombine", "LdStFP2Int"
, "Number of fp load/store pairs transformed to int"}
;
86STATISTIC(SlicedLoads, "Number of load sliced")static llvm::Statistic SlicedLoads = {"dagcombine", "SlicedLoads"
, "Number of load sliced"}
;
87STATISTIC(NumFPLogicOpsConv, "Number of logic ops converted to fp ops")static llvm::Statistic NumFPLogicOpsConv = {"dagcombine", "NumFPLogicOpsConv"
, "Number of logic ops converted to fp ops"}
;
88
89static cl::opt<bool>
90CombinerGlobalAA("combiner-global-alias-analysis", cl::Hidden,
91 cl::desc("Enable DAG combiner's use of IR alias analysis"));
92
93static cl::opt<bool>
94UseTBAA("combiner-use-tbaa", cl::Hidden, cl::init(true),
95 cl::desc("Enable DAG combiner's use of TBAA"));
96
97#ifndef NDEBUG
98static cl::opt<std::string>
99CombinerAAOnlyFunc("combiner-aa-only-func", cl::Hidden,
100 cl::desc("Only use DAG-combiner alias analysis in this"
101 " function"));
102#endif
103
104/// Hidden option to stress test load slicing, i.e., when this option
105/// is enabled, load slicing bypasses most of its profitability guards.
106static cl::opt<bool>
107StressLoadSlicing("combiner-stress-load-slicing", cl::Hidden,
108 cl::desc("Bypass the profitability model of load slicing"),
109 cl::init(false));
110
111static cl::opt<bool>
112 MaySplitLoadIndex("combiner-split-load-index", cl::Hidden, cl::init(true),
113 cl::desc("DAG combiner may split indexing from loads"));
114
115static cl::opt<bool>
116 EnableStoreMerging("combiner-store-merging", cl::Hidden, cl::init(true),
117 cl::desc("DAG combiner enable merging multiple stores "
118 "into a wider store"));
119
120static cl::opt<unsigned> TokenFactorInlineLimit(
121 "combiner-tokenfactor-inline-limit", cl::Hidden, cl::init(2048),
122 cl::desc("Limit the number of operands to inline for Token Factors"));
123
124static cl::opt<unsigned> StoreMergeDependenceLimit(
125 "combiner-store-merge-dependence-limit", cl::Hidden, cl::init(10),
126 cl::desc("Limit the number of times for the same StoreNode and RootNode "
127 "to bail out in store merging dependence check"));
128
129static cl::opt<bool> EnableReduceLoadOpStoreWidth(
130 "combiner-reduce-load-op-store-width", cl::Hidden, cl::init(true),
131 cl::desc("DAG combiner enable reducing the width of load/op/store "
132 "sequence"));
133
134static cl::opt<bool> EnableShrinkLoadReplaceStoreWithStore(
135 "combiner-shrink-load-replace-store-with-store", cl::Hidden, cl::init(true),
136 cl::desc("DAG combiner enable load/<replace bytes>/store with "
137 "a narrower store"));
138
139static cl::opt<bool> EnableVectorFCopySignExtendRound(
140 "combiner-vector-fcopysign-extend-round", cl::Hidden, cl::init(false),
141 cl::desc(
142 "Enable merging extends and rounds into FCOPYSIGN on vector types"));
143
144namespace {
145
146 class DAGCombiner {
147 SelectionDAG &DAG;
148 const TargetLowering &TLI;
149 const SelectionDAGTargetInfo *STI;
150 CombineLevel Level = BeforeLegalizeTypes;
151 CodeGenOpt::Level OptLevel;
152 bool LegalDAG = false;
153 bool LegalOperations = false;
154 bool LegalTypes = false;
155 bool ForCodeSize;
156 bool DisableGenericCombines;
157
158 /// Worklist of all of the nodes that need to be simplified.
159 ///
160 /// This must behave as a stack -- new nodes to process are pushed onto the
161 /// back and when processing we pop off of the back.
162 ///
163 /// The worklist will not contain duplicates but may contain null entries
164 /// due to nodes being deleted from the underlying DAG.
165 SmallVector<SDNode *, 64> Worklist;
166
167 /// Mapping from an SDNode to its position on the worklist.
168 ///
169 /// This is used to find and remove nodes from the worklist (by nulling
170 /// them) when they are deleted from the underlying DAG. It relies on
171 /// stable indices of nodes within the worklist.
172 DenseMap<SDNode *, unsigned> WorklistMap;
173 /// This records all nodes attempted to add to the worklist since we
174 /// considered a new worklist entry. As we keep do not add duplicate nodes
175 /// in the worklist, this is different from the tail of the worklist.
176 SmallSetVector<SDNode *, 32> PruningList;
177
178 /// Set of nodes which have been combined (at least once).
179 ///
180 /// This is used to allow us to reliably add any operands of a DAG node
181 /// which have not yet been combined to the worklist.
182 SmallPtrSet<SDNode *, 32> CombinedNodes;
183
184 /// Map from candidate StoreNode to the pair of RootNode and count.
185 /// The count is used to track how many times we have seen the StoreNode
186 /// with the same RootNode bail out in dependence check. If we have seen
187 /// the bail out for the same pair many times over a limit, we won't
188 /// consider the StoreNode with the same RootNode as store merging
189 /// candidate again.
190 DenseMap<SDNode *, std::pair<SDNode *, unsigned>> StoreRootCountMap;
191
192 // AA - Used for DAG load/store alias analysis.
193 AliasAnalysis *AA;
194
195 /// When an instruction is simplified, add all users of the instruction to
196 /// the work lists because they might get more simplified now.
197 void AddUsersToWorklist(SDNode *N) {
198 for (SDNode *Node : N->uses())
199 AddToWorklist(Node);
200 }
201
202 /// Convenient shorthand to add a node and all of its user to the worklist.
203 void AddToWorklistWithUsers(SDNode *N) {
204 AddUsersToWorklist(N);
205 AddToWorklist(N);
206 }
207
208 // Prune potentially dangling nodes. This is called after
209 // any visit to a node, but should also be called during a visit after any
210 // failed combine which may have created a DAG node.
211 void clearAddedDanglingWorklistEntries() {
212 // Check any nodes added to the worklist to see if they are prunable.
213 while (!PruningList.empty()) {
214 auto *N = PruningList.pop_back_val();
215 if (N->use_empty())
216 recursivelyDeleteUnusedNodes(N);
217 }
218 }
219
220 SDNode *getNextWorklistEntry() {
221 // Before we do any work, remove nodes that are not in use.
222 clearAddedDanglingWorklistEntries();
223 SDNode *N = nullptr;
224 // The Worklist holds the SDNodes in order, but it may contain null
225 // entries.
226 while (!N && !Worklist.empty()) {
227 N = Worklist.pop_back_val();
228 }
229
230 if (N) {
231 bool GoodWorklistEntry = WorklistMap.erase(N);
232 (void)GoodWorklistEntry;
233 assert(GoodWorklistEntry &&(static_cast <bool> (GoodWorklistEntry && "Found a worklist entry without a corresponding map entry!"
) ? void (0) : __assert_fail ("GoodWorklistEntry && \"Found a worklist entry without a corresponding map entry!\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 234, __extension__
__PRETTY_FUNCTION__))
234 "Found a worklist entry without a corresponding map entry!")(static_cast <bool> (GoodWorklistEntry && "Found a worklist entry without a corresponding map entry!"
) ? void (0) : __assert_fail ("GoodWorklistEntry && \"Found a worklist entry without a corresponding map entry!\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 234, __extension__
__PRETTY_FUNCTION__))
;
235 }
236 return N;
237 }
238
239 /// Call the node-specific routine that folds each particular type of node.
240 SDValue visit(SDNode *N);
241
242 public:
243 DAGCombiner(SelectionDAG &D, AliasAnalysis *AA, CodeGenOpt::Level OL)
244 : DAG(D), TLI(D.getTargetLoweringInfo()),
245 STI(D.getSubtarget().getSelectionDAGInfo()), OptLevel(OL), AA(AA) {
246 ForCodeSize = DAG.shouldOptForSize();
247 DisableGenericCombines = STI && STI->disableGenericCombines(OptLevel);
248
249 MaximumLegalStoreInBits = 0;
250 // We use the minimum store size here, since that's all we can guarantee
251 // for the scalable vector types.
252 for (MVT VT : MVT::all_valuetypes())
253 if (EVT(VT).isSimple() && VT != MVT::Other &&
254 TLI.isTypeLegal(EVT(VT)) &&
255 VT.getSizeInBits().getKnownMinSize() >= MaximumLegalStoreInBits)
256 MaximumLegalStoreInBits = VT.getSizeInBits().getKnownMinSize();
257 }
258
259 void ConsiderForPruning(SDNode *N) {
260 // Mark this for potential pruning.
261 PruningList.insert(N);
262 }
263
264 /// Add to the worklist making sure its instance is at the back (next to be
265 /// processed.)
266 void AddToWorklist(SDNode *N) {
267 assert(N->getOpcode() != ISD::DELETED_NODE &&(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& "Deleted Node added to Worklist") ? void (0) : __assert_fail
("N->getOpcode() != ISD::DELETED_NODE && \"Deleted Node added to Worklist\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 268, __extension__
__PRETTY_FUNCTION__))
268 "Deleted Node added to Worklist")(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& "Deleted Node added to Worklist") ? void (0) : __assert_fail
("N->getOpcode() != ISD::DELETED_NODE && \"Deleted Node added to Worklist\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 268, __extension__
__PRETTY_FUNCTION__))
;
269
270 // Skip handle nodes as they can't usefully be combined and confuse the
271 // zero-use deletion strategy.
272 if (N->getOpcode() == ISD::HANDLENODE)
273 return;
274
275 ConsiderForPruning(N);
276
277 if (WorklistMap.insert(std::make_pair(N, Worklist.size())).second)
278 Worklist.push_back(N);
279 }
280
281 /// Remove all instances of N from the worklist.
282 void removeFromWorklist(SDNode *N) {
283 CombinedNodes.erase(N);
284 PruningList.remove(N);
285 StoreRootCountMap.erase(N);
286
287 auto It = WorklistMap.find(N);
288 if (It == WorklistMap.end())
289 return; // Not in the worklist.
290
291 // Null out the entry rather than erasing it to avoid a linear operation.
292 Worklist[It->second] = nullptr;
293 WorklistMap.erase(It);
294 }
295
296 void deleteAndRecombine(SDNode *N);
297 bool recursivelyDeleteUnusedNodes(SDNode *N);
298
299 /// Replaces all uses of the results of one DAG node with new values.
300 SDValue CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
301 bool AddTo = true);
302
303 /// Replaces all uses of the results of one DAG node with new values.
304 SDValue CombineTo(SDNode *N, SDValue Res, bool AddTo = true) {
305 return CombineTo(N, &Res, 1, AddTo);
306 }
307
308 /// Replaces all uses of the results of one DAG node with new values.
309 SDValue CombineTo(SDNode *N, SDValue Res0, SDValue Res1,
310 bool AddTo = true) {
311 SDValue To[] = { Res0, Res1 };
312 return CombineTo(N, To, 2, AddTo);
313 }
314
315 void CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO);
316
317 private:
318 unsigned MaximumLegalStoreInBits;
319
320 /// Check the specified integer node value to see if it can be simplified or
321 /// if things it uses can be simplified by bit propagation.
322 /// If so, return true.
323 bool SimplifyDemandedBits(SDValue Op) {
324 unsigned BitWidth = Op.getScalarValueSizeInBits();
325 APInt DemandedBits = APInt::getAllOnes(BitWidth);
326 return SimplifyDemandedBits(Op, DemandedBits);
327 }
328
329 bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits) {
330 TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations);
331 KnownBits Known;
332 if (!TLI.SimplifyDemandedBits(Op, DemandedBits, Known, TLO, 0, false))
333 return false;
334
335 // Revisit the node.
336 AddToWorklist(Op.getNode());
337
338 CommitTargetLoweringOpt(TLO);
339 return true;
340 }
341
342 /// Check the specified vector node value to see if it can be simplified or
343 /// if things it uses can be simplified as it only uses some of the
344 /// elements. If so, return true.
345 bool SimplifyDemandedVectorElts(SDValue Op) {
346 // TODO: For now just pretend it cannot be simplified.
347 if (Op.getValueType().isScalableVector())
348 return false;
349
350 unsigned NumElts = Op.getValueType().getVectorNumElements();
351 APInt DemandedElts = APInt::getAllOnes(NumElts);
352 return SimplifyDemandedVectorElts(Op, DemandedElts);
353 }
354
355 bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
356 const APInt &DemandedElts,
357 bool AssumeSingleUse = false);
358 bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedElts,
359 bool AssumeSingleUse = false);
360
361 bool CombineToPreIndexedLoadStore(SDNode *N);
362 bool CombineToPostIndexedLoadStore(SDNode *N);
363 SDValue SplitIndexingFromLoad(LoadSDNode *LD);
364 bool SliceUpLoad(SDNode *N);
365
366 // Scalars have size 0 to distinguish from singleton vectors.
367 SDValue ForwardStoreValueToDirectLoad(LoadSDNode *LD);
368 bool getTruncatedStoreValue(StoreSDNode *ST, SDValue &Val);
369 bool extendLoadedValueToExtension(LoadSDNode *LD, SDValue &Val);
370
371 /// Replace an ISD::EXTRACT_VECTOR_ELT of a load with a narrowed
372 /// load.
373 ///
374 /// \param EVE ISD::EXTRACT_VECTOR_ELT to be replaced.
375 /// \param InVecVT type of the input vector to EVE with bitcasts resolved.
376 /// \param EltNo index of the vector element to load.
377 /// \param OriginalLoad load that EVE came from to be replaced.
378 /// \returns EVE on success SDValue() on failure.
379 SDValue scalarizeExtractedVectorLoad(SDNode *EVE, EVT InVecVT,
380 SDValue EltNo,
381 LoadSDNode *OriginalLoad);
382 void ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad);
383 SDValue PromoteOperand(SDValue Op, EVT PVT, bool &Replace);
384 SDValue SExtPromoteOperand(SDValue Op, EVT PVT);
385 SDValue ZExtPromoteOperand(SDValue Op, EVT PVT);
386 SDValue PromoteIntBinOp(SDValue Op);
387 SDValue PromoteIntShiftOp(SDValue Op);
388 SDValue PromoteExtend(SDValue Op);
389 bool PromoteLoad(SDValue Op);
390
391 /// Call the node-specific routine that knows how to fold each
392 /// particular type of node. If that doesn't do anything, try the
393 /// target-specific DAG combines.
394 SDValue combine(SDNode *N);
395
396 // Visitation implementation - Implement dag node combining for different
397 // node types. The semantics are as follows:
398 // Return Value:
399 // SDValue.getNode() == 0 - No change was made
400 // SDValue.getNode() == N - N was replaced, is dead and has been handled.
401 // otherwise - N should be replaced by the returned Operand.
402 //
403 SDValue visitTokenFactor(SDNode *N);
404 SDValue visitMERGE_VALUES(SDNode *N);
405 SDValue visitADD(SDNode *N);
406 SDValue visitADDLike(SDNode *N);
407 SDValue visitADDLikeCommutative(SDValue N0, SDValue N1, SDNode *LocReference);
408 SDValue visitSUB(SDNode *N);
409 SDValue visitADDSAT(SDNode *N);
410 SDValue visitSUBSAT(SDNode *N);
411 SDValue visitADDC(SDNode *N);
412 SDValue visitADDO(SDNode *N);
413 SDValue visitUADDOLike(SDValue N0, SDValue N1, SDNode *N);
414 SDValue visitSUBC(SDNode *N);
415 SDValue visitSUBO(SDNode *N);
416 SDValue visitADDE(SDNode *N);
417 SDValue visitADDCARRY(SDNode *N);
418 SDValue visitSADDO_CARRY(SDNode *N);
419 SDValue visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn, SDNode *N);
420 SDValue visitSUBE(SDNode *N);
421 SDValue visitSUBCARRY(SDNode *N);
422 SDValue visitSSUBO_CARRY(SDNode *N);
423 SDValue visitMUL(SDNode *N);
424 SDValue visitMULFIX(SDNode *N);
425 SDValue useDivRem(SDNode *N);
426 SDValue visitSDIV(SDNode *N);
427 SDValue visitSDIVLike(SDValue N0, SDValue N1, SDNode *N);
428 SDValue visitUDIV(SDNode *N);
429 SDValue visitUDIVLike(SDValue N0, SDValue N1, SDNode *N);
430 SDValue visitREM(SDNode *N);
431 SDValue visitMULHU(SDNode *N);
432 SDValue visitMULHS(SDNode *N);
433 SDValue visitAVG(SDNode *N);
434 SDValue visitSMUL_LOHI(SDNode *N);
435 SDValue visitUMUL_LOHI(SDNode *N);
436 SDValue visitMULO(SDNode *N);
437 SDValue visitIMINMAX(SDNode *N);
438 SDValue visitAND(SDNode *N);
439 SDValue visitANDLike(SDValue N0, SDValue N1, SDNode *N);
440 SDValue visitOR(SDNode *N);
441 SDValue visitORLike(SDValue N0, SDValue N1, SDNode *N);
442 SDValue visitXOR(SDNode *N);
443 SDValue SimplifyVBinOp(SDNode *N, const SDLoc &DL);
444 SDValue visitSHL(SDNode *N);
445 SDValue visitSRA(SDNode *N);
446 SDValue visitSRL(SDNode *N);
447 SDValue visitFunnelShift(SDNode *N);
448 SDValue visitSHLSAT(SDNode *N);
449 SDValue visitRotate(SDNode *N);
450 SDValue visitABS(SDNode *N);
451 SDValue visitBSWAP(SDNode *N);
452 SDValue visitBITREVERSE(SDNode *N);
453 SDValue visitCTLZ(SDNode *N);
454 SDValue visitCTLZ_ZERO_UNDEF(SDNode *N);
455 SDValue visitCTTZ(SDNode *N);
456 SDValue visitCTTZ_ZERO_UNDEF(SDNode *N);
457 SDValue visitCTPOP(SDNode *N);
458 SDValue visitSELECT(SDNode *N);
459 SDValue visitVSELECT(SDNode *N);
460 SDValue visitSELECT_CC(SDNode *N);
461 SDValue visitSETCC(SDNode *N);
462 SDValue visitSETCCCARRY(SDNode *N);
463 SDValue visitSIGN_EXTEND(SDNode *N);
464 SDValue visitZERO_EXTEND(SDNode *N);
465 SDValue visitANY_EXTEND(SDNode *N);
466 SDValue visitAssertExt(SDNode *N);
467 SDValue visitAssertAlign(SDNode *N);
468 SDValue visitSIGN_EXTEND_INREG(SDNode *N);
469 SDValue visitEXTEND_VECTOR_INREG(SDNode *N);
470 SDValue visitTRUNCATE(SDNode *N);
471 SDValue visitBITCAST(SDNode *N);
472 SDValue visitFREEZE(SDNode *N);
473 SDValue visitBUILD_PAIR(SDNode *N);
474 SDValue visitFADD(SDNode *N);
475 SDValue visitSTRICT_FADD(SDNode *N);
476 SDValue visitFSUB(SDNode *N);
477 SDValue visitFMUL(SDNode *N);
478 SDValue visitFMA(SDNode *N);
479 SDValue visitFDIV(SDNode *N);
480 SDValue visitFREM(SDNode *N);
481 SDValue visitFSQRT(SDNode *N);
482 SDValue visitFCOPYSIGN(SDNode *N);
483 SDValue visitFPOW(SDNode *N);
484 SDValue visitSINT_TO_FP(SDNode *N);
485 SDValue visitUINT_TO_FP(SDNode *N);
486 SDValue visitFP_TO_SINT(SDNode *N);
487 SDValue visitFP_TO_UINT(SDNode *N);
488 SDValue visitFP_ROUND(SDNode *N);
489 SDValue visitFP_EXTEND(SDNode *N);
490 SDValue visitFNEG(SDNode *N);
491 SDValue visitFABS(SDNode *N);
492 SDValue visitFCEIL(SDNode *N);
493 SDValue visitFTRUNC(SDNode *N);
494 SDValue visitFFLOOR(SDNode *N);
495 SDValue visitFMinMax(SDNode *N);
496 SDValue visitBRCOND(SDNode *N);
497 SDValue visitBR_CC(SDNode *N);
498 SDValue visitLOAD(SDNode *N);
499
500 SDValue replaceStoreChain(StoreSDNode *ST, SDValue BetterChain);
501 SDValue replaceStoreOfFPConstant(StoreSDNode *ST);
502
503 SDValue visitSTORE(SDNode *N);
504 SDValue visitLIFETIME_END(SDNode *N);
505 SDValue visitINSERT_VECTOR_ELT(SDNode *N);
506 SDValue visitEXTRACT_VECTOR_ELT(SDNode *N);
507 SDValue visitBUILD_VECTOR(SDNode *N);
508 SDValue visitCONCAT_VECTORS(SDNode *N);
509 SDValue visitEXTRACT_SUBVECTOR(SDNode *N);
510 SDValue visitVECTOR_SHUFFLE(SDNode *N);
511 SDValue visitSCALAR_TO_VECTOR(SDNode *N);
512 SDValue visitINSERT_SUBVECTOR(SDNode *N);
513 SDValue visitMLOAD(SDNode *N);
514 SDValue visitMSTORE(SDNode *N);
515 SDValue visitMGATHER(SDNode *N);
516 SDValue visitMSCATTER(SDNode *N);
517 SDValue visitVPGATHER(SDNode *N);
518 SDValue visitVPSCATTER(SDNode *N);
519 SDValue visitFP_TO_FP16(SDNode *N);
520 SDValue visitFP16_TO_FP(SDNode *N);
521 SDValue visitFP_TO_BF16(SDNode *N);
522 SDValue visitVECREDUCE(SDNode *N);
523 SDValue visitVPOp(SDNode *N);
524
525 SDValue visitFADDForFMACombine(SDNode *N);
526 SDValue visitFSUBForFMACombine(SDNode *N);
527 SDValue visitFMULForFMADistributiveCombine(SDNode *N);
528
529 SDValue XformToShuffleWithZero(SDNode *N);
530 bool reassociationCanBreakAddressingModePattern(unsigned Opc,
531 const SDLoc &DL,
532 SDNode *N,
533 SDValue N0,
534 SDValue N1);
535 SDValue reassociateOpsCommutative(unsigned Opc, const SDLoc &DL, SDValue N0,
536 SDValue N1);
537 SDValue reassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0,
538 SDValue N1, SDNodeFlags Flags);
539
540 SDValue visitShiftByConstant(SDNode *N);
541
542 SDValue foldSelectOfConstants(SDNode *N);
543 SDValue foldVSelectOfConstants(SDNode *N);
544 SDValue foldBinOpIntoSelect(SDNode *BO);
545 bool SimplifySelectOps(SDNode *SELECT, SDValue LHS, SDValue RHS);
546 SDValue hoistLogicOpWithSameOpcodeHands(SDNode *N);
547 SDValue SimplifySelect(const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2);
548 SDValue SimplifySelectCC(const SDLoc &DL, SDValue N0, SDValue N1,
549 SDValue N2, SDValue N3, ISD::CondCode CC,
550 bool NotExtCompare = false);
551 SDValue convertSelectOfFPConstantsToLoadOffset(
552 const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2, SDValue N3,
553 ISD::CondCode CC);
554 SDValue foldSignChangeInBitcast(SDNode *N);
555 SDValue foldSelectCCToShiftAnd(const SDLoc &DL, SDValue N0, SDValue N1,
556 SDValue N2, SDValue N3, ISD::CondCode CC);
557 SDValue foldSelectOfBinops(SDNode *N);
558 SDValue foldSextSetcc(SDNode *N);
559 SDValue foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1,
560 const SDLoc &DL);
561 SDValue foldSubToUSubSat(EVT DstVT, SDNode *N);
562 SDValue unfoldMaskedMerge(SDNode *N);
563 SDValue unfoldExtremeBitClearingToShifts(SDNode *N);
564 SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
565 const SDLoc &DL, bool foldBooleans);
566 SDValue rebuildSetCC(SDValue N);
567
568 bool isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
569 SDValue &CC, bool MatchStrict = false) const;
570 bool isOneUseSetCC(SDValue N) const;
571
572 SDValue SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
573 unsigned HiOp);
574 SDValue CombineConsecutiveLoads(SDNode *N, EVT VT);
575 SDValue CombineExtLoad(SDNode *N);
576 SDValue CombineZExtLogicopShiftLoad(SDNode *N);
577 SDValue combineRepeatedFPDivisors(SDNode *N);
578 SDValue combineInsertEltToShuffle(SDNode *N, unsigned InsIndex);
579 SDValue ConstantFoldBITCASTofBUILD_VECTOR(SDNode *, EVT);
580 SDValue BuildSDIV(SDNode *N);
581 SDValue BuildSDIVPow2(SDNode *N);
582 SDValue BuildUDIV(SDNode *N);
583 SDValue BuildSREMPow2(SDNode *N);
584 SDValue buildOptimizedSREM(SDValue N0, SDValue N1, SDNode *N);
585 SDValue BuildLogBase2(SDValue V, const SDLoc &DL);
586 SDValue BuildDivEstimate(SDValue N, SDValue Op, SDNodeFlags Flags);
587 SDValue buildRsqrtEstimate(SDValue Op, SDNodeFlags Flags);
588 SDValue buildSqrtEstimate(SDValue Op, SDNodeFlags Flags);
589 SDValue buildSqrtEstimateImpl(SDValue Op, SDNodeFlags Flags, bool Recip);
590 SDValue buildSqrtNROneConst(SDValue Arg, SDValue Est, unsigned Iterations,
591 SDNodeFlags Flags, bool Reciprocal);
592 SDValue buildSqrtNRTwoConst(SDValue Arg, SDValue Est, unsigned Iterations,
593 SDNodeFlags Flags, bool Reciprocal);
594 SDValue MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
595 bool DemandHighBits = true);
596 SDValue MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1);
597 SDValue MatchRotatePosNeg(SDValue Shifted, SDValue Pos, SDValue Neg,
598 SDValue InnerPos, SDValue InnerNeg, bool HasPos,
599 unsigned PosOpcode, unsigned NegOpcode,
600 const SDLoc &DL);
601 SDValue MatchFunnelPosNeg(SDValue N0, SDValue N1, SDValue Pos, SDValue Neg,
602 SDValue InnerPos, SDValue InnerNeg, bool HasPos,
603 unsigned PosOpcode, unsigned NegOpcode,
604 const SDLoc &DL);
605 SDValue MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL);
606 SDValue MatchLoadCombine(SDNode *N);
607 SDValue mergeTruncStores(StoreSDNode *N);
608 SDValue reduceLoadWidth(SDNode *N);
609 SDValue ReduceLoadOpStoreWidth(SDNode *N);
610 SDValue splitMergedValStore(StoreSDNode *ST);
611 SDValue TransformFPLoadStorePair(SDNode *N);
612 SDValue convertBuildVecZextToZext(SDNode *N);
613 SDValue reduceBuildVecExtToExtBuildVec(SDNode *N);
614 SDValue reduceBuildVecTruncToBitCast(SDNode *N);
615 SDValue reduceBuildVecToShuffle(SDNode *N);
616 SDValue createBuildVecShuffle(const SDLoc &DL, SDNode *N,
617 ArrayRef<int> VectorMask, SDValue VecIn1,
618 SDValue VecIn2, unsigned LeftIdx,
619 bool DidSplitVec);
620 SDValue matchVSelectOpSizesWithSetCC(SDNode *Cast);
621
622 /// Walk up chain skipping non-aliasing memory nodes,
623 /// looking for aliasing nodes and adding them to the Aliases vector.
624 void GatherAllAliases(SDNode *N, SDValue OriginalChain,
625 SmallVectorImpl<SDValue> &Aliases);
626
627 /// Return true if there is any possibility that the two addresses overlap.
628 bool mayAlias(SDNode *Op0, SDNode *Op1) const;
629
630 /// Walk up chain skipping non-aliasing memory nodes, looking for a better
631 /// chain (aliasing node.)
632 SDValue FindBetterChain(SDNode *N, SDValue Chain);
633
634 /// Try to replace a store and any possibly adjacent stores on
635 /// consecutive chains with better chains. Return true only if St is
636 /// replaced.
637 ///
638 /// Notice that other chains may still be replaced even if the function
639 /// returns false.
640 bool findBetterNeighborChains(StoreSDNode *St);
641
642 // Helper for findBetterNeighborChains. Walk up store chain add additional
643 // chained stores that do not overlap and can be parallelized.
644 bool parallelizeChainedStores(StoreSDNode *St);
645
646 /// Holds a pointer to an LSBaseSDNode as well as information on where it
647 /// is located in a sequence of memory operations connected by a chain.
648 struct MemOpLink {
649 // Ptr to the mem node.
650 LSBaseSDNode *MemNode;
651
652 // Offset from the base ptr.
653 int64_t OffsetFromBase;
654
655 MemOpLink(LSBaseSDNode *N, int64_t Offset)
656 : MemNode(N), OffsetFromBase(Offset) {}
657 };
658
659 // Classify the origin of a stored value.
660 enum class StoreSource { Unknown, Constant, Extract, Load };
661 StoreSource getStoreSource(SDValue StoreVal) {
662 switch (StoreVal.getOpcode()) {
663 case ISD::Constant:
664 case ISD::ConstantFP:
665 return StoreSource::Constant;
666 case ISD::EXTRACT_VECTOR_ELT:
667 case ISD::EXTRACT_SUBVECTOR:
668 return StoreSource::Extract;
669 case ISD::LOAD:
670 return StoreSource::Load;
671 default:
672 return StoreSource::Unknown;
673 }
674 }
675
676 /// This is a helper function for visitMUL to check the profitability
677 /// of folding (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2).
678 /// MulNode is the original multiply, AddNode is (add x, c1),
679 /// and ConstNode is c2.
680 bool isMulAddWithConstProfitable(SDNode *MulNode, SDValue AddNode,
681 SDValue ConstNode);
682
683 /// This is a helper function for visitAND and visitZERO_EXTEND. Returns
684 /// true if the (and (load x) c) pattern matches an extload. ExtVT returns
685 /// the type of the loaded value to be extended.
686 bool isAndLoadExtLoad(ConstantSDNode *AndC, LoadSDNode *LoadN,
687 EVT LoadResultTy, EVT &ExtVT);
688
689 /// Helper function to calculate whether the given Load/Store can have its
690 /// width reduced to ExtVT.
691 bool isLegalNarrowLdSt(LSBaseSDNode *LDSTN, ISD::LoadExtType ExtType,
692 EVT &MemVT, unsigned ShAmt = 0);
693
694 /// Used by BackwardsPropagateMask to find suitable loads.
695 bool SearchForAndLoads(SDNode *N, SmallVectorImpl<LoadSDNode*> &Loads,
696 SmallPtrSetImpl<SDNode*> &NodesWithConsts,
697 ConstantSDNode *Mask, SDNode *&NodeToMask);
698 /// Attempt to propagate a given AND node back to load leaves so that they
699 /// can be combined into narrow loads.
700 bool BackwardsPropagateMask(SDNode *N);
701
702 /// Helper function for mergeConsecutiveStores which merges the component
703 /// store chains.
704 SDValue getMergeStoreChains(SmallVectorImpl<MemOpLink> &StoreNodes,
705 unsigned NumStores);
706
707 /// This is a helper function for mergeConsecutiveStores. When the source
708 /// elements of the consecutive stores are all constants or all extracted
709 /// vector elements, try to merge them into one larger store introducing
710 /// bitcasts if necessary. \return True if a merged store was created.
711 bool mergeStoresOfConstantsOrVecElts(SmallVectorImpl<MemOpLink> &StoreNodes,
712 EVT MemVT, unsigned NumStores,
713 bool IsConstantSrc, bool UseVector,
714 bool UseTrunc);
715
716 /// This is a helper function for mergeConsecutiveStores. Stores that
717 /// potentially may be merged with St are placed in StoreNodes. RootNode is
718 /// a chain predecessor to all store candidates.
719 void getStoreMergeCandidates(StoreSDNode *St,
720 SmallVectorImpl<MemOpLink> &StoreNodes,
721 SDNode *&Root);
722
723 /// Helper function for mergeConsecutiveStores. Checks if candidate stores
724 /// have indirect dependency through their operands. RootNode is the
725 /// predecessor to all stores calculated by getStoreMergeCandidates and is
726 /// used to prune the dependency check. \return True if safe to merge.
727 bool checkMergeStoreCandidatesForDependencies(
728 SmallVectorImpl<MemOpLink> &StoreNodes, unsigned NumStores,
729 SDNode *RootNode);
730
731 /// This is a helper function for mergeConsecutiveStores. Given a list of
732 /// store candidates, find the first N that are consecutive in memory.
733 /// Returns 0 if there are not at least 2 consecutive stores to try merging.
734 unsigned getConsecutiveStores(SmallVectorImpl<MemOpLink> &StoreNodes,
735 int64_t ElementSizeBytes) const;
736
737 /// This is a helper function for mergeConsecutiveStores. It is used for
738 /// store chains that are composed entirely of constant values.
739 bool tryStoreMergeOfConstants(SmallVectorImpl<MemOpLink> &StoreNodes,
740 unsigned NumConsecutiveStores,
741 EVT MemVT, SDNode *Root, bool AllowVectors);
742
743 /// This is a helper function for mergeConsecutiveStores. It is used for
744 /// store chains that are composed entirely of extracted vector elements.
745 /// When extracting multiple vector elements, try to store them in one
746 /// vector store rather than a sequence of scalar stores.
747 bool tryStoreMergeOfExtracts(SmallVectorImpl<MemOpLink> &StoreNodes,
748 unsigned NumConsecutiveStores, EVT MemVT,
749 SDNode *Root);
750
751 /// This is a helper function for mergeConsecutiveStores. It is used for
752 /// store chains that are composed entirely of loaded values.
753 bool tryStoreMergeOfLoads(SmallVectorImpl<MemOpLink> &StoreNodes,
754 unsigned NumConsecutiveStores, EVT MemVT,
755 SDNode *Root, bool AllowVectors,
756 bool IsNonTemporalStore, bool IsNonTemporalLoad);
757
758 /// Merge consecutive store operations into a wide store.
759 /// This optimization uses wide integers or vectors when possible.
760 /// \return true if stores were merged.
761 bool mergeConsecutiveStores(StoreSDNode *St);
762
763 /// Try to transform a truncation where C is a constant:
764 /// (trunc (and X, C)) -> (and (trunc X), (trunc C))
765 ///
766 /// \p N needs to be a truncation and its first operand an AND. Other
767 /// requirements are checked by the function (e.g. that trunc is
768 /// single-use) and if missed an empty SDValue is returned.
769 SDValue distributeTruncateThroughAnd(SDNode *N);
770
771 /// Helper function to determine whether the target supports operation
772 /// given by \p Opcode for type \p VT, that is, whether the operation
773 /// is legal or custom before legalizing operations, and whether is
774 /// legal (but not custom) after legalization.
775 bool hasOperation(unsigned Opcode, EVT VT) {
776 return TLI.isOperationLegalOrCustom(Opcode, VT, LegalOperations);
777 }
778
779 public:
780 /// Runs the dag combiner on all nodes in the work list
781 void Run(CombineLevel AtLevel);
782
783 SelectionDAG &getDAG() const { return DAG; }
784
785 /// Returns a type large enough to hold any valid shift amount - before type
786 /// legalization these can be huge.
787 EVT getShiftAmountTy(EVT LHSTy) {
788 assert(LHSTy.isInteger() && "Shift amount is not an integer type!")(static_cast <bool> (LHSTy.isInteger() && "Shift amount is not an integer type!"
) ? void (0) : __assert_fail ("LHSTy.isInteger() && \"Shift amount is not an integer type!\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 788, __extension__
__PRETTY_FUNCTION__))
;
789 return TLI.getShiftAmountTy(LHSTy, DAG.getDataLayout(), LegalTypes);
790 }
791
792 /// This method returns true if we are running before type legalization or
793 /// if the specified VT is legal.
794 bool isTypeLegal(const EVT &VT) {
795 if (!LegalTypes) return true;
796 return TLI.isTypeLegal(VT);
797 }
798
799 /// Convenience wrapper around TargetLowering::getSetCCResultType
800 EVT getSetCCResultType(EVT VT) const {
801 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
802 }
803
804 void ExtendSetCCUses(const SmallVectorImpl<SDNode *> &SetCCs,
805 SDValue OrigLoad, SDValue ExtLoad,
806 ISD::NodeType ExtType);
807 };
808
809/// This class is a DAGUpdateListener that removes any deleted
810/// nodes from the worklist.
811class WorklistRemover : public SelectionDAG::DAGUpdateListener {
812 DAGCombiner &DC;
813
814public:
815 explicit WorklistRemover(DAGCombiner &dc)
816 : SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
817
818 void NodeDeleted(SDNode *N, SDNode *E) override {
819 DC.removeFromWorklist(N);
820 }
821};
822
823class WorklistInserter : public SelectionDAG::DAGUpdateListener {
824 DAGCombiner &DC;
825
826public:
827 explicit WorklistInserter(DAGCombiner &dc)
828 : SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
829
830 // FIXME: Ideally we could add N to the worklist, but this causes exponential
831 // compile time costs in large DAGs, e.g. Halide.
832 void NodeInserted(SDNode *N) override { DC.ConsiderForPruning(N); }
833};
834
835} // end anonymous namespace
836
837//===----------------------------------------------------------------------===//
838// TargetLowering::DAGCombinerInfo implementation
839//===----------------------------------------------------------------------===//
840
841void TargetLowering::DAGCombinerInfo::AddToWorklist(SDNode *N) {
842 ((DAGCombiner*)DC)->AddToWorklist(N);
843}
844
845SDValue TargetLowering::DAGCombinerInfo::
846CombineTo(SDNode *N, ArrayRef<SDValue> To, bool AddTo) {
847 return ((DAGCombiner*)DC)->CombineTo(N, &To[0], To.size(), AddTo);
848}
849
850SDValue TargetLowering::DAGCombinerInfo::
851CombineTo(SDNode *N, SDValue Res, bool AddTo) {
852 return ((DAGCombiner*)DC)->CombineTo(N, Res, AddTo);
853}
854
855SDValue TargetLowering::DAGCombinerInfo::
856CombineTo(SDNode *N, SDValue Res0, SDValue Res1, bool AddTo) {
857 return ((DAGCombiner*)DC)->CombineTo(N, Res0, Res1, AddTo);
858}
859
860bool TargetLowering::DAGCombinerInfo::
861recursivelyDeleteUnusedNodes(SDNode *N) {
862 return ((DAGCombiner*)DC)->recursivelyDeleteUnusedNodes(N);
863}
864
865void TargetLowering::DAGCombinerInfo::
866CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
867 return ((DAGCombiner*)DC)->CommitTargetLoweringOpt(TLO);
868}
869
870//===----------------------------------------------------------------------===//
871// Helper Functions
872//===----------------------------------------------------------------------===//
873
874void DAGCombiner::deleteAndRecombine(SDNode *N) {
875 removeFromWorklist(N);
876
877 // If the operands of this node are only used by the node, they will now be
878 // dead. Make sure to re-visit them and recursively delete dead nodes.
879 for (const SDValue &Op : N->ops())
880 // For an operand generating multiple values, one of the values may
881 // become dead allowing further simplification (e.g. split index
882 // arithmetic from an indexed load).
883 if (Op->hasOneUse() || Op->getNumValues() > 1)
884 AddToWorklist(Op.getNode());
885
886 DAG.DeleteNode(N);
887}
888
889// APInts must be the same size for most operations, this helper
890// function zero extends the shorter of the pair so that they match.
891// We provide an Offset so that we can create bitwidths that won't overflow.
892static void zeroExtendToMatch(APInt &LHS, APInt &RHS, unsigned Offset = 0) {
893 unsigned Bits = Offset + std::max(LHS.getBitWidth(), RHS.getBitWidth());
894 LHS = LHS.zext(Bits);
895 RHS = RHS.zext(Bits);
896}
897
898// Return true if this node is a setcc, or is a select_cc
899// that selects between the target values used for true and false, making it
900// equivalent to a setcc. Also, set the incoming LHS, RHS, and CC references to
901// the appropriate nodes based on the type of node we are checking. This
902// simplifies life a bit for the callers.
903bool DAGCombiner::isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
904 SDValue &CC, bool MatchStrict) const {
905 if (N.getOpcode() == ISD::SETCC) {
906 LHS = N.getOperand(0);
907 RHS = N.getOperand(1);
908 CC = N.getOperand(2);
909 return true;
910 }
911
912 if (MatchStrict &&
913 (N.getOpcode() == ISD::STRICT_FSETCC ||
914 N.getOpcode() == ISD::STRICT_FSETCCS)) {
915 LHS = N.getOperand(1);
916 RHS = N.getOperand(2);
917 CC = N.getOperand(3);
918 return true;
919 }
920
921 if (N.getOpcode() != ISD::SELECT_CC || !TLI.isConstTrueVal(N.getOperand(2)) ||
922 !TLI.isConstFalseVal(N.getOperand(3)))
923 return false;
924
925 if (TLI.getBooleanContents(N.getValueType()) ==
926 TargetLowering::UndefinedBooleanContent)
927 return false;
928
929 LHS = N.getOperand(0);
930 RHS = N.getOperand(1);
931 CC = N.getOperand(4);
932 return true;
933}
934
935/// Return true if this is a SetCC-equivalent operation with only one use.
936/// If this is true, it allows the users to invert the operation for free when
937/// it is profitable to do so.
938bool DAGCombiner::isOneUseSetCC(SDValue N) const {
939 SDValue N0, N1, N2;
940 if (isSetCCEquivalent(N, N0, N1, N2) && N->hasOneUse())
941 return true;
942 return false;
943}
944
945static bool isConstantSplatVectorMaskForType(SDNode *N, EVT ScalarTy) {
946 if (!ScalarTy.isSimple())
947 return false;
948
949 uint64_t MaskForTy = 0ULL;
950 switch (ScalarTy.getSimpleVT().SimpleTy) {
951 case MVT::i8:
952 MaskForTy = 0xFFULL;
953 break;
954 case MVT::i16:
955 MaskForTy = 0xFFFFULL;
956 break;
957 case MVT::i32:
958 MaskForTy = 0xFFFFFFFFULL;
959 break;
960 default:
961 return false;
962 break;
963 }
964
965 APInt Val;
966 if (ISD::isConstantSplatVector(N, Val))
967 return Val.getLimitedValue() == MaskForTy;
968
969 return false;
970}
971
972// Determines if it is a constant integer or a splat/build vector of constant
973// integers (and undefs).
974// Do not permit build vector implicit truncation.
975static bool isConstantOrConstantVector(SDValue N, bool NoOpaques = false) {
976 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N))
977 return !(Const->isOpaque() && NoOpaques);
978 if (N.getOpcode() != ISD::BUILD_VECTOR && N.getOpcode() != ISD::SPLAT_VECTOR)
979 return false;
980 unsigned BitWidth = N.getScalarValueSizeInBits();
981 for (const SDValue &Op : N->op_values()) {
982 if (Op.isUndef())
983 continue;
984 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Op);
985 if (!Const || Const->getAPIntValue().getBitWidth() != BitWidth ||
986 (Const->isOpaque() && NoOpaques))
987 return false;
988 }
989 return true;
990}
991
992// Determines if a BUILD_VECTOR is composed of all-constants possibly mixed with
993// undef's.
994static bool isAnyConstantBuildVector(SDValue V, bool NoOpaques = false) {
995 if (V.getOpcode() != ISD::BUILD_VECTOR)
996 return false;
997 return isConstantOrConstantVector(V, NoOpaques) ||
998 ISD::isBuildVectorOfConstantFPSDNodes(V.getNode());
999}
1000
1001// Determine if this an indexed load with an opaque target constant index.
1002static bool canSplitIdx(LoadSDNode *LD) {
1003 return MaySplitLoadIndex &&
1004 (LD->getOperand(2).getOpcode() != ISD::TargetConstant ||
1005 !cast<ConstantSDNode>(LD->getOperand(2))->isOpaque());
1006}
1007
1008bool DAGCombiner::reassociationCanBreakAddressingModePattern(unsigned Opc,
1009 const SDLoc &DL,
1010 SDNode *N,
1011 SDValue N0,
1012 SDValue N1) {
1013 // Currently this only tries to ensure we don't undo the GEP splits done by
1014 // CodeGenPrepare when shouldConsiderGEPOffsetSplit is true. To ensure this,
1015 // we check if the following transformation would be problematic:
1016 // (load/store (add, (add, x, offset1), offset2)) ->
1017 // (load/store (add, x, offset1+offset2)).
1018
1019 // (load/store (add, (add, x, y), offset2)) ->
1020 // (load/store (add, (add, x, offset2), y)).
1021
1022 if (Opc != ISD::ADD || N0.getOpcode() != ISD::ADD)
1023 return false;
1024
1025 auto *C2 = dyn_cast<ConstantSDNode>(N1);
1026 if (!C2)
1027 return false;
1028
1029 const APInt &C2APIntVal = C2->getAPIntValue();
1030 if (C2APIntVal.getSignificantBits() > 64)
1031 return false;
1032
1033 if (auto *C1 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
1034 if (N0.hasOneUse())
1035 return false;
1036
1037 const APInt &C1APIntVal = C1->getAPIntValue();
1038 const APInt CombinedValueIntVal = C1APIntVal + C2APIntVal;
1039 if (CombinedValueIntVal.getSignificantBits() > 64)
1040 return false;
1041 const int64_t CombinedValue = CombinedValueIntVal.getSExtValue();
1042
1043 for (SDNode *Node : N->uses()) {
1044 if (auto *LoadStore = dyn_cast<MemSDNode>(Node)) {
1045 // Is x[offset2] already not a legal addressing mode? If so then
1046 // reassociating the constants breaks nothing (we test offset2 because
1047 // that's the one we hope to fold into the load or store).
1048 TargetLoweringBase::AddrMode AM;
1049 AM.HasBaseReg = true;
1050 AM.BaseOffs = C2APIntVal.getSExtValue();
1051 EVT VT = LoadStore->getMemoryVT();
1052 unsigned AS = LoadStore->getAddressSpace();
1053 Type *AccessTy = VT.getTypeForEVT(*DAG.getContext());
1054 if (!TLI.isLegalAddressingMode(DAG.getDataLayout(), AM, AccessTy, AS))
1055 continue;
1056
1057 // Would x[offset1+offset2] still be a legal addressing mode?
1058 AM.BaseOffs = CombinedValue;
1059 if (!TLI.isLegalAddressingMode(DAG.getDataLayout(), AM, AccessTy, AS))
1060 return true;
1061 }
1062 }
1063 } else {
1064 if (auto *GA = dyn_cast<GlobalAddressSDNode>(N0.getOperand(1)))
1065 if (GA->getOpcode() == ISD::GlobalAddress && TLI.isOffsetFoldingLegal(GA))
1066 return false;
1067
1068 for (SDNode *Node : N->uses()) {
1069 auto *LoadStore = dyn_cast<MemSDNode>(Node);
1070 if (!LoadStore)
1071 return false;
1072
1073 // Is x[offset2] a legal addressing mode? If so then
1074 // reassociating the constants breaks address pattern
1075 TargetLoweringBase::AddrMode AM;
1076 AM.HasBaseReg = true;
1077 AM.BaseOffs = C2APIntVal.getSExtValue();
1078 EVT VT = LoadStore->getMemoryVT();
1079 unsigned AS = LoadStore->getAddressSpace();
1080 Type *AccessTy = VT.getTypeForEVT(*DAG.getContext());
1081 if (!TLI.isLegalAddressingMode(DAG.getDataLayout(), AM, AccessTy, AS))
1082 return false;
1083 }
1084 return true;
1085 }
1086
1087 return false;
1088}
1089
1090// Helper for DAGCombiner::reassociateOps. Try to reassociate an expression
1091// such as (Opc N0, N1), if \p N0 is the same kind of operation as \p Opc.
1092SDValue DAGCombiner::reassociateOpsCommutative(unsigned Opc, const SDLoc &DL,
1093 SDValue N0, SDValue N1) {
1094 EVT VT = N0.getValueType();
1095
1096 if (N0.getOpcode() != Opc)
1097 return SDValue();
1098
1099 SDValue N00 = N0.getOperand(0);
1100 SDValue N01 = N0.getOperand(1);
1101
1102 if (DAG.isConstantIntBuildVectorOrConstantInt(peekThroughBitcasts(N01))) {
1103 if (DAG.isConstantIntBuildVectorOrConstantInt(peekThroughBitcasts(N1))) {
1104 // Reassociate: (op (op x, c1), c2) -> (op x, (op c1, c2))
1105 if (SDValue OpNode = DAG.FoldConstantArithmetic(Opc, DL, VT, {N01, N1}))
1106 return DAG.getNode(Opc, DL, VT, N00, OpNode);
1107 return SDValue();
1108 }
1109 if (TLI.isReassocProfitable(DAG, N0, N1)) {
1110 // Reassociate: (op (op x, c1), y) -> (op (op x, y), c1)
1111 // iff (op x, c1) has one use
1112 SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N00, N1);
1113 return DAG.getNode(Opc, DL, VT, OpNode, N01);
1114 }
1115 }
1116
1117 // Check for repeated operand logic simplifications.
1118 if (Opc == ISD::AND || Opc == ISD::OR) {
1119 // (N00 & N01) & N00 --> N00 & N01
1120 // (N00 & N01) & N01 --> N00 & N01
1121 // (N00 | N01) | N00 --> N00 | N01
1122 // (N00 | N01) | N01 --> N00 | N01
1123 if (N1 == N00 || N1 == N01)
1124 return N0;
1125 }
1126 if (Opc == ISD::XOR) {
1127 // (N00 ^ N01) ^ N00 --> N01
1128 if (N1 == N00)
1129 return N01;
1130 // (N00 ^ N01) ^ N01 --> N00
1131 if (N1 == N01)
1132 return N00;
1133 }
1134
1135 if (TLI.isReassocProfitable(DAG, N0, N1)) {
1136 if (N1 != N01) {
1137 // Reassociate if (op N00, N1) already exist
1138 if (SDNode *NE = DAG.getNodeIfExists(Opc, DAG.getVTList(VT), {N00, N1})) {
1139 // if Op (Op N00, N1), N01 already exist
1140 // we need to stop reassciate to avoid dead loop
1141 if (!DAG.doesNodeExist(Opc, DAG.getVTList(VT), {SDValue(NE, 0), N01}))
1142 return DAG.getNode(Opc, DL, VT, SDValue(NE, 0), N01);
1143 }
1144 }
1145
1146 if (N1 != N00) {
1147 // Reassociate if (op N01, N1) already exist
1148 if (SDNode *NE = DAG.getNodeIfExists(Opc, DAG.getVTList(VT), {N01, N1})) {
1149 // if Op (Op N01, N1), N00 already exist
1150 // we need to stop reassciate to avoid dead loop
1151 if (!DAG.doesNodeExist(Opc, DAG.getVTList(VT), {SDValue(NE, 0), N00}))
1152 return DAG.getNode(Opc, DL, VT, SDValue(NE, 0), N00);
1153 }
1154 }
1155 }
1156
1157 return SDValue();
1158}
1159
1160// Try to reassociate commutative binops.
1161SDValue DAGCombiner::reassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0,
1162 SDValue N1, SDNodeFlags Flags) {
1163 assert(TLI.isCommutativeBinOp(Opc) && "Operation not commutative.")(static_cast <bool> (TLI.isCommutativeBinOp(Opc) &&
"Operation not commutative.") ? void (0) : __assert_fail ("TLI.isCommutativeBinOp(Opc) && \"Operation not commutative.\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 1163, __extension__
__PRETTY_FUNCTION__))
;
1164
1165 // Floating-point reassociation is not allowed without loose FP math.
1166 if (N0.getValueType().isFloatingPoint() ||
1167 N1.getValueType().isFloatingPoint())
1168 if (!Flags.hasAllowReassociation() || !Flags.hasNoSignedZeros())
1169 return SDValue();
1170
1171 if (SDValue Combined = reassociateOpsCommutative(Opc, DL, N0, N1))
1172 return Combined;
1173 if (SDValue Combined = reassociateOpsCommutative(Opc, DL, N1, N0))
1174 return Combined;
1175 return SDValue();
1176}
1177
1178SDValue DAGCombiner::CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
1179 bool AddTo) {
1180 assert(N->getNumValues() == NumTo && "Broken CombineTo call!")(static_cast <bool> (N->getNumValues() == NumTo &&
"Broken CombineTo call!") ? void (0) : __assert_fail ("N->getNumValues() == NumTo && \"Broken CombineTo call!\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 1180, __extension__
__PRETTY_FUNCTION__))
;
1181 ++NodesCombined;
1182 LLVM_DEBUG(dbgs() << "\nReplacing.1 "; N->dump(&DAG); dbgs() << "\nWith: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.1 "; N->dump
(&DAG); dbgs() << "\nWith: "; To[0].dump(&DAG);
dbgs() << " and " << NumTo - 1 << " other values\n"
; } } while (false)
1183 To[0].dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.1 "; N->dump
(&DAG); dbgs() << "\nWith: "; To[0].dump(&DAG);
dbgs() << " and " << NumTo - 1 << " other values\n"
; } } while (false)
1184 dbgs() << " and " << NumTo - 1 << " other values\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.1 "; N->dump
(&DAG); dbgs() << "\nWith: "; To[0].dump(&DAG);
dbgs() << " and " << NumTo - 1 << " other values\n"
; } } while (false)
;
1185 for (unsigned i = 0, e = NumTo; i != e; ++i)
1186 assert((!To[i].getNode() ||(static_cast <bool> ((!To[i].getNode() || N->getValueType
(i) == To[i].getValueType()) && "Cannot combine value to value of different type!"
) ? void (0) : __assert_fail ("(!To[i].getNode() || N->getValueType(i) == To[i].getValueType()) && \"Cannot combine value to value of different type!\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 1188, __extension__
__PRETTY_FUNCTION__))
1187 N->getValueType(i) == To[i].getValueType()) &&(static_cast <bool> ((!To[i].getNode() || N->getValueType
(i) == To[i].getValueType()) && "Cannot combine value to value of different type!"
) ? void (0) : __assert_fail ("(!To[i].getNode() || N->getValueType(i) == To[i].getValueType()) && \"Cannot combine value to value of different type!\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 1188, __extension__
__PRETTY_FUNCTION__))
1188 "Cannot combine value to value of different type!")(static_cast <bool> ((!To[i].getNode() || N->getValueType
(i) == To[i].getValueType()) && "Cannot combine value to value of different type!"
) ? void (0) : __assert_fail ("(!To[i].getNode() || N->getValueType(i) == To[i].getValueType()) && \"Cannot combine value to value of different type!\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 1188, __extension__
__PRETTY_FUNCTION__))
;
1189
1190 WorklistRemover DeadNodes(*this);
1191 DAG.ReplaceAllUsesWith(N, To);
1192 if (AddTo) {
1193 // Push the new nodes and any users onto the worklist
1194 for (unsigned i = 0, e = NumTo; i != e; ++i) {
1195 if (To[i].getNode())
1196 AddToWorklistWithUsers(To[i].getNode());
1197 }
1198 }
1199
1200 // Finally, if the node is now dead, remove it from the graph. The node
1201 // may not be dead if the replacement process recursively simplified to
1202 // something else needing this node.
1203 if (N->use_empty())
1204 deleteAndRecombine(N);
1205 return SDValue(N, 0);
1206}
1207
1208void DAGCombiner::
1209CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
1210 // Replace the old value with the new one.
1211 ++NodesCombined;
1212 LLVM_DEBUG(dbgs() << "\nReplacing.2 "; TLO.Old.dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.dump
(&DAG); dbgs() << "\nWith: "; TLO.New.dump(&DAG
); dbgs() << '\n'; } } while (false)
1213 dbgs() << "\nWith: "; TLO.New.dump(&DAG); dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.dump
(&DAG); dbgs() << "\nWith: "; TLO.New.dump(&DAG
); dbgs() << '\n'; } } while (false)
;
1214
1215 // Replace all uses.
1216 DAG.ReplaceAllUsesOfValueWith(TLO.Old, TLO.New);
1217
1218 // Push the new node and any (possibly new) users onto the worklist.
1219 AddToWorklistWithUsers(TLO.New.getNode());
1220
1221 // Finally, if the node is now dead, remove it from the graph.
1222 recursivelyDeleteUnusedNodes(TLO.Old.getNode());
1223}
1224
1225/// Check the specified integer node value to see if it can be simplified or if
1226/// things it uses can be simplified by bit propagation. If so, return true.
1227bool DAGCombiner::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
1228 const APInt &DemandedElts,
1229 bool AssumeSingleUse) {
1230 TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations);
1231 KnownBits Known;
1232 if (!TLI.SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO, 0,
1233 AssumeSingleUse))
1234 return false;
1235
1236 // Revisit the node.
1237 AddToWorklist(Op.getNode());
1238
1239 CommitTargetLoweringOpt(TLO);
1240 return true;
1241}
1242
1243/// Check the specified vector node value to see if it can be simplified or
1244/// if things it uses can be simplified as it only uses some of the elements.
1245/// If so, return true.
1246bool DAGCombiner::SimplifyDemandedVectorElts(SDValue Op,
1247 const APInt &DemandedElts,
1248 bool AssumeSingleUse) {
1249 TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations);
1250 APInt KnownUndef, KnownZero;
1251 if (!TLI.SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero,
1252 TLO, 0, AssumeSingleUse))
1253 return false;
1254
1255 // Revisit the node.
1256 AddToWorklist(Op.getNode());
1257
1258 CommitTargetLoweringOpt(TLO);
1259 return true;
1260}
1261
1262void DAGCombiner::ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad) {
1263 SDLoc DL(Load);
1264 EVT VT = Load->getValueType(0);
1265 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, VT, SDValue(ExtLoad, 0));
1266
1267 LLVM_DEBUG(dbgs() << "\nReplacing.9 "; Load->dump(&DAG); dbgs() << "\nWith: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.9 "; Load->
dump(&DAG); dbgs() << "\nWith: "; Trunc.dump(&DAG
); dbgs() << '\n'; } } while (false)
1268 Trunc.dump(&DAG); dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.9 "; Load->
dump(&DAG); dbgs() << "\nWith: "; Trunc.dump(&DAG
); dbgs() << '\n'; } } while (false)
;
1269
1270 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), Trunc);
1271 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), SDValue(ExtLoad, 1));
1272
1273 AddToWorklist(Trunc.getNode());
1274 recursivelyDeleteUnusedNodes(Load);
1275}
1276
1277SDValue DAGCombiner::PromoteOperand(SDValue Op, EVT PVT, bool &Replace) {
1278 Replace = false;
1279 SDLoc DL(Op);
1280 if (ISD::isUNINDEXEDLoad(Op.getNode())) {
1281 LoadSDNode *LD = cast<LoadSDNode>(Op);
1282 EVT MemVT = LD->getMemoryVT();
1283 ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD) ? ISD::EXTLOAD
1284 : LD->getExtensionType();
1285 Replace = true;
1286 return DAG.getExtLoad(ExtType, DL, PVT,
1287 LD->getChain(), LD->getBasePtr(),
1288 MemVT, LD->getMemOperand());
1289 }
1290
1291 unsigned Opc = Op.getOpcode();
1292 switch (Opc) {
1293 default: break;
1294 case ISD::AssertSext:
1295 if (SDValue Op0 = SExtPromoteOperand(Op.getOperand(0), PVT))
1296 return DAG.getNode(ISD::AssertSext, DL, PVT, Op0, Op.getOperand(1));
1297 break;
1298 case ISD::AssertZext:
1299 if (SDValue Op0 = ZExtPromoteOperand(Op.getOperand(0), PVT))
1300 return DAG.getNode(ISD::AssertZext, DL, PVT, Op0, Op.getOperand(1));
1301 break;
1302 case ISD::Constant: {
1303 unsigned ExtOpc =
1304 Op.getValueType().isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
1305 return DAG.getNode(ExtOpc, DL, PVT, Op);
1306 }
1307 }
1308
1309 if (!TLI.isOperationLegal(ISD::ANY_EXTEND, PVT))
1310 return SDValue();
1311 return DAG.getNode(ISD::ANY_EXTEND, DL, PVT, Op);
1312}
1313
1314SDValue DAGCombiner::SExtPromoteOperand(SDValue Op, EVT PVT) {
1315 if (!TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, PVT))
1316 return SDValue();
1317 EVT OldVT = Op.getValueType();
1318 SDLoc DL(Op);
1319 bool Replace = false;
1320 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
1321 if (!NewOp.getNode())
1322 return SDValue();
1323 AddToWorklist(NewOp.getNode());
1324
1325 if (Replace)
1326 ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
1327 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, NewOp.getValueType(), NewOp,
1328 DAG.getValueType(OldVT));
1329}
1330
1331SDValue DAGCombiner::ZExtPromoteOperand(SDValue Op, EVT PVT) {
1332 EVT OldVT = Op.getValueType();
1333 SDLoc DL(Op);
1334 bool Replace = false;
1335 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
1336 if (!NewOp.getNode())
1337 return SDValue();
1338 AddToWorklist(NewOp.getNode());
1339
1340 if (Replace)
1341 ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
1342 return DAG.getZeroExtendInReg(NewOp, DL, OldVT);
1343}
1344
1345/// Promote the specified integer binary operation if the target indicates it is
1346/// beneficial. e.g. On x86, it's usually better to promote i16 operations to
1347/// i32 since i16 instructions are longer.
1348SDValue DAGCombiner::PromoteIntBinOp(SDValue Op) {
1349 if (!LegalOperations)
1350 return SDValue();
1351
1352 EVT VT = Op.getValueType();
1353 if (VT.isVector() || !VT.isInteger())
1354 return SDValue();
1355
1356 // If operation type is 'undesirable', e.g. i16 on x86, consider
1357 // promoting it.
1358 unsigned Opc = Op.getOpcode();
1359 if (TLI.isTypeDesirableForOp(Opc, VT))
1360 return SDValue();
1361
1362 EVT PVT = VT;
1363 // Consult target whether it is a good idea to promote this operation and
1364 // what's the right type to promote it to.
1365 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1366 assert(PVT != VT && "Don't know what type to promote to!")(static_cast <bool> (PVT != VT && "Don't know what type to promote to!"
) ? void (0) : __assert_fail ("PVT != VT && \"Don't know what type to promote to!\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 1366, __extension__
__PRETTY_FUNCTION__))
;
1367
1368 LLVM_DEBUG(dbgs() << "\nPromoting "; Op.dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; Op.dump(&
DAG); } } while (false)
;
1369
1370 bool Replace0 = false;
1371 SDValue N0 = Op.getOperand(0);
1372 SDValue NN0 = PromoteOperand(N0, PVT, Replace0);
1373
1374 bool Replace1 = false;
1375 SDValue N1 = Op.getOperand(1);
1376 SDValue NN1 = PromoteOperand(N1, PVT, Replace1);
1377 SDLoc DL(Op);
1378
1379 SDValue RV =
1380 DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, NN0, NN1));
1381
1382 // We are always replacing N0/N1's use in N and only need additional
1383 // replacements if there are additional uses.
1384 // Note: We are checking uses of the *nodes* (SDNode) rather than values
1385 // (SDValue) here because the node may reference multiple values
1386 // (for example, the chain value of a load node).
1387 Replace0 &= !N0->hasOneUse();
1388 Replace1 &= (N0 != N1) && !N1->hasOneUse();
1389
1390 // Combine Op here so it is preserved past replacements.
1391 CombineTo(Op.getNode(), RV);
1392
1393 // If operands have a use ordering, make sure we deal with
1394 // predecessor first.
1395 if (Replace0 && Replace1 && N0->isPredecessorOf(N1.getNode())) {
1396 std::swap(N0, N1);
1397 std::swap(NN0, NN1);
1398 }
1399
1400 if (Replace0) {
1401 AddToWorklist(NN0.getNode());
1402 ReplaceLoadWithPromotedLoad(N0.getNode(), NN0.getNode());
1403 }
1404 if (Replace1) {
1405 AddToWorklist(NN1.getNode());
1406 ReplaceLoadWithPromotedLoad(N1.getNode(), NN1.getNode());
1407 }
1408 return Op;
1409 }
1410 return SDValue();
1411}
1412
1413/// Promote the specified integer shift operation if the target indicates it is
1414/// beneficial. e.g. On x86, it's usually better to promote i16 operations to
1415/// i32 since i16 instructions are longer.
1416SDValue DAGCombiner::PromoteIntShiftOp(SDValue Op) {
1417 if (!LegalOperations)
1418 return SDValue();
1419
1420 EVT VT = Op.getValueType();
1421 if (VT.isVector() || !VT.isInteger())
1422 return SDValue();
1423
1424 // If operation type is 'undesirable', e.g. i16 on x86, consider
1425 // promoting it.
1426 unsigned Opc = Op.getOpcode();
1427 if (TLI.isTypeDesirableForOp(Opc, VT))
1428 return SDValue();
1429
1430 EVT PVT = VT;
1431 // Consult target whether it is a good idea to promote this operation and
1432 // what's the right type to promote it to.
1433 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1434 assert(PVT != VT && "Don't know what type to promote to!")(static_cast <bool> (PVT != VT && "Don't know what type to promote to!"
) ? void (0) : __assert_fail ("PVT != VT && \"Don't know what type to promote to!\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 1434, __extension__
__PRETTY_FUNCTION__))
;
1435
1436 LLVM_DEBUG(dbgs() << "\nPromoting "; Op.dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; Op.dump(&
DAG); } } while (false)
;
1437
1438 bool Replace = false;
1439 SDValue N0 = Op.getOperand(0);
1440 if (Opc == ISD::SRA)
1441 N0 = SExtPromoteOperand(N0, PVT);
1442 else if (Opc == ISD::SRL)
1443 N0 = ZExtPromoteOperand(N0, PVT);
1444 else
1445 N0 = PromoteOperand(N0, PVT, Replace);
1446
1447 if (!N0.getNode())
1448 return SDValue();
1449
1450 SDLoc DL(Op);
1451 SDValue N1 = Op.getOperand(1);
1452 SDValue RV =
1453 DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, N0, N1));
1454
1455 if (Replace)
1456 ReplaceLoadWithPromotedLoad(Op.getOperand(0).getNode(), N0.getNode());
1457
1458 // Deal with Op being deleted.
1459 if (Op && Op.getOpcode() != ISD::DELETED_NODE)
1460 return RV;
1461 }
1462 return SDValue();
1463}
1464
1465SDValue DAGCombiner::PromoteExtend(SDValue Op) {
1466 if (!LegalOperations)
1467 return SDValue();
1468
1469 EVT VT = Op.getValueType();
1470 if (VT.isVector() || !VT.isInteger())
1471 return SDValue();
1472
1473 // If operation type is 'undesirable', e.g. i16 on x86, consider
1474 // promoting it.
1475 unsigned Opc = Op.getOpcode();
1476 if (TLI.isTypeDesirableForOp(Opc, VT))
1477 return SDValue();
1478
1479 EVT PVT = VT;
1480 // Consult target whether it is a good idea to promote this operation and
1481 // what's the right type to promote it to.
1482 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1483 assert(PVT != VT && "Don't know what type to promote to!")(static_cast <bool> (PVT != VT && "Don't know what type to promote to!"
) ? void (0) : __assert_fail ("PVT != VT && \"Don't know what type to promote to!\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 1483, __extension__
__PRETTY_FUNCTION__))
;
1484 // fold (aext (aext x)) -> (aext x)
1485 // fold (aext (zext x)) -> (zext x)
1486 // fold (aext (sext x)) -> (sext x)
1487 LLVM_DEBUG(dbgs() << "\nPromoting "; Op.dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; Op.dump(&
DAG); } } while (false)
;
1488 return DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, Op.getOperand(0));
1489 }
1490 return SDValue();
1491}
1492
1493bool DAGCombiner::PromoteLoad(SDValue Op) {
1494 if (!LegalOperations)
1495 return false;
1496
1497 if (!ISD::isUNINDEXEDLoad(Op.getNode()))
1498 return false;
1499
1500 EVT VT = Op.getValueType();
1501 if (VT.isVector() || !VT.isInteger())
1502 return false;
1503
1504 // If operation type is 'undesirable', e.g. i16 on x86, consider
1505 // promoting it.
1506 unsigned Opc = Op.getOpcode();
1507 if (TLI.isTypeDesirableForOp(Opc, VT))
1508 return false;
1509
1510 EVT PVT = VT;
1511 // Consult target whether it is a good idea to promote this operation and
1512 // what's the right type to promote it to.
1513 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1514 assert(PVT != VT && "Don't know what type to promote to!")(static_cast <bool> (PVT != VT && "Don't know what type to promote to!"
) ? void (0) : __assert_fail ("PVT != VT && \"Don't know what type to promote to!\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 1514, __extension__
__PRETTY_FUNCTION__))
;
1515
1516 SDLoc DL(Op);
1517 SDNode *N = Op.getNode();
1518 LoadSDNode *LD = cast<LoadSDNode>(N);
1519 EVT MemVT = LD->getMemoryVT();
1520 ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD) ? ISD::EXTLOAD
1521 : LD->getExtensionType();
1522 SDValue NewLD = DAG.getExtLoad(ExtType, DL, PVT,
1523 LD->getChain(), LD->getBasePtr(),
1524 MemVT, LD->getMemOperand());
1525 SDValue Result = DAG.getNode(ISD::TRUNCATE, DL, VT, NewLD);
1526
1527 LLVM_DEBUG(dbgs() << "\nPromoting "; N->dump(&DAG); dbgs() << "\nTo: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; N->dump(
&DAG); dbgs() << "\nTo: "; Result.dump(&DAG); dbgs
() << '\n'; } } while (false)
1528 Result.dump(&DAG); dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; N->dump(
&DAG); dbgs() << "\nTo: "; Result.dump(&DAG); dbgs
() << '\n'; } } while (false)
;
1529
1530 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result);
1531 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), NewLD.getValue(1));
1532
1533 AddToWorklist(Result.getNode());
1534 recursivelyDeleteUnusedNodes(N);
1535 return true;
1536 }
1537
1538 return false;
1539}
1540
1541/// Recursively delete a node which has no uses and any operands for
1542/// which it is the only use.
1543///
1544/// Note that this both deletes the nodes and removes them from the worklist.
1545/// It also adds any nodes who have had a user deleted to the worklist as they
1546/// may now have only one use and subject to other combines.
1547bool DAGCombiner::recursivelyDeleteUnusedNodes(SDNode *N) {
1548 if (!N->use_empty())
1549 return false;
1550
1551 SmallSetVector<SDNode *, 16> Nodes;
1552 Nodes.insert(N);
1553 do {
1554 N = Nodes.pop_back_val();
1555 if (!N)
1556 continue;
1557
1558 if (N->use_empty()) {
1559 for (const SDValue &ChildN : N->op_values())
1560 Nodes.insert(ChildN.getNode());
1561
1562 removeFromWorklist(N);
1563 DAG.DeleteNode(N);
1564 } else {
1565 AddToWorklist(N);
1566 }
1567 } while (!Nodes.empty());
1568 return true;
1569}
1570
1571//===----------------------------------------------------------------------===//
1572// Main DAG Combiner implementation
1573//===----------------------------------------------------------------------===//
1574
1575void DAGCombiner::Run(CombineLevel AtLevel) {
1576 // set the instance variables, so that the various visit routines may use it.
1577 Level = AtLevel;
1578 LegalDAG = Level >= AfterLegalizeDAG;
1579 LegalOperations = Level >= AfterLegalizeVectorOps;
1580 LegalTypes = Level >= AfterLegalizeTypes;
1581
1582 WorklistInserter AddNodes(*this);
1583
1584 // Add all the dag nodes to the worklist.
1585 for (SDNode &Node : DAG.allnodes())
1586 AddToWorklist(&Node);
1587
1588 // Create a dummy node (which is not added to allnodes), that adds a reference
1589 // to the root node, preventing it from being deleted, and tracking any
1590 // changes of the root.
1591 HandleSDNode Dummy(DAG.getRoot());
1592
1593 // While we have a valid worklist entry node, try to combine it.
1594 while (SDNode *N = getNextWorklistEntry()) {
1595 // If N has no uses, it is dead. Make sure to revisit all N's operands once
1596 // N is deleted from the DAG, since they too may now be dead or may have a
1597 // reduced number of uses, allowing other xforms.
1598 if (recursivelyDeleteUnusedNodes(N))
1599 continue;
1600
1601 WorklistRemover DeadNodes(*this);
1602
1603 // If this combine is running after legalizing the DAG, re-legalize any
1604 // nodes pulled off the worklist.
1605 if (LegalDAG) {
1606 SmallSetVector<SDNode *, 16> UpdatedNodes;
1607 bool NIsValid = DAG.LegalizeOp(N, UpdatedNodes);
1608
1609 for (SDNode *LN : UpdatedNodes)
1610 AddToWorklistWithUsers(LN);
1611
1612 if (!NIsValid)
1613 continue;
1614 }
1615
1616 LLVM_DEBUG(dbgs() << "\nCombining: "; N->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nCombining: "; N->dump
(&DAG); } } while (false)
;
1617
1618 // Add any operands of the new node which have not yet been combined to the
1619 // worklist as well. Because the worklist uniques things already, this
1620 // won't repeatedly process the same operand.
1621 CombinedNodes.insert(N);
1622 for (const SDValue &ChildN : N->op_values())
1623 if (!CombinedNodes.count(ChildN.getNode()))
1624 AddToWorklist(ChildN.getNode());
1625
1626 SDValue RV = combine(N);
1627
1628 if (!RV.getNode())
1629 continue;
1630
1631 ++NodesCombined;
1632
1633 // If we get back the same node we passed in, rather than a new node or
1634 // zero, we know that the node must have defined multiple values and
1635 // CombineTo was used. Since CombineTo takes care of the worklist
1636 // mechanics for us, we have no work to do in this case.
1637 if (RV.getNode() == N)
1638 continue;
1639
1640 assert(N->getOpcode() != ISD::DELETED_NODE &&(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& RV.getOpcode() != ISD::DELETED_NODE && "Node was deleted but visit returned new node!"
) ? void (0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && RV.getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned new node!\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 1642, __extension__
__PRETTY_FUNCTION__))
1641 RV.getOpcode() != ISD::DELETED_NODE &&(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& RV.getOpcode() != ISD::DELETED_NODE && "Node was deleted but visit returned new node!"
) ? void (0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && RV.getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned new node!\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 1642, __extension__
__PRETTY_FUNCTION__))
1642 "Node was deleted but visit returned new node!")(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& RV.getOpcode() != ISD::DELETED_NODE && "Node was deleted but visit returned new node!"
) ? void (0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && RV.getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned new node!\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 1642, __extension__
__PRETTY_FUNCTION__))
;
1643
1644 LLVM_DEBUG(dbgs() << " ... into: "; RV.dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << " ... into: "; RV.dump(&
DAG); } } while (false)
;
1645
1646 if (N->getNumValues() == RV->getNumValues())
1647 DAG.ReplaceAllUsesWith(N, RV.getNode());
1648 else {
1649 assert(N->getValueType(0) == RV.getValueType() &&(static_cast <bool> (N->getValueType(0) == RV.getValueType
() && N->getNumValues() == 1 && "Type mismatch"
) ? void (0) : __assert_fail ("N->getValueType(0) == RV.getValueType() && N->getNumValues() == 1 && \"Type mismatch\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 1650, __extension__
__PRETTY_FUNCTION__))
1650 N->getNumValues() == 1 && "Type mismatch")(static_cast <bool> (N->getValueType(0) == RV.getValueType
() && N->getNumValues() == 1 && "Type mismatch"
) ? void (0) : __assert_fail ("N->getValueType(0) == RV.getValueType() && N->getNumValues() == 1 && \"Type mismatch\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 1650, __extension__
__PRETTY_FUNCTION__))
;
1651 DAG.ReplaceAllUsesWith(N, &RV);
1652 }
1653
1654 // Push the new node and any users onto the worklist. Omit this if the
1655 // new node is the EntryToken (e.g. if a store managed to get optimized
1656 // out), because re-visiting the EntryToken and its users will not uncover
1657 // any additional opportunities, but there may be a large number of such
1658 // users, potentially causing compile time explosion.
1659 if (RV.getOpcode() != ISD::EntryToken) {
1660 AddToWorklist(RV.getNode());
1661 AddUsersToWorklist(RV.getNode());
1662 }
1663
1664 // Finally, if the node is now dead, remove it from the graph. The node
1665 // may not be dead if the replacement process recursively simplified to
1666 // something else needing this node. This will also take care of adding any
1667 // operands which have lost a user to the worklist.
1668 recursivelyDeleteUnusedNodes(N);
1669 }
1670
1671 // If the root changed (e.g. it was a dead load, update the root).
1672 DAG.setRoot(Dummy.getValue());
1673 DAG.RemoveDeadNodes();
1674}
1675
1676SDValue DAGCombiner::visit(SDNode *N) {
1677 switch (N->getOpcode()) {
1678 default: break;
1679 case ISD::TokenFactor: return visitTokenFactor(N);
1680 case ISD::MERGE_VALUES: return visitMERGE_VALUES(N);
1681 case ISD::ADD: return visitADD(N);
1682 case ISD::SUB: return visitSUB(N);
1683 case ISD::SADDSAT:
1684 case ISD::UADDSAT: return visitADDSAT(N);
1685 case ISD::SSUBSAT:
1686 case ISD::USUBSAT: return visitSUBSAT(N);
1687 case ISD::ADDC: return visitADDC(N);
1688 case ISD::SADDO:
1689 case ISD::UADDO: return visitADDO(N);
1690 case ISD::SUBC: return visitSUBC(N);
1691 case ISD::SSUBO:
1692 case ISD::USUBO: return visitSUBO(N);
1693 case ISD::ADDE: return visitADDE(N);
1694 case ISD::ADDCARRY: return visitADDCARRY(N);
1695 case ISD::SADDO_CARRY: return visitSADDO_CARRY(N);
1696 case ISD::SUBE: return visitSUBE(N);
1697 case ISD::SUBCARRY: return visitSUBCARRY(N);
1698 case ISD::SSUBO_CARRY: return visitSSUBO_CARRY(N);
1699 case ISD::SMULFIX:
1700 case ISD::SMULFIXSAT:
1701 case ISD::UMULFIX:
1702 case ISD::UMULFIXSAT: return visitMULFIX(N);
1703 case ISD::MUL: return visitMUL(N);
1704 case ISD::SDIV: return visitSDIV(N);
1705 case ISD::UDIV: return visitUDIV(N);
1706 case ISD::SREM:
1707 case ISD::UREM: return visitREM(N);
1708 case ISD::MULHU: return visitMULHU(N);
1709 case ISD::MULHS: return visitMULHS(N);
1710 case ISD::AVGFLOORS:
1711 case ISD::AVGFLOORU:
1712 case ISD::AVGCEILS:
1713 case ISD::AVGCEILU: return visitAVG(N);
1714 case ISD::SMUL_LOHI: return visitSMUL_LOHI(N);
1715 case ISD::UMUL_LOHI: return visitUMUL_LOHI(N);
1716 case ISD::SMULO:
1717 case ISD::UMULO: return visitMULO(N);
1718 case ISD::SMIN:
1719 case ISD::SMAX:
1720 case ISD::UMIN:
1721 case ISD::UMAX: return visitIMINMAX(N);
1722 case ISD::AND: return visitAND(N);
1723 case ISD::OR: return visitOR(N);
1724 case ISD::XOR: return visitXOR(N);
1725 case ISD::SHL: return visitSHL(N);
1726 case ISD::SRA: return visitSRA(N);
1727 case ISD::SRL: return visitSRL(N);
1728 case ISD::ROTR:
1729 case ISD::ROTL: return visitRotate(N);
1730 case ISD::FSHL:
1731 case ISD::FSHR: return visitFunnelShift(N);
1732 case ISD::SSHLSAT:
1733 case ISD::USHLSAT: return visitSHLSAT(N);
1734 case ISD::ABS: return visitABS(N);
1735 case ISD::BSWAP: return visitBSWAP(N);
1736 case ISD::BITREVERSE: return visitBITREVERSE(N);
1737 case ISD::CTLZ: return visitCTLZ(N);
1738 case ISD::CTLZ_ZERO_UNDEF: return visitCTLZ_ZERO_UNDEF(N);
1739 case ISD::CTTZ: return visitCTTZ(N);
1740 case ISD::CTTZ_ZERO_UNDEF: return visitCTTZ_ZERO_UNDEF(N);
1741 case ISD::CTPOP: return visitCTPOP(N);
1742 case ISD::SELECT: return visitSELECT(N);
1743 case ISD::VSELECT: return visitVSELECT(N);
1744 case ISD::SELECT_CC: return visitSELECT_CC(N);
1745 case ISD::SETCC: return visitSETCC(N);
1746 case ISD::SETCCCARRY: return visitSETCCCARRY(N);
1747 case ISD::SIGN_EXTEND: return visitSIGN_EXTEND(N);
1748 case ISD::ZERO_EXTEND: return visitZERO_EXTEND(N);
1749 case ISD::ANY_EXTEND: return visitANY_EXTEND(N);
1750 case ISD::AssertSext:
1751 case ISD::AssertZext: return visitAssertExt(N);
1752 case ISD::AssertAlign: return visitAssertAlign(N);
1753 case ISD::SIGN_EXTEND_INREG: return visitSIGN_EXTEND_INREG(N);
1754 case ISD::SIGN_EXTEND_VECTOR_INREG:
1755 case ISD::ZERO_EXTEND_VECTOR_INREG: return visitEXTEND_VECTOR_INREG(N);
1756 case ISD::TRUNCATE: return visitTRUNCATE(N);
1757 case ISD::BITCAST: return visitBITCAST(N);
1758 case ISD::BUILD_PAIR: return visitBUILD_PAIR(N);
1759 case ISD::FADD: return visitFADD(N);
1760 case ISD::STRICT_FADD: return visitSTRICT_FADD(N);
1761 case ISD::FSUB: return visitFSUB(N);
1762 case ISD::FMUL: return visitFMUL(N);
1763 case ISD::FMA: return visitFMA(N);
1764 case ISD::FDIV: return visitFDIV(N);
1765 case ISD::FREM: return visitFREM(N);
1766 case ISD::FSQRT: return visitFSQRT(N);
1767 case ISD::FCOPYSIGN: return visitFCOPYSIGN(N);
1768 case ISD::FPOW: return visitFPOW(N);
1769 case ISD::SINT_TO_FP: return visitSINT_TO_FP(N);
1770 case ISD::UINT_TO_FP: return visitUINT_TO_FP(N);
1771 case ISD::FP_TO_SINT: return visitFP_TO_SINT(N);
1772 case ISD::FP_TO_UINT: return visitFP_TO_UINT(N);
1773 case ISD::FP_ROUND: return visitFP_ROUND(N);
1774 case ISD::FP_EXTEND: return visitFP_EXTEND(N);
1775 case ISD::FNEG: return visitFNEG(N);
1776 case ISD::FABS: return visitFABS(N);
1777 case ISD::FFLOOR: return visitFFLOOR(N);
1778 case ISD::FMINNUM:
1779 case ISD::FMAXNUM:
1780 case ISD::FMINIMUM:
1781 case ISD::FMAXIMUM: return visitFMinMax(N);
1782 case ISD::FCEIL: return visitFCEIL(N);
1783 case ISD::FTRUNC: return visitFTRUNC(N);
1784 case ISD::BRCOND: return visitBRCOND(N);
1785 case ISD::BR_CC: return visitBR_CC(N);
1786 case ISD::LOAD: return visitLOAD(N);
1787 case ISD::STORE: return visitSTORE(N);
1788 case ISD::INSERT_VECTOR_ELT: return visitINSERT_VECTOR_ELT(N);
1789 case ISD::EXTRACT_VECTOR_ELT: return visitEXTRACT_VECTOR_ELT(N);
1790 case ISD::BUILD_VECTOR: return visitBUILD_VECTOR(N);
1791 case ISD::CONCAT_VECTORS: return visitCONCAT_VECTORS(N);
1792 case ISD::EXTRACT_SUBVECTOR: return visitEXTRACT_SUBVECTOR(N);
1793 case ISD::VECTOR_SHUFFLE: return visitVECTOR_SHUFFLE(N);
1794 case ISD::SCALAR_TO_VECTOR: return visitSCALAR_TO_VECTOR(N);
1795 case ISD::INSERT_SUBVECTOR: return visitINSERT_SUBVECTOR(N);
1796 case ISD::MGATHER: return visitMGATHER(N);
1797 case ISD::MLOAD: return visitMLOAD(N);
1798 case ISD::MSCATTER: return visitMSCATTER(N);
1799 case ISD::MSTORE: return visitMSTORE(N);
1800 case ISD::LIFETIME_END: return visitLIFETIME_END(N);
1801 case ISD::FP_TO_FP16: return visitFP_TO_FP16(N);
1802 case ISD::FP16_TO_FP: return visitFP16_TO_FP(N);
1803 case ISD::FP_TO_BF16: return visitFP_TO_BF16(N);
1804 case ISD::FREEZE: return visitFREEZE(N);
1805 case ISD::VECREDUCE_FADD:
1806 case ISD::VECREDUCE_FMUL:
1807 case ISD::VECREDUCE_ADD:
1808 case ISD::VECREDUCE_MUL:
1809 case ISD::VECREDUCE_AND:
1810 case ISD::VECREDUCE_OR:
1811 case ISD::VECREDUCE_XOR:
1812 case ISD::VECREDUCE_SMAX:
1813 case ISD::VECREDUCE_SMIN:
1814 case ISD::VECREDUCE_UMAX:
1815 case ISD::VECREDUCE_UMIN:
1816 case ISD::VECREDUCE_FMAX:
1817 case ISD::VECREDUCE_FMIN: return visitVECREDUCE(N);
1818#define BEGIN_REGISTER_VP_SDNODE(SDOPC, ...) case ISD::SDOPC:
1819#include "llvm/IR/VPIntrinsics.def"
1820 return visitVPOp(N);
1821 }
1822 return SDValue();
1823}
1824
1825SDValue DAGCombiner::combine(SDNode *N) {
1826 SDValue RV;
1827 if (!DisableGenericCombines)
1828 RV = visit(N);
1829
1830 // If nothing happened, try a target-specific DAG combine.
1831 if (!RV.getNode()) {
1832 assert(N->getOpcode() != ISD::DELETED_NODE &&(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& "Node was deleted but visit returned NULL!") ? void
(0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned NULL!\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 1833, __extension__
__PRETTY_FUNCTION__))
1833 "Node was deleted but visit returned NULL!")(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& "Node was deleted but visit returned NULL!") ? void
(0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned NULL!\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 1833, __extension__
__PRETTY_FUNCTION__))
;
1834
1835 if (N->getOpcode() >= ISD::BUILTIN_OP_END ||
1836 TLI.hasTargetDAGCombine((ISD::NodeType)N->getOpcode())) {
1837
1838 // Expose the DAG combiner to the target combiner impls.
1839 TargetLowering::DAGCombinerInfo
1840 DagCombineInfo(DAG, Level, false, this);
1841
1842 RV = TLI.PerformDAGCombine(N, DagCombineInfo);
1843 }
1844 }
1845
1846 // If nothing happened still, try promoting the operation.
1847 if (!RV.getNode()) {
1848 switch (N->getOpcode()) {
1849 default: break;
1850 case ISD::ADD:
1851 case ISD::SUB:
1852 case ISD::MUL:
1853 case ISD::AND:
1854 case ISD::OR:
1855 case ISD::XOR:
1856 RV = PromoteIntBinOp(SDValue(N, 0));
1857 break;
1858 case ISD::SHL:
1859 case ISD::SRA:
1860 case ISD::SRL:
1861 RV = PromoteIntShiftOp(SDValue(N, 0));
1862 break;
1863 case ISD::SIGN_EXTEND:
1864 case ISD::ZERO_EXTEND:
1865 case ISD::ANY_EXTEND:
1866 RV = PromoteExtend(SDValue(N, 0));
1867 break;
1868 case ISD::LOAD:
1869 if (PromoteLoad(SDValue(N, 0)))
1870 RV = SDValue(N, 0);
1871 break;
1872 }
1873 }
1874
1875 // If N is a commutative binary node, try to eliminate it if the commuted
1876 // version is already present in the DAG.
1877 if (!RV.getNode() && TLI.isCommutativeBinOp(N->getOpcode())) {
1878 SDValue N0 = N->getOperand(0);
1879 SDValue N1 = N->getOperand(1);
1880
1881 // Constant operands are canonicalized to RHS.
1882 if (N0 != N1 && (isa<ConstantSDNode>(N0) || !isa<ConstantSDNode>(N1))) {
1883 SDValue Ops[] = {N1, N0};
1884 SDNode *CSENode = DAG.getNodeIfExists(N->getOpcode(), N->getVTList(), Ops,
1885 N->getFlags());
1886 if (CSENode)
1887 return SDValue(CSENode, 0);
1888 }
1889 }
1890
1891 return RV;
1892}
1893
1894/// Given a node, return its input chain if it has one, otherwise return a null
1895/// sd operand.
1896static SDValue getInputChainForNode(SDNode *N) {
1897 if (unsigned NumOps = N->getNumOperands()) {
1898 if (N->getOperand(0).getValueType() == MVT::Other)
1899 return N->getOperand(0);
1900 if (N->getOperand(NumOps-1).getValueType() == MVT::Other)
1901 return N->getOperand(NumOps-1);
1902 for (unsigned i = 1; i < NumOps-1; ++i)
1903 if (N->getOperand(i).getValueType() == MVT::Other)
1904 return N->getOperand(i);
1905 }
1906 return SDValue();
1907}
1908
1909SDValue DAGCombiner::visitTokenFactor(SDNode *N) {
1910 // If N has two operands, where one has an input chain equal to the other,
1911 // the 'other' chain is redundant.
1912 if (N->getNumOperands() == 2) {
1913 if (getInputChainForNode(N->getOperand(0).getNode()) == N->getOperand(1))
1914 return N->getOperand(0);
1915 if (getInputChainForNode(N->getOperand(1).getNode()) == N->getOperand(0))
1916 return N->getOperand(1);
1917 }
1918
1919 // Don't simplify token factors if optnone.
1920 if (OptLevel == CodeGenOpt::None)
1921 return SDValue();
1922
1923 // Don't simplify the token factor if the node itself has too many operands.
1924 if (N->getNumOperands() > TokenFactorInlineLimit)
1925 return SDValue();
1926
1927 // If the sole user is a token factor, we should make sure we have a
1928 // chance to merge them together. This prevents TF chains from inhibiting
1929 // optimizations.
1930 if (N->hasOneUse() && N->use_begin()->getOpcode() == ISD::TokenFactor)
1931 AddToWorklist(*(N->use_begin()));
1932
1933 SmallVector<SDNode *, 8> TFs; // List of token factors to visit.
1934 SmallVector<SDValue, 8> Ops; // Ops for replacing token factor.
1935 SmallPtrSet<SDNode*, 16> SeenOps;
1936 bool Changed = false; // If we should replace this token factor.
1937
1938 // Start out with this token factor.
1939 TFs.push_back(N);
1940
1941 // Iterate through token factors. The TFs grows when new token factors are
1942 // encountered.
1943 for (unsigned i = 0; i < TFs.size(); ++i) {
1944 // Limit number of nodes to inline, to avoid quadratic compile times.
1945 // We have to add the outstanding Token Factors to Ops, otherwise we might
1946 // drop Ops from the resulting Token Factors.
1947 if (Ops.size() > TokenFactorInlineLimit) {
1948 for (unsigned j = i; j < TFs.size(); j++)
1949 Ops.emplace_back(TFs[j], 0);
1950 // Drop unprocessed Token Factors from TFs, so we do not add them to the
1951 // combiner worklist later.
1952 TFs.resize(i);
1953 break;
1954 }
1955
1956 SDNode *TF = TFs[i];
1957 // Check each of the operands.
1958 for (const SDValue &Op : TF->op_values()) {
1959 switch (Op.getOpcode()) {
1960 case ISD::EntryToken:
1961 // Entry tokens don't need to be added to the list. They are
1962 // redundant.
1963 Changed = true;
1964 break;
1965
1966 case ISD::TokenFactor:
1967 if (Op.hasOneUse() && !is_contained(TFs, Op.getNode())) {
1968 // Queue up for processing.
1969 TFs.push_back(Op.getNode());
1970 Changed = true;
1971 break;
1972 }
1973 [[fallthrough]];
1974
1975 default:
1976 // Only add if it isn't already in the list.
1977 if (SeenOps.insert(Op.getNode()).second)
1978 Ops.push_back(Op);
1979 else
1980 Changed = true;
1981 break;
1982 }
1983 }
1984 }
1985
1986 // Re-visit inlined Token Factors, to clean them up in case they have been
1987 // removed. Skip the first Token Factor, as this is the current node.
1988 for (unsigned i = 1, e = TFs.size(); i < e; i++)
1989 AddToWorklist(TFs[i]);
1990
1991 // Remove Nodes that are chained to another node in the list. Do so
1992 // by walking up chains breath-first stopping when we've seen
1993 // another operand. In general we must climb to the EntryNode, but we can exit
1994 // early if we find all remaining work is associated with just one operand as
1995 // no further pruning is possible.
1996
1997 // List of nodes to search through and original Ops from which they originate.
1998 SmallVector<std::pair<SDNode *, unsigned>, 8> Worklist;
1999 SmallVector<unsigned, 8> OpWorkCount; // Count of work for each Op.
2000 SmallPtrSet<SDNode *, 16> SeenChains;
2001 bool DidPruneOps = false;
2002
2003 unsigned NumLeftToConsider = 0;
2004 for (const SDValue &Op : Ops) {
2005 Worklist.push_back(std::make_pair(Op.getNode(), NumLeftToConsider++));
2006 OpWorkCount.push_back(1);
2007 }
2008
2009 auto AddToWorklist = [&](unsigned CurIdx, SDNode *Op, unsigned OpNumber) {
2010 // If this is an Op, we can remove the op from the list. Remark any
2011 // search associated with it as from the current OpNumber.
2012 if (SeenOps.contains(Op)) {
2013 Changed = true;
2014 DidPruneOps = true;
2015 unsigned OrigOpNumber = 0;
2016 while (OrigOpNumber < Ops.size() && Ops[OrigOpNumber].getNode() != Op)
2017 OrigOpNumber++;
2018 assert((OrigOpNumber != Ops.size()) &&(static_cast <bool> ((OrigOpNumber != Ops.size()) &&
"expected to find TokenFactor Operand") ? void (0) : __assert_fail
("(OrigOpNumber != Ops.size()) && \"expected to find TokenFactor Operand\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 2019, __extension__
__PRETTY_FUNCTION__))
2019 "expected to find TokenFactor Operand")(static_cast <bool> ((OrigOpNumber != Ops.size()) &&
"expected to find TokenFactor Operand") ? void (0) : __assert_fail
("(OrigOpNumber != Ops.size()) && \"expected to find TokenFactor Operand\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 2019, __extension__
__PRETTY_FUNCTION__))
;
2020 // Re-mark worklist from OrigOpNumber to OpNumber
2021 for (unsigned i = CurIdx + 1; i < Worklist.size(); ++i) {
2022 if (Worklist[i].second == OrigOpNumber) {
2023 Worklist[i].second = OpNumber;
2024 }
2025 }
2026 OpWorkCount[OpNumber] += OpWorkCount[OrigOpNumber];
2027 OpWorkCount[OrigOpNumber] = 0;
2028 NumLeftToConsider--;
2029 }
2030 // Add if it's a new chain
2031 if (SeenChains.insert(Op).second) {
2032 OpWorkCount[OpNumber]++;
2033 Worklist.push_back(std::make_pair(Op, OpNumber));
2034 }
2035 };
2036
2037 for (unsigned i = 0; i < Worklist.size() && i < 1024; ++i) {
2038 // We need at least be consider at least 2 Ops to prune.
2039 if (NumLeftToConsider <= 1)
2040 break;
2041 auto CurNode = Worklist[i].first;
2042 auto CurOpNumber = Worklist[i].second;
2043 assert((OpWorkCount[CurOpNumber] > 0) &&(static_cast <bool> ((OpWorkCount[CurOpNumber] > 0) &&
"Node should not appear in worklist") ? void (0) : __assert_fail
("(OpWorkCount[CurOpNumber] > 0) && \"Node should not appear in worklist\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 2044, __extension__
__PRETTY_FUNCTION__))
2044 "Node should not appear in worklist")(static_cast <bool> ((OpWorkCount[CurOpNumber] > 0) &&
"Node should not appear in worklist") ? void (0) : __assert_fail
("(OpWorkCount[CurOpNumber] > 0) && \"Node should not appear in worklist\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 2044, __extension__
__PRETTY_FUNCTION__))
;
2045 switch (CurNode->getOpcode()) {
2046 case ISD::EntryToken:
2047 // Hitting EntryToken is the only way for the search to terminate without
2048 // hitting
2049 // another operand's search. Prevent us from marking this operand
2050 // considered.
2051 NumLeftToConsider++;
2052 break;
2053 case ISD::TokenFactor:
2054 for (const SDValue &Op : CurNode->op_values())
2055 AddToWorklist(i, Op.getNode(), CurOpNumber);
2056 break;
2057 case ISD::LIFETIME_START:
2058 case ISD::LIFETIME_END:
2059 case ISD::CopyFromReg:
2060 case ISD::CopyToReg:
2061 AddToWorklist(i, CurNode->getOperand(0).getNode(), CurOpNumber);
2062 break;
2063 default:
2064 if (auto *MemNode = dyn_cast<MemSDNode>(CurNode))
2065 AddToWorklist(i, MemNode->getChain().getNode(), CurOpNumber);
2066 break;
2067 }
2068 OpWorkCount[CurOpNumber]--;
2069 if (OpWorkCount[CurOpNumber] == 0)
2070 NumLeftToConsider--;
2071 }
2072
2073 // If we've changed things around then replace token factor.
2074 if (Changed) {
2075 SDValue Result;
2076 if (Ops.empty()) {
2077 // The entry token is the only possible outcome.
2078 Result = DAG.getEntryNode();
2079 } else {
2080 if (DidPruneOps) {
2081 SmallVector<SDValue, 8> PrunedOps;
2082 //
2083 for (const SDValue &Op : Ops) {
2084 if (SeenChains.count(Op.getNode()) == 0)
2085 PrunedOps.push_back(Op);
2086 }
2087 Result = DAG.getTokenFactor(SDLoc(N), PrunedOps);
2088 } else {
2089 Result = DAG.getTokenFactor(SDLoc(N), Ops);
2090 }
2091 }
2092 return Result;
2093 }
2094 return SDValue();
2095}
2096
2097/// MERGE_VALUES can always be eliminated.
2098SDValue DAGCombiner::visitMERGE_VALUES(SDNode *N) {
2099 WorklistRemover DeadNodes(*this);
2100 // Replacing results may cause a different MERGE_VALUES to suddenly
2101 // be CSE'd with N, and carry its uses with it. Iterate until no
2102 // uses remain, to ensure that the node can be safely deleted.
2103 // First add the users of this node to the work list so that they
2104 // can be tried again once they have new operands.
2105 AddUsersToWorklist(N);
2106 do {
2107 // Do as a single replacement to avoid rewalking use lists.
2108 SmallVector<SDValue, 8> Ops;
2109 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
2110 Ops.push_back(N->getOperand(i));
2111 DAG.ReplaceAllUsesWith(N, Ops.data());
2112 } while (!N->use_empty());
2113 deleteAndRecombine(N);
2114 return SDValue(N, 0); // Return N so it doesn't get rechecked!
2115}
2116
2117/// If \p N is a ConstantSDNode with isOpaque() == false return it casted to a
2118/// ConstantSDNode pointer else nullptr.
2119static ConstantSDNode *getAsNonOpaqueConstant(SDValue N) {
2120 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N);
2121 return Const != nullptr && !Const->isOpaque() ? Const : nullptr;
2122}
2123
2124/// Return true if 'Use' is a load or a store that uses N as its base pointer
2125/// and that N may be folded in the load / store addressing mode.
2126static bool canFoldInAddressingMode(SDNode *N, SDNode *Use, SelectionDAG &DAG,
2127 const TargetLowering &TLI) {
2128 EVT VT;
2129 unsigned AS;
2130
2131 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Use)) {
2132 if (LD->isIndexed() || LD->getBasePtr().getNode() != N)
2133 return false;
2134 VT = LD->getMemoryVT();
2135 AS = LD->getAddressSpace();
2136 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Use)) {
2137 if (ST->isIndexed() || ST->getBasePtr().getNode() != N)
2138 return false;
2139 VT = ST->getMemoryVT();
2140 AS = ST->getAddressSpace();
2141 } else if (MaskedLoadSDNode *LD = dyn_cast<MaskedLoadSDNode>(Use)) {
2142 if (LD->isIndexed() || LD->getBasePtr().getNode() != N)
2143 return false;
2144 VT = LD->getMemoryVT();
2145 AS = LD->getAddressSpace();
2146 } else if (MaskedStoreSDNode *ST = dyn_cast<MaskedStoreSDNode>(Use)) {
2147 if (ST->isIndexed() || ST->getBasePtr().getNode() != N)
2148 return false;
2149 VT = ST->getMemoryVT();
2150 AS = ST->getAddressSpace();
2151 } else {
2152 return false;
2153 }
2154
2155 TargetLowering::AddrMode AM;
2156 if (N->getOpcode() == ISD::ADD) {
2157 AM.HasBaseReg = true;
2158 ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1));
2159 if (Offset)
2160 // [reg +/- imm]
2161 AM.BaseOffs = Offset->getSExtValue();
2162 else
2163 // [reg +/- reg]
2164 AM.Scale = 1;
2165 } else if (N->getOpcode() == ISD::SUB) {
2166 AM.HasBaseReg = true;
2167 ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1));
2168 if (Offset)
2169 // [reg +/- imm]
2170 AM.BaseOffs = -Offset->getSExtValue();
2171 else
2172 // [reg +/- reg]
2173 AM.Scale = 1;
2174 } else {
2175 return false;
2176 }
2177
2178 return TLI.isLegalAddressingMode(DAG.getDataLayout(), AM,
2179 VT.getTypeForEVT(*DAG.getContext()), AS);
2180}
2181
2182/// This inverts a canonicalization in IR that replaces a variable select arm
2183/// with an identity constant. Codegen improves if we re-use the variable
2184/// operand rather than load a constant. This can also be converted into a
2185/// masked vector operation if the target supports it.
2186static SDValue foldSelectWithIdentityConstant(SDNode *N, SelectionDAG &DAG,
2187 bool ShouldCommuteOperands) {
2188 // Match a select as operand 1. The identity constant that we are looking for
2189 // is only valid as operand 1 of a non-commutative binop.
2190 SDValue N0 = N->getOperand(0);
2191 SDValue N1 = N->getOperand(1);
2192 if (ShouldCommuteOperands)
2193 std::swap(N0, N1);
2194
2195 // TODO: Should this apply to scalar select too?
2196 if (N1.getOpcode() != ISD::VSELECT || !N1.hasOneUse())
2197 return SDValue();
2198
2199 unsigned Opcode = N->getOpcode();
2200 EVT VT = N->getValueType(0);
2201 SDValue Cond = N1.getOperand(0);
2202 SDValue TVal = N1.getOperand(1);
2203 SDValue FVal = N1.getOperand(2);
2204
2205 // This transform increases uses of N0, so freeze it to be safe.
2206 // binop N0, (vselect Cond, IDC, FVal) --> vselect Cond, N0, (binop N0, FVal)
2207 unsigned OpNo = ShouldCommuteOperands ? 0 : 1;
2208 if (isNeutralConstant(Opcode, N->getFlags(), TVal, OpNo)) {
2209 SDValue F0 = DAG.getFreeze(N0);
2210 SDValue NewBO = DAG.getNode(Opcode, SDLoc(N), VT, F0, FVal, N->getFlags());
2211 return DAG.getSelect(SDLoc(N), VT, Cond, F0, NewBO);
2212 }
2213 // binop N0, (vselect Cond, TVal, IDC) --> vselect Cond, (binop N0, TVal), N0
2214 if (isNeutralConstant(Opcode, N->getFlags(), FVal, OpNo)) {
2215 SDValue F0 = DAG.getFreeze(N0);
2216 SDValue NewBO = DAG.getNode(Opcode, SDLoc(N), VT, F0, TVal, N->getFlags());
2217 return DAG.getSelect(SDLoc(N), VT, Cond, NewBO, F0);
2218 }
2219
2220 return SDValue();
2221}
2222
2223SDValue DAGCombiner::foldBinOpIntoSelect(SDNode *BO) {
2224 assert(TLI.isBinOp(BO->getOpcode()) && BO->getNumValues() == 1 &&(static_cast <bool> (TLI.isBinOp(BO->getOpcode()) &&
BO->getNumValues() == 1 && "Unexpected binary operator"
) ? void (0) : __assert_fail ("TLI.isBinOp(BO->getOpcode()) && BO->getNumValues() == 1 && \"Unexpected binary operator\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 2225, __extension__
__PRETTY_FUNCTION__))
2225 "Unexpected binary operator")(static_cast <bool> (TLI.isBinOp(BO->getOpcode()) &&
BO->getNumValues() == 1 && "Unexpected binary operator"
) ? void (0) : __assert_fail ("TLI.isBinOp(BO->getOpcode()) && BO->getNumValues() == 1 && \"Unexpected binary operator\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 2225, __extension__
__PRETTY_FUNCTION__))
;
2226
2227 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2228 auto BinOpcode = BO->getOpcode();
2229 EVT VT = BO->getValueType(0);
2230 if (TLI.shouldFoldSelectWithIdentityConstant(BinOpcode, VT)) {
2231 if (SDValue Sel = foldSelectWithIdentityConstant(BO, DAG, false))
2232 return Sel;
2233
2234 if (TLI.isCommutativeBinOp(BO->getOpcode()))
2235 if (SDValue Sel = foldSelectWithIdentityConstant(BO, DAG, true))
2236 return Sel;
2237 }
2238
2239 // Don't do this unless the old select is going away. We want to eliminate the
2240 // binary operator, not replace a binop with a select.
2241 // TODO: Handle ISD::SELECT_CC.
2242 unsigned SelOpNo = 0;
2243 SDValue Sel = BO->getOperand(0);
2244 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse()) {
2245 SelOpNo = 1;
2246 Sel = BO->getOperand(1);
2247 }
2248
2249 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse())
2250 return SDValue();
2251
2252 SDValue CT = Sel.getOperand(1);
2253 if (!isConstantOrConstantVector(CT, true) &&
2254 !DAG.isConstantFPBuildVectorOrConstantFP(CT))
2255 return SDValue();
2256
2257 SDValue CF = Sel.getOperand(2);
2258 if (!isConstantOrConstantVector(CF, true) &&
2259 !DAG.isConstantFPBuildVectorOrConstantFP(CF))
2260 return SDValue();
2261
2262 // Bail out if any constants are opaque because we can't constant fold those.
2263 // The exception is "and" and "or" with either 0 or -1 in which case we can
2264 // propagate non constant operands into select. I.e.:
2265 // and (select Cond, 0, -1), X --> select Cond, 0, X
2266 // or X, (select Cond, -1, 0) --> select Cond, -1, X
2267 bool CanFoldNonConst =
2268 (BinOpcode == ISD::AND || BinOpcode == ISD::OR) &&
2269 (isNullOrNullSplat(CT) || isAllOnesOrAllOnesSplat(CT)) &&
2270 (isNullOrNullSplat(CF) || isAllOnesOrAllOnesSplat(CF));
2271
2272 SDValue CBO = BO->getOperand(SelOpNo ^ 1);
2273 if (!CanFoldNonConst &&
2274 !isConstantOrConstantVector(CBO, true) &&
2275 !DAG.isConstantFPBuildVectorOrConstantFP(CBO))
2276 return SDValue();
2277
2278 // We have a select-of-constants followed by a binary operator with a
2279 // constant. Eliminate the binop by pulling the constant math into the select.
2280 // Example: add (select Cond, CT, CF), CBO --> select Cond, CT + CBO, CF + CBO
2281 SDLoc DL(Sel);
2282 SDValue NewCT = SelOpNo ? DAG.getNode(BinOpcode, DL, VT, CBO, CT)
2283 : DAG.getNode(BinOpcode, DL, VT, CT, CBO);
2284 if (!CanFoldNonConst && !NewCT.isUndef() &&
2285 !isConstantOrConstantVector(NewCT, true) &&
2286 !DAG.isConstantFPBuildVectorOrConstantFP(NewCT))
2287 return SDValue();
2288
2289 SDValue NewCF = SelOpNo ? DAG.getNode(BinOpcode, DL, VT, CBO, CF)
2290 : DAG.getNode(BinOpcode, DL, VT, CF, CBO);
2291 if (!CanFoldNonConst && !NewCF.isUndef() &&
2292 !isConstantOrConstantVector(NewCF, true) &&
2293 !DAG.isConstantFPBuildVectorOrConstantFP(NewCF))
2294 return SDValue();
2295
2296 SDValue SelectOp = DAG.getSelect(DL, VT, Sel.getOperand(0), NewCT, NewCF);
2297 SelectOp->setFlags(BO->getFlags());
2298 return SelectOp;
2299}
2300
2301static SDValue foldAddSubBoolOfMaskedVal(SDNode *N, SelectionDAG &DAG) {
2302 assert((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) &&(static_cast <bool> ((N->getOpcode() == ISD::ADD || N
->getOpcode() == ISD::SUB) && "Expecting add or sub"
) ? void (0) : __assert_fail ("(N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) && \"Expecting add or sub\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 2303, __extension__
__PRETTY_FUNCTION__))
2303 "Expecting add or sub")(static_cast <bool> ((N->getOpcode() == ISD::ADD || N
->getOpcode() == ISD::SUB) && "Expecting add or sub"
) ? void (0) : __assert_fail ("(N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) && \"Expecting add or sub\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 2303, __extension__
__PRETTY_FUNCTION__))
;
2304
2305 // Match a constant operand and a zext operand for the math instruction:
2306 // add Z, C
2307 // sub C, Z
2308 bool IsAdd = N->getOpcode() == ISD::ADD;
2309 SDValue C = IsAdd ? N->getOperand(1) : N->getOperand(0);
2310 SDValue Z = IsAdd ? N->getOperand(0) : N->getOperand(1);
2311 auto *CN = dyn_cast<ConstantSDNode>(C);
2312 if (!CN || Z.getOpcode() != ISD::ZERO_EXTEND)
2313 return SDValue();
2314
2315 // Match the zext operand as a setcc of a boolean.
2316 if (Z.getOperand(0).getOpcode() != ISD::SETCC ||
2317 Z.getOperand(0).getValueType() != MVT::i1)
2318 return SDValue();
2319
2320 // Match the compare as: setcc (X & 1), 0, eq.
2321 SDValue SetCC = Z.getOperand(0);
2322 ISD::CondCode CC = cast<CondCodeSDNode>(SetCC->getOperand(2))->get();
2323 if (CC != ISD::SETEQ || !isNullConstant(SetCC.getOperand(1)) ||
2324 SetCC.getOperand(0).getOpcode() != ISD::AND ||
2325 !isOneConstant(SetCC.getOperand(0).getOperand(1)))
2326 return SDValue();
2327
2328 // We are adding/subtracting a constant and an inverted low bit. Turn that
2329 // into a subtract/add of the low bit with incremented/decremented constant:
2330 // add (zext i1 (seteq (X & 1), 0)), C --> sub C+1, (zext (X & 1))
2331 // sub C, (zext i1 (seteq (X & 1), 0)) --> add C-1, (zext (X & 1))
2332 EVT VT = C.getValueType();
2333 SDLoc DL(N);
2334 SDValue LowBit = DAG.getZExtOrTrunc(SetCC.getOperand(0), DL, VT);
2335 SDValue C1 = IsAdd ? DAG.getConstant(CN->getAPIntValue() + 1, DL, VT) :
2336 DAG.getConstant(CN->getAPIntValue() - 1, DL, VT);
2337 return DAG.getNode(IsAdd ? ISD::SUB : ISD::ADD, DL, VT, C1, LowBit);
2338}
2339
2340/// Try to fold a 'not' shifted sign-bit with add/sub with constant operand into
2341/// a shift and add with a different constant.
2342static SDValue foldAddSubOfSignBit(SDNode *N, SelectionDAG &DAG) {
2343 assert((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) &&(static_cast <bool> ((N->getOpcode() == ISD::ADD || N
->getOpcode() == ISD::SUB) && "Expecting add or sub"
) ? void (0) : __assert_fail ("(N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) && \"Expecting add or sub\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 2344, __extension__
__PRETTY_FUNCTION__))
2344 "Expecting add or sub")(static_cast <bool> ((N->getOpcode() == ISD::ADD || N
->getOpcode() == ISD::SUB) && "Expecting add or sub"
) ? void (0) : __assert_fail ("(N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) && \"Expecting add or sub\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 2344, __extension__
__PRETTY_FUNCTION__))
;
2345
2346 // We need a constant operand for the add/sub, and the other operand is a
2347 // logical shift right: add (srl), C or sub C, (srl).
2348 bool IsAdd = N->getOpcode() == ISD::ADD;
2349 SDValue ConstantOp = IsAdd ? N->getOperand(1) : N->getOperand(0);
2350 SDValue ShiftOp = IsAdd ? N->getOperand(0) : N->getOperand(1);
2351 if (!DAG.isConstantIntBuildVectorOrConstantInt(ConstantOp) ||
2352 ShiftOp.getOpcode() != ISD::SRL)
2353 return SDValue();
2354
2355 // The shift must be of a 'not' value.
2356 SDValue Not = ShiftOp.getOperand(0);
2357 if (!Not.hasOneUse() || !isBitwiseNot(Not))
2358 return SDValue();
2359
2360 // The shift must be moving the sign bit to the least-significant-bit.
2361 EVT VT = ShiftOp.getValueType();
2362 SDValue ShAmt = ShiftOp.getOperand(1);
2363 ConstantSDNode *ShAmtC = isConstOrConstSplat(ShAmt);
2364 if (!ShAmtC || ShAmtC->getAPIntValue() != (VT.getScalarSizeInBits() - 1))
2365 return SDValue();
2366
2367 // Eliminate the 'not' by adjusting the shift and add/sub constant:
2368 // add (srl (not X), 31), C --> add (sra X, 31), (C + 1)
2369 // sub C, (srl (not X), 31) --> add (srl X, 31), (C - 1)
2370 SDLoc DL(N);
2371 if (SDValue NewC = DAG.FoldConstantArithmetic(
2372 IsAdd ? ISD::ADD : ISD::SUB, DL, VT,
2373 {ConstantOp, DAG.getConstant(1, DL, VT)})) {
2374 SDValue NewShift = DAG.getNode(IsAdd ? ISD::SRA : ISD::SRL, DL, VT,
2375 Not.getOperand(0), ShAmt);
2376 return DAG.getNode(ISD::ADD, DL, VT, NewShift, NewC);
2377 }
2378
2379 return SDValue();
2380}
2381
2382static bool isADDLike(SDValue V, const SelectionDAG &DAG) {
2383 unsigned Opcode = V.getOpcode();
2384 if (Opcode == ISD::OR)
2385 return DAG.haveNoCommonBitsSet(V.getOperand(0), V.getOperand(1));
2386 if (Opcode == ISD::XOR)
2387 return isMinSignedConstant(V.getOperand(1));
2388 return false;
2389}
2390
2391/// Try to fold a node that behaves like an ADD (note that N isn't necessarily
2392/// an ISD::ADD here, it could for example be an ISD::OR if we know that there
2393/// are no common bits set in the operands).
2394SDValue DAGCombiner::visitADDLike(SDNode *N) {
2395 SDValue N0 = N->getOperand(0);
2396 SDValue N1 = N->getOperand(1);
2397 EVT VT = N0.getValueType();
2398 SDLoc DL(N);
2399
2400 // fold (add x, undef) -> undef
2401 if (N0.isUndef())
2402 return N0;
2403 if (N1.isUndef())
2404 return N1;
2405
2406 // fold (add c1, c2) -> c1+c2
2407 if (SDValue C = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N0, N1}))
2408 return C;
2409
2410 // canonicalize constant to RHS
2411 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
2412 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
2413 return DAG.getNode(ISD::ADD, DL, VT, N1, N0);
2414
2415 // fold vector ops
2416 if (VT.isVector()) {
2417 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
2418 return FoldedVOp;
2419
2420 // fold (add x, 0) -> x, vector edition
2421 if (ISD::isConstantSplatVectorAllZeros(N1.getNode()))
2422 return N0;
2423 }
2424
2425 // fold (add x, 0) -> x
2426 if (isNullConstant(N1))
2427 return N0;
2428
2429 if (N0.getOpcode() == ISD::SUB) {
2430 SDValue N00 = N0.getOperand(0);
2431 SDValue N01 = N0.getOperand(1);
2432
2433 // fold ((A-c1)+c2) -> (A+(c2-c1))
2434 if (SDValue Sub = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N1, N01}))
2435 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), Sub);
2436
2437 // fold ((c1-A)+c2) -> (c1+c2)-A
2438 if (SDValue Add = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N1, N00}))
2439 return DAG.getNode(ISD::SUB, DL, VT, Add, N0.getOperand(1));
2440 }
2441
2442 // add (sext i1 X), 1 -> zext (not i1 X)
2443 // We don't transform this pattern:
2444 // add (zext i1 X), -1 -> sext (not i1 X)
2445 // because most (?) targets generate better code for the zext form.
2446 if (N0.getOpcode() == ISD::SIGN_EXTEND && N0.hasOneUse() &&
2447 isOneOrOneSplat(N1)) {
2448 SDValue X = N0.getOperand(0);
2449 if ((!LegalOperations ||
2450 (TLI.isOperationLegal(ISD::XOR, X.getValueType()) &&
2451 TLI.isOperationLegal(ISD::ZERO_EXTEND, VT))) &&
2452 X.getScalarValueSizeInBits() == 1) {
2453 SDValue Not = DAG.getNOT(DL, X, X.getValueType());
2454 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Not);
2455 }
2456 }
2457
2458 // Fold (add (or x, c0), c1) -> (add x, (c0 + c1))
2459 // iff (or x, c0) is equivalent to (add x, c0).
2460 // Fold (add (xor x, c0), c1) -> (add x, (c0 + c1))
2461 // iff (xor x, c0) is equivalent to (add x, c0).
2462 if (isADDLike(N0, DAG)) {
2463 SDValue N01 = N0.getOperand(1);
2464 if (SDValue Add = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N1, N01}))
2465 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), Add);
2466 }
2467
2468 if (SDValue NewSel = foldBinOpIntoSelect(N))
2469 return NewSel;
2470
2471 // reassociate add
2472 if (!reassociationCanBreakAddressingModePattern(ISD::ADD, DL, N, N0, N1)) {
2473 if (SDValue RADD = reassociateOps(ISD::ADD, DL, N0, N1, N->getFlags()))
2474 return RADD;
2475
2476 // Reassociate (add (or x, c), y) -> (add add(x, y), c)) if (or x, c) is
2477 // equivalent to (add x, c).
2478 // Reassociate (add (xor x, c), y) -> (add add(x, y), c)) if (xor x, c) is
2479 // equivalent to (add x, c).
2480 auto ReassociateAddOr = [&](SDValue N0, SDValue N1) {
2481 if (isADDLike(N0, DAG) && N0.hasOneUse() &&
2482 isConstantOrConstantVector(N0.getOperand(1), /* NoOpaque */ true)) {
2483 return DAG.getNode(ISD::ADD, DL, VT,
2484 DAG.getNode(ISD::ADD, DL, VT, N1, N0.getOperand(0)),
2485 N0.getOperand(1));
2486 }
2487 return SDValue();
2488 };
2489 if (SDValue Add = ReassociateAddOr(N0, N1))
2490 return Add;
2491 if (SDValue Add = ReassociateAddOr(N1, N0))
2492 return Add;
2493 }
2494 // fold ((0-A) + B) -> B-A
2495 if (N0.getOpcode() == ISD::SUB && isNullOrNullSplat(N0.getOperand(0)))
2496 return DAG.getNode(ISD::SUB, DL, VT, N1, N0.getOperand(1));
2497
2498 // fold (A + (0-B)) -> A-B
2499 if (N1.getOpcode() == ISD::SUB && isNullOrNullSplat(N1.getOperand(0)))
2500 return DAG.getNode(ISD::SUB, DL, VT, N0, N1.getOperand(1));
2501
2502 // fold (A+(B-A)) -> B
2503 if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(1))
2504 return N1.getOperand(0);
2505
2506 // fold ((B-A)+A) -> B
2507 if (N0.getOpcode() == ISD::SUB && N1 == N0.getOperand(1))
2508 return N0.getOperand(0);
2509
2510 // fold ((A-B)+(C-A)) -> (C-B)
2511 if (N0.getOpcode() == ISD::SUB && N1.getOpcode() == ISD::SUB &&
2512 N0.getOperand(0) == N1.getOperand(1))
2513 return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0),
2514 N0.getOperand(1));
2515
2516 // fold ((A-B)+(B-C)) -> (A-C)
2517 if (N0.getOpcode() == ISD::SUB && N1.getOpcode() == ISD::SUB &&
2518 N0.getOperand(1) == N1.getOperand(0))
2519 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0),
2520 N1.getOperand(1));
2521
2522 // fold (A+(B-(A+C))) to (B-C)
2523 if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD &&
2524 N0 == N1.getOperand(1).getOperand(0))
2525 return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0),
2526 N1.getOperand(1).getOperand(1));
2527
2528 // fold (A+(B-(C+A))) to (B-C)
2529 if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD &&
2530 N0 == N1.getOperand(1).getOperand(1))
2531 return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0),
2532 N1.getOperand(1).getOperand(0));
2533
2534 // fold (A+((B-A)+or-C)) to (B+or-C)
2535 if ((N1.getOpcode() == ISD::SUB || N1.getOpcode() == ISD::ADD) &&
2536 N1.getOperand(0).getOpcode() == ISD::SUB &&
2537 N0 == N1.getOperand(0).getOperand(1))
2538 return DAG.getNode(N1.getOpcode(), DL, VT, N1.getOperand(0).getOperand(0),
2539 N1.getOperand(1));
2540
2541 // fold (A-B)+(C-D) to (A+C)-(B+D) when A or C is constant
2542 if (N0.getOpcode() == ISD::SUB && N1.getOpcode() == ISD::SUB &&
2543 N0->hasOneUse() && N1->hasOneUse()) {
2544 SDValue N00 = N0.getOperand(0);
2545 SDValue N01 = N0.getOperand(1);
2546 SDValue N10 = N1.getOperand(0);
2547 SDValue N11 = N1.getOperand(1);
2548
2549 if (isConstantOrConstantVector(N00) || isConstantOrConstantVector(N10))
2550 return DAG.getNode(ISD::SUB, DL, VT,
2551 DAG.getNode(ISD::ADD, SDLoc(N0), VT, N00, N10),
2552 DAG.getNode(ISD::ADD, SDLoc(N1), VT, N01, N11));
2553 }
2554
2555 // fold (add (umax X, C), -C) --> (usubsat X, C)
2556 if (N0.getOpcode() == ISD::UMAX && hasOperation(ISD::USUBSAT, VT)) {
2557 auto MatchUSUBSAT = [](ConstantSDNode *Max, ConstantSDNode *Op) {
2558 return (!Max && !Op) ||
2559 (Max && Op && Max->getAPIntValue() == (-Op->getAPIntValue()));
2560 };
2561 if (ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchUSUBSAT,
2562 /*AllowUndefs*/ true))
2563 return DAG.getNode(ISD::USUBSAT, DL, VT, N0.getOperand(0),
2564 N0.getOperand(1));
2565 }
2566
2567 if (SimplifyDemandedBits(SDValue(N, 0)))
2568 return SDValue(N, 0);
2569
2570 if (isOneOrOneSplat(N1)) {
2571 // fold (add (xor a, -1), 1) -> (sub 0, a)
2572 if (isBitwiseNot(N0))
2573 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
2574 N0.getOperand(0));
2575
2576 // fold (add (add (xor a, -1), b), 1) -> (sub b, a)
2577 if (N0.getOpcode() == ISD::ADD) {
2578 SDValue A, Xor;
2579
2580 if (isBitwiseNot(N0.getOperand(0))) {
2581 A = N0.getOperand(1);
2582 Xor = N0.getOperand(0);
2583 } else if (isBitwiseNot(N0.getOperand(1))) {
2584 A = N0.getOperand(0);
2585 Xor = N0.getOperand(1);
2586 }
2587
2588 if (Xor)
2589 return DAG.getNode(ISD::SUB, DL, VT, A, Xor.getOperand(0));
2590 }
2591
2592 // Look for:
2593 // add (add x, y), 1
2594 // And if the target does not like this form then turn into:
2595 // sub y, (xor x, -1)
2596 if (!TLI.preferIncOfAddToSubOfNot(VT) && N0.getOpcode() == ISD::ADD &&
2597 N0.hasOneUse()) {
2598 SDValue Not = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(0),
2599 DAG.getAllOnesConstant(DL, VT));
2600 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(1), Not);
2601 }
2602 }
2603
2604 // (x - y) + -1 -> add (xor y, -1), x
2605 if (N0.getOpcode() == ISD::SUB && N0.hasOneUse() &&
2606 isAllOnesOrAllOnesSplat(N1)) {
2607 SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(1), N1);
2608 return DAG.getNode(ISD::ADD, DL, VT, Xor, N0.getOperand(0));
2609 }
2610
2611 if (SDValue Combined = visitADDLikeCommutative(N0, N1, N))
2612 return Combined;
2613
2614 if (SDValue Combined = visitADDLikeCommutative(N1, N0, N))
2615 return Combined;
2616
2617 return SDValue();
2618}
2619
2620SDValue DAGCombiner::visitADD(SDNode *N) {
2621 SDValue N0 = N->getOperand(0);
2622 SDValue N1 = N->getOperand(1);
2623 EVT VT = N0.getValueType();
2624 SDLoc DL(N);
2625
2626 if (SDValue Combined = visitADDLike(N))
2627 return Combined;
2628
2629 if (SDValue V = foldAddSubBoolOfMaskedVal(N, DAG))
2630 return V;
2631
2632 if (SDValue V = foldAddSubOfSignBit(N, DAG))
2633 return V;
2634
2635 // fold (a+b) -> (a|b) iff a and b share no bits.
2636 if ((!LegalOperations || TLI.isOperationLegal(ISD::OR, VT)) &&
2637 DAG.haveNoCommonBitsSet(N0, N1))
2638 return DAG.getNode(ISD::OR, DL, VT, N0, N1);
2639
2640 // Fold (add (vscale * C0), (vscale * C1)) to (vscale * (C0 + C1)).
2641 if (N0.getOpcode() == ISD::VSCALE && N1.getOpcode() == ISD::VSCALE) {
2642 const APInt &C0 = N0->getConstantOperandAPInt(0);
2643 const APInt &C1 = N1->getConstantOperandAPInt(0);
2644 return DAG.getVScale(DL, VT, C0 + C1);
2645 }
2646
2647 // fold a+vscale(c1)+vscale(c2) -> a+vscale(c1+c2)
2648 if (N0.getOpcode() == ISD::ADD &&
2649 N0.getOperand(1).getOpcode() == ISD::VSCALE &&
2650 N1.getOpcode() == ISD::VSCALE) {
2651 const APInt &VS0 = N0.getOperand(1)->getConstantOperandAPInt(0);
2652 const APInt &VS1 = N1->getConstantOperandAPInt(0);
2653 SDValue VS = DAG.getVScale(DL, VT, VS0 + VS1);
2654 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), VS);
2655 }
2656
2657 // Fold (add step_vector(c1), step_vector(c2) to step_vector(c1+c2))
2658 if (N0.getOpcode() == ISD::STEP_VECTOR &&
2659 N1.getOpcode() == ISD::STEP_VECTOR) {
2660 const APInt &C0 = N0->getConstantOperandAPInt(0);
2661 const APInt &C1 = N1->getConstantOperandAPInt(0);
2662 APInt NewStep = C0 + C1;
2663 return DAG.getStepVector(DL, VT, NewStep);
2664 }
2665
2666 // Fold a + step_vector(c1) + step_vector(c2) to a + step_vector(c1+c2)
2667 if (N0.getOpcode() == ISD::ADD &&
2668 N0.getOperand(1).getOpcode() == ISD::STEP_VECTOR &&
2669 N1.getOpcode() == ISD::STEP_VECTOR) {
2670 const APInt &SV0 = N0.getOperand(1)->getConstantOperandAPInt(0);
2671 const APInt &SV1 = N1->getConstantOperandAPInt(0);
2672 APInt NewStep = SV0 + SV1;
2673 SDValue SV = DAG.getStepVector(DL, VT, NewStep);
2674 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), SV);
2675 }
2676
2677 return SDValue();
2678}
2679
2680SDValue DAGCombiner::visitADDSAT(SDNode *N) {
2681 unsigned Opcode = N->getOpcode();
2682 SDValue N0 = N->getOperand(0);
2683 SDValue N1 = N->getOperand(1);
2684 EVT VT = N0.getValueType();
2685 SDLoc DL(N);
2686
2687 // fold (add_sat x, undef) -> -1
2688 if (N0.isUndef() || N1.isUndef())
2689 return DAG.getAllOnesConstant(DL, VT);
2690
2691 // fold (add_sat c1, c2) -> c3
2692 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1}))
2693 return C;
2694
2695 // canonicalize constant to RHS
2696 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
2697 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
2698 return DAG.getNode(Opcode, DL, VT, N1, N0);
2699
2700 // fold vector ops
2701 if (VT.isVector()) {
2702 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
2703 return FoldedVOp;
2704
2705 // fold (add_sat x, 0) -> x, vector edition
2706 if (ISD::isConstantSplatVectorAllZeros(N1.getNode()))
2707 return N0;
2708 }
2709
2710 // fold (add_sat x, 0) -> x
2711 if (isNullConstant(N1))
2712 return N0;
2713
2714 // If it cannot overflow, transform into an add.
2715 if (Opcode == ISD::UADDSAT)
2716 if (DAG.computeOverflowKind(N0, N1) == SelectionDAG::OFK_Never)
2717 return DAG.getNode(ISD::ADD, DL, VT, N0, N1);
2718
2719 return SDValue();
2720}
2721
2722static SDValue getAsCarry(const TargetLowering &TLI, SDValue V) {
2723 bool Masked = false;
2724
2725 // First, peel away TRUNCATE/ZERO_EXTEND/AND nodes due to legalization.
2726 while (true) {
2727 if (V.getOpcode() == ISD::TRUNCATE || V.getOpcode() == ISD::ZERO_EXTEND) {
2728 V = V.getOperand(0);
2729 continue;
2730 }
2731
2732 if (V.getOpcode() == ISD::AND && isOneConstant(V.getOperand(1))) {
2733 Masked = true;
2734 V = V.getOperand(0);
2735 continue;
2736 }
2737
2738 break;
2739 }
2740
2741 // If this is not a carry, return.
2742 if (V.getResNo() != 1)
2743 return SDValue();
2744
2745 if (V.getOpcode() != ISD::ADDCARRY && V.getOpcode() != ISD::SUBCARRY &&
2746 V.getOpcode() != ISD::UADDO && V.getOpcode() != ISD::USUBO)
2747 return SDValue();
2748
2749 EVT VT = V->getValueType(0);
2750 if (!TLI.isOperationLegalOrCustom(V.getOpcode(), VT))
2751 return SDValue();
2752
2753 // If the result is masked, then no matter what kind of bool it is we can
2754 // return. If it isn't, then we need to make sure the bool type is either 0 or
2755 // 1 and not other values.
2756 if (Masked ||
2757 TLI.getBooleanContents(V.getValueType()) ==
2758 TargetLoweringBase::ZeroOrOneBooleanContent)
2759 return V;
2760
2761 return SDValue();
2762}
2763
2764/// Given the operands of an add/sub operation, see if the 2nd operand is a
2765/// masked 0/1 whose source operand is actually known to be 0/-1. If so, invert
2766/// the opcode and bypass the mask operation.
2767static SDValue foldAddSubMasked1(bool IsAdd, SDValue N0, SDValue N1,
2768 SelectionDAG &DAG, const SDLoc &DL) {
2769 if (N1.getOpcode() != ISD::AND || !isOneOrOneSplat(N1->getOperand(1)))
2770 return SDValue();
2771
2772 EVT VT = N0.getValueType();
2773 if (DAG.ComputeNumSignBits(N1.getOperand(0)) != VT.getScalarSizeInBits())
2774 return SDValue();
2775
2776 // add N0, (and (AssertSext X, i1), 1) --> sub N0, X
2777 // sub N0, (and (AssertSext X, i1), 1) --> add N0, X
2778 return DAG.getNode(IsAdd ? ISD::SUB : ISD::ADD, DL, VT, N0, N1.getOperand(0));
2779}
2780
2781/// Helper for doing combines based on N0 and N1 being added to each other.
2782SDValue DAGCombiner::visitADDLikeCommutative(SDValue N0, SDValue N1,
2783 SDNode *LocReference) {
2784 EVT VT = N0.getValueType();
2785 SDLoc DL(LocReference);
2786
2787 // fold (add x, shl(0 - y, n)) -> sub(x, shl(y, n))
2788 if (N1.getOpcode() == ISD::SHL && N1.getOperand(0).getOpcode() == ISD::SUB &&
2789 isNullOrNullSplat(N1.getOperand(0).getOperand(0)))
2790 return DAG.getNode(ISD::SUB, DL, VT, N0,
2791 DAG.getNode(ISD::SHL, DL, VT,
2792 N1.getOperand(0).getOperand(1),
2793 N1.getOperand(1)));
2794
2795 if (SDValue V = foldAddSubMasked1(true, N0, N1, DAG, DL))
2796 return V;
2797
2798 // Look for:
2799 // add (add x, 1), y
2800 // And if the target does not like this form then turn into:
2801 // sub y, (xor x, -1)
2802 if (!TLI.preferIncOfAddToSubOfNot(VT) && N0.getOpcode() == ISD::ADD &&
2803 N0.hasOneUse() && isOneOrOneSplat(N0.getOperand(1))) {
2804 SDValue Not = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(0),
2805 DAG.getAllOnesConstant(DL, VT));
2806 return DAG.getNode(ISD::SUB, DL, VT, N1, Not);
2807 }
2808
2809 if (N0.getOpcode() == ISD::SUB && N0.hasOneUse()) {
2810 // Hoist one-use subtraction by non-opaque constant:
2811 // (x - C) + y -> (x + y) - C
2812 // This is necessary because SUB(X,C) -> ADD(X,-C) doesn't work for vectors.
2813 if (isConstantOrConstantVector(N0.getOperand(1), /*NoOpaques=*/true)) {
2814 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), N1);
2815 return DAG.getNode(ISD::SUB, DL, VT, Add, N0.getOperand(1));
2816 }
2817 // Hoist one-use subtraction from non-opaque constant:
2818 // (C - x) + y -> (y - x) + C
2819 if (isConstantOrConstantVector(N0.getOperand(0), /*NoOpaques=*/true)) {
2820 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N1, N0.getOperand(1));
2821 return DAG.getNode(ISD::ADD, DL, VT, Sub, N0.getOperand(0));
2822 }
2823 }
2824
2825 // If the target's bool is represented as 0/1, prefer to make this 'sub 0/1'
2826 // rather than 'add 0/-1' (the zext should get folded).
2827 // add (sext i1 Y), X --> sub X, (zext i1 Y)
2828 if (N0.getOpcode() == ISD::SIGN_EXTEND &&
2829 N0.getOperand(0).getScalarValueSizeInBits() == 1 &&
2830 TLI.getBooleanContents(VT) == TargetLowering::ZeroOrOneBooleanContent) {
2831 SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0));
2832 return DAG.getNode(ISD::SUB, DL, VT, N1, ZExt);
2833 }
2834
2835 // add X, (sextinreg Y i1) -> sub X, (and Y 1)
2836 if (N1.getOpcode() == ISD::SIGN_EXTEND_INREG) {
2837 VTSDNode *TN = cast<VTSDNode>(N1.getOperand(1));
2838 if (TN->getVT() == MVT::i1) {
2839 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0),
2840 DAG.getConstant(1, DL, VT));
2841 return DAG.getNode(ISD::SUB, DL, VT, N0, ZExt);
2842 }
2843 }
2844
2845 // (add X, (addcarry Y, 0, Carry)) -> (addcarry X, Y, Carry)
2846 if (N1.getOpcode() == ISD::ADDCARRY && isNullConstant(N1.getOperand(1)) &&
2847 N1.getResNo() == 0)
2848 return DAG.getNode(ISD::ADDCARRY, DL, N1->getVTList(),
2849 N0, N1.getOperand(0), N1.getOperand(2));
2850
2851 // (add X, Carry) -> (addcarry X, 0, Carry)
2852 if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT))
2853 if (SDValue Carry = getAsCarry(TLI, N1))
2854 return DAG.getNode(ISD::ADDCARRY, DL,
2855 DAG.getVTList(VT, Carry.getValueType()), N0,
2856 DAG.getConstant(0, DL, VT), Carry);
2857
2858 return SDValue();
2859}
2860
2861SDValue DAGCombiner::visitADDC(SDNode *N) {
2862 SDValue N0 = N->getOperand(0);
2863 SDValue N1 = N->getOperand(1);
2864 EVT VT = N0.getValueType();
2865 SDLoc DL(N);
2866
2867 // If the flag result is dead, turn this into an ADD.
2868 if (!N->hasAnyUseOfValue(1))
2869 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2870 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2871
2872 // canonicalize constant to RHS.
2873 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
2874 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2875 if (N0C && !N1C)
2876 return DAG.getNode(ISD::ADDC, DL, N->getVTList(), N1, N0);
2877
2878 // fold (addc x, 0) -> x + no carry out
2879 if (isNullConstant(N1))
2880 return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE,
2881 DL, MVT::Glue));
2882
2883 // If it cannot overflow, transform into an add.
2884 if (DAG.computeOverflowKind(N0, N1) == SelectionDAG::OFK_Never)
2885 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2886 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2887
2888 return SDValue();
2889}
2890
2891/**
2892 * Flips a boolean if it is cheaper to compute. If the Force parameters is set,
2893 * then the flip also occurs if computing the inverse is the same cost.
2894 * This function returns an empty SDValue in case it cannot flip the boolean
2895 * without increasing the cost of the computation. If you want to flip a boolean
2896 * no matter what, use DAG.getLogicalNOT.
2897 */
2898static SDValue extractBooleanFlip(SDValue V, SelectionDAG &DAG,
2899 const TargetLowering &TLI,
2900 bool Force) {
2901 if (Force && isa<ConstantSDNode>(V))
2902 return DAG.getLogicalNOT(SDLoc(V), V, V.getValueType());
2903
2904 if (V.getOpcode() != ISD::XOR)
2905 return SDValue();
2906
2907 ConstantSDNode *Const = isConstOrConstSplat(V.getOperand(1), false);
2908 if (!Const)
2909 return SDValue();
2910
2911 EVT VT = V.getValueType();
2912
2913 bool IsFlip = false;
2914 switch(TLI.getBooleanContents(VT)) {
2915 case TargetLowering::ZeroOrOneBooleanContent:
2916 IsFlip = Const->isOne();
2917 break;
2918 case TargetLowering::ZeroOrNegativeOneBooleanContent:
2919 IsFlip = Const->isAllOnes();
2920 break;
2921 case TargetLowering::UndefinedBooleanContent:
2922 IsFlip = (Const->getAPIntValue() & 0x01) == 1;
2923 break;
2924 }
2925
2926 if (IsFlip)
2927 return V.getOperand(0);
2928 if (Force)
2929 return DAG.getLogicalNOT(SDLoc(V), V, V.getValueType());
2930 return SDValue();
2931}
2932
2933SDValue DAGCombiner::visitADDO(SDNode *N) {
2934 SDValue N0 = N->getOperand(0);
2935 SDValue N1 = N->getOperand(1);
2936 EVT VT = N0.getValueType();
2937 bool IsSigned = (ISD::SADDO == N->getOpcode());
2938
2939 EVT CarryVT = N->getValueType(1);
2940 SDLoc DL(N);
2941
2942 // If the flag result is dead, turn this into an ADD.
2943 if (!N->hasAnyUseOfValue(1))
2944 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2945 DAG.getUNDEF(CarryVT));
2946
2947 // canonicalize constant to RHS.
2948 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
2949 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
2950 return DAG.getNode(N->getOpcode(), DL, N->getVTList(), N1, N0);
2951
2952 // fold (addo x, 0) -> x + no carry out
2953 if (isNullOrNullSplat(N1))
2954 return CombineTo(N, N0, DAG.getConstant(0, DL, CarryVT));
2955
2956 if (!IsSigned) {
2957 // If it cannot overflow, transform into an add.
2958 if (DAG.computeOverflowKind(N0, N1) == SelectionDAG::OFK_Never)
2959 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2960 DAG.getConstant(0, DL, CarryVT));
2961
2962 // fold (uaddo (xor a, -1), 1) -> (usub 0, a) and flip carry.
2963 if (isBitwiseNot(N0) && isOneOrOneSplat(N1)) {
2964 SDValue Sub = DAG.getNode(ISD::USUBO, DL, N->getVTList(),
2965 DAG.getConstant(0, DL, VT), N0.getOperand(0));
2966 return CombineTo(
2967 N, Sub, DAG.getLogicalNOT(DL, Sub.getValue(1), Sub->getValueType(1)));
2968 }
2969
2970 if (SDValue Combined = visitUADDOLike(N0, N1, N))
2971 return Combined;
2972
2973 if (SDValue Combined = visitUADDOLike(N1, N0, N))
2974 return Combined;
2975 }
2976
2977 return SDValue();
2978}
2979
2980SDValue DAGCombiner::visitUADDOLike(SDValue N0, SDValue N1, SDNode *N) {
2981 EVT VT = N0.getValueType();
2982 if (VT.isVector())
2983 return SDValue();
2984
2985 // (uaddo X, (addcarry Y, 0, Carry)) -> (addcarry X, Y, Carry)
2986 // If Y + 1 cannot overflow.
2987 if (N1.getOpcode() == ISD::ADDCARRY && isNullConstant(N1.getOperand(1))) {
2988 SDValue Y = N1.getOperand(0);
2989 SDValue One = DAG.getConstant(1, SDLoc(N), Y.getValueType());
2990 if (DAG.computeOverflowKind(Y, One) == SelectionDAG::OFK_Never)
2991 return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0, Y,
2992 N1.getOperand(2));
2993 }
2994
2995 // (uaddo X, Carry) -> (addcarry X, 0, Carry)
2996 if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT))
2997 if (SDValue Carry = getAsCarry(TLI, N1))
2998 return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0,
2999 DAG.getConstant(0, SDLoc(N), VT), Carry);
3000
3001 return SDValue();
3002}
3003
3004SDValue DAGCombiner::visitADDE(SDNode *N) {
3005 SDValue N0 = N->getOperand(0);
3006 SDValue N1 = N->getOperand(1);
3007 SDValue CarryIn = N->getOperand(2);
3008
3009 // canonicalize constant to RHS
3010 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
3011 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
3012 if (N0C && !N1C)
3013 return DAG.getNode(ISD::ADDE, SDLoc(N), N->getVTList(),
3014 N1, N0, CarryIn);
3015
3016 // fold (adde x, y, false) -> (addc x, y)
3017 if (CarryIn.getOpcode() == ISD::CARRY_FALSE)
3018 return DAG.getNode(ISD::ADDC, SDLoc(N), N->getVTList(), N0, N1);
3019
3020 return SDValue();
3021}
3022
3023SDValue DAGCombiner::visitADDCARRY(SDNode *N) {
3024 SDValue N0 = N->getOperand(0);
3025 SDValue N1 = N->getOperand(1);
3026 SDValue CarryIn = N->getOperand(2);
3027 SDLoc DL(N);
3028
3029 // canonicalize constant to RHS
3030 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
3031 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
3032 if (N0C && !N1C)
3033 return DAG.getNode(ISD::ADDCARRY, DL, N->getVTList(), N1, N0, CarryIn);
3034
3035 // fold (addcarry x, y, false) -> (uaddo x, y)
3036 if (isNullConstant(CarryIn)) {
3037 if (!LegalOperations ||
3038 TLI.isOperationLegalOrCustom(ISD::UADDO, N->getValueType(0)))
3039 return DAG.getNode(ISD::UADDO, DL, N->getVTList(), N0, N1);
3040 }
3041
3042 // fold (addcarry 0, 0, X) -> (and (ext/trunc X), 1) and no carry.
3043 if (isNullConstant(N0) && isNullConstant(N1)) {
3044 EVT VT = N0.getValueType();
3045 EVT CarryVT = CarryIn.getValueType();
3046 SDValue CarryExt = DAG.getBoolExtOrTrunc(CarryIn, DL, VT, CarryVT);
3047 AddToWorklist(CarryExt.getNode());
3048 return CombineTo(N, DAG.getNode(ISD::AND, DL, VT, CarryExt,
3049 DAG.getConstant(1, DL, VT)),
3050 DAG.getConstant(0, DL, CarryVT));
3051 }
3052
3053 if (SDValue Combined = visitADDCARRYLike(N0, N1, CarryIn, N))
3054 return Combined;
3055
3056 if (SDValue Combined = visitADDCARRYLike(N1, N0, CarryIn, N))
3057 return Combined;
3058
3059 return SDValue();
3060}
3061
3062SDValue DAGCombiner::visitSADDO_CARRY(SDNode *N) {
3063 SDValue N0 = N->getOperand(0);
3064 SDValue N1 = N->getOperand(1);
3065 SDValue CarryIn = N->getOperand(2);
3066 SDLoc DL(N);
3067
3068 // canonicalize constant to RHS
3069 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
3070 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
3071 if (N0C && !N1C)
3072 return DAG.getNode(ISD::SADDO_CARRY, DL, N->getVTList(), N1, N0, CarryIn);
3073
3074 // fold (saddo_carry x, y, false) -> (saddo x, y)
3075 if (isNullConstant(CarryIn)) {
3076 if (!LegalOperations ||
3077 TLI.isOperationLegalOrCustom(ISD::SADDO, N->getValueType(0)))
3078 return DAG.getNode(ISD::SADDO, DL, N->getVTList(), N0, N1);
3079 }
3080
3081 return SDValue();
3082}
3083
3084/**
3085 * If we are facing some sort of diamond carry propapagtion pattern try to
3086 * break it up to generate something like:
3087 * (addcarry X, 0, (addcarry A, B, Z):Carry)
3088 *
3089 * The end result is usually an increase in operation required, but because the
3090 * carry is now linearized, other tranforms can kick in and optimize the DAG.
3091 *
3092 * Patterns typically look something like
3093 * (uaddo A, B)
3094 * / \
3095 * Carry Sum
3096 * | \
3097 * | (addcarry *, 0, Z)
3098 * | /
3099 * \ Carry
3100 * | /
3101 * (addcarry X, *, *)
3102 *
3103 * But numerous variation exist. Our goal is to identify A, B, X and Z and
3104 * produce a combine with a single path for carry propagation.
3105 */
3106static SDValue combineADDCARRYDiamond(DAGCombiner &Combiner, SelectionDAG &DAG,
3107 SDValue X, SDValue Carry0, SDValue Carry1,
3108 SDNode *N) {
3109 if (Carry1.getResNo() != 1 || Carry0.getResNo() != 1)
3110 return SDValue();
3111 if (Carry1.getOpcode() != ISD::UADDO)
3112 return SDValue();
3113
3114 SDValue Z;
3115
3116 /**
3117 * First look for a suitable Z. It will present itself in the form of
3118 * (addcarry Y, 0, Z) or its equivalent (uaddo Y, 1) for Z=true
3119 */
3120 if (Carry0.getOpcode() == ISD::ADDCARRY &&
3121 isNullConstant(Carry0.getOperand(1))) {
3122 Z = Carry0.getOperand(2);
3123 } else if (Carry0.getOpcode() == ISD::UADDO &&
3124 isOneConstant(Carry0.getOperand(1))) {
3125 EVT VT = Combiner.getSetCCResultType(Carry0.getValueType());
3126 Z = DAG.getConstant(1, SDLoc(Carry0.getOperand(1)), VT);
3127 } else {
3128 // We couldn't find a suitable Z.
3129 return SDValue();
3130 }
3131
3132
3133 auto cancelDiamond = [&](SDValue A,SDValue B) {
3134 SDLoc DL(N);
3135 SDValue NewY = DAG.getNode(ISD::ADDCARRY, DL, Carry0->getVTList(), A, B, Z);
3136 Combiner.AddToWorklist(NewY.getNode());
3137 return DAG.getNode(ISD::ADDCARRY, DL, N->getVTList(), X,
3138 DAG.getConstant(0, DL, X.getValueType()),
3139 NewY.getValue(1));
3140 };
3141
3142 /**
3143 * (uaddo A, B)
3144 * |
3145 * Sum
3146 * |
3147 * (addcarry *, 0, Z)
3148 */
3149 if (Carry0.getOperand(0) == Carry1.getValue(0)) {
3150 return cancelDiamond(Carry1.getOperand(0), Carry1.getOperand(1));
3151 }
3152
3153 /**
3154 * (addcarry A, 0, Z)
3155 * |
3156 * Sum
3157 * |
3158 * (uaddo *, B)
3159 */
3160 if (Carry1.getOperand(0) == Carry0.getValue(0)) {
3161 return cancelDiamond(Carry0.getOperand(0), Carry1.getOperand(1));
3162 }
3163
3164 if (Carry1.getOperand(1) == Carry0.getValue(0)) {
3165 return cancelDiamond(Carry1.getOperand(0), Carry0.getOperand(0));
3166 }
3167
3168 return SDValue();
3169}
3170
3171// If we are facing some sort of diamond carry/borrow in/out pattern try to
3172// match patterns like:
3173//
3174// (uaddo A, B) CarryIn
3175// | \ |
3176// | \ |
3177// PartialSum PartialCarryOutX /
3178// | | /
3179// | ____|____________/
3180// | / |
3181// (uaddo *, *) \________
3182// | \ \
3183// | \ |
3184// | PartialCarryOutY |
3185// | \ |
3186// | \ /
3187// AddCarrySum | ______/
3188// | /
3189// CarryOut = (or *, *)
3190//
3191// And generate ADDCARRY (or SUBCARRY) with two result values:
3192//
3193// {AddCarrySum, CarryOut} = (addcarry A, B, CarryIn)
3194//
3195// Our goal is to identify A, B, and CarryIn and produce ADDCARRY/SUBCARRY with
3196// a single path for carry/borrow out propagation:
3197static SDValue combineCarryDiamond(SelectionDAG &DAG, const TargetLowering &TLI,
3198 SDValue N0, SDValue N1, SDNode *N) {
3199 SDValue Carry0 = getAsCarry(TLI, N0);
3200 if (!Carry0)
3201 return SDValue();
3202 SDValue Carry1 = getAsCarry(TLI, N1);
3203 if (!Carry1)
3204 return SDValue();
3205
3206 unsigned Opcode = Carry0.getOpcode();
3207 if (Opcode != Carry1.getOpcode())
3208 return SDValue();
3209 if (Opcode != ISD::UADDO && Opcode != ISD::USUBO)
3210 return SDValue();
3211
3212 // Canonicalize the add/sub of A and B (the top node in the above ASCII art)
3213 // as Carry0 and the add/sub of the carry in as Carry1 (the middle node).
3214 if (Carry1.getNode()->isOperandOf(Carry0.getNode()))
3215 std::swap(Carry0, Carry1);
3216
3217 // Check if nodes are connected in expected way.
3218 if (Carry1.getOperand(0) != Carry0.getValue(0) &&
3219 Carry1.getOperand(1) != Carry0.getValue(0))
3220 return SDValue();
3221
3222 // The carry in value must be on the righthand side for subtraction.
3223 unsigned CarryInOperandNum =
3224 Carry1.getOperand(0) == Carry0.getValue(0) ? 1 : 0;
3225 if (Opcode == ISD::USUBO && CarryInOperandNum != 1)
3226 return SDValue();
3227 SDValue CarryIn = Carry1.getOperand(CarryInOperandNum);
3228
3229 unsigned NewOp = Opcode == ISD::UADDO ? ISD::ADDCARRY : ISD::SUBCARRY;
3230 if (!TLI.isOperationLegalOrCustom(NewOp, Carry0.getValue(0).getValueType()))
3231 return SDValue();
3232
3233 // Verify that the carry/borrow in is plausibly a carry/borrow bit.
3234 // TODO: make getAsCarry() aware of how partial carries are merged.
3235 if (CarryIn.getOpcode() != ISD::ZERO_EXTEND)
3236 return SDValue();
3237 CarryIn = CarryIn.getOperand(0);
3238 if (CarryIn.getValueType() != MVT::i1)
3239 return SDValue();
3240
3241 SDLoc DL(N);
3242 SDValue Merged =
3243 DAG.getNode(NewOp, DL, Carry1->getVTList(), Carry0.getOperand(0),
3244 Carry0.getOperand(1), CarryIn);
3245
3246 // Please note that because we have proven that the result of the UADDO/USUBO
3247 // of A and B feeds into the UADDO/USUBO that does the carry/borrow in, we can
3248 // therefore prove that if the first UADDO/USUBO overflows, the second
3249 // UADDO/USUBO cannot. For example consider 8-bit numbers where 0xFF is the
3250 // maximum value.
3251 //
3252 // 0xFF + 0xFF == 0xFE with carry but 0xFE + 1 does not carry
3253 // 0x00 - 0xFF == 1 with a carry/borrow but 1 - 1 == 0 (no carry/borrow)
3254 //
3255 // This is important because it means that OR and XOR can be used to merge
3256 // carry flags; and that AND can return a constant zero.
3257 //
3258 // TODO: match other operations that can merge flags (ADD, etc)
3259 DAG.ReplaceAllUsesOfValueWith(Carry1.getValue(0), Merged.getValue(0));
3260 if (N->getOpcode() == ISD::AND)
3261 return DAG.getConstant(0, DL, MVT::i1);
3262 return Merged.getValue(1);
3263}
3264
3265SDValue DAGCombiner::visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn,
3266 SDNode *N) {
3267 // fold (addcarry (xor a, -1), b, c) -> (subcarry b, a, !c) and flip carry.
3268 if (isBitwiseNot(N0))
3269 if (SDValue NotC = extractBooleanFlip(CarryIn, DAG, TLI, true)) {
3270 SDLoc DL(N);
3271 SDValue Sub = DAG.getNode(ISD::SUBCARRY, DL, N->getVTList(), N1,
3272 N0.getOperand(0), NotC);
3273 return CombineTo(
3274 N, Sub, DAG.getLogicalNOT(DL, Sub.getValue(1), Sub->getValueType(1)));
3275 }
3276
3277 // Iff the flag result is dead:
3278 // (addcarry (add|uaddo X, Y), 0, Carry) -> (addcarry X, Y, Carry)
3279 // Don't do this if the Carry comes from the uaddo. It won't remove the uaddo
3280 // or the dependency between the instructions.
3281 if ((N0.getOpcode() == ISD::ADD ||
3282 (N0.getOpcode() == ISD::UADDO && N0.getResNo() == 0 &&
3283 N0.getValue(1) != CarryIn)) &&
3284 isNullConstant(N1) && !N->hasAnyUseOfValue(1))
3285 return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(),
3286 N0.getOperand(0), N0.getOperand(1), CarryIn);
3287
3288 /**
3289 * When one of the addcarry argument is itself a carry, we may be facing
3290 * a diamond carry propagation. In which case we try to transform the DAG
3291 * to ensure linear carry propagation if that is possible.
3292 */
3293 if (auto Y = getAsCarry(TLI, N1)) {
3294 // Because both are carries, Y and Z can be swapped.
3295 if (auto R = combineADDCARRYDiamond(*this, DAG, N0, Y, CarryIn, N))
3296 return R;
3297 if (auto R = combineADDCARRYDiamond(*this, DAG, N0, CarryIn, Y, N))
3298 return R;
3299 }
3300
3301 return SDValue();
3302}
3303
3304// Attempt to create a USUBSAT(LHS, RHS) node with DstVT, performing a
3305// clamp/truncation if necessary.
3306static SDValue getTruncatedUSUBSAT(EVT DstVT, EVT SrcVT, SDValue LHS,
3307 SDValue RHS, SelectionDAG &DAG,
3308 const SDLoc &DL) {
3309 assert(DstVT.getScalarSizeInBits() <= SrcVT.getScalarSizeInBits() &&(static_cast <bool> (DstVT.getScalarSizeInBits() <= SrcVT
.getScalarSizeInBits() && "Illegal truncation") ? void
(0) : __assert_fail ("DstVT.getScalarSizeInBits() <= SrcVT.getScalarSizeInBits() && \"Illegal truncation\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 3310, __extension__
__PRETTY_FUNCTION__))
3310 "Illegal truncation")(static_cast <bool> (DstVT.getScalarSizeInBits() <= SrcVT
.getScalarSizeInBits() && "Illegal truncation") ? void
(0) : __assert_fail ("DstVT.getScalarSizeInBits() <= SrcVT.getScalarSizeInBits() && \"Illegal truncation\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 3310, __extension__
__PRETTY_FUNCTION__))
;
3311
3312 if (DstVT == SrcVT)
3313 return DAG.getNode(ISD::USUBSAT, DL, DstVT, LHS, RHS);
3314
3315 // If the LHS is zero-extended then we can perform the USUBSAT as DstVT by
3316 // clamping RHS.
3317 APInt UpperBits = APInt::getBitsSetFrom(SrcVT.getScalarSizeInBits(),
3318 DstVT.getScalarSizeInBits());
3319 if (!DAG.MaskedValueIsZero(LHS, UpperBits))
3320 return SDValue();
3321
3322 SDValue SatLimit =
3323 DAG.getConstant(APInt::getLowBitsSet(SrcVT.getScalarSizeInBits(),
3324 DstVT.getScalarSizeInBits()),
3325 DL, SrcVT);
3326 RHS = DAG.getNode(ISD::UMIN, DL, SrcVT, RHS, SatLimit);
3327 RHS = DAG.getNode(ISD::TRUNCATE, DL, DstVT, RHS);
3328 LHS = DAG.getNode(ISD::TRUNCATE, DL, DstVT, LHS);
3329 return DAG.getNode(ISD::USUBSAT, DL, DstVT, LHS, RHS);
3330}
3331
3332// Try to find umax(a,b) - b or a - umin(a,b) patterns that may be converted to
3333// usubsat(a,b), optionally as a truncated type.
3334SDValue DAGCombiner::foldSubToUSubSat(EVT DstVT, SDNode *N) {
3335 if (N->getOpcode() != ISD::SUB ||
3336 !(!LegalOperations || hasOperation(ISD::USUBSAT, DstVT)))
3337 return SDValue();
3338
3339 EVT SubVT = N->getValueType(0);
3340 SDValue Op0 = N->getOperand(0);
3341 SDValue Op1 = N->getOperand(1);
3342
3343 // Try to find umax(a,b) - b or a - umin(a,b) patterns
3344 // they may be converted to usubsat(a,b).
3345 if (Op0.getOpcode() == ISD::UMAX && Op0.hasOneUse()) {
3346 SDValue MaxLHS = Op0.getOperand(0);
3347 SDValue MaxRHS = Op0.getOperand(1);
3348 if (MaxLHS == Op1)
3349 return getTruncatedUSUBSAT(DstVT, SubVT, MaxRHS, Op1, DAG, SDLoc(N));
3350 if (MaxRHS == Op1)
3351 return getTruncatedUSUBSAT(DstVT, SubVT, MaxLHS, Op1, DAG, SDLoc(N));
3352 }
3353
3354 if (Op1.getOpcode() == ISD::UMIN && Op1.hasOneUse()) {
3355 SDValue MinLHS = Op1.getOperand(0);
3356 SDValue MinRHS = Op1.getOperand(1);
3357 if (MinLHS == Op0)
3358 return getTruncatedUSUBSAT(DstVT, SubVT, Op0, MinRHS, DAG, SDLoc(N));
3359 if (MinRHS == Op0)
3360 return getTruncatedUSUBSAT(DstVT, SubVT, Op0, MinLHS, DAG, SDLoc(N));
3361 }
3362
3363 // sub(a,trunc(umin(zext(a),b))) -> usubsat(a,trunc(umin(b,SatLimit)))
3364 if (Op1.getOpcode() == ISD::TRUNCATE &&
3365 Op1.getOperand(0).getOpcode() == ISD::UMIN &&
3366 Op1.getOperand(0).hasOneUse()) {
3367 SDValue MinLHS = Op1.getOperand(0).getOperand(0);
3368 SDValue MinRHS = Op1.getOperand(0).getOperand(1);
3369 if (MinLHS.getOpcode() == ISD::ZERO_EXTEND && MinLHS.getOperand(0) == Op0)
3370 return getTruncatedUSUBSAT(DstVT, MinLHS.getValueType(), MinLHS, MinRHS,
3371 DAG, SDLoc(N));
3372 if (MinRHS.getOpcode() == ISD::ZERO_EXTEND && MinRHS.getOperand(0) == Op0)
3373 return getTruncatedUSUBSAT(DstVT, MinLHS.getValueType(), MinRHS, MinLHS,
3374 DAG, SDLoc(N));
3375 }
3376
3377 return SDValue();
3378}
3379
3380// Since it may not be valid to emit a fold to zero for vector initializers
3381// check if we can before folding.
3382static SDValue tryFoldToZero(const SDLoc &DL, const TargetLowering &TLI, EVT VT,
3383 SelectionDAG &DAG, bool LegalOperations) {
3384 if (!VT.isVector())
3385 return DAG.getConstant(0, DL, VT);
3386 if (!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT))
3387 return DAG.getConstant(0, DL, VT);
3388 return SDValue();
3389}
3390
3391SDValue DAGCombiner::visitSUB(SDNode *N) {
3392 SDValue N0 = N->getOperand(0);
3393 SDValue N1 = N->getOperand(1);
3394 EVT VT = N0.getValueType();
3395 SDLoc DL(N);
3396
3397 auto PeekThroughFreeze = [](SDValue N) {
3398 if (N->getOpcode() == ISD::FREEZE && N.hasOneUse())
3399 return N->getOperand(0);
3400 return N;
3401 };
3402
3403 // fold (sub x, x) -> 0
3404 // FIXME: Refactor this and xor and other similar operations together.
3405 if (PeekThroughFreeze(N0) == PeekThroughFreeze(N1))
3406 return tryFoldToZero(DL, TLI, VT, DAG, LegalOperations);
3407
3408 // fold (sub c1, c2) -> c3
3409 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N0, N1}))
3410 return C;
3411
3412 // fold vector ops
3413 if (VT.isVector()) {
3414 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
3415 return FoldedVOp;
3416
3417 // fold (sub x, 0) -> x, vector edition
3418 if (ISD::isConstantSplatVectorAllZeros(N1.getNode()))
3419 return N0;
3420 }
3421
3422 if (SDValue NewSel = foldBinOpIntoSelect(N))
3423 return NewSel;
3424
3425 ConstantSDNode *N1C = getAsNonOpaqueConstant(N1);
3426
3427 // fold (sub x, c) -> (add x, -c)
3428 if (N1C) {
3429 return DAG.getNode(ISD::ADD, DL, VT, N0,
3430 DAG.getConstant(-N1C->getAPIntValue(), DL, VT));
3431 }
3432
3433 if (isNullOrNullSplat(N0)) {
3434 unsigned BitWidth = VT.getScalarSizeInBits();
3435 // Right-shifting everything out but the sign bit followed by negation is
3436 // the same as flipping arithmetic/logical shift type without the negation:
3437 // -(X >>u 31) -> (X >>s 31)
3438 // -(X >>s 31) -> (X >>u 31)
3439 if (N1->getOpcode() == ISD::SRA || N1->getOpcode() == ISD::SRL) {
3440 ConstantSDNode *ShiftAmt = isConstOrConstSplat(N1.getOperand(1));
3441 if (ShiftAmt && ShiftAmt->getAPIntValue() == (BitWidth - 1)) {
3442 auto NewSh = N1->getOpcode() == ISD::SRA ? ISD::SRL : ISD::SRA;
3443 if (!LegalOperations || TLI.isOperationLegal(NewSh, VT))
3444 return DAG.getNode(NewSh, DL, VT, N1.getOperand(0), N1.getOperand(1));
3445 }
3446 }
3447
3448 // 0 - X --> 0 if the sub is NUW.
3449 if (N->getFlags().hasNoUnsignedWrap())
3450 return N0;
3451
3452 if (DAG.MaskedValueIsZero(N1, ~APInt::getSignMask(BitWidth))) {
3453 // N1 is either 0 or the minimum signed value. If the sub is NSW, then
3454 // N1 must be 0 because negating the minimum signed value is undefined.
3455 if (N->getFlags().hasNoSignedWrap())
3456 return N0;
3457
3458 // 0 - X --> X if X is 0 or the minimum signed value.
3459 return N1;
3460 }
3461
3462 // Convert 0 - abs(x).
3463 if (N1.getOpcode() == ISD::ABS && N1.hasOneUse() &&
3464 !TLI.isOperationLegalOrCustom(ISD::ABS, VT))
3465 if (SDValue Result = TLI.expandABS(N1.getNode(), DAG, true))
3466 return Result;
3467
3468 // Fold neg(splat(neg(x)) -> splat(x)
3469 if (VT.isVector()) {
3470 SDValue N1S = DAG.getSplatValue(N1, true);
3471 if (N1S && N1S.getOpcode() == ISD::SUB &&
3472 isNullConstant(N1S.getOperand(0))) {
3473 if (VT.isScalableVector())
3474 return DAG.getSplatVector(VT, DL, N1S.getOperand(1));
3475 return DAG.getSplatBuildVector(VT, DL, N1S.getOperand(1));
3476 }
3477 }
3478 }
3479
3480 // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1)
3481 if (isAllOnesOrAllOnesSplat(N0))
3482 return DAG.getNode(ISD::XOR, DL, VT, N1, N0);
3483
3484 // fold (A - (0-B)) -> A+B
3485 if (N1.getOpcode() == ISD::SUB && isNullOrNullSplat(N1.getOperand(0)))
3486 return DAG.getNode(ISD::ADD, DL, VT, N0, N1.getOperand(1));
3487
3488 // fold A-(A-B) -> B
3489 if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(0))
3490 return N1.getOperand(1);
3491
3492 // fold (A+B)-A -> B
3493 if (N0.getOpcode() == ISD::ADD && N0.getOperand(0) == N1)
3494 return N0.getOperand(1);
3495
3496 // fold (A+B)-B -> A
3497 if (N0.getOpcode() == ISD::ADD && N0.getOperand(1) == N1)
3498 return N0.getOperand(0);
3499
3500 // fold (A+C1)-C2 -> A+(C1-C2)
3501 if (N0.getOpcode() == ISD::ADD) {
3502 SDValue N01 = N0.getOperand(1);
3503 if (SDValue NewC = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N01, N1}))
3504 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), NewC);
3505 }
3506
3507 // fold C2-(A+C1) -> (C2-C1)-A
3508 if (N1.getOpcode() == ISD::ADD) {
3509 SDValue N11 = N1.getOperand(1);
3510 if (SDValue NewC = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N0, N11}))
3511 return DAG.getNode(ISD::SUB, DL, VT, NewC, N1.getOperand(0));
3512 }
3513
3514 // fold (A-C1)-C2 -> A-(C1+C2)
3515 if (N0.getOpcode() == ISD::SUB) {
3516 SDValue N01 = N0.getOperand(1);
3517 if (SDValue NewC = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N01, N1}))
3518 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), NewC);
3519 }
3520
3521 // fold (c1-A)-c2 -> (c1-c2)-A
3522 if (N0.getOpcode() == ISD::SUB) {
3523 SDValue N00 = N0.getOperand(0);
3524 if (SDValue NewC = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N00, N1}))
3525 return DAG.getNode(ISD::SUB, DL, VT, NewC, N0.getOperand(1));
3526 }
3527
3528 // fold ((A+(B+or-C))-B) -> A+or-C
3529 if (N0.getOpcode() == ISD::ADD &&
3530 (N0.getOperand(1).getOpcode() == ISD::SUB ||
3531 N0.getOperand(1).getOpcode() == ISD::ADD) &&
3532 N0.getOperand(1).getOperand(0) == N1)
3533 return DAG.getNode(N0.getOperand(1).getOpcode(), DL, VT, N0.getOperand(0),
3534 N0.getOperand(1).getOperand(1));
3535
3536 // fold ((A+(C+B))-B) -> A+C
3537 if (N0.getOpcode() == ISD::ADD && N0.getOperand(1).getOpcode() == ISD::ADD &&
3538 N0.getOperand(1).getOperand(1) == N1)
3539 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0),
3540 N0.getOperand(1).getOperand(0));
3541
3542 // fold ((A-(B-C))-C) -> A-B
3543 if (N0.getOpcode() == ISD::SUB && N0.getOperand(1).getOpcode() == ISD::SUB &&
3544 N0.getOperand(1).getOperand(1) == N1)
3545 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0),
3546 N0.getOperand(1).getOperand(0));
3547
3548 // fold (A-(B-C)) -> A+(C-B)
3549 if (N1.getOpcode() == ISD::SUB && N1.hasOneUse())
3550 return DAG.getNode(ISD::ADD, DL, VT, N0,
3551 DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(1),
3552 N1.getOperand(0)));
3553
3554 // A - (A & B) -> A & (~B)
3555 if (N1.getOpcode() == ISD::AND) {
3556 SDValue A = N1.getOperand(0);
3557 SDValue B = N1.getOperand(1);
3558 if (A != N0)
3559 std::swap(A, B);
3560 if (A == N0 &&
3561 (N1.hasOneUse() || isConstantOrConstantVector(B, /*NoOpaques=*/true))) {
3562 SDValue InvB =
3563 DAG.getNode(ISD::XOR, DL, VT, B, DAG.getAllOnesConstant(DL, VT));
3564 return DAG.getNode(ISD::AND, DL, VT, A, InvB);
3565 }
3566 }
3567
3568 // fold (X - (-Y * Z)) -> (X + (Y * Z))
3569 if (N1.getOpcode() == ISD::MUL && N1.hasOneUse()) {
3570 if (N1.getOperand(0).getOpcode() == ISD::SUB &&
3571 isNullOrNullSplat(N1.getOperand(0).getOperand(0))) {
3572 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT,
3573 N1.getOperand(0).getOperand(1),
3574 N1.getOperand(1));
3575 return DAG.getNode(ISD::ADD, DL, VT, N0, Mul);
3576 }
3577 if (N1.getOperand(1).getOpcode() == ISD::SUB &&
3578 isNullOrNullSplat(N1.getOperand(1).getOperand(0))) {
3579 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT,
3580 N1.getOperand(0),
3581 N1.getOperand(1).getOperand(1));
3582 return DAG.getNode(ISD::ADD, DL, VT, N0, Mul);
3583 }
3584 }
3585
3586 // If either operand of a sub is undef, the result is undef
3587 if (N0.isUndef())
3588 return N0;
3589 if (N1.isUndef())
3590 return N1;
3591
3592 if (SDValue V = foldAddSubBoolOfMaskedVal(N, DAG))
3593 return V;
3594
3595 if (SDValue V = foldAddSubOfSignBit(N, DAG))
3596 return V;
3597
3598 if (SDValue V = foldAddSubMasked1(false, N0, N1, DAG, SDLoc(N)))
3599 return V;
3600
3601 if (SDValue V = foldSubToUSubSat(VT, N))
3602 return V;
3603
3604 // (x - y) - 1 -> add (xor y, -1), x
3605 if (N0.getOpcode() == ISD::SUB && N0.hasOneUse() && isOneOrOneSplat(N1)) {
3606 SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(1),
3607 DAG.getAllOnesConstant(DL, VT));
3608 return DAG.getNode(ISD::ADD, DL, VT, Xor, N0.getOperand(0));
3609 }
3610
3611 // Look for:
3612 // sub y, (xor x, -1)
3613 // And if the target does not like this form then turn into:
3614 // add (add x, y), 1
3615 if (TLI.preferIncOfAddToSubOfNot(VT) && N1.hasOneUse() && isBitwiseNot(N1)) {
3616 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, N1.getOperand(0));
3617 return DAG.getNode(ISD::ADD, DL, VT, Add, DAG.getConstant(1, DL, VT));
3618 }
3619
3620 // Hoist one-use addition by non-opaque constant:
3621 // (x + C) - y -> (x - y) + C
3622 if (N0.getOpcode() == ISD::ADD && N0.hasOneUse() &&
3623 isConstantOrConstantVector(N0.getOperand(1), /*NoOpaques=*/true)) {
3624 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), N1);
3625 return DAG.getNode(ISD::ADD, DL, VT, Sub, N0.getOperand(1));
3626 }
3627 // y - (x + C) -> (y - x) - C
3628 if (N1.getOpcode() == ISD::ADD && N1.hasOneUse() &&
3629 isConstantOrConstantVector(N1.getOperand(1), /*NoOpaques=*/true)) {
3630 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, N1.getOperand(0));
3631 return DAG.getNode(ISD::SUB, DL, VT, Sub, N1.getOperand(1));
3632 }
3633 // (x - C) - y -> (x - y) - C
3634 // This is necessary because SUB(X,C) -> ADD(X,-C) doesn't work for vectors.
3635 if (N0.getOpcode() == ISD::SUB && N0.hasOneUse() &&
3636 isConstantOrConstantVector(N0.getOperand(1), /*NoOpaques=*/true)) {
3637 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), N1);
3638 return DAG.getNode(ISD::SUB, DL, VT, Sub, N0.getOperand(1));
3639 }
3640 // (C - x) - y -> C - (x + y)
3641 if (N0.getOpcode() == ISD::SUB && N0.hasOneUse() &&
3642 isConstantOrConstantVector(N0.getOperand(0), /*NoOpaques=*/true)) {
3643 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(1), N1);
3644 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), Add);
3645 }
3646
3647 // If the target's bool is represented as 0/-1, prefer to make this 'add 0/-1'
3648 // rather than 'sub 0/1' (the sext should get folded).
3649 // sub X, (zext i1 Y) --> add X, (sext i1 Y)
3650 if (N1.getOpcode() == ISD::ZERO_EXTEND &&
3651 N1.getOperand(0).getScalarValueSizeInBits() == 1 &&
3652 TLI.getBooleanContents(VT) ==
3653 TargetLowering::ZeroOrNegativeOneBooleanContent) {
3654 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, N1.getOperand(0));
3655 return DAG.getNode(ISD::ADD, DL, VT, N0, SExt);
3656 }
3657
3658 // fold Y = sra (X, size(X)-1); sub (xor (X, Y), Y) -> (abs X)
3659 if (TLI.isOperationLegalOrCustom(ISD::ABS, VT)) {
3660 if (N0.getOpcode() == ISD::XOR && N1.getOpcode() == ISD::SRA) {
3661 SDValue X0 = N0.getOperand(0), X1 = N0.getOperand(1);
3662 SDValue S0 = N1.getOperand(0);
3663 if ((X0 == S0 && X1 == N1) || (X0 == N1 && X1 == S0))
3664 if (ConstantSDNode *C = isConstOrConstSplat(N1.getOperand(1)))
3665 if (C->getAPIntValue() == (VT.getScalarSizeInBits() - 1))
3666 return DAG.getNode(ISD::ABS, SDLoc(N), VT, S0);
3667 }
3668 }
3669
3670 // If the relocation model supports it, consider symbol offsets.
3671 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N0))
3672 if (!LegalOperations && TLI.isOffsetFoldingLegal(GA)) {
3673 // fold (sub Sym, c) -> Sym-c
3674 if (N1C && GA->getOpcode() == ISD::GlobalAddress)
3675 return DAG.getGlobalAddress(GA->getGlobal(), SDLoc(N1C), VT,
3676 GA->getOffset() -
3677 (uint64_t)N1C->getSExtValue());
3678 // fold (sub Sym+c1, Sym+c2) -> c1-c2
3679 if (GlobalAddressSDNode *GB = dyn_cast<GlobalAddressSDNode>(N1))
3680 if (GA->getGlobal() == GB->getGlobal())
3681 return DAG.getConstant((uint64_t)GA->getOffset() - GB->getOffset(),
3682 DL, VT);
3683 }
3684
3685 // sub X, (sextinreg Y i1) -> add X, (and Y 1)
3686 if (N1.getOpcode() == ISD::SIGN_EXTEND_INREG) {
3687 VTSDNode *TN = cast<VTSDNode>(N1.getOperand(1));
3688 if (TN->getVT() == MVT::i1) {
3689 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0),
3690 DAG.getConstant(1, DL, VT));
3691 return DAG.getNode(ISD::ADD, DL, VT, N0, ZExt);
3692 }
3693 }
3694
3695 // canonicalize (sub X, (vscale * C)) to (add X, (vscale * -C))
3696 if (N1.getOpcode() == ISD::VSCALE && N1.hasOneUse()) {
3697 const APInt &IntVal = N1.getConstantOperandAPInt(0);
3698 return DAG.getNode(ISD::ADD, DL, VT, N0, DAG.getVScale(DL, VT, -IntVal));
3699 }
3700
3701 // canonicalize (sub X, step_vector(C)) to (add X, step_vector(-C))
3702 if (N1.getOpcode() == ISD::STEP_VECTOR && N1.hasOneUse()) {
3703 APInt NewStep = -N1.getConstantOperandAPInt(0);
3704 return DAG.getNode(ISD::ADD, DL, VT, N0,
3705 DAG.getStepVector(DL, VT, NewStep));
3706 }
3707
3708 // Prefer an add for more folding potential and possibly better codegen:
3709 // sub N0, (lshr N10, width-1) --> add N0, (ashr N10, width-1)
3710 if (!LegalOperations && N1.getOpcode() == ISD::SRL && N1.hasOneUse()) {
3711 SDValue ShAmt = N1.getOperand(1);
3712 ConstantSDNode *ShAmtC = isConstOrConstSplat(ShAmt);
3713 if (ShAmtC &&
3714 ShAmtC->getAPIntValue() == (N1.getScalarValueSizeInBits() - 1)) {
3715 SDValue SRA = DAG.getNode(ISD::SRA, DL, VT, N1.getOperand(0), ShAmt);
3716 return DAG.getNode(ISD::ADD, DL, VT, N0, SRA);
3717 }
3718 }
3719
3720 // As with the previous fold, prefer add for more folding potential.
3721 // Subtracting SMIN/0 is the same as adding SMIN/0:
3722 // N0 - (X << BW-1) --> N0 + (X << BW-1)
3723 if (N1.getOpcode() == ISD::SHL) {
3724 ConstantSDNode *ShlC = isConstOrConstSplat(N1.getOperand(1));
3725 if (ShlC && ShlC->getAPIntValue() == VT.getScalarSizeInBits() - 1)
3726 return DAG.getNode(ISD::ADD, DL, VT, N1, N0);
3727 }
3728
3729 // (sub (subcarry X, 0, Carry), Y) -> (subcarry X, Y, Carry)
3730 if (N0.getOpcode() == ISD::SUBCARRY && isNullConstant(N0.getOperand(1)) &&
3731 N0.getResNo() == 0 && N0.hasOneUse())
3732 return DAG.getNode(ISD::SUBCARRY, DL, N0->getVTList(),
3733 N0.getOperand(0), N1, N0.getOperand(2));
3734
3735 if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT)) {
3736 // (sub Carry, X) -> (addcarry (sub 0, X), 0, Carry)
3737 if (SDValue Carry = getAsCarry(TLI, N0)) {
3738 SDValue X = N1;
3739 SDValue Zero = DAG.getConstant(0, DL, VT);
3740 SDValue NegX = DAG.getNode(ISD::SUB, DL, VT, Zero, X);
3741 return DAG.getNode(ISD::ADDCARRY, DL,
3742 DAG.getVTList(VT, Carry.getValueType()), NegX, Zero,
3743 Carry);
3744 }
3745 }
3746
3747 // If there's no chance of borrowing from adjacent bits, then sub is xor:
3748 // sub C0, X --> xor X, C0
3749 if (ConstantSDNode *C0 = isConstOrConstSplat(N0)) {
3750 if (!C0->isOpaque()) {
3751 const APInt &C0Val = C0->getAPIntValue();
3752 const APInt &MaybeOnes = ~DAG.computeKnownBits(N1).Zero;
3753 if ((C0Val - MaybeOnes) == (C0Val ^ MaybeOnes))
3754 return DAG.getNode(ISD::XOR, DL, VT, N1, N0);
3755 }
3756 }
3757
3758 return SDValue();
3759}
3760
3761SDValue DAGCombiner::visitSUBSAT(SDNode *N) {
3762 SDValue N0 = N->getOperand(0);
3763 SDValue N1 = N->getOperand(1);
3764 EVT VT = N0.getValueType();
3765 SDLoc DL(N);
3766
3767 // fold (sub_sat x, undef) -> 0
3768 if (N0.isUndef() || N1.isUndef())
3769 return DAG.getConstant(0, DL, VT);
3770
3771 // fold (sub_sat x, x) -> 0
3772 if (N0 == N1)
3773 return DAG.getConstant(0, DL, VT);
3774
3775 // fold (sub_sat c1, c2) -> c3
3776 if (SDValue C = DAG.FoldConstantArithmetic(N->getOpcode(), DL, VT, {N0, N1}))
3777 return C;
3778
3779 // fold vector ops
3780 if (VT.isVector()) {
3781 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
3782 return FoldedVOp;
3783
3784 // fold (sub_sat x, 0) -> x, vector edition
3785 if (ISD::isConstantSplatVectorAllZeros(N1.getNode()))
3786 return N0;
3787 }
3788
3789 // fold (sub_sat x, 0) -> x
3790 if (isNullConstant(N1))
3791 return N0;
3792
3793 return SDValue();
3794}
3795
3796SDValue DAGCombiner::visitSUBC(SDNode *N) {
3797 SDValue N0 = N->getOperand(0);
3798 SDValue N1 = N->getOperand(1);
3799 EVT VT = N0.getValueType();
3800 SDLoc DL(N);
3801
3802 // If the flag result is dead, turn this into an SUB.
3803 if (!N->hasAnyUseOfValue(1))
3804 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1),
3805 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
3806
3807 // fold (subc x, x) -> 0 + no borrow
3808 if (N0 == N1)
3809 return CombineTo(N, DAG.getConstant(0, DL, VT),
3810 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
3811
3812 // fold (subc x, 0) -> x + no borrow
3813 if (isNullConstant(N1))
3814 return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
3815
3816 // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1) + no borrow
3817 if (isAllOnesConstant(N0))
3818 return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0),
3819 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
3820
3821 return SDValue();
3822}
3823
3824SDValue DAGCombiner::visitSUBO(SDNode *N) {
3825 SDValue N0 = N->getOperand(0);
3826 SDValue N1 = N->getOperand(1);
3827 EVT VT = N0.getValueType();
3828 bool IsSigned = (ISD::SSUBO == N->getOpcode());
3829
3830 EVT CarryVT = N->getValueType(1);
3831 SDLoc DL(N);
3832
3833 // If the flag result is dead, turn this into an SUB.
3834 if (!N->hasAnyUseOfValue(1))
3835 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1),
3836 DAG.getUNDEF(CarryVT));
3837
3838 // fold (subo x, x) -> 0 + no borrow
3839 if (N0 == N1)
3840 return CombineTo(N, DAG.getConstant(0, DL, VT),
3841 DAG.getConstant(0, DL, CarryVT));
3842
3843 ConstantSDNode *N1C = getAsNonOpaqueConstant(N1);
3844
3845 // fold (subox, c) -> (addo x, -c)
3846 if (IsSigned && N1C && !N1C->getAPIntValue().isMinSignedValue()) {
3847 return DAG.getNode(ISD::SADDO, DL, N->getVTList(), N0,
3848 DAG.getConstant(-N1C->getAPIntValue(), DL, VT));
3849 }
3850
3851 // fold (subo x, 0) -> x + no borrow
3852 if (isNullOrNullSplat(N1))
3853 return CombineTo(N, N0, DAG.getConstant(0, DL, CarryVT));
3854
3855 // Canonicalize (usubo -1, x) -> ~x, i.e. (xor x, -1) + no borrow
3856 if (!IsSigned && isAllOnesOrAllOnesSplat(N0))
3857 return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0),
3858 DAG.getConstant(0, DL, CarryVT));
3859
3860 return SDValue();
3861}
3862
3863SDValue DAGCombiner::visitSUBE(SDNode *N) {
3864 SDValue N0 = N->getOperand(0);
3865 SDValue N1 = N->getOperand(1);
3866 SDValue CarryIn = N->getOperand(2);
3867
3868 // fold (sube x, y, false) -> (subc x, y)
3869 if (CarryIn.getOpcode() == ISD::CARRY_FALSE)
3870 return DAG.getNode(ISD::SUBC, SDLoc(N), N->getVTList(), N0, N1);
3871
3872 return SDValue();
3873}
3874
3875SDValue DAGCombiner::visitSUBCARRY(SDNode *N) {
3876 SDValue N0 = N->getOperand(0);
3877 SDValue N1 = N->getOperand(1);
3878 SDValue CarryIn = N->getOperand(2);
3879
3880 // fold (subcarry x, y, false) -> (usubo x, y)
3881 if (isNullConstant(CarryIn)) {
3882 if (!LegalOperations ||
3883 TLI.isOperationLegalOrCustom(ISD::USUBO, N->getValueType(0)))
3884 return DAG.getNode(ISD::USUBO, SDLoc(N), N->getVTList(), N0, N1);
3885 }
3886
3887 return SDValue();
3888}
3889
3890SDValue DAGCombiner::visitSSUBO_CARRY(SDNode *N) {
3891 SDValue N0 = N->getOperand(0);
3892 SDValue N1 = N->getOperand(1);
3893 SDValue CarryIn = N->getOperand(2);
3894
3895 // fold (ssubo_carry x, y, false) -> (ssubo x, y)
3896 if (isNullConstant(CarryIn)) {
3897 if (!LegalOperations ||
3898 TLI.isOperationLegalOrCustom(ISD::SSUBO, N->getValueType(0)))
3899 return DAG.getNode(ISD::SSUBO, SDLoc(N), N->getVTList(), N0, N1);
3900 }
3901
3902 return SDValue();
3903}
3904
3905// Notice that "mulfix" can be any of SMULFIX, SMULFIXSAT, UMULFIX and
3906// UMULFIXSAT here.
3907SDValue DAGCombiner::visitMULFIX(SDNode *N) {
3908 SDValue N0 = N->getOperand(0);
3909 SDValue N1 = N->getOperand(1);
3910 SDValue Scale = N->getOperand(2);
3911 EVT VT = N0.getValueType();
3912
3913 // fold (mulfix x, undef, scale) -> 0
3914 if (N0.isUndef() || N1.isUndef())
3915 return DAG.getConstant(0, SDLoc(N), VT);
3916
3917 // Canonicalize constant to RHS (vector doesn't have to splat)
3918 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
3919 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
3920 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N1, N0, Scale);
3921
3922 // fold (mulfix x, 0, scale) -> 0
3923 if (isNullConstant(N1))
3924 return DAG.getConstant(0, SDLoc(N), VT);
3925
3926 return SDValue();
3927}
3928
3929SDValue DAGCombiner::visitMUL(SDNode *N) {
3930 SDValue N0 = N->getOperand(0);
3931 SDValue N1 = N->getOperand(1);
3932 EVT VT = N0.getValueType();
3933 SDLoc DL(N);
3934
3935 // fold (mul x, undef) -> 0
3936 if (N0.isUndef() || N1.isUndef())
3937 return DAG.getConstant(0, DL, VT);
3938
3939 // fold (mul c1, c2) -> c1*c2
3940 if (SDValue C = DAG.FoldConstantArithmetic(ISD::MUL, DL, VT, {N0, N1}))
3941 return C;
3942
3943 // canonicalize constant to RHS (vector doesn't have to splat)
3944 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
3945 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
3946 return DAG.getNode(ISD::MUL, DL, VT, N1, N0);
3947
3948 bool N1IsConst = false;
3949 bool N1IsOpaqueConst = false;
3950 APInt ConstValue1;
3951
3952 // fold vector ops
3953 if (VT.isVector()) {
3954 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
3955 return FoldedVOp;
3956
3957 N1IsConst = ISD::isConstantSplatVector(N1.getNode(), ConstValue1);
3958 assert((!N1IsConst ||(static_cast <bool> ((!N1IsConst || ConstValue1.getBitWidth
() == VT.getScalarSizeInBits()) && "Splat APInt should be element width"
) ? void (0) : __assert_fail ("(!N1IsConst || ConstValue1.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 3960, __extension__
__PRETTY_FUNCTION__))
3959 ConstValue1.getBitWidth() == VT.getScalarSizeInBits()) &&(static_cast <bool> ((!N1IsConst || ConstValue1.getBitWidth
() == VT.getScalarSizeInBits()) && "Splat APInt should be element width"
) ? void (0) : __assert_fail ("(!N1IsConst || ConstValue1.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 3960, __extension__
__PRETTY_FUNCTION__))
3960 "Splat APInt should be element width")(static_cast <bool> ((!N1IsConst || ConstValue1.getBitWidth
() == VT.getScalarSizeInBits()) && "Splat APInt should be element width"
) ? void (0) : __assert_fail ("(!N1IsConst || ConstValue1.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 3960, __extension__
__PRETTY_FUNCTION__))
;
3961 } else {
3962 N1IsConst = isa<ConstantSDNode>(N1);
3963 if (N1IsConst) {
3964 ConstValue1 = cast<ConstantSDNode>(N1)->getAPIntValue();
3965 N1IsOpaqueConst = cast<ConstantSDNode>(N1)->isOpaque();
3966 }
3967 }
3968
3969 // fold (mul x, 0) -> 0
3970 if (N1IsConst && ConstValue1.isZero())
3971 return N1;
3972
3973 // fold (mul x, 1) -> x
3974 if (N1IsConst && ConstValue1.isOne())
3975 return N0;
3976
3977 if (SDValue NewSel = foldBinOpIntoSelect(N))
3978 return NewSel;
3979
3980 // fold (mul x, -1) -> 0-x
3981 if (N1IsConst && ConstValue1.isAllOnes())
3982 return DAG.getNode(ISD::SUB, DL, VT,
3983 DAG.getConstant(0, DL, VT), N0);
3984
3985 // fold (mul x, (1 << c)) -> x << c
3986 if (isConstantOrConstantVector(N1, /*NoOpaques*/ true) &&
3987 DAG.isKnownToBeAPowerOfTwo(N1) &&
3988 (!VT.isVector() || Level <= AfterLegalizeVectorOps)) {
3989 SDValue LogBase2 = BuildLogBase2(N1, DL);
3990 EVT ShiftVT = getShiftAmountTy(N0.getValueType());
3991 SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ShiftVT);
3992 return DAG.getNode(ISD::SHL, DL, VT, N0, Trunc);
3993 }
3994
3995 // fold (mul x, -(1 << c)) -> -(x << c) or (-x) << c
3996 if (N1IsConst && !N1IsOpaqueConst && ConstValue1.isNegatedPowerOf2()) {
3997 unsigned Log2Val = (-ConstValue1).logBase2();
3998 // FIXME: If the input is something that is easily negated (e.g. a
3999 // single-use add), we should put the negate there.
4000 return DAG.getNode(ISD::SUB, DL, VT,
4001 DAG.getConstant(0, DL, VT),
4002 DAG.getNode(ISD::SHL, DL, VT, N0,
4003 DAG.getConstant(Log2Val, DL,
4004 getShiftAmountTy(N0.getValueType()))));
4005 }
4006
4007 // Try to transform:
4008 // (1) multiply-by-(power-of-2 +/- 1) into shift and add/sub.
4009 // mul x, (2^N + 1) --> add (shl x, N), x
4010 // mul x, (2^N - 1) --> sub (shl x, N), x
4011 // Examples: x * 33 --> (x << 5) + x
4012 // x * 15 --> (x << 4) - x
4013 // x * -33 --> -((x << 5) + x)
4014 // x * -15 --> -((x << 4) - x) ; this reduces --> x - (x << 4)
4015 // (2) multiply-by-(power-of-2 +/- power-of-2) into shifts and add/sub.
4016 // mul x, (2^N + 2^M) --> (add (shl x, N), (shl x, M))
4017 // mul x, (2^N - 2^M) --> (sub (shl x, N), (shl x, M))
4018 // Examples: x * 0x8800 --> (x << 15) + (x << 11)
4019 // x * 0xf800 --> (x << 16) - (x << 11)
4020 // x * -0x8800 --> -((x << 15) + (x << 11))
4021 // x * -0xf800 --> -((x << 16) - (x << 11)) ; (x << 11) - (x << 16)
4022 if (N1IsConst && TLI.decomposeMulByConstant(*DAG.getContext(), VT, N1)) {
4023 // TODO: We could handle more general decomposition of any constant by
4024 // having the target set a limit on number of ops and making a
4025 // callback to determine that sequence (similar to sqrt expansion).
4026 unsigned MathOp = ISD::DELETED_NODE;
4027 APInt MulC = ConstValue1.abs();
4028 // The constant `2` should be treated as (2^0 + 1).
4029 unsigned TZeros = MulC == 2 ? 0 : MulC.countTrailingZeros();
4030 MulC.lshrInPlace(TZeros);
4031 if ((MulC - 1).isPowerOf2())
4032 MathOp = ISD::ADD;
4033 else if ((MulC + 1).isPowerOf2())
4034 MathOp = ISD::SUB;
4035
4036 if (MathOp != ISD::DELETED_NODE) {
4037 unsigned ShAmt =
4038 MathOp == ISD::ADD ? (MulC - 1).logBase2() : (MulC + 1).logBase2();
4039 ShAmt += TZeros;
4040 assert(ShAmt < VT.getScalarSizeInBits() &&(static_cast <bool> (ShAmt < VT.getScalarSizeInBits(
) && "multiply-by-constant generated out of bounds shift"
) ? void (0) : __assert_fail ("ShAmt < VT.getScalarSizeInBits() && \"multiply-by-constant generated out of bounds shift\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 4041, __extension__
__PRETTY_FUNCTION__))
4041 "multiply-by-constant generated out of bounds shift")(static_cast <bool> (ShAmt < VT.getScalarSizeInBits(
) && "multiply-by-constant generated out of bounds shift"
) ? void (0) : __assert_fail ("ShAmt < VT.getScalarSizeInBits() && \"multiply-by-constant generated out of bounds shift\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 4041, __extension__
__PRETTY_FUNCTION__))
;
4042 SDValue Shl =
4043 DAG.getNode(ISD::SHL, DL, VT, N0, DAG.getConstant(ShAmt, DL, VT));
4044 SDValue R =
4045 TZeros ? DAG.getNode(MathOp, DL, VT, Shl,
4046 DAG.getNode(ISD::SHL, DL, VT, N0,
4047 DAG.getConstant(TZeros, DL, VT)))
4048 : DAG.getNode(MathOp, DL, VT, Shl, N0);
4049 if (ConstValue1.isNegative())
4050 R = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), R);
4051 return R;
4052 }
4053 }
4054
4055 // (mul (shl X, c1), c2) -> (mul X, c2 << c1)
4056 if (N0.getOpcode() == ISD::SHL) {
4057 SDValue N01 = N0.getOperand(1);
4058 if (SDValue C3 = DAG.FoldConstantArithmetic(ISD::SHL, DL, VT, {N1, N01}))
4059 return DAG.getNode(ISD::MUL, DL, VT, N0.getOperand(0), C3);
4060 }
4061
4062 // Change (mul (shl X, C), Y) -> (shl (mul X, Y), C) when the shift has one
4063 // use.
4064 {
4065 SDValue Sh, Y;
4066
4067 // Check for both (mul (shl X, C), Y) and (mul Y, (shl X, C)).
4068 if (N0.getOpcode() == ISD::SHL &&
4069 isConstantOrConstantVector(N0.getOperand(1)) && N0->hasOneUse()) {
4070 Sh = N0; Y = N1;
4071 } else if (N1.getOpcode() == ISD::SHL &&
4072 isConstantOrConstantVector(N1.getOperand(1)) &&
4073 N1->hasOneUse()) {
4074 Sh = N1; Y = N0;
4075 }
4076
4077 if (Sh.getNode()) {
4078 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, Sh.getOperand(0), Y);
4079 return DAG.getNode(ISD::SHL, DL, VT, Mul, Sh.getOperand(1));
4080 }
4081 }
4082
4083 // fold (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2)
4084 if (DAG.isConstantIntBuildVectorOrConstantInt(N1) &&
4085 N0.getOpcode() == ISD::ADD &&
4086 DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1)) &&
4087 isMulAddWithConstProfitable(N, N0, N1))
4088 return DAG.getNode(
4089 ISD::ADD, DL, VT,
4090 DAG.getNode(ISD::MUL, SDLoc(N0), VT, N0.getOperand(0), N1),
4091 DAG.getNode(ISD::MUL, SDLoc(N1), VT, N0.getOperand(1), N1));
4092
4093 // Fold (mul (vscale * C0), C1) to (vscale * (C0 * C1)).
4094 ConstantSDNode *NC1 = isConstOrConstSplat(N1);
4095 if (N0.getOpcode() == ISD::VSCALE && NC1) {
4096 const APInt &C0 = N0.getConstantOperandAPInt(0);
4097 const APInt &C1 = NC1->getAPIntValue();
4098 return DAG.getVScale(DL, VT, C0 * C1);
4099 }
4100
4101 // Fold (mul step_vector(C0), C1) to (step_vector(C0 * C1)).
4102 APInt MulVal;
4103 if (N0.getOpcode() == ISD::STEP_VECTOR &&
4104 ISD::isConstantSplatVector(N1.getNode(), MulVal)) {
4105 const APInt &C0 = N0.getConstantOperandAPInt(0);
4106 APInt NewStep = C0 * MulVal;
4107 return DAG.getStepVector(DL, VT, NewStep);
4108 }
4109
4110 // Fold ((mul x, 0/undef) -> 0,
4111 // (mul x, 1) -> x) -> x)
4112 // -> and(x, mask)
4113 // We can replace vectors with '0' and '1' factors with a clearing mask.
4114 if (VT.isFixedLengthVector()) {
4115 unsigned NumElts = VT.getVectorNumElements();
4116 SmallBitVector ClearMask;
4117 ClearMask.reserve(NumElts);
4118 auto IsClearMask = [&ClearMask](ConstantSDNode *V) {
4119 if (!V || V->isZero()) {
4120 ClearMask.push_back(true);
4121 return true;
4122 }
4123 ClearMask.push_back(false);
4124 return V->isOne();
4125 };
4126 if ((!LegalOperations || TLI.isOperationLegalOrCustom(ISD::AND, VT)) &&
4127 ISD::matchUnaryPredicate(N1, IsClearMask, /*AllowUndefs*/ true)) {
4128 assert(N1.getOpcode() == ISD::BUILD_VECTOR && "Unknown constant vector")(static_cast <bool> (N1.getOpcode() == ISD::BUILD_VECTOR
&& "Unknown constant vector") ? void (0) : __assert_fail
("N1.getOpcode() == ISD::BUILD_VECTOR && \"Unknown constant vector\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 4128, __extension__
__PRETTY_FUNCTION__))
;
4129 EVT LegalSVT = N1.getOperand(0).getValueType();
4130 SDValue Zero = DAG.getConstant(0, DL, LegalSVT);
4131 SDValue AllOnes = DAG.getAllOnesConstant(DL, LegalSVT);
4132 SmallVector<SDValue, 16> Mask(NumElts, AllOnes);
4133 for (unsigned I = 0; I != NumElts; ++I)
4134 if (ClearMask[I])
4135 Mask[I] = Zero;
4136 return DAG.getNode(ISD::AND, DL, VT, N0, DAG.getBuildVector(VT, DL, Mask));
4137 }
4138 }
4139
4140 // reassociate mul
4141 if (SDValue RMUL = reassociateOps(ISD::MUL, DL, N0, N1, N->getFlags()))
4142 return RMUL;
4143
4144 // Simplify the operands using demanded-bits information.
4145 if (SimplifyDemandedBits(SDValue(N, 0)))
4146 return SDValue(N, 0);
4147
4148 return SDValue();
4149}
4150
4151/// Return true if divmod libcall is available.
4152static bool isDivRemLibcallAvailable(SDNode *Node, bool isSigned,
4153 const TargetLowering &TLI) {
4154 RTLIB::Libcall LC;
4155 EVT NodeType = Node->getValueType(0);
4156 if (!NodeType.isSimple())
4157 return false;
4158 switch (NodeType.getSimpleVT().SimpleTy) {
4159 default: return false; // No libcall for vector types.
4160 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
4161 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
4162 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
4163 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
4164 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
4165 }
4166
4167 return TLI.getLibcallName(LC) != nullptr;
4168}
4169
4170/// Issue divrem if both quotient and remainder are needed.
4171SDValue DAGCombiner::useDivRem(SDNode *Node) {
4172 if (Node->use_empty())
4173 return SDValue(); // This is a dead node, leave it alone.
4174
4175 unsigned Opcode = Node->getOpcode();
4176 bool isSigned = (Opcode == ISD::SDIV) || (Opcode == ISD::SREM);
4177 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
4178
4179 // DivMod lib calls can still work on non-legal types if using lib-calls.
4180 EVT VT = Node->getValueType(0);
4181 if (VT.isVector() || !VT.isInteger())
4182 return SDValue();
4183
4184 if (!TLI.isTypeLegal(VT) && !TLI.isOperationCustom(DivRemOpc, VT))
4185 return SDValue();
4186
4187 // If DIVREM is going to get expanded into a libcall,
4188 // but there is no libcall available, then don't combine.
4189 if (!TLI.isOperationLegalOrCustom(DivRemOpc, VT) &&
4190 !isDivRemLibcallAvailable(Node, isSigned, TLI))
4191 return SDValue();
4192
4193 // If div is legal, it's better to do the normal expansion
4194 unsigned OtherOpcode = 0;
4195 if ((Opcode == ISD::SDIV) || (Opcode == ISD::UDIV)) {
4196 OtherOpcode = isSigned ? ISD::SREM : ISD::UREM;
4197 if (TLI.isOperationLegalOrCustom(Opcode, VT))
4198 return SDValue();
4199 } else {
4200 OtherOpcode = isSigned ? ISD::SDIV : ISD::UDIV;
4201 if (TLI.isOperationLegalOrCustom(OtherOpcode, VT))
4202 return SDValue();
4203 }
4204
4205 SDValue Op0 = Node->getOperand(0);
4206 SDValue Op1 = Node->getOperand(1);
4207 SDValue combined;
4208 for (SDNode *User : Op0->uses()) {
4209 if (User == Node || User->getOpcode() == ISD::DELETED_NODE ||
4210 User->use_empty())
4211 continue;
4212 // Convert the other matching node(s), too;
4213 // otherwise, the DIVREM may get target-legalized into something
4214 // target-specific that we won't be able to recognize.
4215 unsigned UserOpc = User->getOpcode();
4216 if ((UserOpc == Opcode || UserOpc == OtherOpcode || UserOpc == DivRemOpc) &&
4217 User->getOperand(0) == Op0 &&
4218 User->getOperand(1) == Op1) {
4219 if (!combined) {
4220 if (UserOpc == OtherOpcode) {
4221 SDVTList VTs = DAG.getVTList(VT, VT);
4222 combined = DAG.getNode(DivRemOpc, SDLoc(Node), VTs, Op0, Op1);
4223 } else if (UserOpc == DivRemOpc) {
4224 combined = SDValue(User, 0);
4225 } else {
4226 assert(UserOpc == Opcode)(static_cast <bool> (UserOpc == Opcode) ? void (0) : __assert_fail
("UserOpc == Opcode", "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4226, __extension__ __PRETTY_FUNCTION__))
;
4227 continue;
4228 }
4229 }
4230 if (UserOpc == ISD::SDIV || UserOpc == ISD::UDIV)
4231 CombineTo(User, combined);
4232 else if (UserOpc == ISD::SREM || UserOpc == ISD::UREM)
4233 CombineTo(User, combined.getValue(1));
4234 }
4235 }
4236 return combined;
4237}
4238
4239static SDValue simplifyDivRem(SDNode *N, SelectionDAG &DAG) {
4240 SDValue N0 = N->getOperand(0);
4241 SDValue N1 = N->getOperand(1);
4242 EVT VT = N->getValueType(0);
4243 SDLoc DL(N);
4244
4245 unsigned Opc = N->getOpcode();
4246 bool IsDiv = (ISD::SDIV == Opc) || (ISD::UDIV == Opc);
4247 ConstantSDNode *N1C = isConstOrConstSplat(N1);
4248
4249 // X / undef -> undef
4250 // X % undef -> undef
4251 // X / 0 -> undef
4252 // X % 0 -> undef
4253 // NOTE: This includes vectors where any divisor element is zero/undef.
4254 if (DAG.isUndef(Opc, {N0, N1}))
4255 return DAG.getUNDEF(VT);
4256
4257 // undef / X -> 0
4258 // undef % X -> 0
4259 if (N0.isUndef())
4260 return DAG.getConstant(0, DL, VT);
4261
4262 // 0 / X -> 0
4263 // 0 % X -> 0
4264 ConstantSDNode *N0C = isConstOrConstSplat(N0);
4265 if (N0C && N0C->isZero())
4266 return N0;
4267
4268 // X / X -> 1
4269 // X % X -> 0
4270 if (N0 == N1)
4271 return DAG.getConstant(IsDiv ? 1 : 0, DL, VT);
4272
4273 // X / 1 -> X
4274 // X % 1 -> 0
4275 // If this is a boolean op (single-bit element type), we can't have
4276 // division-by-zero or remainder-by-zero, so assume the divisor is 1.
4277 // TODO: Similarly, if we're zero-extending a boolean divisor, then assume
4278 // it's a 1.
4279 if ((N1C && N1C->isOne()) || (VT.getScalarType() == MVT::i1))
4280 return IsDiv ? N0 : DAG.getConstant(0, DL, VT);
4281
4282 return SDValue();
4283}
4284
4285SDValue DAGCombiner::visitSDIV(SDNode *N) {
4286 SDValue N0 = N->getOperand(0);
4287 SDValue N1 = N->getOperand(1);
4288 EVT VT = N->getValueType(0);
4289 EVT CCVT = getSetCCResultType(VT);
4290 SDLoc DL(N);
4291
4292 // fold (sdiv c1, c2) -> c1/c2
4293 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SDIV, DL, VT, {N0, N1}))
4294 return C;
4295
4296 // fold vector ops
4297 if (VT.isVector())
4298 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
4299 return FoldedVOp;
4300
4301 // fold (sdiv X, -1) -> 0-X
4302 ConstantSDNode *N1C = isConstOrConstSplat(N1);
4303 if (N1C && N1C->isAllOnes())
4304 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), N0);
4305
4306 // fold (sdiv X, MIN_SIGNED) -> select(X == MIN_SIGNED, 1, 0)
4307 if (N1C && N1C->getAPIntValue().isMinSignedValue())
4308 return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ),
4309 DAG.getConstant(1, DL, VT),
4310 DAG.getConstant(0, DL, VT));
4311
4312 if (SDValue V = simplifyDivRem(N, DAG))
4313 return V;
4314
4315 if (SDValue NewSel = foldBinOpIntoSelect(N))
4316 return NewSel;
4317
4318 // If we know the sign bits of both operands are zero, strength reduce to a
4319 // udiv instead. Handles (X&15) /s 4 -> X&15 >> 2
4320 if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
4321 return DAG.getNode(ISD::UDIV, DL, N1.getValueType(), N0, N1);
4322
4323 if (SDValue V = visitSDIVLike(N0, N1, N)) {
4324 // If the corresponding remainder node exists, update its users with
4325 // (Dividend - (Quotient * Divisor).
4326 if (SDNode *RemNode = DAG.getNodeIfExists(ISD::SREM, N->getVTList(),
4327 { N0, N1 })) {
4328 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, V, N1);
4329 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
4330 AddToWorklist(Mul.getNode());
4331 AddToWorklist(Sub.getNode());
4332 CombineTo(RemNode, Sub);
4333 }
4334 return V;
4335 }
4336
4337 // sdiv, srem -> sdivrem
4338 // If the divisor is constant, then return DIVREM only if isIntDivCheap() is
4339 // true. Otherwise, we break the simplification logic in visitREM().
4340 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
4341 if (!N1C || TLI.isIntDivCheap(N->getValueType(0), Attr))
4342 if (SDValue DivRem = useDivRem(N))
4343 return DivRem;
4344
4345 return SDValue();
4346}
4347
4348static bool isDivisorPowerOfTwo(SDValue Divisor) {
4349 // Helper for determining whether a value is a power-2 constant scalar or a
4350 // vector of such elements.
4351 auto IsPowerOfTwo = [](ConstantSDNode *C) {
4352 if (C->isZero() || C->isOpaque())
4353 return false;
4354 if (C->getAPIntValue().isPowerOf2())
4355 return true;
4356 if (C->getAPIntValue().isNegatedPowerOf2())
4357 return true;
4358 return false;
4359 };
4360
4361 return ISD::matchUnaryPredicate(Divisor, IsPowerOfTwo);
4362}
4363
4364SDValue DAGCombiner::visitSDIVLike(SDValue N0, SDValue N1, SDNode *N) {
4365 SDLoc DL(N);
4366 EVT VT = N->getValueType(0);
4367 EVT CCVT = getSetCCResultType(VT);
4368 unsigned BitWidth = VT.getScalarSizeInBits();
4369
4370 // fold (sdiv X, pow2) -> simple ops after legalize
4371 // FIXME: We check for the exact bit here because the generic lowering gives
4372 // better results in that case. The target-specific lowering should learn how
4373 // to handle exact sdivs efficiently.
4374 if (!N->getFlags().hasExact() && isDivisorPowerOfTwo(N1)) {
4375 // Target-specific implementation of sdiv x, pow2.
4376 if (SDValue Res = BuildSDIVPow2(N))
4377 return Res;
4378
4379 // Create constants that are functions of the shift amount value.
4380 EVT ShiftAmtTy = getShiftAmountTy(N0.getValueType());
4381 SDValue Bits = DAG.getConstant(BitWidth, DL, ShiftAmtTy);
4382 SDValue C1 = DAG.getNode(ISD::CTTZ, DL, VT, N1);
4383 C1 = DAG.getZExtOrTrunc(C1, DL, ShiftAmtTy);
4384 SDValue Inexact = DAG.getNode(ISD::SUB, DL, ShiftAmtTy, Bits, C1);
4385 if (!isConstantOrConstantVector(Inexact))
4386 return SDValue();
4387
4388 // Splat the sign bit into the register
4389 SDValue Sign = DAG.getNode(ISD::SRA, DL, VT, N0,
4390 DAG.getConstant(BitWidth - 1, DL, ShiftAmtTy));
4391 AddToWorklist(Sign.getNode());
4392
4393 // Add (N0 < 0) ? abs2 - 1 : 0;
4394 SDValue Srl = DAG.getNode(ISD::SRL, DL, VT, Sign, Inexact);
4395 AddToWorklist(Srl.getNode());
4396 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Srl);
4397 AddToWorklist(Add.getNode());
4398 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Add, C1);
4399 AddToWorklist(Sra.getNode());
4400
4401 // Special case: (sdiv X, 1) -> X
4402 // Special Case: (sdiv X, -1) -> 0-X
4403 SDValue One = DAG.getConstant(1, DL, VT);
4404 SDValue AllOnes = DAG.getAllOnesConstant(DL, VT);
4405 SDValue IsOne = DAG.getSetCC(DL, CCVT, N1, One, ISD::SETEQ);
4406 SDValue IsAllOnes = DAG.getSetCC(DL, CCVT, N1, AllOnes, ISD::SETEQ);
4407 SDValue IsOneOrAllOnes = DAG.getNode(ISD::OR, DL, CCVT, IsOne, IsAllOnes);
4408 Sra = DAG.getSelect(DL, VT, IsOneOrAllOnes, N0, Sra);
4409
4410 // If dividing by a positive value, we're done. Otherwise, the result must
4411 // be negated.
4412 SDValue Zero = DAG.getConstant(0, DL, VT);
4413 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, Zero, Sra);
4414
4415 // FIXME: Use SELECT_CC once we improve SELECT_CC constant-folding.
4416 SDValue IsNeg = DAG.getSetCC(DL, CCVT, N1, Zero, ISD::SETLT);
4417 SDValue Res = DAG.getSelect(DL, VT, IsNeg, Sub, Sra);
4418 return Res;
4419 }
4420
4421 // If integer divide is expensive and we satisfy the requirements, emit an
4422 // alternate sequence. Targets may check function attributes for size/speed
4423 // trade-offs.
4424 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
4425 if (isConstantOrConstantVector(N1) &&
4426 !TLI.isIntDivCheap(N->getValueType(0), Attr))
4427 if (SDValue Op = BuildSDIV(N))
4428 return Op;
4429
4430 return SDValue();
4431}
4432
4433SDValue DAGCombiner::visitUDIV(SDNode *N) {
4434 SDValue N0 = N->getOperand(0);
4435 SDValue N1 = N->getOperand(1);
4436 EVT VT = N->getValueType(0);
4437 EVT CCVT = getSetCCResultType(VT);
4438 SDLoc DL(N);
4439
4440 // fold (udiv c1, c2) -> c1/c2
4441 if (SDValue C = DAG.FoldConstantArithmetic(ISD::UDIV, DL, VT, {N0, N1}))
4442 return C;
4443
4444 // fold vector ops
4445 if (VT.isVector())
4446 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
4447 return FoldedVOp;
4448
4449 // fold (udiv X, -1) -> select(X == -1, 1, 0)
4450 ConstantSDNode *N1C = isConstOrConstSplat(N1);
4451 if (N1C && N1C->isAllOnes())
4452 return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ),
4453 DAG.getConstant(1, DL, VT),
4454 DAG.getConstant(0, DL, VT));
4455
4456 if (SDValue V = simplifyDivRem(N, DAG))
4457 return V;
4458
4459 if (SDValue NewSel = foldBinOpIntoSelect(N))
4460 return NewSel;
4461
4462 if (SDValue V = visitUDIVLike(N0, N1, N)) {
4463 // If the corresponding remainder node exists, update its users with
4464 // (Dividend - (Quotient * Divisor).
4465 if (SDNode *RemNode = DAG.getNodeIfExists(ISD::UREM, N->getVTList(),
4466 { N0, N1 })) {
4467 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, V, N1);
4468 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
4469 AddToWorklist(Mul.getNode());
4470 AddToWorklist(Sub.getNode());
4471 CombineTo(RemNode, Sub);
4472 }
4473 return V;
4474 }
4475
4476 // sdiv, srem -> sdivrem
4477 // If the divisor is constant, then return DIVREM only if isIntDivCheap() is
4478 // true. Otherwise, we break the simplification logic in visitREM().
4479 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
4480 if (!N1C || TLI.isIntDivCheap(N->getValueType(0), Attr))
4481 if (SDValue DivRem = useDivRem(N))
4482 return DivRem;
4483
4484 return SDValue();
4485}
4486
4487SDValue DAGCombiner::visitUDIVLike(SDValue N0, SDValue N1, SDNode *N) {
4488 SDLoc DL(N);
4489 EVT VT = N->getValueType(0);
4490
4491 // fold (udiv x, (1 << c)) -> x >>u c
4492 if (isConstantOrConstantVector(N1, /*NoOpaques*/ true) &&
4493 DAG.isKnownToBeAPowerOfTwo(N1)) {
4494 SDValue LogBase2 = BuildLogBase2(N1, DL);
4495 AddToWorklist(LogBase2.getNode());
4496
4497 EVT ShiftVT = getShiftAmountTy(N0.getValueType());
4498 SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ShiftVT);
4499 AddToWorklist(Trunc.getNode());
4500 return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc);
4501 }
4502
4503 // fold (udiv x, (shl c, y)) -> x >>u (log2(c)+y) iff c is power of 2
4504 if (N1.getOpcode() == ISD::SHL) {
4505 SDValue N10 = N1.getOperand(0);
4506 if (isConstantOrConstantVector(N10, /*NoOpaques*/ true) &&
4507 DAG.isKnownToBeAPowerOfTwo(N10)) {
4508 SDValue LogBase2 = BuildLogBase2(N10, DL);
4509 AddToWorklist(LogBase2.getNode());
4510
4511 EVT ADDVT = N1.getOperand(1).getValueType();
4512 SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ADDVT);
4513 AddToWorklist(Trunc.getNode());
4514 SDValue Add = DAG.getNode(ISD::ADD, DL, ADDVT, N1.getOperand(1), Trunc);
4515 AddToWorklist(Add.getNode());
4516 return DAG.getNode(ISD::SRL, DL, VT, N0, Add);
4517 }
4518 }
4519
4520 // fold (udiv x, c) -> alternate
4521 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
4522 if (isConstantOrConstantVector(N1) &&
4523 !TLI.isIntDivCheap(N->getValueType(0), Attr))
4524 if (SDValue Op = BuildUDIV(N))
4525 return Op;
4526
4527 return SDValue();
4528}
4529
4530SDValue DAGCombiner::buildOptimizedSREM(SDValue N0, SDValue N1, SDNode *N) {
4531 if (!N->getFlags().hasExact() && isDivisorPowerOfTwo(N1) &&
4532 !DAG.doesNodeExist(ISD::SDIV, N->getVTList(), {N0, N1})) {
4533 // Target-specific implementation of srem x, pow2.
4534 if (SDValue Res = BuildSREMPow2(N))
4535 return Res;
4536 }
4537 return SDValue();
4538}
4539
4540// handles ISD::SREM and ISD::UREM
4541SDValue DAGCombiner::visitREM(SDNode *N) {
4542 unsigned Opcode = N->getOpcode();
4543 SDValue N0 = N->getOperand(0);
4544 SDValue N1 = N->getOperand(1);
4545 EVT VT = N->getValueType(0);
4546 EVT CCVT = getSetCCResultType(VT);
4547
4548 bool isSigned = (Opcode == ISD::SREM);
4549 SDLoc DL(N);
4550
4551 // fold (rem c1, c2) -> c1%c2
4552 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1}))
4553 return C;
4554
4555 // fold (urem X, -1) -> select(FX == -1, 0, FX)
4556 // Freeze the numerator to avoid a miscompile with an undefined value.
4557 if (!isSigned && llvm::isAllOnesOrAllOnesSplat(N1, /*AllowUndefs*/ false)) {
4558 SDValue F0 = DAG.getFreeze(N0);
4559 SDValue EqualsNeg1 = DAG.getSetCC(DL, CCVT, F0, N1, ISD::SETEQ);
4560 return DAG.getSelect(DL, VT, EqualsNeg1, DAG.getConstant(0, DL, VT), F0);
4561 }
4562
4563 if (SDValue V = simplifyDivRem(N, DAG))
4564 return V;
4565
4566 if (SDValue NewSel = foldBinOpIntoSelect(N))
4567 return NewSel;
4568
4569 if (isSigned) {
4570 // If we know the sign bits of both operands are zero, strength reduce to a
4571 // urem instead. Handles (X & 0x0FFFFFFF) %s 16 -> X&15
4572 if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
4573 return DAG.getNode(ISD::UREM, DL, VT, N0, N1);
4574 } else {
4575 if (DAG.isKnownToBeAPowerOfTwo(N1)) {
4576 // fold (urem x, pow2) -> (and x, pow2-1)
4577 SDValue NegOne = DAG.getAllOnesConstant(DL, VT);
4578 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne);
4579 AddToWorklist(Add.getNode());
4580 return DAG.getNode(ISD::AND, DL, VT, N0, Add);
4581 }
4582 // fold (urem x, (shl pow2, y)) -> (and x, (add (shl pow2, y), -1))
4583 // fold (urem x, (lshr pow2, y)) -> (and x, (add (lshr pow2, y), -1))
4584 // TODO: We should sink the following into isKnownToBePowerOfTwo
4585 // using a OrZero parameter analogous to our handling in ValueTracking.
4586 if ((N1.getOpcode() == ISD::SHL || N1.getOpcode() == ISD::SRL) &&
4587 DAG.isKnownToBeAPowerOfTwo(N1.getOperand(0))) {
4588 SDValue NegOne = DAG.getAllOnesConstant(DL, VT);
4589 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne);
4590 AddToWorklist(Add.getNode());
4591 return DAG.getNode(ISD::AND, DL, VT, N0, Add);
4592 }
4593 }
4594
4595 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
4596
4597 // If X/C can be simplified by the division-by-constant logic, lower
4598 // X%C to the equivalent of X-X/C*C.
4599 // Reuse the SDIVLike/UDIVLike combines - to avoid mangling nodes, the
4600 // speculative DIV must not cause a DIVREM conversion. We guard against this
4601 // by skipping the simplification if isIntDivCheap(). When div is not cheap,
4602 // combine will not return a DIVREM. Regardless, checking cheapness here
4603 // makes sense since the simplification results in fatter code.
4604 if (DAG.isKnownNeverZero(N1) && !TLI.isIntDivCheap(VT, Attr)) {
4605 if (isSigned) {
4606 // check if we can build faster implementation for srem
4607 if (SDValue OptimizedRem = buildOptimizedSREM(N0, N1, N))
4608 return OptimizedRem;
4609 }
4610
4611 SDValue OptimizedDiv =
4612 isSigned ? visitSDIVLike(N0, N1, N) : visitUDIVLike(N0, N1, N);
4613 if (OptimizedDiv.getNode() && OptimizedDiv.getNode() != N) {
4614 // If the equivalent Div node also exists, update its users.
4615 unsigned DivOpcode = isSigned ? ISD::SDIV : ISD::UDIV;
4616 if (SDNode *DivNode = DAG.getNodeIfExists(DivOpcode, N->getVTList(),
4617 { N0, N1 }))
4618 CombineTo(DivNode, OptimizedDiv);
4619 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, OptimizedDiv, N1);
4620 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
4621 AddToWorklist(OptimizedDiv.getNode());
4622 AddToWorklist(Mul.getNode());
4623 return Sub;
4624 }
4625 }
4626
4627 // sdiv, srem -> sdivrem
4628 if (SDValue DivRem = useDivRem(N))
4629 return DivRem.getValue(1);
4630
4631 return SDValue();
4632}
4633
4634SDValue DAGCombiner::visitMULHS(SDNode *N) {
4635 SDValue N0 = N->getOperand(0);
4636 SDValue N1 = N->getOperand(1);
4637 EVT VT = N->getValueType(0);
4638 SDLoc DL(N);
4639
4640 // fold (mulhs c1, c2)
4641 if (SDValue C = DAG.FoldConstantArithmetic(ISD::MULHS, DL, VT, {N0, N1}))
4642 return C;
4643
4644 // canonicalize constant to RHS.
4645 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
4646 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
4647 return DAG.getNode(ISD::MULHS, DL, N->getVTList(), N1, N0);
4648
4649 if (VT.isVector()) {
4650 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
4651 return FoldedVOp;
4652
4653 // fold (mulhs x, 0) -> 0
4654 // do not return N1, because undef node may exist.
4655 if (ISD::isConstantSplatVectorAllZeros(N1.getNode()))
4656 return DAG.getConstant(0, DL, VT);
4657 }
4658
4659 // fold (mulhs x, 0) -> 0
4660 if (isNullConstant(N1))
4661 return N1;
4662
4663 // fold (mulhs x, 1) -> (sra x, size(x)-1)
4664 if (isOneConstant(N1))
4665 return DAG.getNode(ISD::SRA, DL, N0.getValueType(), N0,
4666 DAG.getConstant(N0.getScalarValueSizeInBits() - 1, DL,
4667 getShiftAmountTy(N0.getValueType())));
4668
4669 // fold (mulhs x, undef) -> 0
4670 if (N0.isUndef() || N1.isUndef())
4671 return DAG.getConstant(0, DL, VT);
4672
4673 // If the type twice as wide is legal, transform the mulhs to a wider multiply
4674 // plus a shift.
4675 if (!TLI.isOperationLegalOrCustom(ISD::MULHS, VT) && VT.isSimple() &&
4676 !VT.isVector()) {
4677 MVT Simple = VT.getSimpleVT();
4678 unsigned SimpleSize = Simple.getSizeInBits();
4679 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
4680 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
4681 N0 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N0);
4682 N1 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N1);
4683 N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
4684 N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
4685 DAG.getConstant(SimpleSize, DL,
4686 getShiftAmountTy(N1.getValueType())));
4687 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
4688 }
4689 }
4690
4691 return SDValue();
4692}
4693
4694SDValue DAGCombiner::visitMULHU(SDNode *N) {
4695 SDValue N0 = N->getOperand(0);
4696 SDValue N1 = N->getOperand(1);
4697 EVT VT = N->getValueType(0);
4698 SDLoc DL(N);
4699
4700 // fold (mulhu c1, c2)
4701 if (SDValue C = DAG.FoldConstantArithmetic(ISD::MULHU, DL, VT, {N0, N1}))
4702 return C;
4703
4704 // canonicalize constant to RHS.
4705 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
4706 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
4707 return DAG.getNode(ISD::MULHU, DL, N->getVTList(), N1, N0);
4708
4709 if (VT.isVector()) {
4710 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
4711 return FoldedVOp;
4712
4713 // fold (mulhu x, 0) -> 0
4714 // do not return N1, because undef node may exist.
4715 if (ISD::isConstantSplatVectorAllZeros(N1.getNode()))
4716 return DAG.getConstant(0, DL, VT);
4717 }
4718
4719 // fold (mulhu x, 0) -> 0
4720 if (isNullConstant(N1))
4721 return N1;
4722
4723 // fold (mulhu x, 1) -> 0
4724 if (isOneConstant(N1))
4725 return DAG.getConstant(0, DL, N0.getValueType());
4726
4727 // fold (mulhu x, undef) -> 0
4728 if (N0.isUndef() || N1.isUndef())
4729 return DAG.getConstant(0, DL, VT);
4730
4731 // fold (mulhu x, (1 << c)) -> x >> (bitwidth - c)
4732 if (isConstantOrConstantVector(N1, /*NoOpaques*/ true) &&
4733 DAG.isKnownToBeAPowerOfTwo(N1) && hasOperation(ISD::SRL, VT)) {
4734 unsigned NumEltBits = VT.getScalarSizeInBits();
4735 SDValue LogBase2 = BuildLogBase2(N1, DL);
4736 SDValue SRLAmt = DAG.getNode(
4737 ISD::SUB, DL, VT, DAG.getConstant(NumEltBits, DL, VT), LogBase2);
4738 EVT ShiftVT = getShiftAmountTy(N0.getValueType());
4739 SDValue Trunc = DAG.getZExtOrTrunc(SRLAmt, DL, ShiftVT);
4740 return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc);
4741 }
4742
4743 // If the type twice as wide is legal, transform the mulhu to a wider multiply
4744 // plus a shift.
4745 if (!TLI.isOperationLegalOrCustom(ISD::MULHU, VT) && VT.isSimple() &&
4746 !VT.isVector()) {
4747 MVT Simple = VT.getSimpleVT();
4748 unsigned SimpleSize = Simple.getSizeInBits();
4749 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
4750 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
4751 N0 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N0);
4752 N1 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N1);
4753 N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
4754 N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
4755 DAG.getConstant(SimpleSize, DL,
4756 getShiftAmountTy(N1.getValueType())));
4757 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
4758 }
4759 }
4760
4761 // Simplify the operands using demanded-bits information.
4762 // We don't have demanded bits support for MULHU so this just enables constant
4763 // folding based on known bits.
4764 if (SimplifyDemandedBits(SDValue(N, 0)))
4765 return SDValue(N, 0);
4766
4767 return SDValue();
4768}
4769
4770SDValue DAGCombiner::visitAVG(SDNode *N) {
4771 unsigned Opcode = N->getOpcode();
4772 SDValue N0 = N->getOperand(0);
4773 SDValue N1 = N->getOperand(1);
4774 EVT VT = N->getValueType(0);
4775 SDLoc DL(N);
4776
4777 // fold (avg c1, c2)
4778 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1}))
4779 return C;
4780
4781 // canonicalize constant to RHS.
4782 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
4783 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
4784 return DAG.getNode(Opcode, DL, N->getVTList(), N1, N0);
4785
4786 if (VT.isVector()) {
4787 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
4788 return FoldedVOp;
4789
4790 // fold (avgfloor x, 0) -> x >> 1
4791 if (ISD::isConstantSplatVectorAllZeros(N1.getNode())) {
4792 if (Opcode == ISD::AVGFLOORS)
4793 return DAG.getNode(ISD::SRA, DL, VT, N0, DAG.getConstant(1, DL, VT));
4794 if (Opcode == ISD::AVGFLOORU)
4795 return DAG.getNode(ISD::SRL, DL, VT, N0, DAG.getConstant(1, DL, VT));
4796 }
4797 }
4798
4799 // fold (avg x, undef) -> x
4800 if (N0.isUndef())
4801 return N1;
4802 if (N1.isUndef())
4803 return N0;
4804
4805 // TODO If we use avg for scalars anywhere, we can add (avgfl x, 0) -> x >> 1
4806
4807 return SDValue();
4808}
4809
4810/// Perform optimizations common to nodes that compute two values. LoOp and HiOp
4811/// give the opcodes for the two computations that are being performed. Return
4812/// true if a simplification was made.
4813SDValue DAGCombiner::SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
4814 unsigned HiOp) {
4815 // If the high half is not needed, just compute the low half.
4816 bool HiExists = N->hasAnyUseOfValue(1);
4817 if (!HiExists && (!LegalOperations ||
4818 TLI.isOperationLegalOrCustom(LoOp, N->getValueType(0)))) {
4819 SDValue Res = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0), N->ops());
4820 return CombineTo(N, Res, Res);
4821 }
4822
4823 // If the low half is not needed, just compute the high half.
4824 bool LoExists = N->hasAnyUseOfValue(0);
4825 if (!LoExists && (!LegalOperations ||
4826 TLI.isOperationLegalOrCustom(HiOp, N->getValueType(1)))) {
4827 SDValue Res = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1), N->ops());
4828 return CombineTo(N, Res, Res);
4829 }
4830
4831 // If both halves are used, return as it is.
4832 if (LoExists && HiExists)
4833 return SDValue();
4834
4835 // If the two computed results can be simplified separately, separate them.
4836 if (LoExists) {
4837 SDValue Lo = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0), N->ops());
4838 AddToWorklist(Lo.getNode());
4839 SDValue LoOpt = combine(Lo.getNode());
4840 if (LoOpt.getNode() && LoOpt.getNode() != Lo.getNode() &&
4841 (!LegalOperations ||
4842 TLI.isOperationLegalOrCustom(LoOpt.getOpcode(), LoOpt.getValueType())))
4843 return CombineTo(N, LoOpt, LoOpt);
4844 }
4845
4846 if (HiExists) {
4847 SDValue Hi = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1), N->ops());
4848 AddToWorklist(Hi.getNode());
4849 SDValue HiOpt = combine(Hi.getNode());
4850 if (HiOpt.getNode() && HiOpt != Hi &&
4851 (!LegalOperations ||
4852 TLI.isOperationLegalOrCustom(HiOpt.getOpcode(), HiOpt.getValueType())))
4853 return CombineTo(N, HiOpt, HiOpt);
4854 }
4855
4856 return SDValue();
4857}
4858
4859SDValue DAGCombiner::visitSMUL_LOHI(SDNode *N) {
4860 if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHS))
4861 return Res;
4862
4863 SDValue N0 = N->getOperand(0);
4864 SDValue N1 = N->getOperand(1);
4865 EVT VT = N->getValueType(0);
4866 SDLoc DL(N);
4867
4868 // canonicalize constant to RHS (vector doesn't have to splat)
4869 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
4870 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
4871 return DAG.getNode(ISD::SMUL_LOHI, DL, N->getVTList(), N1, N0);
4872
4873 // If the type is twice as wide is legal, transform the mulhu to a wider
4874 // multiply plus a shift.
4875 if (VT.isSimple() && !VT.isVector()) {
4876 MVT Simple = VT.getSimpleVT();
4877 unsigned SimpleSize = Simple.getSizeInBits();
4878 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
4879 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
4880 SDValue Lo = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N0);
4881 SDValue Hi = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N1);
4882 Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
4883 // Compute the high part as N1.
4884 Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
4885 DAG.getConstant(SimpleSize, DL,
4886 getShiftAmountTy(Lo.getValueType())));
4887 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
4888 // Compute the low part as N0.
4889 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
4890 return CombineTo(N, Lo, Hi);
4891 }
4892 }
4893
4894 return SDValue();
4895}
4896
4897SDValue DAGCombiner::visitUMUL_LOHI(SDNode *N) {
4898 if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHU))
4899 return Res;
4900
4901 SDValue N0 = N->getOperand(0);
4902 SDValue N1 = N->getOperand(1);
4903 EVT VT = N->getValueType(0);
4904 SDLoc DL(N);
4905
4906 // canonicalize constant to RHS (vector doesn't have to splat)
4907 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
4908 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
4909 return DAG.getNode(ISD::UMUL_LOHI, DL, N->getVTList(), N1, N0);
4910
4911 // (umul_lohi N0, 0) -> (0, 0)
4912 if (isNullConstant(N1)) {
4913 SDValue Zero = DAG.getConstant(0, DL, VT);
4914 return CombineTo(N, Zero, Zero);
4915 }
4916
4917 // (umul_lohi N0, 1) -> (N0, 0)
4918 if (isOneConstant(N1)) {
4919 SDValue Zero = DAG.getConstant(0, DL, VT);
4920 return CombineTo(N, N0, Zero);
4921 }
4922
4923 // If the type is twice as wide is legal, transform the mulhu to a wider
4924 // multiply plus a shift.
4925 if (VT.isSimple() && !VT.isVector()) {
4926 MVT Simple = VT.getSimpleVT();
4927 unsigned SimpleSize = Simple.getSizeInBits();
4928 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
4929 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
4930 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N0);
4931 SDValue Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N1);
4932 Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
4933 // Compute the high part as N1.
4934 Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
4935 DAG.getConstant(SimpleSize, DL,
4936 getShiftAmountTy(Lo.getValueType())));
4937 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
4938 // Compute the low part as N0.
4939 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
4940 return CombineTo(N, Lo, Hi);
4941 }
4942 }
4943
4944 return SDValue();
4945}
4946
4947SDValue DAGCombiner::visitMULO(SDNode *N) {
4948 SDValue N0 = N->getOperand(0);
4949 SDValue N1 = N->getOperand(1);
4950 EVT VT = N0.getValueType();
4951 bool IsSigned = (ISD::SMULO == N->getOpcode());
4952
4953 EVT CarryVT = N->getValueType(1);
4954 SDLoc DL(N);
4955
4956 ConstantSDNode *N0C = isConstOrConstSplat(N0);
4957 ConstantSDNode *N1C = isConstOrConstSplat(N1);
4958
4959 // fold operation with constant operands.
4960 // TODO: Move this to FoldConstantArithmetic when it supports nodes with
4961 // multiple results.
4962 if (N0C && N1C) {
4963 bool Overflow;
4964 APInt Result =
4965 IsSigned ? N0C->getAPIntValue().smul_ov(N1C->getAPIntValue(), Overflow)
4966 : N0C->getAPIntValue().umul_ov(N1C->getAPIntValue(), Overflow);
4967 return CombineTo(N, DAG.getConstant(Result, DL, VT),
4968 DAG.getBoolConstant(Overflow, DL, CarryVT, CarryVT));
4969 }
4970
4971 // canonicalize constant to RHS.
4972 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
4973 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
4974 return DAG.getNode(N->getOpcode(), DL, N->getVTList(), N1, N0);
4975
4976 // fold (mulo x, 0) -> 0 + no carry out
4977 if (isNullOrNullSplat(N1))
4978 return CombineTo(N, DAG.getConstant(0, DL, VT),
4979 DAG.getConstant(0, DL, CarryVT));
4980
4981 // (mulo x, 2) -> (addo x, x)
4982 // FIXME: This needs a freeze.
4983 if (N1C && N1C->getAPIntValue() == 2 &&
4984 (!IsSigned || VT.getScalarSizeInBits() > 2))
4985 return DAG.getNode(IsSigned ? ISD::SADDO : ISD::UADDO, DL,
4986 N->getVTList(), N0, N0);
4987
4988 if (IsSigned) {
4989 // A 1 bit SMULO overflows if both inputs are 1.
4990 if (VT.getScalarSizeInBits() == 1) {
4991 SDValue And = DAG.getNode(ISD::AND, DL, VT, N0, N1);
4992 return CombineTo(N, And,
4993 DAG.getSetCC(DL, CarryVT, And,
4994 DAG.getConstant(0, DL, VT), ISD::SETNE));
4995 }
4996
4997 // Multiplying n * m significant bits yields a result of n + m significant
4998 // bits. If the total number of significant bits does not exceed the
4999 // result bit width (minus 1), there is no overflow.
5000 unsigned SignBits = DAG.ComputeNumSignBits(N0);
5001 if (SignBits > 1)
5002 SignBits += DAG.ComputeNumSignBits(N1);
5003 if (SignBits > VT.getScalarSizeInBits() + 1)
5004 return CombineTo(N, DAG.getNode(ISD::MUL, DL, VT, N0, N1),
5005 DAG.getConstant(0, DL, CarryVT));
5006 } else {
5007 KnownBits N1Known = DAG.computeKnownBits(N1);
5008 KnownBits N0Known = DAG.computeKnownBits(N0);
5009 bool Overflow;
5010 (void)N0Known.getMaxValue().umul_ov(N1Known.getMaxValue(), Overflow);
5011 if (!Overflow)
5012 return CombineTo(N, DAG.getNode(ISD::MUL, DL, VT, N0, N1),
5013 DAG.getConstant(0, DL, CarryVT));
5014 }
5015
5016 return SDValue();
5017}
5018
5019// Function to calculate whether the Min/Max pair of SDNodes (potentially
5020// swapped around) make a signed saturate pattern, clamping to between a signed
5021// saturate of -2^(BW-1) and 2^(BW-1)-1, or an unsigned saturate of 0 and 2^BW.
5022// Returns the node being clamped and the bitwidth of the clamp in BW. Should
5023// work with both SMIN/SMAX nodes and setcc/select combo. The operands are the
5024// same as SimplifySelectCC. N0<N1 ? N2 : N3.
5025static SDValue isSaturatingMinMax(SDValue N0, SDValue N1, SDValue N2,
5026 SDValue N3, ISD::CondCode CC, unsigned &BW,
5027 bool &Unsigned) {
5028 auto isSignedMinMax = [&](SDValue N0, SDValue N1, SDValue N2, SDValue N3,
5029 ISD::CondCode CC) {
5030 // The compare and select operand should be the same or the select operands
5031 // should be truncated versions of the comparison.
5032 if (N0 != N2 && (N2.getOpcode() != ISD::TRUNCATE || N0 != N2.getOperand(0)))
5033 return 0;
5034 // The constants need to be the same or a truncated version of each other.
5035 ConstantSDNode *N1C = isConstOrConstSplat(N1);
5036 ConstantSDNode *N3C = isConstOrConstSplat(N3);
5037 if (!N1C || !N3C)
5038 return 0;
5039 const APInt &C1 = N1C->getAPIntValue();
5040 const APInt &C2 = N3C->getAPIntValue();
5041 if (C1.getBitWidth() < C2.getBitWidth() || C1 != C2.sext(C1.getBitWidth()))
5042 return 0;
5043 return CC == ISD::SETLT ? ISD::SMIN : (CC == ISD::SETGT ? ISD::SMAX : 0);
5044 };
5045
5046 // Check the initial value is a SMIN/SMAX equivalent.
5047 unsigned Opcode0 = isSignedMinMax(N0, N1, N2, N3, CC);
5048 if (!Opcode0)
5049 return SDValue();
5050
5051 SDValue N00, N01, N02, N03;
5052 ISD::CondCode N0CC;
5053 switch (N0.getOpcode()) {
5054 case ISD::SMIN:
5055 case ISD::SMAX:
5056 N00 = N02 = N0.getOperand(0);
5057 N01 = N03 = N0.getOperand(1);
5058 N0CC = N0.getOpcode() == ISD::SMIN ? ISD::SETLT : ISD::SETGT;
5059 break;
5060 case ISD::SELECT_CC:
5061 N00 = N0.getOperand(0);
5062 N01 = N0.getOperand(1);
5063 N02 = N0.getOperand(2);
5064 N03 = N0.getOperand(3);
5065 N0CC = cast<CondCodeSDNode>(N0.getOperand(4))->get();
5066 break;
5067 case ISD::SELECT:
5068 case ISD::VSELECT:
5069 if (N0.getOperand(0).getOpcode() != ISD::SETCC)
5070 return SDValue();
5071 N00 = N0.getOperand(0).getOperand(0);
5072 N01 = N0.getOperand(0).getOperand(1);
5073 N02 = N0.getOperand(1);
5074 N03 = N0.getOperand(2);
5075 N0CC = cast<CondCodeSDNode>(N0.getOperand(0).getOperand(2))->get();
5076 break;
5077 default:
5078 return SDValue();
5079 }
5080
5081 unsigned Opcode1 = isSignedMinMax(N00, N01, N02, N03, N0CC);
5082 if (!Opcode1 || Opcode0 == Opcode1)
5083 return SDValue();
5084
5085 ConstantSDNode *MinCOp = isConstOrConstSplat(Opcode0 == ISD::SMIN ? N1 : N01);
5086 ConstantSDNode *MaxCOp = isConstOrConstSplat(Opcode0 == ISD::SMIN ? N01 : N1);
5087 if (!MinCOp || !MaxCOp || MinCOp->getValueType(0) != MaxCOp->getValueType(0))
5088 return SDValue();
5089
5090 const APInt &MinC = MinCOp->getAPIntValue();
5091 const APInt &MaxC = MaxCOp->getAPIntValue();
5092 APInt MinCPlus1 = MinC + 1;
5093 if (-MaxC == MinCPlus1 && MinCPlus1.isPowerOf2()) {
5094 BW = MinCPlus1.exactLogBase2() + 1;
5095 Unsigned = false;
5096 return N02;
5097 }
5098
5099 if (MaxC == 0 && MinCPlus1.isPowerOf2()) {
5100 BW = MinCPlus1.exactLogBase2();
5101 Unsigned = true;
5102 return N02;
5103 }
5104
5105 return SDValue();
5106}
5107
5108static SDValue PerformMinMaxFpToSatCombine(SDValue N0, SDValue N1, SDValue N2,
5109 SDValue N3, ISD::CondCode CC,
5110 SelectionDAG &DAG) {
5111 unsigned BW;
5112 bool Unsigned;
5113 SDValue Fp = isSaturatingMinMax(N0, N1, N2, N3, CC, BW, Unsigned);
5114 if (!Fp || Fp.getOpcode() != ISD::FP_TO_SINT)
5115 return SDValue();
5116 EVT FPVT = Fp.getOperand(0).getValueType();
5117 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), BW);
5118 if (FPVT.isVector())
5119 NewVT = EVT::getVectorVT(*DAG.getContext(), NewVT,
5120 FPVT.getVectorElementCount());
5121 unsigned NewOpc = Unsigned ? ISD::FP_TO_UINT_SAT : ISD::FP_TO_SINT_SAT;
5122 if (!DAG.getTargetLoweringInfo().shouldConvertFpToSat(NewOpc, FPVT, NewVT))
5123 return SDValue();
5124 SDLoc DL(Fp);
5125 SDValue Sat = DAG.getNode(NewOpc, DL, NewVT, Fp.getOperand(0),
5126 DAG.getValueType(NewVT.getScalarType()));
5127 return Unsigned ? DAG.getZExtOrTrunc(Sat, DL, N2->getValueType(0))
5128 : DAG.getSExtOrTrunc(Sat, DL, N2->getValueType(0));
5129}
5130
5131static SDValue PerformUMinFpToSatCombine(SDValue N0, SDValue N1, SDValue N2,
5132 SDValue N3, ISD::CondCode CC,
5133 SelectionDAG &DAG) {
5134 // We are looking for UMIN(FPTOUI(X), (2^n)-1), which may have come via a
5135 // select/vselect/select_cc. The two operands pairs for the select (N2/N3) may
5136 // be truncated versions of the the setcc (N0/N1).
5137 if ((N0 != N2 &&
5138 (N2.getOpcode() != ISD::TRUNCATE || N0 != N2.getOperand(0))) ||
5139 N0.getOpcode() != ISD::FP_TO_UINT || CC != ISD::SETULT)
5140 return SDValue();
5141 ConstantSDNode *N1C = isConstOrConstSplat(N1);
5142 ConstantSDNode *N3C = isConstOrConstSplat(N3);
5143 if (!N1C || !N3C)
5144 return SDValue();
5145 const APInt &C1 = N1C->getAPIntValue();
5146 const APInt &C3 = N3C->getAPIntValue();
5147 if (!(C1 + 1).isPowerOf2() || C1.getBitWidth() < C3.getBitWidth() ||
5148 C1 != C3.zext(C1.getBitWidth()))
5149 return SDValue();
5150
5151 unsigned BW = (C1 + 1).exactLogBase2();
5152 EVT FPVT = N0.getOperand(0).getValueType();
5153 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), BW);
5154 if (FPVT.isVector())
5155 NewVT = EVT::getVectorVT(*DAG.getContext(), NewVT,
5156 FPVT.getVectorElementCount());
5157 if (!DAG.getTargetLoweringInfo().shouldConvertFpToSat(ISD::FP_TO_UINT_SAT,
5158 FPVT, NewVT))
5159 return SDValue();
5160
5161 SDValue Sat =
5162 DAG.getNode(ISD::FP_TO_UINT_SAT, SDLoc(N0), NewVT, N0.getOperand(0),
5163 DAG.getValueType(NewVT.getScalarType()));
5164 return DAG.getZExtOrTrunc(Sat, SDLoc(N0), N3.getValueType());
5165}
5166
5167SDValue DAGCombiner::visitIMINMAX(SDNode *N) {
5168 SDValue N0 = N->getOperand(0);
5169 SDValue N1 = N->getOperand(1);
5170 EVT VT = N0.getValueType();
5171 unsigned Opcode = N->getOpcode();
5172 SDLoc DL(N);
5173
5174 // fold operation with constant operands.
5175 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1}))
5176 return C;
5177
5178 // If the operands are the same, this is a no-op.
5179 if (N0 == N1)
5180 return N0;
5181
5182 // canonicalize constant to RHS
5183 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
5184 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
5185 return DAG.getNode(Opcode, DL, VT, N1, N0);
5186
5187 // fold vector ops
5188 if (VT.isVector())
5189 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
5190 return FoldedVOp;
5191
5192 // Is sign bits are zero, flip between UMIN/UMAX and SMIN/SMAX.
5193 // Only do this if the current op isn't legal and the flipped is.
5194 if (!TLI.isOperationLegal(Opcode, VT) &&
5195 (N0.isUndef() || DAG.SignBitIsZero(N0)) &&
5196 (N1.isUndef() || DAG.SignBitIsZero(N1))) {
5197 unsigned AltOpcode;
5198 switch (Opcode) {
5199 case ISD::SMIN: AltOpcode = ISD::UMIN; break;
5200 case ISD::SMAX: AltOpcode = ISD::UMAX; break;
5201 case ISD::UMIN: AltOpcode = ISD::SMIN; break;
5202 case ISD::UMAX: AltOpcode = ISD::SMAX; break;
5203 default: llvm_unreachable("Unknown MINMAX opcode")::llvm::llvm_unreachable_internal("Unknown MINMAX opcode", "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5203)
;
5204 }
5205 if (TLI.isOperationLegal(AltOpcode, VT))
5206 return DAG.getNode(AltOpcode, DL, VT, N0, N1);
5207 }
5208
5209 if (Opcode == ISD::SMIN || Opcode == ISD::SMAX)
5210 if (SDValue S = PerformMinMaxFpToSatCombine(
5211 N0, N1, N0, N1, Opcode == ISD::SMIN ? ISD::SETLT : ISD::SETGT, DAG))
5212 return S;
5213 if (Opcode == ISD::UMIN)
5214 if (SDValue S = PerformUMinFpToSatCombine(N0, N1, N0, N1, ISD::SETULT, DAG))
5215 return S;
5216
5217 // Simplify the operands using demanded-bits information.
5218 if (SimplifyDemandedBits(SDValue(N, 0)))
5219 return SDValue(N, 0);
5220
5221 return SDValue();
5222}
5223
5224/// If this is a bitwise logic instruction and both operands have the same
5225/// opcode, try to sink the other opcode after the logic instruction.
5226SDValue DAGCombiner::hoistLogicOpWithSameOpcodeHands(SDNode *N) {
5227 SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
5228 EVT VT = N0.getValueType();
5229 unsigned LogicOpcode = N->getOpcode();
5230 unsigned HandOpcode = N0.getOpcode();
5231 assert((LogicOpcode == ISD::AND || LogicOpcode == ISD::OR ||(static_cast <bool> ((LogicOpcode == ISD::AND || LogicOpcode
== ISD::OR || LogicOpcode == ISD::XOR) && "Expected logic opcode"
) ? void (0) : __assert_fail ("(LogicOpcode == ISD::AND || LogicOpcode == ISD::OR || LogicOpcode == ISD::XOR) && \"Expected logic opcode\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 5232, __extension__
__PRETTY_FUNCTION__))
5232 LogicOpcode == ISD::XOR) && "Expected logic opcode")(static_cast <bool> ((LogicOpcode == ISD::AND || LogicOpcode
== ISD::OR || LogicOpcode == ISD::XOR) && "Expected logic opcode"
) ? void (0) : __assert_fail ("(LogicOpcode == ISD::AND || LogicOpcode == ISD::OR || LogicOpcode == ISD::XOR) && \"Expected logic opcode\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 5232, __extension__
__PRETTY_FUNCTION__))
;
5233 assert(HandOpcode == N1.getOpcode() && "Bad input!")(static_cast <bool> (HandOpcode == N1.getOpcode() &&
"Bad input!") ? void (0) : __assert_fail ("HandOpcode == N1.getOpcode() && \"Bad input!\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 5233, __extension__
__PRETTY_FUNCTION__))
;
5234
5235 // Bail early if none of these transforms apply.
5236 if (N0.getNumOperands() == 0)
5237 return SDValue();
5238
5239 // FIXME: We should check number of uses of the operands to not increase
5240 // the instruction count for all transforms.
5241
5242 // Handle size-changing casts.
5243 SDValue X = N0.getOperand(0);
5244 SDValue Y = N1.getOperand(0);
5245 EVT XVT = X.getValueType();
5246 SDLoc DL(N);
5247 if (HandOpcode == ISD::ANY_EXTEND || HandOpcode == ISD::ZERO_EXTEND ||
5248 HandOpcode == ISD::SIGN_EXTEND) {
5249 // If both operands have other uses, this transform would create extra
5250 // instructions without eliminating anything.
5251 if (!N0.hasOneUse() && !N1.hasOneUse())
5252 return SDValue();
5253 // We need matching integer source types.
5254 if (XVT != Y.getValueType())
5255 return SDValue();
5256 // Don't create an illegal op during or after legalization. Don't ever
5257 // create an unsupported vector op.
5258 if ((VT.isVector() || LegalOperations) &&
5259 !TLI.isOperationLegalOrCustom(LogicOpcode, XVT))
5260 return SDValue();
5261 // Avoid infinite looping with PromoteIntBinOp.
5262 // TODO: Should we apply desirable/legal constraints to all opcodes?
5263 if (HandOpcode == ISD::ANY_EXTEND && LegalTypes &&
5264 !TLI.isTypeDesirableForOp(LogicOpcode, XVT))
5265 return SDValue();
5266 // logic_op (hand_op X), (hand_op Y) --> hand_op (logic_op X, Y)
5267 SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
5268 return DAG.getNode(HandOpcode, DL, VT, Logic);
5269 }
5270
5271 // logic_op (truncate x), (truncate y) --> truncate (logic_op x, y)
5272 if (HandOpcode == ISD::TRUNCATE) {
5273 // If both operands have other uses, this transform would create extra
5274 // instructions without eliminating anything.
5275 if (!N0.hasOneUse() && !N1.hasOneUse())
5276 return SDValue();
5277 // We need matching source types.
5278 if (XVT != Y.getValueType())
5279 return SDValue();
5280 // Don't create an illegal op during or after legalization.
5281 if (LegalOperations && !TLI.isOperationLegal(LogicOpcode, XVT))
5282 return SDValue();
5283 // Be extra careful sinking truncate. If it's free, there's no benefit in
5284 // widening a binop. Also, don't create a logic op on an illegal type.
5285 if (TLI.isZExtFree(VT, XVT) && TLI.isTruncateFree(XVT, VT))
5286 return SDValue();
5287 if (!TLI.isTypeLegal(XVT))
5288 return SDValue();
5289 SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
5290 return DAG.getNode(HandOpcode, DL, VT, Logic);
5291 }
5292
5293 // For binops SHL/SRL/SRA/AND:
5294 // logic_op (OP x, z), (OP y, z) --> OP (logic_op x, y), z
5295 if ((HandOpcode == ISD::SHL || HandOpcode == ISD::SRL ||
5296 HandOpcode == ISD::SRA || HandOpcode == ISD::AND) &&
5297 N0.getOperand(1) == N1.getOperand(1)) {
5298 // If either operand has other uses, this transform is not an improvement.
5299 if (!N0.hasOneUse() || !N1.hasOneUse())
5300 return SDValue();
5301 SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
5302 return DAG.getNode(HandOpcode, DL, VT, Logic, N0.getOperand(1));
5303 }
5304
5305 // Unary ops: logic_op (bswap x), (bswap y) --> bswap (logic_op x, y)
5306 if (HandOpcode == ISD::BSWAP) {
5307 // If either operand has other uses, this transform is not an improvement.
5308 if (!N0.hasOneUse() || !N1.hasOneUse())
5309 return SDValue();
5310 SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
5311 return DAG.getNode(HandOpcode, DL, VT, Logic);
5312 }
5313
5314 // For funnel shifts FSHL/FSHR:
5315 // logic_op (OP x, x1, s), (OP y, y1, s) -->
5316 // --> OP (logic_op x, y), (logic_op, x1, y1), s
5317 if ((HandOpcode == ISD::FSHL || HandOpcode == ISD::FSHR) &&
5318 N0.getOperand(2) == N1.getOperand(2)) {
5319 if (!N0.hasOneUse() || !N1.hasOneUse())
5320 return SDValue();
5321 SDValue X1 = N0.getOperand(1);
5322 SDValue Y1 = N1.getOperand(1);
5323 SDValue S = N0.getOperand(2);
5324 SDValue Logic0 = DAG.getNode(LogicOpcode, DL, VT, X, Y);
5325 SDValue Logic1 = DAG.getNode(LogicOpcode, DL, VT, X1, Y1);
5326 return DAG.getNode(HandOpcode, DL, VT, Logic0, Logic1, S);
5327 }
5328
5329 // Simplify xor/and/or (bitcast(A), bitcast(B)) -> bitcast(op (A,B))
5330 // Only perform this optimization up until type legalization, before
5331 // LegalizeVectorOprs. LegalizeVectorOprs promotes vector operations by
5332 // adding bitcasts. For example (xor v4i32) is promoted to (v2i64), and
5333 // we don't want to undo this promotion.
5334 // We also handle SCALAR_TO_VECTOR because xor/or/and operations are cheaper
5335 // on scalars.
5336 if ((HandOpcode == ISD::BITCAST || HandOpcode == ISD::SCALAR_TO_VECTOR) &&
5337 Level <= AfterLegalizeTypes) {
5338 // Input types must be integer and the same.
5339 if (XVT.isInteger() && XVT == Y.getValueType() &&
5340 !(VT.isVector() && TLI.isTypeLegal(VT) &&
5341 !XVT.isVector() && !TLI.isTypeLegal(XVT))) {
5342 SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
5343 return DAG.getNode(HandOpcode, DL, VT, Logic);
5344 }
5345 }
5346
5347 // Xor/and/or are indifferent to the swizzle operation (shuffle of one value).
5348 // Simplify xor/and/or (shuff(A), shuff(B)) -> shuff(op (A,B))
5349 // If both shuffles use the same mask, and both shuffle within a single
5350 // vector, then it is worthwhile to move the swizzle after the operation.
5351 // The type-legalizer generates this pattern when loading illegal
5352 // vector types from memory. In many cases this allows additional shuffle
5353 // optimizations.
5354 // There are other cases where moving the shuffle after the xor/and/or
5355 // is profitable even if shuffles don't perform a swizzle.
5356 // If both shuffles use the same mask, and both shuffles have the same first
5357 // or second operand, then it might still be profitable to move the shuffle
5358 // after the xor/and/or operation.
5359 if (HandOpcode == ISD::VECTOR_SHUFFLE && Level < AfterLegalizeDAG) {
5360 auto *SVN0 = cast<ShuffleVectorSDNode>(N0);
5361 auto *SVN1 = cast<ShuffleVectorSDNode>(N1);
5362 assert(X.getValueType() == Y.getValueType() &&(static_cast <bool> (X.getValueType() == Y.getValueType
() && "Inputs to shuffles are not the same type") ? void
(0) : __assert_fail ("X.getValueType() == Y.getValueType() && \"Inputs to shuffles are not the same type\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 5363, __extension__
__PRETTY_FUNCTION__))
5363 "Inputs to shuffles are not the same type")(static_cast <bool> (X.getValueType() == Y.getValueType
() && "Inputs to shuffles are not the same type") ? void
(0) : __assert_fail ("X.getValueType() == Y.getValueType() && \"Inputs to shuffles are not the same type\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 5363, __extension__
__PRETTY_FUNCTION__))
;
5364
5365 // Check that both shuffles use the same mask. The masks are known to be of
5366 // the same length because the result vector type is the same.
5367 // Check also that shuffles have only one use to avoid introducing extra
5368 // instructions.
5369 if (!SVN0->hasOneUse() || !SVN1->hasOneUse() ||
5370 !SVN0->getMask().equals(SVN1->getMask()))
5371 return SDValue();
5372
5373 // Don't try to fold this node if it requires introducing a
5374 // build vector of all zeros that might be illegal at this stage.
5375 SDValue ShOp = N0.getOperand(1);
5376 if (LogicOpcode == ISD::XOR && !ShOp.isUndef())
5377 ShOp = tryFoldToZero(DL, TLI, VT, DAG, LegalOperations);
5378
5379 // (logic_op (shuf (A, C), shuf (B, C))) --> shuf (logic_op (A, B), C)
5380 if (N0.getOperand(1) == N1.getOperand(1) && ShOp.getNode()) {
5381 SDValue Logic = DAG.getNode(LogicOpcode, DL, VT,
5382 N0.getOperand(0), N1.getOperand(0));
5383 return DAG.getVectorShuffle(VT, DL, Logic, ShOp, SVN0->getMask());
5384 }
5385
5386 // Don't try to fold this node if it requires introducing a
5387 // build vector of all zeros that might be illegal at this stage.
5388 ShOp = N0.getOperand(0);
5389 if (LogicOpcode == ISD::XOR && !ShOp.isUndef())
5390 ShOp = tryFoldToZero(DL, TLI, VT, DAG, LegalOperations);
5391
5392 // (logic_op (shuf (C, A), shuf (C, B))) --> shuf (C, logic_op (A, B))
5393 if (N0.getOperand(0) == N1.getOperand(0) && ShOp.getNode()) {
5394 SDValue Logic = DAG.getNode(LogicOpcode, DL, VT, N0.getOperand(1),
5395 N1.getOperand(1));
5396 return DAG.getVectorShuffle(VT, DL, ShOp, Logic, SVN0->getMask());
5397 }
5398 }
5399
5400 return SDValue();
5401}
5402
5403/// Try to make (and/or setcc (LL, LR), setcc (RL, RR)) more efficient.
5404SDValue DAGCombiner::foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1,
5405 const SDLoc &DL) {
5406 SDValue LL, LR, RL, RR, N0CC, N1CC;
5407 if (!isSetCCEquivalent(N0, LL, LR, N0CC) ||
5408 !isSetCCEquivalent(N1, RL, RR, N1CC))
5409 return SDValue();
5410
5411 assert(N0.getValueType() == N1.getValueType() &&(static_cast <bool> (N0.getValueType() == N1.getValueType
() && "Unexpected operand types for bitwise logic op"
) ? void (0) : __assert_fail ("N0.getValueType() == N1.getValueType() && \"Unexpected operand types for bitwise logic op\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 5412, __extension__
__PRETTY_FUNCTION__))
5412 "Unexpected operand types for bitwise logic op")(static_cast <bool> (N0.getValueType() == N1.getValueType
() && "Unexpected operand types for bitwise logic op"
) ? void (0) : __assert_fail ("N0.getValueType() == N1.getValueType() && \"Unexpected operand types for bitwise logic op\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 5412, __extension__
__PRETTY_FUNCTION__))
;
5413 assert(LL.getValueType() == LR.getValueType() &&(static_cast <bool> (LL.getValueType() == LR.getValueType
() && RL.getValueType() == RR.getValueType() &&
"Unexpected operand types for setcc") ? void (0) : __assert_fail
("LL.getValueType() == LR.getValueType() && RL.getValueType() == RR.getValueType() && \"Unexpected operand types for setcc\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 5415, __extension__
__PRETTY_FUNCTION__))
5414 RL.getValueType() == RR.getValueType() &&(static_cast <bool> (LL.getValueType() == LR.getValueType
() && RL.getValueType() == RR.getValueType() &&
"Unexpected operand types for setcc") ? void (0) : __assert_fail
("LL.getValueType() == LR.getValueType() && RL.getValueType() == RR.getValueType() && \"Unexpected operand types for setcc\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 5415, __extension__
__PRETTY_FUNCTION__))
5415 "Unexpected operand types for setcc")(static_cast <bool> (LL.getValueType() == LR.getValueType
() && RL.getValueType() == RR.getValueType() &&
"Unexpected operand types for setcc") ? void (0) : __assert_fail
("LL.getValueType() == LR.getValueType() && RL.getValueType() == RR.getValueType() && \"Unexpected operand types for setcc\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 5415, __extension__
__PRETTY_FUNCTION__))
;
5416
5417 // If we're here post-legalization or the logic op type is not i1, the logic
5418 // op type must match a setcc result type. Also, all folds require new
5419 // operations on the left and right operands, so those types must match.
5420 EVT VT = N0.getValueType();
5421 EVT OpVT = LL.getValueType();
5422 if (LegalOperations || VT.getScalarType() != MVT::i1)
5423 if (VT != getSetCCResultType(OpVT))
5424 return SDValue();
5425 if (OpVT != RL.getValueType())
5426 return SDValue();
5427
5428 ISD::CondCode CC0 = cast<CondCodeSDNode>(N0CC)->get();
5429 ISD::CondCode CC1 = cast<CondCodeSDNode>(N1CC)->get();
5430 bool IsInteger = OpVT.isInteger();
5431 if (LR == RR && CC0 == CC1 && IsInteger) {
5432 bool IsZero = isNullOrNullSplat(LR);
5433 bool IsNeg1 = isAllOnesOrAllOnesSplat(LR);
5434
5435 // All bits clear?
5436 bool AndEqZero = IsAnd && CC1 == ISD::SETEQ && IsZero;
5437 // All sign bits clear?
5438 bool AndGtNeg1 = IsAnd && CC1 == ISD::SETGT && IsNeg1;
5439 // Any bits set?
5440 bool OrNeZero = !IsAnd && CC1 == ISD::SETNE && IsZero;
5441 // Any sign bits set?
5442 bool OrLtZero = !IsAnd && CC1 == ISD::SETLT && IsZero;
5443
5444 // (and (seteq X, 0), (seteq Y, 0)) --> (seteq (or X, Y), 0)
5445 // (and (setgt X, -1), (setgt Y, -1)) --> (setgt (or X, Y), -1)
5446 // (or (setne X, 0), (setne Y, 0)) --> (setne (or X, Y), 0)
5447 // (or (setlt X, 0), (setlt Y, 0)) --> (setlt (or X, Y), 0)
5448 if (AndEqZero || AndGtNeg1 || OrNeZero || OrLtZero) {
5449 SDValue Or = DAG.getNode(ISD::OR, SDLoc(N0), OpVT, LL, RL);
5450 AddToWorklist(Or.getNode());
5451 return DAG.getSetCC(DL, VT, Or, LR, CC1);
5452 }
5453
5454 // All bits set?
5455 bool AndEqNeg1 = IsAnd && CC1 == ISD::SETEQ && IsNeg1;
5456 // All sign bits set?
5457 bool AndLtZero = IsAnd && CC1 == ISD::SETLT && IsZero;
5458 // Any bits clear?
5459 bool OrNeNeg1 = !IsAnd && CC1 == ISD::SETNE && IsNeg1;
5460 // Any sign bits clear?
5461 bool OrGtNeg1 = !IsAnd && CC1 == ISD::SETGT && IsNeg1;
5462
5463 // (and (seteq X, -1), (seteq Y, -1)) --> (seteq (and X, Y), -1)
5464 // (and (setlt X, 0), (setlt Y, 0)) --> (setlt (and X, Y), 0)
5465 // (or (setne X, -1), (setne Y, -1)) --> (setne (and X, Y), -1)
5466 // (or (setgt X, -1), (setgt Y -1)) --> (setgt (and X, Y), -1)
5467 if (AndEqNeg1 || AndLtZero || OrNeNeg1 || OrGtNeg1) {
5468 SDValue And = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, LL, RL);
5469 AddToWorklist(And.getNode());
5470 return DAG.getSetCC(DL, VT, And, LR, CC1);
5471 }
5472 }
5473
5474 // TODO: What is the 'or' equivalent of this fold?
5475 // (and (setne X, 0), (setne X, -1)) --> (setuge (add X, 1), 2)
5476 if (IsAnd && LL == RL && CC0 == CC1 && OpVT.getScalarSizeInBits() > 1 &&
5477 IsInteger && CC0 == ISD::SETNE &&
5478 ((isNullConstant(LR) && isAllOnesConstant(RR)) ||
5479 (isAllOnesConstant(LR) && isNullConstant(RR)))) {
5480 SDValue One = DAG.getConstant(1, DL, OpVT);
5481 SDValue Two = DAG.getConstant(2, DL, OpVT);
5482 SDValue Add = DAG.getNode(ISD::ADD, SDLoc(N0), OpVT, LL, One);
5483 AddToWorklist(Add.getNode());
5484 return DAG.getSetCC(DL, VT, Add, Two, ISD::SETUGE);
5485 }
5486
5487 // Try more general transforms if the predicates match and the only user of
5488 // the compares is the 'and' or 'or'.
5489 if (IsInteger && TLI.convertSetCCLogicToBitwiseLogic(OpVT) && CC0 == CC1 &&
5490 N0.hasOneUse() && N1.hasOneUse()) {
5491 // and (seteq A, B), (seteq C, D) --> seteq (or (xor A, B), (xor C, D)), 0
5492 // or (setne A, B), (setne C, D) --> setne (or (xor A, B), (xor C, D)), 0
5493 if ((IsAnd && CC1 == ISD::SETEQ) || (!IsAnd && CC1 == ISD::SETNE)) {
5494 SDValue XorL = DAG.getNode(ISD::XOR, SDLoc(N0), OpVT, LL, LR);
5495 SDValue XorR = DAG.getNode(ISD::XOR, SDLoc(N1), OpVT, RL, RR);
5496 SDValue Or = DAG.getNode(ISD::OR, DL, OpVT, XorL, XorR);
5497 SDValue Zero = DAG.getConstant(0, DL, OpVT);
5498 return DAG.getSetCC(DL, VT, Or, Zero, CC1);
5499 }
5500
5501 // Turn compare of constants whose difference is 1 bit into add+and+setcc.
5502 if ((IsAnd && CC1 == ISD::SETNE) || (!IsAnd && CC1 == ISD::SETEQ)) {
5503 // Match a shared variable operand and 2 non-opaque constant operands.
5504 auto MatchDiffPow2 = [&](ConstantSDNode *C0, ConstantSDNode *C1) {
5505 // The difference of the constants must be a single bit.
5506 const APInt &CMax =
5507 APIntOps::umax(C0->getAPIntValue(), C1->getAPIntValue());
5508 const APInt &CMin =
5509 APIntOps::umin(C0->getAPIntValue(), C1->getAPIntValue());
5510 return !C0->isOpaque() && !C1->isOpaque() && (CMax - CMin).isPowerOf2();
5511 };
5512 if (LL == RL && ISD::matchBinaryPredicate(LR, RR, MatchDiffPow2)) {
5513 // and/or (setcc X, CMax, ne), (setcc X, CMin, ne/eq) -->
5514 // setcc ((sub X, CMin), ~(CMax - CMin)), 0, ne/eq
5515 SDValue Max = DAG.getNode(ISD::UMAX, DL, OpVT, LR, RR);
5516 SDValue Min = DAG.getNode(ISD::UMIN, DL, OpVT, LR, RR);
5517 SDValue Offset = DAG.getNode(ISD::SUB, DL, OpVT, LL, Min);
5518 SDValue Diff = DAG.getNode(ISD::SUB, DL, OpVT, Max, Min);
5519 SDValue Mask = DAG.getNOT(DL, Diff, OpVT);
5520 SDValue And = DAG.getNode(ISD::AND, DL, OpVT, Offset, Mask);
5521 SDValue Zero = DAG.getConstant(0, DL, OpVT);
5522 return DAG.getSetCC(DL, VT, And, Zero, CC0);
5523 }
5524 }
5525 }
5526
5527 // Canonicalize equivalent operands to LL == RL.
5528 if (LL == RR && LR == RL) {
5529 CC1 = ISD::getSetCCSwappedOperands(CC1);
5530 std::swap(RL, RR);
5531 }
5532
5533 // (and (setcc X, Y, CC0), (setcc X, Y, CC1)) --> (setcc X, Y, NewCC)
5534 // (or (setcc X, Y, CC0), (setcc X, Y, CC1)) --> (setcc X, Y, NewCC)
5535 if (LL == RL && LR == RR) {
5536 ISD::CondCode NewCC = IsAnd ? ISD::getSetCCAndOperation(CC0, CC1, OpVT)
5537 : ISD::getSetCCOrOperation(CC0, CC1, OpVT);
5538 if (NewCC != ISD::SETCC_INVALID &&
5539 (!LegalOperations ||
5540 (TLI.isCondCodeLegal(NewCC, LL.getSimpleValueType()) &&
5541 TLI.isOperationLegal(ISD::SETCC, OpVT))))
5542 return DAG.getSetCC(DL, VT, LL, LR, NewCC);
5543 }
5544
5545 return SDValue();
5546}
5547
5548/// This contains all DAGCombine rules which reduce two values combined by
5549/// an And operation to a single value. This makes them reusable in the context
5550/// of visitSELECT(). Rules involving constants are not included as
5551/// visitSELECT() already handles those cases.
5552SDValue DAGCombiner::visitANDLike(SDValue N0, SDValue N1, SDNode *N) {
5553 EVT VT = N1.getValueType();
5554 SDLoc DL(N);
5555
5556 // fold (and x, undef) -> 0
5557 if (N0.isUndef() || N1.isUndef())
5558 return DAG.getConstant(0, DL, VT);
5559
5560 if (SDValue V = foldLogicOfSetCCs(true, N0, N1, DL))
5561 return V;
5562
5563 // TODO: Rewrite this to return a new 'AND' instead of using CombineTo.
5564 if (N0.getOpcode() == ISD::ADD && N1.getOpcode() == ISD::SRL &&
5565 VT.getSizeInBits() <= 64 && N0->hasOneUse()) {
5566 if (ConstantSDNode *ADDI = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
5567 if (ConstantSDNode *SRLI = dyn_cast<ConstantSDNode>(N1.getOperand(1))) {
5568 // Look for (and (add x, c1), (lshr y, c2)). If C1 wasn't a legal
5569 // immediate for an add, but it is legal if its top c2 bits are set,
5570 // transform the ADD so the immediate doesn't need to be materialized
5571 // in a register.
5572 APInt ADDC = ADDI->getAPIntValue();
5573 APInt SRLC = SRLI->getAPIntValue();
5574 if (ADDC.getMinSignedBits() <= 64 &&
5575 SRLC.ult(VT.getSizeInBits()) &&
5576 !TLI.isLegalAddImmediate(ADDC.getSExtValue())) {
5577 APInt Mask = APInt::getHighBitsSet(VT.getSizeInBits(),
5578 SRLC.getZExtValue());
5579 if (DAG.MaskedValueIsZero(N0.getOperand(1), Mask)) {
5580 ADDC |= Mask;
5581 if (TLI.isLegalAddImmediate(ADDC.getSExtValue())) {
5582 SDLoc DL0(N0);
5583 SDValue NewAdd =
5584 DAG.getNode(ISD::ADD, DL0, VT,
5585 N0.getOperand(0), DAG.getConstant(ADDC, DL, VT));
5586 CombineTo(N0.getNode(), NewAdd);
5587 // Return N so it doesn't get rechecked!
5588 return SDValue(N, 0);
5589 }
5590 }
5591 }
5592 }
5593 }
5594 }
5595
5596 // Reduce bit extract of low half of an integer to the narrower type.
5597 // (and (srl i64:x, K), KMask) ->
5598 // (i64 zero_extend (and (srl (i32 (trunc i64:x)), K)), KMask)
5599 if (N0.getOpcode() == ISD::SRL && N0.hasOneUse()) {
5600 if (ConstantSDNode *CAnd = dyn_cast<ConstantSDNode>(N1)) {
5601 if (ConstantSDNode *CShift = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
5602 unsigned Size = VT.getSizeInBits();
5603 const APInt &AndMask = CAnd->getAPIntValue();
5604 unsigned ShiftBits = CShift->getZExtValue();
5605
5606 // Bail out, this node will probably disappear anyway.
5607 if (ShiftBits == 0)
5608 return SDValue();
5609
5610 unsigned MaskBits = AndMask.countTrailingOnes();
5611 EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), Size / 2);
5612
5613 if (AndMask.isMask() &&
5614 // Required bits must not span the two halves of the integer and
5615 // must fit in the half size type.
5616 (ShiftBits + MaskBits <= Size / 2) &&
5617 TLI.isNarrowingProfitable(VT, HalfVT) &&
5618 TLI.isTypeDesirableForOp(ISD::AND, HalfVT) &&
5619 TLI.isTypeDesirableForOp(ISD::SRL, HalfVT) &&
5620 TLI.isTruncateFree(VT, HalfVT) &&
5621 TLI.isZExtFree(HalfVT, VT)) {
5622 // The isNarrowingProfitable is to avoid regressions on PPC and
5623 // AArch64 which match a few 64-bit bit insert / bit extract patterns
5624 // on downstream users of this. Those patterns could probably be
5625 // extended to handle extensions mixed in.
5626
5627 SDValue SL(N0);
5628 assert(MaskBits <= Size)(static_cast <bool> (MaskBits <= Size) ? void (0) : __assert_fail
("MaskBits <= Size", "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5628, __extension__ __PRETTY_FUNCTION__))
;
5629
5630 // Extracting the highest bit of the low half.
5631 EVT ShiftVT = TLI.getShiftAmountTy(HalfVT, DAG.getDataLayout());
5632 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, HalfVT,
5633 N0.getOperand(0));
5634
5635 SDValue NewMask = DAG.getConstant(AndMask.trunc(Size / 2), SL, HalfVT);
5636 SDValue ShiftK = DAG.getConstant(ShiftBits, SL, ShiftVT);
5637 SDValue Shift = DAG.getNode(ISD::SRL, SL, HalfVT, Trunc, ShiftK);
5638 SDValue And = DAG.getNode(ISD::AND, SL, HalfVT, Shift, NewMask);
5639 return DAG.getNode(ISD::ZERO_EXTEND, SL, VT, And);
5640 }
5641 }
5642 }
5643 }
5644
5645 return SDValue();
5646}
5647
5648bool DAGCombiner::isAndLoadExtLoad(ConstantSDNode *AndC, LoadSDNode *LoadN,
5649 EVT LoadResultTy, EVT &ExtVT) {
5650 if (!AndC->getAPIntValue().isMask())
5651 return false;
5652
5653 unsigned ActiveBits = AndC->getAPIntValue().countTrailingOnes();
5654
5655 ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
5656 EVT LoadedVT = LoadN->getMemoryVT();
5657
5658 if (ExtVT == LoadedVT &&
5659 (!LegalOperations ||
5660 TLI.isLoadExtLegal(ISD::ZEXTLOAD, LoadResultTy, ExtVT))) {
5661 // ZEXTLOAD will match without needing to change the size of the value being
5662 // loaded.
5663 return true;
5664 }
5665
5666 // Do not change the width of a volatile or atomic loads.
5667 if (!LoadN->isSimple())
5668 return false;
5669
5670 // Do not generate loads of non-round integer types since these can
5671 // be expensive (and would be wrong if the type is not byte sized).
5672 if (!LoadedVT.bitsGT(ExtVT) || !ExtVT.isRound())
5673 return false;
5674
5675 if (LegalOperations &&
5676 !TLI.isLoadExtLegal(ISD::ZEXTLOAD, LoadResultTy, ExtVT))
5677 return false;
5678
5679 if (!TLI.shouldReduceLoadWidth(LoadN, ISD::ZEXTLOAD, ExtVT))
5680 return false;
5681
5682 return true;
5683}
5684
5685bool DAGCombiner::isLegalNarrowLdSt(LSBaseSDNode *LDST,
5686 ISD::LoadExtType ExtType, EVT &MemVT,
5687 unsigned ShAmt) {
5688 if (!LDST)
5689 return false;
5690 // Only allow byte offsets.
5691 if (ShAmt % 8)
5692 return false;
5693
5694 // Do not generate loads of non-round integer types since these can
5695 // be expensive (and would be wrong if the type is not byte sized).
5696 if (!MemVT.isRound())
5697 return false;
5698
5699 // Don't change the width of a volatile or atomic loads.
5700 if (!LDST->isSimple())
5701 return false;
5702
5703 EVT LdStMemVT = LDST->getMemoryVT();
5704
5705 // Bail out when changing the scalable property, since we can't be sure that
5706 // we're actually narrowing here.
5707 if (LdStMemVT.isScalableVector() != MemVT.isScalableVector())
5708 return false;
5709
5710 // Verify that we are actually reducing a load width here.
5711 if (LdStMemVT.bitsLT(MemVT))
5712 return false;
5713
5714 // Ensure that this isn't going to produce an unsupported memory access.
5715 if (ShAmt) {
5716 assert(ShAmt % 8 == 0 && "ShAmt is byte offset")(static_cast <bool> (ShAmt % 8 == 0 && "ShAmt is byte offset"
) ? void (0) : __assert_fail ("ShAmt % 8 == 0 && \"ShAmt is byte offset\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 5716, __extension__
__PRETTY_FUNCTION__))
;
5717 const unsigned ByteShAmt = ShAmt / 8;
5718 const Align LDSTAlign = LDST->getAlign();
5719 const Align NarrowAlign = commonAlignment(LDSTAlign, ByteShAmt);
5720 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), MemVT,
5721 LDST->getAddressSpace(), NarrowAlign,
5722 LDST->getMemOperand()->getFlags()))
5723 return false;
5724 }
5725
5726 // It's not possible to generate a constant of extended or untyped type.
5727 EVT PtrType = LDST->getBasePtr().getValueType();
5728 if (PtrType == MVT::Untyped || PtrType.isExtended())
5729 return false;
5730
5731 if (isa<LoadSDNode>(LDST)) {
5732 LoadSDNode *Load = cast<LoadSDNode>(LDST);
5733 // Don't transform one with multiple uses, this would require adding a new
5734 // load.
5735 if (!SDValue(Load, 0).hasOneUse())
5736 return false;
5737
5738 if (LegalOperations &&
5739 !TLI.isLoadExtLegal(ExtType, Load->getValueType(0), MemVT))
5740 return false;
5741
5742 // For the transform to be legal, the load must produce only two values
5743 // (the value loaded and the chain). Don't transform a pre-increment
5744 // load, for example, which produces an extra value. Otherwise the
5745 // transformation is not equivalent, and the downstream logic to replace
5746 // uses gets things wrong.
5747 if (Load->getNumValues() > 2)
5748 return false;
5749
5750 // If the load that we're shrinking is an extload and we're not just
5751 // discarding the extension we can't simply shrink the load. Bail.
5752 // TODO: It would be possible to merge the extensions in some cases.
5753 if (Load->getExtensionType() != ISD::NON_EXTLOAD &&
5754 Load->getMemoryVT().getSizeInBits() < MemVT.getSizeInBits() + ShAmt)
5755 return false;
5756
5757 if (!TLI.shouldReduceLoadWidth(Load, ExtType, MemVT))
5758 return false;
5759 } else {
5760 assert(isa<StoreSDNode>(LDST) && "It is not a Load nor a Store SDNode")(static_cast <bool> (isa<StoreSDNode>(LDST) &&
"It is not a Load nor a Store SDNode") ? void (0) : __assert_fail
("isa<StoreSDNode>(LDST) && \"It is not a Load nor a Store SDNode\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 5760, __extension__
__PRETTY_FUNCTION__))
;
5761 StoreSDNode *Store = cast<StoreSDNode>(LDST);
5762 // Can't write outside the original store
5763 if (Store->getMemoryVT().getSizeInBits() < MemVT.getSizeInBits() + ShAmt)
5764 return false;
5765
5766 if (LegalOperations &&
5767 !TLI.isTruncStoreLegal(Store->getValue().getValueType(), MemVT))
5768 return false;
5769 }
5770 return true;
5771}
5772
5773bool DAGCombiner::SearchForAndLoads(SDNode *N,
5774 SmallVectorImpl<LoadSDNode*> &Loads,
5775 SmallPtrSetImpl<SDNode*> &NodesWithConsts,
5776 ConstantSDNode *Mask,
5777 SDNode *&NodeToMask) {
5778 // Recursively search for the operands, looking for loads which can be
5779 // narrowed.
5780 for (SDValue Op : N->op_values()) {
5781 if (Op.getValueType().isVector())
5782 return false;
5783
5784 // Some constants may need fixing up later if they are too large.
5785 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
5786 if ((N->getOpcode() == ISD::OR || N->getOpcode() == ISD::XOR) &&
5787 (Mask->getAPIntValue() & C->getAPIntValue()) != C->getAPIntValue())
5788 NodesWithConsts.insert(N);
5789 continue;
5790 }
5791
5792 if (!Op.hasOneUse())
5793 return false;
5794
5795 switch(Op.getOpcode()) {
5796 case ISD::LOAD: {
5797 auto *Load = cast<LoadSDNode>(Op);
5798 EVT ExtVT;
5799 if (isAndLoadExtLoad(Mask, Load, Load->getValueType(0), ExtVT) &&
5800 isLegalNarrowLdSt(Load, ISD::ZEXTLOAD, ExtVT)) {
5801
5802 // ZEXTLOAD is already small enough.
5803 if (Load->getExtensionType() == ISD::ZEXTLOAD &&
5804 ExtVT.bitsGE(Load->getMemoryVT()))
5805 continue;
5806
5807 // Use LE to convert equal sized loads to zext.
5808 if (ExtVT.bitsLE(Load->getMemoryVT()))
5809 Loads.push_back(Load);
5810
5811 continue;
5812 }
5813 return false;
5814 }
5815 case ISD::ZERO_EXTEND:
5816 case ISD::AssertZext: {
5817 unsigned ActiveBits = Mask->getAPIntValue().countTrailingOnes();
5818 EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
5819 EVT VT = Op.getOpcode() == ISD::AssertZext ?
5820 cast<VTSDNode>(Op.getOperand(1))->getVT() :
5821 Op.getOperand(0).getValueType();
5822
5823 // We can accept extending nodes if the mask is wider or an equal
5824 // width to the original type.
5825 if (ExtVT.bitsGE(VT))
5826 continue;
5827 break;
5828 }
5829 case ISD::OR:
5830 case ISD::XOR:
5831 case ISD::AND:
5832 if (!SearchForAndLoads(Op.getNode(), Loads, NodesWithConsts, Mask,
5833 NodeToMask))
5834 return false;
5835 continue;
5836 }
5837
5838 // Allow one node which will masked along with any loads found.
5839 if (NodeToMask)
5840 return false;
5841
5842 // Also ensure that the node to be masked only produces one data result.
5843 NodeToMask = Op.getNode();
5844 if (NodeToMask->getNumValues() > 1) {
5845 bool HasValue = false;
5846 for (unsigned i = 0, e = NodeToMask->getNumValues(); i < e; ++i) {
5847 MVT VT = SDValue(NodeToMask, i).getSimpleValueType();
5848 if (VT != MVT::Glue && VT != MVT::Other) {
5849 if (HasValue) {
5850 NodeToMask = nullptr;
5851 return false;
5852 }
5853 HasValue = true;
5854 }
5855 }
5856 assert(HasValue && "Node to be masked has no data result?")(static_cast <bool> (HasValue && "Node to be masked has no data result?"
) ? void (0) : __assert_fail ("HasValue && \"Node to be masked has no data result?\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 5856, __extension__
__PRETTY_FUNCTION__))
;
5857 }
5858 }
5859 return true;
5860}
5861
5862bool DAGCombiner::BackwardsPropagateMask(SDNode *N) {
5863 auto *Mask = dyn_cast<ConstantSDNode>(N->getOperand(1));
5864 if (!Mask)
5865 return false;
5866
5867 if (!Mask->getAPIntValue().isMask())
5868 return false;
5869
5870 // No need to do anything if the and directly uses a load.
5871 if (isa<LoadSDNode>(N->getOperand(0)))
5872 return false;
5873
5874 SmallVector<LoadSDNode*, 8> Loads;
5875 SmallPtrSet<SDNode*, 2> NodesWithConsts;
5876 SDNode *FixupNode = nullptr;
5877 if (SearchForAndLoads(N, Loads, NodesWithConsts, Mask, FixupNode)) {
5878 if (Loads.size() == 0)
5879 return false;
5880
5881 LLVM_DEBUG(dbgs() << "Backwards propagate AND: "; N->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "Backwards propagate AND: "
; N->dump(); } } while (false)
;
5882 SDValue MaskOp = N->getOperand(1);
5883
5884 // If it exists, fixup the single node we allow in the tree that needs
5885 // masking.
5886 if (FixupNode) {
5887 LLVM_DEBUG(dbgs() << "First, need to fix up: "; FixupNode->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "First, need to fix up: "; FixupNode
->dump(); } } while (false)
;
5888 SDValue And = DAG.getNode(ISD::AND, SDLoc(FixupNode),
5889 FixupNode->getValueType(0),
5890 SDValue(FixupNode, 0), MaskOp);
5891 DAG.ReplaceAllUsesOfValueWith(SDValue(FixupNode, 0), And);
5892 if (And.getOpcode() == ISD ::AND)
5893 DAG.UpdateNodeOperands(And.getNode(), SDValue(FixupNode, 0), MaskOp);
5894 }
5895
5896 // Narrow any constants that need it.
5897 for (auto *LogicN : NodesWithConsts) {
5898 SDValue Op0 = LogicN->getOperand(0);
5899 SDValue Op1 = LogicN->getOperand(1);
5900
5901 if (isa<ConstantSDNode>(Op0))
5902 std::swap(Op0, Op1);
5903
5904 SDValue And = DAG.getNode(ISD::AND, SDLoc(Op1), Op1.getValueType(),
5905 Op1, MaskOp);
5906
5907 DAG.UpdateNodeOperands(LogicN, Op0, And);
5908 }
5909
5910 // Create narrow loads.
5911 for (auto *Load : Loads) {
5912 LLVM_DEBUG(dbgs() << "Propagate AND back to: "; Load->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "Propagate AND back to: "; Load
->dump(); } } while (false)
;
5913 SDValue And = DAG.getNode(ISD::AND, SDLoc(Load), Load->getValueType(0),
5914 SDValue(Load, 0), MaskOp);
5915 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), And);
5916 if (And.getOpcode() == ISD ::AND)
5917 And = SDValue(
5918 DAG.UpdateNodeOperands(And.getNode(), SDValue(Load, 0), MaskOp), 0);
5919 SDValue NewLoad = reduceLoadWidth(And.getNode());
5920 assert(NewLoad &&(static_cast <bool> (NewLoad && "Shouldn't be masking the load if it can't be narrowed"
) ? void (0) : __assert_fail ("NewLoad && \"Shouldn't be masking the load if it can't be narrowed\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 5921, __extension__
__PRETTY_FUNCTION__))
5921 "Shouldn't be masking the load if it can't be narrowed")(static_cast <bool> (NewLoad && "Shouldn't be masking the load if it can't be narrowed"
) ? void (0) : __assert_fail ("NewLoad && \"Shouldn't be masking the load if it can't be narrowed\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 5921, __extension__
__PRETTY_FUNCTION__))
;
5922 CombineTo(Load, NewLoad, NewLoad.getValue(1));
5923 }
5924 DAG.ReplaceAllUsesWith(N, N->getOperand(0).getNode());
5925 return true;
5926 }
5927 return false;
5928}
5929
5930// Unfold
5931// x & (-1 'logical shift' y)
5932// To
5933// (x 'opposite logical shift' y) 'logical shift' y
5934// if it is better for performance.
5935SDValue DAGCombiner::unfoldExtremeBitClearingToShifts(SDNode *N) {
5936 assert(N->getOpcode() == ISD::AND)(static_cast <bool> (N->getOpcode() == ISD::AND) ? void
(0) : __assert_fail ("N->getOpcode() == ISD::AND", "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5936, __extension__ __PRETTY_FUNCTION__))
;
5937
5938 SDValue N0 = N->getOperand(0);
5939 SDValue N1 = N->getOperand(1);
5940
5941 // Do we actually prefer shifts over mask?
5942 if (!TLI.shouldFoldMaskToVariableShiftPair(N0))
5943 return SDValue();
5944
5945 // Try to match (-1 '[outer] logical shift' y)
5946 unsigned OuterShift;
5947 unsigned InnerShift; // The opposite direction to the OuterShift.
5948 SDValue Y; // Shift amount.
5949 auto matchMask = [&OuterShift, &InnerShift, &Y](SDValue M) -> bool {
5950 if (!M.hasOneUse())
5951 return false;
5952 OuterShift = M->getOpcode();
5953 if (OuterShift == ISD::SHL)
5954 InnerShift = ISD::SRL;
5955 else if (OuterShift == ISD::SRL)
5956 InnerShift = ISD::SHL;
5957 else
5958 return false;
5959 if (!isAllOnesConstant(M->getOperand(0)))
5960 return false;
5961 Y = M->getOperand(1);
5962 return true;
5963 };
5964
5965 SDValue X;
5966 if (matchMask(N1))
5967 X = N0;
5968 else if (matchMask(N0))
5969 X = N1;
5970 else
5971 return SDValue();
5972
5973 SDLoc DL(N);
5974 EVT VT = N->getValueType(0);
5975
5976 // tmp = x 'opposite logical shift' y
5977 SDValue T0 = DAG.getNode(InnerShift, DL, VT, X, Y);
5978 // ret = tmp 'logical shift' y
5979 SDValue T1 = DAG.getNode(OuterShift, DL, VT, T0, Y);
5980
5981 return T1;
5982}
5983
5984/// Try to replace shift/logic that tests if a bit is clear with mask + setcc.
5985/// For a target with a bit test, this is expected to become test + set and save
5986/// at least 1 instruction.
5987static SDValue combineShiftAnd1ToBitTest(SDNode *And, SelectionDAG &DAG) {
5988 assert(And->getOpcode() == ISD::AND && "Expected an 'and' op")(static_cast <bool> (And->getOpcode() == ISD::AND &&
"Expected an 'and' op") ? void (0) : __assert_fail ("And->getOpcode() == ISD::AND && \"Expected an 'and' op\""
, "llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp", 5988, __extension__
__PRETTY_FUNCTION__))
;
5989
5990 // This is probably not worthwhile without a supported type.
5991 EVT VT = And->getValueType(0);
5992 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5993 if (!TLI.isTypeLegal(VT))
5994 return SDValue();
5995
5996 // Look through an optional extension.
5997 SDValue And0 = And->getOperand(0), And1 = And->getOperand(1);
5998 if (And0.getOpcode() == ISD::ANY_EXTEND && And0.hasOneUse())
5999 And0 = And0.getOperand(0);
6000 if (!isOneConstant(And1) || !And0.hasOneUse())
6001 return SDValue();
6002
6003 SDValue Src = And0;
6004
6005 // Attempt to find a 'not' op.
6006 // TODO: Should we favor test+set even without the 'not' op?
6007 bool FoundNot = false;
6008 if (isBitwiseNot(Src)) {
6009 FoundNot = true;
6010 Src = Src.getOperand(0);
6011
6012 // Look though an optional truncation. The source operand may not be the
6013 // same type as the original 'and', but that is ok because we are masking
6014 // off everything but the low bit.
6015 if (Src.getOpcode() == ISD::TRUNCATE && Src.hasOneUse())
6016 Src = Src.getOperand(0);
6017 }
6018
6019 // Match a shift-right by constant.
6020 if (Src.getOpcode() != ISD::SRL || !Src.hasOneUse())
6021 return SDValue();
6022
6023 // We might have looked through casts that make this transform invalid.
6024 // TODO: If the source type is wider than the result type, do the mask and
6025 // compare in the source type.
6026