Bug Summary

File:llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
Warning:line 17025, column 11
Value stored to 'StartAddress' during its initialization is never read

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name DAGCombiner.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -fhalf-no-semantic-interposition -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -fno-split-dwarf-inlining -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-12/lib/clang/12.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/build-llvm/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/build-llvm/include -I /build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-12/lib/clang/12.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/build-llvm/lib/CodeGen/SelectionDAG -fdebug-prefix-map=/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1=. -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2021-01-24-223304-31662-1 -x c++ /build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
1//===- DAGCombiner.cpp - Implement a DAG node combiner --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass combines dag nodes to form fewer, simpler DAG nodes. It can be run
10// both before and after the DAG is legalized.
11//
12// This pass is not a substitute for the LLVM IR instcombine pass. This pass is
13// primarily intended to handle simplification opportunities that are implicit
14// in the LLVM IR and exposed by the various codegen lowering phases.
15//
16//===----------------------------------------------------------------------===//
17
18#include "llvm/ADT/APFloat.h"
19#include "llvm/ADT/APInt.h"
20#include "llvm/ADT/ArrayRef.h"
21#include "llvm/ADT/DenseMap.h"
22#include "llvm/ADT/IntervalMap.h"
23#include "llvm/ADT/None.h"
24#include "llvm/ADT/Optional.h"
25#include "llvm/ADT/STLExtras.h"
26#include "llvm/ADT/SetVector.h"
27#include "llvm/ADT/SmallBitVector.h"
28#include "llvm/ADT/SmallPtrSet.h"
29#include "llvm/ADT/SmallSet.h"
30#include "llvm/ADT/SmallVector.h"
31#include "llvm/ADT/Statistic.h"
32#include "llvm/Analysis/AliasAnalysis.h"
33#include "llvm/Analysis/MemoryLocation.h"
34#include "llvm/Analysis/TargetLibraryInfo.h"
35#include "llvm/Analysis/VectorUtils.h"
36#include "llvm/CodeGen/DAGCombine.h"
37#include "llvm/CodeGen/ISDOpcodes.h"
38#include "llvm/CodeGen/MachineFrameInfo.h"
39#include "llvm/CodeGen/MachineFunction.h"
40#include "llvm/CodeGen/MachineMemOperand.h"
41#include "llvm/CodeGen/RuntimeLibcalls.h"
42#include "llvm/CodeGen/SelectionDAG.h"
43#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
44#include "llvm/CodeGen/SelectionDAGNodes.h"
45#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
46#include "llvm/CodeGen/TargetLowering.h"
47#include "llvm/CodeGen/TargetRegisterInfo.h"
48#include "llvm/CodeGen/TargetSubtargetInfo.h"
49#include "llvm/CodeGen/ValueTypes.h"
50#include "llvm/IR/Attributes.h"
51#include "llvm/IR/Constant.h"
52#include "llvm/IR/DataLayout.h"
53#include "llvm/IR/DerivedTypes.h"
54#include "llvm/IR/Function.h"
55#include "llvm/IR/LLVMContext.h"
56#include "llvm/IR/Metadata.h"
57#include "llvm/Support/Casting.h"
58#include "llvm/Support/CodeGen.h"
59#include "llvm/Support/CommandLine.h"
60#include "llvm/Support/Compiler.h"
61#include "llvm/Support/Debug.h"
62#include "llvm/Support/ErrorHandling.h"
63#include "llvm/Support/KnownBits.h"
64#include "llvm/Support/MachineValueType.h"
65#include "llvm/Support/MathExtras.h"
66#include "llvm/Support/raw_ostream.h"
67#include "llvm/Target/TargetMachine.h"
68#include "llvm/Target/TargetOptions.h"
69#include <algorithm>
70#include <cassert>
71#include <cstdint>
72#include <functional>
73#include <iterator>
74#include <string>
75#include <tuple>
76#include <utility>
77
78using namespace llvm;
79
80#define DEBUG_TYPE"dagcombine" "dagcombine"
81
82STATISTIC(NodesCombined , "Number of dag nodes combined")static llvm::Statistic NodesCombined = {"dagcombine", "NodesCombined"
, "Number of dag nodes combined"}
;
83STATISTIC(PreIndexedNodes , "Number of pre-indexed nodes created")static llvm::Statistic PreIndexedNodes = {"dagcombine", "PreIndexedNodes"
, "Number of pre-indexed nodes created"}
;
84STATISTIC(PostIndexedNodes, "Number of post-indexed nodes created")static llvm::Statistic PostIndexedNodes = {"dagcombine", "PostIndexedNodes"
, "Number of post-indexed nodes created"}
;
85STATISTIC(OpsNarrowed , "Number of load/op/store narrowed")static llvm::Statistic OpsNarrowed = {"dagcombine", "OpsNarrowed"
, "Number of load/op/store narrowed"}
;
86STATISTIC(LdStFP2Int , "Number of fp load/store pairs transformed to int")static llvm::Statistic LdStFP2Int = {"dagcombine", "LdStFP2Int"
, "Number of fp load/store pairs transformed to int"}
;
87STATISTIC(SlicedLoads, "Number of load sliced")static llvm::Statistic SlicedLoads = {"dagcombine", "SlicedLoads"
, "Number of load sliced"}
;
88STATISTIC(NumFPLogicOpsConv, "Number of logic ops converted to fp ops")static llvm::Statistic NumFPLogicOpsConv = {"dagcombine", "NumFPLogicOpsConv"
, "Number of logic ops converted to fp ops"}
;
89
90static cl::opt<bool>
91CombinerGlobalAA("combiner-global-alias-analysis", cl::Hidden,
92 cl::desc("Enable DAG combiner's use of IR alias analysis"));
93
94static cl::opt<bool>
95UseTBAA("combiner-use-tbaa", cl::Hidden, cl::init(true),
96 cl::desc("Enable DAG combiner's use of TBAA"));
97
98#ifndef NDEBUG
99static cl::opt<std::string>
100CombinerAAOnlyFunc("combiner-aa-only-func", cl::Hidden,
101 cl::desc("Only use DAG-combiner alias analysis in this"
102 " function"));
103#endif
104
105/// Hidden option to stress test load slicing, i.e., when this option
106/// is enabled, load slicing bypasses most of its profitability guards.
107static cl::opt<bool>
108StressLoadSlicing("combiner-stress-load-slicing", cl::Hidden,
109 cl::desc("Bypass the profitability model of load slicing"),
110 cl::init(false));
111
112static cl::opt<bool>
113 MaySplitLoadIndex("combiner-split-load-index", cl::Hidden, cl::init(true),
114 cl::desc("DAG combiner may split indexing from loads"));
115
116static cl::opt<bool>
117 EnableStoreMerging("combiner-store-merging", cl::Hidden, cl::init(true),
118 cl::desc("DAG combiner enable merging multiple stores "
119 "into a wider store"));
120
121static cl::opt<unsigned> TokenFactorInlineLimit(
122 "combiner-tokenfactor-inline-limit", cl::Hidden, cl::init(2048),
123 cl::desc("Limit the number of operands to inline for Token Factors"));
124
125static cl::opt<unsigned> StoreMergeDependenceLimit(
126 "combiner-store-merge-dependence-limit", cl::Hidden, cl::init(10),
127 cl::desc("Limit the number of times for the same StoreNode and RootNode "
128 "to bail out in store merging dependence check"));
129
130static cl::opt<bool> EnableReduceLoadOpStoreWidth(
131 "combiner-reduce-load-op-store-width", cl::Hidden, cl::init(true),
132 cl::desc("DAG cominber enable reducing the width of load/op/store "
133 "sequence"));
134
135static cl::opt<bool> EnableShrinkLoadReplaceStoreWithStore(
136 "combiner-shrink-load-replace-store-with-store", cl::Hidden, cl::init(true),
137 cl::desc("DAG cominber enable load/<replace bytes>/store with "
138 "a narrower store"));
139
140namespace {
141
142 class DAGCombiner {
143 SelectionDAG &DAG;
144 const TargetLowering &TLI;
145 const SelectionDAGTargetInfo *STI;
146 CombineLevel Level;
147 CodeGenOpt::Level OptLevel;
148 bool LegalDAG = false;
149 bool LegalOperations = false;
150 bool LegalTypes = false;
151 bool ForCodeSize;
152 bool DisableGenericCombines;
153
154 /// Worklist of all of the nodes that need to be simplified.
155 ///
156 /// This must behave as a stack -- new nodes to process are pushed onto the
157 /// back and when processing we pop off of the back.
158 ///
159 /// The worklist will not contain duplicates but may contain null entries
160 /// due to nodes being deleted from the underlying DAG.
161 SmallVector<SDNode *, 64> Worklist;
162
163 /// Mapping from an SDNode to its position on the worklist.
164 ///
165 /// This is used to find and remove nodes from the worklist (by nulling
166 /// them) when they are deleted from the underlying DAG. It relies on
167 /// stable indices of nodes within the worklist.
168 DenseMap<SDNode *, unsigned> WorklistMap;
169 /// This records all nodes attempted to add to the worklist since we
170 /// considered a new worklist entry. As we keep do not add duplicate nodes
171 /// in the worklist, this is different from the tail of the worklist.
172 SmallSetVector<SDNode *, 32> PruningList;
173
174 /// Set of nodes which have been combined (at least once).
175 ///
176 /// This is used to allow us to reliably add any operands of a DAG node
177 /// which have not yet been combined to the worklist.
178 SmallPtrSet<SDNode *, 32> CombinedNodes;
179
180 /// Map from candidate StoreNode to the pair of RootNode and count.
181 /// The count is used to track how many times we have seen the StoreNode
182 /// with the same RootNode bail out in dependence check. If we have seen
183 /// the bail out for the same pair many times over a limit, we won't
184 /// consider the StoreNode with the same RootNode as store merging
185 /// candidate again.
186 DenseMap<SDNode *, std::pair<SDNode *, unsigned>> StoreRootCountMap;
187
188 // AA - Used for DAG load/store alias analysis.
189 AliasAnalysis *AA;
190
191 /// When an instruction is simplified, add all users of the instruction to
192 /// the work lists because they might get more simplified now.
193 void AddUsersToWorklist(SDNode *N) {
194 for (SDNode *Node : N->uses())
195 AddToWorklist(Node);
196 }
197
198 /// Convenient shorthand to add a node and all of its user to the worklist.
199 void AddToWorklistWithUsers(SDNode *N) {
200 AddUsersToWorklist(N);
201 AddToWorklist(N);
202 }
203
204 // Prune potentially dangling nodes. This is called after
205 // any visit to a node, but should also be called during a visit after any
206 // failed combine which may have created a DAG node.
207 void clearAddedDanglingWorklistEntries() {
208 // Check any nodes added to the worklist to see if they are prunable.
209 while (!PruningList.empty()) {
210 auto *N = PruningList.pop_back_val();
211 if (N->use_empty())
212 recursivelyDeleteUnusedNodes(N);
213 }
214 }
215
216 SDNode *getNextWorklistEntry() {
217 // Before we do any work, remove nodes that are not in use.
218 clearAddedDanglingWorklistEntries();
219 SDNode *N = nullptr;
220 // The Worklist holds the SDNodes in order, but it may contain null
221 // entries.
222 while (!N && !Worklist.empty()) {
223 N = Worklist.pop_back_val();
224 }
225
226 if (N) {
227 bool GoodWorklistEntry = WorklistMap.erase(N);
228 (void)GoodWorklistEntry;
229 assert(GoodWorklistEntry &&((GoodWorklistEntry && "Found a worklist entry without a corresponding map entry!"
) ? static_cast<void> (0) : __assert_fail ("GoodWorklistEntry && \"Found a worklist entry without a corresponding map entry!\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 230, __PRETTY_FUNCTION__))
230 "Found a worklist entry without a corresponding map entry!")((GoodWorklistEntry && "Found a worklist entry without a corresponding map entry!"
) ? static_cast<void> (0) : __assert_fail ("GoodWorklistEntry && \"Found a worklist entry without a corresponding map entry!\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 230, __PRETTY_FUNCTION__))
;
231 }
232 return N;
233 }
234
235 /// Call the node-specific routine that folds each particular type of node.
236 SDValue visit(SDNode *N);
237
238 public:
239 DAGCombiner(SelectionDAG &D, AliasAnalysis *AA, CodeGenOpt::Level OL)
240 : DAG(D), TLI(D.getTargetLoweringInfo()),
241 STI(D.getSubtarget().getSelectionDAGInfo()),
242 Level(BeforeLegalizeTypes), OptLevel(OL), AA(AA) {
243 ForCodeSize = DAG.shouldOptForSize();
244 DisableGenericCombines = STI && STI->disableGenericCombines(OptLevel);
245
246 MaximumLegalStoreInBits = 0;
247 // We use the minimum store size here, since that's all we can guarantee
248 // for the scalable vector types.
249 for (MVT VT : MVT::all_valuetypes())
250 if (EVT(VT).isSimple() && VT != MVT::Other &&
251 TLI.isTypeLegal(EVT(VT)) &&
252 VT.getSizeInBits().getKnownMinSize() >= MaximumLegalStoreInBits)
253 MaximumLegalStoreInBits = VT.getSizeInBits().getKnownMinSize();
254 }
255
256 void ConsiderForPruning(SDNode *N) {
257 // Mark this for potential pruning.
258 PruningList.insert(N);
259 }
260
261 /// Add to the worklist making sure its instance is at the back (next to be
262 /// processed.)
263 void AddToWorklist(SDNode *N) {
264 assert(N->getOpcode() != ISD::DELETED_NODE &&((N->getOpcode() != ISD::DELETED_NODE && "Deleted Node added to Worklist"
) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && \"Deleted Node added to Worklist\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 265, __PRETTY_FUNCTION__))
265 "Deleted Node added to Worklist")((N->getOpcode() != ISD::DELETED_NODE && "Deleted Node added to Worklist"
) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && \"Deleted Node added to Worklist\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 265, __PRETTY_FUNCTION__))
;
266
267 // Skip handle nodes as they can't usefully be combined and confuse the
268 // zero-use deletion strategy.
269 if (N->getOpcode() == ISD::HANDLENODE)
270 return;
271
272 ConsiderForPruning(N);
273
274 if (WorklistMap.insert(std::make_pair(N, Worklist.size())).second)
275 Worklist.push_back(N);
276 }
277
278 /// Remove all instances of N from the worklist.
279 void removeFromWorklist(SDNode *N) {
280 CombinedNodes.erase(N);
281 PruningList.remove(N);
282 StoreRootCountMap.erase(N);
283
284 auto It = WorklistMap.find(N);
285 if (It == WorklistMap.end())
286 return; // Not in the worklist.
287
288 // Null out the entry rather than erasing it to avoid a linear operation.
289 Worklist[It->second] = nullptr;
290 WorklistMap.erase(It);
291 }
292
293 void deleteAndRecombine(SDNode *N);
294 bool recursivelyDeleteUnusedNodes(SDNode *N);
295
296 /// Replaces all uses of the results of one DAG node with new values.
297 SDValue CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
298 bool AddTo = true);
299
300 /// Replaces all uses of the results of one DAG node with new values.
301 SDValue CombineTo(SDNode *N, SDValue Res, bool AddTo = true) {
302 return CombineTo(N, &Res, 1, AddTo);
303 }
304
305 /// Replaces all uses of the results of one DAG node with new values.
306 SDValue CombineTo(SDNode *N, SDValue Res0, SDValue Res1,
307 bool AddTo = true) {
308 SDValue To[] = { Res0, Res1 };
309 return CombineTo(N, To, 2, AddTo);
310 }
311
312 void CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO);
313
314 private:
315 unsigned MaximumLegalStoreInBits;
316
317 /// Check the specified integer node value to see if it can be simplified or
318 /// if things it uses can be simplified by bit propagation.
319 /// If so, return true.
320 bool SimplifyDemandedBits(SDValue Op) {
321 unsigned BitWidth = Op.getScalarValueSizeInBits();
322 APInt DemandedBits = APInt::getAllOnesValue(BitWidth);
323 return SimplifyDemandedBits(Op, DemandedBits);
324 }
325
326 bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits) {
327 TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations);
328 KnownBits Known;
329 if (!TLI.SimplifyDemandedBits(Op, DemandedBits, Known, TLO, 0, false))
330 return false;
331
332 // Revisit the node.
333 AddToWorklist(Op.getNode());
334
335 CommitTargetLoweringOpt(TLO);
336 return true;
337 }
338
339 /// Check the specified vector node value to see if it can be simplified or
340 /// if things it uses can be simplified as it only uses some of the
341 /// elements. If so, return true.
342 bool SimplifyDemandedVectorElts(SDValue Op) {
343 // TODO: For now just pretend it cannot be simplified.
344 if (Op.getValueType().isScalableVector())
345 return false;
346
347 unsigned NumElts = Op.getValueType().getVectorNumElements();
348 APInt DemandedElts = APInt::getAllOnesValue(NumElts);
349 return SimplifyDemandedVectorElts(Op, DemandedElts);
350 }
351
352 bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
353 const APInt &DemandedElts,
354 bool AssumeSingleUse = false);
355 bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedElts,
356 bool AssumeSingleUse = false);
357
358 bool CombineToPreIndexedLoadStore(SDNode *N);
359 bool CombineToPostIndexedLoadStore(SDNode *N);
360 SDValue SplitIndexingFromLoad(LoadSDNode *LD);
361 bool SliceUpLoad(SDNode *N);
362
363 // Scalars have size 0 to distinguish from singleton vectors.
364 SDValue ForwardStoreValueToDirectLoad(LoadSDNode *LD);
365 bool getTruncatedStoreValue(StoreSDNode *ST, SDValue &Val);
366 bool extendLoadedValueToExtension(LoadSDNode *LD, SDValue &Val);
367
368 /// Replace an ISD::EXTRACT_VECTOR_ELT of a load with a narrowed
369 /// load.
370 ///
371 /// \param EVE ISD::EXTRACT_VECTOR_ELT to be replaced.
372 /// \param InVecVT type of the input vector to EVE with bitcasts resolved.
373 /// \param EltNo index of the vector element to load.
374 /// \param OriginalLoad load that EVE came from to be replaced.
375 /// \returns EVE on success SDValue() on failure.
376 SDValue scalarizeExtractedVectorLoad(SDNode *EVE, EVT InVecVT,
377 SDValue EltNo,
378 LoadSDNode *OriginalLoad);
379 void ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad);
380 SDValue PromoteOperand(SDValue Op, EVT PVT, bool &Replace);
381 SDValue SExtPromoteOperand(SDValue Op, EVT PVT);
382 SDValue ZExtPromoteOperand(SDValue Op, EVT PVT);
383 SDValue PromoteIntBinOp(SDValue Op);
384 SDValue PromoteIntShiftOp(SDValue Op);
385 SDValue PromoteExtend(SDValue Op);
386 bool PromoteLoad(SDValue Op);
387
388 /// Call the node-specific routine that knows how to fold each
389 /// particular type of node. If that doesn't do anything, try the
390 /// target-specific DAG combines.
391 SDValue combine(SDNode *N);
392
393 // Visitation implementation - Implement dag node combining for different
394 // node types. The semantics are as follows:
395 // Return Value:
396 // SDValue.getNode() == 0 - No change was made
397 // SDValue.getNode() == N - N was replaced, is dead and has been handled.
398 // otherwise - N should be replaced by the returned Operand.
399 //
400 SDValue visitTokenFactor(SDNode *N);
401 SDValue visitMERGE_VALUES(SDNode *N);
402 SDValue visitADD(SDNode *N);
403 SDValue visitADDLike(SDNode *N);
404 SDValue visitADDLikeCommutative(SDValue N0, SDValue N1, SDNode *LocReference);
405 SDValue visitSUB(SDNode *N);
406 SDValue visitADDSAT(SDNode *N);
407 SDValue visitSUBSAT(SDNode *N);
408 SDValue visitADDC(SDNode *N);
409 SDValue visitADDO(SDNode *N);
410 SDValue visitUADDOLike(SDValue N0, SDValue N1, SDNode *N);
411 SDValue visitSUBC(SDNode *N);
412 SDValue visitSUBO(SDNode *N);
413 SDValue visitADDE(SDNode *N);
414 SDValue visitADDCARRY(SDNode *N);
415 SDValue visitSADDO_CARRY(SDNode *N);
416 SDValue visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn, SDNode *N);
417 SDValue visitSUBE(SDNode *N);
418 SDValue visitSUBCARRY(SDNode *N);
419 SDValue visitSSUBO_CARRY(SDNode *N);
420 SDValue visitMUL(SDNode *N);
421 SDValue visitMULFIX(SDNode *N);
422 SDValue useDivRem(SDNode *N);
423 SDValue visitSDIV(SDNode *N);
424 SDValue visitSDIVLike(SDValue N0, SDValue N1, SDNode *N);
425 SDValue visitUDIV(SDNode *N);
426 SDValue visitUDIVLike(SDValue N0, SDValue N1, SDNode *N);
427 SDValue visitREM(SDNode *N);
428 SDValue visitMULHU(SDNode *N);
429 SDValue visitMULHS(SDNode *N);
430 SDValue visitSMUL_LOHI(SDNode *N);
431 SDValue visitUMUL_LOHI(SDNode *N);
432 SDValue visitMULO(SDNode *N);
433 SDValue visitIMINMAX(SDNode *N);
434 SDValue visitAND(SDNode *N);
435 SDValue visitANDLike(SDValue N0, SDValue N1, SDNode *N);
436 SDValue visitOR(SDNode *N);
437 SDValue visitORLike(SDValue N0, SDValue N1, SDNode *N);
438 SDValue visitXOR(SDNode *N);
439 SDValue SimplifyVBinOp(SDNode *N);
440 SDValue visitSHL(SDNode *N);
441 SDValue visitSRA(SDNode *N);
442 SDValue visitSRL(SDNode *N);
443 SDValue visitFunnelShift(SDNode *N);
444 SDValue visitRotate(SDNode *N);
445 SDValue visitABS(SDNode *N);
446 SDValue visitBSWAP(SDNode *N);
447 SDValue visitBITREVERSE(SDNode *N);
448 SDValue visitCTLZ(SDNode *N);
449 SDValue visitCTLZ_ZERO_UNDEF(SDNode *N);
450 SDValue visitCTTZ(SDNode *N);
451 SDValue visitCTTZ_ZERO_UNDEF(SDNode *N);
452 SDValue visitCTPOP(SDNode *N);
453 SDValue visitSELECT(SDNode *N);
454 SDValue visitVSELECT(SDNode *N);
455 SDValue visitSELECT_CC(SDNode *N);
456 SDValue visitSETCC(SDNode *N);
457 SDValue visitSETCCCARRY(SDNode *N);
458 SDValue visitSIGN_EXTEND(SDNode *N);
459 SDValue visitZERO_EXTEND(SDNode *N);
460 SDValue visitANY_EXTEND(SDNode *N);
461 SDValue visitAssertExt(SDNode *N);
462 SDValue visitAssertAlign(SDNode *N);
463 SDValue visitSIGN_EXTEND_INREG(SDNode *N);
464 SDValue visitSIGN_EXTEND_VECTOR_INREG(SDNode *N);
465 SDValue visitZERO_EXTEND_VECTOR_INREG(SDNode *N);
466 SDValue visitTRUNCATE(SDNode *N);
467 SDValue visitBITCAST(SDNode *N);
468 SDValue visitFREEZE(SDNode *N);
469 SDValue visitBUILD_PAIR(SDNode *N);
470 SDValue visitFADD(SDNode *N);
471 SDValue visitSTRICT_FADD(SDNode *N);
472 SDValue visitFSUB(SDNode *N);
473 SDValue visitFMUL(SDNode *N);
474 SDValue visitFMA(SDNode *N);
475 SDValue visitFDIV(SDNode *N);
476 SDValue visitFREM(SDNode *N);
477 SDValue visitFSQRT(SDNode *N);
478 SDValue visitFCOPYSIGN(SDNode *N);
479 SDValue visitFPOW(SDNode *N);
480 SDValue visitSINT_TO_FP(SDNode *N);
481 SDValue visitUINT_TO_FP(SDNode *N);
482 SDValue visitFP_TO_SINT(SDNode *N);
483 SDValue visitFP_TO_UINT(SDNode *N);
484 SDValue visitFP_ROUND(SDNode *N);
485 SDValue visitFP_EXTEND(SDNode *N);
486 SDValue visitFNEG(SDNode *N);
487 SDValue visitFABS(SDNode *N);
488 SDValue visitFCEIL(SDNode *N);
489 SDValue visitFTRUNC(SDNode *N);
490 SDValue visitFFLOOR(SDNode *N);
491 SDValue visitFMINNUM(SDNode *N);
492 SDValue visitFMAXNUM(SDNode *N);
493 SDValue visitFMINIMUM(SDNode *N);
494 SDValue visitFMAXIMUM(SDNode *N);
495 SDValue visitBRCOND(SDNode *N);
496 SDValue visitBR_CC(SDNode *N);
497 SDValue visitLOAD(SDNode *N);
498
499 SDValue replaceStoreChain(StoreSDNode *ST, SDValue BetterChain);
500 SDValue replaceStoreOfFPConstant(StoreSDNode *ST);
501
502 SDValue visitSTORE(SDNode *N);
503 SDValue visitLIFETIME_END(SDNode *N);
504 SDValue visitINSERT_VECTOR_ELT(SDNode *N);
505 SDValue visitEXTRACT_VECTOR_ELT(SDNode *N);
506 SDValue visitBUILD_VECTOR(SDNode *N);
507 SDValue visitCONCAT_VECTORS(SDNode *N);
508 SDValue visitEXTRACT_SUBVECTOR(SDNode *N);
509 SDValue visitVECTOR_SHUFFLE(SDNode *N);
510 SDValue visitSCALAR_TO_VECTOR(SDNode *N);
511 SDValue visitINSERT_SUBVECTOR(SDNode *N);
512 SDValue visitMLOAD(SDNode *N);
513 SDValue visitMSTORE(SDNode *N);
514 SDValue visitMGATHER(SDNode *N);
515 SDValue visitMSCATTER(SDNode *N);
516 SDValue visitFP_TO_FP16(SDNode *N);
517 SDValue visitFP16_TO_FP(SDNode *N);
518 SDValue visitVECREDUCE(SDNode *N);
519
520 SDValue visitFADDForFMACombine(SDNode *N);
521 SDValue visitFSUBForFMACombine(SDNode *N);
522 SDValue visitFMULForFMADistributiveCombine(SDNode *N);
523
524 SDValue XformToShuffleWithZero(SDNode *N);
525 bool reassociationCanBreakAddressingModePattern(unsigned Opc,
526 const SDLoc &DL, SDValue N0,
527 SDValue N1);
528 SDValue reassociateOpsCommutative(unsigned Opc, const SDLoc &DL, SDValue N0,
529 SDValue N1);
530 SDValue reassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0,
531 SDValue N1, SDNodeFlags Flags);
532
533 SDValue visitShiftByConstant(SDNode *N);
534
535 SDValue foldSelectOfConstants(SDNode *N);
536 SDValue foldVSelectOfConstants(SDNode *N);
537 SDValue foldBinOpIntoSelect(SDNode *BO);
538 bool SimplifySelectOps(SDNode *SELECT, SDValue LHS, SDValue RHS);
539 SDValue hoistLogicOpWithSameOpcodeHands(SDNode *N);
540 SDValue SimplifySelect(const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2);
541 SDValue SimplifySelectCC(const SDLoc &DL, SDValue N0, SDValue N1,
542 SDValue N2, SDValue N3, ISD::CondCode CC,
543 bool NotExtCompare = false);
544 SDValue convertSelectOfFPConstantsToLoadOffset(
545 const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2, SDValue N3,
546 ISD::CondCode CC);
547 SDValue foldSignChangeInBitcast(SDNode *N);
548 SDValue foldSelectCCToShiftAnd(const SDLoc &DL, SDValue N0, SDValue N1,
549 SDValue N2, SDValue N3, ISD::CondCode CC);
550 SDValue foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1,
551 const SDLoc &DL);
552 SDValue unfoldMaskedMerge(SDNode *N);
553 SDValue unfoldExtremeBitClearingToShifts(SDNode *N);
554 SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
555 const SDLoc &DL, bool foldBooleans);
556 SDValue rebuildSetCC(SDValue N);
557
558 bool isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
559 SDValue &CC, bool MatchStrict = false) const;
560 bool isOneUseSetCC(SDValue N) const;
561
562 SDValue SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
563 unsigned HiOp);
564 SDValue CombineConsecutiveLoads(SDNode *N, EVT VT);
565 SDValue CombineExtLoad(SDNode *N);
566 SDValue CombineZExtLogicopShiftLoad(SDNode *N);
567 SDValue combineRepeatedFPDivisors(SDNode *N);
568 SDValue combineInsertEltToShuffle(SDNode *N, unsigned InsIndex);
569 SDValue ConstantFoldBITCASTofBUILD_VECTOR(SDNode *, EVT);
570 SDValue BuildSDIV(SDNode *N);
571 SDValue BuildSDIVPow2(SDNode *N);
572 SDValue BuildUDIV(SDNode *N);
573 SDValue BuildLogBase2(SDValue V, const SDLoc &DL);
574 SDValue BuildDivEstimate(SDValue N, SDValue Op, SDNodeFlags Flags);
575 SDValue buildRsqrtEstimate(SDValue Op, SDNodeFlags Flags);
576 SDValue buildSqrtEstimate(SDValue Op, SDNodeFlags Flags);
577 SDValue buildSqrtEstimateImpl(SDValue Op, SDNodeFlags Flags, bool Recip);
578 SDValue buildSqrtNROneConst(SDValue Arg, SDValue Est, unsigned Iterations,
579 SDNodeFlags Flags, bool Reciprocal);
580 SDValue buildSqrtNRTwoConst(SDValue Arg, SDValue Est, unsigned Iterations,
581 SDNodeFlags Flags, bool Reciprocal);
582 SDValue MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
583 bool DemandHighBits = true);
584 SDValue MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1);
585 SDValue MatchRotatePosNeg(SDValue Shifted, SDValue Pos, SDValue Neg,
586 SDValue InnerPos, SDValue InnerNeg,
587 unsigned PosOpcode, unsigned NegOpcode,
588 const SDLoc &DL);
589 SDValue MatchFunnelPosNeg(SDValue N0, SDValue N1, SDValue Pos, SDValue Neg,
590 SDValue InnerPos, SDValue InnerNeg,
591 unsigned PosOpcode, unsigned NegOpcode,
592 const SDLoc &DL);
593 SDValue MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL);
594 SDValue MatchLoadCombine(SDNode *N);
595 SDValue mergeTruncStores(StoreSDNode *N);
596 SDValue ReduceLoadWidth(SDNode *N);
597 SDValue ReduceLoadOpStoreWidth(SDNode *N);
598 SDValue splitMergedValStore(StoreSDNode *ST);
599 SDValue TransformFPLoadStorePair(SDNode *N);
600 SDValue convertBuildVecZextToZext(SDNode *N);
601 SDValue reduceBuildVecExtToExtBuildVec(SDNode *N);
602 SDValue reduceBuildVecTruncToBitCast(SDNode *N);
603 SDValue reduceBuildVecToShuffle(SDNode *N);
604 SDValue createBuildVecShuffle(const SDLoc &DL, SDNode *N,
605 ArrayRef<int> VectorMask, SDValue VecIn1,
606 SDValue VecIn2, unsigned LeftIdx,
607 bool DidSplitVec);
608 SDValue matchVSelectOpSizesWithSetCC(SDNode *Cast);
609
610 /// Walk up chain skipping non-aliasing memory nodes,
611 /// looking for aliasing nodes and adding them to the Aliases vector.
612 void GatherAllAliases(SDNode *N, SDValue OriginalChain,
613 SmallVectorImpl<SDValue> &Aliases);
614
615 /// Return true if there is any possibility that the two addresses overlap.
616 bool isAlias(SDNode *Op0, SDNode *Op1) const;
617
618 /// Walk up chain skipping non-aliasing memory nodes, looking for a better
619 /// chain (aliasing node.)
620 SDValue FindBetterChain(SDNode *N, SDValue Chain);
621
622 /// Try to replace a store and any possibly adjacent stores on
623 /// consecutive chains with better chains. Return true only if St is
624 /// replaced.
625 ///
626 /// Notice that other chains may still be replaced even if the function
627 /// returns false.
628 bool findBetterNeighborChains(StoreSDNode *St);
629
630 // Helper for findBetterNeighborChains. Walk up store chain add additional
631 // chained stores that do not overlap and can be parallelized.
632 bool parallelizeChainedStores(StoreSDNode *St);
633
634 /// Holds a pointer to an LSBaseSDNode as well as information on where it
635 /// is located in a sequence of memory operations connected by a chain.
636 struct MemOpLink {
637 // Ptr to the mem node.
638 LSBaseSDNode *MemNode;
639
640 // Offset from the base ptr.
641 int64_t OffsetFromBase;
642
643 MemOpLink(LSBaseSDNode *N, int64_t Offset)
644 : MemNode(N), OffsetFromBase(Offset) {}
645 };
646
647 // Classify the origin of a stored value.
648 enum class StoreSource { Unknown, Constant, Extract, Load };
649 StoreSource getStoreSource(SDValue StoreVal) {
650 switch (StoreVal.getOpcode()) {
651 case ISD::Constant:
652 case ISD::ConstantFP:
653 return StoreSource::Constant;
654 case ISD::EXTRACT_VECTOR_ELT:
655 case ISD::EXTRACT_SUBVECTOR:
656 return StoreSource::Extract;
657 case ISD::LOAD:
658 return StoreSource::Load;
659 default:
660 return StoreSource::Unknown;
661 }
662 }
663
664 /// This is a helper function for visitMUL to check the profitability
665 /// of folding (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2).
666 /// MulNode is the original multiply, AddNode is (add x, c1),
667 /// and ConstNode is c2.
668 bool isMulAddWithConstProfitable(SDNode *MulNode,
669 SDValue &AddNode,
670 SDValue &ConstNode);
671
672 /// This is a helper function for visitAND and visitZERO_EXTEND. Returns
673 /// true if the (and (load x) c) pattern matches an extload. ExtVT returns
674 /// the type of the loaded value to be extended.
675 bool isAndLoadExtLoad(ConstantSDNode *AndC, LoadSDNode *LoadN,
676 EVT LoadResultTy, EVT &ExtVT);
677
678 /// Helper function to calculate whether the given Load/Store can have its
679 /// width reduced to ExtVT.
680 bool isLegalNarrowLdSt(LSBaseSDNode *LDSTN, ISD::LoadExtType ExtType,
681 EVT &MemVT, unsigned ShAmt = 0);
682
683 /// Used by BackwardsPropagateMask to find suitable loads.
684 bool SearchForAndLoads(SDNode *N, SmallVectorImpl<LoadSDNode*> &Loads,
685 SmallPtrSetImpl<SDNode*> &NodesWithConsts,
686 ConstantSDNode *Mask, SDNode *&NodeToMask);
687 /// Attempt to propagate a given AND node back to load leaves so that they
688 /// can be combined into narrow loads.
689 bool BackwardsPropagateMask(SDNode *N);
690
691 /// Helper function for mergeConsecutiveStores which merges the component
692 /// store chains.
693 SDValue getMergeStoreChains(SmallVectorImpl<MemOpLink> &StoreNodes,
694 unsigned NumStores);
695
696 /// This is a helper function for mergeConsecutiveStores. When the source
697 /// elements of the consecutive stores are all constants or all extracted
698 /// vector elements, try to merge them into one larger store introducing
699 /// bitcasts if necessary. \return True if a merged store was created.
700 bool mergeStoresOfConstantsOrVecElts(SmallVectorImpl<MemOpLink> &StoreNodes,
701 EVT MemVT, unsigned NumStores,
702 bool IsConstantSrc, bool UseVector,
703 bool UseTrunc);
704
705 /// This is a helper function for mergeConsecutiveStores. Stores that
706 /// potentially may be merged with St are placed in StoreNodes. RootNode is
707 /// a chain predecessor to all store candidates.
708 void getStoreMergeCandidates(StoreSDNode *St,
709 SmallVectorImpl<MemOpLink> &StoreNodes,
710 SDNode *&Root);
711
712 /// Helper function for mergeConsecutiveStores. Checks if candidate stores
713 /// have indirect dependency through their operands. RootNode is the
714 /// predecessor to all stores calculated by getStoreMergeCandidates and is
715 /// used to prune the dependency check. \return True if safe to merge.
716 bool checkMergeStoreCandidatesForDependencies(
717 SmallVectorImpl<MemOpLink> &StoreNodes, unsigned NumStores,
718 SDNode *RootNode);
719
720 /// This is a helper function for mergeConsecutiveStores. Given a list of
721 /// store candidates, find the first N that are consecutive in memory.
722 /// Returns 0 if there are not at least 2 consecutive stores to try merging.
723 unsigned getConsecutiveStores(SmallVectorImpl<MemOpLink> &StoreNodes,
724 int64_t ElementSizeBytes) const;
725
726 /// This is a helper function for mergeConsecutiveStores. It is used for
727 /// store chains that are composed entirely of constant values.
728 bool tryStoreMergeOfConstants(SmallVectorImpl<MemOpLink> &StoreNodes,
729 unsigned NumConsecutiveStores,
730 EVT MemVT, SDNode *Root, bool AllowVectors);
731
732 /// This is a helper function for mergeConsecutiveStores. It is used for
733 /// store chains that are composed entirely of extracted vector elements.
734 /// When extracting multiple vector elements, try to store them in one
735 /// vector store rather than a sequence of scalar stores.
736 bool tryStoreMergeOfExtracts(SmallVectorImpl<MemOpLink> &StoreNodes,
737 unsigned NumConsecutiveStores, EVT MemVT,
738 SDNode *Root);
739
740 /// This is a helper function for mergeConsecutiveStores. It is used for
741 /// store chains that are composed entirely of loaded values.
742 bool tryStoreMergeOfLoads(SmallVectorImpl<MemOpLink> &StoreNodes,
743 unsigned NumConsecutiveStores, EVT MemVT,
744 SDNode *Root, bool AllowVectors,
745 bool IsNonTemporalStore, bool IsNonTemporalLoad);
746
747 /// Merge consecutive store operations into a wide store.
748 /// This optimization uses wide integers or vectors when possible.
749 /// \return true if stores were merged.
750 bool mergeConsecutiveStores(StoreSDNode *St);
751
752 /// Try to transform a truncation where C is a constant:
753 /// (trunc (and X, C)) -> (and (trunc X), (trunc C))
754 ///
755 /// \p N needs to be a truncation and its first operand an AND. Other
756 /// requirements are checked by the function (e.g. that trunc is
757 /// single-use) and if missed an empty SDValue is returned.
758 SDValue distributeTruncateThroughAnd(SDNode *N);
759
760 /// Helper function to determine whether the target supports operation
761 /// given by \p Opcode for type \p VT, that is, whether the operation
762 /// is legal or custom before legalizing operations, and whether is
763 /// legal (but not custom) after legalization.
764 bool hasOperation(unsigned Opcode, EVT VT) {
765 return TLI.isOperationLegalOrCustom(Opcode, VT, LegalOperations);
766 }
767
768 public:
769 /// Runs the dag combiner on all nodes in the work list
770 void Run(CombineLevel AtLevel);
771
772 SelectionDAG &getDAG() const { return DAG; }
773
774 /// Returns a type large enough to hold any valid shift amount - before type
775 /// legalization these can be huge.
776 EVT getShiftAmountTy(EVT LHSTy) {
777 assert(LHSTy.isInteger() && "Shift amount is not an integer type!")((LHSTy.isInteger() && "Shift amount is not an integer type!"
) ? static_cast<void> (0) : __assert_fail ("LHSTy.isInteger() && \"Shift amount is not an integer type!\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 777, __PRETTY_FUNCTION__))
;
778 return TLI.getShiftAmountTy(LHSTy, DAG.getDataLayout(), LegalTypes);
779 }
780
781 /// This method returns true if we are running before type legalization or
782 /// if the specified VT is legal.
783 bool isTypeLegal(const EVT &VT) {
784 if (!LegalTypes) return true;
785 return TLI.isTypeLegal(VT);
786 }
787
788 /// Convenience wrapper around TargetLowering::getSetCCResultType
789 EVT getSetCCResultType(EVT VT) const {
790 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
791 }
792
793 void ExtendSetCCUses(const SmallVectorImpl<SDNode *> &SetCCs,
794 SDValue OrigLoad, SDValue ExtLoad,
795 ISD::NodeType ExtType);
796 };
797
798/// This class is a DAGUpdateListener that removes any deleted
799/// nodes from the worklist.
800class WorklistRemover : public SelectionDAG::DAGUpdateListener {
801 DAGCombiner &DC;
802
803public:
804 explicit WorklistRemover(DAGCombiner &dc)
805 : SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
806
807 void NodeDeleted(SDNode *N, SDNode *E) override {
808 DC.removeFromWorklist(N);
809 }
810};
811
812class WorklistInserter : public SelectionDAG::DAGUpdateListener {
813 DAGCombiner &DC;
814
815public:
816 explicit WorklistInserter(DAGCombiner &dc)
817 : SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
818
819 // FIXME: Ideally we could add N to the worklist, but this causes exponential
820 // compile time costs in large DAGs, e.g. Halide.
821 void NodeInserted(SDNode *N) override { DC.ConsiderForPruning(N); }
822};
823
824} // end anonymous namespace
825
826//===----------------------------------------------------------------------===//
827// TargetLowering::DAGCombinerInfo implementation
828//===----------------------------------------------------------------------===//
829
830void TargetLowering::DAGCombinerInfo::AddToWorklist(SDNode *N) {
831 ((DAGCombiner*)DC)->AddToWorklist(N);
832}
833
834SDValue TargetLowering::DAGCombinerInfo::
835CombineTo(SDNode *N, ArrayRef<SDValue> To, bool AddTo) {
836 return ((DAGCombiner*)DC)->CombineTo(N, &To[0], To.size(), AddTo);
837}
838
839SDValue TargetLowering::DAGCombinerInfo::
840CombineTo(SDNode *N, SDValue Res, bool AddTo) {
841 return ((DAGCombiner*)DC)->CombineTo(N, Res, AddTo);
842}
843
844SDValue TargetLowering::DAGCombinerInfo::
845CombineTo(SDNode *N, SDValue Res0, SDValue Res1, bool AddTo) {
846 return ((DAGCombiner*)DC)->CombineTo(N, Res0, Res1, AddTo);
847}
848
849bool TargetLowering::DAGCombinerInfo::
850recursivelyDeleteUnusedNodes(SDNode *N) {
851 return ((DAGCombiner*)DC)->recursivelyDeleteUnusedNodes(N);
852}
853
854void TargetLowering::DAGCombinerInfo::
855CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
856 return ((DAGCombiner*)DC)->CommitTargetLoweringOpt(TLO);
857}
858
859//===----------------------------------------------------------------------===//
860// Helper Functions
861//===----------------------------------------------------------------------===//
862
863void DAGCombiner::deleteAndRecombine(SDNode *N) {
864 removeFromWorklist(N);
865
866 // If the operands of this node are only used by the node, they will now be
867 // dead. Make sure to re-visit them and recursively delete dead nodes.
868 for (const SDValue &Op : N->ops())
869 // For an operand generating multiple values, one of the values may
870 // become dead allowing further simplification (e.g. split index
871 // arithmetic from an indexed load).
872 if (Op->hasOneUse() || Op->getNumValues() > 1)
873 AddToWorklist(Op.getNode());
874
875 DAG.DeleteNode(N);
876}
877
878// APInts must be the same size for most operations, this helper
879// function zero extends the shorter of the pair so that they match.
880// We provide an Offset so that we can create bitwidths that won't overflow.
881static void zeroExtendToMatch(APInt &LHS, APInt &RHS, unsigned Offset = 0) {
882 unsigned Bits = Offset + std::max(LHS.getBitWidth(), RHS.getBitWidth());
883 LHS = LHS.zextOrSelf(Bits);
884 RHS = RHS.zextOrSelf(Bits);
885}
886
887// Return true if this node is a setcc, or is a select_cc
888// that selects between the target values used for true and false, making it
889// equivalent to a setcc. Also, set the incoming LHS, RHS, and CC references to
890// the appropriate nodes based on the type of node we are checking. This
891// simplifies life a bit for the callers.
892bool DAGCombiner::isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
893 SDValue &CC, bool MatchStrict) const {
894 if (N.getOpcode() == ISD::SETCC) {
895 LHS = N.getOperand(0);
896 RHS = N.getOperand(1);
897 CC = N.getOperand(2);
898 return true;
899 }
900
901 if (MatchStrict &&
902 (N.getOpcode() == ISD::STRICT_FSETCC ||
903 N.getOpcode() == ISD::STRICT_FSETCCS)) {
904 LHS = N.getOperand(1);
905 RHS = N.getOperand(2);
906 CC = N.getOperand(3);
907 return true;
908 }
909
910 if (N.getOpcode() != ISD::SELECT_CC ||
911 !TLI.isConstTrueVal(N.getOperand(2).getNode()) ||
912 !TLI.isConstFalseVal(N.getOperand(3).getNode()))
913 return false;
914
915 if (TLI.getBooleanContents(N.getValueType()) ==
916 TargetLowering::UndefinedBooleanContent)
917 return false;
918
919 LHS = N.getOperand(0);
920 RHS = N.getOperand(1);
921 CC = N.getOperand(4);
922 return true;
923}
924
925/// Return true if this is a SetCC-equivalent operation with only one use.
926/// If this is true, it allows the users to invert the operation for free when
927/// it is profitable to do so.
928bool DAGCombiner::isOneUseSetCC(SDValue N) const {
929 SDValue N0, N1, N2;
930 if (isSetCCEquivalent(N, N0, N1, N2) && N.getNode()->hasOneUse())
931 return true;
932 return false;
933}
934
935static bool isConstantSplatVectorMaskForType(SDNode *N, EVT ScalarTy) {
936 if (!ScalarTy.isSimple())
937 return false;
938
939 uint64_t MaskForTy = 0ULL;
940 switch (ScalarTy.getSimpleVT().SimpleTy) {
941 case MVT::i8:
942 MaskForTy = 0xFFULL;
943 break;
944 case MVT::i16:
945 MaskForTy = 0xFFFFULL;
946 break;
947 case MVT::i32:
948 MaskForTy = 0xFFFFFFFFULL;
949 break;
950 default:
951 return false;
952 break;
953 }
954
955 APInt Val;
956 if (ISD::isConstantSplatVector(N, Val))
957 return Val.getLimitedValue() == MaskForTy;
958
959 return false;
960}
961
962// Determines if it is a constant integer or a build vector of constant
963// integers (and undefs).
964// Do not permit build vector implicit truncation.
965static bool isConstantOrConstantVector(SDValue N, bool NoOpaques = false) {
966 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N))
967 return !(Const->isOpaque() && NoOpaques);
968 if (N.getOpcode() != ISD::BUILD_VECTOR)
969 return false;
970 unsigned BitWidth = N.getScalarValueSizeInBits();
971 for (const SDValue &Op : N->op_values()) {
972 if (Op.isUndef())
973 continue;
974 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Op);
975 if (!Const || Const->getAPIntValue().getBitWidth() != BitWidth ||
976 (Const->isOpaque() && NoOpaques))
977 return false;
978 }
979 return true;
980}
981
982// Determines if a BUILD_VECTOR is composed of all-constants possibly mixed with
983// undef's.
984static bool isAnyConstantBuildVector(SDValue V, bool NoOpaques = false) {
985 if (V.getOpcode() != ISD::BUILD_VECTOR)
986 return false;
987 return isConstantOrConstantVector(V, NoOpaques) ||
988 ISD::isBuildVectorOfConstantFPSDNodes(V.getNode());
989}
990
991// Determine if this an indexed load with an opaque target constant index.
992static bool canSplitIdx(LoadSDNode *LD) {
993 return MaySplitLoadIndex &&
994 (LD->getOperand(2).getOpcode() != ISD::TargetConstant ||
995 !cast<ConstantSDNode>(LD->getOperand(2))->isOpaque());
996}
997
998bool DAGCombiner::reassociationCanBreakAddressingModePattern(unsigned Opc,
999 const SDLoc &DL,
1000 SDValue N0,
1001 SDValue N1) {
1002 // Currently this only tries to ensure we don't undo the GEP splits done by
1003 // CodeGenPrepare when shouldConsiderGEPOffsetSplit is true. To ensure this,
1004 // we check if the following transformation would be problematic:
1005 // (load/store (add, (add, x, offset1), offset2)) ->
1006 // (load/store (add, x, offset1+offset2)).
1007
1008 if (Opc != ISD::ADD || N0.getOpcode() != ISD::ADD)
1009 return false;
1010
1011 if (N0.hasOneUse())
1012 return false;
1013
1014 auto *C1 = dyn_cast<ConstantSDNode>(N0.getOperand(1));
1015 auto *C2 = dyn_cast<ConstantSDNode>(N1);
1016 if (!C1 || !C2)
1017 return false;
1018
1019 const APInt &C1APIntVal = C1->getAPIntValue();
1020 const APInt &C2APIntVal = C2->getAPIntValue();
1021 if (C1APIntVal.getBitWidth() > 64 || C2APIntVal.getBitWidth() > 64)
1022 return false;
1023
1024 const APInt CombinedValueIntVal = C1APIntVal + C2APIntVal;
1025 if (CombinedValueIntVal.getBitWidth() > 64)
1026 return false;
1027 const int64_t CombinedValue = CombinedValueIntVal.getSExtValue();
1028
1029 for (SDNode *Node : N0->uses()) {
1030 auto LoadStore = dyn_cast<MemSDNode>(Node);
1031 if (LoadStore) {
1032 // Is x[offset2] already not a legal addressing mode? If so then
1033 // reassociating the constants breaks nothing (we test offset2 because
1034 // that's the one we hope to fold into the load or store).
1035 TargetLoweringBase::AddrMode AM;
1036 AM.HasBaseReg = true;
1037 AM.BaseOffs = C2APIntVal.getSExtValue();
1038 EVT VT = LoadStore->getMemoryVT();
1039 unsigned AS = LoadStore->getAddressSpace();
1040 Type *AccessTy = VT.getTypeForEVT(*DAG.getContext());
1041 if (!TLI.isLegalAddressingMode(DAG.getDataLayout(), AM, AccessTy, AS))
1042 continue;
1043
1044 // Would x[offset1+offset2] still be a legal addressing mode?
1045 AM.BaseOffs = CombinedValue;
1046 if (!TLI.isLegalAddressingMode(DAG.getDataLayout(), AM, AccessTy, AS))
1047 return true;
1048 }
1049 }
1050
1051 return false;
1052}
1053
1054// Helper for DAGCombiner::reassociateOps. Try to reassociate an expression
1055// such as (Opc N0, N1), if \p N0 is the same kind of operation as \p Opc.
1056SDValue DAGCombiner::reassociateOpsCommutative(unsigned Opc, const SDLoc &DL,
1057 SDValue N0, SDValue N1) {
1058 EVT VT = N0.getValueType();
1059
1060 if (N0.getOpcode() != Opc)
1061 return SDValue();
1062
1063 if (DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1))) {
1064 if (DAG.isConstantIntBuildVectorOrConstantInt(N1)) {
1065 // Reassociate: (op (op x, c1), c2) -> (op x, (op c1, c2))
1066 if (SDValue OpNode =
1067 DAG.FoldConstantArithmetic(Opc, DL, VT, {N0.getOperand(1), N1}))
1068 return DAG.getNode(Opc, DL, VT, N0.getOperand(0), OpNode);
1069 return SDValue();
1070 }
1071 if (N0.hasOneUse()) {
1072 // Reassociate: (op (op x, c1), y) -> (op (op x, y), c1)
1073 // iff (op x, c1) has one use
1074 SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N0.getOperand(0), N1);
1075 if (!OpNode.getNode())
1076 return SDValue();
1077 return DAG.getNode(Opc, DL, VT, OpNode, N0.getOperand(1));
1078 }
1079 }
1080 return SDValue();
1081}
1082
1083// Try to reassociate commutative binops.
1084SDValue DAGCombiner::reassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0,
1085 SDValue N1, SDNodeFlags Flags) {
1086 assert(TLI.isCommutativeBinOp(Opc) && "Operation not commutative.")((TLI.isCommutativeBinOp(Opc) && "Operation not commutative."
) ? static_cast<void> (0) : __assert_fail ("TLI.isCommutativeBinOp(Opc) && \"Operation not commutative.\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1086, __PRETTY_FUNCTION__))
;
1087
1088 // Floating-point reassociation is not allowed without loose FP math.
1089 if (N0.getValueType().isFloatingPoint() ||
1090 N1.getValueType().isFloatingPoint())
1091 if (!Flags.hasAllowReassociation() || !Flags.hasNoSignedZeros())
1092 return SDValue();
1093
1094 if (SDValue Combined = reassociateOpsCommutative(Opc, DL, N0, N1))
1095 return Combined;
1096 if (SDValue Combined = reassociateOpsCommutative(Opc, DL, N1, N0))
1097 return Combined;
1098 return SDValue();
1099}
1100
1101SDValue DAGCombiner::CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
1102 bool AddTo) {
1103 assert(N->getNumValues() == NumTo && "Broken CombineTo call!")((N->getNumValues() == NumTo && "Broken CombineTo call!"
) ? static_cast<void> (0) : __assert_fail ("N->getNumValues() == NumTo && \"Broken CombineTo call!\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1103, __PRETTY_FUNCTION__))
;
1104 ++NodesCombined;
1105 LLVM_DEBUG(dbgs() << "\nReplacing.1 "; N->dump(&DAG); dbgs() << "\nWith: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.1 "; N->dump
(&DAG); dbgs() << "\nWith: "; To[0].getNode()->dump
(&DAG); dbgs() << " and " << NumTo - 1 <<
" other values\n"; } } while (false)
1106 To[0].getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.1 "; N->dump
(&DAG); dbgs() << "\nWith: "; To[0].getNode()->dump
(&DAG); dbgs() << " and " << NumTo - 1 <<
" other values\n"; } } while (false)
1107 dbgs() << " and " << NumTo - 1 << " other values\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.1 "; N->dump
(&DAG); dbgs() << "\nWith: "; To[0].getNode()->dump
(&DAG); dbgs() << " and " << NumTo - 1 <<
" other values\n"; } } while (false)
;
1108 for (unsigned i = 0, e = NumTo; i != e; ++i)
1109 assert((!To[i].getNode() ||(((!To[i].getNode() || N->getValueType(i) == To[i].getValueType
()) && "Cannot combine value to value of different type!"
) ? static_cast<void> (0) : __assert_fail ("(!To[i].getNode() || N->getValueType(i) == To[i].getValueType()) && \"Cannot combine value to value of different type!\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1111, __PRETTY_FUNCTION__))
1110 N->getValueType(i) == To[i].getValueType()) &&(((!To[i].getNode() || N->getValueType(i) == To[i].getValueType
()) && "Cannot combine value to value of different type!"
) ? static_cast<void> (0) : __assert_fail ("(!To[i].getNode() || N->getValueType(i) == To[i].getValueType()) && \"Cannot combine value to value of different type!\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1111, __PRETTY_FUNCTION__))
1111 "Cannot combine value to value of different type!")(((!To[i].getNode() || N->getValueType(i) == To[i].getValueType
()) && "Cannot combine value to value of different type!"
) ? static_cast<void> (0) : __assert_fail ("(!To[i].getNode() || N->getValueType(i) == To[i].getValueType()) && \"Cannot combine value to value of different type!\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1111, __PRETTY_FUNCTION__))
;
1112
1113 WorklistRemover DeadNodes(*this);
1114 DAG.ReplaceAllUsesWith(N, To);
1115 if (AddTo) {
1116 // Push the new nodes and any users onto the worklist
1117 for (unsigned i = 0, e = NumTo; i != e; ++i) {
1118 if (To[i].getNode()) {
1119 AddToWorklist(To[i].getNode());
1120 AddUsersToWorklist(To[i].getNode());
1121 }
1122 }
1123 }
1124
1125 // Finally, if the node is now dead, remove it from the graph. The node
1126 // may not be dead if the replacement process recursively simplified to
1127 // something else needing this node.
1128 if (N->use_empty())
1129 deleteAndRecombine(N);
1130 return SDValue(N, 0);
1131}
1132
1133void DAGCombiner::
1134CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
1135 // Replace the old value with the new one.
1136 ++NodesCombined;
1137 LLVM_DEBUG(dbgs() << "\nReplacing.2 "; TLO.Old.getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.getNode
()->dump(&DAG); dbgs() << "\nWith: "; TLO.New.getNode
()->dump(&DAG); dbgs() << '\n'; } } while (false
)
1138 dbgs() << "\nWith: "; TLO.New.getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.getNode
()->dump(&DAG); dbgs() << "\nWith: "; TLO.New.getNode
()->dump(&DAG); dbgs() << '\n'; } } while (false
)
1139 dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.getNode
()->dump(&DAG); dbgs() << "\nWith: "; TLO.New.getNode
()->dump(&DAG); dbgs() << '\n'; } } while (false
)
;
1140
1141 // Replace all uses. If any nodes become isomorphic to other nodes and
1142 // are deleted, make sure to remove them from our worklist.
1143 WorklistRemover DeadNodes(*this);
1144 DAG.ReplaceAllUsesOfValueWith(TLO.Old, TLO.New);
1145
1146 // Push the new node and any (possibly new) users onto the worklist.
1147 AddToWorklistWithUsers(TLO.New.getNode());
1148
1149 // Finally, if the node is now dead, remove it from the graph. The node
1150 // may not be dead if the replacement process recursively simplified to
1151 // something else needing this node.
1152 if (TLO.Old.getNode()->use_empty())
1153 deleteAndRecombine(TLO.Old.getNode());
1154}
1155
1156/// Check the specified integer node value to see if it can be simplified or if
1157/// things it uses can be simplified by bit propagation. If so, return true.
1158bool DAGCombiner::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
1159 const APInt &DemandedElts,
1160 bool AssumeSingleUse) {
1161 TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations);
1162 KnownBits Known;
1163 if (!TLI.SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO, 0,
1164 AssumeSingleUse))
1165 return false;
1166
1167 // Revisit the node.
1168 AddToWorklist(Op.getNode());
1169
1170 CommitTargetLoweringOpt(TLO);
1171 return true;
1172}
1173
1174/// Check the specified vector node value to see if it can be simplified or
1175/// if things it uses can be simplified as it only uses some of the elements.
1176/// If so, return true.
1177bool DAGCombiner::SimplifyDemandedVectorElts(SDValue Op,
1178 const APInt &DemandedElts,
1179 bool AssumeSingleUse) {
1180 TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations);
1181 APInt KnownUndef, KnownZero;
1182 if (!TLI.SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero,
1183 TLO, 0, AssumeSingleUse))
1184 return false;
1185
1186 // Revisit the node.
1187 AddToWorklist(Op.getNode());
1188
1189 CommitTargetLoweringOpt(TLO);
1190 return true;
1191}
1192
1193void DAGCombiner::ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad) {
1194 SDLoc DL(Load);
1195 EVT VT = Load->getValueType(0);
1196 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, VT, SDValue(ExtLoad, 0));
1197
1198 LLVM_DEBUG(dbgs() << "\nReplacing.9 "; Load->dump(&DAG); dbgs() << "\nWith: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.9 "; Load->
dump(&DAG); dbgs() << "\nWith: "; Trunc.getNode()->
dump(&DAG); dbgs() << '\n'; } } while (false)
1199 Trunc.getNode()->dump(&DAG); dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.9 "; Load->
dump(&DAG); dbgs() << "\nWith: "; Trunc.getNode()->
dump(&DAG); dbgs() << '\n'; } } while (false)
;
1200 WorklistRemover DeadNodes(*this);
1201 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), Trunc);
1202 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), SDValue(ExtLoad, 1));
1203 deleteAndRecombine(Load);
1204 AddToWorklist(Trunc.getNode());
1205}
1206
1207SDValue DAGCombiner::PromoteOperand(SDValue Op, EVT PVT, bool &Replace) {
1208 Replace = false;
1209 SDLoc DL(Op);
1210 if (ISD::isUNINDEXEDLoad(Op.getNode())) {
1211 LoadSDNode *LD = cast<LoadSDNode>(Op);
1212 EVT MemVT = LD->getMemoryVT();
1213 ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD) ? ISD::EXTLOAD
1214 : LD->getExtensionType();
1215 Replace = true;
1216 return DAG.getExtLoad(ExtType, DL, PVT,
1217 LD->getChain(), LD->getBasePtr(),
1218 MemVT, LD->getMemOperand());
1219 }
1220
1221 unsigned Opc = Op.getOpcode();
1222 switch (Opc) {
1223 default: break;
1224 case ISD::AssertSext:
1225 if (SDValue Op0 = SExtPromoteOperand(Op.getOperand(0), PVT))
1226 return DAG.getNode(ISD::AssertSext, DL, PVT, Op0, Op.getOperand(1));
1227 break;
1228 case ISD::AssertZext:
1229 if (SDValue Op0 = ZExtPromoteOperand(Op.getOperand(0), PVT))
1230 return DAG.getNode(ISD::AssertZext, DL, PVT, Op0, Op.getOperand(1));
1231 break;
1232 case ISD::Constant: {
1233 unsigned ExtOpc =
1234 Op.getValueType().isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
1235 return DAG.getNode(ExtOpc, DL, PVT, Op);
1236 }
1237 }
1238
1239 if (!TLI.isOperationLegal(ISD::ANY_EXTEND, PVT))
1240 return SDValue();
1241 return DAG.getNode(ISD::ANY_EXTEND, DL, PVT, Op);
1242}
1243
1244SDValue DAGCombiner::SExtPromoteOperand(SDValue Op, EVT PVT) {
1245 if (!TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, PVT))
1246 return SDValue();
1247 EVT OldVT = Op.getValueType();
1248 SDLoc DL(Op);
1249 bool Replace = false;
1250 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
1251 if (!NewOp.getNode())
1252 return SDValue();
1253 AddToWorklist(NewOp.getNode());
1254
1255 if (Replace)
1256 ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
1257 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, NewOp.getValueType(), NewOp,
1258 DAG.getValueType(OldVT));
1259}
1260
1261SDValue DAGCombiner::ZExtPromoteOperand(SDValue Op, EVT PVT) {
1262 EVT OldVT = Op.getValueType();
1263 SDLoc DL(Op);
1264 bool Replace = false;
1265 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
1266 if (!NewOp.getNode())
1267 return SDValue();
1268 AddToWorklist(NewOp.getNode());
1269
1270 if (Replace)
1271 ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
1272 return DAG.getZeroExtendInReg(NewOp, DL, OldVT);
1273}
1274
1275/// Promote the specified integer binary operation if the target indicates it is
1276/// beneficial. e.g. On x86, it's usually better to promote i16 operations to
1277/// i32 since i16 instructions are longer.
1278SDValue DAGCombiner::PromoteIntBinOp(SDValue Op) {
1279 if (!LegalOperations)
1280 return SDValue();
1281
1282 EVT VT = Op.getValueType();
1283 if (VT.isVector() || !VT.isInteger())
1284 return SDValue();
1285
1286 // If operation type is 'undesirable', e.g. i16 on x86, consider
1287 // promoting it.
1288 unsigned Opc = Op.getOpcode();
1289 if (TLI.isTypeDesirableForOp(Opc, VT))
1290 return SDValue();
1291
1292 EVT PVT = VT;
1293 // Consult target whether it is a good idea to promote this operation and
1294 // what's the right type to promote it to.
1295 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1296 assert(PVT != VT && "Don't know what type to promote to!")((PVT != VT && "Don't know what type to promote to!")
? static_cast<void> (0) : __assert_fail ("PVT != VT && \"Don't know what type to promote to!\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1296, __PRETTY_FUNCTION__))
;
1297
1298 LLVM_DEBUG(dbgs() << "\nPromoting "; Op.getNode()->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; Op.getNode(
)->dump(&DAG); } } while (false)
;
1299
1300 bool Replace0 = false;
1301 SDValue N0 = Op.getOperand(0);
1302 SDValue NN0 = PromoteOperand(N0, PVT, Replace0);
1303
1304 bool Replace1 = false;
1305 SDValue N1 = Op.getOperand(1);
1306 SDValue NN1 = PromoteOperand(N1, PVT, Replace1);
1307 SDLoc DL(Op);
1308
1309 SDValue RV =
1310 DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, NN0, NN1));
1311
1312 // We are always replacing N0/N1's use in N and only need additional
1313 // replacements if there are additional uses.
1314 // Note: We are checking uses of the *nodes* (SDNode) rather than values
1315 // (SDValue) here because the node may reference multiple values
1316 // (for example, the chain value of a load node).
1317 Replace0 &= !N0->hasOneUse();
1318 Replace1 &= (N0 != N1) && !N1->hasOneUse();
1319
1320 // Combine Op here so it is preserved past replacements.
1321 CombineTo(Op.getNode(), RV);
1322
1323 // If operands have a use ordering, make sure we deal with
1324 // predecessor first.
1325 if (Replace0 && Replace1 && N0.getNode()->isPredecessorOf(N1.getNode())) {
1326 std::swap(N0, N1);
1327 std::swap(NN0, NN1);
1328 }
1329
1330 if (Replace0) {
1331 AddToWorklist(NN0.getNode());
1332 ReplaceLoadWithPromotedLoad(N0.getNode(), NN0.getNode());
1333 }
1334 if (Replace1) {
1335 AddToWorklist(NN1.getNode());
1336 ReplaceLoadWithPromotedLoad(N1.getNode(), NN1.getNode());
1337 }
1338 return Op;
1339 }
1340 return SDValue();
1341}
1342
1343/// Promote the specified integer shift operation if the target indicates it is
1344/// beneficial. e.g. On x86, it's usually better to promote i16 operations to
1345/// i32 since i16 instructions are longer.
1346SDValue DAGCombiner::PromoteIntShiftOp(SDValue Op) {
1347 if (!LegalOperations)
1348 return SDValue();
1349
1350 EVT VT = Op.getValueType();
1351 if (VT.isVector() || !VT.isInteger())
1352 return SDValue();
1353
1354 // If operation type is 'undesirable', e.g. i16 on x86, consider
1355 // promoting it.
1356 unsigned Opc = Op.getOpcode();
1357 if (TLI.isTypeDesirableForOp(Opc, VT))
1358 return SDValue();
1359
1360 EVT PVT = VT;
1361 // Consult target whether it is a good idea to promote this operation and
1362 // what's the right type to promote it to.
1363 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1364 assert(PVT != VT && "Don't know what type to promote to!")((PVT != VT && "Don't know what type to promote to!")
? static_cast<void> (0) : __assert_fail ("PVT != VT && \"Don't know what type to promote to!\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1364, __PRETTY_FUNCTION__))
;
1365
1366 LLVM_DEBUG(dbgs() << "\nPromoting "; Op.getNode()->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; Op.getNode(
)->dump(&DAG); } } while (false)
;
1367
1368 bool Replace = false;
1369 SDValue N0 = Op.getOperand(0);
1370 SDValue N1 = Op.getOperand(1);
1371 if (Opc == ISD::SRA)
1372 N0 = SExtPromoteOperand(N0, PVT);
1373 else if (Opc == ISD::SRL)
1374 N0 = ZExtPromoteOperand(N0, PVT);
1375 else
1376 N0 = PromoteOperand(N0, PVT, Replace);
1377
1378 if (!N0.getNode())
1379 return SDValue();
1380
1381 SDLoc DL(Op);
1382 SDValue RV =
1383 DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, N0, N1));
1384
1385 if (Replace)
1386 ReplaceLoadWithPromotedLoad(Op.getOperand(0).getNode(), N0.getNode());
1387
1388 // Deal with Op being deleted.
1389 if (Op && Op.getOpcode() != ISD::DELETED_NODE)
1390 return RV;
1391 }
1392 return SDValue();
1393}
1394
1395SDValue DAGCombiner::PromoteExtend(SDValue Op) {
1396 if (!LegalOperations)
1397 return SDValue();
1398
1399 EVT VT = Op.getValueType();
1400 if (VT.isVector() || !VT.isInteger())
1401 return SDValue();
1402
1403 // If operation type is 'undesirable', e.g. i16 on x86, consider
1404 // promoting it.
1405 unsigned Opc = Op.getOpcode();
1406 if (TLI.isTypeDesirableForOp(Opc, VT))
1407 return SDValue();
1408
1409 EVT PVT = VT;
1410 // Consult target whether it is a good idea to promote this operation and
1411 // what's the right type to promote it to.
1412 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1413 assert(PVT != VT && "Don't know what type to promote to!")((PVT != VT && "Don't know what type to promote to!")
? static_cast<void> (0) : __assert_fail ("PVT != VT && \"Don't know what type to promote to!\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1413, __PRETTY_FUNCTION__))
;
1414 // fold (aext (aext x)) -> (aext x)
1415 // fold (aext (zext x)) -> (zext x)
1416 // fold (aext (sext x)) -> (sext x)
1417 LLVM_DEBUG(dbgs() << "\nPromoting "; Op.getNode()->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; Op.getNode(
)->dump(&DAG); } } while (false)
;
1418 return DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, Op.getOperand(0));
1419 }
1420 return SDValue();
1421}
1422
1423bool DAGCombiner::PromoteLoad(SDValue Op) {
1424 if (!LegalOperations)
1425 return false;
1426
1427 if (!ISD::isUNINDEXEDLoad(Op.getNode()))
1428 return false;
1429
1430 EVT VT = Op.getValueType();
1431 if (VT.isVector() || !VT.isInteger())
1432 return false;
1433
1434 // If operation type is 'undesirable', e.g. i16 on x86, consider
1435 // promoting it.
1436 unsigned Opc = Op.getOpcode();
1437 if (TLI.isTypeDesirableForOp(Opc, VT))
1438 return false;
1439
1440 EVT PVT = VT;
1441 // Consult target whether it is a good idea to promote this operation and
1442 // what's the right type to promote it to.
1443 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1444 assert(PVT != VT && "Don't know what type to promote to!")((PVT != VT && "Don't know what type to promote to!")
? static_cast<void> (0) : __assert_fail ("PVT != VT && \"Don't know what type to promote to!\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1444, __PRETTY_FUNCTION__))
;
1445
1446 SDLoc DL(Op);
1447 SDNode *N = Op.getNode();
1448 LoadSDNode *LD = cast<LoadSDNode>(N);
1449 EVT MemVT = LD->getMemoryVT();
1450 ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD) ? ISD::EXTLOAD
1451 : LD->getExtensionType();
1452 SDValue NewLD = DAG.getExtLoad(ExtType, DL, PVT,
1453 LD->getChain(), LD->getBasePtr(),
1454 MemVT, LD->getMemOperand());
1455 SDValue Result = DAG.getNode(ISD::TRUNCATE, DL, VT, NewLD);
1456
1457 LLVM_DEBUG(dbgs() << "\nPromoting "; N->dump(&DAG); dbgs() << "\nTo: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; N->dump(
&DAG); dbgs() << "\nTo: "; Result.getNode()->dump
(&DAG); dbgs() << '\n'; } } while (false)
1458 Result.getNode()->dump(&DAG); dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; N->dump(
&DAG); dbgs() << "\nTo: "; Result.getNode()->dump
(&DAG); dbgs() << '\n'; } } while (false)
;
1459 WorklistRemover DeadNodes(*this);
1460 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result);
1461 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), NewLD.getValue(1));
1462 deleteAndRecombine(N);
1463 AddToWorklist(Result.getNode());
1464 return true;
1465 }
1466 return false;
1467}
1468
1469/// Recursively delete a node which has no uses and any operands for
1470/// which it is the only use.
1471///
1472/// Note that this both deletes the nodes and removes them from the worklist.
1473/// It also adds any nodes who have had a user deleted to the worklist as they
1474/// may now have only one use and subject to other combines.
1475bool DAGCombiner::recursivelyDeleteUnusedNodes(SDNode *N) {
1476 if (!N->use_empty())
1477 return false;
1478
1479 SmallSetVector<SDNode *, 16> Nodes;
1480 Nodes.insert(N);
1481 do {
1482 N = Nodes.pop_back_val();
1483 if (!N)
1484 continue;
1485
1486 if (N->use_empty()) {
1487 for (const SDValue &ChildN : N->op_values())
1488 Nodes.insert(ChildN.getNode());
1489
1490 removeFromWorklist(N);
1491 DAG.DeleteNode(N);
1492 } else {
1493 AddToWorklist(N);
1494 }
1495 } while (!Nodes.empty());
1496 return true;
1497}
1498
1499//===----------------------------------------------------------------------===//
1500// Main DAG Combiner implementation
1501//===----------------------------------------------------------------------===//
1502
1503void DAGCombiner::Run(CombineLevel AtLevel) {
1504 // set the instance variables, so that the various visit routines may use it.
1505 Level = AtLevel;
1506 LegalDAG = Level >= AfterLegalizeDAG;
1507 LegalOperations = Level >= AfterLegalizeVectorOps;
1508 LegalTypes = Level >= AfterLegalizeTypes;
1509
1510 WorklistInserter AddNodes(*this);
1511
1512 // Add all the dag nodes to the worklist.
1513 for (SDNode &Node : DAG.allnodes())
1514 AddToWorklist(&Node);
1515
1516 // Create a dummy node (which is not added to allnodes), that adds a reference
1517 // to the root node, preventing it from being deleted, and tracking any
1518 // changes of the root.
1519 HandleSDNode Dummy(DAG.getRoot());
1520
1521 // While we have a valid worklist entry node, try to combine it.
1522 while (SDNode *N = getNextWorklistEntry()) {
1523 // If N has no uses, it is dead. Make sure to revisit all N's operands once
1524 // N is deleted from the DAG, since they too may now be dead or may have a
1525 // reduced number of uses, allowing other xforms.
1526 if (recursivelyDeleteUnusedNodes(N))
1527 continue;
1528
1529 WorklistRemover DeadNodes(*this);
1530
1531 // If this combine is running after legalizing the DAG, re-legalize any
1532 // nodes pulled off the worklist.
1533 if (LegalDAG) {
1534 SmallSetVector<SDNode *, 16> UpdatedNodes;
1535 bool NIsValid = DAG.LegalizeOp(N, UpdatedNodes);
1536
1537 for (SDNode *LN : UpdatedNodes)
1538 AddToWorklistWithUsers(LN);
1539
1540 if (!NIsValid)
1541 continue;
1542 }
1543
1544 LLVM_DEBUG(dbgs() << "\nCombining: "; N->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nCombining: "; N->dump
(&DAG); } } while (false)
;
1545
1546 // Add any operands of the new node which have not yet been combined to the
1547 // worklist as well. Because the worklist uniques things already, this
1548 // won't repeatedly process the same operand.
1549 CombinedNodes.insert(N);
1550 for (const SDValue &ChildN : N->op_values())
1551 if (!CombinedNodes.count(ChildN.getNode()))
1552 AddToWorklist(ChildN.getNode());
1553
1554 SDValue RV = combine(N);
1555
1556 if (!RV.getNode())
1557 continue;
1558
1559 ++NodesCombined;
1560
1561 // If we get back the same node we passed in, rather than a new node or
1562 // zero, we know that the node must have defined multiple values and
1563 // CombineTo was used. Since CombineTo takes care of the worklist
1564 // mechanics for us, we have no work to do in this case.
1565 if (RV.getNode() == N)
1566 continue;
1567
1568 assert(N->getOpcode() != ISD::DELETED_NODE &&((N->getOpcode() != ISD::DELETED_NODE && RV.getOpcode
() != ISD::DELETED_NODE && "Node was deleted but visit returned new node!"
) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && RV.getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned new node!\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1570, __PRETTY_FUNCTION__))
1569 RV.getOpcode() != ISD::DELETED_NODE &&((N->getOpcode() != ISD::DELETED_NODE && RV.getOpcode
() != ISD::DELETED_NODE && "Node was deleted but visit returned new node!"
) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && RV.getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned new node!\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1570, __PRETTY_FUNCTION__))
1570 "Node was deleted but visit returned new node!")((N->getOpcode() != ISD::DELETED_NODE && RV.getOpcode
() != ISD::DELETED_NODE && "Node was deleted but visit returned new node!"
) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && RV.getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned new node!\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1570, __PRETTY_FUNCTION__))
;
1571
1572 LLVM_DEBUG(dbgs() << " ... into: "; RV.getNode()->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << " ... into: "; RV.getNode()
->dump(&DAG); } } while (false)
;
1573
1574 if (N->getNumValues() == RV.getNode()->getNumValues())
1575 DAG.ReplaceAllUsesWith(N, RV.getNode());
1576 else {
1577 assert(N->getValueType(0) == RV.getValueType() &&((N->getValueType(0) == RV.getValueType() && N->
getNumValues() == 1 && "Type mismatch") ? static_cast
<void> (0) : __assert_fail ("N->getValueType(0) == RV.getValueType() && N->getNumValues() == 1 && \"Type mismatch\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1578, __PRETTY_FUNCTION__))
1578 N->getNumValues() == 1 && "Type mismatch")((N->getValueType(0) == RV.getValueType() && N->
getNumValues() == 1 && "Type mismatch") ? static_cast
<void> (0) : __assert_fail ("N->getValueType(0) == RV.getValueType() && N->getNumValues() == 1 && \"Type mismatch\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1578, __PRETTY_FUNCTION__))
;
1579 DAG.ReplaceAllUsesWith(N, &RV);
1580 }
1581
1582 // Push the new node and any users onto the worklist. Omit this if the
1583 // new node is the EntryToken (e.g. if a store managed to get optimized
1584 // out), because re-visiting the EntryToken and its users will not uncover
1585 // any additional opportunities, but there may be a large number of such
1586 // users, potentially causing compile time explosion.
1587 if (RV.getOpcode() != ISD::EntryToken) {
1588 AddToWorklist(RV.getNode());
1589 AddUsersToWorklist(RV.getNode());
1590 }
1591
1592 // Finally, if the node is now dead, remove it from the graph. The node
1593 // may not be dead if the replacement process recursively simplified to
1594 // something else needing this node. This will also take care of adding any
1595 // operands which have lost a user to the worklist.
1596 recursivelyDeleteUnusedNodes(N);
1597 }
1598
1599 // If the root changed (e.g. it was a dead load, update the root).
1600 DAG.setRoot(Dummy.getValue());
1601 DAG.RemoveDeadNodes();
1602}
1603
1604SDValue DAGCombiner::visit(SDNode *N) {
1605 switch (N->getOpcode()) {
1606 default: break;
1607 case ISD::TokenFactor: return visitTokenFactor(N);
1608 case ISD::MERGE_VALUES: return visitMERGE_VALUES(N);
1609 case ISD::ADD: return visitADD(N);
1610 case ISD::SUB: return visitSUB(N);
1611 case ISD::SADDSAT:
1612 case ISD::UADDSAT: return visitADDSAT(N);
1613 case ISD::SSUBSAT:
1614 case ISD::USUBSAT: return visitSUBSAT(N);
1615 case ISD::ADDC: return visitADDC(N);
1616 case ISD::SADDO:
1617 case ISD::UADDO: return visitADDO(N);
1618 case ISD::SUBC: return visitSUBC(N);
1619 case ISD::SSUBO:
1620 case ISD::USUBO: return visitSUBO(N);
1621 case ISD::ADDE: return visitADDE(N);
1622 case ISD::ADDCARRY: return visitADDCARRY(N);
1623 case ISD::SADDO_CARRY: return visitSADDO_CARRY(N);
1624 case ISD::SUBE: return visitSUBE(N);
1625 case ISD::SUBCARRY: return visitSUBCARRY(N);
1626 case ISD::SSUBO_CARRY: return visitSSUBO_CARRY(N);
1627 case ISD::SMULFIX:
1628 case ISD::SMULFIXSAT:
1629 case ISD::UMULFIX:
1630 case ISD::UMULFIXSAT: return visitMULFIX(N);
1631 case ISD::MUL: return visitMUL(N);
1632 case ISD::SDIV: return visitSDIV(N);
1633 case ISD::UDIV: return visitUDIV(N);
1634 case ISD::SREM:
1635 case ISD::UREM: return visitREM(N);
1636 case ISD::MULHU: return visitMULHU(N);
1637 case ISD::MULHS: return visitMULHS(N);
1638 case ISD::SMUL_LOHI: return visitSMUL_LOHI(N);
1639 case ISD::UMUL_LOHI: return visitUMUL_LOHI(N);
1640 case ISD::SMULO:
1641 case ISD::UMULO: return visitMULO(N);
1642 case ISD::SMIN:
1643 case ISD::SMAX:
1644 case ISD::UMIN:
1645 case ISD::UMAX: return visitIMINMAX(N);
1646 case ISD::AND: return visitAND(N);
1647 case ISD::OR: return visitOR(N);
1648 case ISD::XOR: return visitXOR(N);
1649 case ISD::SHL: return visitSHL(N);
1650 case ISD::SRA: return visitSRA(N);
1651 case ISD::SRL: return visitSRL(N);
1652 case ISD::ROTR:
1653 case ISD::ROTL: return visitRotate(N);
1654 case ISD::FSHL:
1655 case ISD::FSHR: return visitFunnelShift(N);
1656 case ISD::ABS: return visitABS(N);
1657 case ISD::BSWAP: return visitBSWAP(N);
1658 case ISD::BITREVERSE: return visitBITREVERSE(N);
1659 case ISD::CTLZ: return visitCTLZ(N);
1660 case ISD::CTLZ_ZERO_UNDEF: return visitCTLZ_ZERO_UNDEF(N);
1661 case ISD::CTTZ: return visitCTTZ(N);
1662 case ISD::CTTZ_ZERO_UNDEF: return visitCTTZ_ZERO_UNDEF(N);
1663 case ISD::CTPOP: return visitCTPOP(N);
1664 case ISD::SELECT: return visitSELECT(N);
1665 case ISD::VSELECT: return visitVSELECT(N);
1666 case ISD::SELECT_CC: return visitSELECT_CC(N);
1667 case ISD::SETCC: return visitSETCC(N);
1668 case ISD::SETCCCARRY: return visitSETCCCARRY(N);
1669 case ISD::SIGN_EXTEND: return visitSIGN_EXTEND(N);
1670 case ISD::ZERO_EXTEND: return visitZERO_EXTEND(N);
1671 case ISD::ANY_EXTEND: return visitANY_EXTEND(N);
1672 case ISD::AssertSext:
1673 case ISD::AssertZext: return visitAssertExt(N);
1674 case ISD::AssertAlign: return visitAssertAlign(N);
1675 case ISD::SIGN_EXTEND_INREG: return visitSIGN_EXTEND_INREG(N);
1676 case ISD::SIGN_EXTEND_VECTOR_INREG: return visitSIGN_EXTEND_VECTOR_INREG(N);
1677 case ISD::ZERO_EXTEND_VECTOR_INREG: return visitZERO_EXTEND_VECTOR_INREG(N);
1678 case ISD::TRUNCATE: return visitTRUNCATE(N);
1679 case ISD::BITCAST: return visitBITCAST(N);
1680 case ISD::BUILD_PAIR: return visitBUILD_PAIR(N);
1681 case ISD::FADD: return visitFADD(N);
1682 case ISD::STRICT_FADD: return visitSTRICT_FADD(N);
1683 case ISD::FSUB: return visitFSUB(N);
1684 case ISD::FMUL: return visitFMUL(N);
1685 case ISD::FMA: return visitFMA(N);
1686 case ISD::FDIV: return visitFDIV(N);
1687 case ISD::FREM: return visitFREM(N);
1688 case ISD::FSQRT: return visitFSQRT(N);
1689 case ISD::FCOPYSIGN: return visitFCOPYSIGN(N);
1690 case ISD::FPOW: return visitFPOW(N);
1691 case ISD::SINT_TO_FP: return visitSINT_TO_FP(N);
1692 case ISD::UINT_TO_FP: return visitUINT_TO_FP(N);
1693 case ISD::FP_TO_SINT: return visitFP_TO_SINT(N);
1694 case ISD::FP_TO_UINT: return visitFP_TO_UINT(N);
1695 case ISD::FP_ROUND: return visitFP_ROUND(N);
1696 case ISD::FP_EXTEND: return visitFP_EXTEND(N);
1697 case ISD::FNEG: return visitFNEG(N);
1698 case ISD::FABS: return visitFABS(N);
1699 case ISD::FFLOOR: return visitFFLOOR(N);
1700 case ISD::FMINNUM: return visitFMINNUM(N);
1701 case ISD::FMAXNUM: return visitFMAXNUM(N);
1702 case ISD::FMINIMUM: return visitFMINIMUM(N);
1703 case ISD::FMAXIMUM: return visitFMAXIMUM(N);
1704 case ISD::FCEIL: return visitFCEIL(N);
1705 case ISD::FTRUNC: return visitFTRUNC(N);
1706 case ISD::BRCOND: return visitBRCOND(N);
1707 case ISD::BR_CC: return visitBR_CC(N);
1708 case ISD::LOAD: return visitLOAD(N);
1709 case ISD::STORE: return visitSTORE(N);
1710 case ISD::INSERT_VECTOR_ELT: return visitINSERT_VECTOR_ELT(N);
1711 case ISD::EXTRACT_VECTOR_ELT: return visitEXTRACT_VECTOR_ELT(N);
1712 case ISD::BUILD_VECTOR: return visitBUILD_VECTOR(N);
1713 case ISD::CONCAT_VECTORS: return visitCONCAT_VECTORS(N);
1714 case ISD::EXTRACT_SUBVECTOR: return visitEXTRACT_SUBVECTOR(N);
1715 case ISD::VECTOR_SHUFFLE: return visitVECTOR_SHUFFLE(N);
1716 case ISD::SCALAR_TO_VECTOR: return visitSCALAR_TO_VECTOR(N);
1717 case ISD::INSERT_SUBVECTOR: return visitINSERT_SUBVECTOR(N);
1718 case ISD::MGATHER: return visitMGATHER(N);
1719 case ISD::MLOAD: return visitMLOAD(N);
1720 case ISD::MSCATTER: return visitMSCATTER(N);
1721 case ISD::MSTORE: return visitMSTORE(N);
1722 case ISD::LIFETIME_END: return visitLIFETIME_END(N);
1723 case ISD::FP_TO_FP16: return visitFP_TO_FP16(N);
1724 case ISD::FP16_TO_FP: return visitFP16_TO_FP(N);
1725 case ISD::FREEZE: return visitFREEZE(N);
1726 case ISD::VECREDUCE_FADD:
1727 case ISD::VECREDUCE_FMUL:
1728 case ISD::VECREDUCE_ADD:
1729 case ISD::VECREDUCE_MUL:
1730 case ISD::VECREDUCE_AND:
1731 case ISD::VECREDUCE_OR:
1732 case ISD::VECREDUCE_XOR:
1733 case ISD::VECREDUCE_SMAX:
1734 case ISD::VECREDUCE_SMIN:
1735 case ISD::VECREDUCE_UMAX:
1736 case ISD::VECREDUCE_UMIN:
1737 case ISD::VECREDUCE_FMAX:
1738 case ISD::VECREDUCE_FMIN: return visitVECREDUCE(N);
1739 }
1740 return SDValue();
1741}
1742
1743SDValue DAGCombiner::combine(SDNode *N) {
1744 SDValue RV;
1745 if (!DisableGenericCombines)
1746 RV = visit(N);
1747
1748 // If nothing happened, try a target-specific DAG combine.
1749 if (!RV.getNode()) {
1750 assert(N->getOpcode() != ISD::DELETED_NODE &&((N->getOpcode() != ISD::DELETED_NODE && "Node was deleted but visit returned NULL!"
) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned NULL!\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1751, __PRETTY_FUNCTION__))
1751 "Node was deleted but visit returned NULL!")((N->getOpcode() != ISD::DELETED_NODE && "Node was deleted but visit returned NULL!"
) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned NULL!\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1751, __PRETTY_FUNCTION__))
;
1752
1753 if (N->getOpcode() >= ISD::BUILTIN_OP_END ||
1754 TLI.hasTargetDAGCombine((ISD::NodeType)N->getOpcode())) {
1755
1756 // Expose the DAG combiner to the target combiner impls.
1757 TargetLowering::DAGCombinerInfo
1758 DagCombineInfo(DAG, Level, false, this);
1759
1760 RV = TLI.PerformDAGCombine(N, DagCombineInfo);
1761 }
1762 }
1763
1764 // If nothing happened still, try promoting the operation.
1765 if (!RV.getNode()) {
1766 switch (N->getOpcode()) {
1767 default: break;
1768 case ISD::ADD:
1769 case ISD::SUB:
1770 case ISD::MUL:
1771 case ISD::AND:
1772 case ISD::OR:
1773 case ISD::XOR:
1774 RV = PromoteIntBinOp(SDValue(N, 0));
1775 break;
1776 case ISD::SHL:
1777 case ISD::SRA:
1778 case ISD::SRL:
1779 RV = PromoteIntShiftOp(SDValue(N, 0));
1780 break;
1781 case ISD::SIGN_EXTEND:
1782 case ISD::ZERO_EXTEND:
1783 case ISD::ANY_EXTEND:
1784 RV = PromoteExtend(SDValue(N, 0));
1785 break;
1786 case ISD::LOAD:
1787 if (PromoteLoad(SDValue(N, 0)))
1788 RV = SDValue(N, 0);
1789 break;
1790 }
1791 }
1792
1793 // If N is a commutative binary node, try to eliminate it if the commuted
1794 // version is already present in the DAG.
1795 if (!RV.getNode() && TLI.isCommutativeBinOp(N->getOpcode()) &&
1796 N->getNumValues() == 1) {
1797 SDValue N0 = N->getOperand(0);
1798 SDValue N1 = N->getOperand(1);
1799
1800 // Constant operands are canonicalized to RHS.
1801 if (N0 != N1 && (isa<ConstantSDNode>(N0) || !isa<ConstantSDNode>(N1))) {
1802 SDValue Ops[] = {N1, N0};
1803 SDNode *CSENode = DAG.getNodeIfExists(N->getOpcode(), N->getVTList(), Ops,
1804 N->getFlags());
1805 if (CSENode)
1806 return SDValue(CSENode, 0);
1807 }
1808 }
1809
1810 return RV;
1811}
1812
1813/// Given a node, return its input chain if it has one, otherwise return a null
1814/// sd operand.
1815static SDValue getInputChainForNode(SDNode *N) {
1816 if (unsigned NumOps = N->getNumOperands()) {
1817 if (N->getOperand(0).getValueType() == MVT::Other)
1818 return N->getOperand(0);
1819 if (N->getOperand(NumOps-1).getValueType() == MVT::Other)
1820 return N->getOperand(NumOps-1);
1821 for (unsigned i = 1; i < NumOps-1; ++i)
1822 if (N->getOperand(i).getValueType() == MVT::Other)
1823 return N->getOperand(i);
1824 }
1825 return SDValue();
1826}
1827
1828SDValue DAGCombiner::visitTokenFactor(SDNode *N) {
1829 // If N has two operands, where one has an input chain equal to the other,
1830 // the 'other' chain is redundant.
1831 if (N->getNumOperands() == 2) {
1832 if (getInputChainForNode(N->getOperand(0).getNode()) == N->getOperand(1))
1833 return N->getOperand(0);
1834 if (getInputChainForNode(N->getOperand(1).getNode()) == N->getOperand(0))
1835 return N->getOperand(1);
1836 }
1837
1838 // Don't simplify token factors if optnone.
1839 if (OptLevel == CodeGenOpt::None)
1840 return SDValue();
1841
1842 // Don't simplify the token factor if the node itself has too many operands.
1843 if (N->getNumOperands() > TokenFactorInlineLimit)
1844 return SDValue();
1845
1846 // If the sole user is a token factor, we should make sure we have a
1847 // chance to merge them together. This prevents TF chains from inhibiting
1848 // optimizations.
1849 if (N->hasOneUse() && N->use_begin()->getOpcode() == ISD::TokenFactor)
1850 AddToWorklist(*(N->use_begin()));
1851
1852 SmallVector<SDNode *, 8> TFs; // List of token factors to visit.
1853 SmallVector<SDValue, 8> Ops; // Ops for replacing token factor.
1854 SmallPtrSet<SDNode*, 16> SeenOps;
1855 bool Changed = false; // If we should replace this token factor.
1856
1857 // Start out with this token factor.
1858 TFs.push_back(N);
1859
1860 // Iterate through token factors. The TFs grows when new token factors are
1861 // encountered.
1862 for (unsigned i = 0; i < TFs.size(); ++i) {
1863 // Limit number of nodes to inline, to avoid quadratic compile times.
1864 // We have to add the outstanding Token Factors to Ops, otherwise we might
1865 // drop Ops from the resulting Token Factors.
1866 if (Ops.size() > TokenFactorInlineLimit) {
1867 for (unsigned j = i; j < TFs.size(); j++)
1868 Ops.emplace_back(TFs[j], 0);
1869 // Drop unprocessed Token Factors from TFs, so we do not add them to the
1870 // combiner worklist later.
1871 TFs.resize(i);
1872 break;
1873 }
1874
1875 SDNode *TF = TFs[i];
1876 // Check each of the operands.
1877 for (const SDValue &Op : TF->op_values()) {
1878 switch (Op.getOpcode()) {
1879 case ISD::EntryToken:
1880 // Entry tokens don't need to be added to the list. They are
1881 // redundant.
1882 Changed = true;
1883 break;
1884
1885 case ISD::TokenFactor:
1886 if (Op.hasOneUse() && !is_contained(TFs, Op.getNode())) {
1887 // Queue up for processing.
1888 TFs.push_back(Op.getNode());
1889 Changed = true;
1890 break;
1891 }
1892 LLVM_FALLTHROUGH[[gnu::fallthrough]];
1893
1894 default:
1895 // Only add if it isn't already in the list.
1896 if (SeenOps.insert(Op.getNode()).second)
1897 Ops.push_back(Op);
1898 else
1899 Changed = true;
1900 break;
1901 }
1902 }
1903 }
1904
1905 // Re-visit inlined Token Factors, to clean them up in case they have been
1906 // removed. Skip the first Token Factor, as this is the current node.
1907 for (unsigned i = 1, e = TFs.size(); i < e; i++)
1908 AddToWorklist(TFs[i]);
1909
1910 // Remove Nodes that are chained to another node in the list. Do so
1911 // by walking up chains breath-first stopping when we've seen
1912 // another operand. In general we must climb to the EntryNode, but we can exit
1913 // early if we find all remaining work is associated with just one operand as
1914 // no further pruning is possible.
1915
1916 // List of nodes to search through and original Ops from which they originate.
1917 SmallVector<std::pair<SDNode *, unsigned>, 8> Worklist;
1918 SmallVector<unsigned, 8> OpWorkCount; // Count of work for each Op.
1919 SmallPtrSet<SDNode *, 16> SeenChains;
1920 bool DidPruneOps = false;
1921
1922 unsigned NumLeftToConsider = 0;
1923 for (const SDValue &Op : Ops) {
1924 Worklist.push_back(std::make_pair(Op.getNode(), NumLeftToConsider++));
1925 OpWorkCount.push_back(1);
1926 }
1927
1928 auto AddToWorklist = [&](unsigned CurIdx, SDNode *Op, unsigned OpNumber) {
1929 // If this is an Op, we can remove the op from the list. Remark any
1930 // search associated with it as from the current OpNumber.
1931 if (SeenOps.contains(Op)) {
1932 Changed = true;
1933 DidPruneOps = true;
1934 unsigned OrigOpNumber = 0;
1935 while (OrigOpNumber < Ops.size() && Ops[OrigOpNumber].getNode() != Op)
1936 OrigOpNumber++;
1937 assert((OrigOpNumber != Ops.size()) &&(((OrigOpNumber != Ops.size()) && "expected to find TokenFactor Operand"
) ? static_cast<void> (0) : __assert_fail ("(OrigOpNumber != Ops.size()) && \"expected to find TokenFactor Operand\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1938, __PRETTY_FUNCTION__))
1938 "expected to find TokenFactor Operand")(((OrigOpNumber != Ops.size()) && "expected to find TokenFactor Operand"
) ? static_cast<void> (0) : __assert_fail ("(OrigOpNumber != Ops.size()) && \"expected to find TokenFactor Operand\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1938, __PRETTY_FUNCTION__))
;
1939 // Re-mark worklist from OrigOpNumber to OpNumber
1940 for (unsigned i = CurIdx + 1; i < Worklist.size(); ++i) {
1941 if (Worklist[i].second == OrigOpNumber) {
1942 Worklist[i].second = OpNumber;
1943 }
1944 }
1945 OpWorkCount[OpNumber] += OpWorkCount[OrigOpNumber];
1946 OpWorkCount[OrigOpNumber] = 0;
1947 NumLeftToConsider--;
1948 }
1949 // Add if it's a new chain
1950 if (SeenChains.insert(Op).second) {
1951 OpWorkCount[OpNumber]++;
1952 Worklist.push_back(std::make_pair(Op, OpNumber));
1953 }
1954 };
1955
1956 for (unsigned i = 0; i < Worklist.size() && i < 1024; ++i) {
1957 // We need at least be consider at least 2 Ops to prune.
1958 if (NumLeftToConsider <= 1)
1959 break;
1960 auto CurNode = Worklist[i].first;
1961 auto CurOpNumber = Worklist[i].second;
1962 assert((OpWorkCount[CurOpNumber] > 0) &&(((OpWorkCount[CurOpNumber] > 0) && "Node should not appear in worklist"
) ? static_cast<void> (0) : __assert_fail ("(OpWorkCount[CurOpNumber] > 0) && \"Node should not appear in worklist\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1963, __PRETTY_FUNCTION__))
1963 "Node should not appear in worklist")(((OpWorkCount[CurOpNumber] > 0) && "Node should not appear in worklist"
) ? static_cast<void> (0) : __assert_fail ("(OpWorkCount[CurOpNumber] > 0) && \"Node should not appear in worklist\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1963, __PRETTY_FUNCTION__))
;
1964 switch (CurNode->getOpcode()) {
1965 case ISD::EntryToken:
1966 // Hitting EntryToken is the only way for the search to terminate without
1967 // hitting
1968 // another operand's search. Prevent us from marking this operand
1969 // considered.
1970 NumLeftToConsider++;
1971 break;
1972 case ISD::TokenFactor:
1973 for (const SDValue &Op : CurNode->op_values())
1974 AddToWorklist(i, Op.getNode(), CurOpNumber);
1975 break;
1976 case ISD::LIFETIME_START:
1977 case ISD::LIFETIME_END:
1978 case ISD::CopyFromReg:
1979 case ISD::CopyToReg:
1980 AddToWorklist(i, CurNode->getOperand(0).getNode(), CurOpNumber);
1981 break;
1982 default:
1983 if (auto *MemNode = dyn_cast<MemSDNode>(CurNode))
1984 AddToWorklist(i, MemNode->getChain().getNode(), CurOpNumber);
1985 break;
1986 }
1987 OpWorkCount[CurOpNumber]--;
1988 if (OpWorkCount[CurOpNumber] == 0)
1989 NumLeftToConsider--;
1990 }
1991
1992 // If we've changed things around then replace token factor.
1993 if (Changed) {
1994 SDValue Result;
1995 if (Ops.empty()) {
1996 // The entry token is the only possible outcome.
1997 Result = DAG.getEntryNode();
1998 } else {
1999 if (DidPruneOps) {
2000 SmallVector<SDValue, 8> PrunedOps;
2001 //
2002 for (const SDValue &Op : Ops) {
2003 if (SeenChains.count(Op.getNode()) == 0)
2004 PrunedOps.push_back(Op);
2005 }
2006 Result = DAG.getTokenFactor(SDLoc(N), PrunedOps);
2007 } else {
2008 Result = DAG.getTokenFactor(SDLoc(N), Ops);
2009 }
2010 }
2011 return Result;
2012 }
2013 return SDValue();
2014}
2015
2016/// MERGE_VALUES can always be eliminated.
2017SDValue DAGCombiner::visitMERGE_VALUES(SDNode *N) {
2018 WorklistRemover DeadNodes(*this);
2019 // Replacing results may cause a different MERGE_VALUES to suddenly
2020 // be CSE'd with N, and carry its uses with it. Iterate until no
2021 // uses remain, to ensure that the node can be safely deleted.
2022 // First add the users of this node to the work list so that they
2023 // can be tried again once they have new operands.
2024 AddUsersToWorklist(N);
2025 do {
2026 // Do as a single replacement to avoid rewalking use lists.
2027 SmallVector<SDValue, 8> Ops;
2028 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
2029 Ops.push_back(N->getOperand(i));
2030 DAG.ReplaceAllUsesWith(N, Ops.data());
2031 } while (!N->use_empty());
2032 deleteAndRecombine(N);
2033 return SDValue(N, 0); // Return N so it doesn't get rechecked!
2034}
2035
2036/// If \p N is a ConstantSDNode with isOpaque() == false return it casted to a
2037/// ConstantSDNode pointer else nullptr.
2038static ConstantSDNode *getAsNonOpaqueConstant(SDValue N) {
2039 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N);
2040 return Const != nullptr && !Const->isOpaque() ? Const : nullptr;
2041}
2042
2043/// Return true if 'Use' is a load or a store that uses N as its base pointer
2044/// and that N may be folded in the load / store addressing mode.
2045static bool canFoldInAddressingMode(SDNode *N, SDNode *Use, SelectionDAG &DAG,
2046 const TargetLowering &TLI) {
2047 EVT VT;
2048 unsigned AS;
2049
2050 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Use)) {
2051 if (LD->isIndexed() || LD->getBasePtr().getNode() != N)
2052 return false;
2053 VT = LD->getMemoryVT();
2054 AS = LD->getAddressSpace();
2055 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Use)) {
2056 if (ST->isIndexed() || ST->getBasePtr().getNode() != N)
2057 return false;
2058 VT = ST->getMemoryVT();
2059 AS = ST->getAddressSpace();
2060 } else if (MaskedLoadSDNode *LD = dyn_cast<MaskedLoadSDNode>(Use)) {
2061 if (LD->isIndexed() || LD->getBasePtr().getNode() != N)
2062 return false;
2063 VT = LD->getMemoryVT();
2064 AS = LD->getAddressSpace();
2065 } else if (MaskedStoreSDNode *ST = dyn_cast<MaskedStoreSDNode>(Use)) {
2066 if (ST->isIndexed() || ST->getBasePtr().getNode() != N)
2067 return false;
2068 VT = ST->getMemoryVT();
2069 AS = ST->getAddressSpace();
2070 } else
2071 return false;
2072
2073 TargetLowering::AddrMode AM;
2074 if (N->getOpcode() == ISD::ADD) {
2075 AM.HasBaseReg = true;
2076 ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1));
2077 if (Offset)
2078 // [reg +/- imm]
2079 AM.BaseOffs = Offset->getSExtValue();
2080 else
2081 // [reg +/- reg]
2082 AM.Scale = 1;
2083 } else if (N->getOpcode() == ISD::SUB) {
2084 AM.HasBaseReg = true;
2085 ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1));
2086 if (Offset)
2087 // [reg +/- imm]
2088 AM.BaseOffs = -Offset->getSExtValue();
2089 else
2090 // [reg +/- reg]
2091 AM.Scale = 1;
2092 } else
2093 return false;
2094
2095 return TLI.isLegalAddressingMode(DAG.getDataLayout(), AM,
2096 VT.getTypeForEVT(*DAG.getContext()), AS);
2097}
2098
2099SDValue DAGCombiner::foldBinOpIntoSelect(SDNode *BO) {
2100 assert(TLI.isBinOp(BO->getOpcode()) && BO->getNumValues() == 1 &&((TLI.isBinOp(BO->getOpcode()) && BO->getNumValues
() == 1 && "Unexpected binary operator") ? static_cast
<void> (0) : __assert_fail ("TLI.isBinOp(BO->getOpcode()) && BO->getNumValues() == 1 && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2101, __PRETTY_FUNCTION__))
2101 "Unexpected binary operator")((TLI.isBinOp(BO->getOpcode()) && BO->getNumValues
() == 1 && "Unexpected binary operator") ? static_cast
<void> (0) : __assert_fail ("TLI.isBinOp(BO->getOpcode()) && BO->getNumValues() == 1 && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2101, __PRETTY_FUNCTION__))
;
2102
2103 // Don't do this unless the old select is going away. We want to eliminate the
2104 // binary operator, not replace a binop with a select.
2105 // TODO: Handle ISD::SELECT_CC.
2106 unsigned SelOpNo = 0;
2107 SDValue Sel = BO->getOperand(0);
2108 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse()) {
2109 SelOpNo = 1;
2110 Sel = BO->getOperand(1);
2111 }
2112
2113 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse())
2114 return SDValue();
2115
2116 SDValue CT = Sel.getOperand(1);
2117 if (!isConstantOrConstantVector(CT, true) &&
2118 !DAG.isConstantFPBuildVectorOrConstantFP(CT))
2119 return SDValue();
2120
2121 SDValue CF = Sel.getOperand(2);
2122 if (!isConstantOrConstantVector(CF, true) &&
2123 !DAG.isConstantFPBuildVectorOrConstantFP(CF))
2124 return SDValue();
2125
2126 // Bail out if any constants are opaque because we can't constant fold those.
2127 // The exception is "and" and "or" with either 0 or -1 in which case we can
2128 // propagate non constant operands into select. I.e.:
2129 // and (select Cond, 0, -1), X --> select Cond, 0, X
2130 // or X, (select Cond, -1, 0) --> select Cond, -1, X
2131 auto BinOpcode = BO->getOpcode();
2132 bool CanFoldNonConst =
2133 (BinOpcode == ISD::AND || BinOpcode == ISD::OR) &&
2134 (isNullOrNullSplat(CT) || isAllOnesOrAllOnesSplat(CT)) &&
2135 (isNullOrNullSplat(CF) || isAllOnesOrAllOnesSplat(CF));
2136
2137 SDValue CBO = BO->getOperand(SelOpNo ^ 1);
2138 if (!CanFoldNonConst &&
2139 !isConstantOrConstantVector(CBO, true) &&
2140 !DAG.isConstantFPBuildVectorOrConstantFP(CBO))
2141 return SDValue();
2142
2143 EVT VT = BO->getValueType(0);
2144
2145 // We have a select-of-constants followed by a binary operator with a
2146 // constant. Eliminate the binop by pulling the constant math into the select.
2147 // Example: add (select Cond, CT, CF), CBO --> select Cond, CT + CBO, CF + CBO
2148 SDLoc DL(Sel);
2149 SDValue NewCT = SelOpNo ? DAG.getNode(BinOpcode, DL, VT, CBO, CT)
2150 : DAG.getNode(BinOpcode, DL, VT, CT, CBO);
2151 if (!CanFoldNonConst && !NewCT.isUndef() &&
2152 !isConstantOrConstantVector(NewCT, true) &&
2153 !DAG.isConstantFPBuildVectorOrConstantFP(NewCT))
2154 return SDValue();
2155
2156 SDValue NewCF = SelOpNo ? DAG.getNode(BinOpcode, DL, VT, CBO, CF)
2157 : DAG.getNode(BinOpcode, DL, VT, CF, CBO);
2158 if (!CanFoldNonConst && !NewCF.isUndef() &&
2159 !isConstantOrConstantVector(NewCF, true) &&
2160 !DAG.isConstantFPBuildVectorOrConstantFP(NewCF))
2161 return SDValue();
2162
2163 SDValue SelectOp = DAG.getSelect(DL, VT, Sel.getOperand(0), NewCT, NewCF);
2164 SelectOp->setFlags(BO->getFlags());
2165 return SelectOp;
2166}
2167
2168static SDValue foldAddSubBoolOfMaskedVal(SDNode *N, SelectionDAG &DAG) {
2169 assert((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) &&(((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::
SUB) && "Expecting add or sub") ? static_cast<void
> (0) : __assert_fail ("(N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) && \"Expecting add or sub\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2170, __PRETTY_FUNCTION__))
2170 "Expecting add or sub")(((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::
SUB) && "Expecting add or sub") ? static_cast<void
> (0) : __assert_fail ("(N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) && \"Expecting add or sub\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2170, __PRETTY_FUNCTION__))
;
2171
2172 // Match a constant operand and a zext operand for the math instruction:
2173 // add Z, C
2174 // sub C, Z
2175 bool IsAdd = N->getOpcode() == ISD::ADD;
2176 SDValue C = IsAdd ? N->getOperand(1) : N->getOperand(0);
2177 SDValue Z = IsAdd ? N->getOperand(0) : N->getOperand(1);
2178 auto *CN = dyn_cast<ConstantSDNode>(C);
2179 if (!CN || Z.getOpcode() != ISD::ZERO_EXTEND)
2180 return SDValue();
2181
2182 // Match the zext operand as a setcc of a boolean.
2183 if (Z.getOperand(0).getOpcode() != ISD::SETCC ||
2184 Z.getOperand(0).getValueType() != MVT::i1)
2185 return SDValue();
2186
2187 // Match the compare as: setcc (X & 1), 0, eq.
2188 SDValue SetCC = Z.getOperand(0);
2189 ISD::CondCode CC = cast<CondCodeSDNode>(SetCC->getOperand(2))->get();
2190 if (CC != ISD::SETEQ || !isNullConstant(SetCC.getOperand(1)) ||
2191 SetCC.getOperand(0).getOpcode() != ISD::AND ||
2192 !isOneConstant(SetCC.getOperand(0).getOperand(1)))
2193 return SDValue();
2194
2195 // We are adding/subtracting a constant and an inverted low bit. Turn that
2196 // into a subtract/add of the low bit with incremented/decremented constant:
2197 // add (zext i1 (seteq (X & 1), 0)), C --> sub C+1, (zext (X & 1))
2198 // sub C, (zext i1 (seteq (X & 1), 0)) --> add C-1, (zext (X & 1))
2199 EVT VT = C.getValueType();
2200 SDLoc DL(N);
2201 SDValue LowBit = DAG.getZExtOrTrunc(SetCC.getOperand(0), DL, VT);
2202 SDValue C1 = IsAdd ? DAG.getConstant(CN->getAPIntValue() + 1, DL, VT) :
2203 DAG.getConstant(CN->getAPIntValue() - 1, DL, VT);
2204 return DAG.getNode(IsAdd ? ISD::SUB : ISD::ADD, DL, VT, C1, LowBit);
2205}
2206
2207/// Try to fold a 'not' shifted sign-bit with add/sub with constant operand into
2208/// a shift and add with a different constant.
2209static SDValue foldAddSubOfSignBit(SDNode *N, SelectionDAG &DAG) {
2210 assert((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) &&(((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::
SUB) && "Expecting add or sub") ? static_cast<void
> (0) : __assert_fail ("(N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) && \"Expecting add or sub\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2211, __PRETTY_FUNCTION__))
2211 "Expecting add or sub")(((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::
SUB) && "Expecting add or sub") ? static_cast<void
> (0) : __assert_fail ("(N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) && \"Expecting add or sub\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2211, __PRETTY_FUNCTION__))
;
2212
2213 // We need a constant operand for the add/sub, and the other operand is a
2214 // logical shift right: add (srl), C or sub C, (srl).
2215 bool IsAdd = N->getOpcode() == ISD::ADD;
2216 SDValue ConstantOp = IsAdd ? N->getOperand(1) : N->getOperand(0);
2217 SDValue ShiftOp = IsAdd ? N->getOperand(0) : N->getOperand(1);
2218 if (!DAG.isConstantIntBuildVectorOrConstantInt(ConstantOp) ||
2219 ShiftOp.getOpcode() != ISD::SRL)
2220 return SDValue();
2221
2222 // The shift must be of a 'not' value.
2223 SDValue Not = ShiftOp.getOperand(0);
2224 if (!Not.hasOneUse() || !isBitwiseNot(Not))
2225 return SDValue();
2226
2227 // The shift must be moving the sign bit to the least-significant-bit.
2228 EVT VT = ShiftOp.getValueType();
2229 SDValue ShAmt = ShiftOp.getOperand(1);
2230 ConstantSDNode *ShAmtC = isConstOrConstSplat(ShAmt);
2231 if (!ShAmtC || ShAmtC->getAPIntValue() != (VT.getScalarSizeInBits() - 1))
2232 return SDValue();
2233
2234 // Eliminate the 'not' by adjusting the shift and add/sub constant:
2235 // add (srl (not X), 31), C --> add (sra X, 31), (C + 1)
2236 // sub C, (srl (not X), 31) --> add (srl X, 31), (C - 1)
2237 SDLoc DL(N);
2238 auto ShOpcode = IsAdd ? ISD::SRA : ISD::SRL;
2239 SDValue NewShift = DAG.getNode(ShOpcode, DL, VT, Not.getOperand(0), ShAmt);
2240 if (SDValue NewC =
2241 DAG.FoldConstantArithmetic(IsAdd ? ISD::ADD : ISD::SUB, DL, VT,
2242 {ConstantOp, DAG.getConstant(1, DL, VT)}))
2243 return DAG.getNode(ISD::ADD, DL, VT, NewShift, NewC);
2244 return SDValue();
2245}
2246
2247/// Try to fold a node that behaves like an ADD (note that N isn't necessarily
2248/// an ISD::ADD here, it could for example be an ISD::OR if we know that there
2249/// are no common bits set in the operands).
2250SDValue DAGCombiner::visitADDLike(SDNode *N) {
2251 SDValue N0 = N->getOperand(0);
2252 SDValue N1 = N->getOperand(1);
2253 EVT VT = N0.getValueType();
2254 SDLoc DL(N);
2255
2256 // fold vector ops
2257 if (VT.isVector()) {
2258 if (SDValue FoldedVOp = SimplifyVBinOp(N))
2259 return FoldedVOp;
2260
2261 // fold (add x, 0) -> x, vector edition
2262 if (ISD::isBuildVectorAllZeros(N1.getNode()))
2263 return N0;
2264 if (ISD::isBuildVectorAllZeros(N0.getNode()))
2265 return N1;
2266 }
2267
2268 // fold (add x, undef) -> undef
2269 if (N0.isUndef())
2270 return N0;
2271
2272 if (N1.isUndef())
2273 return N1;
2274
2275 if (DAG.isConstantIntBuildVectorOrConstantInt(N0)) {
2276 // canonicalize constant to RHS
2277 if (!DAG.isConstantIntBuildVectorOrConstantInt(N1))
2278 return DAG.getNode(ISD::ADD, DL, VT, N1, N0);
2279 // fold (add c1, c2) -> c1+c2
2280 return DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N0, N1});
2281 }
2282
2283 // fold (add x, 0) -> x
2284 if (isNullConstant(N1))
2285 return N0;
2286
2287 if (isConstantOrConstantVector(N1, /* NoOpaque */ true)) {
2288 // fold ((A-c1)+c2) -> (A+(c2-c1))
2289 if (N0.getOpcode() == ISD::SUB &&
2290 isConstantOrConstantVector(N0.getOperand(1), /* NoOpaque */ true)) {
2291 SDValue Sub =
2292 DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N1, N0.getOperand(1)});
2293 assert(Sub && "Constant folding failed")((Sub && "Constant folding failed") ? static_cast<
void> (0) : __assert_fail ("Sub && \"Constant folding failed\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2293, __PRETTY_FUNCTION__))
;
2294 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), Sub);
2295 }
2296
2297 // fold ((c1-A)+c2) -> (c1+c2)-A
2298 if (N0.getOpcode() == ISD::SUB &&
2299 isConstantOrConstantVector(N0.getOperand(0), /* NoOpaque */ true)) {
2300 SDValue Add =
2301 DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N1, N0.getOperand(0)});
2302 assert(Add && "Constant folding failed")((Add && "Constant folding failed") ? static_cast<
void> (0) : __assert_fail ("Add && \"Constant folding failed\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2302, __PRETTY_FUNCTION__))
;
2303 return DAG.getNode(ISD::SUB, DL, VT, Add, N0.getOperand(1));
2304 }
2305
2306 // add (sext i1 X), 1 -> zext (not i1 X)
2307 // We don't transform this pattern:
2308 // add (zext i1 X), -1 -> sext (not i1 X)
2309 // because most (?) targets generate better code for the zext form.
2310 if (N0.getOpcode() == ISD::SIGN_EXTEND && N0.hasOneUse() &&
2311 isOneOrOneSplat(N1)) {
2312 SDValue X = N0.getOperand(0);
2313 if ((!LegalOperations ||
2314 (TLI.isOperationLegal(ISD::XOR, X.getValueType()) &&
2315 TLI.isOperationLegal(ISD::ZERO_EXTEND, VT))) &&
2316 X.getScalarValueSizeInBits() == 1) {
2317 SDValue Not = DAG.getNOT(DL, X, X.getValueType());
2318 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Not);
2319 }
2320 }
2321
2322 // Fold (add (or x, c0), c1) -> (add x, (c0 + c1)) if (or x, c0) is
2323 // equivalent to (add x, c0).
2324 if (N0.getOpcode() == ISD::OR &&
2325 isConstantOrConstantVector(N0.getOperand(1), /* NoOpaque */ true) &&
2326 DAG.haveNoCommonBitsSet(N0.getOperand(0), N0.getOperand(1))) {
2327 if (SDValue Add0 = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT,
2328 {N1, N0.getOperand(1)}))
2329 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), Add0);
2330 }
2331 }
2332
2333 if (SDValue NewSel = foldBinOpIntoSelect(N))
2334 return NewSel;
2335
2336 // reassociate add
2337 if (!reassociationCanBreakAddressingModePattern(ISD::ADD, DL, N0, N1)) {
2338 if (SDValue RADD = reassociateOps(ISD::ADD, DL, N0, N1, N->getFlags()))
2339 return RADD;
2340 }
2341 // fold ((0-A) + B) -> B-A
2342 if (N0.getOpcode() == ISD::SUB && isNullOrNullSplat(N0.getOperand(0)))
2343 return DAG.getNode(ISD::SUB, DL, VT, N1, N0.getOperand(1));
2344
2345 // fold (A + (0-B)) -> A-B
2346 if (N1.getOpcode() == ISD::SUB && isNullOrNullSplat(N1.getOperand(0)))
2347 return DAG.getNode(ISD::SUB, DL, VT, N0, N1.getOperand(1));
2348
2349 // fold (A+(B-A)) -> B
2350 if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(1))
2351 return N1.getOperand(0);
2352
2353 // fold ((B-A)+A) -> B
2354 if (N0.getOpcode() == ISD::SUB && N1 == N0.getOperand(1))
2355 return N0.getOperand(0);
2356
2357 // fold ((A-B)+(C-A)) -> (C-B)
2358 if (N0.getOpcode() == ISD::SUB && N1.getOpcode() == ISD::SUB &&
2359 N0.getOperand(0) == N1.getOperand(1))
2360 return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0),
2361 N0.getOperand(1));
2362
2363 // fold ((A-B)+(B-C)) -> (A-C)
2364 if (N0.getOpcode() == ISD::SUB && N1.getOpcode() == ISD::SUB &&
2365 N0.getOperand(1) == N1.getOperand(0))
2366 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0),
2367 N1.getOperand(1));
2368
2369 // fold (A+(B-(A+C))) to (B-C)
2370 if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD &&
2371 N0 == N1.getOperand(1).getOperand(0))
2372 return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0),
2373 N1.getOperand(1).getOperand(1));
2374
2375 // fold (A+(B-(C+A))) to (B-C)
2376 if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD &&
2377 N0 == N1.getOperand(1).getOperand(1))
2378 return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0),
2379 N1.getOperand(1).getOperand(0));
2380
2381 // fold (A+((B-A)+or-C)) to (B+or-C)
2382 if ((N1.getOpcode() == ISD::SUB || N1.getOpcode() == ISD::ADD) &&
2383 N1.getOperand(0).getOpcode() == ISD::SUB &&
2384 N0 == N1.getOperand(0).getOperand(1))
2385 return DAG.getNode(N1.getOpcode(), DL, VT, N1.getOperand(0).getOperand(0),
2386 N1.getOperand(1));
2387
2388 // fold (A-B)+(C-D) to (A+C)-(B+D) when A or C is constant
2389 if (N0.getOpcode() == ISD::SUB && N1.getOpcode() == ISD::SUB) {
2390 SDValue N00 = N0.getOperand(0);
2391 SDValue N01 = N0.getOperand(1);
2392 SDValue N10 = N1.getOperand(0);
2393 SDValue N11 = N1.getOperand(1);
2394
2395 if (isConstantOrConstantVector(N00) || isConstantOrConstantVector(N10))
2396 return DAG.getNode(ISD::SUB, DL, VT,
2397 DAG.getNode(ISD::ADD, SDLoc(N0), VT, N00, N10),
2398 DAG.getNode(ISD::ADD, SDLoc(N1), VT, N01, N11));
2399 }
2400
2401 // fold (add (umax X, C), -C) --> (usubsat X, C)
2402 if (N0.getOpcode() == ISD::UMAX && hasOperation(ISD::USUBSAT, VT)) {
2403 auto MatchUSUBSAT = [](ConstantSDNode *Max, ConstantSDNode *Op) {
2404 return (!Max && !Op) ||
2405 (Max && Op && Max->getAPIntValue() == (-Op->getAPIntValue()));
2406 };
2407 if (ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchUSUBSAT,
2408 /*AllowUndefs*/ true))
2409 return DAG.getNode(ISD::USUBSAT, DL, VT, N0.getOperand(0),
2410 N0.getOperand(1));
2411 }
2412
2413 if (SimplifyDemandedBits(SDValue(N, 0)))
2414 return SDValue(N, 0);
2415
2416 if (isOneOrOneSplat(N1)) {
2417 // fold (add (xor a, -1), 1) -> (sub 0, a)
2418 if (isBitwiseNot(N0))
2419 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
2420 N0.getOperand(0));
2421
2422 // fold (add (add (xor a, -1), b), 1) -> (sub b, a)
2423 if (N0.getOpcode() == ISD::ADD ||
2424 N0.getOpcode() == ISD::UADDO ||
2425 N0.getOpcode() == ISD::SADDO) {
2426 SDValue A, Xor;
2427
2428 if (isBitwiseNot(N0.getOperand(0))) {
2429 A = N0.getOperand(1);
2430 Xor = N0.getOperand(0);
2431 } else if (isBitwiseNot(N0.getOperand(1))) {
2432 A = N0.getOperand(0);
2433 Xor = N0.getOperand(1);
2434 }
2435
2436 if (Xor)
2437 return DAG.getNode(ISD::SUB, DL, VT, A, Xor.getOperand(0));
2438 }
2439
2440 // Look for:
2441 // add (add x, y), 1
2442 // And if the target does not like this form then turn into:
2443 // sub y, (xor x, -1)
2444 if (!TLI.preferIncOfAddToSubOfNot(VT) && N0.hasOneUse() &&
2445 N0.getOpcode() == ISD::ADD) {
2446 SDValue Not = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(0),
2447 DAG.getAllOnesConstant(DL, VT));
2448 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(1), Not);
2449 }
2450 }
2451
2452 // (x - y) + -1 -> add (xor y, -1), x
2453 if (N0.hasOneUse() && N0.getOpcode() == ISD::SUB &&
2454 isAllOnesOrAllOnesSplat(N1)) {
2455 SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(1), N1);
2456 return DAG.getNode(ISD::ADD, DL, VT, Xor, N0.getOperand(0));
2457 }
2458
2459 if (SDValue Combined = visitADDLikeCommutative(N0, N1, N))
2460 return Combined;
2461
2462 if (SDValue Combined = visitADDLikeCommutative(N1, N0, N))
2463 return Combined;
2464
2465 return SDValue();
2466}
2467
2468SDValue DAGCombiner::visitADD(SDNode *N) {
2469 SDValue N0 = N->getOperand(0);
2470 SDValue N1 = N->getOperand(1);
2471 EVT VT = N0.getValueType();
2472 SDLoc DL(N);
2473
2474 if (SDValue Combined = visitADDLike(N))
2475 return Combined;
2476
2477 if (SDValue V = foldAddSubBoolOfMaskedVal(N, DAG))
2478 return V;
2479
2480 if (SDValue V = foldAddSubOfSignBit(N, DAG))
2481 return V;
2482
2483 // fold (a+b) -> (a|b) iff a and b share no bits.
2484 if ((!LegalOperations || TLI.isOperationLegal(ISD::OR, VT)) &&
2485 DAG.haveNoCommonBitsSet(N0, N1))
2486 return DAG.getNode(ISD::OR, DL, VT, N0, N1);
2487
2488 // Fold (add (vscale * C0), (vscale * C1)) to (vscale * (C0 + C1)).
2489 if (N0.getOpcode() == ISD::VSCALE && N1.getOpcode() == ISD::VSCALE) {
2490 const APInt &C0 = N0->getConstantOperandAPInt(0);
2491 const APInt &C1 = N1->getConstantOperandAPInt(0);
2492 return DAG.getVScale(DL, VT, C0 + C1);
2493 }
2494
2495 // fold a+vscale(c1)+vscale(c2) -> a+vscale(c1+c2)
2496 if ((N0.getOpcode() == ISD::ADD) &&
2497 (N0.getOperand(1).getOpcode() == ISD::VSCALE) &&
2498 (N1.getOpcode() == ISD::VSCALE)) {
2499 const APInt &VS0 = N0.getOperand(1)->getConstantOperandAPInt(0);
2500 const APInt &VS1 = N1->getConstantOperandAPInt(0);
2501 SDValue VS = DAG.getVScale(DL, VT, VS0 + VS1);
2502 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), VS);
2503 }
2504
2505 return SDValue();
2506}
2507
2508SDValue DAGCombiner::visitADDSAT(SDNode *N) {
2509 unsigned Opcode = N->getOpcode();
2510 SDValue N0 = N->getOperand(0);
2511 SDValue N1 = N->getOperand(1);
2512 EVT VT = N0.getValueType();
2513 SDLoc DL(N);
2514
2515 // fold vector ops
2516 if (VT.isVector()) {
2517 // TODO SimplifyVBinOp
2518
2519 // fold (add_sat x, 0) -> x, vector edition
2520 if (ISD::isBuildVectorAllZeros(N1.getNode()))
2521 return N0;
2522 if (ISD::isBuildVectorAllZeros(N0.getNode()))
2523 return N1;
2524 }
2525
2526 // fold (add_sat x, undef) -> -1
2527 if (N0.isUndef() || N1.isUndef())
2528 return DAG.getAllOnesConstant(DL, VT);
2529
2530 if (DAG.isConstantIntBuildVectorOrConstantInt(N0)) {
2531 // canonicalize constant to RHS
2532 if (!DAG.isConstantIntBuildVectorOrConstantInt(N1))
2533 return DAG.getNode(Opcode, DL, VT, N1, N0);
2534 // fold (add_sat c1, c2) -> c3
2535 return DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1});
2536 }
2537
2538 // fold (add_sat x, 0) -> x
2539 if (isNullConstant(N1))
2540 return N0;
2541
2542 // If it cannot overflow, transform into an add.
2543 if (Opcode == ISD::UADDSAT)
2544 if (DAG.computeOverflowKind(N0, N1) == SelectionDAG::OFK_Never)
2545 return DAG.getNode(ISD::ADD, DL, VT, N0, N1);
2546
2547 return SDValue();
2548}
2549
2550static SDValue getAsCarry(const TargetLowering &TLI, SDValue V) {
2551 bool Masked = false;
2552
2553 // First, peel away TRUNCATE/ZERO_EXTEND/AND nodes due to legalization.
2554 while (true) {
2555 if (V.getOpcode() == ISD::TRUNCATE || V.getOpcode() == ISD::ZERO_EXTEND) {
2556 V = V.getOperand(0);
2557 continue;
2558 }
2559
2560 if (V.getOpcode() == ISD::AND && isOneConstant(V.getOperand(1))) {
2561 Masked = true;
2562 V = V.getOperand(0);
2563 continue;
2564 }
2565
2566 break;
2567 }
2568
2569 // If this is not a carry, return.
2570 if (V.getResNo() != 1)
2571 return SDValue();
2572
2573 if (V.getOpcode() != ISD::ADDCARRY && V.getOpcode() != ISD::SUBCARRY &&
2574 V.getOpcode() != ISD::UADDO && V.getOpcode() != ISD::USUBO)
2575 return SDValue();
2576
2577 EVT VT = V.getNode()->getValueType(0);
2578 if (!TLI.isOperationLegalOrCustom(V.getOpcode(), VT))
2579 return SDValue();
2580
2581 // If the result is masked, then no matter what kind of bool it is we can
2582 // return. If it isn't, then we need to make sure the bool type is either 0 or
2583 // 1 and not other values.
2584 if (Masked ||
2585 TLI.getBooleanContents(V.getValueType()) ==
2586 TargetLoweringBase::ZeroOrOneBooleanContent)
2587 return V;
2588
2589 return SDValue();
2590}
2591
2592/// Given the operands of an add/sub operation, see if the 2nd operand is a
2593/// masked 0/1 whose source operand is actually known to be 0/-1. If so, invert
2594/// the opcode and bypass the mask operation.
2595static SDValue foldAddSubMasked1(bool IsAdd, SDValue N0, SDValue N1,
2596 SelectionDAG &DAG, const SDLoc &DL) {
2597 if (N1.getOpcode() != ISD::AND || !isOneOrOneSplat(N1->getOperand(1)))
2598 return SDValue();
2599
2600 EVT VT = N0.getValueType();
2601 if (DAG.ComputeNumSignBits(N1.getOperand(0)) != VT.getScalarSizeInBits())
2602 return SDValue();
2603
2604 // add N0, (and (AssertSext X, i1), 1) --> sub N0, X
2605 // sub N0, (and (AssertSext X, i1), 1) --> add N0, X
2606 return DAG.getNode(IsAdd ? ISD::SUB : ISD::ADD, DL, VT, N0, N1.getOperand(0));
2607}
2608
2609/// Helper for doing combines based on N0 and N1 being added to each other.
2610SDValue DAGCombiner::visitADDLikeCommutative(SDValue N0, SDValue N1,
2611 SDNode *LocReference) {
2612 EVT VT = N0.getValueType();
2613 SDLoc DL(LocReference);
2614
2615 // fold (add x, shl(0 - y, n)) -> sub(x, shl(y, n))
2616 if (N1.getOpcode() == ISD::SHL && N1.getOperand(0).getOpcode() == ISD::SUB &&
2617 isNullOrNullSplat(N1.getOperand(0).getOperand(0)))
2618 return DAG.getNode(ISD::SUB, DL, VT, N0,
2619 DAG.getNode(ISD::SHL, DL, VT,
2620 N1.getOperand(0).getOperand(1),
2621 N1.getOperand(1)));
2622
2623 if (SDValue V = foldAddSubMasked1(true, N0, N1, DAG, DL))
2624 return V;
2625
2626 // Look for:
2627 // add (add x, 1), y
2628 // And if the target does not like this form then turn into:
2629 // sub y, (xor x, -1)
2630 if (!TLI.preferIncOfAddToSubOfNot(VT) && N0.hasOneUse() &&
2631 N0.getOpcode() == ISD::ADD && isOneOrOneSplat(N0.getOperand(1))) {
2632 SDValue Not = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(0),
2633 DAG.getAllOnesConstant(DL, VT));
2634 return DAG.getNode(ISD::SUB, DL, VT, N1, Not);
2635 }
2636
2637 // Hoist one-use subtraction by non-opaque constant:
2638 // (x - C) + y -> (x + y) - C
2639 // This is necessary because SUB(X,C) -> ADD(X,-C) doesn't work for vectors.
2640 if (N0.hasOneUse() && N0.getOpcode() == ISD::SUB &&
2641 isConstantOrConstantVector(N0.getOperand(1), /*NoOpaques=*/true)) {
2642 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), N1);
2643 return DAG.getNode(ISD::SUB, DL, VT, Add, N0.getOperand(1));
2644 }
2645 // Hoist one-use subtraction from non-opaque constant:
2646 // (C - x) + y -> (y - x) + C
2647 if (N0.hasOneUse() && N0.getOpcode() == ISD::SUB &&
2648 isConstantOrConstantVector(N0.getOperand(0), /*NoOpaques=*/true)) {
2649 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N1, N0.getOperand(1));
2650 return DAG.getNode(ISD::ADD, DL, VT, Sub, N0.getOperand(0));
2651 }
2652
2653 // If the target's bool is represented as 0/1, prefer to make this 'sub 0/1'
2654 // rather than 'add 0/-1' (the zext should get folded).
2655 // add (sext i1 Y), X --> sub X, (zext i1 Y)
2656 if (N0.getOpcode() == ISD::SIGN_EXTEND &&
2657 N0.getOperand(0).getScalarValueSizeInBits() == 1 &&
2658 TLI.getBooleanContents(VT) == TargetLowering::ZeroOrOneBooleanContent) {
2659 SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0));
2660 return DAG.getNode(ISD::SUB, DL, VT, N1, ZExt);
2661 }
2662
2663 // add X, (sextinreg Y i1) -> sub X, (and Y 1)
2664 if (N1.getOpcode() == ISD::SIGN_EXTEND_INREG) {
2665 VTSDNode *TN = cast<VTSDNode>(N1.getOperand(1));
2666 if (TN->getVT() == MVT::i1) {
2667 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0),
2668 DAG.getConstant(1, DL, VT));
2669 return DAG.getNode(ISD::SUB, DL, VT, N0, ZExt);
2670 }
2671 }
2672
2673 // (add X, (addcarry Y, 0, Carry)) -> (addcarry X, Y, Carry)
2674 if (N1.getOpcode() == ISD::ADDCARRY && isNullConstant(N1.getOperand(1)) &&
2675 N1.getResNo() == 0)
2676 return DAG.getNode(ISD::ADDCARRY, DL, N1->getVTList(),
2677 N0, N1.getOperand(0), N1.getOperand(2));
2678
2679 // (add X, Carry) -> (addcarry X, 0, Carry)
2680 if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT))
2681 if (SDValue Carry = getAsCarry(TLI, N1))
2682 return DAG.getNode(ISD::ADDCARRY, DL,
2683 DAG.getVTList(VT, Carry.getValueType()), N0,
2684 DAG.getConstant(0, DL, VT), Carry);
2685
2686 return SDValue();
2687}
2688
2689SDValue DAGCombiner::visitADDC(SDNode *N) {
2690 SDValue N0 = N->getOperand(0);
2691 SDValue N1 = N->getOperand(1);
2692 EVT VT = N0.getValueType();
2693 SDLoc DL(N);
2694
2695 // If the flag result is dead, turn this into an ADD.
2696 if (!N->hasAnyUseOfValue(1))
2697 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2698 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2699
2700 // canonicalize constant to RHS.
2701 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
2702 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2703 if (N0C && !N1C)
2704 return DAG.getNode(ISD::ADDC, DL, N->getVTList(), N1, N0);
2705
2706 // fold (addc x, 0) -> x + no carry out
2707 if (isNullConstant(N1))
2708 return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE,
2709 DL, MVT::Glue));
2710
2711 // If it cannot overflow, transform into an add.
2712 if (DAG.computeOverflowKind(N0, N1) == SelectionDAG::OFK_Never)
2713 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2714 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2715
2716 return SDValue();
2717}
2718
2719/**
2720 * Flips a boolean if it is cheaper to compute. If the Force parameters is set,
2721 * then the flip also occurs if computing the inverse is the same cost.
2722 * This function returns an empty SDValue in case it cannot flip the boolean
2723 * without increasing the cost of the computation. If you want to flip a boolean
2724 * no matter what, use DAG.getLogicalNOT.
2725 */
2726static SDValue extractBooleanFlip(SDValue V, SelectionDAG &DAG,
2727 const TargetLowering &TLI,
2728 bool Force) {
2729 if (Force && isa<ConstantSDNode>(V))
2730 return DAG.getLogicalNOT(SDLoc(V), V, V.getValueType());
2731
2732 if (V.getOpcode() != ISD::XOR)
2733 return SDValue();
2734
2735 ConstantSDNode *Const = isConstOrConstSplat(V.getOperand(1), false);
2736 if (!Const)
2737 return SDValue();
2738
2739 EVT VT = V.getValueType();
2740
2741 bool IsFlip = false;
2742 switch(TLI.getBooleanContents(VT)) {
2743 case TargetLowering::ZeroOrOneBooleanContent:
2744 IsFlip = Const->isOne();
2745 break;
2746 case TargetLowering::ZeroOrNegativeOneBooleanContent:
2747 IsFlip = Const->isAllOnesValue();
2748 break;
2749 case TargetLowering::UndefinedBooleanContent:
2750 IsFlip = (Const->getAPIntValue() & 0x01) == 1;
2751 break;
2752 }
2753
2754 if (IsFlip)
2755 return V.getOperand(0);
2756 if (Force)
2757 return DAG.getLogicalNOT(SDLoc(V), V, V.getValueType());
2758 return SDValue();
2759}
2760
2761SDValue DAGCombiner::visitADDO(SDNode *N) {
2762 SDValue N0 = N->getOperand(0);
2763 SDValue N1 = N->getOperand(1);
2764 EVT VT = N0.getValueType();
2765 bool IsSigned = (ISD::SADDO == N->getOpcode());
2766
2767 EVT CarryVT = N->getValueType(1);
2768 SDLoc DL(N);
2769
2770 // If the flag result is dead, turn this into an ADD.
2771 if (!N->hasAnyUseOfValue(1))
2772 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2773 DAG.getUNDEF(CarryVT));
2774
2775 // canonicalize constant to RHS.
2776 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
2777 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
2778 return DAG.getNode(N->getOpcode(), DL, N->getVTList(), N1, N0);
2779
2780 // fold (addo x, 0) -> x + no carry out
2781 if (isNullOrNullSplat(N1))
2782 return CombineTo(N, N0, DAG.getConstant(0, DL, CarryVT));
2783
2784 if (!IsSigned) {
2785 // If it cannot overflow, transform into an add.
2786 if (DAG.computeOverflowKind(N0, N1) == SelectionDAG::OFK_Never)
2787 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2788 DAG.getConstant(0, DL, CarryVT));
2789
2790 // fold (uaddo (xor a, -1), 1) -> (usub 0, a) and flip carry.
2791 if (isBitwiseNot(N0) && isOneOrOneSplat(N1)) {
2792 SDValue Sub = DAG.getNode(ISD::USUBO, DL, N->getVTList(),
2793 DAG.getConstant(0, DL, VT), N0.getOperand(0));
2794 return CombineTo(
2795 N, Sub, DAG.getLogicalNOT(DL, Sub.getValue(1), Sub->getValueType(1)));
2796 }
2797
2798 if (SDValue Combined = visitUADDOLike(N0, N1, N))
2799 return Combined;
2800
2801 if (SDValue Combined = visitUADDOLike(N1, N0, N))
2802 return Combined;
2803 }
2804
2805 return SDValue();
2806}
2807
2808SDValue DAGCombiner::visitUADDOLike(SDValue N0, SDValue N1, SDNode *N) {
2809 EVT VT = N0.getValueType();
2810 if (VT.isVector())
2811 return SDValue();
2812
2813 // (uaddo X, (addcarry Y, 0, Carry)) -> (addcarry X, Y, Carry)
2814 // If Y + 1 cannot overflow.
2815 if (N1.getOpcode() == ISD::ADDCARRY && isNullConstant(N1.getOperand(1))) {
2816 SDValue Y = N1.getOperand(0);
2817 SDValue One = DAG.getConstant(1, SDLoc(N), Y.getValueType());
2818 if (DAG.computeOverflowKind(Y, One) == SelectionDAG::OFK_Never)
2819 return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0, Y,
2820 N1.getOperand(2));
2821 }
2822
2823 // (uaddo X, Carry) -> (addcarry X, 0, Carry)
2824 if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT))
2825 if (SDValue Carry = getAsCarry(TLI, N1))
2826 return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0,
2827 DAG.getConstant(0, SDLoc(N), VT), Carry);
2828
2829 return SDValue();
2830}
2831
2832SDValue DAGCombiner::visitADDE(SDNode *N) {
2833 SDValue N0 = N->getOperand(0);
2834 SDValue N1 = N->getOperand(1);
2835 SDValue CarryIn = N->getOperand(2);
2836
2837 // canonicalize constant to RHS
2838 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
2839 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2840 if (N0C && !N1C)
2841 return DAG.getNode(ISD::ADDE, SDLoc(N), N->getVTList(),
2842 N1, N0, CarryIn);
2843
2844 // fold (adde x, y, false) -> (addc x, y)
2845 if (CarryIn.getOpcode() == ISD::CARRY_FALSE)
2846 return DAG.getNode(ISD::ADDC, SDLoc(N), N->getVTList(), N0, N1);
2847
2848 return SDValue();
2849}
2850
2851SDValue DAGCombiner::visitADDCARRY(SDNode *N) {
2852 SDValue N0 = N->getOperand(0);
2853 SDValue N1 = N->getOperand(1);
2854 SDValue CarryIn = N->getOperand(2);
2855 SDLoc DL(N);
2856
2857 // canonicalize constant to RHS
2858 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
2859 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2860 if (N0C && !N1C)
2861 return DAG.getNode(ISD::ADDCARRY, DL, N->getVTList(), N1, N0, CarryIn);
2862
2863 // fold (addcarry x, y, false) -> (uaddo x, y)
2864 if (isNullConstant(CarryIn)) {
2865 if (!LegalOperations ||
2866 TLI.isOperationLegalOrCustom(ISD::UADDO, N->getValueType(0)))
2867 return DAG.getNode(ISD::UADDO, DL, N->getVTList(), N0, N1);
2868 }
2869
2870 // fold (addcarry 0, 0, X) -> (and (ext/trunc X), 1) and no carry.
2871 if (isNullConstant(N0) && isNullConstant(N1)) {
2872 EVT VT = N0.getValueType();
2873 EVT CarryVT = CarryIn.getValueType();
2874 SDValue CarryExt = DAG.getBoolExtOrTrunc(CarryIn, DL, VT, CarryVT);
2875 AddToWorklist(CarryExt.getNode());
2876 return CombineTo(N, DAG.getNode(ISD::AND, DL, VT, CarryExt,
2877 DAG.getConstant(1, DL, VT)),
2878 DAG.getConstant(0, DL, CarryVT));
2879 }
2880
2881 if (SDValue Combined = visitADDCARRYLike(N0, N1, CarryIn, N))
2882 return Combined;
2883
2884 if (SDValue Combined = visitADDCARRYLike(N1, N0, CarryIn, N))
2885 return Combined;
2886
2887 return SDValue();
2888}
2889
2890SDValue DAGCombiner::visitSADDO_CARRY(SDNode *N) {
2891 SDValue N0 = N->getOperand(0);
2892 SDValue N1 = N->getOperand(1);
2893 SDValue CarryIn = N->getOperand(2);
2894 SDLoc DL(N);
2895
2896 // canonicalize constant to RHS
2897 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
2898 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2899 if (N0C && !N1C)
2900 return DAG.getNode(ISD::SADDO_CARRY, DL, N->getVTList(), N1, N0, CarryIn);
2901
2902 // fold (saddo_carry x, y, false) -> (saddo x, y)
2903 if (isNullConstant(CarryIn)) {
2904 if (!LegalOperations ||
2905 TLI.isOperationLegalOrCustom(ISD::SADDO, N->getValueType(0)))
2906 return DAG.getNode(ISD::SADDO, DL, N->getVTList(), N0, N1);
2907 }
2908
2909 return SDValue();
2910}
2911
2912/**
2913 * If we are facing some sort of diamond carry propapagtion pattern try to
2914 * break it up to generate something like:
2915 * (addcarry X, 0, (addcarry A, B, Z):Carry)
2916 *
2917 * The end result is usually an increase in operation required, but because the
2918 * carry is now linearized, other tranforms can kick in and optimize the DAG.
2919 *
2920 * Patterns typically look something like
2921 * (uaddo A, B)
2922 * / \
2923 * Carry Sum
2924 * | \
2925 * | (addcarry *, 0, Z)
2926 * | /
2927 * \ Carry
2928 * | /
2929 * (addcarry X, *, *)
2930 *
2931 * But numerous variation exist. Our goal is to identify A, B, X and Z and
2932 * produce a combine with a single path for carry propagation.
2933 */
2934static SDValue combineADDCARRYDiamond(DAGCombiner &Combiner, SelectionDAG &DAG,
2935 SDValue X, SDValue Carry0, SDValue Carry1,
2936 SDNode *N) {
2937 if (Carry1.getResNo() != 1 || Carry0.getResNo() != 1)
2938 return SDValue();
2939 if (Carry1.getOpcode() != ISD::UADDO)
2940 return SDValue();
2941
2942 SDValue Z;
2943
2944 /**
2945 * First look for a suitable Z. It will present itself in the form of
2946 * (addcarry Y, 0, Z) or its equivalent (uaddo Y, 1) for Z=true
2947 */
2948 if (Carry0.getOpcode() == ISD::ADDCARRY &&
2949 isNullConstant(Carry0.getOperand(1))) {
2950 Z = Carry0.getOperand(2);
2951 } else if (Carry0.getOpcode() == ISD::UADDO &&
2952 isOneConstant(Carry0.getOperand(1))) {
2953 EVT VT = Combiner.getSetCCResultType(Carry0.getValueType());
2954 Z = DAG.getConstant(1, SDLoc(Carry0.getOperand(1)), VT);
2955 } else {
2956 // We couldn't find a suitable Z.
2957 return SDValue();
2958 }
2959
2960
2961 auto cancelDiamond = [&](SDValue A,SDValue B) {
2962 SDLoc DL(N);
2963 SDValue NewY = DAG.getNode(ISD::ADDCARRY, DL, Carry0->getVTList(), A, B, Z);
2964 Combiner.AddToWorklist(NewY.getNode());
2965 return DAG.getNode(ISD::ADDCARRY, DL, N->getVTList(), X,
2966 DAG.getConstant(0, DL, X.getValueType()),
2967 NewY.getValue(1));
2968 };
2969
2970 /**
2971 * (uaddo A, B)
2972 * |
2973 * Sum
2974 * |
2975 * (addcarry *, 0, Z)
2976 */
2977 if (Carry0.getOperand(0) == Carry1.getValue(0)) {
2978 return cancelDiamond(Carry1.getOperand(0), Carry1.getOperand(1));
2979 }
2980
2981 /**
2982 * (addcarry A, 0, Z)
2983 * |
2984 * Sum
2985 * |
2986 * (uaddo *, B)
2987 */
2988 if (Carry1.getOperand(0) == Carry0.getValue(0)) {
2989 return cancelDiamond(Carry0.getOperand(0), Carry1.getOperand(1));
2990 }
2991
2992 if (Carry1.getOperand(1) == Carry0.getValue(0)) {
2993 return cancelDiamond(Carry1.getOperand(0), Carry0.getOperand(0));
2994 }
2995
2996 return SDValue();
2997}
2998
2999// If we are facing some sort of diamond carry/borrow in/out pattern try to
3000// match patterns like:
3001//
3002// (uaddo A, B) CarryIn
3003// | \ |
3004// | \ |
3005// PartialSum PartialCarryOutX /
3006// | | /
3007// | ____|____________/
3008// | / |
3009// (uaddo *, *) \________
3010// | \ \
3011// | \ |
3012// | PartialCarryOutY |
3013// | \ |
3014// | \ /
3015// AddCarrySum | ______/
3016// | /
3017// CarryOut = (or *, *)
3018//
3019// And generate ADDCARRY (or SUBCARRY) with two result values:
3020//
3021// {AddCarrySum, CarryOut} = (addcarry A, B, CarryIn)
3022//
3023// Our goal is to identify A, B, and CarryIn and produce ADDCARRY/SUBCARRY with
3024// a single path for carry/borrow out propagation:
3025static SDValue combineCarryDiamond(DAGCombiner &Combiner, SelectionDAG &DAG,
3026 const TargetLowering &TLI, SDValue Carry0,
3027 SDValue Carry1, SDNode *N) {
3028 if (Carry0.getResNo() != 1 || Carry1.getResNo() != 1)
3029 return SDValue();
3030 unsigned Opcode = Carry0.getOpcode();
3031 if (Opcode != Carry1.getOpcode())
3032 return SDValue();
3033 if (Opcode != ISD::UADDO && Opcode != ISD::USUBO)
3034 return SDValue();
3035
3036 // Canonicalize the add/sub of A and B as Carry0 and the add/sub of the
3037 // carry/borrow in as Carry1. (The top and middle uaddo nodes respectively in
3038 // the above ASCII art.)
3039 if (Carry1.getOperand(0) != Carry0.getValue(0) &&
3040 Carry1.getOperand(1) != Carry0.getValue(0))
3041 std::swap(Carry0, Carry1);
3042 if (Carry1.getOperand(0) != Carry0.getValue(0) &&
3043 Carry1.getOperand(1) != Carry0.getValue(0))
3044 return SDValue();
3045
3046 // The carry in value must be on the righthand side for subtraction.
3047 unsigned CarryInOperandNum =
3048 Carry1.getOperand(0) == Carry0.getValue(0) ? 1 : 0;
3049 if (Opcode == ISD::USUBO && CarryInOperandNum != 1)
3050 return SDValue();
3051 SDValue CarryIn = Carry1.getOperand(CarryInOperandNum);
3052
3053 unsigned NewOp = Opcode == ISD::UADDO ? ISD::ADDCARRY : ISD::SUBCARRY;
3054 if (!TLI.isOperationLegalOrCustom(NewOp, Carry0.getValue(0).getValueType()))
3055 return SDValue();
3056
3057 // Verify that the carry/borrow in is plausibly a carry/borrow bit.
3058 // TODO: make getAsCarry() aware of how partial carries are merged.
3059 if (CarryIn.getOpcode() != ISD::ZERO_EXTEND)
3060 return SDValue();
3061 CarryIn = CarryIn.getOperand(0);
3062 if (CarryIn.getValueType() != MVT::i1)
3063 return SDValue();
3064
3065 SDLoc DL(N);
3066 SDValue Merged =
3067 DAG.getNode(NewOp, DL, Carry1->getVTList(), Carry0.getOperand(0),
3068 Carry0.getOperand(1), CarryIn);
3069
3070 // Please note that because we have proven that the result of the UADDO/USUBO
3071 // of A and B feeds into the UADDO/USUBO that does the carry/borrow in, we can
3072 // therefore prove that if the first UADDO/USUBO overflows, the second
3073 // UADDO/USUBO cannot. For example consider 8-bit numbers where 0xFF is the
3074 // maximum value.
3075 //
3076 // 0xFF + 0xFF == 0xFE with carry but 0xFE + 1 does not carry
3077 // 0x00 - 0xFF == 1 with a carry/borrow but 1 - 1 == 0 (no carry/borrow)
3078 //
3079 // This is important because it means that OR and XOR can be used to merge
3080 // carry flags; and that AND can return a constant zero.
3081 //
3082 // TODO: match other operations that can merge flags (ADD, etc)
3083 DAG.ReplaceAllUsesOfValueWith(Carry1.getValue(0), Merged.getValue(0));
3084 if (N->getOpcode() == ISD::AND)
3085 return DAG.getConstant(0, DL, MVT::i1);
3086 return Merged.getValue(1);
3087}
3088
3089SDValue DAGCombiner::visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn,
3090 SDNode *N) {
3091 // fold (addcarry (xor a, -1), b, c) -> (subcarry b, a, !c) and flip carry.
3092 if (isBitwiseNot(N0))
3093 if (SDValue NotC = extractBooleanFlip(CarryIn, DAG, TLI, true)) {
3094 SDLoc DL(N);
3095 SDValue Sub = DAG.getNode(ISD::SUBCARRY, DL, N->getVTList(), N1,
3096 N0.getOperand(0), NotC);
3097 return CombineTo(
3098 N, Sub, DAG.getLogicalNOT(DL, Sub.getValue(1), Sub->getValueType(1)));
3099 }
3100
3101 // Iff the flag result is dead:
3102 // (addcarry (add|uaddo X, Y), 0, Carry) -> (addcarry X, Y, Carry)
3103 // Don't do this if the Carry comes from the uaddo. It won't remove the uaddo
3104 // or the dependency between the instructions.
3105 if ((N0.getOpcode() == ISD::ADD ||
3106 (N0.getOpcode() == ISD::UADDO && N0.getResNo() == 0 &&
3107 N0.getValue(1) != CarryIn)) &&
3108 isNullConstant(N1) && !N->hasAnyUseOfValue(1))
3109 return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(),
3110 N0.getOperand(0), N0.getOperand(1), CarryIn);
3111
3112 /**
3113 * When one of the addcarry argument is itself a carry, we may be facing
3114 * a diamond carry propagation. In which case we try to transform the DAG
3115 * to ensure linear carry propagation if that is possible.
3116 */
3117 if (auto Y = getAsCarry(TLI, N1)) {
3118 // Because both are carries, Y and Z can be swapped.
3119 if (auto R = combineADDCARRYDiamond(*this, DAG, N0, Y, CarryIn, N))
3120 return R;
3121 if (auto R = combineADDCARRYDiamond(*this, DAG, N0, CarryIn, Y, N))
3122 return R;
3123 }
3124
3125 return SDValue();
3126}
3127
3128// Since it may not be valid to emit a fold to zero for vector initializers
3129// check if we can before folding.
3130static SDValue tryFoldToZero(const SDLoc &DL, const TargetLowering &TLI, EVT VT,
3131 SelectionDAG &DAG, bool LegalOperations) {
3132 if (!VT.isVector())
3133 return DAG.getConstant(0, DL, VT);
3134 if (!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT))
3135 return DAG.getConstant(0, DL, VT);
3136 return SDValue();
3137}
3138
3139SDValue DAGCombiner::visitSUB(SDNode *N) {
3140 SDValue N0 = N->getOperand(0);
3141 SDValue N1 = N->getOperand(1);
3142 EVT VT = N0.getValueType();
3143 SDLoc DL(N);
3144
3145 // fold vector ops
3146 if (VT.isVector()) {
3147 if (SDValue FoldedVOp = SimplifyVBinOp(N))
3148 return FoldedVOp;
3149
3150 // fold (sub x, 0) -> x, vector edition
3151 if (ISD::isBuildVectorAllZeros(N1.getNode()))
3152 return N0;
3153 }
3154
3155 // fold (sub x, x) -> 0
3156 // FIXME: Refactor this and xor and other similar operations together.
3157 if (N0 == N1)
3158 return tryFoldToZero(DL, TLI, VT, DAG, LegalOperations);
3159
3160 // fold (sub c1, c2) -> c3
3161 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N0, N1}))
3162 return C;
3163
3164 if (SDValue NewSel = foldBinOpIntoSelect(N))
3165 return NewSel;
3166
3167 ConstantSDNode *N1C = getAsNonOpaqueConstant(N1);
3168
3169 // fold (sub x, c) -> (add x, -c)
3170 if (N1C) {
3171 return DAG.getNode(ISD::ADD, DL, VT, N0,
3172 DAG.getConstant(-N1C->getAPIntValue(), DL, VT));
3173 }
3174
3175 if (isNullOrNullSplat(N0)) {
3176 unsigned BitWidth = VT.getScalarSizeInBits();
3177 // Right-shifting everything out but the sign bit followed by negation is
3178 // the same as flipping arithmetic/logical shift type without the negation:
3179 // -(X >>u 31) -> (X >>s 31)
3180 // -(X >>s 31) -> (X >>u 31)
3181 if (N1->getOpcode() == ISD::SRA || N1->getOpcode() == ISD::SRL) {
3182 ConstantSDNode *ShiftAmt = isConstOrConstSplat(N1.getOperand(1));
3183 if (ShiftAmt && ShiftAmt->getAPIntValue() == (BitWidth - 1)) {
3184 auto NewSh = N1->getOpcode() == ISD::SRA ? ISD::SRL : ISD::SRA;
3185 if (!LegalOperations || TLI.isOperationLegal(NewSh, VT))
3186 return DAG.getNode(NewSh, DL, VT, N1.getOperand(0), N1.getOperand(1));
3187 }
3188 }
3189
3190 // 0 - X --> 0 if the sub is NUW.
3191 if (N->getFlags().hasNoUnsignedWrap())
3192 return N0;
3193
3194 if (DAG.MaskedValueIsZero(N1, ~APInt::getSignMask(BitWidth))) {
3195 // N1 is either 0 or the minimum signed value. If the sub is NSW, then
3196 // N1 must be 0 because negating the minimum signed value is undefined.
3197 if (N->getFlags().hasNoSignedWrap())
3198 return N0;
3199
3200 // 0 - X --> X if X is 0 or the minimum signed value.
3201 return N1;
3202 }
3203
3204 // Convert 0 - abs(x).
3205 SDValue Result;
3206 if (N1->getOpcode() == ISD::ABS &&
3207 !TLI.isOperationLegalOrCustom(ISD::ABS, VT) &&
3208 TLI.expandABS(N1.getNode(), Result, DAG, true))
3209 return Result;
3210 }
3211
3212 // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1)
3213 if (isAllOnesOrAllOnesSplat(N0))
3214 return DAG.getNode(ISD::XOR, DL, VT, N1, N0);
3215
3216 // fold (A - (0-B)) -> A+B
3217 if (N1.getOpcode() == ISD::SUB && isNullOrNullSplat(N1.getOperand(0)))
3218 return DAG.getNode(ISD::ADD, DL, VT, N0, N1.getOperand(1));
3219
3220 // fold A-(A-B) -> B
3221 if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(0))
3222 return N1.getOperand(1);
3223
3224 // fold (A+B)-A -> B
3225 if (N0.getOpcode() == ISD::ADD && N0.getOperand(0) == N1)
3226 return N0.getOperand(1);
3227
3228 // fold (A+B)-B -> A
3229 if (N0.getOpcode() == ISD::ADD && N0.getOperand(1) == N1)
3230 return N0.getOperand(0);
3231
3232 // fold (A+C1)-C2 -> A+(C1-C2)
3233 if (N0.getOpcode() == ISD::ADD &&
3234 isConstantOrConstantVector(N1, /* NoOpaques */ true) &&
3235 isConstantOrConstantVector(N0.getOperand(1), /* NoOpaques */ true)) {
3236 SDValue NewC =
3237 DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N0.getOperand(1), N1});
3238 assert(NewC && "Constant folding failed")((NewC && "Constant folding failed") ? static_cast<
void> (0) : __assert_fail ("NewC && \"Constant folding failed\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3238, __PRETTY_FUNCTION__))
;
3239 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), NewC);
3240 }
3241
3242 // fold C2-(A+C1) -> (C2-C1)-A
3243 if (N1.getOpcode() == ISD::ADD) {
3244 SDValue N11 = N1.getOperand(1);
3245 if (isConstantOrConstantVector(N0, /* NoOpaques */ true) &&
3246 isConstantOrConstantVector(N11, /* NoOpaques */ true)) {
3247 SDValue NewC = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N0, N11});
3248 assert(NewC && "Constant folding failed")((NewC && "Constant folding failed") ? static_cast<
void> (0) : __assert_fail ("NewC && \"Constant folding failed\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3248, __PRETTY_FUNCTION__))
;
3249 return DAG.getNode(ISD::SUB, DL, VT, NewC, N1.getOperand(0));
3250 }
3251 }
3252
3253 // fold (A-C1)-C2 -> A-(C1+C2)
3254 if (N0.getOpcode() == ISD::SUB &&
3255 isConstantOrConstantVector(N1, /* NoOpaques */ true) &&
3256 isConstantOrConstantVector(N0.getOperand(1), /* NoOpaques */ true)) {
3257 SDValue NewC =
3258 DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N0.getOperand(1), N1});
3259 assert(NewC && "Constant folding failed")((NewC && "Constant folding failed") ? static_cast<
void> (0) : __assert_fail ("NewC && \"Constant folding failed\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3259, __PRETTY_FUNCTION__))
;
3260 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), NewC);
3261 }
3262
3263 // fold (c1-A)-c2 -> (c1-c2)-A
3264 if (N0.getOpcode() == ISD::SUB &&
3265 isConstantOrConstantVector(N1, /* NoOpaques */ true) &&
3266 isConstantOrConstantVector(N0.getOperand(0), /* NoOpaques */ true)) {
3267 SDValue NewC =
3268 DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N0.getOperand(0), N1});
3269 assert(NewC && "Constant folding failed")((NewC && "Constant folding failed") ? static_cast<
void> (0) : __assert_fail ("NewC && \"Constant folding failed\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3269, __PRETTY_FUNCTION__))
;
3270 return DAG.getNode(ISD::SUB, DL, VT, NewC, N0.getOperand(1));
3271 }
3272
3273 // fold ((A+(B+or-C))-B) -> A+or-C
3274 if (N0.getOpcode() == ISD::ADD &&
3275 (N0.getOperand(1).getOpcode() == ISD::SUB ||
3276 N0.getOperand(1).getOpcode() == ISD::ADD) &&
3277 N0.getOperand(1).getOperand(0) == N1)
3278 return DAG.getNode(N0.getOperand(1).getOpcode(), DL, VT, N0.getOperand(0),
3279 N0.getOperand(1).getOperand(1));
3280
3281 // fold ((A+(C+B))-B) -> A+C
3282 if (N0.getOpcode() == ISD::ADD && N0.getOperand(1).getOpcode() == ISD::ADD &&
3283 N0.getOperand(1).getOperand(1) == N1)
3284 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0),
3285 N0.getOperand(1).getOperand(0));
3286
3287 // fold ((A-(B-C))-C) -> A-B
3288 if (N0.getOpcode() == ISD::SUB && N0.getOperand(1).getOpcode() == ISD::SUB &&
3289 N0.getOperand(1).getOperand(1) == N1)
3290 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0),
3291 N0.getOperand(1).getOperand(0));
3292
3293 // fold (A-(B-C)) -> A+(C-B)
3294 if (N1.getOpcode() == ISD::SUB && N1.hasOneUse())
3295 return DAG.getNode(ISD::ADD, DL, VT, N0,
3296 DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(1),
3297 N1.getOperand(0)));
3298
3299 // A - (A & B) -> A & (~B)
3300 if (N1.getOpcode() == ISD::AND) {
3301 SDValue A = N1.getOperand(0);
3302 SDValue B = N1.getOperand(1);
3303 if (A != N0)
3304 std::swap(A, B);
3305 if (A == N0 &&
3306 (N1.hasOneUse() || isConstantOrConstantVector(B, /*NoOpaques=*/true))) {
3307 SDValue InvB =
3308 DAG.getNode(ISD::XOR, DL, VT, B, DAG.getAllOnesConstant(DL, VT));
3309 return DAG.getNode(ISD::AND, DL, VT, A, InvB);
3310 }
3311 }
3312
3313 // fold (X - (-Y * Z)) -> (X + (Y * Z))
3314 if (N1.getOpcode() == ISD::MUL && N1.hasOneUse()) {
3315 if (N1.getOperand(0).getOpcode() == ISD::SUB &&
3316 isNullOrNullSplat(N1.getOperand(0).getOperand(0))) {
3317 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT,
3318 N1.getOperand(0).getOperand(1),
3319 N1.getOperand(1));
3320 return DAG.getNode(ISD::ADD, DL, VT, N0, Mul);
3321 }
3322 if (N1.getOperand(1).getOpcode() == ISD::SUB &&
3323 isNullOrNullSplat(N1.getOperand(1).getOperand(0))) {
3324 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT,
3325 N1.getOperand(0),
3326 N1.getOperand(1).getOperand(1));
3327 return DAG.getNode(ISD::ADD, DL, VT, N0, Mul);
3328 }
3329 }
3330
3331 // If either operand of a sub is undef, the result is undef
3332 if (N0.isUndef())
3333 return N0;
3334 if (N1.isUndef())
3335 return N1;
3336
3337 if (SDValue V = foldAddSubBoolOfMaskedVal(N, DAG))
3338 return V;
3339
3340 if (SDValue V = foldAddSubOfSignBit(N, DAG))
3341 return V;
3342
3343 if (SDValue V = foldAddSubMasked1(false, N0, N1, DAG, SDLoc(N)))
3344 return V;
3345
3346 // (x - y) - 1 -> add (xor y, -1), x
3347 if (N0.hasOneUse() && N0.getOpcode() == ISD::SUB && isOneOrOneSplat(N1)) {
3348 SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(1),
3349 DAG.getAllOnesConstant(DL, VT));
3350 return DAG.getNode(ISD::ADD, DL, VT, Xor, N0.getOperand(0));
3351 }
3352
3353 // Look for:
3354 // sub y, (xor x, -1)
3355 // And if the target does not like this form then turn into:
3356 // add (add x, y), 1
3357 if (TLI.preferIncOfAddToSubOfNot(VT) && N1.hasOneUse() && isBitwiseNot(N1)) {
3358 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, N1.getOperand(0));
3359 return DAG.getNode(ISD::ADD, DL, VT, Add, DAG.getConstant(1, DL, VT));
3360 }
3361
3362 // Hoist one-use addition by non-opaque constant:
3363 // (x + C) - y -> (x - y) + C
3364 if (N0.hasOneUse() && N0.getOpcode() == ISD::ADD &&
3365 isConstantOrConstantVector(N0.getOperand(1), /*NoOpaques=*/true)) {
3366 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), N1);
3367 return DAG.getNode(ISD::ADD, DL, VT, Sub, N0.getOperand(1));
3368 }
3369 // y - (x + C) -> (y - x) - C
3370 if (N1.hasOneUse() && N1.getOpcode() == ISD::ADD &&
3371 isConstantOrConstantVector(N1.getOperand(1), /*NoOpaques=*/true)) {
3372 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, N1.getOperand(0));
3373 return DAG.getNode(ISD::SUB, DL, VT, Sub, N1.getOperand(1));
3374 }
3375 // (x - C) - y -> (x - y) - C
3376 // This is necessary because SUB(X,C) -> ADD(X,-C) doesn't work for vectors.
3377 if (N0.hasOneUse() && N0.getOpcode() == ISD::SUB &&
3378 isConstantOrConstantVector(N0.getOperand(1), /*NoOpaques=*/true)) {
3379 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), N1);
3380 return DAG.getNode(ISD::SUB, DL, VT, Sub, N0.getOperand(1));
3381 }
3382 // (C - x) - y -> C - (x + y)
3383 if (N0.hasOneUse() && N0.getOpcode() == ISD::SUB &&
3384 isConstantOrConstantVector(N0.getOperand(0), /*NoOpaques=*/true)) {
3385 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(1), N1);
3386 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), Add);
3387 }
3388
3389 // If the target's bool is represented as 0/-1, prefer to make this 'add 0/-1'
3390 // rather than 'sub 0/1' (the sext should get folded).
3391 // sub X, (zext i1 Y) --> add X, (sext i1 Y)
3392 if (N1.getOpcode() == ISD::ZERO_EXTEND &&
3393 N1.getOperand(0).getScalarValueSizeInBits() == 1 &&
3394 TLI.getBooleanContents(VT) ==
3395 TargetLowering::ZeroOrNegativeOneBooleanContent) {
3396 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, N1.getOperand(0));
3397 return DAG.getNode(ISD::ADD, DL, VT, N0, SExt);
3398 }
3399
3400 // fold Y = sra (X, size(X)-1); sub (xor (X, Y), Y) -> (abs X)
3401 if (TLI.isOperationLegalOrCustom(ISD::ABS, VT)) {
3402 if (N0.getOpcode() == ISD::XOR && N1.getOpcode() == ISD::SRA) {
3403 SDValue X0 = N0.getOperand(0), X1 = N0.getOperand(1);
3404 SDValue S0 = N1.getOperand(0);
3405 if ((X0 == S0 && X1 == N1) || (X0 == N1 && X1 == S0))
3406 if (ConstantSDNode *C = isConstOrConstSplat(N1.getOperand(1)))
3407 if (C->getAPIntValue() == (VT.getScalarSizeInBits() - 1))
3408 return DAG.getNode(ISD::ABS, SDLoc(N), VT, S0);
3409 }
3410 }
3411
3412 // If the relocation model supports it, consider symbol offsets.
3413 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N0))
3414 if (!LegalOperations && TLI.isOffsetFoldingLegal(GA)) {
3415 // fold (sub Sym, c) -> Sym-c
3416 if (N1C && GA->getOpcode() == ISD::GlobalAddress)
3417 return DAG.getGlobalAddress(GA->getGlobal(), SDLoc(N1C), VT,
3418 GA->getOffset() -
3419 (uint64_t)N1C->getSExtValue());
3420 // fold (sub Sym+c1, Sym+c2) -> c1-c2
3421 if (GlobalAddressSDNode *GB = dyn_cast<GlobalAddressSDNode>(N1))
3422 if (GA->getGlobal() == GB->getGlobal())
3423 return DAG.getConstant((uint64_t)GA->getOffset() - GB->getOffset(),
3424 DL, VT);
3425 }
3426
3427 // sub X, (sextinreg Y i1) -> add X, (and Y 1)
3428 if (N1.getOpcode() == ISD::SIGN_EXTEND_INREG) {
3429 VTSDNode *TN = cast<VTSDNode>(N1.getOperand(1));
3430 if (TN->getVT() == MVT::i1) {
3431 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0),
3432 DAG.getConstant(1, DL, VT));
3433 return DAG.getNode(ISD::ADD, DL, VT, N0, ZExt);
3434 }
3435 }
3436
3437 // canonicalize (sub X, (vscale * C)) to (add X, (vscale * -C))
3438 if (N1.getOpcode() == ISD::VSCALE) {
3439 const APInt &IntVal = N1.getConstantOperandAPInt(0);
3440 return DAG.getNode(ISD::ADD, DL, VT, N0, DAG.getVScale(DL, VT, -IntVal));
3441 }
3442
3443 // Prefer an add for more folding potential and possibly better codegen:
3444 // sub N0, (lshr N10, width-1) --> add N0, (ashr N10, width-1)
3445 if (!LegalOperations && N1.getOpcode() == ISD::SRL && N1.hasOneUse()) {
3446 SDValue ShAmt = N1.getOperand(1);
3447 ConstantSDNode *ShAmtC = isConstOrConstSplat(ShAmt);
3448 if (ShAmtC &&
3449 ShAmtC->getAPIntValue() == (N1.getScalarValueSizeInBits() - 1)) {
3450 SDValue SRA = DAG.getNode(ISD::SRA, DL, VT, N1.getOperand(0), ShAmt);
3451 return DAG.getNode(ISD::ADD, DL, VT, N0, SRA);
3452 }
3453 }
3454
3455 if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT)) {
3456 // (sub Carry, X) -> (addcarry (sub 0, X), 0, Carry)
3457 if (SDValue Carry = getAsCarry(TLI, N0)) {
3458 SDValue X = N1;
3459 SDValue Zero = DAG.getConstant(0, DL, VT);
3460 SDValue NegX = DAG.getNode(ISD::SUB, DL, VT, Zero, X);
3461 return DAG.getNode(ISD::ADDCARRY, DL,
3462 DAG.getVTList(VT, Carry.getValueType()), NegX, Zero,
3463 Carry);
3464 }
3465 }
3466
3467 return SDValue();
3468}
3469
3470SDValue DAGCombiner::visitSUBSAT(SDNode *N) {
3471 SDValue N0 = N->getOperand(0);
3472 SDValue N1 = N->getOperand(1);
3473 EVT VT = N0.getValueType();
3474 SDLoc DL(N);
3475
3476 // fold vector ops
3477 if (VT.isVector()) {
3478 // TODO SimplifyVBinOp
3479
3480 // fold (sub_sat x, 0) -> x, vector edition
3481 if (ISD::isBuildVectorAllZeros(N1.getNode()))
3482 return N0;
3483 }
3484
3485 // fold (sub_sat x, undef) -> 0
3486 if (N0.isUndef() || N1.isUndef())
3487 return DAG.getConstant(0, DL, VT);
3488
3489 // fold (sub_sat x, x) -> 0
3490 if (N0 == N1)
3491 return DAG.getConstant(0, DL, VT);
3492
3493 // fold (sub_sat c1, c2) -> c3
3494 if (SDValue C = DAG.FoldConstantArithmetic(N->getOpcode(), DL, VT, {N0, N1}))
3495 return C;
3496
3497 // fold (sub_sat x, 0) -> x
3498 if (isNullConstant(N1))
3499 return N0;
3500
3501 return SDValue();
3502}
3503
3504SDValue DAGCombiner::visitSUBC(SDNode *N) {
3505 SDValue N0 = N->getOperand(0);
3506 SDValue N1 = N->getOperand(1);
3507 EVT VT = N0.getValueType();
3508 SDLoc DL(N);
3509
3510 // If the flag result is dead, turn this into an SUB.
3511 if (!N->hasAnyUseOfValue(1))
3512 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1),
3513 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
3514
3515 // fold (subc x, x) -> 0 + no borrow
3516 if (N0 == N1)
3517 return CombineTo(N, DAG.getConstant(0, DL, VT),
3518 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
3519
3520 // fold (subc x, 0) -> x + no borrow
3521 if (isNullConstant(N1))
3522 return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
3523
3524 // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1) + no borrow
3525 if (isAllOnesConstant(N0))
3526 return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0),
3527 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
3528
3529 return SDValue();
3530}
3531
3532SDValue DAGCombiner::visitSUBO(SDNode *N) {
3533 SDValue N0 = N->getOperand(0);
3534 SDValue N1 = N->getOperand(1);
3535 EVT VT = N0.getValueType();
3536 bool IsSigned = (ISD::SSUBO == N->getOpcode());
3537
3538 EVT CarryVT = N->getValueType(1);
3539 SDLoc DL(N);
3540
3541 // If the flag result is dead, turn this into an SUB.
3542 if (!N->hasAnyUseOfValue(1))
3543 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1),
3544 DAG.getUNDEF(CarryVT));
3545
3546 // fold (subo x, x) -> 0 + no borrow
3547 if (N0 == N1)
3548 return CombineTo(N, DAG.getConstant(0, DL, VT),
3549 DAG.getConstant(0, DL, CarryVT));
3550
3551 ConstantSDNode *N1C = getAsNonOpaqueConstant(N1);
3552
3553 // fold (subox, c) -> (addo x, -c)
3554 if (IsSigned && N1C && !N1C->getAPIntValue().isMinSignedValue()) {
3555 return DAG.getNode(ISD::SADDO, DL, N->getVTList(), N0,
3556 DAG.getConstant(-N1C->getAPIntValue(), DL, VT));
3557 }
3558
3559 // fold (subo x, 0) -> x + no borrow
3560 if (isNullOrNullSplat(N1))
3561 return CombineTo(N, N0, DAG.getConstant(0, DL, CarryVT));
3562
3563 // Canonicalize (usubo -1, x) -> ~x, i.e. (xor x, -1) + no borrow
3564 if (!IsSigned && isAllOnesOrAllOnesSplat(N0))
3565 return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0),
3566 DAG.getConstant(0, DL, CarryVT));
3567
3568 return SDValue();
3569}
3570
3571SDValue DAGCombiner::visitSUBE(SDNode *N) {
3572 SDValue N0 = N->getOperand(0);
3573 SDValue N1 = N->getOperand(1);
3574 SDValue CarryIn = N->getOperand(2);
3575
3576 // fold (sube x, y, false) -> (subc x, y)
3577 if (CarryIn.getOpcode() == ISD::CARRY_FALSE)
3578 return DAG.getNode(ISD::SUBC, SDLoc(N), N->getVTList(), N0, N1);
3579
3580 return SDValue();
3581}
3582
3583SDValue DAGCombiner::visitSUBCARRY(SDNode *N) {
3584 SDValue N0 = N->getOperand(0);
3585 SDValue N1 = N->getOperand(1);
3586 SDValue CarryIn = N->getOperand(2);
3587
3588 // fold (subcarry x, y, false) -> (usubo x, y)
3589 if (isNullConstant(CarryIn)) {
3590 if (!LegalOperations ||
3591 TLI.isOperationLegalOrCustom(ISD::USUBO, N->getValueType(0)))
3592 return DAG.getNode(ISD::USUBO, SDLoc(N), N->getVTList(), N0, N1);
3593 }
3594
3595 return SDValue();
3596}
3597
3598SDValue DAGCombiner::visitSSUBO_CARRY(SDNode *N) {
3599 SDValue N0 = N->getOperand(0);
3600 SDValue N1 = N->getOperand(1);
3601 SDValue CarryIn = N->getOperand(2);
3602
3603 // fold (ssubo_carry x, y, false) -> (ssubo x, y)
3604 if (isNullConstant(CarryIn)) {
3605 if (!LegalOperations ||
3606 TLI.isOperationLegalOrCustom(ISD::SSUBO, N->getValueType(0)))
3607 return DAG.getNode(ISD::SSUBO, SDLoc(N), N->getVTList(), N0, N1);
3608 }
3609
3610 return SDValue();
3611}
3612
3613// Notice that "mulfix" can be any of SMULFIX, SMULFIXSAT, UMULFIX and
3614// UMULFIXSAT here.
3615SDValue DAGCombiner::visitMULFIX(SDNode *N) {
3616 SDValue N0 = N->getOperand(0);
3617 SDValue N1 = N->getOperand(1);
3618 SDValue Scale = N->getOperand(2);
3619 EVT VT = N0.getValueType();
3620
3621 // fold (mulfix x, undef, scale) -> 0
3622 if (N0.isUndef() || N1.isUndef())
3623 return DAG.getConstant(0, SDLoc(N), VT);
3624
3625 // Canonicalize constant to RHS (vector doesn't have to splat)
3626 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
3627 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
3628 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N1, N0, Scale);
3629
3630 // fold (mulfix x, 0, scale) -> 0
3631 if (isNullConstant(N1))
3632 return DAG.getConstant(0, SDLoc(N), VT);
3633
3634 return SDValue();
3635}
3636
3637SDValue DAGCombiner::visitMUL(SDNode *N) {
3638 SDValue N0 = N->getOperand(0);
3639 SDValue N1 = N->getOperand(1);
3640 EVT VT = N0.getValueType();
3641
3642 // fold (mul x, undef) -> 0
3643 if (N0.isUndef() || N1.isUndef())
3644 return DAG.getConstant(0, SDLoc(N), VT);
3645
3646 bool N1IsConst = false;
3647 bool N1IsOpaqueConst = false;
3648 APInt ConstValue1;
3649
3650 // fold vector ops
3651 if (VT.isVector()) {
3652 if (SDValue FoldedVOp = SimplifyVBinOp(N))
3653 return FoldedVOp;
3654
3655 N1IsConst = ISD::isConstantSplatVector(N1.getNode(), ConstValue1);
3656 assert((!N1IsConst ||(((!N1IsConst || ConstValue1.getBitWidth() == VT.getScalarSizeInBits
()) && "Splat APInt should be element width") ? static_cast
<void> (0) : __assert_fail ("(!N1IsConst || ConstValue1.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3658, __PRETTY_FUNCTION__))
3657 ConstValue1.getBitWidth() == VT.getScalarSizeInBits()) &&(((!N1IsConst || ConstValue1.getBitWidth() == VT.getScalarSizeInBits
()) && "Splat APInt should be element width") ? static_cast
<void> (0) : __assert_fail ("(!N1IsConst || ConstValue1.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3658, __PRETTY_FUNCTION__))
3658 "Splat APInt should be element width")(((!N1IsConst || ConstValue1.getBitWidth() == VT.getScalarSizeInBits
()) && "Splat APInt should be element width") ? static_cast
<void> (0) : __assert_fail ("(!N1IsConst || ConstValue1.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3658, __PRETTY_FUNCTION__))
;
3659 } else {
3660 N1IsConst = isa<ConstantSDNode>(N1);
3661 if (N1IsConst) {
3662 ConstValue1 = cast<ConstantSDNode>(N1)->getAPIntValue();
3663 N1IsOpaqueConst = cast<ConstantSDNode>(N1)->isOpaque();
3664 }
3665 }
3666
3667 // fold (mul c1, c2) -> c1*c2
3668 if (SDValue C = DAG.FoldConstantArithmetic(ISD::MUL, SDLoc(N), VT, {N0, N1}))
3669 return C;
3670
3671 // canonicalize constant to RHS (vector doesn't have to splat)
3672 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
3673 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
3674 return DAG.getNode(ISD::MUL, SDLoc(N), VT, N1, N0);
3675
3676 // fold (mul x, 0) -> 0
3677 if (N1IsConst && ConstValue1.isNullValue())
3678 return N1;
3679
3680 // fold (mul x, 1) -> x
3681 if (N1IsConst && ConstValue1.isOneValue())
3682 return N0;
3683
3684 if (SDValue NewSel = foldBinOpIntoSelect(N))
3685 return NewSel;
3686
3687 // fold (mul x, -1) -> 0-x
3688 if (N1IsConst && ConstValue1.isAllOnesValue()) {
3689 SDLoc DL(N);
3690 return DAG.getNode(ISD::SUB, DL, VT,
3691 DAG.getConstant(0, DL, VT), N0);
3692 }
3693
3694 // fold (mul x, (1 << c)) -> x << c
3695 if (isConstantOrConstantVector(N1, /*NoOpaques*/ true) &&
3696 DAG.isKnownToBeAPowerOfTwo(N1) &&
3697 (!VT.isVector() || Level <= AfterLegalizeVectorOps)) {
3698 SDLoc DL(N);
3699 SDValue LogBase2 = BuildLogBase2(N1, DL);
3700 EVT ShiftVT = getShiftAmountTy(N0.getValueType());
3701 SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ShiftVT);
3702 return DAG.getNode(ISD::SHL, DL, VT, N0, Trunc);
3703 }
3704
3705 // fold (mul x, -(1 << c)) -> -(x << c) or (-x) << c
3706 if (N1IsConst && !N1IsOpaqueConst && (-ConstValue1).isPowerOf2()) {
3707 unsigned Log2Val = (-ConstValue1).logBase2();
3708 SDLoc DL(N);
3709 // FIXME: If the input is something that is easily negated (e.g. a
3710 // single-use add), we should put the negate there.
3711 return DAG.getNode(ISD::SUB, DL, VT,
3712 DAG.getConstant(0, DL, VT),
3713 DAG.getNode(ISD::SHL, DL, VT, N0,
3714 DAG.getConstant(Log2Val, DL,
3715 getShiftAmountTy(N0.getValueType()))));
3716 }
3717
3718 // Try to transform:
3719 // (1) multiply-by-(power-of-2 +/- 1) into shift and add/sub.
3720 // mul x, (2^N + 1) --> add (shl x, N), x
3721 // mul x, (2^N - 1) --> sub (shl x, N), x
3722 // Examples: x * 33 --> (x << 5) + x
3723 // x * 15 --> (x << 4) - x
3724 // x * -33 --> -((x << 5) + x)
3725 // x * -15 --> -((x << 4) - x) ; this reduces --> x - (x << 4)
3726 // (2) multiply-by-(power-of-2 +/- power-of-2) into shifts and add/sub.
3727 // mul x, (2^N + 2^M) --> (add (shl x, N), (shl x, M))
3728 // mul x, (2^N - 2^M) --> (sub (shl x, N), (shl x, M))
3729 // Examples: x * 0x8800 --> (x << 15) + (x << 11)
3730 // x * 0xf800 --> (x << 16) - (x << 11)
3731 // x * -0x8800 --> -((x << 15) + (x << 11))
3732 // x * -0xf800 --> -((x << 16) - (x << 11)) ; (x << 11) - (x << 16)
3733 if (N1IsConst && TLI.decomposeMulByConstant(*DAG.getContext(), VT, N1)) {
3734 // TODO: We could handle more general decomposition of any constant by
3735 // having the target set a limit on number of ops and making a
3736 // callback to determine that sequence (similar to sqrt expansion).
3737 unsigned MathOp = ISD::DELETED_NODE;
3738 APInt MulC = ConstValue1.abs();
3739 // The constant `2` should be treated as (2^0 + 1).
3740 unsigned TZeros = MulC == 2 ? 0 : MulC.countTrailingZeros();
3741 MulC.lshrInPlace(TZeros);
3742 if ((MulC - 1).isPowerOf2())
3743 MathOp = ISD::ADD;
3744 else if ((MulC + 1).isPowerOf2())
3745 MathOp = ISD::SUB;
3746
3747 if (MathOp != ISD::DELETED_NODE) {
3748 unsigned ShAmt =
3749 MathOp == ISD::ADD ? (MulC - 1).logBase2() : (MulC + 1).logBase2();
3750 ShAmt += TZeros;
3751 assert(ShAmt < VT.getScalarSizeInBits() &&((ShAmt < VT.getScalarSizeInBits() && "multiply-by-constant generated out of bounds shift"
) ? static_cast<void> (0) : __assert_fail ("ShAmt < VT.getScalarSizeInBits() && \"multiply-by-constant generated out of bounds shift\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3752, __PRETTY_FUNCTION__))
3752 "multiply-by-constant generated out of bounds shift")((ShAmt < VT.getScalarSizeInBits() && "multiply-by-constant generated out of bounds shift"
) ? static_cast<void> (0) : __assert_fail ("ShAmt < VT.getScalarSizeInBits() && \"multiply-by-constant generated out of bounds shift\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3752, __PRETTY_FUNCTION__))
;
3753 SDLoc DL(N);
3754 SDValue Shl =
3755 DAG.getNode(ISD::SHL, DL, VT, N0, DAG.getConstant(ShAmt, DL, VT));
3756 SDValue R =
3757 TZeros ? DAG.getNode(MathOp, DL, VT, Shl,
3758 DAG.getNode(ISD::SHL, DL, VT, N0,
3759 DAG.getConstant(TZeros, DL, VT)))
3760 : DAG.getNode(MathOp, DL, VT, Shl, N0);
3761 if (ConstValue1.isNegative())
3762 R = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), R);
3763 return R;
3764 }
3765 }
3766
3767 // (mul (shl X, c1), c2) -> (mul X, c2 << c1)
3768 if (N0.getOpcode() == ISD::SHL &&
3769 isConstantOrConstantVector(N1, /* NoOpaques */ true) &&
3770 isConstantOrConstantVector(N0.getOperand(1), /* NoOpaques */ true)) {
3771 SDValue C3 = DAG.getNode(ISD::SHL, SDLoc(N), VT, N1, N0.getOperand(1));
3772 if (isConstantOrConstantVector(C3))
3773 return DAG.getNode(ISD::MUL, SDLoc(N), VT, N0.getOperand(0), C3);
3774 }
3775
3776 // Change (mul (shl X, C), Y) -> (shl (mul X, Y), C) when the shift has one
3777 // use.
3778 {
3779 SDValue Sh(nullptr, 0), Y(nullptr, 0);
3780
3781 // Check for both (mul (shl X, C), Y) and (mul Y, (shl X, C)).
3782 if (N0.getOpcode() == ISD::SHL &&
3783 isConstantOrConstantVector(N0.getOperand(1)) &&
3784 N0.getNode()->hasOneUse()) {
3785 Sh = N0; Y = N1;
3786 } else if (N1.getOpcode() == ISD::SHL &&
3787 isConstantOrConstantVector(N1.getOperand(1)) &&
3788 N1.getNode()->hasOneUse()) {
3789 Sh = N1; Y = N0;
3790 }
3791
3792 if (Sh.getNode()) {
3793 SDValue Mul = DAG.getNode(ISD::MUL, SDLoc(N), VT, Sh.getOperand(0), Y);
3794 return DAG.getNode(ISD::SHL, SDLoc(N), VT, Mul, Sh.getOperand(1));
3795 }
3796 }
3797
3798 // fold (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2)
3799 if (DAG.isConstantIntBuildVectorOrConstantInt(N1) &&
3800 N0.getOpcode() == ISD::ADD &&
3801 DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1)) &&
3802 isMulAddWithConstProfitable(N, N0, N1))
3803 return DAG.getNode(ISD::ADD, SDLoc(N), VT,
3804 DAG.getNode(ISD::MUL, SDLoc(N0), VT,
3805 N0.getOperand(0), N1),
3806 DAG.getNode(ISD::MUL, SDLoc(N1), VT,
3807 N0.getOperand(1), N1));
3808
3809 // Fold (mul (vscale * C0), C1) to (vscale * (C0 * C1)).
3810 if (N0.getOpcode() == ISD::VSCALE)
3811 if (ConstantSDNode *NC1 = isConstOrConstSplat(N1)) {
3812 const APInt &C0 = N0.getConstantOperandAPInt(0);
3813 const APInt &C1 = NC1->getAPIntValue();
3814 return DAG.getVScale(SDLoc(N), VT, C0 * C1);
3815 }
3816
3817 // Fold ((mul x, 0/undef) -> 0,
3818 // (mul x, 1) -> x) -> x)
3819 // -> and(x, mask)
3820 // We can replace vectors with '0' and '1' factors with a clearing mask.
3821 if (VT.isFixedLengthVector()) {
3822 unsigned NumElts = VT.getVectorNumElements();
3823 SmallBitVector ClearMask;
3824 ClearMask.reserve(NumElts);
3825 auto IsClearMask = [&ClearMask](ConstantSDNode *V) {
3826 if (!V || V->isNullValue()) {
3827 ClearMask.push_back(true);
3828 return true;
3829 }
3830 ClearMask.push_back(false);
3831 return V->isOne();
3832 };
3833 if ((!LegalOperations || TLI.isOperationLegalOrCustom(ISD::AND, VT)) &&
3834 ISD::matchUnaryPredicate(N1, IsClearMask, /*AllowUndefs*/ true)) {
3835 assert(N1.getOpcode() == ISD::BUILD_VECTOR && "Unknown constant vector")((N1.getOpcode() == ISD::BUILD_VECTOR && "Unknown constant vector"
) ? static_cast<void> (0) : __assert_fail ("N1.getOpcode() == ISD::BUILD_VECTOR && \"Unknown constant vector\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3835, __PRETTY_FUNCTION__))
;
3836 SDLoc DL(N);
3837 EVT LegalSVT = N1.getOperand(0).getValueType();
3838 SDValue Zero = DAG.getConstant(0, DL, LegalSVT);
3839 SDValue AllOnes = DAG.getAllOnesConstant(DL, LegalSVT);
3840 SmallVector<SDValue, 16> Mask(NumElts, AllOnes);
3841 for (unsigned I = 0; I != NumElts; ++I)
3842 if (ClearMask[I])
3843 Mask[I] = Zero;
3844 return DAG.getNode(ISD::AND, DL, VT, N0, DAG.getBuildVector(VT, DL, Mask));
3845 }
3846 }
3847
3848 // reassociate mul
3849 if (SDValue RMUL = reassociateOps(ISD::MUL, SDLoc(N), N0, N1, N->getFlags()))
3850 return RMUL;
3851
3852 return SDValue();
3853}
3854
3855/// Return true if divmod libcall is available.
3856static bool isDivRemLibcallAvailable(SDNode *Node, bool isSigned,
3857 const TargetLowering &TLI) {
3858 RTLIB::Libcall LC;
3859 EVT NodeType = Node->getValueType(0);
3860 if (!NodeType.isSimple())
3861 return false;
3862 switch (NodeType.getSimpleVT().SimpleTy) {
3863 default: return false; // No libcall for vector types.
3864 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
3865 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
3866 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
3867 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
3868 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
3869 }
3870
3871 return TLI.getLibcallName(LC) != nullptr;
3872}
3873
3874/// Issue divrem if both quotient and remainder are needed.
3875SDValue DAGCombiner::useDivRem(SDNode *Node) {
3876 if (Node->use_empty())
3877 return SDValue(); // This is a dead node, leave it alone.
3878
3879 unsigned Opcode = Node->getOpcode();
3880 bool isSigned = (Opcode == ISD::SDIV) || (Opcode == ISD::SREM);
3881 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3882
3883 // DivMod lib calls can still work on non-legal types if using lib-calls.
3884 EVT VT = Node->getValueType(0);
3885 if (VT.isVector() || !VT.isInteger())
3886 return SDValue();
3887
3888 if (!TLI.isTypeLegal(VT) && !TLI.isOperationCustom(DivRemOpc, VT))
3889 return SDValue();
3890
3891 // If DIVREM is going to get expanded into a libcall,
3892 // but there is no libcall available, then don't combine.
3893 if (!TLI.isOperationLegalOrCustom(DivRemOpc, VT) &&
3894 !isDivRemLibcallAvailable(Node, isSigned, TLI))
3895 return SDValue();
3896
3897 // If div is legal, it's better to do the normal expansion
3898 unsigned OtherOpcode = 0;
3899 if ((Opcode == ISD::SDIV) || (Opcode == ISD::UDIV)) {
3900 OtherOpcode = isSigned ? ISD::SREM : ISD::UREM;
3901 if (TLI.isOperationLegalOrCustom(Opcode, VT))
3902 return SDValue();
3903 } else {
3904 OtherOpcode = isSigned ? ISD::SDIV : ISD::UDIV;
3905 if (TLI.isOperationLegalOrCustom(OtherOpcode, VT))
3906 return SDValue();
3907 }
3908
3909 SDValue Op0 = Node->getOperand(0);
3910 SDValue Op1 = Node->getOperand(1);
3911 SDValue combined;
3912 for (SDNode::use_iterator UI = Op0.getNode()->use_begin(),
3913 UE = Op0.getNode()->use_end(); UI != UE; ++UI) {
3914 SDNode *User = *UI;
3915 if (User == Node || User->getOpcode() == ISD::DELETED_NODE ||
3916 User->use_empty())
3917 continue;
3918 // Convert the other matching node(s), too;
3919 // otherwise, the DIVREM may get target-legalized into something
3920 // target-specific that we won't be able to recognize.
3921 unsigned UserOpc = User->getOpcode();
3922 if ((UserOpc == Opcode || UserOpc == OtherOpcode || UserOpc == DivRemOpc) &&
3923 User->getOperand(0) == Op0 &&
3924 User->getOperand(1) == Op1) {
3925 if (!combined) {
3926 if (UserOpc == OtherOpcode) {
3927 SDVTList VTs = DAG.getVTList(VT, VT);
3928 combined = DAG.getNode(DivRemOpc, SDLoc(Node), VTs, Op0, Op1);
3929 } else if (UserOpc == DivRemOpc) {
3930 combined = SDValue(User, 0);
3931 } else {
3932 assert(UserOpc == Opcode)((UserOpc == Opcode) ? static_cast<void> (0) : __assert_fail
("UserOpc == Opcode", "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3932, __PRETTY_FUNCTION__))
;
3933 continue;
3934 }
3935 }
3936 if (UserOpc == ISD::SDIV || UserOpc == ISD::UDIV)
3937 CombineTo(User, combined);
3938 else if (UserOpc == ISD::SREM || UserOpc == ISD::UREM)
3939 CombineTo(User, combined.getValue(1));
3940 }
3941 }
3942 return combined;
3943}
3944
3945static SDValue simplifyDivRem(SDNode *N, SelectionDAG &DAG) {
3946 SDValue N0 = N->getOperand(0);
3947 SDValue N1 = N->getOperand(1);
3948 EVT VT = N->getValueType(0);
3949 SDLoc DL(N);
3950
3951 unsigned Opc = N->getOpcode();
3952 bool IsDiv = (ISD::SDIV == Opc) || (ISD::UDIV == Opc);
3953 ConstantSDNode *N1C = isConstOrConstSplat(N1);
3954
3955 // X / undef -> undef
3956 // X % undef -> undef
3957 // X / 0 -> undef
3958 // X % 0 -> undef
3959 // NOTE: This includes vectors where any divisor element is zero/undef.
3960 if (DAG.isUndef(Opc, {N0, N1}))
3961 return DAG.getUNDEF(VT);
3962
3963 // undef / X -> 0
3964 // undef % X -> 0
3965 if (N0.isUndef())
3966 return DAG.getConstant(0, DL, VT);
3967
3968 // 0 / X -> 0
3969 // 0 % X -> 0
3970 ConstantSDNode *N0C = isConstOrConstSplat(N0);
3971 if (N0C && N0C->isNullValue())
3972 return N0;
3973
3974 // X / X -> 1
3975 // X % X -> 0
3976 if (N0 == N1)
3977 return DAG.getConstant(IsDiv ? 1 : 0, DL, VT);
3978
3979 // X / 1 -> X
3980 // X % 1 -> 0
3981 // If this is a boolean op (single-bit element type), we can't have
3982 // division-by-zero or remainder-by-zero, so assume the divisor is 1.
3983 // TODO: Similarly, if we're zero-extending a boolean divisor, then assume
3984 // it's a 1.
3985 if ((N1C && N1C->isOne()) || (VT.getScalarType() == MVT::i1))
3986 return IsDiv ? N0 : DAG.getConstant(0, DL, VT);
3987
3988 return SDValue();
3989}
3990
3991SDValue DAGCombiner::visitSDIV(SDNode *N) {
3992 SDValue N0 = N->getOperand(0);
3993 SDValue N1 = N->getOperand(1);
3994 EVT VT = N->getValueType(0);
3995 EVT CCVT = getSetCCResultType(VT);
3996
3997 // fold vector ops
3998 if (VT.isVector())
3999 if (SDValue FoldedVOp = SimplifyVBinOp(N))
4000 return FoldedVOp;
4001
4002 SDLoc DL(N);
4003
4004 // fold (sdiv c1, c2) -> c1/c2
4005 ConstantSDNode *N1C = isConstOrConstSplat(N1);
4006 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SDIV, DL, VT, {N0, N1}))
4007 return C;
4008
4009 // fold (sdiv X, -1) -> 0-X
4010 if (N1C && N1C->isAllOnesValue())
4011 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), N0);
4012
4013 // fold (sdiv X, MIN_SIGNED) -> select(X == MIN_SIGNED, 1, 0)
4014 if (N1C && N1C->getAPIntValue().isMinSignedValue())
4015 return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ),
4016 DAG.getConstant(1, DL, VT),
4017 DAG.getConstant(0, DL, VT));
4018
4019 if (SDValue V = simplifyDivRem(N, DAG))
4020 return V;
4021
4022 if (SDValue NewSel = foldBinOpIntoSelect(N))
4023 return NewSel;
4024
4025 // If we know the sign bits of both operands are zero, strength reduce to a
4026 // udiv instead. Handles (X&15) /s 4 -> X&15 >> 2
4027 if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
4028 return DAG.getNode(ISD::UDIV, DL, N1.getValueType(), N0, N1);
4029
4030 if (SDValue V = visitSDIVLike(N0, N1, N)) {
4031 // If the corresponding remainder node exists, update its users with
4032 // (Dividend - (Quotient * Divisor).
4033 if (SDNode *RemNode = DAG.getNodeIfExists(ISD::SREM, N->getVTList(),
4034 { N0, N1 })) {
4035 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, V, N1);
4036 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
4037 AddToWorklist(Mul.getNode());
4038 AddToWorklist(Sub.getNode());
4039 CombineTo(RemNode, Sub);
4040 }
4041 return V;
4042 }
4043
4044 // sdiv, srem -> sdivrem
4045 // If the divisor is constant, then return DIVREM only if isIntDivCheap() is
4046 // true. Otherwise, we break the simplification logic in visitREM().
4047 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
4048 if (!N1C || TLI.isIntDivCheap(N->getValueType(0), Attr))
4049 if (SDValue DivRem = useDivRem(N))
4050 return DivRem;
4051
4052 return SDValue();
4053}
4054
4055SDValue DAGCombiner::visitSDIVLike(SDValue N0, SDValue N1, SDNode *N) {
4056 SDLoc DL(N);
4057 EVT VT = N->getValueType(0);
4058 EVT CCVT = getSetCCResultType(VT);
4059 unsigned BitWidth = VT.getScalarSizeInBits();
4060
4061 // Helper for determining whether a value is a power-2 constant scalar or a
4062 // vector of such elements.
4063 auto IsPowerOfTwo = [](ConstantSDNode *C) {
4064 if (C->isNullValue() || C->isOpaque())
4065 return false;
4066 if (C->getAPIntValue().isPowerOf2())
4067 return true;
4068 if ((-C->getAPIntValue()).isPowerOf2())
4069 return true;
4070 return false;
4071 };
4072
4073 // fold (sdiv X, pow2) -> simple ops after legalize
4074 // FIXME: We check for the exact bit here because the generic lowering gives
4075 // better results in that case. The target-specific lowering should learn how
4076 // to handle exact sdivs efficiently.
4077 if (!N->getFlags().hasExact() && ISD::matchUnaryPredicate(N1, IsPowerOfTwo)) {
4078 // Target-specific implementation of sdiv x, pow2.
4079 if (SDValue Res = BuildSDIVPow2(N))
4080 return Res;
4081
4082 // Create constants that are functions of the shift amount value.
4083 EVT ShiftAmtTy = getShiftAmountTy(N0.getValueType());
4084 SDValue Bits = DAG.getConstant(BitWidth, DL, ShiftAmtTy);
4085 SDValue C1 = DAG.getNode(ISD::CTTZ, DL, VT, N1);
4086 C1 = DAG.getZExtOrTrunc(C1, DL, ShiftAmtTy);
4087 SDValue Inexact = DAG.getNode(ISD::SUB, DL, ShiftAmtTy, Bits, C1);
4088 if (!isConstantOrConstantVector(Inexact))
4089 return SDValue();
4090
4091 // Splat the sign bit into the register
4092 SDValue Sign = DAG.getNode(ISD::SRA, DL, VT, N0,
4093 DAG.getConstant(BitWidth - 1, DL, ShiftAmtTy));
4094 AddToWorklist(Sign.getNode());
4095
4096 // Add (N0 < 0) ? abs2 - 1 : 0;
4097 SDValue Srl = DAG.getNode(ISD::SRL, DL, VT, Sign, Inexact);
4098 AddToWorklist(Srl.getNode());
4099 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Srl);
4100 AddToWorklist(Add.getNode());
4101 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Add, C1);
4102 AddToWorklist(Sra.getNode());
4103
4104 // Special case: (sdiv X, 1) -> X
4105 // Special Case: (sdiv X, -1) -> 0-X
4106 SDValue One = DAG.getConstant(1, DL, VT);
4107 SDValue AllOnes = DAG.getAllOnesConstant(DL, VT);
4108 SDValue IsOne = DAG.getSetCC(DL, CCVT, N1, One, ISD::SETEQ);
4109 SDValue IsAllOnes = DAG.getSetCC(DL, CCVT, N1, AllOnes, ISD::SETEQ);
4110 SDValue IsOneOrAllOnes = DAG.getNode(ISD::OR, DL, CCVT, IsOne, IsAllOnes);
4111 Sra = DAG.getSelect(DL, VT, IsOneOrAllOnes, N0, Sra);
4112
4113 // If dividing by a positive value, we're done. Otherwise, the result must
4114 // be negated.
4115 SDValue Zero = DAG.getConstant(0, DL, VT);
4116 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, Zero, Sra);
4117
4118 // FIXME: Use SELECT_CC once we improve SELECT_CC constant-folding.
4119 SDValue IsNeg = DAG.getSetCC(DL, CCVT, N1, Zero, ISD::SETLT);
4120 SDValue Res = DAG.getSelect(DL, VT, IsNeg, Sub, Sra);
4121 return Res;
4122 }
4123
4124 // If integer divide is expensive and we satisfy the requirements, emit an
4125 // alternate sequence. Targets may check function attributes for size/speed
4126 // trade-offs.
4127 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
4128 if (isConstantOrConstantVector(N1) &&
4129 !TLI.isIntDivCheap(N->getValueType(0), Attr))
4130 if (SDValue Op = BuildSDIV(N))
4131 return Op;
4132
4133 return SDValue();
4134}
4135
4136SDValue DAGCombiner::visitUDIV(SDNode *N) {
4137 SDValue N0 = N->getOperand(0);
4138 SDValue N1 = N->getOperand(1);
4139 EVT VT = N->getValueType(0);
4140 EVT CCVT = getSetCCResultType(VT);
4141
4142 // fold vector ops
4143 if (VT.isVector())
4144 if (SDValue FoldedVOp = SimplifyVBinOp(N))
4145 return FoldedVOp;
4146
4147 SDLoc DL(N);
4148
4149 // fold (udiv c1, c2) -> c1/c2
4150 ConstantSDNode *N1C = isConstOrConstSplat(N1);
4151 if (SDValue C = DAG.FoldConstantArithmetic(ISD::UDIV, DL, VT, {N0, N1}))
4152 return C;
4153
4154 // fold (udiv X, -1) -> select(X == -1, 1, 0)
4155 if (N1C && N1C->getAPIntValue().isAllOnesValue())
4156 return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ),
4157 DAG.getConstant(1, DL, VT),
4158 DAG.getConstant(0, DL, VT));
4159
4160 if (SDValue V = simplifyDivRem(N, DAG))
4161 return V;
4162
4163 if (SDValue NewSel = foldBinOpIntoSelect(N))
4164 return NewSel;
4165
4166 if (SDValue V = visitUDIVLike(N0, N1, N)) {
4167 // If the corresponding remainder node exists, update its users with
4168 // (Dividend - (Quotient * Divisor).
4169 if (SDNode *RemNode = DAG.getNodeIfExists(ISD::UREM, N->getVTList(),
4170 { N0, N1 })) {
4171 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, V, N1);
4172 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
4173 AddToWorklist(Mul.getNode());
4174 AddToWorklist(Sub.getNode());
4175 CombineTo(RemNode, Sub);
4176 }
4177 return V;
4178 }
4179
4180 // sdiv, srem -> sdivrem
4181 // If the divisor is constant, then return DIVREM only if isIntDivCheap() is
4182 // true. Otherwise, we break the simplification logic in visitREM().
4183 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
4184 if (!N1C || TLI.isIntDivCheap(N->getValueType(0), Attr))
4185 if (SDValue DivRem = useDivRem(N))
4186 return DivRem;
4187
4188 return SDValue();
4189}
4190
4191SDValue DAGCombiner::visitUDIVLike(SDValue N0, SDValue N1, SDNode *N) {
4192 SDLoc DL(N);
4193 EVT VT = N->getValueType(0);
4194
4195 // fold (udiv x, (1 << c)) -> x >>u c
4196 if (isConstantOrConstantVector(N1, /*NoOpaques*/ true) &&
4197 DAG.isKnownToBeAPowerOfTwo(N1)) {
4198 SDValue LogBase2 = BuildLogBase2(N1, DL);
4199 AddToWorklist(LogBase2.getNode());
4200
4201 EVT ShiftVT = getShiftAmountTy(N0.getValueType());
4202 SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ShiftVT);
4203 AddToWorklist(Trunc.getNode());
4204 return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc);
4205 }
4206
4207 // fold (udiv x, (shl c, y)) -> x >>u (log2(c)+y) iff c is power of 2
4208 if (N1.getOpcode() == ISD::SHL) {
4209 SDValue N10 = N1.getOperand(0);
4210 if (isConstantOrConstantVector(N10, /*NoOpaques*/ true) &&
4211 DAG.isKnownToBeAPowerOfTwo(N10)) {
4212 SDValue LogBase2 = BuildLogBase2(N10, DL);
4213 AddToWorklist(LogBase2.getNode());
4214
4215 EVT ADDVT = N1.getOperand(1).getValueType();
4216 SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ADDVT);
4217 AddToWorklist(Trunc.getNode());
4218 SDValue Add = DAG.getNode(ISD::ADD, DL, ADDVT, N1.getOperand(1), Trunc);
4219 AddToWorklist(Add.getNode());
4220 return DAG.getNode(ISD::SRL, DL, VT, N0, Add);
4221 }
4222 }
4223
4224 // fold (udiv x, c) -> alternate
4225 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
4226 if (isConstantOrConstantVector(N1) &&
4227 !TLI.isIntDivCheap(N->getValueType(0), Attr))
4228 if (SDValue Op = BuildUDIV(N))
4229 return Op;
4230
4231 return SDValue();
4232}
4233
4234// handles ISD::SREM and ISD::UREM
4235SDValue DAGCombiner::visitREM(SDNode *N) {
4236 unsigned Opcode = N->getOpcode();
4237 SDValue N0 = N->getOperand(0);
4238 SDValue N1 = N->getOperand(1);
4239 EVT VT = N->getValueType(0);
4240 EVT CCVT = getSetCCResultType(VT);
4241
4242 bool isSigned = (Opcode == ISD::SREM);
4243 SDLoc DL(N);
4244
4245 // fold (rem c1, c2) -> c1%c2
4246 ConstantSDNode *N1C = isConstOrConstSplat(N1);
4247 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1}))
4248 return C;
4249
4250 // fold (urem X, -1) -> select(X == -1, 0, x)
4251 if (!isSigned && N1C && N1C->getAPIntValue().isAllOnesValue())
4252 return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ),
4253 DAG.getConstant(0, DL, VT), N0);
4254
4255 if (SDValue V = simplifyDivRem(N, DAG))
4256 return V;
4257
4258 if (SDValue NewSel = foldBinOpIntoSelect(N))
4259 return NewSel;
4260
4261 if (isSigned) {
4262 // If we know the sign bits of both operands are zero, strength reduce to a
4263 // urem instead. Handles (X & 0x0FFFFFFF) %s 16 -> X&15
4264 if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
4265 return DAG.getNode(ISD::UREM, DL, VT, N0, N1);
4266 } else {
4267 if (DAG.isKnownToBeAPowerOfTwo(N1)) {
4268 // fold (urem x, pow2) -> (and x, pow2-1)
4269 SDValue NegOne = DAG.getAllOnesConstant(DL, VT);
4270 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne);
4271 AddToWorklist(Add.getNode());
4272 return DAG.getNode(ISD::AND, DL, VT, N0, Add);
4273 }
4274 if (N1.getOpcode() == ISD::SHL &&
4275 DAG.isKnownToBeAPowerOfTwo(N1.getOperand(0))) {
4276 // fold (urem x, (shl pow2, y)) -> (and x, (add (shl pow2, y), -1))
4277 SDValue NegOne = DAG.getAllOnesConstant(DL, VT);
4278 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne);
4279 AddToWorklist(Add.getNode());
4280 return DAG.getNode(ISD::AND, DL, VT, N0, Add);
4281 }
4282 }
4283
4284 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
4285
4286 // If X/C can be simplified by the division-by-constant logic, lower
4287 // X%C to the equivalent of X-X/C*C.
4288 // Reuse the SDIVLike/UDIVLike combines - to avoid mangling nodes, the
4289 // speculative DIV must not cause a DIVREM conversion. We guard against this
4290 // by skipping the simplification if isIntDivCheap(). When div is not cheap,
4291 // combine will not return a DIVREM. Regardless, checking cheapness here
4292 // makes sense since the simplification results in fatter code.
4293 if (DAG.isKnownNeverZero(N1) && !TLI.isIntDivCheap(VT, Attr)) {
4294 SDValue OptimizedDiv =
4295 isSigned ? visitSDIVLike(N0, N1, N) : visitUDIVLike(N0, N1, N);
4296 if (OptimizedDiv.getNode()) {
4297 // If the equivalent Div node also exists, update its users.
4298 unsigned DivOpcode = isSigned ? ISD::SDIV : ISD::UDIV;
4299 if (SDNode *DivNode = DAG.getNodeIfExists(DivOpcode, N->getVTList(),
4300 { N0, N1 }))
4301 CombineTo(DivNode, OptimizedDiv);
4302 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, OptimizedDiv, N1);
4303 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
4304 AddToWorklist(OptimizedDiv.getNode());
4305 AddToWorklist(Mul.getNode());
4306 return Sub;
4307 }
4308 }
4309
4310 // sdiv, srem -> sdivrem
4311 if (SDValue DivRem = useDivRem(N))
4312 return DivRem.getValue(1);
4313
4314 return SDValue();
4315}
4316
4317SDValue DAGCombiner::visitMULHS(SDNode *N) {
4318 SDValue N0 = N->getOperand(0);
4319 SDValue N1 = N->getOperand(1);
4320 EVT VT = N->getValueType(0);
4321 SDLoc DL(N);
4322
4323 if (VT.isVector()) {
4324 // fold (mulhs x, 0) -> 0
4325 // do not return N0/N1, because undef node may exist.
4326 if (ISD::isBuildVectorAllZeros(N0.getNode()) ||
4327 ISD::isBuildVectorAllZeros(N1.getNode()))
4328 return DAG.getConstant(0, DL, VT);
4329 }
4330
4331 // fold (mulhs x, 0) -> 0
4332 if (isNullConstant(N1))
4333 return N1;
4334 // fold (mulhs x, 1) -> (sra x, size(x)-1)
4335 if (isOneConstant(N1))
4336 return DAG.getNode(ISD::SRA, DL, N0.getValueType(), N0,
4337 DAG.getConstant(N0.getScalarValueSizeInBits() - 1, DL,
4338 getShiftAmountTy(N0.getValueType())));
4339
4340 // fold (mulhs x, undef) -> 0
4341 if (N0.isUndef() || N1.isUndef())
4342 return DAG.getConstant(0, DL, VT);
4343
4344 // If the type twice as wide is legal, transform the mulhs to a wider multiply
4345 // plus a shift.
4346 if (!TLI.isOperationLegalOrCustom(ISD::MULHS, VT) && VT.isSimple() &&
4347 !VT.isVector()) {
4348 MVT Simple = VT.getSimpleVT();
4349 unsigned SimpleSize = Simple.getSizeInBits();
4350 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
4351 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
4352 N0 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N0);
4353 N1 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N1);
4354 N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
4355 N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
4356 DAG.getConstant(SimpleSize, DL,
4357 getShiftAmountTy(N1.getValueType())));
4358 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
4359 }
4360 }
4361
4362 return SDValue();
4363}
4364
4365SDValue DAGCombiner::visitMULHU(SDNode *N) {
4366 SDValue N0 = N->getOperand(0);
4367 SDValue N1 = N->getOperand(1);
4368 EVT VT = N->getValueType(0);
4369 SDLoc DL(N);
4370
4371 if (VT.isVector()) {
4372 // fold (mulhu x, 0) -> 0
4373 // do not return N0/N1, because undef node may exist.
4374 if (ISD::isBuildVectorAllZeros(N0.getNode()) ||
4375 ISD::isBuildVectorAllZeros(N1.getNode()))
4376 return DAG.getConstant(0, DL, VT);
4377 }
4378
4379 // fold (mulhu x, 0) -> 0
4380 if (isNullConstant(N1))
4381 return N1;
4382 // fold (mulhu x, 1) -> 0
4383 if (isOneConstant(N1))
4384 return DAG.getConstant(0, DL, N0.getValueType());
4385 // fold (mulhu x, undef) -> 0
4386 if (N0.isUndef() || N1.isUndef())
4387 return DAG.getConstant(0, DL, VT);
4388
4389 // fold (mulhu x, (1 << c)) -> x >> (bitwidth - c)
4390 if (isConstantOrConstantVector(N1, /*NoOpaques*/ true) &&
4391 DAG.isKnownToBeAPowerOfTwo(N1) && hasOperation(ISD::SRL, VT)) {
4392 unsigned NumEltBits = VT.getScalarSizeInBits();
4393 SDValue LogBase2 = BuildLogBase2(N1, DL);
4394 SDValue SRLAmt = DAG.getNode(
4395 ISD::SUB, DL, VT, DAG.getConstant(NumEltBits, DL, VT), LogBase2);
4396 EVT ShiftVT = getShiftAmountTy(N0.getValueType());
4397 SDValue Trunc = DAG.getZExtOrTrunc(SRLAmt, DL, ShiftVT);
4398 return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc);
4399 }
4400
4401 // If the type twice as wide is legal, transform the mulhu to a wider multiply
4402 // plus a shift.
4403 if (!TLI.isOperationLegalOrCustom(ISD::MULHU, VT) && VT.isSimple() &&
4404 !VT.isVector()) {
4405 MVT Simple = VT.getSimpleVT();
4406 unsigned SimpleSize = Simple.getSizeInBits();
4407 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
4408 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
4409 N0 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N0);
4410 N1 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N1);
4411 N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
4412 N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
4413 DAG.getConstant(SimpleSize, DL,
4414 getShiftAmountTy(N1.getValueType())));
4415 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
4416 }
4417 }
4418
4419 return SDValue();
4420}
4421
4422/// Perform optimizations common to nodes that compute two values. LoOp and HiOp
4423/// give the opcodes for the two computations that are being performed. Return
4424/// true if a simplification was made.
4425SDValue DAGCombiner::SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
4426 unsigned HiOp) {
4427 // If the high half is not needed, just compute the low half.
4428 bool HiExists = N->hasAnyUseOfValue(1);
4429 if (!HiExists && (!LegalOperations ||
4430 TLI.isOperationLegalOrCustom(LoOp, N->getValueType(0)))) {
4431 SDValue Res = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0), N->ops());
4432 return CombineTo(N, Res, Res);
4433 }
4434
4435 // If the low half is not needed, just compute the high half.
4436 bool LoExists = N->hasAnyUseOfValue(0);
4437 if (!LoExists && (!LegalOperations ||
4438 TLI.isOperationLegalOrCustom(HiOp, N->getValueType(1)))) {
4439 SDValue Res = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1), N->ops());
4440 return CombineTo(N, Res, Res);
4441 }
4442
4443 // If both halves are used, return as it is.
4444 if (LoExists && HiExists)
4445 return SDValue();
4446
4447 // If the two computed results can be simplified separately, separate them.
4448 if (LoExists) {
4449 SDValue Lo = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0), N->ops());
4450 AddToWorklist(Lo.getNode());
4451 SDValue LoOpt = combine(Lo.getNode());
4452 if (LoOpt.getNode() && LoOpt.getNode() != Lo.getNode() &&
4453 (!LegalOperations ||
4454 TLI.isOperationLegalOrCustom(LoOpt.getOpcode(), LoOpt.getValueType())))
4455 return CombineTo(N, LoOpt, LoOpt);
4456 }
4457
4458 if (HiExists) {
4459 SDValue Hi = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1), N->ops());
4460 AddToWorklist(Hi.getNode());
4461 SDValue HiOpt = combine(Hi.getNode());
4462 if (HiOpt.getNode() && HiOpt != Hi &&
4463 (!LegalOperations ||
4464 TLI.isOperationLegalOrCustom(HiOpt.getOpcode(), HiOpt.getValueType())))
4465 return CombineTo(N, HiOpt, HiOpt);
4466 }
4467
4468 return SDValue();
4469}
4470
4471SDValue DAGCombiner::visitSMUL_LOHI(SDNode *N) {
4472 if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHS))
4473 return Res;
4474
4475 EVT VT = N->getValueType(0);
4476 SDLoc DL(N);
4477
4478 // If the type is twice as wide is legal, transform the mulhu to a wider
4479 // multiply plus a shift.
4480 if (VT.isSimple() && !VT.isVector()) {
4481 MVT Simple = VT.getSimpleVT();
4482 unsigned SimpleSize = Simple.getSizeInBits();
4483 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
4484 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
4485 SDValue Lo = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(0));
4486 SDValue Hi = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(1));
4487 Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
4488 // Compute the high part as N1.
4489 Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
4490 DAG.getConstant(SimpleSize, DL,
4491 getShiftAmountTy(Lo.getValueType())));
4492 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
4493 // Compute the low part as N0.
4494 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
4495 return CombineTo(N, Lo, Hi);
4496 }
4497 }
4498
4499 return SDValue();
4500}
4501
4502SDValue DAGCombiner::visitUMUL_LOHI(SDNode *N) {
4503 if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHU))
4504 return Res;
4505
4506 EVT VT = N->getValueType(0);
4507 SDLoc DL(N);
4508
4509 // (umul_lohi N0, 0) -> (0, 0)
4510 if (isNullConstant(N->getOperand(1))) {
4511 SDValue Zero = DAG.getConstant(0, DL, VT);
4512 return CombineTo(N, Zero, Zero);
4513 }
4514
4515 // (umul_lohi N0, 1) -> (N0, 0)
4516 if (isOneConstant(N->getOperand(1))) {
4517 SDValue Zero = DAG.getConstant(0, DL, VT);
4518 return CombineTo(N, N->getOperand(0), Zero);
4519 }
4520
4521 // If the type is twice as wide is legal, transform the mulhu to a wider
4522 // multiply plus a shift.
4523 if (VT.isSimple() && !VT.isVector()) {
4524 MVT Simple = VT.getSimpleVT();
4525 unsigned SimpleSize = Simple.getSizeInBits();
4526 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
4527 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
4528 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(0));
4529 SDValue Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(1));
4530 Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
4531 // Compute the high part as N1.
4532 Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
4533 DAG.getConstant(SimpleSize, DL,
4534 getShiftAmountTy(Lo.getValueType())));
4535 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
4536 // Compute the low part as N0.
4537 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
4538 return CombineTo(N, Lo, Hi);
4539 }
4540 }
4541
4542 return SDValue();
4543}
4544
4545SDValue DAGCombiner::visitMULO(SDNode *N) {
4546 SDValue N0 = N->getOperand(0);
4547 SDValue N1 = N->getOperand(1);
4548 EVT VT = N0.getValueType();
4549 bool IsSigned = (ISD::SMULO == N->getOpcode());
4550
4551 EVT CarryVT = N->getValueType(1);
4552 SDLoc DL(N);
4553
4554 // canonicalize constant to RHS.
4555 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
4556 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
4557 return DAG.getNode(N->getOpcode(), DL, N->getVTList(), N1, N0);
4558
4559 // fold (mulo x, 0) -> 0 + no carry out
4560 if (isNullOrNullSplat(N1))
4561 return CombineTo(N, DAG.getConstant(0, DL, VT),
4562 DAG.getConstant(0, DL, CarryVT));
4563
4564 // (mulo x, 2) -> (addo x, x)
4565 if (ConstantSDNode *C2 = isConstOrConstSplat(N1))
4566 if (C2->getAPIntValue() == 2)
4567 return DAG.getNode(IsSigned ? ISD::SADDO : ISD::UADDO, DL,
4568 N->getVTList(), N0, N0);
4569
4570 return SDValue();
4571}
4572
4573SDValue DAGCombiner::visitIMINMAX(SDNode *N) {
4574 SDValue N0 = N->getOperand(0);
4575 SDValue N1 = N->getOperand(1);
4576 EVT VT = N0.getValueType();
4577 unsigned Opcode = N->getOpcode();
4578
4579 // fold vector ops
4580 if (VT.isVector())
4581 if (SDValue FoldedVOp = SimplifyVBinOp(N))
4582 return FoldedVOp;
4583
4584 // fold operation with constant operands.
4585 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, SDLoc(N), VT, {N0, N1}))
4586 return C;
4587
4588 // canonicalize constant to RHS
4589 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
4590 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
4591 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N1, N0);
4592
4593 // Is sign bits are zero, flip between UMIN/UMAX and SMIN/SMAX.
4594 // Only do this if the current op isn't legal and the flipped is.
4595 if (!TLI.isOperationLegal(Opcode, VT) &&
4596 (N0.isUndef() || DAG.SignBitIsZero(N0)) &&
4597 (N1.isUndef() || DAG.SignBitIsZero(N1))) {
4598 unsigned AltOpcode;
4599 switch (Opcode) {
4600 case ISD::SMIN: AltOpcode = ISD::UMIN; break;
4601 case ISD::SMAX: AltOpcode = ISD::UMAX; break;
4602 case ISD::UMIN: AltOpcode = ISD::SMIN; break;
4603 case ISD::UMAX: AltOpcode = ISD::SMAX; break;
4604 default: llvm_unreachable("Unknown MINMAX opcode")::llvm::llvm_unreachable_internal("Unknown MINMAX opcode", "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4604)
;
4605 }
4606 if (TLI.isOperationLegal(AltOpcode, VT))
4607 return DAG.getNode(AltOpcode, SDLoc(N), VT, N0, N1);
4608 }
4609
4610 // Simplify the operands using demanded-bits information.
4611 if (SimplifyDemandedBits(SDValue(N, 0)))
4612 return SDValue(N, 0);
4613
4614 return SDValue();
4615}
4616
4617/// If this is a bitwise logic instruction and both operands have the same
4618/// opcode, try to sink the other opcode after the logic instruction.
4619SDValue DAGCombiner::hoistLogicOpWithSameOpcodeHands(SDNode *N) {
4620 SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
4621 EVT VT = N0.getValueType();
4622 unsigned LogicOpcode = N->getOpcode();
4623 unsigned HandOpcode = N0.getOpcode();
4624 assert((LogicOpcode == ISD::AND || LogicOpcode == ISD::OR ||(((LogicOpcode == ISD::AND || LogicOpcode == ISD::OR || LogicOpcode
== ISD::XOR) && "Expected logic opcode") ? static_cast
<void> (0) : __assert_fail ("(LogicOpcode == ISD::AND || LogicOpcode == ISD::OR || LogicOpcode == ISD::XOR) && \"Expected logic opcode\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4625, __PRETTY_FUNCTION__))
4625 LogicOpcode == ISD::XOR) && "Expected logic opcode")(((LogicOpcode == ISD::AND || LogicOpcode == ISD::OR || LogicOpcode
== ISD::XOR) && "Expected logic opcode") ? static_cast
<void> (0) : __assert_fail ("(LogicOpcode == ISD::AND || LogicOpcode == ISD::OR || LogicOpcode == ISD::XOR) && \"Expected logic opcode\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4625, __PRETTY_FUNCTION__))
;
4626 assert(HandOpcode == N1.getOpcode() && "Bad input!")((HandOpcode == N1.getOpcode() && "Bad input!") ? static_cast
<void> (0) : __assert_fail ("HandOpcode == N1.getOpcode() && \"Bad input!\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4626, __PRETTY_FUNCTION__))
;
4627
4628 // Bail early if none of these transforms apply.
4629 if (N0.getNumOperands() == 0)
4630 return SDValue();
4631
4632 // FIXME: We should check number of uses of the operands to not increase
4633 // the instruction count for all transforms.
4634
4635 // Handle size-changing casts.
4636 SDValue X = N0.getOperand(0);
4637 SDValue Y = N1.getOperand(0);
4638 EVT XVT = X.getValueType();
4639 SDLoc DL(N);
4640 if (HandOpcode == ISD::ANY_EXTEND || HandOpcode == ISD::ZERO_EXTEND ||
4641 HandOpcode == ISD::SIGN_EXTEND) {
4642 // If both operands have other uses, this transform would create extra
4643 // instructions without eliminating anything.
4644 if (!N0.hasOneUse() && !N1.hasOneUse())
4645 return SDValue();
4646 // We need matching integer source types.
4647 if (XVT != Y.getValueType())
4648 return SDValue();
4649 // Don't create an illegal op during or after legalization. Don't ever
4650 // create an unsupported vector op.
4651 if ((VT.isVector() || LegalOperations) &&
4652 !TLI.isOperationLegalOrCustom(LogicOpcode, XVT))
4653 return SDValue();
4654 // Avoid infinite looping with PromoteIntBinOp.
4655 // TODO: Should we apply desirable/legal constraints to all opcodes?
4656 if (HandOpcode == ISD::ANY_EXTEND && LegalTypes &&
4657 !TLI.isTypeDesirableForOp(LogicOpcode, XVT))
4658 return SDValue();
4659 // logic_op (hand_op X), (hand_op Y) --> hand_op (logic_op X, Y)
4660 SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
4661 return DAG.getNode(HandOpcode, DL, VT, Logic);
4662 }
4663
4664 // logic_op (truncate x), (truncate y) --> truncate (logic_op x, y)
4665 if (HandOpcode == ISD::TRUNCATE) {
4666 // If both operands have other uses, this transform would create extra
4667 // instructions without eliminating anything.
4668 if (!N0.hasOneUse() && !N1.hasOneUse())
4669 return SDValue();
4670 // We need matching source types.
4671 if (XVT != Y.getValueType())
4672 return SDValue();
4673 // Don't create an illegal op during or after legalization.
4674 if (LegalOperations && !TLI.isOperationLegal(LogicOpcode, XVT))
4675 return SDValue();
4676 // Be extra careful sinking truncate. If it's free, there's no benefit in
4677 // widening a binop. Also, don't create a logic op on an illegal type.
4678 if (TLI.isZExtFree(VT, XVT) && TLI.isTruncateFree(XVT, VT))
4679 return SDValue();
4680 if (!TLI.isTypeLegal(XVT))
4681 return SDValue();
4682 SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
4683 return DAG.getNode(HandOpcode, DL, VT, Logic);
4684 }
4685
4686 // For binops SHL/SRL/SRA/AND:
4687 // logic_op (OP x, z), (OP y, z) --> OP (logic_op x, y), z
4688 if ((HandOpcode == ISD::SHL || HandOpcode == ISD::SRL ||
4689 HandOpcode == ISD::SRA || HandOpcode == ISD::AND) &&
4690 N0.getOperand(1) == N1.getOperand(1)) {
4691 // If either operand has other uses, this transform is not an improvement.
4692 if (!N0.hasOneUse() || !N1.hasOneUse())
4693 return SDValue();
4694 SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
4695 return DAG.getNode(HandOpcode, DL, VT, Logic, N0.getOperand(1));
4696 }
4697
4698 // Unary ops: logic_op (bswap x), (bswap y) --> bswap (logic_op x, y)
4699 if (HandOpcode == ISD::BSWAP) {
4700 // If either operand has other uses, this transform is not an improvement.
4701 if (!N0.hasOneUse() || !N1.hasOneUse())
4702 return SDValue();
4703 SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
4704 return DAG.getNode(HandOpcode, DL, VT, Logic);
4705 }
4706
4707 // Simplify xor/and/or (bitcast(A), bitcast(B)) -> bitcast(op (A,B))
4708 // Only perform this optimization up until type legalization, before
4709 // LegalizeVectorOprs. LegalizeVectorOprs promotes vector operations by
4710 // adding bitcasts. For example (xor v4i32) is promoted to (v2i64), and
4711 // we don't want to undo this promotion.
4712 // We also handle SCALAR_TO_VECTOR because xor/or/and operations are cheaper
4713 // on scalars.
4714 if ((HandOpcode == ISD::BITCAST || HandOpcode == ISD::SCALAR_TO_VECTOR) &&
4715 Level <= AfterLegalizeTypes) {
4716 // Input types must be integer and the same.
4717 if (XVT.isInteger() && XVT == Y.getValueType() &&
4718 !(VT.isVector() && TLI.isTypeLegal(VT) &&
4719 !XVT.isVector() && !TLI.isTypeLegal(XVT))) {
4720 SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
4721 return DAG.getNode(HandOpcode, DL, VT, Logic);
4722 }
4723 }
4724
4725 // Xor/and/or are indifferent to the swizzle operation (shuffle of one value).
4726 // Simplify xor/and/or (shuff(A), shuff(B)) -> shuff(op (A,B))
4727 // If both shuffles use the same mask, and both shuffle within a single
4728 // vector, then it is worthwhile to move the swizzle after the operation.
4729 // The type-legalizer generates this pattern when loading illegal
4730 // vector types from memory. In many cases this allows additional shuffle
4731 // optimizations.
4732 // There are other cases where moving the shuffle after the xor/and/or
4733 // is profitable even if shuffles don't perform a swizzle.
4734 // If both shuffles use the same mask, and both shuffles have the same first
4735 // or second operand, then it might still be profitable to move the shuffle
4736 // after the xor/and/or operation.
4737 if (HandOpcode == ISD::VECTOR_SHUFFLE && Level < AfterLegalizeDAG) {
4738 auto *SVN0 = cast<ShuffleVectorSDNode>(N0);
4739 auto *SVN1 = cast<ShuffleVectorSDNode>(N1);
4740 assert(X.getValueType() == Y.getValueType() &&((X.getValueType() == Y.getValueType() && "Inputs to shuffles are not the same type"
) ? static_cast<void> (0) : __assert_fail ("X.getValueType() == Y.getValueType() && \"Inputs to shuffles are not the same type\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4741, __PRETTY_FUNCTION__))
4741 "Inputs to shuffles are not the same type")((X.getValueType() == Y.getValueType() && "Inputs to shuffles are not the same type"
) ? static_cast<void> (0) : __assert_fail ("X.getValueType() == Y.getValueType() && \"Inputs to shuffles are not the same type\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4741, __PRETTY_FUNCTION__))
;
4742
4743 // Check that both shuffles use the same mask. The masks are known to be of
4744 // the same length because the result vector type is the same.
4745 // Check also that shuffles have only one use to avoid introducing extra
4746 // instructions.
4747 if (!SVN0->hasOneUse() || !SVN1->hasOneUse() ||
4748 !SVN0->getMask().equals(SVN1->getMask()))
4749 return SDValue();
4750
4751 // Don't try to fold this node if it requires introducing a
4752 // build vector of all zeros that might be illegal at this stage.
4753 SDValue ShOp = N0.getOperand(1);
4754 if (LogicOpcode == ISD::XOR && !ShOp.isUndef())
4755 ShOp = tryFoldToZero(DL, TLI, VT, DAG, LegalOperations);
4756
4757 // (logic_op (shuf (A, C), shuf (B, C))) --> shuf (logic_op (A, B), C)
4758 if (N0.getOperand(1) == N1.getOperand(1) && ShOp.getNode()) {
4759 SDValue Logic = DAG.getNode(LogicOpcode, DL, VT,
4760 N0.getOperand(0), N1.getOperand(0));
4761 return DAG.getVectorShuffle(VT, DL, Logic, ShOp, SVN0->getMask());
4762 }
4763
4764 // Don't try to fold this node if it requires introducing a
4765 // build vector of all zeros that might be illegal at this stage.
4766 ShOp = N0.getOperand(0);
4767 if (LogicOpcode == ISD::XOR && !ShOp.isUndef())
4768 ShOp = tryFoldToZero(DL, TLI, VT, DAG, LegalOperations);
4769
4770 // (logic_op (shuf (C, A), shuf (C, B))) --> shuf (C, logic_op (A, B))
4771 if (N0.getOperand(0) == N1.getOperand(0) && ShOp.getNode()) {
4772 SDValue Logic = DAG.getNode(LogicOpcode, DL, VT, N0.getOperand(1),
4773 N1.getOperand(1));
4774 return DAG.getVectorShuffle(VT, DL, ShOp, Logic, SVN0->getMask());
4775 }
4776 }
4777
4778 return SDValue();
4779}
4780
4781/// Try to make (and/or setcc (LL, LR), setcc (RL, RR)) more efficient.
4782SDValue DAGCombiner::foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1,
4783 const SDLoc &DL) {
4784 SDValue LL, LR, RL, RR, N0CC, N1CC;
4785 if (!isSetCCEquivalent(N0, LL, LR, N0CC) ||
4786 !isSetCCEquivalent(N1, RL, RR, N1CC))
4787 return SDValue();
4788
4789 assert(N0.getValueType() == N1.getValueType() &&((N0.getValueType() == N1.getValueType() && "Unexpected operand types for bitwise logic op"
) ? static_cast<void> (0) : __assert_fail ("N0.getValueType() == N1.getValueType() && \"Unexpected operand types for bitwise logic op\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4790, __PRETTY_FUNCTION__))
4790 "Unexpected operand types for bitwise logic op")((N0.getValueType() == N1.getValueType() && "Unexpected operand types for bitwise logic op"
) ? static_cast<void> (0) : __assert_fail ("N0.getValueType() == N1.getValueType() && \"Unexpected operand types for bitwise logic op\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4790, __PRETTY_FUNCTION__))
;
4791 assert(LL.getValueType() == LR.getValueType() &&((LL.getValueType() == LR.getValueType() && RL.getValueType
() == RR.getValueType() && "Unexpected operand types for setcc"
) ? static_cast<void> (0) : __assert_fail ("LL.getValueType() == LR.getValueType() && RL.getValueType() == RR.getValueType() && \"Unexpected operand types for setcc\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4793, __PRETTY_FUNCTION__))
4792 RL.getValueType() == RR.getValueType() &&((LL.getValueType() == LR.getValueType() && RL.getValueType
() == RR.getValueType() && "Unexpected operand types for setcc"
) ? static_cast<void> (0) : __assert_fail ("LL.getValueType() == LR.getValueType() && RL.getValueType() == RR.getValueType() && \"Unexpected operand types for setcc\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4793, __PRETTY_FUNCTION__))
4793 "Unexpected operand types for setcc")((LL.getValueType() == LR.getValueType() && RL.getValueType
() == RR.getValueType() && "Unexpected operand types for setcc"
) ? static_cast<void> (0) : __assert_fail ("LL.getValueType() == LR.getValueType() && RL.getValueType() == RR.getValueType() && \"Unexpected operand types for setcc\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4793, __PRETTY_FUNCTION__))
;
4794
4795 // If we're here post-legalization or the logic op type is not i1, the logic
4796 // op type must match a setcc result type. Also, all folds require new
4797 // operations on the left and right operands, so those types must match.
4798 EVT VT = N0.getValueType();
4799 EVT OpVT = LL.getValueType();
4800 if (LegalOperations || VT.getScalarType() != MVT::i1)
4801 if (VT != getSetCCResultType(OpVT))
4802 return SDValue();
4803 if (OpVT != RL.getValueType())
4804 return SDValue();
4805
4806 ISD::CondCode CC0 = cast<CondCodeSDNode>(N0CC)->get();
4807 ISD::CondCode CC1 = cast<CondCodeSDNode>(N1CC)->get();
4808 bool IsInteger = OpVT.isInteger();
4809 if (LR == RR && CC0 == CC1 && IsInteger) {
4810 bool IsZero = isNullOrNullSplat(LR);
4811 bool IsNeg1 = isAllOnesOrAllOnesSplat(LR);
4812
4813 // All bits clear?
4814 bool AndEqZero = IsAnd && CC1 == ISD::SETEQ && IsZero;
4815 // All sign bits clear?
4816 bool AndGtNeg1 = IsAnd && CC1 == ISD::SETGT && IsNeg1;
4817 // Any bits set?
4818 bool OrNeZero = !IsAnd && CC1 == ISD::SETNE && IsZero;
4819 // Any sign bits set?
4820 bool OrLtZero = !IsAnd && CC1 == ISD::SETLT && IsZero;
4821
4822 // (and (seteq X, 0), (seteq Y, 0)) --> (seteq (or X, Y), 0)
4823 // (and (setgt X, -1), (setgt Y, -1)) --> (setgt (or X, Y), -1)
4824 // (or (setne X, 0), (setne Y, 0)) --> (setne (or X, Y), 0)
4825 // (or (setlt X, 0), (setlt Y, 0)) --> (setlt (or X, Y), 0)
4826 if (AndEqZero || AndGtNeg1 || OrNeZero || OrLtZero) {
4827 SDValue Or = DAG.getNode(ISD::OR, SDLoc(N0), OpVT, LL, RL);
4828 AddToWorklist(Or.getNode());
4829 return DAG.getSetCC(DL, VT, Or, LR, CC1);
4830 }
4831
4832 // All bits set?
4833 bool AndEqNeg1 = IsAnd && CC1 == ISD::SETEQ && IsNeg1;
4834 // All sign bits set?
4835 bool AndLtZero = IsAnd && CC1 == ISD::SETLT && IsZero;
4836 // Any bits clear?
4837 bool OrNeNeg1 = !IsAnd && CC1 == ISD::SETNE && IsNeg1;
4838 // Any sign bits clear?
4839 bool OrGtNeg1 = !IsAnd && CC1 == ISD::SETGT && IsNeg1;
4840
4841 // (and (seteq X, -1), (seteq Y, -1)) --> (seteq (and X, Y), -1)
4842 // (and (setlt X, 0), (setlt Y, 0)) --> (setlt (and X, Y), 0)
4843 // (or (setne X, -1), (setne Y, -1)) --> (setne (and X, Y), -1)
4844 // (or (setgt X, -1), (setgt Y -1)) --> (setgt (and X, Y), -1)
4845 if (AndEqNeg1 || AndLtZero || OrNeNeg1 || OrGtNeg1) {
4846 SDValue And = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, LL, RL);
4847 AddToWorklist(And.getNode());
4848 return DAG.getSetCC(DL, VT, And, LR, CC1);
4849 }
4850 }
4851
4852 // TODO: What is the 'or' equivalent of this fold?
4853 // (and (setne X, 0), (setne X, -1)) --> (setuge (add X, 1), 2)
4854 if (IsAnd && LL == RL && CC0 == CC1 && OpVT.getScalarSizeInBits() > 1 &&
4855 IsInteger && CC0 == ISD::SETNE &&
4856 ((isNullConstant(LR) && isAllOnesConstant(RR)) ||
4857 (isAllOnesConstant(LR) && isNullConstant(RR)))) {
4858 SDValue One = DAG.getConstant(1, DL, OpVT);
4859 SDValue Two = DAG.getConstant(2, DL, OpVT);
4860 SDValue Add = DAG.getNode(ISD::ADD, SDLoc(N0), OpVT, LL, One);
4861 AddToWorklist(Add.getNode());
4862 return DAG.getSetCC(DL, VT, Add, Two, ISD::SETUGE);
4863 }
4864
4865 // Try more general transforms if the predicates match and the only user of
4866 // the compares is the 'and' or 'or'.
4867 if (IsInteger && TLI.convertSetCCLogicToBitwiseLogic(OpVT) && CC0 == CC1 &&
4868 N0.hasOneUse() && N1.hasOneUse()) {
4869 // and (seteq A, B), (seteq C, D) --> seteq (or (xor A, B), (xor C, D)), 0
4870 // or (setne A, B), (setne C, D) --> setne (or (xor A, B), (xor C, D)), 0
4871 if ((IsAnd && CC1 == ISD::SETEQ) || (!IsAnd && CC1 == ISD::SETNE)) {
4872 SDValue XorL = DAG.getNode(ISD::XOR, SDLoc(N0), OpVT, LL, LR);
4873 SDValue XorR = DAG.getNode(ISD::XOR, SDLoc(N1), OpVT, RL, RR);
4874 SDValue Or = DAG.getNode(ISD::OR, DL, OpVT, XorL, XorR);
4875 SDValue Zero = DAG.getConstant(0, DL, OpVT);
4876 return DAG.getSetCC(DL, VT, Or, Zero, CC1);
4877 }
4878
4879 // Turn compare of constants whose difference is 1 bit into add+and+setcc.
4880 // TODO - support non-uniform vector amounts.
4881 if ((IsAnd && CC1 == ISD::SETNE) || (!IsAnd && CC1 == ISD::SETEQ)) {
4882 // Match a shared variable operand and 2 non-opaque constant operands.
4883 ConstantSDNode *C0 = isConstOrConstSplat(LR);
4884 ConstantSDNode *C1 = isConstOrConstSplat(RR);
4885 if (LL == RL && C0 && C1 && !C0->isOpaque() && !C1->isOpaque()) {
4886 // Canonicalize larger constant as C0.
4887 if (C1->getAPIntValue().ugt(C0->getAPIntValue()))
4888 std::swap(C0, C1);
4889
4890 // The difference of the constants must be a single bit.
4891 const APInt &C0Val = C0->getAPIntValue();
4892 const APInt &C1Val = C1->getAPIntValue();
4893 if ((C0Val - C1Val).isPowerOf2()) {
4894 // and/or (setcc X, C0, ne), (setcc X, C1, ne/eq) -->
4895 // setcc ((add X, -C1), ~(C0 - C1)), 0, ne/eq
4896 SDValue OffsetC = DAG.getConstant(-C1Val, DL, OpVT);
4897 SDValue Add = DAG.getNode(ISD::ADD, DL, OpVT, LL, OffsetC);
4898 SDValue MaskC = DAG.getConstant(~(C0Val - C1Val), DL, OpVT);
4899 SDValue And = DAG.getNode(ISD::AND, DL, OpVT, Add, MaskC);
4900 SDValue Zero = DAG.getConstant(0, DL, OpVT);
4901 return DAG.getSetCC(DL, VT, And, Zero, CC0);
4902 }
4903 }
4904 }
4905 }
4906
4907 // Canonicalize equivalent operands to LL == RL.
4908 if (LL == RR && LR == RL) {
4909 CC1 = ISD::getSetCCSwappedOperands(CC1);
4910 std::swap(RL, RR);
4911 }
4912
4913 // (and (setcc X, Y, CC0), (setcc X, Y, CC1)) --> (setcc X, Y, NewCC)
4914 // (or (setcc X, Y, CC0), (setcc X, Y, CC1)) --> (setcc X, Y, NewCC)
4915 if (LL == RL && LR == RR) {
4916 ISD::CondCode NewCC = IsAnd ? ISD::getSetCCAndOperation(CC0, CC1, OpVT)
4917 : ISD::getSetCCOrOperation(CC0, CC1, OpVT);
4918 if (NewCC != ISD::SETCC_INVALID &&
4919 (!LegalOperations ||
4920 (TLI.isCondCodeLegal(NewCC, LL.getSimpleValueType()) &&
4921 TLI.isOperationLegal(ISD::SETCC, OpVT))))
4922 return DAG.getSetCC(DL, VT, LL, LR, NewCC);
4923 }
4924
4925 return SDValue();
4926}
4927
4928/// This contains all DAGCombine rules which reduce two values combined by
4929/// an And operation to a single value. This makes them reusable in the context
4930/// of visitSELECT(). Rules involving constants are not included as
4931/// visitSELECT() already handles those cases.
4932SDValue DAGCombiner::visitANDLike(SDValue N0, SDValue N1, SDNode *N) {
4933 EVT VT = N1.getValueType();
4934 SDLoc DL(N);
4935
4936 // fold (and x, undef) -> 0
4937 if (N0.isUndef() || N1.isUndef())
4938 return DAG.getConstant(0, DL, VT);
4939
4940 if (SDValue V = foldLogicOfSetCCs(true, N0, N1, DL))
4941 return V;
4942
4943 if (N0.getOpcode() == ISD::ADD && N1.getOpcode() == ISD::SRL &&
4944 VT.getSizeInBits() <= 64) {
4945 if (ConstantSDNode *ADDI = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
4946 if (ConstantSDNode *SRLI = dyn_cast<ConstantSDNode>(N1.getOperand(1))) {
4947 // Look for (and (add x, c1), (lshr y, c2)). If C1 wasn't a legal
4948 // immediate for an add, but it is legal if its top c2 bits are set,
4949 // transform the ADD so the immediate doesn't need to be materialized
4950 // in a register.
4951 APInt ADDC = ADDI->getAPIntValue();
4952 APInt SRLC = SRLI->getAPIntValue();
4953 if (ADDC.getMinSignedBits() <= 64 &&
4954 SRLC.ult(VT.getSizeInBits()) &&
4955 !TLI.isLegalAddImmediate(ADDC.getSExtValue())) {
4956 APInt Mask = APInt::getHighBitsSet(VT.getSizeInBits(),
4957 SRLC.getZExtValue());
4958 if (DAG.MaskedValueIsZero(N0.getOperand(1), Mask)) {
4959 ADDC |= Mask;
4960 if (TLI.isLegalAddImmediate(ADDC.getSExtValue())) {
4961 SDLoc DL0(N0);
4962 SDValue NewAdd =
4963 DAG.getNode(ISD::ADD, DL0, VT,
4964 N0.getOperand(0), DAG.getConstant(ADDC, DL, VT));
4965 CombineTo(N0.getNode(), NewAdd);
4966 // Return N so it doesn't get rechecked!
4967 return SDValue(N, 0);
4968 }
4969 }
4970 }
4971 }
4972 }
4973 }
4974
4975 // Reduce bit extract of low half of an integer to the narrower type.
4976 // (and (srl i64:x, K), KMask) ->
4977 // (i64 zero_extend (and (srl (i32 (trunc i64:x)), K)), KMask)
4978 if (N0.getOpcode() == ISD::SRL && N0.hasOneUse()) {
4979 if (ConstantSDNode *CAnd = dyn_cast<ConstantSDNode>(N1)) {
4980 if (ConstantSDNode *CShift = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
4981 unsigned Size = VT.getSizeInBits();
4982 const APInt &AndMask = CAnd->getAPIntValue();
4983 unsigned ShiftBits = CShift->getZExtValue();
4984
4985 // Bail out, this node will probably disappear anyway.
4986 if (ShiftBits == 0)
4987 return SDValue();
4988
4989 unsigned MaskBits = AndMask.countTrailingOnes();
4990 EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), Size / 2);
4991
4992 if (AndMask.isMask() &&
4993 // Required bits must not span the two halves of the integer and
4994 // must fit in the half size type.
4995 (ShiftBits + MaskBits <= Size / 2) &&
4996 TLI.isNarrowingProfitable(VT, HalfVT) &&
4997 TLI.isTypeDesirableForOp(ISD::AND, HalfVT) &&
4998 TLI.isTypeDesirableForOp(ISD::SRL, HalfVT) &&
4999 TLI.isTruncateFree(VT, HalfVT) &&
5000 TLI.isZExtFree(HalfVT, VT)) {
5001 // The isNarrowingProfitable is to avoid regressions on PPC and
5002 // AArch64 which match a few 64-bit bit insert / bit extract patterns
5003 // on downstream users of this. Those patterns could probably be
5004 // extended to handle extensions mixed in.
5005
5006 SDValue SL(N0);
5007 assert(MaskBits <= Size)((MaskBits <= Size) ? static_cast<void> (0) : __assert_fail
("MaskBits <= Size", "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5007, __PRETTY_FUNCTION__))
;
5008
5009 // Extracting the highest bit of the low half.
5010 EVT ShiftVT = TLI.getShiftAmountTy(HalfVT, DAG.getDataLayout());
5011 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, HalfVT,
5012 N0.getOperand(0));
5013
5014 SDValue NewMask = DAG.getConstant(AndMask.trunc(Size / 2), SL, HalfVT);
5015 SDValue ShiftK = DAG.getConstant(ShiftBits, SL, ShiftVT);
5016 SDValue Shift = DAG.getNode(ISD::SRL, SL, HalfVT, Trunc, ShiftK);
5017 SDValue And = DAG.getNode(ISD::AND, SL, HalfVT, Shift, NewMask);
5018 return DAG.getNode(ISD::ZERO_EXTEND, SL, VT, And);
5019 }
5020 }
5021 }
5022 }
5023
5024 return SDValue();
5025}
5026
5027bool DAGCombiner::isAndLoadExtLoad(ConstantSDNode *AndC, LoadSDNode *LoadN,
5028 EVT LoadResultTy, EVT &ExtVT) {
5029 if (!AndC->getAPIntValue().isMask())
5030 return false;
5031
5032 unsigned ActiveBits = AndC->getAPIntValue().countTrailingOnes();
5033
5034 ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
5035 EVT LoadedVT = LoadN->getMemoryVT();
5036
5037 if (ExtVT == LoadedVT &&
5038 (!LegalOperations ||
5039 TLI.isLoadExtLegal(ISD::ZEXTLOAD, LoadResultTy, ExtVT))) {
5040 // ZEXTLOAD will match without needing to change the size of the value being
5041 // loaded.
5042 return true;
5043 }
5044
5045 // Do not change the width of a volatile or atomic loads.
5046 if (!LoadN->isSimple())
5047 return false;
5048
5049 // Do not generate loads of non-round integer types since these can
5050 // be expensive (and would be wrong if the type is not byte sized).
5051 if (!LoadedVT.bitsGT(ExtVT) || !ExtVT.isRound())
5052 return false;
5053
5054 if (LegalOperations &&
5055 !TLI.isLoadExtLegal(ISD::ZEXTLOAD, LoadResultTy, ExtVT))
5056 return false;
5057
5058 if (!TLI.shouldReduceLoadWidth(LoadN, ISD::ZEXTLOAD, ExtVT))
5059 return false;
5060
5061 return true;
5062}
5063
5064bool DAGCombiner::isLegalNarrowLdSt(LSBaseSDNode *LDST,
5065 ISD::LoadExtType ExtType, EVT &MemVT,
5066 unsigned ShAmt) {
5067 if (!LDST)
5068 return false;
5069 // Only allow byte offsets.
5070 if (ShAmt % 8)
5071 return false;
5072
5073 // Do not generate loads of non-round integer types since these can
5074 // be expensive (and would be wrong if the type is not byte sized).
5075 if (!MemVT.isRound())
5076 return false;
5077
5078 // Don't change the width of a volatile or atomic loads.
5079 if (!LDST->isSimple())
5080 return false;
5081
5082 EVT LdStMemVT = LDST->getMemoryVT();
5083
5084 // Bail out when changing the scalable property, since we can't be sure that
5085 // we're actually narrowing here.
5086 if (LdStMemVT.isScalableVector() != MemVT.isScalableVector())
5087 return false;
5088
5089 // Verify that we are actually reducing a load width here.
5090 if (LdStMemVT.bitsLT(MemVT))
5091 return false;
5092
5093 // Ensure that this isn't going to produce an unsupported memory access.
5094 if (ShAmt) {
5095 assert(ShAmt % 8 == 0 && "ShAmt is byte offset")((ShAmt % 8 == 0 && "ShAmt is byte offset") ? static_cast
<void> (0) : __assert_fail ("ShAmt % 8 == 0 && \"ShAmt is byte offset\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5095, __PRETTY_FUNCTION__))
;
5096 const unsigned ByteShAmt = ShAmt / 8;
5097 const Align LDSTAlign = LDST->getAlign();
5098 const Align NarrowAlign = commonAlignment(LDSTAlign, ByteShAmt);
5099 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), MemVT,
5100 LDST->getAddressSpace(), NarrowAlign,
5101 LDST->getMemOperand()->getFlags()))
5102 return false;
5103 }
5104
5105 // It's not possible to generate a constant of extended or untyped type.
5106 EVT PtrType = LDST->getBasePtr().getValueType();
5107 if (PtrType == MVT::Untyped || PtrType.isExtended())
5108 return false;
5109
5110 if (isa<LoadSDNode>(LDST)) {
5111 LoadSDNode *Load = cast<LoadSDNode>(LDST);
5112 // Don't transform one with multiple uses, this would require adding a new
5113 // load.
5114 if (!SDValue(Load, 0).hasOneUse())
5115 return false;
5116
5117 if (LegalOperations &&
5118 !TLI.isLoadExtLegal(ExtType, Load->getValueType(0), MemVT))
5119 return false;
5120
5121 // For the transform to be legal, the load must produce only two values
5122 // (the value loaded and the chain). Don't transform a pre-increment
5123 // load, for example, which produces an extra value. Otherwise the
5124 // transformation is not equivalent, and the downstream logic to replace
5125 // uses gets things wrong.
5126 if (Load->getNumValues() > 2)
5127 return false;
5128
5129 // If the load that we're shrinking is an extload and we're not just
5130 // discarding the extension we can't simply shrink the load. Bail.
5131 // TODO: It would be possible to merge the extensions in some cases.
5132 if (Load->getExtensionType() != ISD::NON_EXTLOAD &&
5133 Load->getMemoryVT().getSizeInBits() < MemVT.getSizeInBits() + ShAmt)
5134 return false;
5135
5136 if (!TLI.shouldReduceLoadWidth(Load, ExtType, MemVT))
5137 return false;
5138 } else {
5139 assert(isa<StoreSDNode>(LDST) && "It is not a Load nor a Store SDNode")((isa<StoreSDNode>(LDST) && "It is not a Load nor a Store SDNode"
) ? static_cast<void> (0) : __assert_fail ("isa<StoreSDNode>(LDST) && \"It is not a Load nor a Store SDNode\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5139, __PRETTY_FUNCTION__))
;
5140 StoreSDNode *Store = cast<StoreSDNode>(LDST);
5141 // Can't write outside the original store
5142 if (Store->getMemoryVT().getSizeInBits() < MemVT.getSizeInBits() + ShAmt)
5143 return false;
5144
5145 if (LegalOperations &&
5146 !TLI.isTruncStoreLegal(Store->getValue().getValueType(), MemVT))
5147 return false;
5148 }
5149 return true;
5150}
5151
5152bool DAGCombiner::SearchForAndLoads(SDNode *N,
5153 SmallVectorImpl<LoadSDNode*> &Loads,
5154 SmallPtrSetImpl<SDNode*> &NodesWithConsts,
5155 ConstantSDNode *Mask,
5156 SDNode *&NodeToMask) {
5157 // Recursively search for the operands, looking for loads which can be
5158 // narrowed.
5159 for (SDValue Op : N->op_values()) {
5160 if (Op.getValueType().isVector())
5161 return false;
5162
5163 // Some constants may need fixing up later if they are too large.
5164 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
5165 if ((N->getOpcode() == ISD::OR || N->getOpcode() == ISD::XOR) &&
5166 (Mask->getAPIntValue() & C->getAPIntValue()) != C->getAPIntValue())
5167 NodesWithConsts.insert(N);
5168 continue;
5169 }
5170
5171 if (!Op.hasOneUse())
5172 return false;
5173
5174 switch(Op.getOpcode()) {
5175 case ISD::LOAD: {
5176 auto *Load = cast<LoadSDNode>(Op);
5177 EVT ExtVT;
5178 if (isAndLoadExtLoad(Mask, Load, Load->getValueType(0), ExtVT) &&
5179 isLegalNarrowLdSt(Load, ISD::ZEXTLOAD, ExtVT)) {
5180
5181 // ZEXTLOAD is already small enough.
5182 if (Load->getExtensionType() == ISD::ZEXTLOAD &&
5183 ExtVT.bitsGE(Load->getMemoryVT()))
5184 continue;
5185
5186 // Use LE to convert equal sized loads to zext.
5187 if (ExtVT.bitsLE(Load->getMemoryVT()))
5188 Loads.push_back(Load);
5189
5190 continue;
5191 }
5192 return false;
5193 }
5194 case ISD::ZERO_EXTEND:
5195 case ISD::AssertZext: {
5196 unsigned ActiveBits = Mask->getAPIntValue().countTrailingOnes();
5197 EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
5198 EVT VT = Op.getOpcode() == ISD::AssertZext ?
5199 cast<VTSDNode>(Op.getOperand(1))->getVT() :
5200 Op.getOperand(0).getValueType();
5201
5202 // We can accept extending nodes if the mask is wider or an equal
5203 // width to the original type.
5204 if (ExtVT.bitsGE(VT))
5205 continue;
5206 break;
5207 }
5208 case ISD::OR:
5209 case ISD::XOR:
5210 case ISD::AND:
5211 if (!SearchForAndLoads(Op.getNode(), Loads, NodesWithConsts, Mask,
5212 NodeToMask))
5213 return false;
5214 continue;
5215 }
5216
5217 // Allow one node which will masked along with any loads found.
5218 if (NodeToMask)
5219 return false;
5220
5221 // Also ensure that the node to be masked only produces one data result.
5222 NodeToMask = Op.getNode();
5223 if (NodeToMask->getNumValues() > 1) {
5224 bool HasValue = false;
5225 for (unsigned i = 0, e = NodeToMask->getNumValues(); i < e; ++i) {
5226 MVT VT = SDValue(NodeToMask, i).getSimpleValueType();
5227 if (VT != MVT::Glue && VT != MVT::Other) {
5228 if (HasValue) {
5229 NodeToMask = nullptr;
5230 return false;
5231 }
5232 HasValue = true;
5233 }
5234 }
5235 assert(HasValue && "Node to be masked has no data result?")((HasValue && "Node to be masked has no data result?"
) ? static_cast<void> (0) : __assert_fail ("HasValue && \"Node to be masked has no data result?\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5235, __PRETTY_FUNCTION__))
;
5236 }
5237 }
5238 return true;
5239}
5240
5241bool DAGCombiner::BackwardsPropagateMask(SDNode *N) {
5242 auto *Mask = dyn_cast<ConstantSDNode>(N->getOperand(1));
5243 if (!Mask)
5244 return false;
5245
5246 if (!Mask->getAPIntValue().isMask())
5247 return false;
5248
5249 // No need to do anything if the and directly uses a load.
5250 if (isa<LoadSDNode>(N->getOperand(0)))
5251 return false;
5252
5253 SmallVector<LoadSDNode*, 8> Loads;
5254 SmallPtrSet<SDNode*, 2> NodesWithConsts;
5255 SDNode *FixupNode = nullptr;
5256 if (SearchForAndLoads(N, Loads, NodesWithConsts, Mask, FixupNode)) {
5257 if (Loads.size() == 0)
5258 return false;
5259
5260 LLVM_DEBUG(dbgs() << "Backwards propagate AND: "; N->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "Backwards propagate AND: "
; N->dump(); } } while (false)
;
5261 SDValue MaskOp = N->getOperand(1);
5262
5263 // If it exists, fixup the single node we allow in the tree that needs
5264 // masking.
5265 if (FixupNode) {
5266 LLVM_DEBUG(dbgs() << "First, need to fix up: "; FixupNode->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "First, need to fix up: "; FixupNode
->dump(); } } while (false)
;
5267 SDValue And = DAG.getNode(ISD::AND, SDLoc(FixupNode),
5268 FixupNode->getValueType(0),
5269 SDValue(FixupNode, 0), MaskOp);
5270 DAG.ReplaceAllUsesOfValueWith(SDValue(FixupNode, 0), And);
5271 if (And.getOpcode() == ISD ::AND)
5272 DAG.UpdateNodeOperands(And.getNode(), SDValue(FixupNode, 0), MaskOp);
5273 }
5274
5275 // Narrow any constants that need it.
5276 for (auto *LogicN : NodesWithConsts) {
5277 SDValue Op0 = LogicN->getOperand(0);
5278 SDValue Op1 = LogicN->getOperand(1);
5279
5280 if (isa<ConstantSDNode>(Op0))
5281 std::swap(Op0, Op1);
5282
5283 SDValue And = DAG.getNode(ISD::AND, SDLoc(Op1), Op1.getValueType(),
5284 Op1, MaskOp);
5285
5286 DAG.UpdateNodeOperands(LogicN, Op0, And);
5287 }
5288
5289 // Create narrow loads.
5290 for (auto *Load : Loads) {
5291 LLVM_DEBUG(dbgs() << "Propagate AND back to: "; Load->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "Propagate AND back to: "; Load
->dump(); } } while (false)
;
5292 SDValue And = DAG.getNode(ISD::AND, SDLoc(Load), Load->getValueType(0),
5293 SDValue(Load, 0), MaskOp);
5294 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), And);
5295 if (And.getOpcode() == ISD ::AND)
5296 And = SDValue(
5297 DAG.UpdateNodeOperands(And.getNode(), SDValue(Load, 0), MaskOp), 0);
5298 SDValue NewLoad = ReduceLoadWidth(And.getNode());
5299 assert(NewLoad &&((NewLoad && "Shouldn't be masking the load if it can't be narrowed"
) ? static_cast<void> (0) : __assert_fail ("NewLoad && \"Shouldn't be masking the load if it can't be narrowed\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5300, __PRETTY_FUNCTION__))
5300 "Shouldn't be masking the load if it can't be narrowed")((NewLoad && "Shouldn't be masking the load if it can't be narrowed"
) ? static_cast<void> (0) : __assert_fail ("NewLoad && \"Shouldn't be masking the load if it can't be narrowed\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5300, __PRETTY_FUNCTION__))
;
5301 CombineTo(Load, NewLoad, NewLoad.getValue(1));
5302 }
5303 DAG.ReplaceAllUsesWith(N, N->getOperand(0).getNode());
5304 return true;
5305 }
5306 return false;
5307}
5308
5309// Unfold
5310// x & (-1 'logical shift' y)
5311// To
5312// (x 'opposite logical shift' y) 'logical shift' y
5313// if it is better for performance.
5314SDValue DAGCombiner::unfoldExtremeBitClearingToShifts(SDNode *N) {
5315 assert(N->getOpcode() == ISD::AND)((N->getOpcode() == ISD::AND) ? static_cast<void> (0
) : __assert_fail ("N->getOpcode() == ISD::AND", "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5315, __PRETTY_FUNCTION__))
;
5316
5317 SDValue N0 = N->getOperand(0);
5318 SDValue N1 = N->getOperand(1);
5319
5320 // Do we actually prefer shifts over mask?
5321 if (!TLI.shouldFoldMaskToVariableShiftPair(N0))
5322 return SDValue();
5323
5324 // Try to match (-1 '[outer] logical shift' y)
5325 unsigned OuterShift;
5326 unsigned InnerShift; // The opposite direction to the OuterShift.
5327 SDValue Y; // Shift amount.
5328 auto matchMask = [&OuterShift, &InnerShift, &Y](SDValue M) -> bool {
5329 if (!M.hasOneUse())
5330 return false;
5331 OuterShift = M->getOpcode();
5332 if (OuterShift == ISD::SHL)
5333 InnerShift = ISD::SRL;
5334 else if (OuterShift == ISD::SRL)
5335 InnerShift = ISD::SHL;
5336 else
5337 return false;
5338 if (!isAllOnesConstant(M->getOperand(0)))
5339 return false;
5340 Y = M->getOperand(1);
5341 return true;
5342 };
5343
5344 SDValue X;
5345 if (matchMask(N1))
5346 X = N0;
5347 else if (matchMask(N0))
5348 X = N1;
5349 else
5350 return SDValue();
5351
5352 SDLoc DL(N);
5353 EVT VT = N->getValueType(0);
5354
5355 // tmp = x 'opposite logical shift' y
5356 SDValue T0 = DAG.getNode(InnerShift, DL, VT, X, Y);
5357 // ret = tmp 'logical shift' y
5358 SDValue T1 = DAG.getNode(OuterShift, DL, VT, T0, Y);
5359
5360 return T1;
5361}
5362
5363/// Try to replace shift/logic that tests if a bit is clear with mask + setcc.
5364/// For a target with a bit test, this is expected to become test + set and save
5365/// at least 1 instruction.
5366static SDValue combineShiftAnd1ToBitTest(SDNode *And, SelectionDAG &DAG) {
5367 assert(And->getOpcode() == ISD::AND && "Expected an 'and' op")((And->getOpcode() == ISD::AND && "Expected an 'and' op"
) ? static_cast<void> (0) : __assert_fail ("And->getOpcode() == ISD::AND && \"Expected an 'and' op\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5367, __PRETTY_FUNCTION__))
;
5368
5369 // This is probably not worthwhile without a supported type.
5370 EVT VT = And->getValueType(0);
5371 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5372 if (!TLI.isTypeLegal(VT))
5373 return SDValue();
5374
5375 // Look through an optional extension and find a 'not'.
5376 // TODO: Should we favor test+set even without the 'not' op?
5377 SDValue Not = And->getOperand(0), And1 = And->getOperand(1);
5378 if (Not.getOpcode() == ISD::ANY_EXTEND)
5379 Not = Not.getOperand(0);
5380 if (!isBitwiseNot(Not) || !Not.hasOneUse() || !isOneConstant(And1))
5381 return SDValue();
5382
5383 // Look though an optional truncation. The source operand may not be the same
5384 // type as the original 'and', but that is ok because we are masking off
5385 // everything but the low bit.
5386 SDValue Srl = Not.getOperand(0);
5387 if (Srl.getOpcode() == ISD::TRUNCATE)
5388 Srl = Srl.getOperand(0);
5389
5390 // Match a shift-right by constant.
5391 if (Srl.getOpcode() != ISD::SRL || !Srl.hasOneUse() ||
5392 !isa<ConstantSDNode>(Srl.getOperand(1)))
5393 return SDValue();
5394
5395 // We might have looked through casts that make this transform invalid.
5396 // TODO: If the source type is wider than the result type, do the mask and
5397 // compare in the source type.
5398 const APInt &ShiftAmt = Srl.getConstantOperandAPInt(1);
5399 unsigned VTBitWidth = VT.getSizeInBits();
5400 if (ShiftAmt.uge(VTBitWidth))
5401 return SDValue();
5402
5403 // Turn this into a bit-test pattern using mask op + setcc:
5404 // and (not (srl X, C)), 1 --> (and X, 1<<C) == 0
5405 SDLoc DL(And);
5406 SDValue X = DAG.getZExtOrTrunc(Srl.getOperand(0), DL, VT);
5407 EVT CCVT = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
5408 SDValue Mask = DAG.getConstant(
5409 APInt::getOneBitSet(VTBitWidth, ShiftAmt.getZExtValue()), DL, VT);
5410 SDValue NewAnd = DAG.getNode(ISD::AND, DL, VT, X, Mask);
5411 SDValue Zero = DAG.getConstant(0, DL, VT);
5412 SDValue Setcc = DAG.getSetCC(DL, CCVT, NewAnd, Zero, ISD::SETEQ);
5413 return DAG.getZExtOrTrunc(Setcc, DL, VT);
5414}
5415
5416SDValue DAGCombiner::visitAND(SDNode *N) {
5417 SDValue N0 = N->getOperand(0);
5418 SDValue N1 = N->getOperand(1);
5419 EVT VT = N1.getValueType();
5420
5421 // x & x --> x
5422 if (N0 == N1)
5423 return N0;
5424
5425 // fold vector ops
5426 if (VT.isVector()) {
5427 if (SDValue FoldedVOp = SimplifyVBinOp(N))
5428 return FoldedVOp;
5429
5430 // fold (and x, 0) -> 0, vector edition
5431 if (ISD::isBuildVectorAllZeros(N0.getNode()))
5432 // do not return N0, because undef node may exist in N0
5433 return DAG.getConstant(APInt::getNullValue(N0.getScalarValueSizeInBits()),
5434 SDLoc(N), N0.getValueType());
5435 if (ISD::isBuildVectorAllZeros(N1.getNode()))
5436 // do not return N1, because undef node may exist in N1
5437 return DAG.getConstant(APInt::getNullValue(N1.getScalarValueSizeInBits()),
5438 SDLoc(N), N1.getValueType());
5439
5440 // fold (and x, -1) -> x, vector edition
5441 if (ISD::isBuildVectorAllOnes(N0.getNode()))
5442 return N1;
5443 if (ISD::isBuildVectorAllOnes(N1.getNode()))
5444 return N0;
5445
5446 // fold (and (masked_load) (build_vec (x, ...))) to zext_masked_load
5447 auto *MLoad = dyn_cast<MaskedLoadSDNode>(N0);
5448 auto *BVec = dyn_cast<BuildVectorSDNode>(N1);
5449 if (MLoad && BVec && MLoad->getExtensionType() == ISD::EXTLOAD &&
5450 N0.hasOneUse() && N1.hasOneUse()) {
5451 EVT LoadVT = MLoad->getMemoryVT();
5452 EVT ExtVT = VT;
5453 if (TLI.isLoadExtLegal(ISD::ZEXTLOAD, ExtVT, LoadVT)) {
5454 // For this AND to be a zero extension of the masked load the elements
5455 // of the BuildVec must mask the bottom bits of the extended element
5456 // type
5457 if (ConstantSDNode *Splat = BVec->getConstantSplatNode()) {
5458 uint64_t ElementSize =
5459 LoadVT.getVectorElementType().getScalarSizeInBits();
5460 if (Splat->getAPIntValue().isMask(ElementSize)) {
5461 return DAG.getMaskedLoad(
5462 ExtVT, SDLoc(N), MLoad->getChain(), MLoad->getBasePtr(),
5463 MLoad->getOffset(), MLoad->getMask(), MLoad->getPassThru(),
5464 LoadVT, MLoad->getMemOperand(), MLoad->getAddressingMode(),
5465 ISD::ZEXTLOAD, MLoad->isExpandingLoad());
5466 }
5467 }
5468 }
5469 }
5470 }
5471
5472 // fold (and c1, c2) -> c1&c2
5473 ConstantSDNode *N1C = isConstOrConstSplat(N1);
5474 if (SDValue C = DAG.FoldConstantArithmetic(ISD::AND, SDLoc(N), VT, {N0, N1}))
5475 return C;
5476
5477 // canonicalize constant to RHS
5478 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
5479 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
5480 return DAG.getNode(ISD::AND, SDLoc(N), VT, N1, N0);
5481
5482 // fold (and x, -1) -> x
5483 if (isAllOnesConstant(N1))
5484 return N0;
5485
5486 // if (and x, c) is known to be zero, return 0
5487 unsigned BitWidth = VT.getScalarSizeInBits();
5488 if (N1C && DAG.MaskedValueIsZero(SDValue(N, 0),
5489 APInt::getAllOnesValue(BitWidth)))
5490 return DAG.getConstant(0, SDLoc(N), VT);
5491
5492 if (SDValue NewSel = foldBinOpIntoSelect(N))
5493 return NewSel;
5494
5495 // reassociate and
5496 if (SDValue RAND = reassociateOps(ISD::AND, SDLoc(N), N0, N1, N->getFlags()))
5497 return RAND;
5498
5499 // Try to convert a constant mask AND into a shuffle clear mask.
5500 if (VT.isVector())
5501 if (SDValue Shuffle = XformToShuffleWithZero(N))
5502 return Shuffle;
5503
5504 if (SDValue Combined = combineCarryDiamond(*this, DAG, TLI, N0, N1, N))
5505 return Combined;
5506
5507 // fold (and (or x, C), D) -> D if (C & D) == D
5508 auto MatchSubset = [](ConstantSDNode *LHS, ConstantSDNode *RHS) {
5509 return RHS->getAPIntValue().isSubsetOf(LHS->getAPIntValue());
5510 };
5511 if (N0.getOpcode() == ISD::OR &&
5512 ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchSubset))
5513 return N1;
5514 // fold (and (any_ext V), c) -> (zero_ext V) if 'and' only clears top bits.
5515 if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) {
5516 SDValue N0Op0 = N0.getOperand(0);
5517 APInt Mask = ~N1C->getAPIntValue();
5518 Mask = Mask.trunc(N0Op0.getScalarValueSizeInBits());
5519 if (DAG.MaskedValueIsZero(N0Op0, Mask)) {
5520 SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N),
5521 N0.getValueType(), N0Op0);
5522
5523 // Replace uses of the AND with uses of the Zero extend node.
5524 CombineTo(N, Zext);
5525
5526 // We actually want to replace all uses of the any_extend with the
5527 // zero_extend, to avoid duplicating things. This will later cause this
5528 // AND to be folded.
5529 CombineTo(N0.getNode(), Zext);
5530 return SDValue(N, 0); // Return N so it doesn't get rechecked!
5531 }
5532 }
5533
5534 // similarly fold (and (X (load ([non_ext|any_ext|zero_ext] V))), c) ->
5535 // (X (load ([non_ext|zero_ext] V))) if 'and' only clears top bits which must
5536 // already be zero by virtue of the width of the base type of the load.
5537 //
5538 // the 'X' node here can either be nothing or an extract_vector_elt to catch
5539 // more cases.
5540 if ((N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
5541 N0.getValueSizeInBits() == N0.getOperand(0).getScalarValueSizeInBits() &&
5542 N0.getOperand(0).getOpcode() == ISD::LOAD &&
5543 N0.getOperand(0).getResNo() == 0) ||
5544 (N0.getOpcode() == ISD::LOAD && N0.getResNo() == 0)) {
5545 LoadSDNode *Load = cast<LoadSDNode>( (N0.getOpcode() == ISD::LOAD) ?
5546 N0 : N0.getOperand(0) );
5547
5548 // Get the constant (if applicable) the zero'th operand is being ANDed with.
5549 // This can be a pure constant or a vector splat, in which case we treat the
5550 // vector as a scalar and use the splat value.
5551 APInt Constant = APInt::getNullValue(1);
5552 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) {
5553 Constant = C->getAPIntValue();
5554 } else if (BuildVectorSDNode *Vector = dyn_cast<BuildVectorSDNode>(N1)) {
5555 APInt SplatValue, SplatUndef;
5556 unsigned SplatBitSize;
5557 bool HasAnyUndefs;
5558 bool IsSplat = Vector->isConstantSplat(SplatValue, SplatUndef,
5559 SplatBitSize, HasAnyUndefs);
5560 if (IsSplat) {
5561 // Undef bits can contribute to a possible optimisation if set, so
5562 // set them.
5563 SplatValue |= SplatUndef;
5564
5565 // The splat value may be something like "0x00FFFFFF", which means 0 for
5566 // the first vector value and FF for the rest, repeating. We need a mask
5567 // that will apply equally to all members of the vector, so AND all the
5568 // lanes of the constant together.
5569 unsigned EltBitWidth = Vector->getValueType(0).getScalarSizeInBits();
5570
5571 // If the splat value has been compressed to a bitlength lower
5572 // than the size of the vector lane, we need to re-expand it to
5573 // the lane size.
5574 if (EltBitWidth > SplatBitSize)
5575 for (SplatValue = SplatValue.zextOrTrunc(EltBitWidth);
5576 SplatBitSize < EltBitWidth; SplatBitSize = SplatBitSize * 2)
5577 SplatValue |= SplatValue.shl(SplatBitSize);
5578
5579 // Make sure that variable 'Constant' is only set if 'SplatBitSize' is a
5580 // multiple of 'BitWidth'. Otherwise, we could propagate a wrong value.
5581 if ((SplatBitSize % EltBitWidth) == 0) {
5582 Constant = APInt::getAllOnesValue(EltBitWidth);
5583 for (unsigned i = 0, n = (SplatBitSize / EltBitWidth); i < n; ++i)
5584 Constant &= SplatValue.extractBits(EltBitWidth, i * EltBitWidth);
5585 }
5586 }
5587 }
5588
5589 // If we want to change an EXTLOAD to a ZEXTLOAD, ensure a ZEXTLOAD is
5590 // actually legal and isn't going to get expanded, else this is a false
5591 // optimisation.
5592 bool CanZextLoadProfitably = TLI.isLoadExtLegal(ISD::ZEXTLOAD,
5593 Load->getValueType(0),
5594 Load->getMemoryVT());
5595
5596 // Resize the constant to the same size as the original memory access before
5597 // extension. If it is still the AllOnesValue then this AND is completely
5598 // unneeded.
5599 Constant = Constant.zextOrTrunc(Load->getMemoryVT().getScalarSizeInBits());
5600
5601 bool B;
5602 switch (Load->getExtensionType()) {
5603 default: B = false; break;
5604 case ISD::EXTLOAD: B = CanZextLoadProfitably; break;
5605 case ISD::ZEXTLOAD:
5606 case ISD::NON_EXTLOAD: B = true; break;
5607 }
5608
5609 if (B && Constant.isAllOnesValue()) {
5610 // If the load type was an EXTLOAD, convert to ZEXTLOAD in order to
5611 // preserve semantics once we get rid of the AND.
5612 SDValue NewLoad(Load, 0);
5613
5614 // Fold the AND away. NewLoad may get replaced immediately.
5615 CombineTo(N, (N0.getNode() == Load) ? NewLoad : N0);
5616
5617 if (Load->getExtensionType() == ISD::EXTLOAD) {
5618 NewLoad = DAG.getLoad(Load->getAddressingMode(), ISD::ZEXTLOAD,
5619 Load->getValueType(0), SDLoc(Load),
5620 Load->getChain(), Load->getBasePtr(),
5621 Load->getOffset(), Load->getMemoryVT(),
5622 Load->getMemOperand());
5623 // Replace uses of the EXTLOAD with the new ZEXTLOAD.
5624 if (Load->getNumValues() == 3) {
5625 // PRE/POST_INC loads have 3 values.
5626 SDValue To[] = { NewLoad.getValue(0), NewLoad.getValue(1),
5627 NewLoad.getValue(2) };
5628 CombineTo(Load, To, 3, true);
5629 } else {
5630 CombineTo(Load, NewLoad.getValue(0), NewLoad.getValue(1));
5631 }
5632 }
5633
5634 return SDValue(N, 0); // Return N so it doesn't get rechecked!
5635 }
5636 }
5637
5638 // fold (and (masked_gather x)) -> (zext_masked_gather x)
5639 if (auto *GN0 = dyn_cast<MaskedGatherSDNode>(N0)) {
5640 EVT MemVT = GN0->getMemoryVT();
5641 EVT ScalarVT = MemVT.getScalarType();
5642
5643 if (SDValue(GN0, 0).hasOneUse() &&
5644 isConstantSplatVectorMaskForType(N1.getNode(), ScalarVT) &&
5645 TLI.isVectorLoadExtDesirable(SDValue(SDValue(GN0, 0)))) {
5646 SDValue Ops[] = {GN0->getChain(), GN0->getPassThru(), GN0->getMask(),
5647 GN0->getBasePtr(), GN0->getIndex(), GN0->getScale()};
5648
5649 SDValue ZExtLoad = DAG.getMaskedGather(
5650 DAG.getVTList(VT, MVT::Other), MemVT, SDLoc(N), Ops,
5651 GN0->getMemOperand(), GN0->getIndexType(), ISD::ZEXTLOAD);
5652
5653 CombineTo(N, ZExtLoad);
5654 AddToWorklist(ZExtLoad.getNode());
5655 // Avoid recheck of N.
5656 return SDValue(N, 0);
5657 }
5658 }
5659
5660 // fold (and (load x), 255) -> (zextload x, i8)
5661 // fold (and (extload x, i16), 255) -> (zextload x, i8)
5662 // fold (and (any_ext (extload x, i16)), 255) -> (zextload x, i8)
5663 if (!VT.isVector() && N1C && (N0.getOpcode() == ISD::LOAD ||
5664 (N0.getOpcode() == ISD::ANY_EXTEND &&
5665 N0.getOperand(0).getOpcode() == ISD::LOAD))) {
5666 if (SDValue Res = ReduceLoadWidth(N)) {
5667 LoadSDNode *LN0 = N0->getOpcode() == ISD::ANY_EXTEND
5668 ? cast<LoadSDNode>(N0.getOperand(0)) : cast<LoadSDNode>(N0);
5669 AddToWorklist(N);
5670 DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 0), Res);
5671 return SDValue(N, 0);
5672 }
5673 }
5674
5675 if (LegalTypes) {
5676 // Attempt to propagate the AND back up to the leaves which, if they're
5677 // loads, can be combined to narrow loads and the AND node can be removed.
5678 // Perform after legalization so that extend nodes will already be
5679 // combined into the loads.
5680 if (BackwardsPropagateMask(N))
5681 return SDValue(N, 0);
5682 }
5683
5684 if (SDValue Combined = visitANDLike(N0, N1, N))
5685 return Combined;
5686
5687 // Simplify: (and (op x...), (op y...)) -> (op (and x, y))
5688 if (N0.getOpcode() == N1.getOpcode())
5689 if (SDValue V = hoistLogicOpWithSameOpcodeHands(N))
5690 return V;
5691
5692 // Masking the negated extension of a boolean is just the zero-extended
5693 // boolean:
5694 // and (sub 0, zext(bool X)), 1 --> zext(bool X)
5695 // and (sub 0, sext(bool X)), 1 --> zext(bool X)
5696 //
5697 // Note: the SimplifyDemandedBits fold below can make an information-losing
5698 // transform, and then we have no way to find this better fold.
5699 if (N1C && N1C->isOne() && N0.getOpcode() == ISD::SUB) {
5700 if (isNullOrNullSplat(N0.getOperand(0))) {
5701 SDValue SubRHS = N0.getOperand(1);
5702 if (SubRHS.getOpcode() == ISD::ZERO_EXTEND &&
5703 SubRHS.getOperand(0).getScalarValueSizeInBits() == 1)
5704 return SubRHS;
5705 if (SubRHS.getOpcode() == ISD::SIGN_EXTEND &&
5706 SubRHS.getOperand(0).getScalarValueSizeInBits() == 1)
5707 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, SubRHS.getOperand(0));
5708 }
5709 }
5710
5711 // fold (and (sign_extend_inreg x, i16 to i32), 1) -> (and x, 1)
5712 // fold (and (sra)) -> (and (srl)) when possible.
5713 if (SimplifyDemandedBits(SDValue(N, 0)))
5714 return SDValue(N, 0);
5715
5716 // fold (zext_inreg (extload x)) -> (zextload x)
5717 // fold (zext_inreg (sextload x)) -> (zextload x) iff load has one use
5718 if (ISD::isUNINDEXEDLoad(N0.getNode()) &&
5719 (ISD::isEXTLoad(N0.getNode()) ||
5720 (ISD::isSEXTLoad(N0.getNode()) && N0.hasOneUse()))) {
5721 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
5722 EVT MemVT = LN0->getMemoryVT();
5723 // If we zero all the possible extended bits, then we can turn this into
5724 // a zextload if we are running before legalize or the operation is legal.
5725 unsigned ExtBitSize = N1.getScalarValueSizeInBits();
5726 unsigned MemBitSize = MemVT.getScalarSizeInBits();
5727 APInt ExtBits = APInt::getHighBitsSet(ExtBitSize, ExtBitSize - MemBitSize);
5728 if (DAG.MaskedValueIsZero(N1, ExtBits) &&
5729 ((!LegalOperations && LN0->isSimple()) ||
5730 TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT))) {
5731 SDValue ExtLoad =
5732 DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT, LN0->getChain(),
5733 LN0->getBasePtr(), MemVT, LN0->getMemOperand());
5734 AddToWorklist(N);
5735 CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
5736 return SDValue(N, 0); // Return N so it doesn't get rechecked!
5737 }
5738 }
5739
5740 // fold (and (or (srl N, 8), (shl N, 8)), 0xffff) -> (srl (bswap N), const)
5741 if (N1C && N1C->getAPIntValue() == 0xffff && N0.getOpcode() == ISD::OR) {
5742 if (SDValue BSwap = MatchBSwapHWordLow(N0.getNode(), N0.getOperand(0),
5743 N0.getOperand(1), false))
5744 return BSwap;
5745 }
5746
5747 if (SDValue Shifts = unfoldExtremeBitClearingToShifts(N))
5748 return Shifts;
5749
5750 if (TLI.hasBitTest(N0, N1))
5751 if (SDValue V = combineShiftAnd1ToBitTest(N, DAG))
5752 return V;
5753
5754 // Recognize the following pattern:
5755 //
5756 // AndVT = (and (sign_extend NarrowVT to AndVT) #bitmask)
5757 //
5758 // where bitmask is a mask that clears the upper bits of AndVT. The
5759 // number of bits in bitmask must be a power of two.
5760 auto IsAndZeroExtMask = [](SDValue LHS, SDValue RHS) {
5761 if (LHS->getOpcode() != ISD::SIGN_EXTEND)
5762 return false;
5763
5764 auto *C = dyn_cast<ConstantSDNode>(RHS);
5765 if (!C)
5766 return false;
5767
5768 if (!C->getAPIntValue().isMask(
5769 LHS.getOperand(0).getValueType().getFixedSizeInBits()))
5770 return false;
5771
5772 return true;
5773 };
5774
5775 // Replace (and (sign_extend ...) #bitmask) with (zero_extend ...).
5776 if (IsAndZeroExtMask(N0, N1))
5777 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, N0.getOperand(0));
5778
5779 return SDValue();
5780}
5781
5782/// Match (a >> 8) | (a << 8) as (bswap a) >> 16.
5783SDValue DAGCombiner::MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
5784 bool DemandHighBits) {
5785 if (!LegalOperations)
5786 return SDValue();
5787
5788 EVT VT = N->getValueType(0);
5789 if (VT != MVT::i64 && VT != MVT::i32 && VT != MVT::i16)
5790 return SDValue();
5791 if (!TLI.isOperationLegalOrCustom(ISD::BSWAP, VT))
5792 return SDValue();
5793
5794 // Recognize (and (shl a, 8), 0xff00), (and (srl a, 8), 0xff)
5795 bool LookPassAnd0 = false;
5796 bool LookPassAnd1 = false;
5797 if (N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::SRL)
5798 std::swap(N0, N1);
5799 if (N1.getOpcode() == ISD::AND && N1.getOperand(0).getOpcode() == ISD::SHL)
5800 std::swap(N0, N1);
5801 if (N0.getOpcode() == ISD::AND) {
5802 if (!N0.getNode()->hasOneUse())
5803 return SDValue();
5804 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
5805 // Also handle 0xffff since the LHS is guaranteed to have zeros there.
5806 // This is needed for X86.
5807 if (!N01C || (N01C->getZExtValue() != 0xFF00 &&
5808 N01C->getZExtValue() != 0xFFFF))
5809 return SDValue();
5810 N0 = N0.getOperand(0);
5811 LookPassAnd0 = true;
5812 }
5813
5814 if (N1.getOpcode() == ISD::AND) {
5815 if (!N1.getNode()->hasOneUse())
5816 return SDValue();
5817 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
5818 if (!N11C || N11C->getZExtValue() != 0xFF)
5819 return SDValue();
5820 N1 = N1.getOperand(0);
5821 LookPassAnd1 = true;
5822 }
5823
5824 if (N0.getOpcode() == ISD::SRL && N1.getOpcode() == ISD::SHL)
5825 std::swap(N0, N1);
5826 if (N0.getOpcode() != ISD::SHL || N1.getOpcode() != ISD::SRL)
5827 return SDValue();
5828 if (!N0.getNode()->hasOneUse() || !N1.getNode()->hasOneUse())
5829 return SDValue();
5830
5831 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
5832 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
5833 if (!N01C || !N11C)
5834 return SDValue();
5835 if (N01C->getZExtValue() != 8 || N11C->getZExtValue() != 8)
5836 return SDValue();
5837
5838 // Look for (shl (and a, 0xff), 8), (srl (and a, 0xff00), 8)
5839 SDValue N00 = N0->getOperand(0);
5840 if (!LookPassAnd0 && N00.getOpcode() == ISD::AND) {
5841 if (!N00.getNode()->hasOneUse())
5842 return SDValue();
5843 ConstantSDNode *N001C = dyn_cast<ConstantSDNode>(N00.getOperand(1));
5844 if (!N001C || N001C->getZExtValue() != 0xFF)
5845 return SDValue();
5846 N00 = N00.getOperand(0);
5847 LookPassAnd0 = true;
5848 }
5849
5850 SDValue N10 = N1->getOperand(0);
5851 if (!LookPassAnd1 && N10.getOpcode() == ISD::AND) {
5852 if (!N10.getNode()->hasOneUse())
5853 return SDValue();
5854 ConstantSDNode *N101C = dyn_cast<ConstantSDNode>(N10.getOperand(1));
5855 // Also allow 0xFFFF since the bits will be shifted out. This is needed
5856 // for X86.
5857 if (!N101C || (N101C->getZExtValue() != 0xFF00 &&
5858 N101C->getZExtValue() != 0xFFFF))
5859 return SDValue();
5860 N10 = N10.getOperand(0);
5861 LookPassAnd1 = true;
5862 }
5863
5864 if (N00 != N10)
5865 return SDValue();
5866
5867 // Make sure everything beyond the low halfword gets set to zero since the SRL
5868 // 16 will clear the top bits.
5869 unsigned OpSizeInBits = VT.getSizeInBits();
5870 if (DemandHighBits && OpSizeInBits > 16) {
5871 // If the left-shift isn't masked out then the only way this is a bswap is
5872 // if all bits beyond the low 8 are 0. In that case the entire pattern
5873 // reduces to a left shift anyway: leave it for other parts of the combiner.
5874 if (!LookPassAnd0)
5875 return SDValue();
5876
5877 // However, if the right shift isn't masked out then it might be because
5878 // it's not needed. See if we can spot that too.
5879 if (!LookPassAnd1 &&
5880 !DAG.MaskedValueIsZero(
5881 N10, APInt::getHighBitsSet(OpSizeInBits, OpSizeInBits - 16)))
5882 return SDValue();
5883 }
5884
5885 SDValue Res = DAG.getNode(ISD::BSWAP, SDLoc(N), VT, N00);
5886 if (OpSizeInBits > 16) {
5887 SDLoc DL(N);
5888 Res = DAG.getNode(ISD::SRL, DL, VT, Res,
5889 DAG.getConstant(OpSizeInBits - 16, DL,
5890 getShiftAmountTy(VT)));
5891 }
5892 return Res;
5893}
5894
5895/// Return true if the specified node is an element that makes up a 32-bit
5896/// packed halfword byteswap.
5897/// ((x & 0x000000ff) << 8) |
5898/// ((x & 0x0000ff00) >> 8) |
5899/// ((x & 0x00ff0000) << 8) |
5900/// ((x & 0xff000000) >> 8)
5901static bool isBSwapHWordElement(SDValue N, MutableArrayRef<SDNode *> Parts) {
5902 if (!N.getNode()->hasOneUse())
5903 return false;
5904
5905 unsigned Opc = N.getOpcode();
5906 if (Opc != ISD::AND && Opc != ISD::SHL && Opc != ISD::SRL)
5907 return false;
5908
5909 SDValue N0 = N.getOperand(0);
5910 unsigned Opc0 = N0.getOpcode();
5911 if (Opc0 != ISD::AND && Opc0 != ISD::SHL && Opc0 != ISD::SRL)
5912 return false;
5913
5914 ConstantSDNode *N1C = nullptr;
5915 // SHL or SRL: look upstream for AND mask operand
5916 if (Opc == ISD::AND)
5917 N1C = dyn_cast<ConstantSDNode>(N.getOperand(1));
5918 else if (Opc0 == ISD::AND)
5919 N1C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
5920 if (!N1C)
5921 return false;
5922
5923 unsigned MaskByteOffset;
5924 switch (N1C->getZExtValue()) {
5925 default:
5926 return false;
5927 case 0xFF: MaskByteOffset = 0; break;
5928 case 0xFF00: MaskByteOffset = 1; break;
5929 case 0xFFFF:
5930 // In case demanded bits didn't clear the bits that will be shifted out.
5931 // This is needed for X86.
5932 if (Opc == ISD::SRL || (Opc == ISD::AND && Opc0 == ISD::SHL)) {
5933 MaskByteOffset = 1;
5934 break;
5935 }
5936 return false;
5937 case 0xFF0000: MaskByteOffset = 2; break;
5938 case 0xFF000000: MaskByteOffset = 3; break;
5939 }
5940
5941 // Look for (x & 0xff) << 8 as well as ((x << 8) & 0xff00).
5942 if (Opc == ISD::AND) {
5943 if (MaskByteOffset == 0 || MaskByteOffset == 2) {
5944 // (x >> 8) & 0xff
5945 // (x >> 8) & 0xff0000
5946 if (Opc0 != ISD::SRL)
5947 return false;
5948 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
5949 if (!C || C->getZExtValue() != 8)
5950 return false;
5951 } else {
5952 // (x << 8) & 0xff00
5953 // (x << 8) & 0xff000000
5954 if (Opc0 != ISD::SHL)
5955 return false;
5956 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
5957 if (!C || C->getZExtValue() != 8)
5958 return false;
5959 }
5960 } else if (Opc == ISD::SHL) {
5961 // (x & 0xff) << 8
5962 // (x & 0xff0000) << 8
5963 if (MaskByteOffset != 0 && MaskByteOffset != 2)
5964 return false;
5965 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
5966 if (!C || C->getZExtValue() != 8)
5967 return false;
5968 } else { // Opc == ISD::SRL
5969 // (x & 0xff00) >> 8
5970 // (x & 0xff000000) >> 8
5971 if (MaskByteOffset != 1 && MaskByteOffset != 3)
5972 return false;
5973 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
5974 if (!C || C->getZExtValue() != 8)
5975 return false;
5976 }
5977
5978 if (Parts[MaskByteOffset])
5979 return false;
5980
5981 Parts[MaskByteOffset] = N0.getOperand(0).getNode();
5982 return true;
5983}
5984
5985// Match 2 elements of a packed halfword bswap.
5986static bool isBSwapHWordPair(SDValue N, MutableArrayRef<SDNode *> Parts) {
5987 if (N.getOpcode() == ISD::OR)
5988 return isBSwapHWordElement(N.getOperand(0), Parts) &&
5989 isBSwapHWordElement(N.getOperand(1), Parts);
5990
5991 if (N.getOpcode() == ISD::SRL && N.getOperand(0).getOpcode() == ISD::BSWAP) {
5992 ConstantSDNode *C = isConstOrConstSplat(N.getOperand(1));
5993 if (!C || C->getAPIntValue() != 16)
5994 return false;
5995 Parts[0] = Parts[1] = N.getOperand(0).getOperand(0).getNode();
5996 return true;
5997 }
5998
5999 return false;
6000}
6001
6002// Match this pattern:
6003// (or (and (shl (A, 8)), 0xff00ff00), (and (srl (A, 8)), 0x00ff00ff))
6004// And rewrite this to:
6005// (rotr (bswap A), 16)
6006static SDValue matchBSwapHWordOrAndAnd(const TargetLowering &TLI,
6007 SelectionDAG &DAG, SDNode *N, SDValue N0,
6008 SDValue N1, EVT VT, EVT ShiftAmountTy) {
6009 assert(N->getOpcode() == ISD::OR && VT == MVT::i32 &&((N->getOpcode() == ISD::OR && VT == MVT::i32 &&
"MatchBSwapHWordOrAndAnd: expecting i32") ? static_cast<void
> (0) : __assert_fail ("N->getOpcode() == ISD::OR && VT == MVT::i32 && \"MatchBSwapHWordOrAndAnd: expecting i32\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 6010, __PRETTY_FUNCTION__))
6010 "MatchBSwapHWordOrAndAnd: expecting i32")((N->getOpcode() == ISD::OR && VT == MVT::i32 &&
"MatchBSwapHWordOrAndAnd: expecting i32") ? static_cast<void
> (0) : __assert_fail ("N->getOpcode() == ISD::OR && VT == MVT::i32 && \"MatchBSwapHWordOrAndAnd: expecting i32\""
, "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 6010, __PRETTY_FUNCTION__))
;
6011 if (!TLI.isOperationLegalOrCustom(ISD::ROTR, VT))
6012 return SDValue();
6013 if (N0.getOpcode() != ISD::AND || N1.getOpcode() != ISD::AND)
6014 return SDValue();
6015 // TODO: this is too restrictive; lifting this restriction requires more tests
6016 if (!N0->hasOneUse() || !N1->hasOneUse())
6017 return SDValue();
6018 ConstantSDNode *Mask0 = isConstOrConstSplat(N0.getOperand(1));
6019 ConstantSDNode *Mask1 = isConstOrConstSplat(N1.getOperand(1));
6020 if (!Mask0 || !Mask1)
6021 return SDValue();
6022 if (Mask0->getAPIntValue() != 0xff00ff00 ||
6023 Mask1->getAPIntValue() != 0x00ff00ff)
6024 return SDValue();
6025 SDValue Shift0 = N0.getOperand(0);
6026 SDValue Shift1 = N1.getOperand(0);
6027 if (Shift0.getOpcode() != ISD::SHL || Shift1.getOpcode() != ISD::SRL)
6028 return SDValue();
6029 ConstantSDNode *ShiftAmt0 = isConstOrConstSplat(Shift0.getOperand(1));
6030 ConstantSDNode *ShiftAmt1 = isConstOrConstSplat(Shift1.getOperand(1));
6031 if (!ShiftAmt0 || !ShiftAmt1)
6032 return SDValue();
6033 if (ShiftAmt0->getAPIntValue() != 8 || ShiftAmt1->getAPIntValue() != 8)
6034 return SDValue();
6035 if (Shift0.getOperand(0) != Shift1.getOperand(0))
6036 return SDValue();
6037
6038 SDLoc DL(N);
6039 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT, Shift0.getOperand(0));
6040 SDValue ShAmt = DAG.getConstant(16, DL, ShiftAmountTy);
6041 return DAG.getNode(ISD::ROTR, DL, VT, BSwap, ShAmt);
6042}
6043
6044/// Match a 32-bit packed halfword bswap. That is
6045/// ((x & 0x000000ff) << 8) |
6046/// ((x & 0x0000ff00) >> 8) |
6047/// ((x & 0x00ff0000) << 8) |
6048/// ((x & 0xff000000) >> 8)
6049/// => (rotl (bswap x), 16)
6050SDValue DAGCombiner::MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1) {
6051 if (!LegalOperations)
6052 return SDValue();
6053
6054 EVT VT = N->getValueType(0);
6055 if (VT != MVT::i32)
6056 return SDValue();
6057 if (!TLI.isOperationLegalOrCustom(ISD::BSWAP, VT))