Bug Summary

File:lib/CodeGen/SelectionDAG/DAGCombiner.cpp
Warning:line 391, column 32
The result of the right shift is undefined due to shifting by '64', which is greater or equal to the width of type 'llvm::APInt::WordType'

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name DAGCombiner.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-eagerly-assume -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -mrelocation-model pic -pic-level 2 -mthread-model posix -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -momit-leaf-frame-pointer -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-7/lib/clang/7.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-7~svn326246/build-llvm/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-7~svn326246/build-llvm/include -I /build/llvm-toolchain-snapshot-7~svn326246/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/7.3.0/../../../../include/c++/7.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/7.3.0/../../../../include/x86_64-linux-gnu/c++/7.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/7.3.0/../../../../include/x86_64-linux-gnu/c++/7.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/7.3.0/../../../../include/c++/7.3.0/backward -internal-isystem /usr/include/clang/7.0.0/include/ -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-7/lib/clang/7.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++11 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-7~svn326246/build-llvm/lib/CodeGen/SelectionDAG -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-checker optin.performance.Padding -analyzer-output=html -analyzer-config stable-report-filename=true -o /tmp/scan-build-2018-02-28-041547-14988-1 -x c++ /build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

1//===- DAGCombiner.cpp - Implement a DAG node combiner --------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This pass combines dag nodes to form fewer, simpler DAG nodes. It can be run
11// both before and after the DAG is legalized.
12//
13// This pass is not a substitute for the LLVM IR instcombine pass. This pass is
14// primarily intended to handle simplification opportunities that are implicit
15// in the LLVM IR and exposed by the various codegen lowering phases.
16//
17//===----------------------------------------------------------------------===//
18
19#include "llvm/ADT/APFloat.h"
20#include "llvm/ADT/APInt.h"
21#include "llvm/ADT/ArrayRef.h"
22#include "llvm/ADT/DenseMap.h"
23#include "llvm/ADT/None.h"
24#include "llvm/ADT/Optional.h"
25#include "llvm/ADT/STLExtras.h"
26#include "llvm/ADT/SetVector.h"
27#include "llvm/ADT/SmallBitVector.h"
28#include "llvm/ADT/SmallPtrSet.h"
29#include "llvm/ADT/SmallSet.h"
30#include "llvm/ADT/SmallVector.h"
31#include "llvm/ADT/Statistic.h"
32#include "llvm/Analysis/AliasAnalysis.h"
33#include "llvm/Analysis/MemoryLocation.h"
34#include "llvm/CodeGen/DAGCombine.h"
35#include "llvm/CodeGen/ISDOpcodes.h"
36#include "llvm/CodeGen/MachineFrameInfo.h"
37#include "llvm/CodeGen/MachineFunction.h"
38#include "llvm/CodeGen/MachineMemOperand.h"
39#include "llvm/CodeGen/MachineValueType.h"
40#include "llvm/CodeGen/RuntimeLibcalls.h"
41#include "llvm/CodeGen/SelectionDAG.h"
42#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
43#include "llvm/CodeGen/SelectionDAGNodes.h"
44#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
45#include "llvm/CodeGen/TargetLowering.h"
46#include "llvm/CodeGen/TargetRegisterInfo.h"
47#include "llvm/CodeGen/TargetSubtargetInfo.h"
48#include "llvm/CodeGen/ValueTypes.h"
49#include "llvm/IR/Attributes.h"
50#include "llvm/IR/Constant.h"
51#include "llvm/IR/DataLayout.h"
52#include "llvm/IR/DerivedTypes.h"
53#include "llvm/IR/Function.h"
54#include "llvm/IR/LLVMContext.h"
55#include "llvm/IR/Metadata.h"
56#include "llvm/Support/Casting.h"
57#include "llvm/Support/CodeGen.h"
58#include "llvm/Support/CommandLine.h"
59#include "llvm/Support/Compiler.h"
60#include "llvm/Support/Debug.h"
61#include "llvm/Support/ErrorHandling.h"
62#include "llvm/Support/KnownBits.h"
63#include "llvm/Support/MathExtras.h"
64#include "llvm/Support/raw_ostream.h"
65#include "llvm/Target/TargetMachine.h"
66#include "llvm/Target/TargetOptions.h"
67#include <algorithm>
68#include <cassert>
69#include <cstdint>
70#include <functional>
71#include <iterator>
72#include <string>
73#include <tuple>
74#include <utility>
75#include <vector>
76
77using namespace llvm;
78
79#define DEBUG_TYPE"dagcombine" "dagcombine"
80
81STATISTIC(NodesCombined , "Number of dag nodes combined")static llvm::Statistic NodesCombined = {"dagcombine", "NodesCombined"
, "Number of dag nodes combined", {0}, {false}}
;
82STATISTIC(PreIndexedNodes , "Number of pre-indexed nodes created")static llvm::Statistic PreIndexedNodes = {"dagcombine", "PreIndexedNodes"
, "Number of pre-indexed nodes created", {0}, {false}}
;
83STATISTIC(PostIndexedNodes, "Number of post-indexed nodes created")static llvm::Statistic PostIndexedNodes = {"dagcombine", "PostIndexedNodes"
, "Number of post-indexed nodes created", {0}, {false}}
;
84STATISTIC(OpsNarrowed , "Number of load/op/store narrowed")static llvm::Statistic OpsNarrowed = {"dagcombine", "OpsNarrowed"
, "Number of load/op/store narrowed", {0}, {false}}
;
85STATISTIC(LdStFP2Int , "Number of fp load/store pairs transformed to int")static llvm::Statistic LdStFP2Int = {"dagcombine", "LdStFP2Int"
, "Number of fp load/store pairs transformed to int", {0}, {false
}}
;
86STATISTIC(SlicedLoads, "Number of load sliced")static llvm::Statistic SlicedLoads = {"dagcombine", "SlicedLoads"
, "Number of load sliced", {0}, {false}}
;
87
88static cl::opt<bool>
89CombinerGlobalAA("combiner-global-alias-analysis", cl::Hidden,
90 cl::desc("Enable DAG combiner's use of IR alias analysis"));
91
92static cl::opt<bool>
93UseTBAA("combiner-use-tbaa", cl::Hidden, cl::init(true),
94 cl::desc("Enable DAG combiner's use of TBAA"));
95
96#ifndef NDEBUG
97static cl::opt<std::string>
98CombinerAAOnlyFunc("combiner-aa-only-func", cl::Hidden,
99 cl::desc("Only use DAG-combiner alias analysis in this"
100 " function"));
101#endif
102
103/// Hidden option to stress test load slicing, i.e., when this option
104/// is enabled, load slicing bypasses most of its profitability guards.
105static cl::opt<bool>
106StressLoadSlicing("combiner-stress-load-slicing", cl::Hidden,
107 cl::desc("Bypass the profitability model of load slicing"),
108 cl::init(false));
109
110static cl::opt<bool>
111 MaySplitLoadIndex("combiner-split-load-index", cl::Hidden, cl::init(true),
112 cl::desc("DAG combiner may split indexing from loads"));
113
114namespace {
115
116 class DAGCombiner {
117 SelectionDAG &DAG;
118 const TargetLowering &TLI;
119 CombineLevel Level;
120 CodeGenOpt::Level OptLevel;
121 bool LegalOperations = false;
122 bool LegalTypes = false;
123 bool ForCodeSize;
124
125 /// \brief Worklist of all of the nodes that need to be simplified.
126 ///
127 /// This must behave as a stack -- new nodes to process are pushed onto the
128 /// back and when processing we pop off of the back.
129 ///
130 /// The worklist will not contain duplicates but may contain null entries
131 /// due to nodes being deleted from the underlying DAG.
132 SmallVector<SDNode *, 64> Worklist;
133
134 /// \brief Mapping from an SDNode to its position on the worklist.
135 ///
136 /// This is used to find and remove nodes from the worklist (by nulling
137 /// them) when they are deleted from the underlying DAG. It relies on
138 /// stable indices of nodes within the worklist.
139 DenseMap<SDNode *, unsigned> WorklistMap;
140
141 /// \brief Set of nodes which have been combined (at least once).
142 ///
143 /// This is used to allow us to reliably add any operands of a DAG node
144 /// which have not yet been combined to the worklist.
145 SmallPtrSet<SDNode *, 32> CombinedNodes;
146
147 // AA - Used for DAG load/store alias analysis.
148 AliasAnalysis *AA;
149
150 /// When an instruction is simplified, add all users of the instruction to
151 /// the work lists because they might get more simplified now.
152 void AddUsersToWorklist(SDNode *N) {
153 for (SDNode *Node : N->uses())
154 AddToWorklist(Node);
155 }
156
157 /// Call the node-specific routine that folds each particular type of node.
158 SDValue visit(SDNode *N);
159
160 public:
161 DAGCombiner(SelectionDAG &D, AliasAnalysis *AA, CodeGenOpt::Level OL)
162 : DAG(D), TLI(D.getTargetLoweringInfo()), Level(BeforeLegalizeTypes),
163 OptLevel(OL), AA(AA) {
164 ForCodeSize = DAG.getMachineFunction().getFunction().optForSize();
165
166 MaximumLegalStoreInBits = 0;
167 for (MVT VT : MVT::all_valuetypes())
168 if (EVT(VT).isSimple() && VT != MVT::Other &&
169 TLI.isTypeLegal(EVT(VT)) &&
170 VT.getSizeInBits() >= MaximumLegalStoreInBits)
171 MaximumLegalStoreInBits = VT.getSizeInBits();
172 }
173
174 /// Add to the worklist making sure its instance is at the back (next to be
175 /// processed.)
176 void AddToWorklist(SDNode *N) {
177 assert(N->getOpcode() != ISD::DELETED_NODE &&(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& "Deleted Node added to Worklist") ? void (0) : __assert_fail
("N->getOpcode() != ISD::DELETED_NODE && \"Deleted Node added to Worklist\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 178, __extension__ __PRETTY_FUNCTION__))
178 "Deleted Node added to Worklist")(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& "Deleted Node added to Worklist") ? void (0) : __assert_fail
("N->getOpcode() != ISD::DELETED_NODE && \"Deleted Node added to Worklist\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 178, __extension__ __PRETTY_FUNCTION__))
;
179
180 // Skip handle nodes as they can't usefully be combined and confuse the
181 // zero-use deletion strategy.
182 if (N->getOpcode() == ISD::HANDLENODE)
183 return;
184
185 if (WorklistMap.insert(std::make_pair(N, Worklist.size())).second)
186 Worklist.push_back(N);
187 }
188
189 /// Remove all instances of N from the worklist.
190 void removeFromWorklist(SDNode *N) {
191 CombinedNodes.erase(N);
192
193 auto It = WorklistMap.find(N);
194 if (It == WorklistMap.end())
195 return; // Not in the worklist.
196
197 // Null out the entry rather than erasing it to avoid a linear operation.
198 Worklist[It->second] = nullptr;
199 WorklistMap.erase(It);
200 }
201
202 void deleteAndRecombine(SDNode *N);
203 bool recursivelyDeleteUnusedNodes(SDNode *N);
204
205 /// Replaces all uses of the results of one DAG node with new values.
206 SDValue CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
207 bool AddTo = true);
208
209 /// Replaces all uses of the results of one DAG node with new values.
210 SDValue CombineTo(SDNode *N, SDValue Res, bool AddTo = true) {
211 return CombineTo(N, &Res, 1, AddTo);
212 }
213
214 /// Replaces all uses of the results of one DAG node with new values.
215 SDValue CombineTo(SDNode *N, SDValue Res0, SDValue Res1,
216 bool AddTo = true) {
217 SDValue To[] = { Res0, Res1 };
218 return CombineTo(N, To, 2, AddTo);
219 }
220
221 void CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO);
222
223 private:
224 unsigned MaximumLegalStoreInBits;
225
226 /// Check the specified integer node value to see if it can be simplified or
227 /// if things it uses can be simplified by bit propagation.
228 /// If so, return true.
229 bool SimplifyDemandedBits(SDValue Op) {
230 unsigned BitWidth = Op.getScalarValueSizeInBits();
231 APInt Demanded = APInt::getAllOnesValue(BitWidth);
232 return SimplifyDemandedBits(Op, Demanded);
233 }
234
235 /// Check the specified vector node value to see if it can be simplified or
236 /// if things it uses can be simplified as it only uses some of the
237 /// elements. If so, return true.
238 bool SimplifyDemandedVectorElts(SDValue Op) {
239 unsigned NumElts = Op.getValueType().getVectorNumElements();
240 APInt Demanded = APInt::getAllOnesValue(NumElts);
241 return SimplifyDemandedVectorElts(Op, Demanded);
242 }
243
244 bool SimplifyDemandedBits(SDValue Op, const APInt &Demanded);
245 bool SimplifyDemandedVectorElts(SDValue Op, const APInt &Demanded);
246
247 bool CombineToPreIndexedLoadStore(SDNode *N);
248 bool CombineToPostIndexedLoadStore(SDNode *N);
249 SDValue SplitIndexingFromLoad(LoadSDNode *LD);
250 bool SliceUpLoad(SDNode *N);
251
252 /// \brief Replace an ISD::EXTRACT_VECTOR_ELT of a load with a narrowed
253 /// load.
254 ///
255 /// \param EVE ISD::EXTRACT_VECTOR_ELT to be replaced.
256 /// \param InVecVT type of the input vector to EVE with bitcasts resolved.
257 /// \param EltNo index of the vector element to load.
258 /// \param OriginalLoad load that EVE came from to be replaced.
259 /// \returns EVE on success SDValue() on failure.
260 SDValue ReplaceExtractVectorEltOfLoadWithNarrowedLoad(
261 SDNode *EVE, EVT InVecVT, SDValue EltNo, LoadSDNode *OriginalLoad);
262 void ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad);
263 SDValue PromoteOperand(SDValue Op, EVT PVT, bool &Replace);
264 SDValue SExtPromoteOperand(SDValue Op, EVT PVT);
265 SDValue ZExtPromoteOperand(SDValue Op, EVT PVT);
266 SDValue PromoteIntBinOp(SDValue Op);
267 SDValue PromoteIntShiftOp(SDValue Op);
268 SDValue PromoteExtend(SDValue Op);
269 bool PromoteLoad(SDValue Op);
270
271 void ExtendSetCCUses(const SmallVectorImpl<SDNode *> &SetCCs,
272 SDValue OrigLoad, SDValue ExtLoad,
273 const SDLoc &DL,
274 ISD::NodeType ExtType);
275
276 /// Call the node-specific routine that knows how to fold each
277 /// particular type of node. If that doesn't do anything, try the
278 /// target-specific DAG combines.
279 SDValue combine(SDNode *N);
280
281 // Visitation implementation - Implement dag node combining for different
282 // node types. The semantics are as follows:
283 // Return Value:
284 // SDValue.getNode() == 0 - No change was made
285 // SDValue.getNode() == N - N was replaced, is dead and has been handled.
286 // otherwise - N should be replaced by the returned Operand.
287 //
288 SDValue visitTokenFactor(SDNode *N);
289 SDValue visitMERGE_VALUES(SDNode *N);
290 SDValue visitADD(SDNode *N);
291 SDValue visitADDLike(SDValue N0, SDValue N1, SDNode *LocReference);
292 SDValue visitSUB(SDNode *N);
293 SDValue visitADDC(SDNode *N);
294 SDValue visitUADDO(SDNode *N);
295 SDValue visitUADDOLike(SDValue N0, SDValue N1, SDNode *N);
296 SDValue visitSUBC(SDNode *N);
297 SDValue visitUSUBO(SDNode *N);
298 SDValue visitADDE(SDNode *N);
299 SDValue visitADDCARRY(SDNode *N);
300 SDValue visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn, SDNode *N);
301 SDValue visitSUBE(SDNode *N);
302 SDValue visitSUBCARRY(SDNode *N);
303 SDValue visitMUL(SDNode *N);
304 SDValue useDivRem(SDNode *N);
305 SDValue visitSDIV(SDNode *N);
306 SDValue visitUDIV(SDNode *N);
307 SDValue visitREM(SDNode *N);
308 SDValue visitMULHU(SDNode *N);
309 SDValue visitMULHS(SDNode *N);
310 SDValue visitSMUL_LOHI(SDNode *N);
311 SDValue visitUMUL_LOHI(SDNode *N);
312 SDValue visitSMULO(SDNode *N);
313 SDValue visitUMULO(SDNode *N);
314 SDValue visitIMINMAX(SDNode *N);
315 SDValue visitAND(SDNode *N);
316 SDValue visitANDLike(SDValue N0, SDValue N1, SDNode *LocReference);
317 SDValue visitOR(SDNode *N);
318 SDValue visitORLike(SDValue N0, SDValue N1, SDNode *LocReference);
319 SDValue visitXOR(SDNode *N);
320 SDValue SimplifyVBinOp(SDNode *N);
321 SDValue visitSHL(SDNode *N);
322 SDValue visitSRA(SDNode *N);
323 SDValue visitSRL(SDNode *N);
324 SDValue visitRotate(SDNode *N);
325 SDValue visitABS(SDNode *N);
326 SDValue visitBSWAP(SDNode *N);
327 SDValue visitBITREVERSE(SDNode *N);
328 SDValue visitCTLZ(SDNode *N);
329 SDValue visitCTLZ_ZERO_UNDEF(SDNode *N);
330 SDValue visitCTTZ(SDNode *N);
331 SDValue visitCTTZ_ZERO_UNDEF(SDNode *N);
332 SDValue visitCTPOP(SDNode *N);
333 SDValue visitSELECT(SDNode *N);
334 SDValue visitVSELECT(SDNode *N);
335 SDValue visitSELECT_CC(SDNode *N);
336 SDValue visitSETCC(SDNode *N);
337 SDValue visitSETCCE(SDNode *N);
338 SDValue visitSETCCCARRY(SDNode *N);
339 SDValue visitSIGN_EXTEND(SDNode *N);
340 SDValue visitZERO_EXTEND(SDNode *N);
341 SDValue visitANY_EXTEND(SDNode *N);
342 SDValue visitAssertExt(SDNode *N);
343 SDValue visitSIGN_EXTEND_INREG(SDNode *N);
344 SDValue visitSIGN_EXTEND_VECTOR_INREG(SDNode *N);
345 SDValue visitZERO_EXTEND_VECTOR_INREG(SDNode *N);
346 SDValue visitTRUNCATE(SDNode *N);
347 SDValue visitBITCAST(SDNode *N);
348 SDValue visitBUILD_PAIR(SDNode *N);
349 SDValue visitFADD(SDNode *N);
350 SDValue visitFSUB(SDNode *N);
351 SDValue visitFMUL(SDNode *N);
352 SDValue visitFMA(SDNode *N);
353 SDValue visitFDIV(SDNode *N);
354 SDValue visitFREM(SDNode *N);
355 SDValue visitFSQRT(SDNode *N);
356 SDValue visitFCOPYSIGN(SDNode *N);
357 SDValue visitSINT_TO_FP(SDNode *N);
358 SDValue visitUINT_TO_FP(SDNode *N);
359 SDValue visitFP_TO_SINT(SDNode *N);
360 SDValue visitFP_TO_UINT(SDNode *N);
361 SDValue visitFP_ROUND(SDNode *N);
362 SDValue visitFP_ROUND_INREG(SDNode *N);
363 SDValue visitFP_EXTEND(SDNode *N);
364 SDValue visitFNEG(SDNode *N);
365 SDValue visitFABS(SDNode *N);
366 SDValue visitFCEIL(SDNode *N);
367 SDValue visitFTRUNC(SDNode *N);
368 SDValue visitFFLOOR(SDNode *N);
369 SDValue visitFMINNUM(SDNode *N);
370 SDValue visitFMAXNUM(SDNode *N);
371 SDValue visitBRCOND(SDNode *N);
372 SDValue visitBR_CC(SDNode *N);
373 SDValue visitLOAD(SDNode *N);
374
375 SDValue replaceStoreChain(StoreSDNode *ST, SDValue BetterChain);
376 SDValue replaceStoreOfFPConstant(StoreSDNode *ST);
377
378 SDValue visitSTORE(SDNode *N);
379 SDValue visitINSERT_VECTOR_ELT(SDNode *N);
380 SDValue visitEXTRACT_VECTOR_ELT(SDNode *N);
381 SDValue visitBUILD_VECTOR(SDNode *N);
382 SDValue visitCONCAT_VECTORS(SDNode *N);
383 SDValue visitEXTRACT_SUBVECTOR(SDNode *N);
384 SDValue visitVECTOR_SHUFFLE(SDNode *N);
385 SDValue visitSCALAR_TO_VECTOR(SDNode *N);
386 SDValue visitINSERT_SUBVECTOR(SDNode *N);
387 SDValue visitMLOAD(SDNode *N);
388 SDValue visitMSTORE(SDNode *N);
389 SDValue visitMGATHER(SDNode *N);
390 SDValue visitMSCATTER(SDNode *N);
391 SDValue visitFP_TO_FP16(SDNode *N);
392 SDValue visitFP16_TO_FP(SDNode *N);
393
394 SDValue visitFADDForFMACombine(SDNode *N);
395 SDValue visitFSUBForFMACombine(SDNode *N);
396 SDValue visitFMULForFMADistributiveCombine(SDNode *N);
397
398 SDValue XformToShuffleWithZero(SDNode *N);
399 SDValue ReassociateOps(unsigned Opc, const SDLoc &DL, SDValue LHS,
400 SDValue RHS);
401
402 SDValue visitShiftByConstant(SDNode *N, ConstantSDNode *Amt);
403
404 SDValue foldSelectOfConstants(SDNode *N);
405 SDValue foldVSelectOfConstants(SDNode *N);
406 SDValue foldBinOpIntoSelect(SDNode *BO);
407 bool SimplifySelectOps(SDNode *SELECT, SDValue LHS, SDValue RHS);
408 SDValue SimplifyBinOpWithSameOpcodeHands(SDNode *N);
409 SDValue SimplifySelect(const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2);
410 SDValue SimplifySelectCC(const SDLoc &DL, SDValue N0, SDValue N1,
411 SDValue N2, SDValue N3, ISD::CondCode CC,
412 bool NotExtCompare = false);
413 SDValue foldSelectCCToShiftAnd(const SDLoc &DL, SDValue N0, SDValue N1,
414 SDValue N2, SDValue N3, ISD::CondCode CC);
415 SDValue foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1,
416 const SDLoc &DL);
417 SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
418 const SDLoc &DL, bool foldBooleans);
419 SDValue rebuildSetCC(SDValue N);
420
421 bool isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
422 SDValue &CC) const;
423 bool isOneUseSetCC(SDValue N) const;
424
425 SDValue SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
426 unsigned HiOp);
427 SDValue CombineConsecutiveLoads(SDNode *N, EVT VT);
428 SDValue CombineExtLoad(SDNode *N);
429 SDValue combineRepeatedFPDivisors(SDNode *N);
430 SDValue combineInsertEltToShuffle(SDNode *N, unsigned InsIndex);
431 SDValue ConstantFoldBITCASTofBUILD_VECTOR(SDNode *, EVT);
432 SDValue BuildSDIV(SDNode *N);
433 SDValue BuildSDIVPow2(SDNode *N);
434 SDValue BuildUDIV(SDNode *N);
435 SDValue BuildLogBase2(SDValue Op, const SDLoc &DL);
436 SDValue BuildReciprocalEstimate(SDValue Op, SDNodeFlags Flags);
437 SDValue buildRsqrtEstimate(SDValue Op, SDNodeFlags Flags);
438 SDValue buildSqrtEstimate(SDValue Op, SDNodeFlags Flags);
439 SDValue buildSqrtEstimateImpl(SDValue Op, SDNodeFlags Flags, bool Recip);
440 SDValue buildSqrtNROneConst(SDValue Op, SDValue Est, unsigned Iterations,
441 SDNodeFlags Flags, bool Reciprocal);
442 SDValue buildSqrtNRTwoConst(SDValue Op, SDValue Est, unsigned Iterations,
443 SDNodeFlags Flags, bool Reciprocal);
444 SDValue MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
445 bool DemandHighBits = true);
446 SDValue MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1);
447 SDNode *MatchRotatePosNeg(SDValue Shifted, SDValue Pos, SDValue Neg,
448 SDValue InnerPos, SDValue InnerNeg,
449 unsigned PosOpcode, unsigned NegOpcode,
450 const SDLoc &DL);
451 SDNode *MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL);
452 SDValue MatchLoadCombine(SDNode *N);
453 SDValue ReduceLoadWidth(SDNode *N);
454 SDValue ReduceLoadOpStoreWidth(SDNode *N);
455 SDValue splitMergedValStore(StoreSDNode *ST);
456 SDValue TransformFPLoadStorePair(SDNode *N);
457 SDValue reduceBuildVecExtToExtBuildVec(SDNode *N);
458 SDValue reduceBuildVecConvertToConvertBuildVec(SDNode *N);
459 SDValue reduceBuildVecToShuffle(SDNode *N);
460 SDValue createBuildVecShuffle(const SDLoc &DL, SDNode *N,
461 ArrayRef<int> VectorMask, SDValue VecIn1,
462 SDValue VecIn2, unsigned LeftIdx);
463 SDValue matchVSelectOpSizesWithSetCC(SDNode *N);
464
465 /// Walk up chain skipping non-aliasing memory nodes,
466 /// looking for aliasing nodes and adding them to the Aliases vector.
467 void GatherAllAliases(SDNode *N, SDValue OriginalChain,
468 SmallVectorImpl<SDValue> &Aliases);
469
470 /// Return true if there is any possibility that the two addresses overlap.
471 bool isAlias(LSBaseSDNode *Op0, LSBaseSDNode *Op1) const;
472
473 /// Walk up chain skipping non-aliasing memory nodes, looking for a better
474 /// chain (aliasing node.)
475 SDValue FindBetterChain(SDNode *N, SDValue Chain);
476
477 /// Try to replace a store and any possibly adjacent stores on
478 /// consecutive chains with better chains. Return true only if St is
479 /// replaced.
480 ///
481 /// Notice that other chains may still be replaced even if the function
482 /// returns false.
483 bool findBetterNeighborChains(StoreSDNode *St);
484
485 /// Match "(X shl/srl V1) & V2" where V2 may not be present.
486 bool MatchRotateHalf(SDValue Op, SDValue &Shift, SDValue &Mask);
487
488 /// Holds a pointer to an LSBaseSDNode as well as information on where it
489 /// is located in a sequence of memory operations connected by a chain.
490 struct MemOpLink {
491 // Ptr to the mem node.
492 LSBaseSDNode *MemNode;
493
494 // Offset from the base ptr.
495 int64_t OffsetFromBase;
496
497 MemOpLink(LSBaseSDNode *N, int64_t Offset)
498 : MemNode(N), OffsetFromBase(Offset) {}
499 };
500
501 /// This is a helper function for visitMUL to check the profitability
502 /// of folding (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2).
503 /// MulNode is the original multiply, AddNode is (add x, c1),
504 /// and ConstNode is c2.
505 bool isMulAddWithConstProfitable(SDNode *MulNode,
506 SDValue &AddNode,
507 SDValue &ConstNode);
508
509 /// This is a helper function for visitAND and visitZERO_EXTEND. Returns
510 /// true if the (and (load x) c) pattern matches an extload. ExtVT returns
511 /// the type of the loaded value to be extended.
512 bool isAndLoadExtLoad(ConstantSDNode *AndC, LoadSDNode *LoadN,
513 EVT LoadResultTy, EVT &ExtVT);
514
515 /// Helper function to calculate whether the given Load can have its
516 /// width reduced to ExtVT.
517 bool isLegalNarrowLoad(LoadSDNode *LoadN, ISD::LoadExtType ExtType,
518 EVT &ExtVT, unsigned ShAmt = 0);
519
520 /// Used by BackwardsPropagateMask to find suitable loads.
521 bool SearchForAndLoads(SDNode *N, SmallPtrSetImpl<LoadSDNode*> &Loads,
522 SmallPtrSetImpl<SDNode*> &NodeWithConsts,
523 ConstantSDNode *Mask, SDNode *&UncombinedNode);
524 /// Attempt to propagate a given AND node back to load leaves so that they
525 /// can be combined into narrow loads.
526 bool BackwardsPropagateMask(SDNode *N, SelectionDAG &DAG);
527
528 /// Helper function for MergeConsecutiveStores which merges the
529 /// component store chains.
530 SDValue getMergeStoreChains(SmallVectorImpl<MemOpLink> &StoreNodes,
531 unsigned NumStores);
532
533 /// This is a helper function for MergeConsecutiveStores. When the
534 /// source elements of the consecutive stores are all constants or
535 /// all extracted vector elements, try to merge them into one
536 /// larger store introducing bitcasts if necessary. \return True
537 /// if a merged store was created.
538 bool MergeStoresOfConstantsOrVecElts(SmallVectorImpl<MemOpLink> &StoreNodes,
539 EVT MemVT, unsigned NumStores,
540 bool IsConstantSrc, bool UseVector,
541 bool UseTrunc);
542
543 /// This is a helper function for MergeConsecutiveStores. Stores
544 /// that potentially may be merged with St are placed in
545 /// StoreNodes.
546 void getStoreMergeCandidates(StoreSDNode *St,
547 SmallVectorImpl<MemOpLink> &StoreNodes);
548
549 /// Helper function for MergeConsecutiveStores. Checks if
550 /// candidate stores have indirect dependency through their
551 /// operands. \return True if safe to merge.
552 bool checkMergeStoreCandidatesForDependencies(
553 SmallVectorImpl<MemOpLink> &StoreNodes, unsigned NumStores);
554
555 /// Merge consecutive store operations into a wide store.
556 /// This optimization uses wide integers or vectors when possible.
557 /// \return number of stores that were merged into a merged store (the
558 /// affected nodes are stored as a prefix in \p StoreNodes).
559 bool MergeConsecutiveStores(StoreSDNode *N);
560
561 /// \brief Try to transform a truncation where C is a constant:
562 /// (trunc (and X, C)) -> (and (trunc X), (trunc C))
563 ///
564 /// \p N needs to be a truncation and its first operand an AND. Other
565 /// requirements are checked by the function (e.g. that trunc is
566 /// single-use) and if missed an empty SDValue is returned.
567 SDValue distributeTruncateThroughAnd(SDNode *N);
568
569 public:
570 /// Runs the dag combiner on all nodes in the work list
571 void Run(CombineLevel AtLevel);
572
573 SelectionDAG &getDAG() const { return DAG; }
574
575 /// Returns a type large enough to hold any valid shift amount - before type
576 /// legalization these can be huge.
577 EVT getShiftAmountTy(EVT LHSTy) {
578 assert(LHSTy.isInteger() && "Shift amount is not an integer type!")(static_cast <bool> (LHSTy.isInteger() && "Shift amount is not an integer type!"
) ? void (0) : __assert_fail ("LHSTy.isInteger() && \"Shift amount is not an integer type!\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 578, __extension__ __PRETTY_FUNCTION__))
;
579 return TLI.getShiftAmountTy(LHSTy, DAG.getDataLayout(), LegalTypes);
580 }
581
582 /// This method returns true if we are running before type legalization or
583 /// if the specified VT is legal.
584 bool isTypeLegal(const EVT &VT) {
585 if (!LegalTypes) return true;
586 return TLI.isTypeLegal(VT);
587 }
588
589 /// Convenience wrapper around TargetLowering::getSetCCResultType
590 EVT getSetCCResultType(EVT VT) const {
591 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
592 }
593 };
594
595/// This class is a DAGUpdateListener that removes any deleted
596/// nodes from the worklist.
597class WorklistRemover : public SelectionDAG::DAGUpdateListener {
598 DAGCombiner &DC;
599
600public:
601 explicit WorklistRemover(DAGCombiner &dc)
602 : SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
603
604 void NodeDeleted(SDNode *N, SDNode *E) override {
605 DC.removeFromWorklist(N);
606 }
607};
608
609} // end anonymous namespace
610
611//===----------------------------------------------------------------------===//
612// TargetLowering::DAGCombinerInfo implementation
613//===----------------------------------------------------------------------===//
614
615void TargetLowering::DAGCombinerInfo::AddToWorklist(SDNode *N) {
616 ((DAGCombiner*)DC)->AddToWorklist(N);
617}
618
619SDValue TargetLowering::DAGCombinerInfo::
620CombineTo(SDNode *N, ArrayRef<SDValue> To, bool AddTo) {
621 return ((DAGCombiner*)DC)->CombineTo(N, &To[0], To.size(), AddTo);
622}
623
624SDValue TargetLowering::DAGCombinerInfo::
625CombineTo(SDNode *N, SDValue Res, bool AddTo) {
626 return ((DAGCombiner*)DC)->CombineTo(N, Res, AddTo);
627}
628
629SDValue TargetLowering::DAGCombinerInfo::
630CombineTo(SDNode *N, SDValue Res0, SDValue Res1, bool AddTo) {
631 return ((DAGCombiner*)DC)->CombineTo(N, Res0, Res1, AddTo);
632}
633
634void TargetLowering::DAGCombinerInfo::
635CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
636 return ((DAGCombiner*)DC)->CommitTargetLoweringOpt(TLO);
637}
638
639//===----------------------------------------------------------------------===//
640// Helper Functions
641//===----------------------------------------------------------------------===//
642
643void DAGCombiner::deleteAndRecombine(SDNode *N) {
644 removeFromWorklist(N);
645
646 // If the operands of this node are only used by the node, they will now be
647 // dead. Make sure to re-visit them and recursively delete dead nodes.
648 for (const SDValue &Op : N->ops())
649 // For an operand generating multiple values, one of the values may
650 // become dead allowing further simplification (e.g. split index
651 // arithmetic from an indexed load).
652 if (Op->hasOneUse() || Op->getNumValues() > 1)
653 AddToWorklist(Op.getNode());
654
655 DAG.DeleteNode(N);
656}
657
658/// Return 1 if we can compute the negated form of the specified expression for
659/// the same cost as the expression itself, or 2 if we can compute the negated
660/// form more cheaply than the expression itself.
661static char isNegatibleForFree(SDValue Op, bool LegalOperations,
662 const TargetLowering &TLI,
663 const TargetOptions *Options,
664 unsigned Depth = 0) {
665 // fneg is removable even if it has multiple uses.
666 if (Op.getOpcode() == ISD::FNEG) return 2;
667
668 // Don't allow anything with multiple uses.
669 if (!Op.hasOneUse()) return 0;
670
671 // Don't recurse exponentially.
672 if (Depth > 6) return 0;
673
674 switch (Op.getOpcode()) {
675 default: return false;
676 case ISD::ConstantFP: {
677 if (!LegalOperations)
678 return 1;
679
680 // Don't invert constant FP values after legalization unless the target says
681 // the negated constant is legal.
682 EVT VT = Op.getValueType();
683 return TLI.isOperationLegal(ISD::ConstantFP, VT) ||
684 TLI.isFPImmLegal(neg(cast<ConstantFPSDNode>(Op)->getValueAPF()), VT);
685 }
686 case ISD::FADD:
687 // FIXME: determine better conditions for this xform.
688 if (!Options->UnsafeFPMath) return 0;
689
690 // After operation legalization, it might not be legal to create new FSUBs.
691 if (LegalOperations &&
692 !TLI.isOperationLegalOrCustom(ISD::FSUB, Op.getValueType()))
693 return 0;
694
695 // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
696 if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI,
697 Options, Depth + 1))
698 return V;
699 // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
700 return isNegatibleForFree(Op.getOperand(1), LegalOperations, TLI, Options,
701 Depth + 1);
702 case ISD::FSUB:
703 // We can't turn -(A-B) into B-A when we honor signed zeros.
704 if (!Options->NoSignedZerosFPMath &&
705 !Op.getNode()->getFlags().hasNoSignedZeros())
706 return 0;
707
708 // fold (fneg (fsub A, B)) -> (fsub B, A)
709 return 1;
710
711 case ISD::FMUL:
712 case ISD::FDIV:
713 if (Options->HonorSignDependentRoundingFPMath()) return 0;
714
715 // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y) or (fmul X, (fneg Y))
716 if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI,
717 Options, Depth + 1))
718 return V;
719
720 return isNegatibleForFree(Op.getOperand(1), LegalOperations, TLI, Options,
721 Depth + 1);
722
723 case ISD::FP_EXTEND:
724 case ISD::FP_ROUND:
725 case ISD::FSIN:
726 return isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI, Options,
727 Depth + 1);
728 }
729}
730
731/// If isNegatibleForFree returns true, return the newly negated expression.
732static SDValue GetNegatedExpression(SDValue Op, SelectionDAG &DAG,
733 bool LegalOperations, unsigned Depth = 0) {
734 const TargetOptions &Options = DAG.getTarget().Options;
735 // fneg is removable even if it has multiple uses.
736 if (Op.getOpcode() == ISD::FNEG) return Op.getOperand(0);
737
738 // Don't allow anything with multiple uses.
739 assert(Op.hasOneUse() && "Unknown reuse!")(static_cast <bool> (Op.hasOneUse() && "Unknown reuse!"
) ? void (0) : __assert_fail ("Op.hasOneUse() && \"Unknown reuse!\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 739, __extension__ __PRETTY_FUNCTION__))
;
740
741 assert(Depth <= 6 && "GetNegatedExpression doesn't match isNegatibleForFree")(static_cast <bool> (Depth <= 6 && "GetNegatedExpression doesn't match isNegatibleForFree"
) ? void (0) : __assert_fail ("Depth <= 6 && \"GetNegatedExpression doesn't match isNegatibleForFree\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 741, __extension__ __PRETTY_FUNCTION__))
;
742
743 const SDNodeFlags Flags = Op.getNode()->getFlags();
744
745 switch (Op.getOpcode()) {
746 default: llvm_unreachable("Unknown code")::llvm::llvm_unreachable_internal("Unknown code", "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 746)
;
747 case ISD::ConstantFP: {
748 APFloat V = cast<ConstantFPSDNode>(Op)->getValueAPF();
749 V.changeSign();
750 return DAG.getConstantFP(V, SDLoc(Op), Op.getValueType());
751 }
752 case ISD::FADD:
753 // FIXME: determine better conditions for this xform.
754 assert(Options.UnsafeFPMath)(static_cast <bool> (Options.UnsafeFPMath) ? void (0) :
__assert_fail ("Options.UnsafeFPMath", "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 754, __extension__ __PRETTY_FUNCTION__))
;
755
756 // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
757 if (isNegatibleForFree(Op.getOperand(0), LegalOperations,
758 DAG.getTargetLoweringInfo(), &Options, Depth+1))
759 return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
760 GetNegatedExpression(Op.getOperand(0), DAG,
761 LegalOperations, Depth+1),
762 Op.getOperand(1), Flags);
763 // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
764 return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
765 GetNegatedExpression(Op.getOperand(1), DAG,
766 LegalOperations, Depth+1),
767 Op.getOperand(0), Flags);
768 case ISD::FSUB:
769 // fold (fneg (fsub 0, B)) -> B
770 if (ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(Op.getOperand(0)))
771 if (N0CFP->isZero())
772 return Op.getOperand(1);
773
774 // fold (fneg (fsub A, B)) -> (fsub B, A)
775 return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
776 Op.getOperand(1), Op.getOperand(0), Flags);
777
778 case ISD::FMUL:
779 case ISD::FDIV:
780 assert(!Options.HonorSignDependentRoundingFPMath())(static_cast <bool> (!Options.HonorSignDependentRoundingFPMath
()) ? void (0) : __assert_fail ("!Options.HonorSignDependentRoundingFPMath()"
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 780, __extension__ __PRETTY_FUNCTION__))
;
781
782 // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y)
783 if (isNegatibleForFree(Op.getOperand(0), LegalOperations,
784 DAG.getTargetLoweringInfo(), &Options, Depth+1))
785 return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
786 GetNegatedExpression(Op.getOperand(0), DAG,
787 LegalOperations, Depth+1),
788 Op.getOperand(1), Flags);
789
790 // fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y))
791 return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
792 Op.getOperand(0),
793 GetNegatedExpression(Op.getOperand(1), DAG,
794 LegalOperations, Depth+1), Flags);
795
796 case ISD::FP_EXTEND:
797 case ISD::FSIN:
798 return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
799 GetNegatedExpression(Op.getOperand(0), DAG,
800 LegalOperations, Depth+1));
801 case ISD::FP_ROUND:
802 return DAG.getNode(ISD::FP_ROUND, SDLoc(Op), Op.getValueType(),
803 GetNegatedExpression(Op.getOperand(0), DAG,
804 LegalOperations, Depth+1),
805 Op.getOperand(1));
806 }
807}
808
809// APInts must be the same size for most operations, this helper
810// function zero extends the shorter of the pair so that they match.
811// We provide an Offset so that we can create bitwidths that won't overflow.
812static void zeroExtendToMatch(APInt &LHS, APInt &RHS, unsigned Offset = 0) {
813 unsigned Bits = Offset + std::max(LHS.getBitWidth(), RHS.getBitWidth());
814 LHS = LHS.zextOrSelf(Bits);
815 RHS = RHS.zextOrSelf(Bits);
816}
817
818// Return true if this node is a setcc, or is a select_cc
819// that selects between the target values used for true and false, making it
820// equivalent to a setcc. Also, set the incoming LHS, RHS, and CC references to
821// the appropriate nodes based on the type of node we are checking. This
822// simplifies life a bit for the callers.
823bool DAGCombiner::isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
824 SDValue &CC) const {
825 if (N.getOpcode() == ISD::SETCC) {
826 LHS = N.getOperand(0);
827 RHS = N.getOperand(1);
828 CC = N.getOperand(2);
829 return true;
830 }
831
832 if (N.getOpcode() != ISD::SELECT_CC ||
833 !TLI.isConstTrueVal(N.getOperand(2).getNode()) ||
834 !TLI.isConstFalseVal(N.getOperand(3).getNode()))
835 return false;
836
837 if (TLI.getBooleanContents(N.getValueType()) ==
838 TargetLowering::UndefinedBooleanContent)
839 return false;
840
841 LHS = N.getOperand(0);
842 RHS = N.getOperand(1);
843 CC = N.getOperand(4);
844 return true;
845}
846
847/// Return true if this is a SetCC-equivalent operation with only one use.
848/// If this is true, it allows the users to invert the operation for free when
849/// it is profitable to do so.
850bool DAGCombiner::isOneUseSetCC(SDValue N) const {
851 SDValue N0, N1, N2;
852 if (isSetCCEquivalent(N, N0, N1, N2) && N.getNode()->hasOneUse())
853 return true;
854 return false;
855}
856
857// \brief Returns the SDNode if it is a constant float BuildVector
858// or constant float.
859static SDNode *isConstantFPBuildVectorOrConstantFP(SDValue N) {
860 if (isa<ConstantFPSDNode>(N))
861 return N.getNode();
862 if (ISD::isBuildVectorOfConstantFPSDNodes(N.getNode()))
863 return N.getNode();
864 return nullptr;
865}
866
867// Determines if it is a constant integer or a build vector of constant
868// integers (and undefs).
869// Do not permit build vector implicit truncation.
870static bool isConstantOrConstantVector(SDValue N, bool NoOpaques = false) {
871 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N))
872 return !(Const->isOpaque() && NoOpaques);
873 if (N.getOpcode() != ISD::BUILD_VECTOR)
874 return false;
875 unsigned BitWidth = N.getScalarValueSizeInBits();
876 for (const SDValue &Op : N->op_values()) {
877 if (Op.isUndef())
878 continue;
879 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Op);
880 if (!Const || Const->getAPIntValue().getBitWidth() != BitWidth ||
881 (Const->isOpaque() && NoOpaques))
882 return false;
883 }
884 return true;
885}
886
887// Determines if it is a constant null integer or a splatted vector of a
888// constant null integer (with no undefs).
889// Build vector implicit truncation is not an issue for null values.
890static bool isNullConstantOrNullSplatConstant(SDValue N) {
891 if (ConstantSDNode *Splat = isConstOrConstSplat(N))
892 return Splat->isNullValue();
893 return false;
894}
895
896// Determines if it is a constant integer of one or a splatted vector of a
897// constant integer of one (with no undefs).
898// Do not permit build vector implicit truncation.
899static bool isOneConstantOrOneSplatConstant(SDValue N) {
900 unsigned BitWidth = N.getScalarValueSizeInBits();
901 if (ConstantSDNode *Splat = isConstOrConstSplat(N))
902 return Splat->isOne() && Splat->getAPIntValue().getBitWidth() == BitWidth;
903 return false;
904}
905
906// Determines if it is a constant integer of all ones or a splatted vector of a
907// constant integer of all ones (with no undefs).
908// Do not permit build vector implicit truncation.
909static bool isAllOnesConstantOrAllOnesSplatConstant(SDValue N) {
910 unsigned BitWidth = N.getScalarValueSizeInBits();
911 if (ConstantSDNode *Splat = isConstOrConstSplat(N))
912 return Splat->isAllOnesValue() &&
913 Splat->getAPIntValue().getBitWidth() == BitWidth;
914 return false;
915}
916
917// Determines if a BUILD_VECTOR is composed of all-constants possibly mixed with
918// undef's.
919static bool isAnyConstantBuildVector(const SDNode *N) {
920 return ISD::isBuildVectorOfConstantSDNodes(N) ||
921 ISD::isBuildVectorOfConstantFPSDNodes(N);
922}
923
924SDValue DAGCombiner::ReassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0,
925 SDValue N1) {
926 EVT VT = N0.getValueType();
927 if (N0.getOpcode() == Opc) {
928 if (SDNode *L = DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1))) {
929 if (SDNode *R = DAG.isConstantIntBuildVectorOrConstantInt(N1)) {
930 // reassoc. (op (op x, c1), c2) -> (op x, (op c1, c2))
931 if (SDValue OpNode = DAG.FoldConstantArithmetic(Opc, DL, VT, L, R))
932 return DAG.getNode(Opc, DL, VT, N0.getOperand(0), OpNode);
933 return SDValue();
934 }
935 if (N0.hasOneUse()) {
936 // reassoc. (op (op x, c1), y) -> (op (op x, y), c1) iff x+c1 has one
937 // use
938 SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N0.getOperand(0), N1);
939 if (!OpNode.getNode())
940 return SDValue();
941 AddToWorklist(OpNode.getNode());
942 return DAG.getNode(Opc, DL, VT, OpNode, N0.getOperand(1));
943 }
944 }
945 }
946
947 if (N1.getOpcode() == Opc) {
948 if (SDNode *R = DAG.isConstantIntBuildVectorOrConstantInt(N1.getOperand(1))) {
949 if (SDNode *L = DAG.isConstantIntBuildVectorOrConstantInt(N0)) {
950 // reassoc. (op c2, (op x, c1)) -> (op x, (op c1, c2))
951 if (SDValue OpNode = DAG.FoldConstantArithmetic(Opc, DL, VT, R, L))
952 return DAG.getNode(Opc, DL, VT, N1.getOperand(0), OpNode);
953 return SDValue();
954 }
955 if (N1.hasOneUse()) {
956 // reassoc. (op x, (op y, c1)) -> (op (op x, y), c1) iff x+c1 has one
957 // use
958 SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N0, N1.getOperand(0));
959 if (!OpNode.getNode())
960 return SDValue();
961 AddToWorklist(OpNode.getNode());
962 return DAG.getNode(Opc, DL, VT, OpNode, N1.getOperand(1));
963 }
964 }
965 }
966
967 return SDValue();
968}
969
970SDValue DAGCombiner::CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
971 bool AddTo) {
972 assert(N->getNumValues() == NumTo && "Broken CombineTo call!")(static_cast <bool> (N->getNumValues() == NumTo &&
"Broken CombineTo call!") ? void (0) : __assert_fail ("N->getNumValues() == NumTo && \"Broken CombineTo call!\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 972, __extension__ __PRETTY_FUNCTION__))
;
973 ++NodesCombined;
974 DEBUG(dbgs() << "\nReplacing.1 ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.1 "; N->dump
(&DAG); dbgs() << "\nWith: "; To[0].getNode()->dump
(&DAG); dbgs() << " and " << NumTo-1 <<
" other values\n"; } } while (false)
975 N->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.1 "; N->dump
(&DAG); dbgs() << "\nWith: "; To[0].getNode()->dump
(&DAG); dbgs() << " and " << NumTo-1 <<
" other values\n"; } } while (false)
976 dbgs() << "\nWith: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.1 "; N->dump
(&DAG); dbgs() << "\nWith: "; To[0].getNode()->dump
(&DAG); dbgs() << " and " << NumTo-1 <<
" other values\n"; } } while (false)
977 To[0].getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.1 "; N->dump
(&DAG); dbgs() << "\nWith: "; To[0].getNode()->dump
(&DAG); dbgs() << " and " << NumTo-1 <<
" other values\n"; } } while (false)
978 dbgs() << " and " << NumTo-1 << " other values\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.1 "; N->dump
(&DAG); dbgs() << "\nWith: "; To[0].getNode()->dump
(&DAG); dbgs() << " and " << NumTo-1 <<
" other values\n"; } } while (false)
;
979 for (unsigned i = 0, e = NumTo; i != e; ++i)
980 assert((!To[i].getNode() ||(static_cast <bool> ((!To[i].getNode() || N->getValueType
(i) == To[i].getValueType()) && "Cannot combine value to value of different type!"
) ? void (0) : __assert_fail ("(!To[i].getNode() || N->getValueType(i) == To[i].getValueType()) && \"Cannot combine value to value of different type!\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 982, __extension__ __PRETTY_FUNCTION__))
981 N->getValueType(i) == To[i].getValueType()) &&(static_cast <bool> ((!To[i].getNode() || N->getValueType
(i) == To[i].getValueType()) && "Cannot combine value to value of different type!"
) ? void (0) : __assert_fail ("(!To[i].getNode() || N->getValueType(i) == To[i].getValueType()) && \"Cannot combine value to value of different type!\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 982, __extension__ __PRETTY_FUNCTION__))
982 "Cannot combine value to value of different type!")(static_cast <bool> ((!To[i].getNode() || N->getValueType
(i) == To[i].getValueType()) && "Cannot combine value to value of different type!"
) ? void (0) : __assert_fail ("(!To[i].getNode() || N->getValueType(i) == To[i].getValueType()) && \"Cannot combine value to value of different type!\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 982, __extension__ __PRETTY_FUNCTION__))
;
983
984 WorklistRemover DeadNodes(*this);
985 DAG.ReplaceAllUsesWith(N, To);
986 if (AddTo) {
987 // Push the new nodes and any users onto the worklist
988 for (unsigned i = 0, e = NumTo; i != e; ++i) {
989 if (To[i].getNode()) {
990 AddToWorklist(To[i].getNode());
991 AddUsersToWorklist(To[i].getNode());
992 }
993 }
994 }
995
996 // Finally, if the node is now dead, remove it from the graph. The node
997 // may not be dead if the replacement process recursively simplified to
998 // something else needing this node.
999 if (N->use_empty())
1000 deleteAndRecombine(N);
1001 return SDValue(N, 0);
1002}
1003
1004void DAGCombiner::
1005CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
1006 // Replace all uses. If any nodes become isomorphic to other nodes and
1007 // are deleted, make sure to remove them from our worklist.
1008 WorklistRemover DeadNodes(*this);
1009 DAG.ReplaceAllUsesOfValueWith(TLO.Old, TLO.New);
1010
1011 // Push the new node and any (possibly new) users onto the worklist.
1012 AddToWorklist(TLO.New.getNode());
1013 AddUsersToWorklist(TLO.New.getNode());
1014
1015 // Finally, if the node is now dead, remove it from the graph. The node
1016 // may not be dead if the replacement process recursively simplified to
1017 // something else needing this node.
1018 if (TLO.Old.getNode()->use_empty())
1019 deleteAndRecombine(TLO.Old.getNode());
1020}
1021
1022/// Check the specified integer node value to see if it can be simplified or if
1023/// things it uses can be simplified by bit propagation. If so, return true.
1024bool DAGCombiner::SimplifyDemandedBits(SDValue Op, const APInt &Demanded) {
1025 TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations);
1026 KnownBits Known;
1027 if (!TLI.SimplifyDemandedBits(Op, Demanded, Known, TLO))
1028 return false;
1029
1030 // Revisit the node.
1031 AddToWorklist(Op.getNode());
1032
1033 // Replace the old value with the new one.
1034 ++NodesCombined;
1035 DEBUG(dbgs() << "\nReplacing.2 ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.getNode
()->dump(&DAG); dbgs() << "\nWith: "; TLO.New.getNode
()->dump(&DAG); dbgs() << '\n'; } } while (false
)
1036 TLO.Old.getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.getNode
()->dump(&DAG); dbgs() << "\nWith: "; TLO.New.getNode
()->dump(&DAG); dbgs() << '\n'; } } while (false
)
1037 dbgs() << "\nWith: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.getNode
()->dump(&DAG); dbgs() << "\nWith: "; TLO.New.getNode
()->dump(&DAG); dbgs() << '\n'; } } while (false
)
1038 TLO.New.getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.getNode
()->dump(&DAG); dbgs() << "\nWith: "; TLO.New.getNode
()->dump(&DAG); dbgs() << '\n'; } } while (false
)
1039 dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.getNode
()->dump(&DAG); dbgs() << "\nWith: "; TLO.New.getNode
()->dump(&DAG); dbgs() << '\n'; } } while (false
)
;
1040
1041 CommitTargetLoweringOpt(TLO);
1042 return true;
1043}
1044
1045/// Check the specified vector node value to see if it can be simplified or
1046/// if things it uses can be simplified as it only uses some of the elements.
1047/// If so, return true.
1048bool DAGCombiner::SimplifyDemandedVectorElts(SDValue Op,
1049 const APInt &Demanded) {
1050 TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations);
1051 APInt KnownUndef, KnownZero;
1052 if (!TLI.SimplifyDemandedVectorElts(Op, Demanded, KnownUndef, KnownZero, TLO))
1053 return false;
1054
1055 // Revisit the node.
1056 AddToWorklist(Op.getNode());
1057
1058 // Replace the old value with the new one.
1059 ++NodesCombined;
1060 DEBUG(dbgs() << "\nReplacing.2 "; TLO.Old.getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.getNode
()->dump(&DAG); dbgs() << "\nWith: "; TLO.New.getNode
()->dump(&DAG); dbgs() << '\n'; } } while (false
)
1061 dbgs() << "\nWith: "; TLO.New.getNode()->dump(&DAG); dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.getNode
()->dump(&DAG); dbgs() << "\nWith: "; TLO.New.getNode
()->dump(&DAG); dbgs() << '\n'; } } while (false
)
;
1062
1063 CommitTargetLoweringOpt(TLO);
1064 return true;
1065}
1066
1067void DAGCombiner::ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad) {
1068 SDLoc DL(Load);
1069 EVT VT = Load->getValueType(0);
1070 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, VT, SDValue(ExtLoad, 0));
1071
1072 DEBUG(dbgs() << "\nReplacing.9 ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.9 "; Load->
dump(&DAG); dbgs() << "\nWith: "; Trunc.getNode()->
dump(&DAG); dbgs() << '\n'; } } while (false)
1073 Load->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.9 "; Load->
dump(&DAG); dbgs() << "\nWith: "; Trunc.getNode()->
dump(&DAG); dbgs() << '\n'; } } while (false)
1074 dbgs() << "\nWith: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.9 "; Load->
dump(&DAG); dbgs() << "\nWith: "; Trunc.getNode()->
dump(&DAG); dbgs() << '\n'; } } while (false)
1075 Trunc.getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.9 "; Load->
dump(&DAG); dbgs() << "\nWith: "; Trunc.getNode()->
dump(&DAG); dbgs() << '\n'; } } while (false)
1076 dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.9 "; Load->
dump(&DAG); dbgs() << "\nWith: "; Trunc.getNode()->
dump(&DAG); dbgs() << '\n'; } } while (false)
;
1077 WorklistRemover DeadNodes(*this);
1078 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), Trunc);
1079 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), SDValue(ExtLoad, 1));
1080 deleteAndRecombine(Load);
1081 AddToWorklist(Trunc.getNode());
1082}
1083
1084SDValue DAGCombiner::PromoteOperand(SDValue Op, EVT PVT, bool &Replace) {
1085 Replace = false;
1086 SDLoc DL(Op);
1087 if (ISD::isUNINDEXEDLoad(Op.getNode())) {
1088 LoadSDNode *LD = cast<LoadSDNode>(Op);
1089 EVT MemVT = LD->getMemoryVT();
1090 ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD)
1091 ? (TLI.isLoadExtLegal(ISD::ZEXTLOAD, PVT, MemVT) ? ISD::ZEXTLOAD
1092 : ISD::EXTLOAD)
1093 : LD->getExtensionType();
1094 Replace = true;
1095 return DAG.getExtLoad(ExtType, DL, PVT,
1096 LD->getChain(), LD->getBasePtr(),
1097 MemVT, LD->getMemOperand());
1098 }
1099
1100 unsigned Opc = Op.getOpcode();
1101 switch (Opc) {
1102 default: break;
1103 case ISD::AssertSext:
1104 if (SDValue Op0 = SExtPromoteOperand(Op.getOperand(0), PVT))
1105 return DAG.getNode(ISD::AssertSext, DL, PVT, Op0, Op.getOperand(1));
1106 break;
1107 case ISD::AssertZext:
1108 if (SDValue Op0 = ZExtPromoteOperand(Op.getOperand(0), PVT))
1109 return DAG.getNode(ISD::AssertZext, DL, PVT, Op0, Op.getOperand(1));
1110 break;
1111 case ISD::Constant: {
1112 unsigned ExtOpc =
1113 Op.getValueType().isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
1114 return DAG.getNode(ExtOpc, DL, PVT, Op);
1115 }
1116 }
1117
1118 if (!TLI.isOperationLegal(ISD::ANY_EXTEND, PVT))
1119 return SDValue();
1120 return DAG.getNode(ISD::ANY_EXTEND, DL, PVT, Op);
1121}
1122
1123SDValue DAGCombiner::SExtPromoteOperand(SDValue Op, EVT PVT) {
1124 if (!TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, PVT))
1125 return SDValue();
1126 EVT OldVT = Op.getValueType();
1127 SDLoc DL(Op);
1128 bool Replace = false;
1129 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
1130 if (!NewOp.getNode())
1131 return SDValue();
1132 AddToWorklist(NewOp.getNode());
1133
1134 if (Replace)
1135 ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
1136 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, NewOp.getValueType(), NewOp,
1137 DAG.getValueType(OldVT));
1138}
1139
1140SDValue DAGCombiner::ZExtPromoteOperand(SDValue Op, EVT PVT) {
1141 EVT OldVT = Op.getValueType();
1142 SDLoc DL(Op);
1143 bool Replace = false;
1144 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
1145 if (!NewOp.getNode())
1146 return SDValue();
1147 AddToWorklist(NewOp.getNode());
1148
1149 if (Replace)
1150 ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
1151 return DAG.getZeroExtendInReg(NewOp, DL, OldVT);
1152}
1153
1154/// Promote the specified integer binary operation if the target indicates it is
1155/// beneficial. e.g. On x86, it's usually better to promote i16 operations to
1156/// i32 since i16 instructions are longer.
1157SDValue DAGCombiner::PromoteIntBinOp(SDValue Op) {
1158 if (!LegalOperations)
1159 return SDValue();
1160
1161 EVT VT = Op.getValueType();
1162 if (VT.isVector() || !VT.isInteger())
1163 return SDValue();
1164
1165 // If operation type is 'undesirable', e.g. i16 on x86, consider
1166 // promoting it.
1167 unsigned Opc = Op.getOpcode();
1168 if (TLI.isTypeDesirableForOp(Opc, VT))
1169 return SDValue();
1170
1171 EVT PVT = VT;
1172 // Consult target whether it is a good idea to promote this operation and
1173 // what's the right type to promote it to.
1174 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1175 assert(PVT != VT && "Don't know what type to promote to!")(static_cast <bool> (PVT != VT && "Don't know what type to promote to!"
) ? void (0) : __assert_fail ("PVT != VT && \"Don't know what type to promote to!\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1175, __extension__ __PRETTY_FUNCTION__))
;
1176
1177 DEBUG(dbgs() << "\nPromoting "; Op.getNode()->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; Op.getNode(
)->dump(&DAG); } } while (false)
;
1178
1179 bool Replace0 = false;
1180 SDValue N0 = Op.getOperand(0);
1181 SDValue NN0 = PromoteOperand(N0, PVT, Replace0);
1182
1183 bool Replace1 = false;
1184 SDValue N1 = Op.getOperand(1);
1185 SDValue NN1 = PromoteOperand(N1, PVT, Replace1);
1186 SDLoc DL(Op);
1187
1188 SDValue RV =
1189 DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, NN0, NN1));
1190
1191 // We are always replacing N0/N1's use in N and only need
1192 // additional replacements if there are additional uses.
1193 Replace0 &= !N0->hasOneUse();
1194 Replace1 &= (N0 != N1) && !N1->hasOneUse();
1195
1196 // Combine Op here so it is preserved past replacements.
1197 CombineTo(Op.getNode(), RV);
1198
1199 // If operands have a use ordering, make sure we deal with
1200 // predecessor first.
1201 if (Replace0 && Replace1 && N0.getNode()->isPredecessorOf(N1.getNode())) {
1202 std::swap(N0, N1);
1203 std::swap(NN0, NN1);
1204 }
1205
1206 if (Replace0) {
1207 AddToWorklist(NN0.getNode());
1208 ReplaceLoadWithPromotedLoad(N0.getNode(), NN0.getNode());
1209 }
1210 if (Replace1) {
1211 AddToWorklist(NN1.getNode());
1212 ReplaceLoadWithPromotedLoad(N1.getNode(), NN1.getNode());
1213 }
1214 return Op;
1215 }
1216 return SDValue();
1217}
1218
1219/// Promote the specified integer shift operation if the target indicates it is
1220/// beneficial. e.g. On x86, it's usually better to promote i16 operations to
1221/// i32 since i16 instructions are longer.
1222SDValue DAGCombiner::PromoteIntShiftOp(SDValue Op) {
1223 if (!LegalOperations)
1224 return SDValue();
1225
1226 EVT VT = Op.getValueType();
1227 if (VT.isVector() || !VT.isInteger())
1228 return SDValue();
1229
1230 // If operation type is 'undesirable', e.g. i16 on x86, consider
1231 // promoting it.
1232 unsigned Opc = Op.getOpcode();
1233 if (TLI.isTypeDesirableForOp(Opc, VT))
1234 return SDValue();
1235
1236 EVT PVT = VT;
1237 // Consult target whether it is a good idea to promote this operation and
1238 // what's the right type to promote it to.
1239 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1240 assert(PVT != VT && "Don't know what type to promote to!")(static_cast <bool> (PVT != VT && "Don't know what type to promote to!"
) ? void (0) : __assert_fail ("PVT != VT && \"Don't know what type to promote to!\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1240, __extension__ __PRETTY_FUNCTION__))
;
1241
1242 DEBUG(dbgs() << "\nPromoting "; Op.getNode()->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; Op.getNode(
)->dump(&DAG); } } while (false)
;
1243
1244 bool Replace = false;
1245 SDValue N0 = Op.getOperand(0);
1246 SDValue N1 = Op.getOperand(1);
1247 if (Opc == ISD::SRA)
1248 N0 = SExtPromoteOperand(N0, PVT);
1249 else if (Opc == ISD::SRL)
1250 N0 = ZExtPromoteOperand(N0, PVT);
1251 else
1252 N0 = PromoteOperand(N0, PVT, Replace);
1253
1254 if (!N0.getNode())
1255 return SDValue();
1256
1257 SDLoc DL(Op);
1258 SDValue RV =
1259 DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, N0, N1));
1260
1261 AddToWorklist(N0.getNode());
1262 if (Replace)
1263 ReplaceLoadWithPromotedLoad(Op.getOperand(0).getNode(), N0.getNode());
1264
1265 // Deal with Op being deleted.
1266 if (Op && Op.getOpcode() != ISD::DELETED_NODE)
1267 return RV;
1268 }
1269 return SDValue();
1270}
1271
1272SDValue DAGCombiner::PromoteExtend(SDValue Op) {
1273 if (!LegalOperations)
1274 return SDValue();
1275
1276 EVT VT = Op.getValueType();
1277 if (VT.isVector() || !VT.isInteger())
1278 return SDValue();
1279
1280 // If operation type is 'undesirable', e.g. i16 on x86, consider
1281 // promoting it.
1282 unsigned Opc = Op.getOpcode();
1283 if (TLI.isTypeDesirableForOp(Opc, VT))
1284 return SDValue();
1285
1286 EVT PVT = VT;
1287 // Consult target whether it is a good idea to promote this operation and
1288 // what's the right type to promote it to.
1289 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1290 assert(PVT != VT && "Don't know what type to promote to!")(static_cast <bool> (PVT != VT && "Don't know what type to promote to!"
) ? void (0) : __assert_fail ("PVT != VT && \"Don't know what type to promote to!\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1290, __extension__ __PRETTY_FUNCTION__))
;
1291 // fold (aext (aext x)) -> (aext x)
1292 // fold (aext (zext x)) -> (zext x)
1293 // fold (aext (sext x)) -> (sext x)
1294 DEBUG(dbgs() << "\nPromoting ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; Op.getNode(
)->dump(&DAG); } } while (false)
1295 Op.getNode()->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; Op.getNode(
)->dump(&DAG); } } while (false)
;
1296 return DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, Op.getOperand(0));
1297 }
1298 return SDValue();
1299}
1300
1301bool DAGCombiner::PromoteLoad(SDValue Op) {
1302 if (!LegalOperations)
1303 return false;
1304
1305 if (!ISD::isUNINDEXEDLoad(Op.getNode()))
1306 return false;
1307
1308 EVT VT = Op.getValueType();
1309 if (VT.isVector() || !VT.isInteger())
1310 return false;
1311
1312 // If operation type is 'undesirable', e.g. i16 on x86, consider
1313 // promoting it.
1314 unsigned Opc = Op.getOpcode();
1315 if (TLI.isTypeDesirableForOp(Opc, VT))
1316 return false;
1317
1318 EVT PVT = VT;
1319 // Consult target whether it is a good idea to promote this operation and
1320 // what's the right type to promote it to.
1321 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1322 assert(PVT != VT && "Don't know what type to promote to!")(static_cast <bool> (PVT != VT && "Don't know what type to promote to!"
) ? void (0) : __assert_fail ("PVT != VT && \"Don't know what type to promote to!\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1322, __extension__ __PRETTY_FUNCTION__))
;
1323
1324 SDLoc DL(Op);
1325 SDNode *N = Op.getNode();
1326 LoadSDNode *LD = cast<LoadSDNode>(N);
1327 EVT MemVT = LD->getMemoryVT();
1328 ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD)
1329 ? (TLI.isLoadExtLegal(ISD::ZEXTLOAD, PVT, MemVT) ? ISD::ZEXTLOAD
1330 : ISD::EXTLOAD)
1331 : LD->getExtensionType();
1332 SDValue NewLD = DAG.getExtLoad(ExtType, DL, PVT,
1333 LD->getChain(), LD->getBasePtr(),
1334 MemVT, LD->getMemOperand());
1335 SDValue Result = DAG.getNode(ISD::TRUNCATE, DL, VT, NewLD);
1336
1337 DEBUG(dbgs() << "\nPromoting ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; N->dump(
&DAG); dbgs() << "\nTo: "; Result.getNode()->dump
(&DAG); dbgs() << '\n'; } } while (false)
1338 N->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; N->dump(
&DAG); dbgs() << "\nTo: "; Result.getNode()->dump
(&DAG); dbgs() << '\n'; } } while (false)
1339 dbgs() << "\nTo: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; N->dump(
&DAG); dbgs() << "\nTo: "; Result.getNode()->dump
(&DAG); dbgs() << '\n'; } } while (false)
1340 Result.getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; N->dump(
&DAG); dbgs() << "\nTo: "; Result.getNode()->dump
(&DAG); dbgs() << '\n'; } } while (false)
1341 dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; N->dump(
&DAG); dbgs() << "\nTo: "; Result.getNode()->dump
(&DAG); dbgs() << '\n'; } } while (false)
;
1342 WorklistRemover DeadNodes(*this);
1343 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result);
1344 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), NewLD.getValue(1));
1345 deleteAndRecombine(N);
1346 AddToWorklist(Result.getNode());
1347 return true;
1348 }
1349 return false;
1350}
1351
1352/// \brief Recursively delete a node which has no uses and any operands for
1353/// which it is the only use.
1354///
1355/// Note that this both deletes the nodes and removes them from the worklist.
1356/// It also adds any nodes who have had a user deleted to the worklist as they
1357/// may now have only one use and subject to other combines.
1358bool DAGCombiner::recursivelyDeleteUnusedNodes(SDNode *N) {
1359 if (!N->use_empty())
1360 return false;
1361
1362 SmallSetVector<SDNode *, 16> Nodes;
1363 Nodes.insert(N);
1364 do {
1365 N = Nodes.pop_back_val();
1366 if (!N)
1367 continue;
1368
1369 if (N->use_empty()) {
1370 for (const SDValue &ChildN : N->op_values())
1371 Nodes.insert(ChildN.getNode());
1372
1373 removeFromWorklist(N);
1374 DAG.DeleteNode(N);
1375 } else {
1376 AddToWorklist(N);
1377 }
1378 } while (!Nodes.empty());
1379 return true;
1380}
1381
1382//===----------------------------------------------------------------------===//
1383// Main DAG Combiner implementation
1384//===----------------------------------------------------------------------===//
1385
1386void DAGCombiner::Run(CombineLevel AtLevel) {
1387 // set the instance variables, so that the various visit routines may use it.
1388 Level = AtLevel;
1389 LegalOperations = Level >= AfterLegalizeVectorOps;
1390 LegalTypes = Level >= AfterLegalizeTypes;
1391
1392 // Add all the dag nodes to the worklist.
1393 for (SDNode &Node : DAG.allnodes())
1394 AddToWorklist(&Node);
1395
1396 // Create a dummy node (which is not added to allnodes), that adds a reference
1397 // to the root node, preventing it from being deleted, and tracking any
1398 // changes of the root.
1399 HandleSDNode Dummy(DAG.getRoot());
1400
1401 // While the worklist isn't empty, find a node and try to combine it.
1402 while (!WorklistMap.empty()) {
1403 SDNode *N;
1404 // The Worklist holds the SDNodes in order, but it may contain null entries.
1405 do {
1406 N = Worklist.pop_back_val();
1407 } while (!N);
1408
1409 bool GoodWorklistEntry = WorklistMap.erase(N);
1410 (void)GoodWorklistEntry;
1411 assert(GoodWorklistEntry &&(static_cast <bool> (GoodWorklistEntry && "Found a worklist entry without a corresponding map entry!"
) ? void (0) : __assert_fail ("GoodWorklistEntry && \"Found a worklist entry without a corresponding map entry!\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1412, __extension__ __PRETTY_FUNCTION__))
1412 "Found a worklist entry without a corresponding map entry!")(static_cast <bool> (GoodWorklistEntry && "Found a worklist entry without a corresponding map entry!"
) ? void (0) : __assert_fail ("GoodWorklistEntry && \"Found a worklist entry without a corresponding map entry!\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1412, __extension__ __PRETTY_FUNCTION__))
;
1413
1414 // If N has no uses, it is dead. Make sure to revisit all N's operands once
1415 // N is deleted from the DAG, since they too may now be dead or may have a
1416 // reduced number of uses, allowing other xforms.
1417 if (recursivelyDeleteUnusedNodes(N))
1418 continue;
1419
1420 WorklistRemover DeadNodes(*this);
1421
1422 // If this combine is running after legalizing the DAG, re-legalize any
1423 // nodes pulled off the worklist.
1424 if (Level == AfterLegalizeDAG) {
1425 SmallSetVector<SDNode *, 16> UpdatedNodes;
1426 bool NIsValid = DAG.LegalizeOp(N, UpdatedNodes);
1427
1428 for (SDNode *LN : UpdatedNodes) {
1429 AddToWorklist(LN);
1430 AddUsersToWorklist(LN);
1431 }
1432 if (!NIsValid)
1433 continue;
1434 }
1435
1436 DEBUG(dbgs() << "\nCombining: "; N->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nCombining: "; N->dump
(&DAG); } } while (false)
;
1437
1438 // Add any operands of the new node which have not yet been combined to the
1439 // worklist as well. Because the worklist uniques things already, this
1440 // won't repeatedly process the same operand.
1441 CombinedNodes.insert(N);
1442 for (const SDValue &ChildN : N->op_values())
1443 if (!CombinedNodes.count(ChildN.getNode()))
1444 AddToWorklist(ChildN.getNode());
1445
1446 SDValue RV = combine(N);
1447
1448 if (!RV.getNode())
1449 continue;
1450
1451 ++NodesCombined;
1452
1453 // If we get back the same node we passed in, rather than a new node or
1454 // zero, we know that the node must have defined multiple values and
1455 // CombineTo was used. Since CombineTo takes care of the worklist
1456 // mechanics for us, we have no work to do in this case.
1457 if (RV.getNode() == N)
1458 continue;
1459
1460 assert(N->getOpcode() != ISD::DELETED_NODE &&(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& RV.getOpcode() != ISD::DELETED_NODE && "Node was deleted but visit returned new node!"
) ? void (0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && RV.getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned new node!\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1462, __extension__ __PRETTY_FUNCTION__))
1461 RV.getOpcode() != ISD::DELETED_NODE &&(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& RV.getOpcode() != ISD::DELETED_NODE && "Node was deleted but visit returned new node!"
) ? void (0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && RV.getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned new node!\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1462, __extension__ __PRETTY_FUNCTION__))
1462 "Node was deleted but visit returned new node!")(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& RV.getOpcode() != ISD::DELETED_NODE && "Node was deleted but visit returned new node!"
) ? void (0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && RV.getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned new node!\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1462, __extension__ __PRETTY_FUNCTION__))
;
1463
1464 DEBUG(dbgs() << " ... into: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << " ... into: "; RV.getNode()
->dump(&DAG); } } while (false)
1465 RV.getNode()->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << " ... into: "; RV.getNode()
->dump(&DAG); } } while (false)
;
1466
1467 if (N->getNumValues() == RV.getNode()->getNumValues())
1468 DAG.ReplaceAllUsesWith(N, RV.getNode());
1469 else {
1470 assert(N->getValueType(0) == RV.getValueType() &&(static_cast <bool> (N->getValueType(0) == RV.getValueType
() && N->getNumValues() == 1 && "Type mismatch"
) ? void (0) : __assert_fail ("N->getValueType(0) == RV.getValueType() && N->getNumValues() == 1 && \"Type mismatch\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1471, __extension__ __PRETTY_FUNCTION__))
1471 N->getNumValues() == 1 && "Type mismatch")(static_cast <bool> (N->getValueType(0) == RV.getValueType
() && N->getNumValues() == 1 && "Type mismatch"
) ? void (0) : __assert_fail ("N->getValueType(0) == RV.getValueType() && N->getNumValues() == 1 && \"Type mismatch\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1471, __extension__ __PRETTY_FUNCTION__))
;
1472 DAG.ReplaceAllUsesWith(N, &RV);
1473 }
1474
1475 // Push the new node and any users onto the worklist
1476 AddToWorklist(RV.getNode());
1477 AddUsersToWorklist(RV.getNode());
1478
1479 // Finally, if the node is now dead, remove it from the graph. The node
1480 // may not be dead if the replacement process recursively simplified to
1481 // something else needing this node. This will also take care of adding any
1482 // operands which have lost a user to the worklist.
1483 recursivelyDeleteUnusedNodes(N);
1484 }
1485
1486 // If the root changed (e.g. it was a dead load, update the root).
1487 DAG.setRoot(Dummy.getValue());
1488 DAG.RemoveDeadNodes();
1489}
1490
1491SDValue DAGCombiner::visit(SDNode *N) {
1492 switch (N->getOpcode()) {
1493 default: break;
1494 case ISD::TokenFactor: return visitTokenFactor(N);
1495 case ISD::MERGE_VALUES: return visitMERGE_VALUES(N);
1496 case ISD::ADD: return visitADD(N);
1497 case ISD::SUB: return visitSUB(N);
1498 case ISD::ADDC: return visitADDC(N);
1499 case ISD::UADDO: return visitUADDO(N);
1500 case ISD::SUBC: return visitSUBC(N);
1501 case ISD::USUBO: return visitUSUBO(N);
1502 case ISD::ADDE: return visitADDE(N);
1503 case ISD::ADDCARRY: return visitADDCARRY(N);
1504 case ISD::SUBE: return visitSUBE(N);
1505 case ISD::SUBCARRY: return visitSUBCARRY(N);
1506 case ISD::MUL: return visitMUL(N);
1507 case ISD::SDIV: return visitSDIV(N);
1508 case ISD::UDIV: return visitUDIV(N);
1509 case ISD::SREM:
1510 case ISD::UREM: return visitREM(N);
1511 case ISD::MULHU: return visitMULHU(N);
1512 case ISD::MULHS: return visitMULHS(N);
1513 case ISD::SMUL_LOHI: return visitSMUL_LOHI(N);
1514 case ISD::UMUL_LOHI: return visitUMUL_LOHI(N);
1515 case ISD::SMULO: return visitSMULO(N);
1516 case ISD::UMULO: return visitUMULO(N);
1517 case ISD::SMIN:
1518 case ISD::SMAX:
1519 case ISD::UMIN:
1520 case ISD::UMAX: return visitIMINMAX(N);
1521 case ISD::AND: return visitAND(N);
1522 case ISD::OR: return visitOR(N);
1523 case ISD::XOR: return visitXOR(N);
1524 case ISD::SHL: return visitSHL(N);
1525 case ISD::SRA: return visitSRA(N);
1526 case ISD::SRL: return visitSRL(N);
1527 case ISD::ROTR:
1528 case ISD::ROTL: return visitRotate(N);
1529 case ISD::ABS: return visitABS(N);
1530 case ISD::BSWAP: return visitBSWAP(N);
1531 case ISD::BITREVERSE: return visitBITREVERSE(N);
1532 case ISD::CTLZ: return visitCTLZ(N);
1533 case ISD::CTLZ_ZERO_UNDEF: return visitCTLZ_ZERO_UNDEF(N);
1534 case ISD::CTTZ: return visitCTTZ(N);
1535 case ISD::CTTZ_ZERO_UNDEF: return visitCTTZ_ZERO_UNDEF(N);
1536 case ISD::CTPOP: return visitCTPOP(N);
1537 case ISD::SELECT: return visitSELECT(N);
1538 case ISD::VSELECT: return visitVSELECT(N);
1539 case ISD::SELECT_CC: return visitSELECT_CC(N);
1540 case ISD::SETCC: return visitSETCC(N);
1541 case ISD::SETCCE: return visitSETCCE(N);
1542 case ISD::SETCCCARRY: return visitSETCCCARRY(N);
1543 case ISD::SIGN_EXTEND: return visitSIGN_EXTEND(N);
1544 case ISD::ZERO_EXTEND: return visitZERO_EXTEND(N);
1545 case ISD::ANY_EXTEND: return visitANY_EXTEND(N);
1546 case ISD::AssertSext:
1547 case ISD::AssertZext: return visitAssertExt(N);
1548 case ISD::SIGN_EXTEND_INREG: return visitSIGN_EXTEND_INREG(N);
1549 case ISD::SIGN_EXTEND_VECTOR_INREG: return visitSIGN_EXTEND_VECTOR_INREG(N);
1550 case ISD::ZERO_EXTEND_VECTOR_INREG: return visitZERO_EXTEND_VECTOR_INREG(N);
1551 case ISD::TRUNCATE: return visitTRUNCATE(N);
1552 case ISD::BITCAST: return visitBITCAST(N);
1553 case ISD::BUILD_PAIR: return visitBUILD_PAIR(N);
1554 case ISD::FADD: return visitFADD(N);
1555 case ISD::FSUB: return visitFSUB(N);
1556 case ISD::FMUL: return visitFMUL(N);
1557 case ISD::FMA: return visitFMA(N);
1558 case ISD::FDIV: return visitFDIV(N);
1559 case ISD::FREM: return visitFREM(N);
1560 case ISD::FSQRT: return visitFSQRT(N);
1561 case ISD::FCOPYSIGN: return visitFCOPYSIGN(N);
1562 case ISD::SINT_TO_FP: return visitSINT_TO_FP(N);
1563 case ISD::UINT_TO_FP: return visitUINT_TO_FP(N);
1564 case ISD::FP_TO_SINT: return visitFP_TO_SINT(N);
1565 case ISD::FP_TO_UINT: return visitFP_TO_UINT(N);
1566 case ISD::FP_ROUND: return visitFP_ROUND(N);
1567 case ISD::FP_ROUND_INREG: return visitFP_ROUND_INREG(N);
1568 case ISD::FP_EXTEND: return visitFP_EXTEND(N);
1569 case ISD::FNEG: return visitFNEG(N);
1570 case ISD::FABS: return visitFABS(N);
1571 case ISD::FFLOOR: return visitFFLOOR(N);
1572 case ISD::FMINNUM: return visitFMINNUM(N);
1573 case ISD::FMAXNUM: return visitFMAXNUM(N);
1574 case ISD::FCEIL: return visitFCEIL(N);
1575 case ISD::FTRUNC: return visitFTRUNC(N);
1576 case ISD::BRCOND: return visitBRCOND(N);
1577 case ISD::BR_CC: return visitBR_CC(N);
1578 case ISD::LOAD: return visitLOAD(N);
1579 case ISD::STORE: return visitSTORE(N);
1580 case ISD::INSERT_VECTOR_ELT: return visitINSERT_VECTOR_ELT(N);
1581 case ISD::EXTRACT_VECTOR_ELT: return visitEXTRACT_VECTOR_ELT(N);
1582 case ISD::BUILD_VECTOR: return visitBUILD_VECTOR(N);
1583 case ISD::CONCAT_VECTORS: return visitCONCAT_VECTORS(N);
1584 case ISD::EXTRACT_SUBVECTOR: return visitEXTRACT_SUBVECTOR(N);
1585 case ISD::VECTOR_SHUFFLE: return visitVECTOR_SHUFFLE(N);
1586 case ISD::SCALAR_TO_VECTOR: return visitSCALAR_TO_VECTOR(N);
1587 case ISD::INSERT_SUBVECTOR: return visitINSERT_SUBVECTOR(N);
1588 case ISD::MGATHER: return visitMGATHER(N);
1589 case ISD::MLOAD: return visitMLOAD(N);
1590 case ISD::MSCATTER: return visitMSCATTER(N);
1591 case ISD::MSTORE: return visitMSTORE(N);
1592 case ISD::FP_TO_FP16: return visitFP_TO_FP16(N);
1593 case ISD::FP16_TO_FP: return visitFP16_TO_FP(N);
1594 }
1595 return SDValue();
1596}
1597
1598SDValue DAGCombiner::combine(SDNode *N) {
1599 SDValue RV = visit(N);
1600
1601 // If nothing happened, try a target-specific DAG combine.
1602 if (!RV.getNode()) {
1603 assert(N->getOpcode() != ISD::DELETED_NODE &&(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& "Node was deleted but visit returned NULL!") ? void
(0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned NULL!\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1604, __extension__ __PRETTY_FUNCTION__))
1604 "Node was deleted but visit returned NULL!")(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& "Node was deleted but visit returned NULL!") ? void
(0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned NULL!\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1604, __extension__ __PRETTY_FUNCTION__))
;
1605
1606 if (N->getOpcode() >= ISD::BUILTIN_OP_END ||
1607 TLI.hasTargetDAGCombine((ISD::NodeType)N->getOpcode())) {
1608
1609 // Expose the DAG combiner to the target combiner impls.
1610 TargetLowering::DAGCombinerInfo
1611 DagCombineInfo(DAG, Level, false, this);
1612
1613 RV = TLI.PerformDAGCombine(N, DagCombineInfo);
1614 }
1615 }
1616
1617 // If nothing happened still, try promoting the operation.
1618 if (!RV.getNode()) {
1619 switch (N->getOpcode()) {
1620 default: break;
1621 case ISD::ADD:
1622 case ISD::SUB:
1623 case ISD::MUL:
1624 case ISD::AND:
1625 case ISD::OR:
1626 case ISD::XOR:
1627 RV = PromoteIntBinOp(SDValue(N, 0));
1628 break;
1629 case ISD::SHL:
1630 case ISD::SRA:
1631 case ISD::SRL:
1632 RV = PromoteIntShiftOp(SDValue(N, 0));
1633 break;
1634 case ISD::SIGN_EXTEND:
1635 case ISD::ZERO_EXTEND:
1636 case ISD::ANY_EXTEND:
1637 RV = PromoteExtend(SDValue(N, 0));
1638 break;
1639 case ISD::LOAD:
1640 if (PromoteLoad(SDValue(N, 0)))
1641 RV = SDValue(N, 0);
1642 break;
1643 }
1644 }
1645
1646 // If N is a commutative binary node, try eliminate it if the commuted
1647 // version is already present in the DAG.
1648 if (!RV.getNode() && TLI.isCommutativeBinOp(N->getOpcode()) &&
1649 N->getNumValues() == 1) {
1650 SDValue N0 = N->getOperand(0);
1651 SDValue N1 = N->getOperand(1);
1652
1653 // Constant operands are canonicalized to RHS.
1654 if (N0 != N1 && (isa<ConstantSDNode>(N0) || !isa<ConstantSDNode>(N1))) {
1655 SDValue Ops[] = {N1, N0};
1656 SDNode *CSENode = DAG.getNodeIfExists(N->getOpcode(), N->getVTList(), Ops,
1657 N->getFlags());
1658 if (CSENode)
1659 return SDValue(CSENode, 0);
1660 }
1661 }
1662
1663 return RV;
1664}
1665
1666/// Given a node, return its input chain if it has one, otherwise return a null
1667/// sd operand.
1668static SDValue getInputChainForNode(SDNode *N) {
1669 if (unsigned NumOps = N->getNumOperands()) {
1670 if (N->getOperand(0).getValueType() == MVT::Other)
1671 return N->getOperand(0);
1672 if (N->getOperand(NumOps-1).getValueType() == MVT::Other)
1673 return N->getOperand(NumOps-1);
1674 for (unsigned i = 1; i < NumOps-1; ++i)
1675 if (N->getOperand(i).getValueType() == MVT::Other)
1676 return N->getOperand(i);
1677 }
1678 return SDValue();
1679}
1680
1681SDValue DAGCombiner::visitTokenFactor(SDNode *N) {
1682 // If N has two operands, where one has an input chain equal to the other,
1683 // the 'other' chain is redundant.
1684 if (N->getNumOperands() == 2) {
1685 if (getInputChainForNode(N->getOperand(0).getNode()) == N->getOperand(1))
1686 return N->getOperand(0);
1687 if (getInputChainForNode(N->getOperand(1).getNode()) == N->getOperand(0))
1688 return N->getOperand(1);
1689 }
1690
1691 SmallVector<SDNode *, 8> TFs; // List of token factors to visit.
1692 SmallVector<SDValue, 8> Ops; // Ops for replacing token factor.
1693 SmallPtrSet<SDNode*, 16> SeenOps;
1694 bool Changed = false; // If we should replace this token factor.
1695
1696 // Start out with this token factor.
1697 TFs.push_back(N);
1698
1699 // Iterate through token factors. The TFs grows when new token factors are
1700 // encountered.
1701 for (unsigned i = 0; i < TFs.size(); ++i) {
1702 SDNode *TF = TFs[i];
1703
1704 // Check each of the operands.
1705 for (const SDValue &Op : TF->op_values()) {
1706 switch (Op.getOpcode()) {
1707 case ISD::EntryToken:
1708 // Entry tokens don't need to be added to the list. They are
1709 // redundant.
1710 Changed = true;
1711 break;
1712
1713 case ISD::TokenFactor:
1714 if (Op.hasOneUse() && !is_contained(TFs, Op.getNode())) {
1715 // Queue up for processing.
1716 TFs.push_back(Op.getNode());
1717 // Clean up in case the token factor is removed.
1718 AddToWorklist(Op.getNode());
1719 Changed = true;
1720 break;
1721 }
1722 LLVM_FALLTHROUGH[[clang::fallthrough]];
1723
1724 default:
1725 // Only add if it isn't already in the list.
1726 if (SeenOps.insert(Op.getNode()).second)
1727 Ops.push_back(Op);
1728 else
1729 Changed = true;
1730 break;
1731 }
1732 }
1733 }
1734
1735 // Remove Nodes that are chained to another node in the list. Do so
1736 // by walking up chains breath-first stopping when we've seen
1737 // another operand. In general we must climb to the EntryNode, but we can exit
1738 // early if we find all remaining work is associated with just one operand as
1739 // no further pruning is possible.
1740
1741 // List of nodes to search through and original Ops from which they originate.
1742 SmallVector<std::pair<SDNode *, unsigned>, 8> Worklist;
1743 SmallVector<unsigned, 8> OpWorkCount; // Count of work for each Op.
1744 SmallPtrSet<SDNode *, 16> SeenChains;
1745 bool DidPruneOps = false;
1746
1747 unsigned NumLeftToConsider = 0;
1748 for (const SDValue &Op : Ops) {
1749 Worklist.push_back(std::make_pair(Op.getNode(), NumLeftToConsider++));
1750 OpWorkCount.push_back(1);
1751 }
1752
1753 auto AddToWorklist = [&](unsigned CurIdx, SDNode *Op, unsigned OpNumber) {
1754 // If this is an Op, we can remove the op from the list. Remark any
1755 // search associated with it as from the current OpNumber.
1756 if (SeenOps.count(Op) != 0) {
1757 Changed = true;
1758 DidPruneOps = true;
1759 unsigned OrigOpNumber = 0;
1760 while (OrigOpNumber < Ops.size() && Ops[OrigOpNumber].getNode() != Op)
1761 OrigOpNumber++;
1762 assert((OrigOpNumber != Ops.size()) &&(static_cast <bool> ((OrigOpNumber != Ops.size()) &&
"expected to find TokenFactor Operand") ? void (0) : __assert_fail
("(OrigOpNumber != Ops.size()) && \"expected to find TokenFactor Operand\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1763, __extension__ __PRETTY_FUNCTION__))
1763 "expected to find TokenFactor Operand")(static_cast <bool> ((OrigOpNumber != Ops.size()) &&
"expected to find TokenFactor Operand") ? void (0) : __assert_fail
("(OrigOpNumber != Ops.size()) && \"expected to find TokenFactor Operand\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1763, __extension__ __PRETTY_FUNCTION__))
;
1764 // Re-mark worklist from OrigOpNumber to OpNumber
1765 for (unsigned i = CurIdx + 1; i < Worklist.size(); ++i) {
1766 if (Worklist[i].second == OrigOpNumber) {
1767 Worklist[i].second = OpNumber;
1768 }
1769 }
1770 OpWorkCount[OpNumber] += OpWorkCount[OrigOpNumber];
1771 OpWorkCount[OrigOpNumber] = 0;
1772 NumLeftToConsider--;
1773 }
1774 // Add if it's a new chain
1775 if (SeenChains.insert(Op).second) {
1776 OpWorkCount[OpNumber]++;
1777 Worklist.push_back(std::make_pair(Op, OpNumber));
1778 }
1779 };
1780
1781 for (unsigned i = 0; i < Worklist.size() && i < 1024; ++i) {
1782 // We need at least be consider at least 2 Ops to prune.
1783 if (NumLeftToConsider <= 1)
1784 break;
1785 auto CurNode = Worklist[i].first;
1786 auto CurOpNumber = Worklist[i].second;
1787 assert((OpWorkCount[CurOpNumber] > 0) &&(static_cast <bool> ((OpWorkCount[CurOpNumber] > 0) &&
"Node should not appear in worklist") ? void (0) : __assert_fail
("(OpWorkCount[CurOpNumber] > 0) && \"Node should not appear in worklist\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1788, __extension__ __PRETTY_FUNCTION__))
1788 "Node should not appear in worklist")(static_cast <bool> ((OpWorkCount[CurOpNumber] > 0) &&
"Node should not appear in worklist") ? void (0) : __assert_fail
("(OpWorkCount[CurOpNumber] > 0) && \"Node should not appear in worklist\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1788, __extension__ __PRETTY_FUNCTION__))
;
1789 switch (CurNode->getOpcode()) {
1790 case ISD::EntryToken:
1791 // Hitting EntryToken is the only way for the search to terminate without
1792 // hitting
1793 // another operand's search. Prevent us from marking this operand
1794 // considered.
1795 NumLeftToConsider++;
1796 break;
1797 case ISD::TokenFactor:
1798 for (const SDValue &Op : CurNode->op_values())
1799 AddToWorklist(i, Op.getNode(), CurOpNumber);
1800 break;
1801 case ISD::CopyFromReg:
1802 case ISD::CopyToReg:
1803 AddToWorklist(i, CurNode->getOperand(0).getNode(), CurOpNumber);
1804 break;
1805 default:
1806 if (auto *MemNode = dyn_cast<MemSDNode>(CurNode))
1807 AddToWorklist(i, MemNode->getChain().getNode(), CurOpNumber);
1808 break;
1809 }
1810 OpWorkCount[CurOpNumber]--;
1811 if (OpWorkCount[CurOpNumber] == 0)
1812 NumLeftToConsider--;
1813 }
1814
1815 // If we've changed things around then replace token factor.
1816 if (Changed) {
1817 SDValue Result;
1818 if (Ops.empty()) {
1819 // The entry token is the only possible outcome.
1820 Result = DAG.getEntryNode();
1821 } else {
1822 if (DidPruneOps) {
1823 SmallVector<SDValue, 8> PrunedOps;
1824 //
1825 for (const SDValue &Op : Ops) {
1826 if (SeenChains.count(Op.getNode()) == 0)
1827 PrunedOps.push_back(Op);
1828 }
1829 Result = DAG.getNode(ISD::TokenFactor, SDLoc(N), MVT::Other, PrunedOps);
1830 } else {
1831 Result = DAG.getNode(ISD::TokenFactor, SDLoc(N), MVT::Other, Ops);
1832 }
1833 }
1834 return Result;
1835 }
1836 return SDValue();
1837}
1838
1839/// MERGE_VALUES can always be eliminated.
1840SDValue DAGCombiner::visitMERGE_VALUES(SDNode *N) {
1841 WorklistRemover DeadNodes(*this);
1842 // Replacing results may cause a different MERGE_VALUES to suddenly
1843 // be CSE'd with N, and carry its uses with it. Iterate until no
1844 // uses remain, to ensure that the node can be safely deleted.
1845 // First add the users of this node to the work list so that they
1846 // can be tried again once they have new operands.
1847 AddUsersToWorklist(N);
1848 do {
1849 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
1850 DAG.ReplaceAllUsesOfValueWith(SDValue(N, i), N->getOperand(i));
1851 } while (!N->use_empty());
1852 deleteAndRecombine(N);
1853 return SDValue(N, 0); // Return N so it doesn't get rechecked!
1854}
1855
1856/// If \p N is a ConstantSDNode with isOpaque() == false return it casted to a
1857/// ConstantSDNode pointer else nullptr.
1858static ConstantSDNode *getAsNonOpaqueConstant(SDValue N) {
1859 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N);
1860 return Const != nullptr && !Const->isOpaque() ? Const : nullptr;
1861}
1862
1863SDValue DAGCombiner::foldBinOpIntoSelect(SDNode *BO) {
1864 auto BinOpcode = BO->getOpcode();
1865 assert((BinOpcode == ISD::ADD || BinOpcode == ISD::SUB ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1874, __extension__ __PRETTY_FUNCTION__))
1866 BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1874, __extension__ __PRETTY_FUNCTION__))
1867 BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1874, __extension__ __PRETTY_FUNCTION__))
1868 BinOpcode == ISD::UREM || BinOpcode == ISD::AND ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1874, __extension__ __PRETTY_FUNCTION__))
1869 BinOpcode == ISD::OR || BinOpcode == ISD::XOR ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1874, __extension__ __PRETTY_FUNCTION__))
1870 BinOpcode == ISD::SHL || BinOpcode == ISD::SRL ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1874, __extension__ __PRETTY_FUNCTION__))
1871 BinOpcode == ISD::SRA || BinOpcode == ISD::FADD ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1874, __extension__ __PRETTY_FUNCTION__))
1872 BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1874, __extension__ __PRETTY_FUNCTION__))
1873 BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) &&(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1874, __extension__ __PRETTY_FUNCTION__))
1874 "Unexpected binary operator")(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1874, __extension__ __PRETTY_FUNCTION__))
;
1875
1876 // Bail out if any constants are opaque because we can't constant fold those.
1877 SDValue C1 = BO->getOperand(1);
1878 if (!isConstantOrConstantVector(C1, true) &&
1879 !isConstantFPBuildVectorOrConstantFP(C1))
1880 return SDValue();
1881
1882 // Don't do this unless the old select is going away. We want to eliminate the
1883 // binary operator, not replace a binop with a select.
1884 // TODO: Handle ISD::SELECT_CC.
1885 SDValue Sel = BO->getOperand(0);
1886 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse())
1887 return SDValue();
1888
1889 SDValue CT = Sel.getOperand(1);
1890 if (!isConstantOrConstantVector(CT, true) &&
1891 !isConstantFPBuildVectorOrConstantFP(CT))
1892 return SDValue();
1893
1894 SDValue CF = Sel.getOperand(2);
1895 if (!isConstantOrConstantVector(CF, true) &&
1896 !isConstantFPBuildVectorOrConstantFP(CF))
1897 return SDValue();
1898
1899 // We have a select-of-constants followed by a binary operator with a
1900 // constant. Eliminate the binop by pulling the constant math into the select.
1901 // Example: add (select Cond, CT, CF), C1 --> select Cond, CT + C1, CF + C1
1902 EVT VT = Sel.getValueType();
1903 SDLoc DL(Sel);
1904 SDValue NewCT = DAG.getNode(BinOpcode, DL, VT, CT, C1);
1905 if (!NewCT.isUndef() &&
1906 !isConstantOrConstantVector(NewCT, true) &&
1907 !isConstantFPBuildVectorOrConstantFP(NewCT))
1908 return SDValue();
1909
1910 SDValue NewCF = DAG.getNode(BinOpcode, DL, VT, CF, C1);
1911 if (!NewCF.isUndef() &&
1912 !isConstantOrConstantVector(NewCF, true) &&
1913 !isConstantFPBuildVectorOrConstantFP(NewCF))
1914 return SDValue();
1915
1916 return DAG.getSelect(DL, VT, Sel.getOperand(0), NewCT, NewCF);
1917}
1918
1919SDValue DAGCombiner::visitADD(SDNode *N) {
1920 SDValue N0 = N->getOperand(0);
1921 SDValue N1 = N->getOperand(1);
1922 EVT VT = N0.getValueType();
1923 SDLoc DL(N);
1924
1925 // fold vector ops
1926 if (VT.isVector()) {
1927 if (SDValue FoldedVOp = SimplifyVBinOp(N))
1928 return FoldedVOp;
1929
1930 // fold (add x, 0) -> x, vector edition
1931 if (ISD::isBuildVectorAllZeros(N1.getNode()))
1932 return N0;
1933 if (ISD::isBuildVectorAllZeros(N0.getNode()))
1934 return N1;
1935 }
1936
1937 // fold (add x, undef) -> undef
1938 if (N0.isUndef())
1939 return N0;
1940
1941 if (N1.isUndef())
1942 return N1;
1943
1944 if (DAG.isConstantIntBuildVectorOrConstantInt(N0)) {
1945 // canonicalize constant to RHS
1946 if (!DAG.isConstantIntBuildVectorOrConstantInt(N1))
1947 return DAG.getNode(ISD::ADD, DL, VT, N1, N0);
1948 // fold (add c1, c2) -> c1+c2
1949 return DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, N0.getNode(),
1950 N1.getNode());
1951 }
1952
1953 // fold (add x, 0) -> x
1954 if (isNullConstant(N1))
1955 return N0;
1956
1957 if (isConstantOrConstantVector(N1, /* NoOpaque */ true)) {
1958 // fold ((c1-A)+c2) -> (c1+c2)-A
1959 if (N0.getOpcode() == ISD::SUB &&
1960 isConstantOrConstantVector(N0.getOperand(0), /* NoOpaque */ true)) {
1961 // FIXME: Adding 2 constants should be handled by FoldConstantArithmetic.
1962 return DAG.getNode(ISD::SUB, DL, VT,
1963 DAG.getNode(ISD::ADD, DL, VT, N1, N0.getOperand(0)),
1964 N0.getOperand(1));
1965 }
1966
1967 // add (sext i1 X), 1 -> zext (not i1 X)
1968 // We don't transform this pattern:
1969 // add (zext i1 X), -1 -> sext (not i1 X)
1970 // because most (?) targets generate better code for the zext form.
1971 if (N0.getOpcode() == ISD::SIGN_EXTEND && N0.hasOneUse() &&
1972 isOneConstantOrOneSplatConstant(N1)) {
1973 SDValue X = N0.getOperand(0);
1974 if ((!LegalOperations ||
1975 (TLI.isOperationLegal(ISD::XOR, X.getValueType()) &&
1976 TLI.isOperationLegal(ISD::ZERO_EXTEND, VT))) &&
1977 X.getScalarValueSizeInBits() == 1) {
1978 SDValue Not = DAG.getNOT(DL, X, X.getValueType());
1979 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Not);
1980 }
1981 }
1982
1983 // Undo the add -> or combine to merge constant offsets from a frame index.
1984 if (N0.getOpcode() == ISD::OR &&
1985 isa<FrameIndexSDNode>(N0.getOperand(0)) &&
1986 isa<ConstantSDNode>(N0.getOperand(1)) &&
1987 DAG.haveNoCommonBitsSet(N0.getOperand(0), N0.getOperand(1))) {
1988 SDValue Add0 = DAG.getNode(ISD::ADD, DL, VT, N1, N0.getOperand(1));
1989 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), Add0);
1990 }
1991 }
1992
1993 if (SDValue NewSel = foldBinOpIntoSelect(N))
1994 return NewSel;
1995
1996 // reassociate add
1997 if (SDValue RADD = ReassociateOps(ISD::ADD, DL, N0, N1))
1998 return RADD;
1999
2000 // fold ((0-A) + B) -> B-A
2001 if (N0.getOpcode() == ISD::SUB &&
2002 isNullConstantOrNullSplatConstant(N0.getOperand(0)))
2003 return DAG.getNode(ISD::SUB, DL, VT, N1, N0.getOperand(1));
2004
2005 // fold (A + (0-B)) -> A-B
2006 if (N1.getOpcode() == ISD::SUB &&
2007 isNullConstantOrNullSplatConstant(N1.getOperand(0)))
2008 return DAG.getNode(ISD::SUB, DL, VT, N0, N1.getOperand(1));
2009
2010 // fold (A+(B-A)) -> B
2011 if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(1))
2012 return N1.getOperand(0);
2013
2014 // fold ((B-A)+A) -> B
2015 if (N0.getOpcode() == ISD::SUB && N1 == N0.getOperand(1))
2016 return N0.getOperand(0);
2017
2018 // fold (A+(B-(A+C))) to (B-C)
2019 if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD &&
2020 N0 == N1.getOperand(1).getOperand(0))
2021 return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0),
2022 N1.getOperand(1).getOperand(1));
2023
2024 // fold (A+(B-(C+A))) to (B-C)
2025 if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD &&
2026 N0 == N1.getOperand(1).getOperand(1))
2027 return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0),
2028 N1.getOperand(1).getOperand(0));
2029
2030 // fold (A+((B-A)+or-C)) to (B+or-C)
2031 if ((N1.getOpcode() == ISD::SUB || N1.getOpcode() == ISD::ADD) &&
2032 N1.getOperand(0).getOpcode() == ISD::SUB &&
2033 N0 == N1.getOperand(0).getOperand(1))
2034 return DAG.getNode(N1.getOpcode(), DL, VT, N1.getOperand(0).getOperand(0),
2035 N1.getOperand(1));
2036
2037 // fold (A-B)+(C-D) to (A+C)-(B+D) when A or C is constant
2038 if (N0.getOpcode() == ISD::SUB && N1.getOpcode() == ISD::SUB) {
2039 SDValue N00 = N0.getOperand(0);
2040 SDValue N01 = N0.getOperand(1);
2041 SDValue N10 = N1.getOperand(0);
2042 SDValue N11 = N1.getOperand(1);
2043
2044 if (isConstantOrConstantVector(N00) || isConstantOrConstantVector(N10))
2045 return DAG.getNode(ISD::SUB, DL, VT,
2046 DAG.getNode(ISD::ADD, SDLoc(N0), VT, N00, N10),
2047 DAG.getNode(ISD::ADD, SDLoc(N1), VT, N01, N11));
2048 }
2049
2050 if (SimplifyDemandedBits(SDValue(N, 0)))
2051 return SDValue(N, 0);
2052
2053 // fold (a+b) -> (a|b) iff a and b share no bits.
2054 if ((!LegalOperations || TLI.isOperationLegal(ISD::OR, VT)) &&
2055 DAG.haveNoCommonBitsSet(N0, N1))
2056 return DAG.getNode(ISD::OR, DL, VT, N0, N1);
2057
2058 if (SDValue Combined = visitADDLike(N0, N1, N))
2059 return Combined;
2060
2061 if (SDValue Combined = visitADDLike(N1, N0, N))
2062 return Combined;
2063
2064 return SDValue();
2065}
2066
2067static SDValue getAsCarry(const TargetLowering &TLI, SDValue V) {
2068 bool Masked = false;
2069
2070 // First, peel away TRUNCATE/ZERO_EXTEND/AND nodes due to legalization.
2071 while (true) {
2072 if (V.getOpcode() == ISD::TRUNCATE || V.getOpcode() == ISD::ZERO_EXTEND) {
2073 V = V.getOperand(0);
2074 continue;
2075 }
2076
2077 if (V.getOpcode() == ISD::AND && isOneConstant(V.getOperand(1))) {
2078 Masked = true;
2079 V = V.getOperand(0);
2080 continue;
2081 }
2082
2083 break;
2084 }
2085
2086 // If this is not a carry, return.
2087 if (V.getResNo() != 1)
2088 return SDValue();
2089
2090 if (V.getOpcode() != ISD::ADDCARRY && V.getOpcode() != ISD::SUBCARRY &&
2091 V.getOpcode() != ISD::UADDO && V.getOpcode() != ISD::USUBO)
2092 return SDValue();
2093
2094 // If the result is masked, then no matter what kind of bool it is we can
2095 // return. If it isn't, then we need to make sure the bool type is either 0 or
2096 // 1 and not other values.
2097 if (Masked ||
2098 TLI.getBooleanContents(V.getValueType()) ==
2099 TargetLoweringBase::ZeroOrOneBooleanContent)
2100 return V;
2101
2102 return SDValue();
2103}
2104
2105SDValue DAGCombiner::visitADDLike(SDValue N0, SDValue N1, SDNode *LocReference) {
2106 EVT VT = N0.getValueType();
2107 SDLoc DL(LocReference);
2108
2109 // fold (add x, shl(0 - y, n)) -> sub(x, shl(y, n))
2110 if (N1.getOpcode() == ISD::SHL && N1.getOperand(0).getOpcode() == ISD::SUB &&
2111 isNullConstantOrNullSplatConstant(N1.getOperand(0).getOperand(0)))
2112 return DAG.getNode(ISD::SUB, DL, VT, N0,
2113 DAG.getNode(ISD::SHL, DL, VT,
2114 N1.getOperand(0).getOperand(1),
2115 N1.getOperand(1)));
2116
2117 if (N1.getOpcode() == ISD::AND) {
2118 SDValue AndOp0 = N1.getOperand(0);
2119 unsigned NumSignBits = DAG.ComputeNumSignBits(AndOp0);
2120 unsigned DestBits = VT.getScalarSizeInBits();
2121
2122 // (add z, (and (sbbl x, x), 1)) -> (sub z, (sbbl x, x))
2123 // and similar xforms where the inner op is either ~0 or 0.
2124 if (NumSignBits == DestBits &&
2125 isOneConstantOrOneSplatConstant(N1->getOperand(1)))
2126 return DAG.getNode(ISD::SUB, DL, VT, N0, AndOp0);
2127 }
2128
2129 // add (sext i1), X -> sub X, (zext i1)
2130 if (N0.getOpcode() == ISD::SIGN_EXTEND &&
2131 N0.getOperand(0).getValueType() == MVT::i1 &&
2132 !TLI.isOperationLegal(ISD::SIGN_EXTEND, MVT::i1)) {
2133 SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0));
2134 return DAG.getNode(ISD::SUB, DL, VT, N1, ZExt);
2135 }
2136
2137 // add X, (sextinreg Y i1) -> sub X, (and Y 1)
2138 if (N1.getOpcode() == ISD::SIGN_EXTEND_INREG) {
2139 VTSDNode *TN = cast<VTSDNode>(N1.getOperand(1));
2140 if (TN->getVT() == MVT::i1) {
2141 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0),
2142 DAG.getConstant(1, DL, VT));
2143 return DAG.getNode(ISD::SUB, DL, VT, N0, ZExt);
2144 }
2145 }
2146
2147 // (add X, (addcarry Y, 0, Carry)) -> (addcarry X, Y, Carry)
2148 if (N1.getOpcode() == ISD::ADDCARRY && isNullConstant(N1.getOperand(1)) &&
2149 N1.getResNo() == 0)
2150 return DAG.getNode(ISD::ADDCARRY, DL, N1->getVTList(),
2151 N0, N1.getOperand(0), N1.getOperand(2));
2152
2153 // (add X, Carry) -> (addcarry X, 0, Carry)
2154 if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT))
2155 if (SDValue Carry = getAsCarry(TLI, N1))
2156 return DAG.getNode(ISD::ADDCARRY, DL,
2157 DAG.getVTList(VT, Carry.getValueType()), N0,
2158 DAG.getConstant(0, DL, VT), Carry);
2159
2160 return SDValue();
2161}
2162
2163SDValue DAGCombiner::visitADDC(SDNode *N) {
2164 SDValue N0 = N->getOperand(0);
2165 SDValue N1 = N->getOperand(1);
2166 EVT VT = N0.getValueType();
2167 SDLoc DL(N);
2168
2169 // If the flag result is dead, turn this into an ADD.
2170 if (!N->hasAnyUseOfValue(1))
2171 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2172 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2173
2174 // canonicalize constant to RHS.
2175 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
2176 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2177 if (N0C && !N1C)
2178 return DAG.getNode(ISD::ADDC, DL, N->getVTList(), N1, N0);
2179
2180 // fold (addc x, 0) -> x + no carry out
2181 if (isNullConstant(N1))
2182 return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE,
2183 DL, MVT::Glue));
2184
2185 // If it cannot overflow, transform into an add.
2186 if (DAG.computeOverflowKind(N0, N1) == SelectionDAG::OFK_Never)
2187 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2188 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2189
2190 return SDValue();
2191}
2192
2193SDValue DAGCombiner::visitUADDO(SDNode *N) {
2194 SDValue N0 = N->getOperand(0);
2195 SDValue N1 = N->getOperand(1);
2196 EVT VT = N0.getValueType();
2197 if (VT.isVector())
2198 return SDValue();
2199
2200 EVT CarryVT = N->getValueType(1);
2201 SDLoc DL(N);
2202
2203 // If the flag result is dead, turn this into an ADD.
2204 if (!N->hasAnyUseOfValue(1))
2205 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2206 DAG.getUNDEF(CarryVT));
2207
2208 // canonicalize constant to RHS.
2209 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
2210 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2211 if (N0C && !N1C)
2212 return DAG.getNode(ISD::UADDO, DL, N->getVTList(), N1, N0);
2213
2214 // fold (uaddo x, 0) -> x + no carry out
2215 if (isNullConstant(N1))
2216 return CombineTo(N, N0, DAG.getConstant(0, DL, CarryVT));
2217
2218 // If it cannot overflow, transform into an add.
2219 if (DAG.computeOverflowKind(N0, N1) == SelectionDAG::OFK_Never)
2220 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2221 DAG.getConstant(0, DL, CarryVT));
2222
2223 if (SDValue Combined = visitUADDOLike(N0, N1, N))
2224 return Combined;
2225
2226 if (SDValue Combined = visitUADDOLike(N1, N0, N))
2227 return Combined;
2228
2229 return SDValue();
2230}
2231
2232SDValue DAGCombiner::visitUADDOLike(SDValue N0, SDValue N1, SDNode *N) {
2233 auto VT = N0.getValueType();
2234
2235 // (uaddo X, (addcarry Y, 0, Carry)) -> (addcarry X, Y, Carry)
2236 // If Y + 1 cannot overflow.
2237 if (N1.getOpcode() == ISD::ADDCARRY && isNullConstant(N1.getOperand(1))) {
2238 SDValue Y = N1.getOperand(0);
2239 SDValue One = DAG.getConstant(1, SDLoc(N), Y.getValueType());
2240 if (DAG.computeOverflowKind(Y, One) == SelectionDAG::OFK_Never)
2241 return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0, Y,
2242 N1.getOperand(2));
2243 }
2244
2245 // (uaddo X, Carry) -> (addcarry X, 0, Carry)
2246 if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT))
2247 if (SDValue Carry = getAsCarry(TLI, N1))
2248 return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0,
2249 DAG.getConstant(0, SDLoc(N), VT), Carry);
2250
2251 return SDValue();
2252}
2253
2254SDValue DAGCombiner::visitADDE(SDNode *N) {
2255 SDValue N0 = N->getOperand(0);
2256 SDValue N1 = N->getOperand(1);
2257 SDValue CarryIn = N->getOperand(2);
2258
2259 // canonicalize constant to RHS
2260 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
2261 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2262 if (N0C && !N1C)
2263 return DAG.getNode(ISD::ADDE, SDLoc(N), N->getVTList(),
2264 N1, N0, CarryIn);
2265
2266 // fold (adde x, y, false) -> (addc x, y)
2267 if (CarryIn.getOpcode() == ISD::CARRY_FALSE)
2268 return DAG.getNode(ISD::ADDC, SDLoc(N), N->getVTList(), N0, N1);
2269
2270 return SDValue();
2271}
2272
2273SDValue DAGCombiner::visitADDCARRY(SDNode *N) {
2274 SDValue N0 = N->getOperand(0);
2275 SDValue N1 = N->getOperand(1);
2276 SDValue CarryIn = N->getOperand(2);
2277 SDLoc DL(N);
2278
2279 // canonicalize constant to RHS
2280 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
2281 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2282 if (N0C && !N1C)
2283 return DAG.getNode(ISD::ADDCARRY, DL, N->getVTList(), N1, N0, CarryIn);
2284
2285 // fold (addcarry x, y, false) -> (uaddo x, y)
2286 if (isNullConstant(CarryIn))
2287 return DAG.getNode(ISD::UADDO, DL, N->getVTList(), N0, N1);
2288
2289 // fold (addcarry 0, 0, X) -> (and (ext/trunc X), 1) and no carry.
2290 if (isNullConstant(N0) && isNullConstant(N1)) {
2291 EVT VT = N0.getValueType();
2292 EVT CarryVT = CarryIn.getValueType();
2293 SDValue CarryExt = DAG.getBoolExtOrTrunc(CarryIn, DL, VT, CarryVT);
2294 AddToWorklist(CarryExt.getNode());
2295 return CombineTo(N, DAG.getNode(ISD::AND, DL, VT, CarryExt,
2296 DAG.getConstant(1, DL, VT)),
2297 DAG.getConstant(0, DL, CarryVT));
2298 }
2299
2300 if (SDValue Combined = visitADDCARRYLike(N0, N1, CarryIn, N))
2301 return Combined;
2302
2303 if (SDValue Combined = visitADDCARRYLike(N1, N0, CarryIn, N))
2304 return Combined;
2305
2306 return SDValue();
2307}
2308
2309SDValue DAGCombiner::visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn,
2310 SDNode *N) {
2311 // Iff the flag result is dead:
2312 // (addcarry (add|uaddo X, Y), 0, Carry) -> (addcarry X, Y, Carry)
2313 if ((N0.getOpcode() == ISD::ADD ||
2314 (N0.getOpcode() == ISD::UADDO && N0.getResNo() == 0)) &&
2315 isNullConstant(N1) && !N->hasAnyUseOfValue(1))
2316 return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(),
2317 N0.getOperand(0), N0.getOperand(1), CarryIn);
2318
2319 /**
2320 * When one of the addcarry argument is itself a carry, we may be facing
2321 * a diamond carry propagation. In which case we try to transform the DAG
2322 * to ensure linear carry propagation if that is possible.
2323 *
2324 * We are trying to get:
2325 * (addcarry X, 0, (addcarry A, B, Z):Carry)
2326 */
2327 if (auto Y = getAsCarry(TLI, N1)) {
2328 /**
2329 * (uaddo A, B)
2330 * / \
2331 * Carry Sum
2332 * | \
2333 * | (addcarry *, 0, Z)
2334 * | /
2335 * \ Carry
2336 * | /
2337 * (addcarry X, *, *)
2338 */
2339 if (Y.getOpcode() == ISD::UADDO &&
2340 CarryIn.getResNo() == 1 &&
2341 CarryIn.getOpcode() == ISD::ADDCARRY &&
2342 isNullConstant(CarryIn.getOperand(1)) &&
2343 CarryIn.getOperand(0) == Y.getValue(0)) {
2344 auto NewY = DAG.getNode(ISD::ADDCARRY, SDLoc(N), Y->getVTList(),
2345 Y.getOperand(0), Y.getOperand(1),
2346 CarryIn.getOperand(2));
2347 AddToWorklist(NewY.getNode());
2348 return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0,
2349 DAG.getConstant(0, SDLoc(N), N0.getValueType()),
2350 NewY.getValue(1));
2351 }
2352 }
2353
2354 return SDValue();
2355}
2356
2357// Since it may not be valid to emit a fold to zero for vector initializers
2358// check if we can before folding.
2359static SDValue tryFoldToZero(const SDLoc &DL, const TargetLowering &TLI, EVT VT,
2360 SelectionDAG &DAG, bool LegalOperations,
2361 bool LegalTypes) {
2362 if (!VT.isVector())
2363 return DAG.getConstant(0, DL, VT);
2364 if (!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT))
2365 return DAG.getConstant(0, DL, VT);
2366 return SDValue();
2367}
2368
2369SDValue DAGCombiner::visitSUB(SDNode *N) {
2370 SDValue N0 = N->getOperand(0);
2371 SDValue N1 = N->getOperand(1);
2372 EVT VT = N0.getValueType();
2373 SDLoc DL(N);
2374
2375 // fold vector ops
2376 if (VT.isVector()) {
2377 if (SDValue FoldedVOp = SimplifyVBinOp(N))
2378 return FoldedVOp;
2379
2380 // fold (sub x, 0) -> x, vector edition
2381 if (ISD::isBuildVectorAllZeros(N1.getNode()))
2382 return N0;
2383 }
2384
2385 // fold (sub x, x) -> 0
2386 // FIXME: Refactor this and xor and other similar operations together.
2387 if (N0 == N1)
2388 return tryFoldToZero(DL, TLI, VT, DAG, LegalOperations, LegalTypes);
2389 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
2390 DAG.isConstantIntBuildVectorOrConstantInt(N1)) {
2391 // fold (sub c1, c2) -> c1-c2
2392 return DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, N0.getNode(),
2393 N1.getNode());
2394 }
2395
2396 if (SDValue NewSel = foldBinOpIntoSelect(N))
2397 return NewSel;
2398
2399 ConstantSDNode *N1C = getAsNonOpaqueConstant(N1);
2400
2401 // fold (sub x, c) -> (add x, -c)
2402 if (N1C) {
2403 return DAG.getNode(ISD::ADD, DL, VT, N0,
2404 DAG.getConstant(-N1C->getAPIntValue(), DL, VT));
2405 }
2406
2407 if (isNullConstantOrNullSplatConstant(N0)) {
2408 unsigned BitWidth = VT.getScalarSizeInBits();
2409 // Right-shifting everything out but the sign bit followed by negation is
2410 // the same as flipping arithmetic/logical shift type without the negation:
2411 // -(X >>u 31) -> (X >>s 31)
2412 // -(X >>s 31) -> (X >>u 31)
2413 if (N1->getOpcode() == ISD::SRA || N1->getOpcode() == ISD::SRL) {
2414 ConstantSDNode *ShiftAmt = isConstOrConstSplat(N1.getOperand(1));
2415 if (ShiftAmt && ShiftAmt->getZExtValue() == BitWidth - 1) {
2416 auto NewSh = N1->getOpcode() == ISD::SRA ? ISD::SRL : ISD::SRA;
2417 if (!LegalOperations || TLI.isOperationLegal(NewSh, VT))
2418 return DAG.getNode(NewSh, DL, VT, N1.getOperand(0), N1.getOperand(1));
2419 }
2420 }
2421
2422 // 0 - X --> 0 if the sub is NUW.
2423 if (N->getFlags().hasNoUnsignedWrap())
2424 return N0;
2425
2426 if (DAG.MaskedValueIsZero(N1, ~APInt::getSignMask(BitWidth))) {
2427 // N1 is either 0 or the minimum signed value. If the sub is NSW, then
2428 // N1 must be 0 because negating the minimum signed value is undefined.
2429 if (N->getFlags().hasNoSignedWrap())
2430 return N0;
2431
2432 // 0 - X --> X if X is 0 or the minimum signed value.
2433 return N1;
2434 }
2435 }
2436
2437 // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1)
2438 if (isAllOnesConstantOrAllOnesSplatConstant(N0))
2439 return DAG.getNode(ISD::XOR, DL, VT, N1, N0);
2440
2441 // fold A-(A-B) -> B
2442 if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(0))
2443 return N1.getOperand(1);
2444
2445 // fold (A+B)-A -> B
2446 if (N0.getOpcode() == ISD::ADD && N0.getOperand(0) == N1)
2447 return N0.getOperand(1);
2448
2449 // fold (A+B)-B -> A
2450 if (N0.getOpcode() == ISD::ADD && N0.getOperand(1) == N1)
2451 return N0.getOperand(0);
2452
2453 // fold C2-(A+C1) -> (C2-C1)-A
2454 if (N1.getOpcode() == ISD::ADD) {
2455 SDValue N11 = N1.getOperand(1);
2456 if (isConstantOrConstantVector(N0, /* NoOpaques */ true) &&
2457 isConstantOrConstantVector(N11, /* NoOpaques */ true)) {
2458 SDValue NewC = DAG.getNode(ISD::SUB, DL, VT, N0, N11);
2459 return DAG.getNode(ISD::SUB, DL, VT, NewC, N1.getOperand(0));
2460 }
2461 }
2462
2463 // fold ((A+(B+or-C))-B) -> A+or-C
2464 if (N0.getOpcode() == ISD::ADD &&
2465 (N0.getOperand(1).getOpcode() == ISD::SUB ||
2466 N0.getOperand(1).getOpcode() == ISD::ADD) &&
2467 N0.getOperand(1).getOperand(0) == N1)
2468 return DAG.getNode(N0.getOperand(1).getOpcode(), DL, VT, N0.getOperand(0),
2469 N0.getOperand(1).getOperand(1));
2470
2471 // fold ((A+(C+B))-B) -> A+C
2472 if (N0.getOpcode() == ISD::ADD && N0.getOperand(1).getOpcode() == ISD::ADD &&
2473 N0.getOperand(1).getOperand(1) == N1)
2474 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0),
2475 N0.getOperand(1).getOperand(0));
2476
2477 // fold ((A-(B-C))-C) -> A-B
2478 if (N0.getOpcode() == ISD::SUB && N0.getOperand(1).getOpcode() == ISD::SUB &&
2479 N0.getOperand(1).getOperand(1) == N1)
2480 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0),
2481 N0.getOperand(1).getOperand(0));
2482
2483 // If either operand of a sub is undef, the result is undef
2484 if (N0.isUndef())
2485 return N0;
2486 if (N1.isUndef())
2487 return N1;
2488
2489 // If the relocation model supports it, consider symbol offsets.
2490 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N0))
2491 if (!LegalOperations && TLI.isOffsetFoldingLegal(GA)) {
2492 // fold (sub Sym, c) -> Sym-c
2493 if (N1C && GA->getOpcode() == ISD::GlobalAddress)
2494 return DAG.getGlobalAddress(GA->getGlobal(), SDLoc(N1C), VT,
2495 GA->getOffset() -
2496 (uint64_t)N1C->getSExtValue());
2497 // fold (sub Sym+c1, Sym+c2) -> c1-c2
2498 if (GlobalAddressSDNode *GB = dyn_cast<GlobalAddressSDNode>(N1))
2499 if (GA->getGlobal() == GB->getGlobal())
2500 return DAG.getConstant((uint64_t)GA->getOffset() - GB->getOffset(),
2501 DL, VT);
2502 }
2503
2504 // sub X, (sextinreg Y i1) -> add X, (and Y 1)
2505 if (N1.getOpcode() == ISD::SIGN_EXTEND_INREG) {
2506 VTSDNode *TN = cast<VTSDNode>(N1.getOperand(1));
2507 if (TN->getVT() == MVT::i1) {
2508 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0),
2509 DAG.getConstant(1, DL, VT));
2510 return DAG.getNode(ISD::ADD, DL, VT, N0, ZExt);
2511 }
2512 }
2513
2514 return SDValue();
2515}
2516
2517SDValue DAGCombiner::visitSUBC(SDNode *N) {
2518 SDValue N0 = N->getOperand(0);
2519 SDValue N1 = N->getOperand(1);
2520 EVT VT = N0.getValueType();
2521 SDLoc DL(N);
2522
2523 // If the flag result is dead, turn this into an SUB.
2524 if (!N->hasAnyUseOfValue(1))
2525 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1),
2526 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2527
2528 // fold (subc x, x) -> 0 + no borrow
2529 if (N0 == N1)
2530 return CombineTo(N, DAG.getConstant(0, DL, VT),
2531 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2532
2533 // fold (subc x, 0) -> x + no borrow
2534 if (isNullConstant(N1))
2535 return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2536
2537 // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1) + no borrow
2538 if (isAllOnesConstant(N0))
2539 return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0),
2540 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2541
2542 return SDValue();
2543}
2544
2545SDValue DAGCombiner::visitUSUBO(SDNode *N) {
2546 SDValue N0 = N->getOperand(0);
2547 SDValue N1 = N->getOperand(1);
2548 EVT VT = N0.getValueType();
2549 if (VT.isVector())
2550 return SDValue();
2551
2552 EVT CarryVT = N->getValueType(1);
2553 SDLoc DL(N);
2554
2555 // If the flag result is dead, turn this into an SUB.
2556 if (!N->hasAnyUseOfValue(1))
2557 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1),
2558 DAG.getUNDEF(CarryVT));
2559
2560 // fold (usubo x, x) -> 0 + no borrow
2561 if (N0 == N1)
2562 return CombineTo(N, DAG.getConstant(0, DL, VT),
2563 DAG.getConstant(0, DL, CarryVT));
2564
2565 // fold (usubo x, 0) -> x + no borrow
2566 if (isNullConstant(N1))
2567 return CombineTo(N, N0, DAG.getConstant(0, DL, CarryVT));
2568
2569 // Canonicalize (usubo -1, x) -> ~x, i.e. (xor x, -1) + no borrow
2570 if (isAllOnesConstant(N0))
2571 return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0),
2572 DAG.getConstant(0, DL, CarryVT));
2573
2574 return SDValue();
2575}
2576
2577SDValue DAGCombiner::visitSUBE(SDNode *N) {
2578 SDValue N0 = N->getOperand(0);
2579 SDValue N1 = N->getOperand(1);
2580 SDValue CarryIn = N->getOperand(2);
2581
2582 // fold (sube x, y, false) -> (subc x, y)
2583 if (CarryIn.getOpcode() == ISD::CARRY_FALSE)
2584 return DAG.getNode(ISD::SUBC, SDLoc(N), N->getVTList(), N0, N1);
2585
2586 return SDValue();
2587}
2588
2589SDValue DAGCombiner::visitSUBCARRY(SDNode *N) {
2590 SDValue N0 = N->getOperand(0);
2591 SDValue N1 = N->getOperand(1);
2592 SDValue CarryIn = N->getOperand(2);
2593
2594 // fold (subcarry x, y, false) -> (usubo x, y)
2595 if (isNullConstant(CarryIn))
2596 return DAG.getNode(ISD::USUBO, SDLoc(N), N->getVTList(), N0, N1);
2597
2598 return SDValue();
2599}
2600
2601SDValue DAGCombiner::visitMUL(SDNode *N) {
2602 SDValue N0 = N->getOperand(0);
2603 SDValue N1 = N->getOperand(1);
2604 EVT VT = N0.getValueType();
2605
2606 // fold (mul x, undef) -> 0
2607 if (N0.isUndef() || N1.isUndef())
2608 return DAG.getConstant(0, SDLoc(N), VT);
2609
2610 bool N0IsConst = false;
2611 bool N1IsConst = false;
2612 bool N1IsOpaqueConst = false;
2613 bool N0IsOpaqueConst = false;
2614 APInt ConstValue0, ConstValue1;
2615 // fold vector ops
2616 if (VT.isVector()) {
2617 if (SDValue FoldedVOp = SimplifyVBinOp(N))
2618 return FoldedVOp;
2619
2620 N0IsConst = ISD::isConstantSplatVector(N0.getNode(), ConstValue0);
2621 N1IsConst = ISD::isConstantSplatVector(N1.getNode(), ConstValue1);
2622 assert((!N0IsConst ||(static_cast <bool> ((!N0IsConst || ConstValue0.getBitWidth
() == VT.getScalarSizeInBits()) && "Splat APInt should be element width"
) ? void (0) : __assert_fail ("(!N0IsConst || ConstValue0.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2624, __extension__ __PRETTY_FUNCTION__))
2623 ConstValue0.getBitWidth() == VT.getScalarSizeInBits()) &&(static_cast <bool> ((!N0IsConst || ConstValue0.getBitWidth
() == VT.getScalarSizeInBits()) && "Splat APInt should be element width"
) ? void (0) : __assert_fail ("(!N0IsConst || ConstValue0.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2624, __extension__ __PRETTY_FUNCTION__))
2624 "Splat APInt should be element width")(static_cast <bool> ((!N0IsConst || ConstValue0.getBitWidth
() == VT.getScalarSizeInBits()) && "Splat APInt should be element width"
) ? void (0) : __assert_fail ("(!N0IsConst || ConstValue0.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2624, __extension__ __PRETTY_FUNCTION__))
;
2625 assert((!N1IsConst ||(static_cast <bool> ((!N1IsConst || ConstValue1.getBitWidth
() == VT.getScalarSizeInBits()) && "Splat APInt should be element width"
) ? void (0) : __assert_fail ("(!N1IsConst || ConstValue1.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2627, __extension__ __PRETTY_FUNCTION__))
2626 ConstValue1.getBitWidth() == VT.getScalarSizeInBits()) &&(static_cast <bool> ((!N1IsConst || ConstValue1.getBitWidth
() == VT.getScalarSizeInBits()) && "Splat APInt should be element width"
) ? void (0) : __assert_fail ("(!N1IsConst || ConstValue1.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2627, __extension__ __PRETTY_FUNCTION__))
2627 "Splat APInt should be element width")(static_cast <bool> ((!N1IsConst || ConstValue1.getBitWidth
() == VT.getScalarSizeInBits()) && "Splat APInt should be element width"
) ? void (0) : __assert_fail ("(!N1IsConst || ConstValue1.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2627, __extension__ __PRETTY_FUNCTION__))
;
2628 } else {
2629 N0IsConst = isa<ConstantSDNode>(N0);
2630 if (N0IsConst) {
2631 ConstValue0 = cast<ConstantSDNode>(N0)->getAPIntValue();
2632 N0IsOpaqueConst = cast<ConstantSDNode>(N0)->isOpaque();
2633 }
2634 N1IsConst = isa<ConstantSDNode>(N1);
2635 if (N1IsConst) {
2636 ConstValue1 = cast<ConstantSDNode>(N1)->getAPIntValue();
2637 N1IsOpaqueConst = cast<ConstantSDNode>(N1)->isOpaque();
2638 }
2639 }
2640
2641 // fold (mul c1, c2) -> c1*c2
2642 if (N0IsConst && N1IsConst && !N0IsOpaqueConst && !N1IsOpaqueConst)
2643 return DAG.FoldConstantArithmetic(ISD::MUL, SDLoc(N), VT,
2644 N0.getNode(), N1.getNode());
2645
2646 // canonicalize constant to RHS (vector doesn't have to splat)
2647 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
2648 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
2649 return DAG.getNode(ISD::MUL, SDLoc(N), VT, N1, N0);
2650 // fold (mul x, 0) -> 0
2651 if (N1IsConst && ConstValue1.isNullValue())
2652 return N1;
2653 // fold (mul x, 1) -> x
2654 if (N1IsConst && ConstValue1.isOneValue())
2655 return N0;
2656
2657 if (SDValue NewSel = foldBinOpIntoSelect(N))
2658 return NewSel;
2659
2660 // fold (mul x, -1) -> 0-x
2661 if (N1IsConst && ConstValue1.isAllOnesValue()) {
2662 SDLoc DL(N);
2663 return DAG.getNode(ISD::SUB, DL, VT,
2664 DAG.getConstant(0, DL, VT), N0);
2665 }
2666 // fold (mul x, (1 << c)) -> x << c
2667 if (isConstantOrConstantVector(N1, /*NoOpaques*/ true) &&
2668 DAG.isKnownToBeAPowerOfTwo(N1) &&
2669 (!VT.isVector() || Level <= AfterLegalizeVectorOps)) {
2670 SDLoc DL(N);
2671 SDValue LogBase2 = BuildLogBase2(N1, DL);
2672 AddToWorklist(LogBase2.getNode());
2673
2674 EVT ShiftVT = getShiftAmountTy(N0.getValueType());
2675 SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ShiftVT);
2676 AddToWorklist(Trunc.getNode());
2677 return DAG.getNode(ISD::SHL, DL, VT, N0, Trunc);
2678 }
2679 // fold (mul x, -(1 << c)) -> -(x << c) or (-x) << c
2680 if (N1IsConst && !N1IsOpaqueConst && (-ConstValue1).isPowerOf2()) {
2681 unsigned Log2Val = (-ConstValue1).logBase2();
2682 SDLoc DL(N);
2683 // FIXME: If the input is something that is easily negated (e.g. a
2684 // single-use add), we should put the negate there.
2685 return DAG.getNode(ISD::SUB, DL, VT,
2686 DAG.getConstant(0, DL, VT),
2687 DAG.getNode(ISD::SHL, DL, VT, N0,
2688 DAG.getConstant(Log2Val, DL,
2689 getShiftAmountTy(N0.getValueType()))));
2690 }
2691
2692 // (mul (shl X, c1), c2) -> (mul X, c2 << c1)
2693 if (N0.getOpcode() == ISD::SHL &&
2694 isConstantOrConstantVector(N1, /* NoOpaques */ true) &&
2695 isConstantOrConstantVector(N0.getOperand(1), /* NoOpaques */ true)) {
2696 SDValue C3 = DAG.getNode(ISD::SHL, SDLoc(N), VT, N1, N0.getOperand(1));
2697 if (isConstantOrConstantVector(C3))
2698 return DAG.getNode(ISD::MUL, SDLoc(N), VT, N0.getOperand(0), C3);
2699 }
2700
2701 // Change (mul (shl X, C), Y) -> (shl (mul X, Y), C) when the shift has one
2702 // use.
2703 {
2704 SDValue Sh(nullptr, 0), Y(nullptr, 0);
2705
2706 // Check for both (mul (shl X, C), Y) and (mul Y, (shl X, C)).
2707 if (N0.getOpcode() == ISD::SHL &&
2708 isConstantOrConstantVector(N0.getOperand(1)) &&
2709 N0.getNode()->hasOneUse()) {
2710 Sh = N0; Y = N1;
2711 } else if (N1.getOpcode() == ISD::SHL &&
2712 isConstantOrConstantVector(N1.getOperand(1)) &&
2713 N1.getNode()->hasOneUse()) {
2714 Sh = N1; Y = N0;
2715 }
2716
2717 if (Sh.getNode()) {
2718 SDValue Mul = DAG.getNode(ISD::MUL, SDLoc(N), VT, Sh.getOperand(0), Y);
2719 return DAG.getNode(ISD::SHL, SDLoc(N), VT, Mul, Sh.getOperand(1));
2720 }
2721 }
2722
2723 // fold (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2)
2724 if (DAG.isConstantIntBuildVectorOrConstantInt(N1) &&
2725 N0.getOpcode() == ISD::ADD &&
2726 DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1)) &&
2727 isMulAddWithConstProfitable(N, N0, N1))
2728 return DAG.getNode(ISD::ADD, SDLoc(N), VT,
2729 DAG.getNode(ISD::MUL, SDLoc(N0), VT,
2730 N0.getOperand(0), N1),
2731 DAG.getNode(ISD::MUL, SDLoc(N1), VT,
2732 N0.getOperand(1), N1));
2733
2734 // reassociate mul
2735 if (SDValue RMUL = ReassociateOps(ISD::MUL, SDLoc(N), N0, N1))
2736 return RMUL;
2737
2738 return SDValue();
2739}
2740
2741/// Return true if divmod libcall is available.
2742static bool isDivRemLibcallAvailable(SDNode *Node, bool isSigned,
2743 const TargetLowering &TLI) {
2744 RTLIB::Libcall LC;
2745 EVT NodeType = Node->getValueType(0);
2746 if (!NodeType.isSimple())
2747 return false;
2748 switch (NodeType.getSimpleVT().SimpleTy) {
2749 default: return false; // No libcall for vector types.
2750 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2751 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2752 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2753 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2754 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2755 }
2756
2757 return TLI.getLibcallName(LC) != nullptr;
2758}
2759
2760/// Issue divrem if both quotient and remainder are needed.
2761SDValue DAGCombiner::useDivRem(SDNode *Node) {
2762 if (Node->use_empty())
2763 return SDValue(); // This is a dead node, leave it alone.
2764
2765 unsigned Opcode = Node->getOpcode();
2766 bool isSigned = (Opcode == ISD::SDIV) || (Opcode == ISD::SREM);
2767 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
2768
2769 // DivMod lib calls can still work on non-legal types if using lib-calls.
2770 EVT VT = Node->getValueType(0);
2771 if (VT.isVector() || !VT.isInteger())
2772 return SDValue();
2773
2774 if (!TLI.isTypeLegal(VT) && !TLI.isOperationCustom(DivRemOpc, VT))
2775 return SDValue();
2776
2777 // If DIVREM is going to get expanded into a libcall,
2778 // but there is no libcall available, then don't combine.
2779 if (!TLI.isOperationLegalOrCustom(DivRemOpc, VT) &&
2780 !isDivRemLibcallAvailable(Node, isSigned, TLI))
2781 return SDValue();
2782
2783 // If div is legal, it's better to do the normal expansion
2784 unsigned OtherOpcode = 0;
2785 if ((Opcode == ISD::SDIV) || (Opcode == ISD::UDIV)) {
2786 OtherOpcode = isSigned ? ISD::SREM : ISD::UREM;
2787 if (TLI.isOperationLegalOrCustom(Opcode, VT))
2788 return SDValue();
2789 } else {
2790 OtherOpcode = isSigned ? ISD::SDIV : ISD::UDIV;
2791 if (TLI.isOperationLegalOrCustom(OtherOpcode, VT))
2792 return SDValue();
2793 }
2794
2795 SDValue Op0 = Node->getOperand(0);
2796 SDValue Op1 = Node->getOperand(1);
2797 SDValue combined;
2798 for (SDNode::use_iterator UI = Op0.getNode()->use_begin(),
2799 UE = Op0.getNode()->use_end(); UI != UE; ++UI) {
2800 SDNode *User = *UI;
2801 if (User == Node || User->use_empty())
2802 continue;
2803 // Convert the other matching node(s), too;
2804 // otherwise, the DIVREM may get target-legalized into something
2805 // target-specific that we won't be able to recognize.
2806 unsigned UserOpc = User->getOpcode();
2807 if ((UserOpc == Opcode || UserOpc == OtherOpcode || UserOpc == DivRemOpc) &&
2808 User->getOperand(0) == Op0 &&
2809 User->getOperand(1) == Op1) {
2810 if (!combined) {
2811 if (UserOpc == OtherOpcode) {
2812 SDVTList VTs = DAG.getVTList(VT, VT);
2813 combined = DAG.getNode(DivRemOpc, SDLoc(Node), VTs, Op0, Op1);
2814 } else if (UserOpc == DivRemOpc) {
2815 combined = SDValue(User, 0);
2816 } else {
2817 assert(UserOpc == Opcode)(static_cast <bool> (UserOpc == Opcode) ? void (0) : __assert_fail
("UserOpc == Opcode", "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2817, __extension__ __PRETTY_FUNCTION__))
;
2818 continue;
2819 }
2820 }
2821 if (UserOpc == ISD::SDIV || UserOpc == ISD::UDIV)
2822 CombineTo(User, combined);
2823 else if (UserOpc == ISD::SREM || UserOpc == ISD::UREM)
2824 CombineTo(User, combined.getValue(1));
2825 }
2826 }
2827 return combined;
2828}
2829
2830static SDValue simplifyDivRem(SDNode *N, SelectionDAG &DAG) {
2831 SDValue N0 = N->getOperand(0);
2832 SDValue N1 = N->getOperand(1);
2833 EVT VT = N->getValueType(0);
2834 SDLoc DL(N);
2835
2836 if (DAG.isUndef(N->getOpcode(), {N0, N1}))
2837 return DAG.getUNDEF(VT);
2838
2839 // undef / X -> 0
2840 // undef % X -> 0
2841 if (N0.isUndef())
2842 return DAG.getConstant(0, DL, VT);
2843
2844 return SDValue();
2845}
2846
2847SDValue DAGCombiner::visitSDIV(SDNode *N) {
2848 SDValue N0 = N->getOperand(0);
2849 SDValue N1 = N->getOperand(1);
2850 EVT VT = N->getValueType(0);
2851
2852 // fold vector ops
2853 if (VT.isVector())
2854 if (SDValue FoldedVOp = SimplifyVBinOp(N))
2855 return FoldedVOp;
2856
2857 SDLoc DL(N);
2858
2859 // fold (sdiv c1, c2) -> c1/c2
2860 ConstantSDNode *N0C = isConstOrConstSplat(N0);
2861 ConstantSDNode *N1C = isConstOrConstSplat(N1);
2862 if (N0C && N1C && !N0C->isOpaque() && !N1C->isOpaque())
2863 return DAG.FoldConstantArithmetic(ISD::SDIV, DL, VT, N0C, N1C);
2864 // fold (sdiv X, 1) -> X
2865 if (N1C && N1C->isOne())
2866 return N0;
2867 // fold (sdiv X, -1) -> 0-X
2868 if (N1C && N1C->isAllOnesValue())
2869 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), N0);
2870
2871 if (SDValue V = simplifyDivRem(N, DAG))
2872 return V;
2873
2874 if (SDValue NewSel = foldBinOpIntoSelect(N))
2875 return NewSel;
2876
2877 // If we know the sign bits of both operands are zero, strength reduce to a
2878 // udiv instead. Handles (X&15) /s 4 -> X&15 >> 2
2879 if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
2880 return DAG.getNode(ISD::UDIV, DL, N1.getValueType(), N0, N1);
2881
2882 // fold (sdiv X, pow2) -> simple ops after legalize
2883 // FIXME: We check for the exact bit here because the generic lowering gives
2884 // better results in that case. The target-specific lowering should learn how
2885 // to handle exact sdivs efficiently.
2886 if (N1C && !N1C->isNullValue() && !N1C->isOpaque() &&
2887 !N->getFlags().hasExact() && (N1C->getAPIntValue().isPowerOf2() ||
2888 (-N1C->getAPIntValue()).isPowerOf2())) {
2889 // Target-specific implementation of sdiv x, pow2.
2890 if (SDValue Res = BuildSDIVPow2(N))
2891 return Res;
2892
2893 unsigned lg2 = N1C->getAPIntValue().countTrailingZeros();
2894
2895 // Splat the sign bit into the register
2896 SDValue SGN =
2897 DAG.getNode(ISD::SRA, DL, VT, N0,
2898 DAG.getConstant(VT.getScalarSizeInBits() - 1, DL,
2899 getShiftAmountTy(N0.getValueType())));
2900 AddToWorklist(SGN.getNode());
2901
2902 // Add (N0 < 0) ? abs2 - 1 : 0;
2903 SDValue SRL =
2904 DAG.getNode(ISD::SRL, DL, VT, SGN,
2905 DAG.getConstant(VT.getScalarSizeInBits() - lg2, DL,
2906 getShiftAmountTy(SGN.getValueType())));
2907 SDValue ADD = DAG.getNode(ISD::ADD, DL, VT, N0, SRL);
2908 AddToWorklist(SRL.getNode());
2909 AddToWorklist(ADD.getNode()); // Divide by pow2
2910 SDValue SRA = DAG.getNode(ISD::SRA, DL, VT, ADD,
2911 DAG.getConstant(lg2, DL,
2912 getShiftAmountTy(ADD.getValueType())));
2913
2914 // If we're dividing by a positive value, we're done. Otherwise, we must
2915 // negate the result.
2916 if (N1C->getAPIntValue().isNonNegative())
2917 return SRA;
2918
2919 AddToWorklist(SRA.getNode());
2920 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), SRA);
2921 }
2922
2923 // If integer divide is expensive and we satisfy the requirements, emit an
2924 // alternate sequence. Targets may check function attributes for size/speed
2925 // trade-offs.
2926 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
2927 if (N1C && !TLI.isIntDivCheap(N->getValueType(0), Attr))
2928 if (SDValue Op = BuildSDIV(N))
2929 return Op;
2930
2931 // sdiv, srem -> sdivrem
2932 // If the divisor is constant, then return DIVREM only if isIntDivCheap() is
2933 // true. Otherwise, we break the simplification logic in visitREM().
2934 if (!N1C || TLI.isIntDivCheap(N->getValueType(0), Attr))
2935 if (SDValue DivRem = useDivRem(N))
2936 return DivRem;
2937
2938 return SDValue();
2939}
2940
2941SDValue DAGCombiner::visitUDIV(SDNode *N) {
2942 SDValue N0 = N->getOperand(0);
2943 SDValue N1 = N->getOperand(1);
2944 EVT VT = N->getValueType(0);
2945
2946 // fold vector ops
2947 if (VT.isVector())
2948 if (SDValue FoldedVOp = SimplifyVBinOp(N))
2949 return FoldedVOp;
2950
2951 SDLoc DL(N);
2952
2953 // fold (udiv c1, c2) -> c1/c2
2954 ConstantSDNode *N0C = isConstOrConstSplat(N0);
2955 ConstantSDNode *N1C = isConstOrConstSplat(N1);
2956 if (N0C && N1C)
2957 if (SDValue Folded = DAG.FoldConstantArithmetic(ISD::UDIV, DL, VT,
2958 N0C, N1C))
2959 return Folded;
2960
2961 if (SDValue V = simplifyDivRem(N, DAG))
2962 return V;
2963
2964 if (SDValue NewSel = foldBinOpIntoSelect(N))
2965 return NewSel;
2966
2967 // fold (udiv x, (1 << c)) -> x >>u c
2968 if (isConstantOrConstantVector(N1, /*NoOpaques*/ true) &&
2969 DAG.isKnownToBeAPowerOfTwo(N1)) {
2970 SDValue LogBase2 = BuildLogBase2(N1, DL);
2971 AddToWorklist(LogBase2.getNode());
2972
2973 EVT ShiftVT = getShiftAmountTy(N0.getValueType());
2974 SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ShiftVT);
2975 AddToWorklist(Trunc.getNode());
2976 return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc);
2977 }
2978
2979 // fold (udiv x, (shl c, y)) -> x >>u (log2(c)+y) iff c is power of 2
2980 if (N1.getOpcode() == ISD::SHL) {
2981 SDValue N10 = N1.getOperand(0);
2982 if (isConstantOrConstantVector(N10, /*NoOpaques*/ true) &&
2983 DAG.isKnownToBeAPowerOfTwo(N10)) {
2984 SDValue LogBase2 = BuildLogBase2(N10, DL);
2985 AddToWorklist(LogBase2.getNode());
2986
2987 EVT ADDVT = N1.getOperand(1).getValueType();
2988 SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ADDVT);
2989 AddToWorklist(Trunc.getNode());
2990 SDValue Add = DAG.getNode(ISD::ADD, DL, ADDVT, N1.getOperand(1), Trunc);
2991 AddToWorklist(Add.getNode());
2992 return DAG.getNode(ISD::SRL, DL, VT, N0, Add);
2993 }
2994 }
2995
2996 // fold (udiv x, c) -> alternate
2997 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
2998 if (N1C && !TLI.isIntDivCheap(N->getValueType(0), Attr))
2999 if (SDValue Op = BuildUDIV(N))
3000 return Op;
3001
3002 // sdiv, srem -> sdivrem
3003 // If the divisor is constant, then return DIVREM only if isIntDivCheap() is
3004 // true. Otherwise, we break the simplification logic in visitREM().
3005 if (!N1C || TLI.isIntDivCheap(N->getValueType(0), Attr))
3006 if (SDValue DivRem = useDivRem(N))
3007 return DivRem;
3008
3009 return SDValue();
3010}
3011
3012// handles ISD::SREM and ISD::UREM
3013SDValue DAGCombiner::visitREM(SDNode *N) {
3014 unsigned Opcode = N->getOpcode();
3015 SDValue N0 = N->getOperand(0);
3016 SDValue N1 = N->getOperand(1);
3017 EVT VT = N->getValueType(0);
3018 bool isSigned = (Opcode == ISD::SREM);
3019 SDLoc DL(N);
3020
3021 // fold (rem c1, c2) -> c1%c2
3022 ConstantSDNode *N0C = isConstOrConstSplat(N0);
3023 ConstantSDNode *N1C = isConstOrConstSplat(N1);
3024 if (N0C && N1C)
3025 if (SDValue Folded = DAG.FoldConstantArithmetic(Opcode, DL, VT, N0C, N1C))
3026 return Folded;
3027
3028 if (SDValue V = simplifyDivRem(N, DAG))
3029 return V;
3030
3031 if (SDValue NewSel = foldBinOpIntoSelect(N))
3032 return NewSel;
3033
3034 if (isSigned) {
3035 // If we know the sign bits of both operands are zero, strength reduce to a
3036 // urem instead. Handles (X & 0x0FFFFFFF) %s 16 -> X&15
3037 if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
3038 return DAG.getNode(ISD::UREM, DL, VT, N0, N1);
3039 } else {
3040 SDValue NegOne = DAG.getAllOnesConstant(DL, VT);
3041 if (DAG.isKnownToBeAPowerOfTwo(N1)) {
3042 // fold (urem x, pow2) -> (and x, pow2-1)
3043 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne);
3044 AddToWorklist(Add.getNode());
3045 return DAG.getNode(ISD::AND, DL, VT, N0, Add);
3046 }
3047 if (N1.getOpcode() == ISD::SHL &&
3048 DAG.isKnownToBeAPowerOfTwo(N1.getOperand(0))) {
3049 // fold (urem x, (shl pow2, y)) -> (and x, (add (shl pow2, y), -1))
3050 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne);
3051 AddToWorklist(Add.getNode());
3052 return DAG.getNode(ISD::AND, DL, VT, N0, Add);
3053 }
3054 }
3055
3056 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
3057
3058 // If X/C can be simplified by the division-by-constant logic, lower
3059 // X%C to the equivalent of X-X/C*C.
3060 // To avoid mangling nodes, this simplification requires that the combine()
3061 // call for the speculative DIV must not cause a DIVREM conversion. We guard
3062 // against this by skipping the simplification if isIntDivCheap(). When
3063 // div is not cheap, combine will not return a DIVREM. Regardless,
3064 // checking cheapness here makes sense since the simplification results in
3065 // fatter code.
3066 if (N1C && !N1C->isNullValue() && !TLI.isIntDivCheap(VT, Attr)) {
3067 unsigned DivOpcode = isSigned ? ISD::SDIV : ISD::UDIV;
3068 SDValue Div = DAG.getNode(DivOpcode, DL, VT, N0, N1);
3069 AddToWorklist(Div.getNode());
3070 SDValue OptimizedDiv = combine(Div.getNode());
3071 if (OptimizedDiv.getNode() && OptimizedDiv.getNode() != Div.getNode()) {
3072 assert((OptimizedDiv.getOpcode() != ISD::UDIVREM) &&(static_cast <bool> ((OptimizedDiv.getOpcode() != ISD::
UDIVREM) && (OptimizedDiv.getOpcode() != ISD::SDIVREM
)) ? void (0) : __assert_fail ("(OptimizedDiv.getOpcode() != ISD::UDIVREM) && (OptimizedDiv.getOpcode() != ISD::SDIVREM)"
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3073, __extension__ __PRETTY_FUNCTION__))
3073 (OptimizedDiv.getOpcode() != ISD::SDIVREM))(static_cast <bool> ((OptimizedDiv.getOpcode() != ISD::
UDIVREM) && (OptimizedDiv.getOpcode() != ISD::SDIVREM
)) ? void (0) : __assert_fail ("(OptimizedDiv.getOpcode() != ISD::UDIVREM) && (OptimizedDiv.getOpcode() != ISD::SDIVREM)"
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3073, __extension__ __PRETTY_FUNCTION__))
;
3074 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, OptimizedDiv, N1);
3075 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
3076 AddToWorklist(Mul.getNode());
3077 return Sub;
3078 }
3079 }
3080
3081 // sdiv, srem -> sdivrem
3082 if (SDValue DivRem = useDivRem(N))
3083 return DivRem.getValue(1);
3084
3085 return SDValue();
3086}
3087
3088SDValue DAGCombiner::visitMULHS(SDNode *N) {
3089 SDValue N0 = N->getOperand(0);
3090 SDValue N1 = N->getOperand(1);
3091 EVT VT = N->getValueType(0);
3092 SDLoc DL(N);
3093
3094 if (VT.isVector()) {
3095 // fold (mulhs x, 0) -> 0
3096 if (ISD::isBuildVectorAllZeros(N1.getNode()))
3097 return N1;
3098 if (ISD::isBuildVectorAllZeros(N0.getNode()))
3099 return N0;
3100 }
3101
3102 // fold (mulhs x, 0) -> 0
3103 if (isNullConstant(N1))
3104 return N1;
3105 // fold (mulhs x, 1) -> (sra x, size(x)-1)
3106 if (isOneConstant(N1))
3107 return DAG.getNode(ISD::SRA, DL, N0.getValueType(), N0,
3108 DAG.getConstant(N0.getValueSizeInBits() - 1, DL,
3109 getShiftAmountTy(N0.getValueType())));
3110
3111 // fold (mulhs x, undef) -> 0
3112 if (N0.isUndef() || N1.isUndef())
3113 return DAG.getConstant(0, DL, VT);
3114
3115 // If the type twice as wide is legal, transform the mulhs to a wider multiply
3116 // plus a shift.
3117 if (VT.isSimple() && !VT.isVector()) {
3118 MVT Simple = VT.getSimpleVT();
3119 unsigned SimpleSize = Simple.getSizeInBits();
3120 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
3121 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
3122 N0 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N0);
3123 N1 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N1);
3124 N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
3125 N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
3126 DAG.getConstant(SimpleSize, DL,
3127 getShiftAmountTy(N1.getValueType())));
3128 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
3129 }
3130 }
3131
3132 return SDValue();
3133}
3134
3135SDValue DAGCombiner::visitMULHU(SDNode *N) {
3136 SDValue N0 = N->getOperand(0);
3137 SDValue N1 = N->getOperand(1);
3138 EVT VT = N->getValueType(0);
3139 SDLoc DL(N);
3140
3141 if (VT.isVector()) {
3142 // fold (mulhu x, 0) -> 0
3143 if (ISD::isBuildVectorAllZeros(N1.getNode()))
3144 return N1;
3145 if (ISD::isBuildVectorAllZeros(N0.getNode()))
3146 return N0;
3147 }
3148
3149 // fold (mulhu x, 0) -> 0
3150 if (isNullConstant(N1))
3151 return N1;
3152 // fold (mulhu x, 1) -> 0
3153 if (isOneConstant(N1))
3154 return DAG.getConstant(0, DL, N0.getValueType());
3155 // fold (mulhu x, undef) -> 0
3156 if (N0.isUndef() || N1.isUndef())
3157 return DAG.getConstant(0, DL, VT);
3158
3159 // If the type twice as wide is legal, transform the mulhu to a wider multiply
3160 // plus a shift.
3161 if (VT.isSimple() && !VT.isVector()) {
3162 MVT Simple = VT.getSimpleVT();
3163 unsigned SimpleSize = Simple.getSizeInBits();
3164 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
3165 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
3166 N0 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N0);
3167 N1 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N1);
3168 N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
3169 N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
3170 DAG.getConstant(SimpleSize, DL,
3171 getShiftAmountTy(N1.getValueType())));
3172 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
3173 }
3174 }
3175
3176 return SDValue();
3177}
3178
3179/// Perform optimizations common to nodes that compute two values. LoOp and HiOp
3180/// give the opcodes for the two computations that are being performed. Return
3181/// true if a simplification was made.
3182SDValue DAGCombiner::SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
3183 unsigned HiOp) {
3184 // If the high half is not needed, just compute the low half.
3185 bool HiExists = N->hasAnyUseOfValue(1);
3186 if (!HiExists &&
3187 (!LegalOperations ||
3188 TLI.isOperationLegalOrCustom(LoOp, N->getValueType(0)))) {
3189 SDValue Res = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0), N->ops());
3190 return CombineTo(N, Res, Res);
3191 }
3192
3193 // If the low half is not needed, just compute the high half.
3194 bool LoExists = N->hasAnyUseOfValue(0);
3195 if (!LoExists &&
3196 (!LegalOperations ||
3197 TLI.isOperationLegal(HiOp, N->getValueType(1)))) {
3198 SDValue Res = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1), N->ops());
3199 return CombineTo(N, Res, Res);
3200 }
3201
3202 // If both halves are used, return as it is.
3203 if (LoExists && HiExists)
3204 return SDValue();
3205
3206 // If the two computed results can be simplified separately, separate them.
3207 if (LoExists) {
3208 SDValue Lo = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0), N->ops());
3209 AddToWorklist(Lo.getNode());
3210 SDValue LoOpt = combine(Lo.getNode());
3211 if (LoOpt.getNode() && LoOpt.getNode() != Lo.getNode() &&
3212 (!LegalOperations ||
3213 TLI.isOperationLegal(LoOpt.getOpcode(), LoOpt.getValueType())))
3214 return CombineTo(N, LoOpt, LoOpt);
3215 }
3216
3217 if (HiExists) {
3218 SDValue Hi = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1), N->ops());
3219 AddToWorklist(Hi.getNode());
3220 SDValue HiOpt = combine(Hi.getNode());
3221 if (HiOpt.getNode() && HiOpt != Hi &&
3222 (!LegalOperations ||
3223 TLI.isOperationLegal(HiOpt.getOpcode(), HiOpt.getValueType())))
3224 return CombineTo(N, HiOpt, HiOpt);
3225 }
3226
3227 return SDValue();
3228}
3229
3230SDValue DAGCombiner::visitSMUL_LOHI(SDNode *N) {
3231 if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHS))
3232 return Res;
3233
3234 EVT VT = N->getValueType(0);
3235 SDLoc DL(N);
3236
3237 // If the type is twice as wide is legal, transform the mulhu to a wider
3238 // multiply plus a shift.
3239 if (VT.isSimple() && !VT.isVector()) {
3240 MVT Simple = VT.getSimpleVT();
3241 unsigned SimpleSize = Simple.getSizeInBits();
3242 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
3243 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
3244 SDValue Lo = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(0));
3245 SDValue Hi = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(1));
3246 Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
3247 // Compute the high part as N1.
3248 Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
3249 DAG.getConstant(SimpleSize, DL,
3250 getShiftAmountTy(Lo.getValueType())));
3251 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
3252 // Compute the low part as N0.
3253 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
3254 return CombineTo(N, Lo, Hi);
3255 }
3256 }
3257
3258 return SDValue();
3259}
3260
3261SDValue DAGCombiner::visitUMUL_LOHI(SDNode *N) {
3262 if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHU))
3263 return Res;
3264
3265 EVT VT = N->getValueType(0);
3266 SDLoc DL(N);
3267
3268 // If the type is twice as wide is legal, transform the mulhu to a wider
3269 // multiply plus a shift.
3270 if (VT.isSimple() && !VT.isVector()) {
3271 MVT Simple = VT.getSimpleVT();
3272 unsigned SimpleSize = Simple.getSizeInBits();
3273 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
3274 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
3275 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(0));
3276 SDValue Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(1));
3277 Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
3278 // Compute the high part as N1.
3279 Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
3280 DAG.getConstant(SimpleSize, DL,
3281 getShiftAmountTy(Lo.getValueType())));
3282 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
3283 // Compute the low part as N0.
3284 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
3285 return CombineTo(N, Lo, Hi);
3286 }
3287 }
3288
3289 return SDValue();
3290}
3291
3292SDValue DAGCombiner::visitSMULO(SDNode *N) {
3293 // (smulo x, 2) -> (saddo x, x)
3294 if (ConstantSDNode *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1)))
3295 if (C2->getAPIntValue() == 2)
3296 return DAG.getNode(ISD::SADDO, SDLoc(N), N->getVTList(),
3297 N->getOperand(0), N->getOperand(0));
3298
3299 return SDValue();
3300}
3301
3302SDValue DAGCombiner::visitUMULO(SDNode *N) {
3303 // (umulo x, 2) -> (uaddo x, x)
3304 if (ConstantSDNode *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1)))
3305 if (C2->getAPIntValue() == 2)
3306 return DAG.getNode(ISD::UADDO, SDLoc(N), N->getVTList(),
3307 N->getOperand(0), N->getOperand(0));
3308
3309 return SDValue();
3310}
3311
3312SDValue DAGCombiner::visitIMINMAX(SDNode *N) {
3313 SDValue N0 = N->getOperand(0);
3314 SDValue N1 = N->getOperand(1);
3315 EVT VT = N0.getValueType();
3316
3317 // fold vector ops
3318 if (VT.isVector())
3319 if (SDValue FoldedVOp = SimplifyVBinOp(N))
3320 return FoldedVOp;
3321
3322 // fold operation with constant operands.
3323 ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
3324 ConstantSDNode *N1C = getAsNonOpaqueConstant(N1);
3325 if (N0C && N1C)
3326 return DAG.FoldConstantArithmetic(N->getOpcode(), SDLoc(N), VT, N0C, N1C);
3327
3328 // canonicalize constant to RHS
3329 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
3330 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
3331 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N1, N0);
3332
3333 // Is sign bits are zero, flip between UMIN/UMAX and SMIN/SMAX.
3334 // Only do this if the current op isn't legal and the flipped is.
3335 unsigned Opcode = N->getOpcode();
3336 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3337 if (!TLI.isOperationLegal(Opcode, VT) &&
3338 (N0.isUndef() || DAG.SignBitIsZero(N0)) &&
3339 (N1.isUndef() || DAG.SignBitIsZero(N1))) {
3340 unsigned AltOpcode;
3341 switch (Opcode) {
3342 case ISD::SMIN: AltOpcode = ISD::UMIN; break;
3343 case ISD::SMAX: AltOpcode = ISD::UMAX; break;
3344 case ISD::UMIN: AltOpcode = ISD::SMIN; break;
3345 case ISD::UMAX: AltOpcode = ISD::SMAX; break;
3346 default: llvm_unreachable("Unknown MINMAX opcode")::llvm::llvm_unreachable_internal("Unknown MINMAX opcode", "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3346)
;
3347 }
3348 if (TLI.isOperationLegal(AltOpcode, VT))
3349 return DAG.getNode(AltOpcode, SDLoc(N), VT, N0, N1);
3350 }
3351
3352 return SDValue();
3353}
3354
3355/// If this is a binary operator with two operands of the same opcode, try to
3356/// simplify it.
3357SDValue DAGCombiner::SimplifyBinOpWithSameOpcodeHands(SDNode *N) {
3358 SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
3359 EVT VT = N0.getValueType();
3360 assert(N0.getOpcode() == N1.getOpcode() && "Bad input!")(static_cast <bool> (N0.getOpcode() == N1.getOpcode() &&
"Bad input!") ? void (0) : __assert_fail ("N0.getOpcode() == N1.getOpcode() && \"Bad input!\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3360, __extension__ __PRETTY_FUNCTION__))
;
3361
3362 // Bail early if none of these transforms apply.
3363 if (N0.getNumOperands() == 0) return SDValue();
3364
3365 // For each of OP in AND/OR/XOR:
3366 // fold (OP (zext x), (zext y)) -> (zext (OP x, y))
3367 // fold (OP (sext x), (sext y)) -> (sext (OP x, y))
3368 // fold (OP (aext x), (aext y)) -> (aext (OP x, y))
3369 // fold (OP (bswap x), (bswap y)) -> (bswap (OP x, y))
3370 // fold (OP (trunc x), (trunc y)) -> (trunc (OP x, y)) (if trunc isn't free)
3371 //
3372 // do not sink logical op inside of a vector extend, since it may combine
3373 // into a vsetcc.
3374 EVT Op0VT = N0.getOperand(0).getValueType();
3375 if ((N0.getOpcode() == ISD::ZERO_EXTEND ||
3376 N0.getOpcode() == ISD::SIGN_EXTEND ||
3377 N0.getOpcode() == ISD::BSWAP ||
3378 // Avoid infinite looping with PromoteIntBinOp.
3379 (N0.getOpcode() == ISD::ANY_EXTEND &&
3380 (!LegalTypes || TLI.isTypeDesirableForOp(N->getOpcode(), Op0VT))) ||
3381 (N0.getOpcode() == ISD::TRUNCATE &&
3382 (!TLI.isZExtFree(VT, Op0VT) ||
3383 !TLI.isTruncateFree(Op0VT, VT)) &&
3384 TLI.isTypeLegal(Op0VT))) &&
3385 !VT.isVector() &&
3386 Op0VT == N1.getOperand(0).getValueType() &&
3387 (!LegalOperations || TLI.isOperationLegal(N->getOpcode(), Op0VT))) {
3388 SDValue ORNode = DAG.getNode(N->getOpcode(), SDLoc(N0),
3389 N0.getOperand(0).getValueType(),
3390 N0.getOperand(0), N1.getOperand(0));
3391 AddToWorklist(ORNode.getNode());
3392 return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, ORNode);
3393 }
3394
3395 // For each of OP in SHL/SRL/SRA/AND...
3396 // fold (and (OP x, z), (OP y, z)) -> (OP (and x, y), z)
3397 // fold (or (OP x, z), (OP y, z)) -> (OP (or x, y), z)
3398 // fold (xor (OP x, z), (OP y, z)) -> (OP (xor x, y), z)
3399 if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL ||
3400 N0.getOpcode() == ISD::SRA || N0.getOpcode() == ISD::AND) &&
3401 N0.getOperand(1) == N1.getOperand(1)) {
3402 SDValue ORNode = DAG.getNode(N->getOpcode(), SDLoc(N0),
3403 N0.getOperand(0).getValueType(),
3404 N0.getOperand(0), N1.getOperand(0));
3405 AddToWorklist(ORNode.getNode());
3406 return DAG.getNode(N0.getOpcode(), SDLoc(N), VT,
3407 ORNode, N0.getOperand(1));
3408 }
3409
3410 // Simplify xor/and/or (bitcast(A), bitcast(B)) -> bitcast(op (A,B))
3411 // Only perform this optimization up until type legalization, before
3412 // LegalizeVectorOprs. LegalizeVectorOprs promotes vector operations by
3413 // adding bitcasts. For example (xor v4i32) is promoted to (v2i64), and
3414 // we don't want to undo this promotion.
3415 // We also handle SCALAR_TO_VECTOR because xor/or/and operations are cheaper
3416 // on scalars.
3417 if ((N0.getOpcode() == ISD::BITCAST ||
3418 N0.getOpcode() == ISD::SCALAR_TO_VECTOR) &&
3419 Level <= AfterLegalizeTypes) {
3420 SDValue In0 = N0.getOperand(0);
3421 SDValue In1 = N1.getOperand(0);
3422 EVT In0Ty = In0.getValueType();
3423 EVT In1Ty = In1.getValueType();
3424 SDLoc DL(N);
3425 // If both incoming values are integers, and the original types are the
3426 // same.
3427 if (In0Ty.isInteger() && In1Ty.isInteger() && In0Ty == In1Ty) {
3428 SDValue Op = DAG.getNode(N->getOpcode(), DL, In0Ty, In0, In1);
3429 SDValue BC = DAG.getNode(N0.getOpcode(), DL, VT, Op);
3430 AddToWorklist(Op.getNode());
3431 return BC;
3432 }
3433 }
3434
3435 // Xor/and/or are indifferent to the swizzle operation (shuffle of one value).
3436 // Simplify xor/and/or (shuff(A), shuff(B)) -> shuff(op (A,B))
3437 // If both shuffles use the same mask, and both shuffle within a single
3438 // vector, then it is worthwhile to move the swizzle after the operation.
3439 // The type-legalizer generates this pattern when loading illegal
3440 // vector types from memory. In many cases this allows additional shuffle
3441 // optimizations.
3442 // There are other cases where moving the shuffle after the xor/and/or
3443 // is profitable even if shuffles don't perform a swizzle.
3444 // If both shuffles use the same mask, and both shuffles have the same first
3445 // or second operand, then it might still be profitable to move the shuffle
3446 // after the xor/and/or operation.
3447 if (N0.getOpcode() == ISD::VECTOR_SHUFFLE && Level < AfterLegalizeDAG) {
3448 ShuffleVectorSDNode *SVN0 = cast<ShuffleVectorSDNode>(N0);
3449 ShuffleVectorSDNode *SVN1 = cast<ShuffleVectorSDNode>(N1);
3450
3451 assert(N0.getOperand(0).getValueType() == N1.getOperand(0).getValueType() &&(static_cast <bool> (N0.getOperand(0).getValueType() ==
N1.getOperand(0).getValueType() && "Inputs to shuffles are not the same type"
) ? void (0) : __assert_fail ("N0.getOperand(0).getValueType() == N1.getOperand(0).getValueType() && \"Inputs to shuffles are not the same type\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3452, __extension__ __PRETTY_FUNCTION__))
3452 "Inputs to shuffles are not the same type")(static_cast <bool> (N0.getOperand(0).getValueType() ==
N1.getOperand(0).getValueType() && "Inputs to shuffles are not the same type"
) ? void (0) : __assert_fail ("N0.getOperand(0).getValueType() == N1.getOperand(0).getValueType() && \"Inputs to shuffles are not the same type\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3452, __extension__ __PRETTY_FUNCTION__))
;
3453
3454 // Check that both shuffles use the same mask. The masks are known to be of
3455 // the same length because the result vector type is the same.
3456 // Check also that shuffles have only one use to avoid introducing extra
3457 // instructions.
3458 if (SVN0->hasOneUse() && SVN1->hasOneUse() &&
3459 SVN0->getMask().equals(SVN1->getMask())) {
3460 SDValue ShOp = N0->getOperand(1);
3461
3462 // Don't try to fold this node if it requires introducing a
3463 // build vector of all zeros that might be illegal at this stage.
3464 if (N->getOpcode() == ISD::XOR && !ShOp.isUndef()) {
3465 if (!LegalTypes)
3466 ShOp = DAG.getConstant(0, SDLoc(N), VT);
3467 else
3468 ShOp = SDValue();
3469 }
3470
3471 // (AND (shuf (A, C), shuf (B, C)) -> shuf (AND (A, B), C)
3472 // (OR (shuf (A, C), shuf (B, C)) -> shuf (OR (A, B), C)
3473 // (XOR (shuf (A, C), shuf (B, C)) -> shuf (XOR (A, B), V_0)
3474 if (N0.getOperand(1) == N1.getOperand(1) && ShOp.getNode()) {
3475 SDValue NewNode = DAG.getNode(N->getOpcode(), SDLoc(N), VT,
3476 N0->getOperand(0), N1->getOperand(0));
3477 AddToWorklist(NewNode.getNode());
3478 return DAG.getVectorShuffle(VT, SDLoc(N), NewNode, ShOp,
3479 SVN0->getMask());
3480 }
3481
3482 // Don't try to fold this node if it requires introducing a
3483 // build vector of all zeros that might be illegal at this stage.
3484 ShOp = N0->getOperand(0);
3485 if (N->getOpcode() == ISD::XOR && !ShOp.isUndef()) {
3486 if (!LegalTypes)
3487 ShOp = DAG.getConstant(0, SDLoc(N), VT);
3488 else
3489 ShOp = SDValue();
3490 }
3491
3492 // (AND (shuf (C, A), shuf (C, B)) -> shuf (C, AND (A, B))
3493 // (OR (shuf (C, A), shuf (C, B)) -> shuf (C, OR (A, B))
3494 // (XOR (shuf (C, A), shuf (C, B)) -> shuf (V_0, XOR (A, B))
3495 if (N0->getOperand(0) == N1->getOperand(0) && ShOp.getNode()) {
3496 SDValue NewNode = DAG.getNode(N->getOpcode(), SDLoc(N), VT,
3497 N0->getOperand(1), N1->getOperand(1));
3498 AddToWorklist(NewNode.getNode());
3499 return DAG.getVectorShuffle(VT, SDLoc(N), ShOp, NewNode,
3500 SVN0->getMask());
3501 }
3502 }
3503 }
3504
3505 return SDValue();
3506}
3507
3508/// Try to make (and/or setcc (LL, LR), setcc (RL, RR)) more efficient.
3509SDValue DAGCombiner::foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1,
3510 const SDLoc &DL) {
3511 SDValue LL, LR, RL, RR, N0CC, N1CC;
3512 if (!isSetCCEquivalent(N0, LL, LR, N0CC) ||
3513 !isSetCCEquivalent(N1, RL, RR, N1CC))
3514 return SDValue();
3515
3516 assert(N0.getValueType() == N1.getValueType() &&(static_cast <bool> (N0.getValueType() == N1.getValueType
() && "Unexpected operand types for bitwise logic op"
) ? void (0) : __assert_fail ("N0.getValueType() == N1.getValueType() && \"Unexpected operand types for bitwise logic op\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3517, __extension__ __PRETTY_FUNCTION__))
3517 "Unexpected operand types for bitwise logic op")(static_cast <bool> (N0.getValueType() == N1.getValueType
() && "Unexpected operand types for bitwise logic op"
) ? void (0) : __assert_fail ("N0.getValueType() == N1.getValueType() && \"Unexpected operand types for bitwise logic op\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3517, __extension__ __PRETTY_FUNCTION__))
;
3518 assert(LL.getValueType() == LR.getValueType() &&(static_cast <bool> (LL.getValueType() == LR.getValueType
() && RL.getValueType() == RR.getValueType() &&
"Unexpected operand types for setcc") ? void (0) : __assert_fail
("LL.getValueType() == LR.getValueType() && RL.getValueType() == RR.getValueType() && \"Unexpected operand types for setcc\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3520, __extension__ __PRETTY_FUNCTION__))
3519 RL.getValueType() == RR.getValueType() &&(static_cast <bool> (LL.getValueType() == LR.getValueType
() && RL.getValueType() == RR.getValueType() &&
"Unexpected operand types for setcc") ? void (0) : __assert_fail
("LL.getValueType() == LR.getValueType() && RL.getValueType() == RR.getValueType() && \"Unexpected operand types for setcc\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3520, __extension__ __PRETTY_FUNCTION__))
3520 "Unexpected operand types for setcc")(static_cast <bool> (LL.getValueType() == LR.getValueType
() && RL.getValueType() == RR.getValueType() &&
"Unexpected operand types for setcc") ? void (0) : __assert_fail
("LL.getValueType() == LR.getValueType() && RL.getValueType() == RR.getValueType() && \"Unexpected operand types for setcc\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3520, __extension__ __PRETTY_FUNCTION__))
;
3521
3522 // If we're here post-legalization or the logic op type is not i1, the logic
3523 // op type must match a setcc result type. Also, all folds require new
3524 // operations on the left and right operands, so those types must match.
3525 EVT VT = N0.getValueType();
3526 EVT OpVT = LL.getValueType();
3527 if (LegalOperations || VT.getScalarType() != MVT::i1)
3528 if (VT != getSetCCResultType(OpVT))
3529 return SDValue();
3530 if (OpVT != RL.getValueType())
3531 return SDValue();
3532
3533 ISD::CondCode CC0 = cast<CondCodeSDNode>(N0CC)->get();
3534 ISD::CondCode CC1 = cast<CondCodeSDNode>(N1CC)->get();
3535 bool IsInteger = OpVT.isInteger();
3536 if (LR == RR && CC0 == CC1 && IsInteger) {
3537 bool IsZero = isNullConstantOrNullSplatConstant(LR);
3538 bool IsNeg1 = isAllOnesConstantOrAllOnesSplatConstant(LR);
3539
3540 // All bits clear?
3541 bool AndEqZero = IsAnd && CC1 == ISD::SETEQ && IsZero;
3542 // All sign bits clear?
3543 bool AndGtNeg1 = IsAnd && CC1 == ISD::SETGT && IsNeg1;
3544 // Any bits set?
3545 bool OrNeZero = !IsAnd && CC1 == ISD::SETNE && IsZero;
3546 // Any sign bits set?
3547 bool OrLtZero = !IsAnd && CC1 == ISD::SETLT && IsZero;
3548
3549 // (and (seteq X, 0), (seteq Y, 0)) --> (seteq (or X, Y), 0)
3550 // (and (setgt X, -1), (setgt Y, -1)) --> (setgt (or X, Y), -1)
3551 // (or (setne X, 0), (setne Y, 0)) --> (setne (or X, Y), 0)
3552 // (or (setlt X, 0), (setlt Y, 0)) --> (setlt (or X, Y), 0)
3553 if (AndEqZero || AndGtNeg1 || OrNeZero || OrLtZero) {
3554 SDValue Or = DAG.getNode(ISD::OR, SDLoc(N0), OpVT, LL, RL);
3555 AddToWorklist(Or.getNode());
3556 return DAG.getSetCC(DL, VT, Or, LR, CC1);
3557 }
3558
3559 // All bits set?
3560 bool AndEqNeg1 = IsAnd && CC1 == ISD::SETEQ && IsNeg1;
3561 // All sign bits set?
3562 bool AndLtZero = IsAnd && CC1 == ISD::SETLT && IsZero;
3563 // Any bits clear?
3564 bool OrNeNeg1 = !IsAnd && CC1 == ISD::SETNE && IsNeg1;
3565 // Any sign bits clear?
3566 bool OrGtNeg1 = !IsAnd && CC1 == ISD::SETGT && IsNeg1;
3567
3568 // (and (seteq X, -1), (seteq Y, -1)) --> (seteq (and X, Y), -1)
3569 // (and (setlt X, 0), (setlt Y, 0)) --> (setlt (and X, Y), 0)
3570 // (or (setne X, -1), (setne Y, -1)) --> (setne (and X, Y), -1)
3571 // (or (setgt X, -1), (setgt Y -1)) --> (setgt (and X, Y), -1)
3572 if (AndEqNeg1 || AndLtZero || OrNeNeg1 || OrGtNeg1) {
3573 SDValue And = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, LL, RL);
3574 AddToWorklist(And.getNode());
3575 return DAG.getSetCC(DL, VT, And, LR, CC1);
3576 }
3577 }
3578
3579 // TODO: What is the 'or' equivalent of this fold?
3580 // (and (setne X, 0), (setne X, -1)) --> (setuge (add X, 1), 2)
3581 if (IsAnd && LL == RL && CC0 == CC1 && OpVT.getScalarSizeInBits() > 1 &&
3582 IsInteger && CC0 == ISD::SETNE &&
3583 ((isNullConstant(LR) && isAllOnesConstant(RR)) ||
3584 (isAllOnesConstant(LR) && isNullConstant(RR)))) {
3585 SDValue One = DAG.getConstant(1, DL, OpVT);
3586 SDValue Two = DAG.getConstant(2, DL, OpVT);
3587 SDValue Add = DAG.getNode(ISD::ADD, SDLoc(N0), OpVT, LL, One);
3588 AddToWorklist(Add.getNode());
3589 return DAG.getSetCC(DL, VT, Add, Two, ISD::SETUGE);
3590 }
3591
3592 // Try more general transforms if the predicates match and the only user of
3593 // the compares is the 'and' or 'or'.
3594 if (IsInteger && TLI.convertSetCCLogicToBitwiseLogic(OpVT) && CC0 == CC1 &&
3595 N0.hasOneUse() && N1.hasOneUse()) {
3596 // and (seteq A, B), (seteq C, D) --> seteq (or (xor A, B), (xor C, D)), 0
3597 // or (setne A, B), (setne C, D) --> setne (or (xor A, B), (xor C, D)), 0
3598 if ((IsAnd && CC1 == ISD::SETEQ) || (!IsAnd && CC1 == ISD::SETNE)) {
3599 SDValue XorL = DAG.getNode(ISD::XOR, SDLoc(N0), OpVT, LL, LR);
3600 SDValue XorR = DAG.getNode(ISD::XOR, SDLoc(N1), OpVT, RL, RR);
3601 SDValue Or = DAG.getNode(ISD::OR, DL, OpVT, XorL, XorR);
3602 SDValue Zero = DAG.getConstant(0, DL, OpVT);
3603 return DAG.getSetCC(DL, VT, Or, Zero, CC1);
3604 }
3605 }
3606
3607 // Canonicalize equivalent operands to LL == RL.
3608 if (LL == RR && LR == RL) {
3609 CC1 = ISD::getSetCCSwappedOperands(CC1);
3610 std::swap(RL, RR);
3611 }
3612
3613 // (and (setcc X, Y, CC0), (setcc X, Y, CC1)) --> (setcc X, Y, NewCC)
3614 // (or (setcc X, Y, CC0), (setcc X, Y, CC1)) --> (setcc X, Y, NewCC)
3615 if (LL == RL && LR == RR) {
3616 ISD::CondCode NewCC = IsAnd ? ISD::getSetCCAndOperation(CC0, CC1, IsInteger)
3617 : ISD::getSetCCOrOperation(CC0, CC1, IsInteger);
3618 if (NewCC != ISD::SETCC_INVALID &&
3619 (!LegalOperations ||
3620 (TLI.isCondCodeLegal(NewCC, LL.getSimpleValueType()) &&
3621 TLI.isOperationLegal(ISD::SETCC, OpVT))))
3622 return DAG.getSetCC(DL, VT, LL, LR, NewCC);
3623 }
3624
3625 return SDValue();
3626}
3627
3628/// This contains all DAGCombine rules which reduce two values combined by
3629/// an And operation to a single value. This makes them reusable in the context
3630/// of visitSELECT(). Rules involving constants are not included as
3631/// visitSELECT() already handles those cases.
3632SDValue DAGCombiner::visitANDLike(SDValue N0, SDValue N1, SDNode *N) {
3633 EVT VT = N1.getValueType();
3634 SDLoc DL(N);
3635
3636 // fold (and x, undef) -> 0
3637 if (N0.isUndef() || N1.isUndef())
3638 return DAG.getConstant(0, DL, VT);
3639
3640 if (SDValue V = foldLogicOfSetCCs(true, N0, N1, DL))
3641 return V;
3642
3643 if (N0.getOpcode() == ISD::ADD && N1.getOpcode() == ISD::SRL &&
3644 VT.getSizeInBits() <= 64) {
3645 if (ConstantSDNode *ADDI = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
3646 if (ConstantSDNode *SRLI = dyn_cast<ConstantSDNode>(N1.getOperand(1))) {
3647 // Look for (and (add x, c1), (lshr y, c2)). If C1 wasn't a legal
3648 // immediate for an add, but it is legal if its top c2 bits are set,
3649 // transform the ADD so the immediate doesn't need to be materialized
3650 // in a register.
3651 APInt ADDC = ADDI->getAPIntValue();
3652 APInt SRLC = SRLI->getAPIntValue();
3653 if (ADDC.getMinSignedBits() <= 64 &&
3654 SRLC.ult(VT.getSizeInBits()) &&
3655 !TLI.isLegalAddImmediate(ADDC.getSExtValue())) {
3656 APInt Mask = APInt::getHighBitsSet(VT.getSizeInBits(),
3657 SRLC.getZExtValue());
3658 if (DAG.MaskedValueIsZero(N0.getOperand(1), Mask)) {
3659 ADDC |= Mask;
3660 if (TLI.isLegalAddImmediate(ADDC.getSExtValue())) {
3661 SDLoc DL0(N0);
3662 SDValue NewAdd =
3663 DAG.getNode(ISD::ADD, DL0, VT,
3664 N0.getOperand(0), DAG.getConstant(ADDC, DL, VT));
3665 CombineTo(N0.getNode(), NewAdd);
3666 // Return N so it doesn't get rechecked!
3667 return SDValue(N, 0);
3668 }
3669 }
3670 }
3671 }
3672 }
3673 }
3674
3675 // Reduce bit extract of low half of an integer to the narrower type.
3676 // (and (srl i64:x, K), KMask) ->
3677 // (i64 zero_extend (and (srl (i32 (trunc i64:x)), K)), KMask)
3678 if (N0.getOpcode() == ISD::SRL && N0.hasOneUse()) {
3679 if (ConstantSDNode *CAnd = dyn_cast<ConstantSDNode>(N1)) {
3680 if (ConstantSDNode *CShift = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
3681 unsigned Size = VT.getSizeInBits();
3682 const APInt &AndMask = CAnd->getAPIntValue();
3683 unsigned ShiftBits = CShift->getZExtValue();
3684
3685 // Bail out, this node will probably disappear anyway.
3686 if (ShiftBits == 0)
3687 return SDValue();
3688
3689 unsigned MaskBits = AndMask.countTrailingOnes();
3690 EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), Size / 2);
3691
3692 if (AndMask.isMask() &&
3693 // Required bits must not span the two halves of the integer and
3694 // must fit in the half size type.
3695 (ShiftBits + MaskBits <= Size / 2) &&
3696 TLI.isNarrowingProfitable(VT, HalfVT) &&
3697 TLI.isTypeDesirableForOp(ISD::AND, HalfVT) &&
3698 TLI.isTypeDesirableForOp(ISD::SRL, HalfVT) &&
3699 TLI.isTruncateFree(VT, HalfVT) &&
3700 TLI.isZExtFree(HalfVT, VT)) {
3701 // The isNarrowingProfitable is to avoid regressions on PPC and
3702 // AArch64 which match a few 64-bit bit insert / bit extract patterns
3703 // on downstream users of this. Those patterns could probably be
3704 // extended to handle extensions mixed in.
3705
3706 SDValue SL(N0);
3707 assert(MaskBits <= Size)(static_cast <bool> (MaskBits <= Size) ? void (0) : __assert_fail
("MaskBits <= Size", "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3707, __extension__ __PRETTY_FUNCTION__))
;
3708
3709 // Extracting the highest bit of the low half.
3710 EVT ShiftVT = TLI.getShiftAmountTy(HalfVT, DAG.getDataLayout());
3711 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, HalfVT,
3712 N0.getOperand(0));
3713
3714 SDValue NewMask = DAG.getConstant(AndMask.trunc(Size / 2), SL, HalfVT);
3715 SDValue ShiftK = DAG.getConstant(ShiftBits, SL, ShiftVT);
3716 SDValue Shift = DAG.getNode(ISD::SRL, SL, HalfVT, Trunc, ShiftK);
3717 SDValue And = DAG.getNode(ISD::AND, SL, HalfVT, Shift, NewMask);
3718 return DAG.getNode(ISD::ZERO_EXTEND, SL, VT, And);
3719 }
3720 }
3721 }
3722 }
3723
3724 return SDValue();
3725}
3726
3727bool DAGCombiner::isAndLoadExtLoad(ConstantSDNode *AndC, LoadSDNode *LoadN,
3728 EVT LoadResultTy, EVT &ExtVT) {
3729 if (!AndC->getAPIntValue().isMask())
3730 return false;
3731
3732 unsigned ActiveBits = AndC->getAPIntValue().countTrailingOnes();
3733
3734 ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
3735 EVT LoadedVT = LoadN->getMemoryVT();
3736
3737 if (ExtVT == LoadedVT &&
3738 (!LegalOperations ||
3739 TLI.isLoadExtLegal(ISD::ZEXTLOAD, LoadResultTy, ExtVT))) {
3740 // ZEXTLOAD will match without needing to change the size of the value being
3741 // loaded.
3742 return true;
3743 }
3744
3745 // Do not change the width of a volatile load.
3746 if (LoadN->isVolatile())
3747 return false;
3748
3749 // Do not generate loads of non-round integer types since these can
3750 // be expensive (and would be wrong if the type is not byte sized).
3751 if (!LoadedVT.bitsGT(ExtVT) || !ExtVT.isRound())
3752 return false;
3753
3754 if (LegalOperations &&
3755 !TLI.isLoadExtLegal(ISD::ZEXTLOAD, LoadResultTy, ExtVT))
3756 return false;
3757
3758 if (!TLI.shouldReduceLoadWidth(LoadN, ISD::ZEXTLOAD, ExtVT))
3759 return false;
3760
3761 return true;
3762}
3763
3764bool DAGCombiner::isLegalNarrowLoad(LoadSDNode *LoadN, ISD::LoadExtType ExtType,
3765 EVT &ExtVT, unsigned ShAmt) {
3766 // Don't transform one with multiple uses, this would require adding a new
3767 // load.
3768 if (!SDValue(LoadN, 0).hasOneUse())
3769 return false;
3770
3771 if (LegalOperations &&
3772 !TLI.isLoadExtLegal(ExtType, LoadN->getValueType(0), ExtVT))
3773 return false;
3774
3775 // Do not generate loads of non-round integer types since these can
3776 // be expensive (and would be wrong if the type is not byte sized).
3777 if (!ExtVT.isRound())
3778 return false;
3779
3780 // Don't change the width of a volatile load.
3781 if (LoadN->isVolatile())
3782 return false;
3783
3784 // Verify that we are actually reducing a load width here.
3785 if (LoadN->getMemoryVT().getSizeInBits() < ExtVT.getSizeInBits())
3786 return false;
3787
3788 // For the transform to be legal, the load must produce only two values
3789 // (the value loaded and the chain). Don't transform a pre-increment
3790 // load, for example, which produces an extra value. Otherwise the
3791 // transformation is not equivalent, and the downstream logic to replace
3792 // uses gets things wrong.
3793 if (LoadN->getNumValues() > 2)
3794 return false;
3795
3796 // If the load that we're shrinking is an extload and we're not just
3797 // discarding the extension we can't simply shrink the load. Bail.
3798 // TODO: It would be possible to merge the extensions in some cases.
3799 if (LoadN->getExtensionType() != ISD::NON_EXTLOAD &&
3800 LoadN->getMemoryVT().getSizeInBits() < ExtVT.getSizeInBits() + ShAmt)
3801 return false;
3802
3803 if (!TLI.shouldReduceLoadWidth(LoadN, ExtType, ExtVT))
3804 return false;
3805
3806 // It's not possible to generate a constant of extended or untyped type.
3807 EVT PtrType = LoadN->getOperand(1).getValueType();
3808 if (PtrType == MVT::Untyped || PtrType.isExtended())
3809 return false;
3810
3811 return true;
3812}
3813
3814bool DAGCombiner::SearchForAndLoads(SDNode *N,
3815 SmallPtrSetImpl<LoadSDNode*> &Loads,
3816 SmallPtrSetImpl<SDNode*> &NodesWithConsts,
3817 ConstantSDNode *Mask,
3818 SDNode *&NodeToMask) {
3819 // Recursively search for the operands, looking for loads which can be
3820 // narrowed.
3821 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) {
3822 SDValue Op = N->getOperand(i);
3823
3824 if (Op.getValueType().isVector())
3825 return false;
3826
3827 // Some constants may need fixing up later if they are too large.
3828 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
3829 if ((N->getOpcode() == ISD::OR || N->getOpcode() == ISD::XOR) &&
3830 (Mask->getAPIntValue() & C->getAPIntValue()) != C->getAPIntValue())
3831 NodesWithConsts.insert(N);
3832 continue;
3833 }
3834
3835 if (!Op.hasOneUse())
3836 return false;
3837
3838 switch(Op.getOpcode()) {
3839 case ISD::LOAD: {
3840 auto *Load = cast<LoadSDNode>(Op);
3841 EVT ExtVT;
3842 if (isAndLoadExtLoad(Mask, Load, Load->getValueType(0), ExtVT) &&
3843 isLegalNarrowLoad(Load, ISD::ZEXTLOAD, ExtVT)) {
3844
3845 // ZEXTLOAD is already small enough.
3846 if (Load->getExtensionType() == ISD::ZEXTLOAD &&
3847 ExtVT.bitsGE(Load->getMemoryVT()))
3848 continue;
3849
3850 // Use LE to convert equal sized loads to zext.
3851 if (ExtVT.bitsLE(Load->getMemoryVT()))
3852 Loads.insert(Load);
3853
3854 continue;
3855 }
3856 return false;
3857 }
3858 case ISD::ZERO_EXTEND:
3859 case ISD::AssertZext: {
3860 unsigned ActiveBits = Mask->getAPIntValue().countTrailingOnes();
3861 EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
3862 EVT VT = Op.getOpcode() == ISD::AssertZext ?
3863 cast<VTSDNode>(Op.getOperand(1))->getVT() :
3864 Op.getOperand(0).getValueType();
3865
3866 // We can accept extending nodes if the mask is wider or an equal
3867 // width to the original type.
3868 if (ExtVT.bitsGE(VT))
3869 continue;
3870 break;
3871 }
3872 case ISD::OR:
3873 case ISD::XOR:
3874 case ISD::AND:
3875 if (!SearchForAndLoads(Op.getNode(), Loads, NodesWithConsts, Mask,
3876 NodeToMask))
3877 return false;
3878 continue;
3879 }
3880
3881 // Allow one node which will masked along with any loads found.
3882 if (NodeToMask)
3883 return false;
3884 NodeToMask = Op.getNode();
3885 }
3886 return true;
3887}
3888
3889bool DAGCombiner::BackwardsPropagateMask(SDNode *N, SelectionDAG &DAG) {
3890 auto *Mask = dyn_cast<ConstantSDNode>(N->getOperand(1));
3891 if (!Mask)
3892 return false;
3893
3894 if (!Mask->getAPIntValue().isMask())
3895 return false;
3896
3897 // No need to do anything if the and directly uses a load.
3898 if (isa<LoadSDNode>(N->getOperand(0)))
3899 return false;
3900
3901 SmallPtrSet<LoadSDNode*, 8> Loads;
3902 SmallPtrSet<SDNode*, 2> NodesWithConsts;
3903 SDNode *FixupNode = nullptr;
3904 if (SearchForAndLoads(N, Loads, NodesWithConsts, Mask, FixupNode)) {
3905 if (Loads.size() == 0)
3906 return false;
3907
3908 DEBUG(dbgs() << "Backwards propagate AND: "; N->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "Backwards propagate AND: "
; N->dump(); } } while (false)
;
3909 SDValue MaskOp = N->getOperand(1);
3910
3911 // If it exists, fixup the single node we allow in the tree that needs
3912 // masking.
3913 if (FixupNode) {
3914 DEBUG(dbgs() << "First, need to fix up: "; FixupNode->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "First, need to fix up: "; FixupNode
->dump(); } } while (false)
;
3915 SDValue And = DAG.getNode(ISD::AND, SDLoc(FixupNode),
3916 FixupNode->getValueType(0),
3917 SDValue(FixupNode, 0), MaskOp);
3918 DAG.ReplaceAllUsesOfValueWith(SDValue(FixupNode, 0), And);
3919 DAG.UpdateNodeOperands(And.getNode(), SDValue(FixupNode, 0),
3920 MaskOp);
3921 }
3922
3923 // Narrow any constants that need it.
3924 for (auto *LogicN : NodesWithConsts) {
3925 SDValue Op0 = LogicN->getOperand(0);
3926 SDValue Op1 = LogicN->getOperand(1);
3927
3928 if (isa<ConstantSDNode>(Op0))
3929 std::swap(Op0, Op1);
3930
3931 SDValue And = DAG.getNode(ISD::AND, SDLoc(Op1), Op1.getValueType(),
3932 Op1, MaskOp);
3933
3934 DAG.UpdateNodeOperands(LogicN, Op0, And);
3935 }
3936
3937 // Create narrow loads.
3938 for (auto *Load : Loads) {
3939 DEBUG(dbgs() << "Propagate AND back to: "; Load->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "Propagate AND back to: "; Load
->dump(); } } while (false)
;
3940 SDValue And = DAG.getNode(ISD::AND, SDLoc(Load), Load->getValueType(0),
3941 SDValue(Load, 0), MaskOp);
3942 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), And);
3943 DAG.UpdateNodeOperands(And.getNode(), SDValue(Load, 0), MaskOp);
3944 SDValue NewLoad = ReduceLoadWidth(And.getNode());
3945 assert(NewLoad &&(static_cast <bool> (NewLoad && "Shouldn't be masking the load if it can't be narrowed"
) ? void (0) : __assert_fail ("NewLoad && \"Shouldn't be masking the load if it can't be narrowed\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3946, __extension__ __PRETTY_FUNCTION__))
3946 "Shouldn't be masking the load if it can't be narrowed")(static_cast <bool> (NewLoad && "Shouldn't be masking the load if it can't be narrowed"
) ? void (0) : __assert_fail ("NewLoad && \"Shouldn't be masking the load if it can't be narrowed\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3946, __extension__ __PRETTY_FUNCTION__))
;
3947 CombineTo(Load, NewLoad, NewLoad.getValue(1));
3948 }
3949 DAG.ReplaceAllUsesWith(N, N->getOperand(0).getNode());
3950 return true;
3951 }
3952 return false;
3953}
3954
3955SDValue DAGCombiner::visitAND(SDNode *N) {
3956 SDValue N0 = N->getOperand(0);
3957 SDValue N1 = N->getOperand(1);
3958 EVT VT = N1.getValueType();
3959
3960 // x & x --> x
3961 if (N0 == N1)
3962 return N0;
3963
3964 // fold vector ops
3965 if (VT.isVector()) {
3966 if (SDValue FoldedVOp = SimplifyVBinOp(N))
3967 return FoldedVOp;
3968
3969 // fold (and x, 0) -> 0, vector edition
3970 if (ISD::isBuildVectorAllZeros(N0.getNode()))
3971 // do not return N0, because undef node may exist in N0
3972 return DAG.getConstant(APInt::getNullValue(N0.getScalarValueSizeInBits()),
3973 SDLoc(N), N0.getValueType());
3974 if (ISD::isBuildVectorAllZeros(N1.getNode()))
3975 // do not return N1, because undef node may exist in N1
3976 return DAG.getConstant(APInt::getNullValue(N1.getScalarValueSizeInBits()),
3977 SDLoc(N), N1.getValueType());
3978
3979 // fold (and x, -1) -> x, vector edition
3980 if (ISD::isBuildVectorAllOnes(N0.getNode()))
3981 return N1;
3982 if (ISD::isBuildVectorAllOnes(N1.getNode()))
3983 return N0;
3984 }
3985
3986 // fold (and c1, c2) -> c1&c2
3987 ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
3988 ConstantSDNode *N1C = isConstOrConstSplat(N1);
3989 if (N0C && N1C && !N1C->isOpaque())
3990 return DAG.FoldConstantArithmetic(ISD::AND, SDLoc(N), VT, N0C, N1C);
3991 // canonicalize constant to RHS
3992 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
3993 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
3994 return DAG.getNode(ISD::AND, SDLoc(N), VT, N1, N0);
3995 // fold (and x, -1) -> x
3996 if (isAllOnesConstant(N1))
3997 return N0;
3998 // if (and x, c) is known to be zero, return 0
3999 unsigned BitWidth = VT.getScalarSizeInBits();
4000 if (N1C && DAG.MaskedValueIsZero(SDValue(N, 0),
4001 APInt::getAllOnesValue(BitWidth)))
4002 return DAG.getConstant(0, SDLoc(N), VT);
4003
4004 if (SDValue NewSel = foldBinOpIntoSelect(N))
4005 return NewSel;
4006
4007 // reassociate and
4008 if (SDValue RAND = ReassociateOps(ISD::AND, SDLoc(N), N0, N1))
4009 return RAND;
4010
4011 // Try to convert a constant mask AND into a shuffle clear mask.
4012 if (VT.isVector())
4013 if (SDValue Shuffle = XformToShuffleWithZero(N))
4014 return Shuffle;
4015
4016 // fold (and (or x, C), D) -> D if (C & D) == D
4017 auto MatchSubset = [](ConstantSDNode *LHS, ConstantSDNode *RHS) {
4018 return RHS->getAPIntValue().isSubsetOf(LHS->getAPIntValue());
4019 };
4020 if (N0.getOpcode() == ISD::OR &&
4021 ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchSubset))
4022 return N1;
4023 // fold (and (any_ext V), c) -> (zero_ext V) if 'and' only clears top bits.
4024 if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) {
4025 SDValue N0Op0 = N0.getOperand(0);
4026 APInt Mask = ~N1C->getAPIntValue();
4027 Mask = Mask.trunc(N0Op0.getScalarValueSizeInBits());
4028 if (DAG.MaskedValueIsZero(N0Op0, Mask)) {
4029 SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N),
4030 N0.getValueType(), N0Op0);
4031
4032 // Replace uses of the AND with uses of the Zero extend node.
4033 CombineTo(N, Zext);
4034
4035 // We actually want to replace all uses of the any_extend with the
4036 // zero_extend, to avoid duplicating things. This will later cause this
4037 // AND to be folded.
4038 CombineTo(N0.getNode(), Zext);
4039 return SDValue(N, 0); // Return N so it doesn't get rechecked!
4040 }
4041 }
4042 // similarly fold (and (X (load ([non_ext|any_ext|zero_ext] V))), c) ->
4043 // (X (load ([non_ext|zero_ext] V))) if 'and' only clears top bits which must
4044 // already be zero by virtue of the width of the base type of the load.
4045 //
4046 // the 'X' node here can either be nothing or an extract_vector_elt to catch
4047 // more cases.
4048 if ((N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
4049 N0.getValueSizeInBits() == N0.getOperand(0).getScalarValueSizeInBits() &&
4050 N0.getOperand(0).getOpcode() == ISD::LOAD &&
4051 N0.getOperand(0).getResNo() == 0) ||
4052 (N0.getOpcode() == ISD::LOAD && N0.getResNo() == 0)) {
4053 LoadSDNode *Load = cast<LoadSDNode>( (N0.getOpcode() == ISD::LOAD) ?
4054 N0 : N0.getOperand(0) );
4055
4056 // Get the constant (if applicable) the zero'th operand is being ANDed with.
4057 // This can be a pure constant or a vector splat, in which case we treat the
4058 // vector as a scalar and use the splat value.
4059 APInt Constant = APInt::getNullValue(1);
4060 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) {
4061 Constant = C->getAPIntValue();
4062 } else if (BuildVectorSDNode *Vector = dyn_cast<BuildVectorSDNode>(N1)) {
4063 APInt SplatValue, SplatUndef;
4064 unsigned SplatBitSize;
4065 bool HasAnyUndefs;
4066 bool IsSplat = Vector->isConstantSplat(SplatValue, SplatUndef,
4067 SplatBitSize, HasAnyUndefs);
4068 if (IsSplat) {
4069 // Undef bits can contribute to a possible optimisation if set, so
4070 // set them.
4071 SplatValue |= SplatUndef;
4072
4073 // The splat value may be something like "0x00FFFFFF", which means 0 for
4074 // the first vector value and FF for the rest, repeating. We need a mask
4075 // that will apply equally to all members of the vector, so AND all the
4076 // lanes of the constant together.
4077 EVT VT = Vector->getValueType(0);
4078 unsigned BitWidth = VT.getScalarSizeInBits();
4079
4080 // If the splat value has been compressed to a bitlength lower
4081 // than the size of the vector lane, we need to re-expand it to
4082 // the lane size.
4083 if (BitWidth > SplatBitSize)
4084 for (SplatValue = SplatValue.zextOrTrunc(BitWidth);
4085 SplatBitSize < BitWidth;
4086 SplatBitSize = SplatBitSize * 2)
4087 SplatValue |= SplatValue.shl(SplatBitSize);
4088
4089 // Make sure that variable 'Constant' is only set if 'SplatBitSize' is a
4090 // multiple of 'BitWidth'. Otherwise, we could propagate a wrong value.
4091 if (SplatBitSize % BitWidth == 0) {
4092 Constant = APInt::getAllOnesValue(BitWidth);
4093 for (unsigned i = 0, n = SplatBitSize/BitWidth; i < n; ++i)
4094 Constant &= SplatValue.lshr(i*BitWidth).zextOrTrunc(BitWidth);
4095 }
4096 }
4097 }
4098
4099 // If we want to change an EXTLOAD to a ZEXTLOAD, ensure a ZEXTLOAD is
4100 // actually legal and isn't going to get expanded, else this is a false
4101 // optimisation.
4102 bool CanZextLoadProfitably = TLI.isLoadExtLegal(ISD::ZEXTLOAD,
4103 Load->getValueType(0),
4104 Load->getMemoryVT());
4105
4106 // Resize the constant to the same size as the original memory access before
4107 // extension. If it is still the AllOnesValue then this AND is completely
4108 // unneeded.
4109 Constant = Constant.zextOrTrunc(Load->getMemoryVT().getScalarSizeInBits());
4110
4111 bool B;
4112 switch (Load->getExtensionType()) {
4113 default: B = false; break;
4114 case ISD::EXTLOAD: B = CanZextLoadProfitably; break;
4115 case ISD::ZEXTLOAD:
4116 case ISD::NON_EXTLOAD: B = true; break;
4117 }
4118
4119 if (B && Constant.isAllOnesValue()) {
4120 // If the load type was an EXTLOAD, convert to ZEXTLOAD in order to
4121 // preserve semantics once we get rid of the AND.
4122 SDValue NewLoad(Load, 0);
4123
4124 // Fold the AND away. NewLoad may get replaced immediately.
4125 CombineTo(N, (N0.getNode() == Load) ? NewLoad : N0);
4126
4127 if (Load->getExtensionType() == ISD::EXTLOAD) {
4128 NewLoad = DAG.getLoad(Load->getAddressingMode(), ISD::ZEXTLOAD,
4129 Load->getValueType(0), SDLoc(Load),
4130 Load->getChain(), Load->getBasePtr(),
4131 Load->getOffset(), Load->getMemoryVT(),
4132 Load->getMemOperand());
4133 // Replace uses of the EXTLOAD with the new ZEXTLOAD.
4134 if (Load->getNumValues() == 3) {
4135 // PRE/POST_INC loads have 3 values.
4136 SDValue To[] = { NewLoad.getValue(0), NewLoad.getValue(1),
4137 NewLoad.getValue(2) };
4138 CombineTo(Load, To, 3, true);
4139 } else {
4140 CombineTo(Load, NewLoad.getValue(0), NewLoad.getValue(1));
4141 }
4142 }
4143
4144 return SDValue(N, 0); // Return N so it doesn't get rechecked!
4145 }
4146 }
4147
4148 // fold (and (load x), 255) -> (zextload x, i8)
4149 // fold (and (extload x, i16), 255) -> (zextload x, i8)
4150 // fold (and (any_ext (extload x, i16)), 255) -> (zextload x, i8)
4151 if (!VT.isVector() && N1C && (N0.getOpcode() == ISD::LOAD ||
4152 (N0.getOpcode() == ISD::ANY_EXTEND &&
4153 N0.getOperand(0).getOpcode() == ISD::LOAD))) {
4154 if (SDValue Res = ReduceLoadWidth(N)) {
4155 LoadSDNode *LN0 = N0->getOpcode() == ISD::ANY_EXTEND
4156 ? cast<LoadSDNode>(N0.getOperand(0)) : cast<LoadSDNode>(N0);
4157
4158 AddToWorklist(N);
4159 CombineTo(LN0, Res, Res.getValue(1));
4160 return SDValue(N, 0);
4161 }
4162 }
4163
4164 if (Level >= AfterLegalizeTypes) {
4165 // Attempt to propagate the AND back up to the leaves which, if they're
4166 // loads, can be combined to narrow loads and the AND node can be removed.
4167 // Perform after legalization so that extend nodes will already be
4168 // combined into the loads.
4169 if (BackwardsPropagateMask(N, DAG)) {
4170 return SDValue(N, 0);
4171 }
4172 }
4173
4174 if (SDValue Combined = visitANDLike(N0, N1, N))
4175 return Combined;
4176
4177 // Simplify: (and (op x...), (op y...)) -> (op (and x, y))
4178 if (N0.getOpcode() == N1.getOpcode())
4179 if (SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N))
4180 return Tmp;
4181
4182 // Masking the negated extension of a boolean is just the zero-extended
4183 // boolean:
4184 // and (sub 0, zext(bool X)), 1 --> zext(bool X)
4185 // and (sub 0, sext(bool X)), 1 --> zext(bool X)
4186 //
4187 // Note: the SimplifyDemandedBits fold below can make an information-losing
4188 // transform, and then we have no way to find this better fold.
4189 if (N1C && N1C->isOne() && N0.getOpcode() == ISD::SUB) {
4190 if (isNullConstantOrNullSplatConstant(N0.getOperand(0))) {
4191 SDValue SubRHS = N0.getOperand(1);
4192 if (SubRHS.getOpcode() == ISD::ZERO_EXTEND &&
4193 SubRHS.getOperand(0).getScalarValueSizeInBits() == 1)
4194 return SubRHS;
4195 if (SubRHS.getOpcode() == ISD::SIGN_EXTEND &&
4196 SubRHS.getOperand(0).getScalarValueSizeInBits() == 1)
4197 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, SubRHS.getOperand(0));
4198 }
4199 }
4200
4201 // fold (and (sign_extend_inreg x, i16 to i32), 1) -> (and x, 1)
4202 // fold (and (sra)) -> (and (srl)) when possible.
4203 if (SimplifyDemandedBits(SDValue(N, 0)))
4204 return SDValue(N, 0);
4205
4206 // fold (zext_inreg (extload x)) -> (zextload x)
4207 if (ISD::isEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode())) {
4208 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
4209 EVT MemVT = LN0->getMemoryVT();
4210 // If we zero all the possible extended bits, then we can turn this into
4211 // a zextload if we are running before legalize or the operation is legal.
4212 unsigned BitWidth = N1.getScalarValueSizeInBits();
4213 if (DAG.MaskedValueIsZero(N1, APInt::getHighBitsSet(BitWidth,
4214 BitWidth - MemVT.getScalarSizeInBits())) &&
4215 ((!LegalOperations && !LN0->isVolatile()) ||
4216 TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT))) {
4217 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT,
4218 LN0->getChain(), LN0->getBasePtr(),
4219 MemVT, LN0->getMemOperand());
4220 AddToWorklist(N);
4221 CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
4222 return SDValue(N, 0); // Return N so it doesn't get rechecked!
4223 }
4224 }
4225 // fold (zext_inreg (sextload x)) -> (zextload x) iff load has one use
4226 if (ISD::isSEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) &&
4227 N0.hasOneUse()) {
4228 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
4229 EVT MemVT = LN0->getMemoryVT();
4230 // If we zero all the possible extended bits, then we can turn this into
4231 // a zextload if we are running before legalize or the operation is legal.
4232 unsigned BitWidth = N1.getScalarValueSizeInBits();
4233 if (DAG.MaskedValueIsZero(N1, APInt::getHighBitsSet(BitWidth,
4234 BitWidth - MemVT.getScalarSizeInBits())) &&
4235 ((!LegalOperations && !LN0->isVolatile()) ||
4236 TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT))) {
4237 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT,
4238 LN0->getChain(), LN0->getBasePtr(),
4239 MemVT, LN0->getMemOperand());
4240 AddToWorklist(N);
4241 CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
4242 return SDValue(N, 0); // Return N so it doesn't get rechecked!
4243 }
4244 }
4245 // fold (and (or (srl N, 8), (shl N, 8)), 0xffff) -> (srl (bswap N), const)
4246 if (N1C && N1C->getAPIntValue() == 0xffff && N0.getOpcode() == ISD::OR) {
4247 if (SDValue BSwap = MatchBSwapHWordLow(N0.getNode(), N0.getOperand(0),
4248 N0.getOperand(1), false))
4249 return BSwap;
4250 }
4251
4252 return SDValue();
4253}
4254
4255/// Match (a >> 8) | (a << 8) as (bswap a) >> 16.
4256SDValue DAGCombiner::MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
4257 bool DemandHighBits) {
4258 if (!LegalOperations)
4259 return SDValue();
4260
4261 EVT VT = N->getValueType(0);
4262 if (VT != MVT::i64 && VT != MVT::i32 && VT != MVT::i16)
4263 return SDValue();
4264 if (!TLI.isOperationLegalOrCustom(ISD::BSWAP, VT))
4265 return SDValue();
4266
4267 // Recognize (and (shl a, 8), 0xff00), (and (srl a, 8), 0xff)
4268 bool LookPassAnd0 = false;
4269 bool LookPassAnd1 = false;
4270 if (N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::SRL)
4271 std::swap(N0, N1);
4272 if (N1.getOpcode() == ISD::AND && N1.getOperand(0).getOpcode() == ISD::SHL)
4273 std::swap(N0, N1);
4274 if (N0.getOpcode() == ISD::AND) {
4275 if (!N0.getNode()->hasOneUse())
4276 return SDValue();
4277 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
4278 if (!N01C || N01C->getZExtValue() != 0xFF00)
4279 return SDValue();
4280 N0 = N0.getOperand(0);
4281 LookPassAnd0 = true;
4282 }
4283
4284 if (N1.getOpcode() == ISD::AND) {
4285 if (!N1.getNode()->hasOneUse())
4286 return SDValue();
4287 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
4288 if (!N11C || N11C->getZExtValue() != 0xFF)
4289 return SDValue();
4290 N1 = N1.getOperand(0);
4291 LookPassAnd1 = true;
4292 }
4293
4294 if (N0.getOpcode() == ISD::SRL && N1.getOpcode() == ISD::SHL)
4295 std::swap(N0, N1);
4296 if (N0.getOpcode() != ISD::SHL || N1.getOpcode() != ISD::SRL)
4297 return SDValue();
4298 if (!N0.getNode()->hasOneUse() || !N1.getNode()->hasOneUse())
4299 return SDValue();
4300
4301 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
4302 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
4303 if (!N01C || !N11C)
4304 return SDValue();
4305 if (N01C->getZExtValue() != 8 || N11C->getZExtValue() != 8)
4306 return SDValue();
4307
4308 // Look for (shl (and a, 0xff), 8), (srl (and a, 0xff00), 8)
4309 SDValue N00 = N0->getOperand(0);
4310 if (!LookPassAnd0 && N00.getOpcode() == ISD::AND) {
4311 if (!N00.getNode()->hasOneUse())
4312 return SDValue();
4313 ConstantSDNode *N001C = dyn_cast<ConstantSDNode>(N00.getOperand(1));
4314 if (!N001C || N001C->getZExtValue() != 0xFF)
4315 return SDValue();
4316 N00 = N00.getOperand(0);
4317 LookPassAnd0 = true;
4318 }
4319
4320 SDValue N10 = N1->getOperand(0);
4321 if (!LookPassAnd1 && N10.getOpcode() == ISD::AND) {
4322 if (!N10.getNode()->hasOneUse())
4323 return SDValue();
4324 ConstantSDNode *N101C = dyn_cast<ConstantSDNode>(N10.getOperand(1));
4325 if (!N101C || N101C->getZExtValue() != 0xFF00)
4326 return SDValue();
4327 N10 = N10.getOperand(0);
4328 LookPassAnd1 = true;
4329 }
4330
4331 if (N00 != N10)
4332 return SDValue();
4333
4334 // Make sure everything beyond the low halfword gets set to zero since the SRL
4335 // 16 will clear the top bits.
4336 unsigned OpSizeInBits = VT.getSizeInBits();
4337 if (DemandHighBits && OpSizeInBits > 16) {
4338 // If the left-shift isn't masked out then the only way this is a bswap is
4339 // if all bits beyond the low 8 are 0. In that case the entire pattern
4340 // reduces to a left shift anyway: leave it for other parts of the combiner.
4341 if (!LookPassAnd0)
4342 return SDValue();
4343
4344 // However, if the right shift isn't masked out then it might be because
4345 // it's not needed. See if we can spot that too.
4346 if (!LookPassAnd1 &&
4347 !DAG.MaskedValueIsZero(
4348 N10, APInt::getHighBitsSet(OpSizeInBits, OpSizeInBits - 16)))
4349 return SDValue();
4350 }
4351
4352 SDValue Res = DAG.getNode(ISD::BSWAP, SDLoc(N), VT, N00);
4353 if (OpSizeInBits > 16) {
4354 SDLoc DL(N);
4355 Res = DAG.getNode(ISD::SRL, DL, VT, Res,
4356 DAG.getConstant(OpSizeInBits - 16, DL,
4357 getShiftAmountTy(VT)));
4358 }
4359 return Res;
4360}
4361
4362/// Return true if the specified node is an element that makes up a 32-bit
4363/// packed halfword byteswap.
4364/// ((x & 0x000000ff) << 8) |
4365/// ((x & 0x0000ff00) >> 8) |
4366/// ((x & 0x00ff0000) << 8) |
4367/// ((x & 0xff000000) >> 8)
4368static bool isBSwapHWordElement(SDValue N, MutableArrayRef<SDNode *> Parts) {
4369 if (!N.getNode()->hasOneUse())
4370 return false;
4371
4372 unsigned Opc = N.getOpcode();
4373 if (Opc != ISD::AND && Opc != ISD::SHL && Opc != ISD::SRL)
4374 return false;
4375
4376 SDValue N0 = N.getOperand(0);
4377 unsigned Opc0 = N0.getOpcode();
4378 if (Opc0 != ISD::AND && Opc0 != ISD::SHL && Opc0 != ISD::SRL)
4379 return false;
4380
4381 ConstantSDNode *N1C = nullptr;
4382 // SHL or SRL: look upstream for AND mask operand
4383 if (Opc == ISD::AND)
4384 N1C = dyn_cast<ConstantSDNode>(N.getOperand(1));
4385 else if (Opc0 == ISD::AND)
4386 N1C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
4387 if (!N1C)
4388 return false;
4389
4390 unsigned MaskByteOffset;
4391 switch (N1C->getZExtValue()) {
4392 default:
4393 return false;
4394 case 0xFF: MaskByteOffset = 0; break;
4395 case 0xFF00: MaskByteOffset = 1; break;
4396 case 0xFF0000: MaskByteOffset = 2; break;
4397 case 0xFF000000: MaskByteOffset = 3; break;
4398 }
4399
4400 // Look for (x & 0xff) << 8 as well as ((x << 8) & 0xff00).
4401 if (Opc == ISD::AND) {
4402 if (MaskByteOffset == 0 || MaskByteOffset == 2) {
4403 // (x >> 8) & 0xff
4404 // (x >> 8) & 0xff0000
4405 if (Opc0 != ISD::SRL)
4406 return false;
4407 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
4408 if (!C || C->getZExtValue() != 8)
4409 return false;
4410 } else {
4411 // (x << 8) & 0xff00
4412 // (x << 8) & 0xff000000
4413 if (Opc0 != ISD::SHL)
4414 return false;
4415 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
4416 if (!C || C->getZExtValue() != 8)
4417 return false;
4418 }
4419 } else if (Opc == ISD::SHL) {
4420 // (x & 0xff) << 8
4421 // (x & 0xff0000) << 8
4422 if (MaskByteOffset != 0 && MaskByteOffset != 2)
4423 return false;
4424 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
4425 if (!C || C->getZExtValue() != 8)
4426 return false;
4427 } else { // Opc == ISD::SRL
4428 // (x & 0xff00) >> 8
4429 // (x & 0xff000000) >> 8
4430 if (MaskByteOffset != 1 && MaskByteOffset != 3)
4431 return false;
4432 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
4433 if (!C || C->getZExtValue() != 8)
4434 return false;
4435 }
4436
4437 if (Parts[MaskByteOffset])
4438 return false;
4439
4440 Parts[MaskByteOffset] = N0.getOperand(0).getNode();
4441 return true;
4442}
4443
4444/// Match a 32-bit packed halfword bswap. That is
4445/// ((x & 0x000000ff) << 8) |
4446/// ((x & 0x0000ff00) >> 8) |
4447/// ((x & 0x00ff0000) << 8) |
4448/// ((x & 0xff000000) >> 8)
4449/// => (rotl (bswap x), 16)
4450SDValue DAGCombiner::MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1) {
4451 if (!LegalOperations)
4452 return SDValue();
4453
4454 EVT VT = N->getValueType(0);
4455 if (VT != MVT::i32)
4456 return SDValue();
4457 if (!TLI.isOperationLegalOrCustom(ISD::BSWAP, VT))
4458 return SDValue();
4459
4460 // Look for either
4461 // (or (or (and), (and)), (or (and), (and)))
4462 // (or (or (or (and), (and)), (and)), (and))
4463 if (N0.getOpcode() != ISD::OR)
4464 return SDValue();
4465 SDValue N00 = N0.getOperand(0);
4466 SDValue N01 = N0.getOperand(1);
4467 SDNode *Parts[4] = {};
4468
4469 if (N1.getOpcode() == ISD::OR &&
4470 N00.getNumOperands() == 2 && N01.getNumOperands() == 2) {
4471 // (or (or (and), (and)), (or (and), (and)))
4472 if (!isBSwapHWordElement(N00, Parts))
4473 return SDValue();
4474
4475 if (!isBSwapHWordElement(N01, Parts))
4476 return SDValue();
4477 SDValue N10 = N1.getOperand(0);
4478 if (!isBSwapHWordElement(N10, Parts))
4479 return SDValue();
4480 SDValue N11 = N1.getOperand(1);
4481 if (!isBSwapHWordElement(N11, Parts))
4482 return SDValue();
4483 } else {
4484 // (or (or (or (and), (and)), (and)), (and))
4485 if (!isBSwapHWordElement(N1, Parts))
4486 return SDValue();
4487 if (!isBSwapHWordElement(N01, Parts))
4488 return SDValue();
4489 if (N00.getOpcode() != ISD::OR)
4490 return SDValue();
4491 SDValue N000 = N00.getOperand(0);
4492 if (!isBSwapHWordElement(N000, Parts))
4493 return SDValue();
4494 SDValue N001 = N00.getOperand(1);
4495 if (!isBSwapHWordElement(N001, Parts))
4496 return SDValue();
4497 }
4498
4499 // Make sure the parts are all coming from the same node.
4500 if (Parts[0] != Parts[1] || Parts[0] != Parts[2] || Parts[0] != Parts[3])
4501 return SDValue();
4502
4503 SDLoc DL(N);
4504 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT,
4505 SDValue(Parts[0], 0));
4506
4507 // Result of the bswap should be rotated by 16. If it's not legal, then
4508 // do (x << 16) | (x >> 16).
4509 SDValue ShAmt = DAG.getConstant(16, DL, getShiftAmountTy(VT));
4510 if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT))
4511 return DAG.getNode(ISD::ROTL, DL, VT, BSwap, ShAmt);
4512 if (TLI.isOperationLegalOrCustom(ISD::ROTR, VT))
4513 return DAG.getNode(ISD::ROTR, DL, VT, BSwap, ShAmt);
4514 return DAG.getNode(ISD::OR, DL, VT,
4515 DAG.getNode(ISD::SHL, DL, VT, BSwap, ShAmt),
4516 DAG.getNode(ISD::SRL, DL, VT, BSwap, ShAmt));
4517}
4518
4519/// This contains all DAGCombine rules which reduce two values combined by
4520/// an Or operation to a single value \see visitANDLike().
4521SDValue DAGCombiner::visitORLike(SDValue N0, SDValue N1, SDNode *N) {
4522 EVT VT = N1.getValueType();
4523 SDLoc DL(N);
4524
4525 // fold (or x, undef) -> -1
4526 if (!LegalOperations && (N0.isUndef() || N1.isUndef()))
4527 return DAG.getAllOnesConstant(DL, VT);
4528
4529 if (SDValue V = foldLogicOfSetCCs(false, N0, N1, DL))
4530 return V;
4531
4532 // (or (and X, C1), (and Y, C2)) -> (and (or X, Y), C3) if possible.
4533 if (N0.getOpcode() == ISD::AND && N1.getOpcode() == ISD::AND &&
4534 // Don't increase # computations.
4535 (N0.getNode()->hasOneUse() || N1.getNode()->hasOneUse())) {
4536 // We can only do this xform if we know that bits from X that are set in C2
4537 // but not in C1 are already zero. Likewise for Y.
4538 if (const ConstantSDNode *N0O1C =
4539 getAsNonOpaqueConstant(N0.getOperand(1))) {
4540 if (const ConstantSDNode *N1O1C =
4541 getAsNonOpaqueConstant(N1.getOperand(1))) {
4542 // We can only do this xform if we know that bits from X that are set in
4543 // C2 but not in C1 are already zero. Likewise for Y.
4544 const APInt &LHSMask = N0O1C->getAPIntValue();
4545 const APInt &RHSMask = N1O1C->getAPIntValue();
4546
4547 if (DAG.MaskedValueIsZero(N0.getOperand(0), RHSMask&~LHSMask) &&
4548 DAG.MaskedValueIsZero(N1.getOperand(0), LHSMask&~RHSMask)) {
4549 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT,
4550 N0.getOperand(0), N1.getOperand(0));
4551 return DAG.getNode(ISD::AND, DL, VT, X,
4552 DAG.getConstant(LHSMask | RHSMask, DL, VT));
4553 }
4554 }
4555 }
4556 }
4557
4558 // (or (and X, M), (and X, N)) -> (and X, (or M, N))
4559 if (N0.getOpcode() == ISD::AND &&
4560 N1.getOpcode() == ISD::AND &&
4561 N0.getOperand(0) == N1.getOperand(0) &&
4562 // Don't increase # computations.
4563 (N0.getNode()->hasOneUse() || N1.getNode()->hasOneUse())) {
4564 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT,
4565 N0.getOperand(1), N1.getOperand(1));
4566 return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), X);
4567 }
4568
4569 return SDValue();
4570}
4571
4572SDValue DAGCombiner::visitOR(SDNode *N) {
4573 SDValue N0 = N->getOperand(0);
4574 SDValue N1 = N->getOperand(1);
4575 EVT VT = N1.getValueType();
4576
4577 // x | x --> x
4578 if (N0 == N1)
4579 return N0;
4580
4581 // fold vector ops
4582 if (VT.isVector()) {
4583 if (SDValue FoldedVOp = SimplifyVBinOp(N))
4584 return FoldedVOp;
4585
4586 // fold (or x, 0) -> x, vector edition
4587 if (ISD::isBuildVectorAllZeros(N0.getNode()))
4588 return N1;
4589 if (ISD::isBuildVectorAllZeros(N1.getNode()))
4590 return N0;
4591
4592 // fold (or x, -1) -> -1, vector edition
4593 if (ISD::isBuildVectorAllOnes(N0.getNode()))
4594 // do not return N0, because undef node may exist in N0
4595 return DAG.getAllOnesConstant(SDLoc(N), N0.getValueType());
4596 if (ISD::isBuildVectorAllOnes(N1.getNode()))
4597 // do not return N1, because undef node may exist in N1
4598 return DAG.getAllOnesConstant(SDLoc(N), N1.getValueType());
4599
4600 // fold (or (shuf A, V_0, MA), (shuf B, V_0, MB)) -> (shuf A, B, Mask)
4601 // Do this only if the resulting shuffle is legal.
4602 if (isa<ShuffleVectorSDNode>(N0) &&
4603 isa<ShuffleVectorSDNode>(N1) &&
4604 // Avoid folding a node with illegal type.
4605 TLI.isTypeLegal(VT)) {
4606 bool ZeroN00 = ISD::isBuildVectorAllZeros(N0.getOperand(0).getNode());
4607 bool ZeroN01 = ISD::isBuildVectorAllZeros(N0.getOperand(1).getNode());
4608 bool ZeroN10 = ISD::isBuildVectorAllZeros(N1.getOperand(0).getNode());
4609 bool ZeroN11 = ISD::isBuildVectorAllZeros(N1.getOperand(1).getNode());
4610 // Ensure both shuffles have a zero input.
4611 if ((ZeroN00 != ZeroN01) && (ZeroN10 != ZeroN11)) {
4612 assert((!ZeroN00 || !ZeroN01) && "Both inputs zero!")(static_cast <bool> ((!ZeroN00 || !ZeroN01) && "Both inputs zero!"
) ? void (0) : __assert_fail ("(!ZeroN00 || !ZeroN01) && \"Both inputs zero!\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4612, __extension__ __PRETTY_FUNCTION__))
;
4613 assert((!ZeroN10 || !ZeroN11) && "Both inputs zero!")(static_cast <bool> ((!ZeroN10 || !ZeroN11) && "Both inputs zero!"
) ? void (0) : __assert_fail ("(!ZeroN10 || !ZeroN11) && \"Both inputs zero!\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4613, __extension__ __PRETTY_FUNCTION__))
;
4614 const ShuffleVectorSDNode *SV0 = cast<ShuffleVectorSDNode>(N0);
4615 const ShuffleVectorSDNode *SV1 = cast<ShuffleVectorSDNode>(N1);
4616 bool CanFold = true;
4617 int NumElts = VT.getVectorNumElements();
4618 SmallVector<int, 4> Mask(NumElts);
4619
4620 for (int i = 0; i != NumElts; ++i) {
4621 int M0 = SV0->getMaskElt(i);
4622 int M1 = SV1->getMaskElt(i);
4623
4624 // Determine if either index is pointing to a zero vector.
4625 bool M0Zero = M0 < 0 || (ZeroN00 == (M0 < NumElts));
4626 bool M1Zero = M1 < 0 || (ZeroN10 == (M1 < NumElts));
4627
4628 // If one element is zero and the otherside is undef, keep undef.
4629 // This also handles the case that both are undef.
4630 if ((M0Zero && M1 < 0) || (M1Zero && M0 < 0)) {
4631 Mask[i] = -1;
4632 continue;
4633 }
4634
4635 // Make sure only one of the elements is zero.
4636 if (M0Zero == M1Zero) {
4637 CanFold = false;
4638 break;
4639 }
4640
4641 assert((M0 >= 0 || M1 >= 0) && "Undef index!")(static_cast <bool> ((M0 >= 0 || M1 >= 0) &&
"Undef index!") ? void (0) : __assert_fail ("(M0 >= 0 || M1 >= 0) && \"Undef index!\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4641, __extension__ __PRETTY_FUNCTION__))
;
4642
4643 // We have a zero and non-zero element. If the non-zero came from
4644 // SV0 make the index a LHS index. If it came from SV1, make it
4645 // a RHS index. We need to mod by NumElts because we don't care
4646 // which operand it came from in the original shuffles.
4647 Mask[i] = M1Zero ? M0 % NumElts : (M1 % NumElts) + NumElts;
4648 }
4649
4650 if (CanFold) {
4651 SDValue NewLHS = ZeroN00 ? N0.getOperand(1) : N0.getOperand(0);
4652 SDValue NewRHS = ZeroN10 ? N1.getOperand(1) : N1.getOperand(0);
4653
4654 bool LegalMask = TLI.isShuffleMaskLegal(Mask, VT);
4655 if (!LegalMask) {
4656 std::swap(NewLHS, NewRHS);
4657 ShuffleVectorSDNode::commuteMask(Mask);
4658 LegalMask = TLI.isShuffleMaskLegal(Mask, VT);
4659 }
4660
4661 if (LegalMask)
4662 return DAG.getVectorShuffle(VT, SDLoc(N), NewLHS, NewRHS, Mask);
4663 }
4664 }
4665 }
4666 }
4667
4668 // fold (or c1, c2) -> c1|c2
4669 ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
4670 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
4671 if (N0C && N1C && !N1C->isOpaque())
4672 return DAG.FoldConstantArithmetic(ISD::OR, SDLoc(N), VT, N0C, N1C);
4673 // canonicalize constant to RHS
4674 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
4675 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
4676 return DAG.getNode(ISD::OR, SDLoc(N), VT, N1, N0);
4677 // fold (or x, 0) -> x
4678 if (isNullConstant(N1))
4679 return N0;
4680 // fold (or x, -1) -> -1
4681 if (isAllOnesConstant(N1))
4682 return N1;
4683
4684 if (SDValue NewSel = foldBinOpIntoSelect(N))
4685 return NewSel;
4686
4687 // fold (or x, c) -> c iff (x & ~c) == 0
4688 if (N1C && DAG.MaskedValueIsZero(N0, ~N1C->getAPIntValue()))
4689 return N1;
4690
4691 if (SDValue Combined = visitORLike(N0, N1, N))
4692 return Combined;
4693
4694 // Recognize halfword bswaps as (bswap + rotl 16) or (bswap + shl 16)
4695 if (SDValue BSwap = MatchBSwapHWord(N, N0, N1))
4696 return BSwap;
4697 if (SDValue BSwap = MatchBSwapHWordLow(N, N0, N1))
4698 return BSwap;
4699
4700 // reassociate or
4701 if (SDValue ROR = ReassociateOps(ISD::OR, SDLoc(N), N0, N1))
4702 return ROR;
4703
4704 // Canonicalize (or (and X, c1), c2) -> (and (or X, c2), c1|c2)
4705 // iff (c1 & c2) != 0.
4706 auto MatchIntersect = [](ConstantSDNode *LHS, ConstantSDNode *RHS) {
4707 return LHS->getAPIntValue().intersects(RHS->getAPIntValue());
4708 };
4709 if (N0.getOpcode() == ISD::AND && N0.getNode()->hasOneUse() &&
4710 ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchIntersect)) {
4711 if (SDValue COR = DAG.FoldConstantArithmetic(
4712 ISD::OR, SDLoc(N1), VT, N1.getNode(), N0.getOperand(1).getNode())) {
4713 SDValue IOR = DAG.getNode(ISD::OR, SDLoc(N0), VT, N0.getOperand(0), N1);
4714 AddToWorklist(IOR.getNode());
4715 return DAG.getNode(ISD::AND, SDLoc(N), VT, COR, IOR);
4716 }
4717 }
4718
4719 // Simplify: (or (op x...), (op y...)) -> (op (or x, y))
4720 if (N0.getOpcode() == N1.getOpcode())
4721 if (SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N))
4722 return Tmp;
4723
4724 // See if this is some rotate idiom.
4725 if (SDNode *Rot = MatchRotate(N0, N1, SDLoc(N)))
4726 return SDValue(Rot, 0);
4727
4728 if (SDValue Load = MatchLoadCombine(N))
4729 return Load;
4730
4731 // Simplify the operands using demanded-bits information.
4732 if (SimplifyDemandedBits(SDValue(N, 0)))
4733 return SDValue(N, 0);
4734
4735 return SDValue();
4736}
4737
4738/// Match "(X shl/srl V1) & V2" where V2 may not be present.
4739bool DAGCombiner::MatchRotateHalf(SDValue Op, SDValue &Shift, SDValue &Mask) {
4740 if (Op.getOpcode() == ISD::AND) {
4741 if (DAG.isConstantIntBuildVectorOrConstantInt(Op.getOperand(1))) {
4742 Mask = Op.getOperand(1);
4743 Op = Op.getOperand(0);
4744 } else {
4745 return false;
4746 }
4747 }
4748
4749 if (Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SHL) {
4750 Shift = Op;
4751 return true;
4752 }
4753
4754 return false;
4755}
4756
4757// Return true if we can prove that, whenever Neg and Pos are both in the
4758// range [0, EltSize), Neg == (Pos == 0 ? 0 : EltSize - Pos). This means that
4759// for two opposing shifts shift1 and shift2 and a value X with OpBits bits:
4760//
4761// (or (shift1 X, Neg), (shift2 X, Pos))
4762//
4763// reduces to a rotate in direction shift2 by Pos or (equivalently) a rotate
4764// in direction shift1 by Neg. The range [0, EltSize) means that we only need
4765// to consider shift amounts with defined behavior.
4766static bool matchRotateSub(SDValue Pos, SDValue Neg, unsigned EltSize) {
4767 // If EltSize is a power of 2 then:
4768 //
4769 // (a) (Pos == 0 ? 0 : EltSize - Pos) == (EltSize - Pos) & (EltSize - 1)
4770 // (b) Neg == Neg & (EltSize - 1) whenever Neg is in [0, EltSize).
4771 //
4772 // So if EltSize is a power of 2 and Neg is (and Neg', EltSize-1), we check
4773 // for the stronger condition:
4774 //
4775 // Neg & (EltSize - 1) == (EltSize - Pos) & (EltSize - 1) [A]
4776 //
4777 // for all Neg and Pos. Since Neg & (EltSize - 1) == Neg' & (EltSize - 1)
4778 // we can just replace Neg with Neg' for the rest of the function.
4779 //
4780 // In other cases we check for the even stronger condition:
4781 //
4782 // Neg == EltSize - Pos [B]
4783 //
4784 // for all Neg and Pos. Note that the (or ...) then invokes undefined
4785 // behavior if Pos == 0 (and consequently Neg == EltSize).
4786 //
4787 // We could actually use [A] whenever EltSize is a power of 2, but the
4788 // only extra cases that it would match are those uninteresting ones
4789 // where Neg and Pos are never in range at the same time. E.g. for
4790 // EltSize == 32, using [A] would allow a Neg of the form (sub 64, Pos)
4791 // as well as (sub 32, Pos), but:
4792 //
4793 // (or (shift1 X, (sub 64, Pos)), (shift2 X, Pos))
4794 //
4795 // always invokes undefined behavior for 32-bit X.
4796 //
4797 // Below, Mask == EltSize - 1 when using [A] and is all-ones otherwise.
4798 unsigned MaskLoBits = 0;
4799 if (Neg.getOpcode() == ISD::AND && isPowerOf2_64(EltSize)) {
4800 if (ConstantSDNode *NegC = isConstOrConstSplat(Neg.getOperand(1))) {
4801 if (NegC->getAPIntValue() == EltSize - 1) {
4802 Neg = Neg.getOperand(0);
4803 MaskLoBits = Log2_64(EltSize);
4804 }
4805 }
4806 }
4807
4808 // Check whether Neg has the form (sub NegC, NegOp1) for some NegC and NegOp1.
4809 if (Neg.getOpcode() != ISD::SUB)
4810 return false;
4811 ConstantSDNode *NegC = isConstOrConstSplat(Neg.getOperand(0));
4812 if (!NegC)
4813 return false;
4814 SDValue NegOp1 = Neg.getOperand(1);
4815
4816 // On the RHS of [A], if Pos is Pos' & (EltSize - 1), just replace Pos with
4817 // Pos'. The truncation is redundant for the purpose of the equality.
4818 if (MaskLoBits && Pos.getOpcode() == ISD::AND)
4819 if (ConstantSDNode *PosC = isConstOrConstSplat(Pos.getOperand(1)))
4820 if (PosC->getAPIntValue() == EltSize - 1)
4821 Pos = Pos.getOperand(0);
4822
4823 // The condition we need is now:
4824 //
4825 // (NegC - NegOp1) & Mask == (EltSize - Pos) & Mask
4826 //
4827 // If NegOp1 == Pos then we need:
4828 //
4829 // EltSize & Mask == NegC & Mask
4830 //
4831 // (because "x & Mask" is a truncation and distributes through subtraction).
4832 APInt Width;
4833 if (Pos == NegOp1)
4834 Width = NegC->getAPIntValue();
4835
4836 // Check for cases where Pos has the form (add NegOp1, PosC) for some PosC.
4837 // Then the condition we want to prove becomes:
4838 //
4839 // (NegC - NegOp1) & Mask == (EltSize - (NegOp1 + PosC)) & Mask
4840 //
4841 // which, again because "x & Mask" is a truncation, becomes:
4842 //
4843 // NegC & Mask == (EltSize - PosC) & Mask
4844 // EltSize & Mask == (NegC + PosC) & Mask
4845 else if (Pos.getOpcode() == ISD::ADD && Pos.getOperand(0) == NegOp1) {
4846 if (ConstantSDNode *PosC = isConstOrConstSplat(Pos.getOperand(1)))
4847 Width = PosC->getAPIntValue() + NegC->getAPIntValue();
4848 else
4849 return false;
4850 } else
4851 return false;
4852
4853 // Now we just need to check that EltSize & Mask == Width & Mask.
4854 if (MaskLoBits)
4855 // EltSize & Mask is 0 since Mask is EltSize - 1.
4856 return Width.getLoBits(MaskLoBits) == 0;
4857 return Width == EltSize;
4858}
4859
4860// A subroutine of MatchRotate used once we have found an OR of two opposite
4861// shifts of Shifted. If Neg == <operand size> - Pos then the OR reduces
4862// to both (PosOpcode Shifted, Pos) and (NegOpcode Shifted, Neg), with the
4863// former being preferred if supported. InnerPos and InnerNeg are Pos and
4864// Neg with outer conversions stripped away.
4865SDNode *DAGCombiner::MatchRotatePosNeg(SDValue Shifted, SDValue Pos,
4866 SDValue Neg, SDValue InnerPos,
4867 SDValue InnerNeg, unsigned PosOpcode,
4868 unsigned NegOpcode, const SDLoc &DL) {
4869 // fold (or (shl x, (*ext y)),
4870 // (srl x, (*ext (sub 32, y)))) ->
4871 // (rotl x, y) or (rotr x, (sub 32, y))
4872 //
4873 // fold (or (shl x, (*ext (sub 32, y))),
4874 // (srl x, (*ext y))) ->
4875 // (rotr x, y) or (rotl x, (sub 32, y))
4876 EVT VT = Shifted.getValueType();
4877 if (matchRotateSub(InnerPos, InnerNeg, VT.getScalarSizeInBits())) {
4878 bool HasPos = TLI.isOperationLegalOrCustom(PosOpcode, VT);
4879 return DAG.getNode(HasPos ? PosOpcode : NegOpcode, DL, VT, Shifted,
4880 HasPos ? Pos : Neg).getNode();
4881 }
4882
4883 return nullptr;
4884}
4885
4886// MatchRotate - Handle an 'or' of two operands. If this is one of the many
4887// idioms for rotate, and if the target supports rotation instructions, generate
4888// a rot[lr].
4889SDNode *DAGCombiner::MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL) {
4890 // Must be a legal type. Expanded 'n promoted things won't work with rotates.
4891 EVT VT = LHS.getValueType();
4892 if (!TLI.isTypeLegal(VT)) return nullptr;
4893
4894 // The target must have at least one rotate flavor.
4895 bool HasROTL = TLI.isOperationLegalOrCustom(ISD::ROTL, VT);
4896 bool HasROTR = TLI.isOperationLegalOrCustom(ISD::ROTR, VT);
4897 if (!HasROTL && !HasROTR) return nullptr;
4898
4899 // Check for truncated rotate.
4900 if (LHS.getOpcode() == ISD::TRUNCATE && RHS.getOpcode() == ISD::TRUNCATE &&
4901 LHS.getOperand(0).getValueType() == RHS.getOperand(0).getValueType()) {
4902 assert(LHS.getValueType() == RHS.getValueType())(static_cast <bool> (LHS.getValueType() == RHS.getValueType
()) ? void (0) : __assert_fail ("LHS.getValueType() == RHS.getValueType()"
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4902, __extension__ __PRETTY_FUNCTION__))
;
4903 if (SDNode *Rot = MatchRotate(LHS.getOperand(0), RHS.getOperand(0), DL)) {
4904 return DAG.getNode(ISD::TRUNCATE, SDLoc(LHS), LHS.getValueType(),
4905 SDValue(Rot, 0)).getNode();
4906 }
4907 }
4908
4909 // Match "(X shl/srl V1) & V2" where V2 may not be present.
4910 SDValue LHSShift; // The shift.
4911 SDValue LHSMask; // AND value if any.
4912 if (!MatchRotateHalf(LHS, LHSShift, LHSMask))
4913 return nullptr; // Not part of a rotate.
4914
4915 SDValue RHSShift; // The shift.
4916 SDValue RHSMask; // AND value if any.
4917 if (!MatchRotateHalf(RHS, RHSShift, RHSMask))
4918 return nullptr; // Not part of a rotate.
4919
4920 if (LHSShift.getOperand(0) != RHSShift.getOperand(0))
4921 return nullptr; // Not shifting the same value.
4922
4923 if (LHSShift.getOpcode() == RHSShift.getOpcode())
4924 return nullptr; // Shifts must disagree.
4925
4926 // Canonicalize shl to left side in a shl/srl pair.
4927 if (RHSShift.getOpcode() == ISD::SHL) {
4928 std::swap(LHS, RHS);
4929 std::swap(LHSShift, RHSShift);
4930 std::swap(LHSMask, RHSMask);
4931 }
4932
4933 unsigned EltSizeInBits = VT.getScalarSizeInBits();
4934 SDValue LHSShiftArg = LHSShift.getOperand(0);
4935 SDValue LHSShiftAmt = LHSShift.getOperand(1);
4936 SDValue RHSShiftArg = RHSShift.getOperand(0);
4937 SDValue RHSShiftAmt = RHSShift.getOperand(1);
4938
4939 // fold (or (shl x, C1), (srl x, C2)) -> (rotl x, C1)
4940 // fold (or (shl x, C1), (srl x, C2)) -> (rotr x, C2)
4941 auto MatchRotateSum = [EltSizeInBits](ConstantSDNode *LHS,
4942 ConstantSDNode *RHS) {
4943 return (LHS->getAPIntValue() + RHS->getAPIntValue()) == EltSizeInBits;
4944 };
4945 if (ISD::matchBinaryPredicate(LHSShiftAmt, RHSShiftAmt, MatchRotateSum)) {
4946 SDValue Rot = DAG.getNode(HasROTL ? ISD::ROTL : ISD::ROTR, DL, VT,
4947 LHSShiftArg, HasROTL ? LHSShiftAmt : RHSShiftAmt);
4948
4949 // If there is an AND of either shifted operand, apply it to the result.
4950 if (LHSMask.getNode() || RHSMask.getNode()) {
4951 SDValue AllOnes = DAG.getAllOnesConstant(DL, VT);
4952 SDValue Mask = AllOnes;
4953
4954 if (LHSMask.getNode()) {
4955 SDValue RHSBits = DAG.getNode(ISD::SRL, DL, VT, AllOnes, RHSShiftAmt);
4956 Mask = DAG.getNode(ISD::AND, DL, VT, Mask,
4957 DAG.getNode(ISD::OR, DL, VT, LHSMask, RHSBits));
4958 }
4959 if (RHSMask.getNode()) {
4960 SDValue LHSBits = DAG.getNode(ISD::SHL, DL, VT, AllOnes, LHSShiftAmt);
4961 Mask = DAG.getNode(ISD::AND, DL, VT, Mask,
4962 DAG.getNode(ISD::OR, DL, VT, RHSMask, LHSBits));
4963 }
4964
4965 Rot = DAG.getNode(ISD::AND, DL, VT, Rot, Mask);
4966 }
4967
4968 return Rot.getNode();
4969 }
4970
4971 // If there is a mask here, and we have a variable shift, we can't be sure
4972 // that we're masking out the right stuff.
4973 if (LHSMask.getNode() || RHSMask.getNode())
4974 return nullptr;
4975
4976 // If the shift amount is sign/zext/any-extended just peel it off.
4977 SDValue LExtOp0 = LHSShiftAmt;
4978 SDValue RExtOp0 = RHSShiftAmt;
4979 if ((LHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND ||
4980 LHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND ||
4981 LHSShiftAmt.getOpcode() == ISD::ANY_EXTEND ||
4982 LHSShiftAmt.getOpcode() == ISD::TRUNCATE) &&
4983 (RHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND ||
4984 RHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND ||
4985 RHSShiftAmt.getOpcode() == ISD::ANY_EXTEND ||
4986 RHSShiftAmt.getOpcode() == ISD::TRUNCATE)) {
4987 LExtOp0 = LHSShiftAmt.getOperand(0);
4988 RExtOp0 = RHSShiftAmt.getOperand(0);
4989 }
4990
4991 SDNode *TryL = MatchRotatePosNeg(LHSShiftArg, LHSShiftAmt, RHSShiftAmt,
4992 LExtOp0, RExtOp0, ISD::ROTL, ISD::ROTR, DL);
4993 if (TryL)
4994 return TryL;
4995
4996 SDNode *TryR = MatchRotatePosNeg(RHSShiftArg, RHSShiftAmt, LHSShiftAmt,
4997 RExtOp0, LExtOp0, ISD::ROTR, ISD::ROTL, DL);
4998 if (TryR)
4999 return TryR;
5000
5001 return nullptr;
5002}
5003
5004namespace {
5005
5006/// Represents known origin of an individual byte in load combine pattern. The
5007/// value of the byte is either constant zero or comes from memory.
5008struct ByteProvider {
5009 // For constant zero providers Load is set to nullptr. For memory providers
5010 // Load represents the node which loads the byte from memory.
5011 // ByteOffset is the offset of the byte in the value produced by the load.
5012 LoadSDNode *Load = nullptr;
5013 unsigned ByteOffset = 0;
5014
5015 ByteProvider() = default;
5016
5017 static ByteProvider getMemory(LoadSDNode *Load, unsigned ByteOffset) {
5018 return ByteProvider(Load, ByteOffset);
5019 }
5020
5021 static ByteProvider getConstantZero() { return ByteProvider(nullptr, 0); }
5022
5023 bool isConstantZero() const { return !Load; }
5024 bool isMemory() const { return Load; }
5025
5026 bool operator==(const ByteProvider &Other) const {
5027 return Other.Load == Load && Other.ByteOffset == ByteOffset;
5028 }
5029
5030private:
5031 ByteProvider(LoadSDNode *Load, unsigned ByteOffset)
5032 : Load(Load), ByteOffset(ByteOffset) {}
5033};
5034
5035} // end anonymous namespace
5036
5037/// Recursively traverses the expression calculating the origin of the requested
5038/// byte of the given value. Returns None if the provider can't be calculated.
5039///
5040/// For all the values except the root of the expression verifies that the value
5041/// has exactly one use and if it's not true return None. This way if the origin
5042/// of the byte is returned it's guaranteed that the values which contribute to
5043/// the byte are not used outside of this expression.
5044///
5045/// Because the parts of the expression are not allowed to have more than one
5046/// use this function iterates over trees, not DAGs. So it never visits the same
5047/// node more than once.
5048static const Optional<ByteProvider>
5049calculateByteProvider(SDValue Op, unsigned Index, unsigned Depth,
5050 bool Root = false) {
5051 // Typical i64 by i8 pattern requires recursion up to 8 calls depth
5052 if (Depth == 10)
5053 return None;
5054
5055 if (!Root && !Op.hasOneUse())
5056 return None;
5057
5058 assert(Op.getValueType().isScalarInteger() && "can't handle other types")(static_cast <bool> (Op.getValueType().isScalarInteger(
) && "can't handle other types") ? void (0) : __assert_fail
("Op.getValueType().isScalarInteger() && \"can't handle other types\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5058, __extension__ __PRETTY_FUNCTION__))
;
5059 unsigned BitWidth = Op.getValueSizeInBits();
5060 if (BitWidth % 8 != 0)
5061 return None;
5062 unsigned ByteWidth = BitWidth / 8;
5063 assert(Index < ByteWidth && "invalid index requested")(static_cast <bool> (Index < ByteWidth && "invalid index requested"
) ? void (0) : __assert_fail ("Index < ByteWidth && \"invalid index requested\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5063, __extension__ __PRETTY_FUNCTION__))
;
5064 (void) ByteWidth;
5065
5066 switch (Op.getOpcode()) {
5067 case ISD::OR: {
5068 auto LHS = calculateByteProvider(Op->getOperand(0), Index, Depth + 1);
5069 if (!LHS)
5070 return None;
5071 auto RHS = calculateByteProvider(Op->getOperand(1), Index, Depth + 1);
5072 if (!RHS)
5073 return None;
5074
5075 if (LHS->isConstantZero())
5076 return RHS;
5077 if (RHS->isConstantZero())
5078 return LHS;
5079 return None;
5080 }
5081 case ISD::SHL: {
5082 auto ShiftOp = dyn_cast<ConstantSDNode>(Op->getOperand(1));
5083 if (!ShiftOp)
5084 return None;
5085
5086 uint64_t BitShift = ShiftOp->getZExtValue();
5087 if (BitShift % 8 != 0)
5088 return None;
5089 uint64_t ByteShift = BitShift / 8;
5090
5091 return Index < ByteShift
5092 ? ByteProvider::getConstantZero()
5093 : calculateByteProvider(Op->getOperand(0), Index - ByteShift,
5094 Depth + 1);
5095 }
5096 case ISD::ANY_EXTEND:
5097 case ISD::SIGN_EXTEND:
5098 case ISD::ZERO_EXTEND: {
5099 SDValue NarrowOp = Op->getOperand(0);
5100 unsigned NarrowBitWidth = NarrowOp.getScalarValueSizeInBits();
5101 if (NarrowBitWidth % 8 != 0)
5102 return None;
5103 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
5104
5105 if (Index >= NarrowByteWidth)
5106 return Op.getOpcode() == ISD::ZERO_EXTEND
5107 ? Optional<ByteProvider>(ByteProvider::getConstantZero())
5108 : None;
5109 return calculateByteProvider(NarrowOp, Index, Depth + 1);
5110 }
5111 case ISD::BSWAP:
5112 return calculateByteProvider(Op->getOperand(0), ByteWidth - Index - 1,
5113 Depth + 1);
5114 case ISD::LOAD: {
5115 auto L = cast<LoadSDNode>(Op.getNode());
5116 if (L->isVolatile() || L->isIndexed())
5117 return None;
5118
5119 unsigned NarrowBitWidth = L->getMemoryVT().getSizeInBits();
5120 if (NarrowBitWidth % 8 != 0)
5121 return None;
5122 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
5123
5124 if (Index >= NarrowByteWidth)
5125 return L->getExtensionType() == ISD::ZEXTLOAD
5126 ? Optional<ByteProvider>(ByteProvider::getConstantZero())
5127 : None;
5128 return ByteProvider::getMemory(L, Index);
5129 }
5130 }
5131
5132 return None;
5133}
5134
5135/// Match a pattern where a wide type scalar value is loaded by several narrow
5136/// loads and combined by shifts and ors. Fold it into a single load or a load
5137/// and a BSWAP if the targets supports it.
5138///
5139/// Assuming little endian target:
5140/// i8 *a = ...
5141/// i32 val = a[0] | (a[1] << 8) | (a[2] << 16) | (a[3] << 24)
5142/// =>
5143/// i32 val = *((i32)a)
5144///
5145/// i8 *a = ...
5146/// i32 val = (a[0] << 24) | (a[1] << 16) | (a[2] << 8) | a[3]
5147/// =>
5148/// i32 val = BSWAP(*((i32)a))
5149///
5150/// TODO: This rule matches complex patterns with OR node roots and doesn't
5151/// interact well with the worklist mechanism. When a part of the pattern is
5152/// updated (e.g. one of the loads) its direct users are put into the worklist,
5153/// but the root node of the pattern which triggers the load combine is not
5154/// necessarily a direct user of the changed node. For example, once the address
5155/// of t28 load is reassociated load combine won't be triggered:
5156/// t25: i32 = add t4, Constant:i32<2>
5157/// t26: i64 = sign_extend t25
5158/// t27: i64 = add t2, t26
5159/// t28: i8,ch = load<LD1[%tmp9]> t0, t27, undef:i64
5160/// t29: i32 = zero_extend t28
5161/// t32: i32 = shl t29, Constant:i8<8>
5162/// t33: i32 = or t23, t32
5163/// As a possible fix visitLoad can check if the load can be a part of a load
5164/// combine pattern and add corresponding OR roots to the worklist.
5165SDValue DAGCombiner::MatchLoadCombine(SDNode *N) {
5166 assert(N->getOpcode() == ISD::OR &&(static_cast <bool> (N->getOpcode() == ISD::OR &&
"Can only match load combining against OR nodes") ? void (0)
: __assert_fail ("N->getOpcode() == ISD::OR && \"Can only match load combining against OR nodes\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5167, __extension__ __PRETTY_FUNCTION__))
5167 "Can only match load combining against OR nodes")(static_cast <bool> (N->getOpcode() == ISD::OR &&
"Can only match load combining against OR nodes") ? void (0)
: __assert_fail ("N->getOpcode() == ISD::OR && \"Can only match load combining against OR nodes\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5167, __extension__ __PRETTY_FUNCTION__))
;
5168
5169 // Handles simple types only
5170 EVT VT = N->getValueType(0);
5171 if (VT != MVT::i16 && VT != MVT::i32 && VT != MVT::i64)
5172 return SDValue();
5173 unsigned ByteWidth = VT.getSizeInBits() / 8;
5174
5175 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5176 // Before legalize we can introduce too wide illegal loads which will be later
5177 // split into legal sized loads. This enables us to combine i64 load by i8
5178 // patterns to a couple of i32 loads on 32 bit targets.
5179 if (LegalOperations && !TLI.isOperationLegal(ISD::LOAD, VT))
5180 return SDValue();
5181
5182 std::function<unsigned(unsigned, unsigned)> LittleEndianByteAt = [](
5183 unsigned BW, unsigned i) { return i; };
5184 std::function<unsigned(unsigned, unsigned)> BigEndianByteAt = [](
5185 unsigned BW, unsigned i) { return BW - i - 1; };
5186
5187 bool IsBigEndianTarget = DAG.getDataLayout().isBigEndian();
5188 auto MemoryByteOffset = [&] (ByteProvider P) {
5189 assert(P.isMemory() && "Must be a memory byte provider")(static_cast <bool> (P.isMemory() && "Must be a memory byte provider"
) ? void (0) : __assert_fail ("P.isMemory() && \"Must be a memory byte provider\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5189, __extension__ __PRETTY_FUNCTION__))
;
5190 unsigned LoadBitWidth = P.Load->getMemoryVT().getSizeInBits();
5191 assert(LoadBitWidth % 8 == 0 &&(static_cast <bool> (LoadBitWidth % 8 == 0 && "can only analyze providers for individual bytes not bit"
) ? void (0) : __assert_fail ("LoadBitWidth % 8 == 0 && \"can only analyze providers for individual bytes not bit\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5192, __extension__ __PRETTY_FUNCTION__))
5192 "can only analyze providers for individual bytes not bit")(static_cast <bool> (LoadBitWidth % 8 == 0 && "can only analyze providers for individual bytes not bit"
) ? void (0) : __assert_fail ("LoadBitWidth % 8 == 0 && \"can only analyze providers for individual bytes not bit\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5192, __extension__ __PRETTY_FUNCTION__))
;
5193 unsigned LoadByteWidth = LoadBitWidth / 8;
5194 return IsBigEndianTarget
5195 ? BigEndianByteAt(LoadByteWidth, P.ByteOffset)
5196 : LittleEndianByteAt(LoadByteWidth, P.ByteOffset);
5197 };
5198
5199 Optional<BaseIndexOffset> Base;
5200 SDValue Chain;
5201
5202 SmallSet<LoadSDNode *, 8> Loads;
5203 Optional<ByteProvider> FirstByteProvider;
5204 int64_t FirstOffset = INT64_MAX(9223372036854775807L);
5205
5206 // Check if all the bytes of the OR we are looking at are loaded from the same
5207 // base address. Collect bytes offsets from Base address in ByteOffsets.
5208 SmallVector<int64_t, 4> ByteOffsets(ByteWidth);
5209 for (unsigned i = 0; i < ByteWidth; i++) {
5210 auto P = calculateByteProvider(SDValue(N, 0), i, 0, /*Root=*/true);
5211 if (!P || !P->isMemory()) // All the bytes must be loaded from memory
5212 return SDValue();
5213
5214 LoadSDNode *L = P->Load;
5215 assert(L->hasNUsesOfValue(1, 0) && !L->isVolatile() && !L->isIndexed() &&(static_cast <bool> (L->hasNUsesOfValue(1, 0) &&
!L->isVolatile() && !L->isIndexed() &&
"Must be enforced by calculateByteProvider") ? void (0) : __assert_fail
("L->hasNUsesOfValue(1, 0) && !L->isVolatile() && !L->isIndexed() && \"Must be enforced by calculateByteProvider\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5216, __extension__ __PRETTY_FUNCTION__))
5216 "Must be enforced by calculateByteProvider")(static_cast <bool> (L->hasNUsesOfValue(1, 0) &&
!L->isVolatile() && !L->isIndexed() &&
"Must be enforced by calculateByteProvider") ? void (0) : __assert_fail
("L->hasNUsesOfValue(1, 0) && !L->isVolatile() && !L->isIndexed() && \"Must be enforced by calculateByteProvider\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5216, __extension__ __PRETTY_FUNCTION__))
;
5217 assert(L->getOffset().isUndef() && "Unindexed load must have undef offset")(static_cast <bool> (L->getOffset().isUndef() &&
"Unindexed load must have undef offset") ? void (0) : __assert_fail
("L->getOffset().isUndef() && \"Unindexed load must have undef offset\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5217, __extension__ __PRETTY_FUNCTION__))
;
5218
5219 // All loads must share the same chain
5220 SDValue LChain = L->getChain();
5221 if (!Chain)
5222 Chain = LChain;
5223 else if (Chain != LChain)
5224 return SDValue();
5225
5226 // Loads must share the same base address
5227 BaseIndexOffset Ptr = BaseIndexOffset::match(L, DAG);
5228 int64_t ByteOffsetFromBase = 0;
5229 if (!Base)
5230 Base = Ptr;
5231 else if (!Base->equalBaseIndex(Ptr, DAG, ByteOffsetFromBase))
5232 return SDValue();
5233
5234 // Calculate the offset of the current byte from the base address
5235 ByteOffsetFromBase += MemoryByteOffset(*P);
5236 ByteOffsets[i] = ByteOffsetFromBase;
5237
5238 // Remember the first byte load
5239 if (ByteOffsetFromBase < FirstOffset) {
5240 FirstByteProvider = P;
5241 FirstOffset = ByteOffsetFromBase;
5242 }
5243
5244 Loads.insert(L);
5245 }
5246 assert(!Loads.empty() && "All the bytes of the value must be loaded from "(static_cast <bool> (!Loads.empty() && "All the bytes of the value must be loaded from "
"memory, so there must be at least one load which produces the value"
) ? void (0) : __assert_fail ("!Loads.empty() && \"All the bytes of the value must be loaded from \" \"memory, so there must be at least one load which produces the value\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5247, __extension__ __PRETTY_FUNCTION__))
5247 "memory, so there must be at least one load which produces the value")(static_cast <bool> (!Loads.empty() && "All the bytes of the value must be loaded from "
"memory, so there must be at least one load which produces the value"
) ? void (0) : __assert_fail ("!Loads.empty() && \"All the bytes of the value must be loaded from \" \"memory, so there must be at least one load which produces the value\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5247, __extension__ __PRETTY_FUNCTION__))
;
5248 assert(Base && "Base address of the accessed memory location must be set")(static_cast <bool> (Base && "Base address of the accessed memory location must be set"
) ? void (0) : __assert_fail ("Base && \"Base address of the accessed memory location must be set\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5248, __extension__ __PRETTY_FUNCTION__))
;
5249 assert(FirstOffset != INT64_MAX && "First byte offset must be set")(static_cast <bool> (FirstOffset != (9223372036854775807L
) && "First byte offset must be set") ? void (0) : __assert_fail
("FirstOffset != INT64_MAX && \"First byte offset must be set\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5249, __extension__ __PRETTY_FUNCTION__))
;
5250
5251 // Check if the bytes of the OR we are looking at match with either big or
5252 // little endian value load
5253 bool BigEndian = true, LittleEndian = true;
5254 for (unsigned i = 0; i < ByteWidth; i++) {
5255 int64_t CurrentByteOffset = ByteOffsets[i] - FirstOffset;
5256 LittleEndian &= CurrentByteOffset == LittleEndianByteAt(ByteWidth, i);
5257 BigEndian &= CurrentByteOffset == BigEndianByteAt(ByteWidth, i);
5258 if (!BigEndian && !LittleEndian)
5259 return SDValue();
5260 }
5261 assert((BigEndian != LittleEndian) && "should be either or")(static_cast <bool> ((BigEndian != LittleEndian) &&
"should be either or") ? void (0) : __assert_fail ("(BigEndian != LittleEndian) && \"should be either or\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5261, __extension__ __PRETTY_FUNCTION__))
;
5262 assert(FirstByteProvider && "must be set")(static_cast <bool> (FirstByteProvider && "must be set"
) ? void (0) : __assert_fail ("FirstByteProvider && \"must be set\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5262, __extension__ __PRETTY_FUNCTION__))
;
5263
5264 // Ensure that the first byte is loaded from zero offset of the first load.
5265 // So the combined value can be loaded from the first load address.
5266 if (MemoryByteOffset(*FirstByteProvider) != 0)
5267 return SDValue();
5268 LoadSDNode *FirstLoad = FirstByteProvider->Load;
5269
5270 // The node we are looking at matches with the pattern, check if we can
5271 // replace it with a single load and bswap if needed.
5272
5273 // If the load needs byte swap check if the target supports it
5274 bool NeedsBswap = IsBigEndianTarget != BigEndian;
5275
5276 // Before legalize we can introduce illegal bswaps which will be later
5277 // converted to an explicit bswap sequence. This way we end up with a single
5278 // load and byte shuffling instead of several loads and byte shuffling.
5279 if (NeedsBswap && LegalOperations && !TLI.isOperationLegal(ISD::BSWAP, VT))
5280 return SDValue();
5281
5282 // Check that a load of the wide type is both allowed and fast on the target
5283 bool Fast = false;
5284 bool Allowed = TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(),
5285 VT, FirstLoad->getAddressSpace(),
5286 FirstLoad->getAlignment(), &Fast);
5287 if (!Allowed || !Fast)
5288 return SDValue();
5289
5290 SDValue NewLoad =
5291 DAG.getLoad(VT, SDLoc(N), Chain, FirstLoad->getBasePtr(),
5292 FirstLoad->getPointerInfo(), FirstLoad->getAlignment());
5293
5294 // Transfer chain users from old loads to the new load.
5295 for (LoadSDNode *L : Loads)
5296 DAG.ReplaceAllUsesOfValueWith(SDValue(L, 1), SDValue(NewLoad.getNode(), 1));
5297
5298 return NeedsBswap ? DAG.getNode(ISD::BSWAP, SDLoc(N), VT, NewLoad) : NewLoad;
5299}
5300
5301SDValue DAGCombiner::visitXOR(SDNode *N) {
5302 SDValue N0 = N->getOperand(0);
5303 SDValue N1 = N->getOperand(1);
5304 EVT VT = N0.getValueType();
5305
5306 // fold vector ops
5307 if (VT.isVector()) {
5308 if (SDValue FoldedVOp = SimplifyVBinOp(N))
5309 return FoldedVOp;
5310
5311 // fold (xor x, 0) -> x, vector edition
5312 if (ISD::isBuildVectorAllZeros(N0.getNode()))
5313 return N1;
5314 if (ISD::isBuildVectorAllZeros(N1.getNode()))
5315 return N0;
5316 }
5317
5318 // fold (xor undef, undef) -> 0. This is a common idiom (misuse).
5319 if (N0.isUndef() && N1.isUndef())
5320 return DAG.getConstant(0, SDLoc(N), VT);
5321 // fold (xor x, undef) -> undef
5322 if (N0.isUndef())
5323 return N0;
5324 if (N1.isUndef())
5325 return N1;
5326 // fold (xor c1, c2) -> c1^c2
5327 ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
5328 ConstantSDNode *N1C = getAsNonOpaqueConstant(N1);
5329 if (N0C && N1C)
5330 return DAG.FoldConstantArithmetic(ISD::XOR, SDLoc(N), VT, N0C, N1C);
5331 // canonicalize constant to RHS
5332 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
5333 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
5334 return DAG.getNode(ISD::XOR, SDLoc(N), VT, N1, N0);
5335 // fold (xor x, 0) -> x
5336 if (isNullConstant(N1))
5337 return N0;
5338
5339 if (SDValue NewSel = foldBinOpIntoSelect(N))
5340 return NewSel;
5341
5342 // reassociate xor
5343 if (SDValue RXOR = ReassociateOps(ISD::XOR, SDLoc(N), N0, N1))
5344 return RXOR;
5345
5346 // fold !(x cc y) -> (x !cc y)
5347 SDValue LHS, RHS, CC;
5348 if (TLI.isConstTrueVal(N1.getNode()) && isSetCCEquivalent(N0, LHS, RHS, CC)) {
5349 bool isInt = LHS.getValueType().isInteger();
5350 ISD::CondCode NotCC = ISD::getSetCCInverse(cast<CondCodeSDNode>(CC)->get(),
5351 isInt);
5352
5353 if (!LegalOperations ||
5354 TLI.isCondCodeLegal(NotCC, LHS.getSimpleValueType())) {
5355 switch (N0.getOpcode()) {
5356 default:
5357 llvm_unreachable("Unhandled SetCC Equivalent!")::llvm::llvm_unreachable_internal("Unhandled SetCC Equivalent!"
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5357)
;
5358 case ISD::SETCC:
5359 return DAG.getSetCC(SDLoc(N0), VT, LHS, RHS, NotCC);
5360 case ISD::SELECT_CC:
5361 return DAG.getSelectCC(SDLoc(N0), LHS, RHS, N0.getOperand(2),
5362 N0.getOperand(3), NotCC);
5363 }
5364 }
5365 }
5366
5367 // fold (not (zext (setcc x, y))) -> (zext (not (setcc x, y)))
5368 if (isOneConstant(N1) && N0.getOpcode() == ISD::ZERO_EXTEND &&
5369 N0.getNode()->hasOneUse() &&
5370 isSetCCEquivalent(N0.getOperand(0), LHS, RHS, CC)){
5371 SDValue V = N0.getOperand(0);
5372 SDLoc DL(N0);
5373 V = DAG.getNode(ISD::XOR, DL, V.getValueType(), V,
5374 DAG.getConstant(1, DL, V.getValueType()));
5375 AddToWorklist(V.getNode());
5376 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, V);
5377 }
5378
5379 // fold (not (or x, y)) -> (and (not x), (not y)) iff x or y are setcc
5380 if (isOneConstant(N1) && VT == MVT::i1 && N0.hasOneUse() &&
5381 (N0.getOpcode() == ISD::OR || N0.getOpcode() == ISD::AND)) {
5382 SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1);
5383 if (isOneUseSetCC(RHS) || isOneUseSetCC(LHS)) {
5384 unsigned NewOpcode = N0.getOpcode() == ISD::AND ? ISD::OR : ISD::AND;
5385 LHS = DAG.getNode(ISD::XOR, SDLoc(LHS), VT, LHS, N1); // LHS = ~LHS
5386 RHS = DAG.getNode(ISD::XOR, SDLoc(RHS), VT, RHS, N1); // RHS = ~RHS
5387 AddToWorklist(LHS.getNode()); AddToWorklist(RHS.getNode());
5388 return DAG.getNode(NewOpcode, SDLoc(N), VT, LHS, RHS);
5389 }
5390 }
5391 // fold (not (or x, y)) -> (and (not x), (not y)) iff x or y are constants
5392 if (isAllOnesConstant(N1) && N0.hasOneUse() &&
5393 (N0.getOpcode() == ISD::OR || N0.getOpcode() == ISD::AND)) {
5394 SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1);
5395 if (isa<ConstantSDNode>(RHS) || isa<ConstantSDNode>(LHS)) {
5396 unsigned NewOpcode = N0.getOpcode() == ISD::AND ? ISD::OR : ISD::AND;
5397 LHS = DAG.getNode(ISD::XOR, SDLoc(LHS), VT, LHS, N1); // LHS = ~LHS
5398 RHS = DAG.getNode(ISD::XOR, SDLoc(RHS), VT, RHS, N1); // RHS = ~RHS
5399 AddToWorklist(LHS.getNode()); AddToWorklist(RHS.getNode());
5400 return DAG.getNode(NewOpcode, SDLoc(N), VT, LHS, RHS);
5401 }
5402 }
5403 // fold (xor (and x, y), y) -> (and (not x), y)
5404 if (N0.getOpcode() == ISD::AND && N0.getNode()->hasOneUse() &&
5405 N0->getOperand(1) == N1) {
5406 SDValue X = N0->getOperand(0);
5407 SDValue NotX = DAG.getNOT(SDLoc(X), X, VT);
5408 AddToWorklist(NotX.getNode());
5409 return DAG.getNode(ISD::AND, SDLoc(N), VT, NotX, N1);
5410 }
5411
5412 // fold Y = sra (X, size(X)-1); xor (add (X, Y), Y) -> (abs X)
5413 unsigned OpSizeInBits = VT.getScalarSizeInBits();
5414 if (N0.getOpcode() == ISD::ADD && N0.getOperand(1) == N1 &&
5415 N1.getOpcode() == ISD::SRA && N1.getOperand(0) == N0.getOperand(0) &&
5416 TLI.isOperationLegalOrCustom(ISD::ABS, VT)) {
5417 if (ConstantSDNode *C = isConstOrConstSplat(N1.getOperand(1)))
5418 if (C->getAPIntValue() == (OpSizeInBits - 1))
5419 return DAG.getNode(ISD::ABS, SDLoc(N), VT, N0.getOperand(0));
5420 }
5421
5422 // fold (xor x, x) -> 0
5423 if (N0 == N1)
5424 return tryFoldToZero(SDLoc(N), TLI, VT, DAG, LegalOperations, LegalTypes);
5425
5426 // fold (xor (shl 1, x), -1) -> (rotl ~1, x)
5427 // Here is a concrete example of this equivalence:
5428 // i16 x == 14
5429 // i16 shl == 1 << 14 == 16384 == 0b0100000000000000
5430 // i16 xor == ~(1 << 14) == 49151 == 0b1011111111111111
5431 //
5432 // =>
5433 //
5434 // i16 ~1 == 0b1111111111111110
5435 // i16 rol(~1, 14) == 0b1011111111111111
5436 //
5437 // Some additional tips to help conceptualize this transform:
5438 // - Try to see the operation as placing a single zero in a value of all ones.
5439 // - There exists no value for x which would allow the result to contain zero.
5440 // - Values of x larger than the bitwidth are undefined and do not require a
5441 // consistent result.
5442 // - Pushing the zero left requires shifting one bits in from the right.
5443 // A rotate left of ~1 is a nice way of achieving the desired result.
5444 if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT) && N0.getOpcode() == ISD::SHL
5445 && isAllOnesConstant(N1) && isOneConstant(N0.getOperand(0))) {
5446 SDLoc DL(N);
5447 return DAG.getNode(ISD::ROTL, DL, VT, DAG.getConstant(~1, DL, VT),
5448 N0.getOperand(1));
5449 }
5450
5451 // Simplify: xor (op x...), (op y...) -> (op (xor x, y))
5452 if (N0.getOpcode() == N1.getOpcode())
5453 if (SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N))
5454 return Tmp;
5455
5456 // Simplify the expression using non-local knowledge.
5457 if (SimplifyDemandedBits(SDValue(N, 0)))
5458 return SDValue(N, 0);
5459
5460 return SDValue();
5461}
5462
5463/// Handle transforms common to the three shifts, when the shift amount is a
5464/// constant.
5465SDValue DAGCombiner::visitShiftByConstant(SDNode *N, ConstantSDNode *Amt) {
5466 SDNode *LHS = N->getOperand(0).getNode();
5467 if (!LHS->hasOneUse()) return SDValue();
5468
5469 // We want to pull some binops through shifts, so that we have (and (shift))
5470 // instead of (shift (and)), likewise for add, or, xor, etc. This sort of
5471 // thing happens with address calculations, so it's important to canonicalize
5472 // it.
5473 bool HighBitSet = false; // Can we transform this if the high bit is set?
5474
5475 switch (LHS->getOpcode()) {
5476 default: return SDValue();
5477 case ISD::OR:
5478 case ISD::XOR:
5479 HighBitSet = false; // We can only transform sra if the high bit is clear.
5480 break;
5481 case ISD::AND:
5482 HighBitSet = true; // We can only transform sra if the high bit is set.
5483 break;
5484 case ISD::ADD:
5485 if (N->getOpcode() != ISD::SHL)
5486 return SDValue(); // only shl(add) not sr[al](add).
5487 HighBitSet = false; // We can only transform sra if the high bit is clear.
5488 break;
5489 }
5490
5491 // We require the RHS of the binop to be a constant and not opaque as well.
5492 ConstantSDNode *BinOpCst = getAsNonOpaqueConstant(LHS->getOperand(1));
5493 if (!BinOpCst) return SDValue();
5494
5495 // FIXME: disable this unless the input to the binop is a shift by a constant
5496 // or is copy/select.Enable this in other cases when figure out it's exactly profitable.
5497 SDNode *BinOpLHSVal = LHS->getOperand(0).getNode();
5498 bool isShift = BinOpLHSVal->getOpcode() == ISD::SHL ||
5499 BinOpLHSVal->getOpcode() == ISD::SRA ||
5500 BinOpLHSVal->getOpcode() == ISD::SRL;
5501 bool isCopyOrSelect = BinOpLHSVal->getOpcode() == ISD::CopyFromReg ||
5502 BinOpLHSVal->getOpcode() == ISD::SELECT;
5503
5504 if ((!isShift || !isa<ConstantSDNode>(BinOpLHSVal->getOperand(1))) &&
5505 !isCopyOrSelect)
5506 return SDValue();
5507
5508 if (isCopyOrSelect && N->hasOneUse())
5509 return SDValue();
5510
5511 EVT VT = N->getValueType(0);
5512
5513 // If this is a signed shift right, and the high bit is modified by the
5514 // logical operation, do not perform the transformation. The highBitSet
5515 // boolean indicates the value of the high bit of the constant which would
5516 // cause it to be modified for this operation.
5517 if (N->getOpcode() == ISD::SRA) {
5518 bool BinOpRHSSignSet = BinOpCst->getAPIntValue().isNegative();
5519 if (BinOpRHSSignSet != HighBitSet)
5520 return SDValue();
5521 }
5522
5523 if (!TLI.isDesirableToCommuteWithShift(LHS))
5524 return SDValue();
5525
5526 // Fold the constants, shifting the binop RHS by the shift amount.
5527 SDValue NewRHS = DAG.getNode(N->getOpcode(), SDLoc(LHS->getOperand(1)),
5528 N->getValueType(0),
5529 LHS->getOperand(1), N->getOperand(1));
5530 assert(isa<ConstantSDNode>(NewRHS) && "Folding was not successful!")(static_cast <bool> (isa<ConstantSDNode>(NewRHS) &&
"Folding was not successful!") ? void (0) : __assert_fail ("isa<ConstantSDNode>(NewRHS) && \"Folding was not successful!\""
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5530, __extension__ __PRETTY_FUNCTION__))
;
5531
5532 // Create the new shift.
5533 SDValue NewShift = DAG.getNode(N->getOpcode(),
5534 SDLoc(LHS->getOperand(0)),
5535 VT, LHS->getOperand(0), N->getOperand(1));
5536
5537 // Create the new binop.
5538 return DAG.getNode(LHS->getOpcode(), SDLoc(N), VT, NewShift, NewRHS);
5539}
5540
5541SDValue DAGCombiner::distributeTruncateThroughAnd(SDNode *N) {
5542 assert(N->getOpcode() == ISD::TRUNCATE)(static_cast <bool> (N->getOpcode() == ISD::TRUNCATE
) ? void (0) : __assert_fail ("N->getOpcode() == ISD::TRUNCATE"
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5542, __extension__ __PRETTY_FUNCTION__))
;
5543 assert(N->getOperand(0).getOpcode() == ISD::AND)(static_cast <bool> (N->getOperand(0).getOpcode() ==
ISD::AND) ? void (0) : __assert_fail ("N->getOperand(0).getOpcode() == ISD::AND"
, "/build/llvm-toolchain-snapshot-7~svn326246/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5543, __extension__ __PRETTY_FUNCTION__))
;
5544
5545 // (truncate:TruncVT (and N00, N01C)) -> (and (truncate:TruncVT N00), TruncC)
5546 if (N->hasOneUse() && N->getOperand(0).hasOneUse()) {
5547 SDValue N01 = N->getOperand(0).getOperand(1);
5548 if (isConstantOrConstantVector(N01, /* NoOpaques */ true)) {
5549 SDLoc DL(N);
5550 EVT TruncVT = N->getValueType(0);
5551 SDValue N00 = N->getOperand(0).getOperand(0);
5552 SDValue Trunc00 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N00);
5553 SDValue Trunc01 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N01);
5554 AddToWorklist(Trunc00.getNode());
5555 AddToWorklist(Trunc01.getNode());
5556 return DAG.getNode(ISD::AND, DL, TruncVT, Trunc00, Trunc01);
5557 }
5558 }
5559
5560 return SDValue();
5561}
5562
5563SDValue DAGCombiner::visitRotate(SDNode *N) {
5564 SDLoc dl(N);
5565 SDValue N0 = N->getOperand(0);
5566 SDValue N1 = N->getOperand(1);
5567 EVT VT = N->getValueType(0);
5568 unsigned Bitsize = VT.getScalarSizeInBits();
5569
5570 // fold (rot x, 0) -> x
5571 if (isNullConstantOrNullSplatConstant(N1))
5572 return N0;
5573
5574 // fold (rot x, c) -> (rot x, c % BitSize)
5575 if (ConstantSDNode *Cst = isConstOrConstSplat(N1)) {
5576 if (Cst->getAPIntValue().uge(Bitsize)) {
5577 uint64_t RotAmt = Cst->getAPIntValue().urem(Bitsize);
5578 return DAG.getNode(N->getOpcode(), dl, VT, N0,
5579 DAG.getConstant(RotAmt, dl, N1.getValueType()));
5580 }
5581 }
5582
5583 // fold (rot* x, (trunc (and y, c))) -> (rot* x, (and (trunc y), (trunc c))).
5584 if (N1.getOpcode() == ISD::TRUNCATE &&
5585 N1.getOperand(0).getOpcode() == ISD::AND) {
5586 if (SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode()))
5587 return DAG.getNode(N->getOpcode(), dl, VT, N0, NewOp1);
5588 }
5589
5590 unsigned NextOp = N0.getOpcode();
5591 // fold (rot* (rot* x, c2), c1) -> (rot* x, c1 +- c2 % bitsize)
5592 if (NextOp == ISD::ROTL || NextOp == ISD::ROTR) {
5593 SDNode *C1 = DAG.isConstantIntBuildVectorOrConstantInt(N1);
5594 SDNode *C2 = DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1));
5595 if (C1 && C2 && C1->getValueType(0) == C2->getValueType(0)) {
5596 EVT ShiftVT = C1->getValueType(0);
5597 bool SameSide = (N->getOpcode() == NextOp);
5598 unsigned CombineOp = SameSide ? ISD::ADD : ISD::SUB;
5599 if (SDValue CombinedShift =
5600 DAG.FoldConstantArithmetic(CombineOp, dl, ShiftVT, C1, C2)) {
5601 SDValue BitsizeC = DAG.getConstant(Bitsize, dl, ShiftVT);
5602 SDValue CombinedShiftNorm = DAG.FoldConstantArithmetic(
5603 ISD::SREM, dl, ShiftVT, CombinedShift.getNode(),
5604 BitsizeC.getNode());
5605 return DAG.getNode(N->getOpcode(), dl, VT, N0->getOperand(0),
5606 CombinedShiftNorm);
5607 }
5608 }
5609 }
5610 return SDValue();
5611}
5612
5613SDValue DAGCombiner::visitSHL(SDNode *N) {
5614 SDValue N0 = N->getOperand(0);
5615 SDValue N1 = N->getOperand(1);
5616 EVT VT = N0.getValueType();
5617 unsigned OpSizeInBits = VT.getScalarSizeInBits();
5618
5619 // fold vector ops
5620 if (VT.isVector()) {
5621 if (SDValue FoldedVOp = SimplifyVBinOp(N))
5622 return FoldedVOp;
5623
5624 BuildVectorSDNode *N1CV = dyn_cast<BuildVectorSDNode>(N1);
5625 // If setcc produces all-one true value then:
5626 // (shl (and (setcc) N01CV) N1CV) -> (and (setcc) N01CV<<N1CV)
5627 if (N1CV && N1CV->isConstant()) {
5628 if (N0.getOpcode() == ISD::AND) {
5629 SDValue N00 = N0->getOperand(0);
5630 SDValue N01 = N0->getOperand(1);
5631 BuildVectorSDNode *N01CV = dyn_cast<BuildVectorSDNode>(N01);
5632
5633 if (N01CV && N01CV->isConstant() && N00.getOpcode() == ISD::SETCC &&
5634 TLI.getBooleanContents(N00.getOperand(0).getValueType()) ==
5635 TargetLowering::ZeroOrNegativeOneBooleanContent) {
5636 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT,
5637 N01CV, N1CV))
5638 return DAG.getNode(ISD::AND, SDLoc(N), VT, N00, C);
5639 }
5640 }
5641 }
5642 }
5643
5644 ConstantSDNode *N1C = isConstOrConstSplat(N1);
5645
5646 // fold (shl c1, c2) -> c1<<c2
5647 ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
5648 if (N0C && N1C && !N1C->isOpaque())
5649 return DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT, N0C, N1C);
5650 // fold (shl 0, x) -> 0
5651 if (isNullConstantOrNullSplatConstant(N0))
5652 return N0;
5653 // fold (shl x, c >= size(x)) -> undef
5654 // NOTE: ALL vector elements must be too big to avoid partial UNDEFs.
5655 auto MatchShiftTooBig = [OpSizeInBits](ConstantSDNode *Val) {
5656 return Val->getAPIntValue().uge(OpSizeInBits);
5657 };
5658 if (ISD::matchUnaryPredicate(N1, MatchShiftTooBig))
5659 return DAG.getUNDEF(VT);
5660 // fold (shl x, 0) -> x
5661 if (N1C && N1C->isNullValue())
5662 return N0;
5663 // fold (shl undef, x) -> 0
5664 if (N0.isUndef())
5665 return DAG.getConstant(0, SDLoc(N), VT);
5666
5667 if (SDValue NewSel = foldBinOpIntoSelect(N))
5668 return NewSel;
5669
5670 // if (shl x, c) is known to be zero, return 0
5671 if (DAG.MaskedValueIsZero(SDValue(N, 0),
5672 APInt::getAllOnesValue(OpSizeInBits)))
5673 return DAG.getConstant(0, SDLoc(N), VT);
5674 // fold (shl x, (trunc (and y, c))) -> (shl x, (and (trunc y), (trunc c))).
5675 if (N1.getOpcode() == ISD::TRUNCATE &&
5676 N1.getOperand(0).getOpcode() == ISD::AND) {
5677 if (SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode()))
5678 return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, NewOp1);
5679 }
5680
5681 if (N1C && SimplifyDemandedBits(SDValue(N, 0)))
5682 return SDValue(N, 0);
5683
5684 // fold (shl (shl x, c1), c2) -> 0 or (shl x, (add c1, c2))
5685 if (N0.getOpcode() == ISD::SHL) {
5686 auto MatchOutOfRange = [OpSizeInBits](ConstantSDNode *LHS,
5687 ConstantSDNode *RHS) {
5688 APInt c1 = LHS->getAPIntValue();
5689 APInt c2 = RHS->getAPIntValue();
5690 zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
5691 return (c1 + c2).uge(OpSizeInBits);
5692 };
5693 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchOutOfRange))
5694 return DAG.getConstant(0, SDLoc(N), VT);
5695
5696 auto MatchInRange = [OpSizeInBits](ConstantSDNode *LHS,
5697 ConstantSDNode *RHS) {
5698 APInt c1 = LHS->getAPIntValue();
5699 APInt c2 = RHS->getAPIntValue();
5700 zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
5701 return (c1 + c2).ult(OpSizeInBits);
5702 };
5703 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchInRange)) {
5704 SDLoc DL(N);
5705 EVT ShiftVT = N1.getValueType();
5706 SDValue Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, N1, N0.getOperand(1));
5707 return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Sum);
5708 }
5709 }
5710
5711 // fold (shl (ext (shl x, c1)), c2) -> (ext (shl x, (add c1, c2)))
5712 // For this to be valid, the second form must not preserve any of the bits
5713 // that are shifted out by the inner shift in the first form. This means
5714 // the outer shift size must be >= the number of bits added by the ext.
5715 // As a corollary, we don't care what kind of ext it is.
5716 if (N1C && (N0.getOpcode() == ISD::ZERO_EXTEND ||
5717 N0.getOpcode() == ISD::ANY_EXTEND ||
5718 N0.getOpcode() == ISD::SIGN_EXTEND) &&
5719 N0.getOperand(0).getOpcode() == ISD::SHL) {
5720 SDValue N0Op0 = N0.getOperand(0);
5721 if (ConstantSDNode *N0Op0C1 = isConstOrConstSplat(N0Op0.getOperand(1))) {
5722 APInt c1 = N0Op0C1->getAPIntValue();
5723 APInt c2 = N1C->getAPIntValue();
5724 zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
5725
5726 EVT InnerShiftVT = N0Op0.getValueType();
5727 uint64_t InnerShiftSize = InnerShiftVT.getScalarSizeInBits();
5728 if (c2.uge(OpSizeInBits - InnerShiftSize)) {
5729 SDLoc DL(N0);
5730 APInt Sum = c1 + c2;
5731 if (Sum.uge(OpSizeInBits))
5732 return DAG.getConstant(0, DL, VT);
5733
5734 return DAG.getNode(
5735 ISD::SHL, DL, VT,
5736 DAG.getNode(N0.getOpcode(), DL, VT, N0Op0->getOperand(0)),
5737 DAG.getConstant(Sum.getZExtValue(), DL, N1.getValueType()));
5738 }
5739 }
5740 }
5741
5742 // fold (shl (zext (srl x, C)), C) -> (zext (shl (srl x, C), C))
5743 // Only fold this if the inner zext has no other uses to avoid increasing
5744 // the total number of instructions.
5745 if (N1C && N0.getOpcode() == ISD::ZERO_EXTEND && N0.hasOneUse() &&
5746 N0.getOperand(0).getOpcode() == ISD::SRL) {
5747 SDValue N0Op0 = N0.getOperand(0);
5748 if (ConstantSDNode *N0Op0C1 = isConstOrConstSplat(N0Op0.getOperand(1))) {
5749 if (N0Op0C1->getAPIntValue().ult(VT.getScalarSizeInBits())) {
5750 uint64_t c1 = N0Op0C1->getZExtValue();
5751 uint64_t c2 = N1C->getZExtValue();
5752 if (c1 == c2) {
5753 SDValue NewOp0 = N0.getOperand(0);
5754 EVT CountVT = NewOp0.getOperand(1).getValueType();
5755 SDLoc DL(N);
5756 SDValue NewSHL = DAG.getNode(ISD::SHL, DL, NewOp0.getValueType(),
5757 NewOp0,
5758 DAG.getConstant(c2, DL, CountVT));
5759 AddToWorklist(NewSHL.getNode());
5760 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N0), VT, NewSHL);
5761 }
5762 }
5763 }
5764 }
5765
5766 // fold (shl (sr[la] exact X, C1), C2) -> (shl X, (C2-C1)) if C1 <= C2
5767 // fold (shl (sr[la] exact X, C1), C2) -> (sr[la] X, (C2-C1)) if C1 > C2
5768 if (N1C && (N0.getOpcode() == ISD::SRL || N0.getOpcode() == ISD::SRA) &&
5769 N0->getFlags().hasExact()) {
5770 if (ConstantSDNode *N0C1 = isConstOrConstSplat(N0.getOperand(1))) {
5771 uint64_t C1 = N0C1->getZExtValue();
5772 uint64_t C2 = N1C->getZExtValue();
5773 SDLoc DL(N);
5774 if (C1 <= C2)
5775 return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0),
5776 DAG.getConstant(C2 - C1, DL, N1.getValueType()));
5777 return DAG.getNode(N0.getOpcode(), DL, VT, N0.getOperand(0),
5778 DAG.getConstant(C1 - C2, DL, N1.getValueType()));
5779 }
5780 }
5781
5782 // fold (shl (srl x, c1), c2) -> (and (shl x, (sub c2, c1), MASK) or
5783 // (and (srl x, (sub c1, c2), MASK)
5784 // Only fold this if the inner shift has no other uses -- if it does, folding
5785 // this will increase the total number of instructions.
5786 if (N1C && N0.getOpcode() == ISD::SRL && N0.hasOneUse()) {
5787 if (ConstantSDNode *N0C1 = isConstOrConstSplat(N0.getOperand(1))) {
5788 uint64_t c1 = N0C1->getZExtValue();
5789 if (c1 < OpSizeInBits) {
5790 uint64_t c2 = N1C->getZExtValue();
5791 APInt Mask = APInt::getHighBitsSet(OpSizeInBits, OpSizeInBits - c1);
5792 SDValue Shift;
5793 if (c2 > c1) {
5794 Mask <<= c2 - c1;
5795 SDLoc DL(N);
5796 Shift = DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0),
5797 DAG.getConstant(c2 - c1, DL, N1.getValueType()));
5798 } else {
5799 Mask.lshrInPlace(c1 - c2);
5800 SDLoc DL(N);
5801 Shift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0),
5802 DAG.getConstant(c1 - c2, DL, N1.getValueType()));
5803 }
5804 SDLoc DL(N0);
5805 return DAG.getNode(ISD::AND, DL, VT, Shift,
5806 DAG.getConstant(Mask, DL, VT));
5807 }
5808 }
5809 }
5810
5811 // fold (shl (sra x, c1), c1) -> (and x, (shl -1, c1))
5812 if (N0.getOpcode() == ISD::SRA && N1 == N0.getOperand(1) &&
5813 isConstantOrConstantVector(N1, /* No Opaques */ true)) {
5814 SDLoc DL(N);
5815 SDValue AllBits = DAG.getAllOnesConstant(DL, VT);
5816 SDValue HiBitsMask = DAG.getNode(ISD::SHL, DL, VT, AllBits, N1);
5817 return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), HiBitsMask);
5818 }
5819
5820 // fold (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2)
5821 // fold (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2)
5822 // Variant of version done on multiply, except mul by a power of 2 is turned
5823 // into a shift.
5824 if ((N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::OR) &&
5825 N0.getNode()->hasOneUse() &&
5826 isConstantOrConstantVector(N1, /* No Opaques */ true) &&
5827 isConstantOrConstantVector(N0.getOperand(1), /* No Opaques */ true)) {
5828 SDValue Shl0 = DAG.getNode(ISD::SHL, SDLoc(N0), VT, N0.getOperand(0), N1);
5829 SDValue Shl1 = DAG.getNode(ISD::SHL, SDLoc(N1), VT, N0.getOperand(1), N1);
5830 AddToWorklist(Shl0.getNode());
5831 AddToWorklist(Shl1.getNode());
5832 return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, Shl0, Shl1);
5833 }
5834
5835 // fold (shl (mul x, c1), c2) -> (mul x, c1 << c2)
5836 if (N0.getOpcode() == ISD::MUL && N0.getNode()->hasOneUse() &&
5837 isConstantOrConstantVector(N1, /* No Opaques */ true) &&
5838 isConstantOrConstantVector(N0.getOperand(1), /* No Opaques */ true)) {
5839 SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(N1), VT, N0.getOperand(1), N1);
5840 if (isConstantOrConstantVector(Shl))
5841 return DAG.getNode(ISD::MUL, SDLoc(N), VT, N0.getOperand(0), Shl);
5842 }
5843
5844 if (N1C && !N1C->isOpaque())
5845 if (SDValue NewSHL = visitShiftByConstant(N, N1C))
5846 return NewSHL;
5847
5848 return SDValue();
5849}
5850
5851SDValue DAGCombiner::visitSRA(SDNode *N) {
5852 SDValue N0 = N->getOperand(0);
5853 SDValue N1 = N->getOperand(1);
5854 EVT VT = N0.getValueType();
5855 unsigned OpSizeInBits = VT.getScalarSizeInBits();
5856
5857 // Arithmetic shifting an all-sign-bit value is a no-op.
5858 // fold (sra 0, x) -> 0
5859 // fold (sra -1, x) -> -1
5860 if (DAG.ComputeNumSignBits(N0) == OpSizeInBits)
5861 return N0;
5862
5863 // fold vector ops
5864 if (VT.isVector())
5865 if (SDValue FoldedVOp = SimplifyVBinOp(N))
5866 return FoldedVOp;
5867
5868 ConstantSDNode *N1C = isConstOrConstSplat(N1);
5869
5870 // fold (sra c1, c2) -> (sra c1, c2)
5871 ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
5872 if (N0C && N1C && !N1C->isOpaque())
5873 return DAG.FoldConstantArithmetic(ISD::SRA, SDLoc(N), VT, N0C, N1C);
5874 // fold (sra x, c >= size(x)) -> undef
5875 // NOTE: ALL vector elements must be too big to avoid partial UNDEFs.
5876 auto MatchShiftTooBig = [OpSizeInBits](ConstantSDNode *Val) {
5877 return Val->getAPIntValue().uge(OpSizeInBits);
5878 };
5879 if (ISD::matchUnaryPredicate(N1, MatchShiftTooBig))
5880 return DAG.getUNDEF(VT);
5881 // fold (sra x, 0) -> x
5882 if (N1C && N1C->isNullValue())
5883 return N0;
5884
5885 if (SDValue NewSel = foldBinOpIntoSelect(N))
5886 return NewSel;
5887
5888 // fold (sra (shl x, c1), c1) -> sext_inreg for some c1 and target supports
5889 // sext_inreg.
5890 if (N1C && N0.getOpcode() == ISD::SHL && N1 == N0.getOperand(1)) {
5891 unsigned LowBits = OpSizeInBits - (unsigned)N1C->getZExtValue();
5892 EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), LowBits);
5893 if (VT.isVector())
5894 ExtVT = EVT::getVectorVT(*DAG.getContext(),
5895 ExtVT, VT.getVectorNumElements());
5896 if ((!LegalOperations ||
5897 TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, ExtVT)))
5898 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT,
5899 N0.getOperand(0), DAG.getValueType(ExtVT));
5900 }
5901
5902 // fold (sra (sra x, c1), c2) -> (sra x, (add c1, c2))
5903 if (N0.getOpcode() == ISD::SRA) {
5904 SDLoc DL(N);
5905 EVT ShiftVT = N1.getValueType();
5906
5907 auto MatchOutOfRange = [OpSizeInBits](ConstantSDNode *LHS,
5908 ConstantSDNode *RHS) {
5909 APInt c1 = LHS->getAPIntValue();
5910 APInt c2 = RHS->getAPIntValue();
5911 zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
5912 return (c1 + c2).uge(OpSizeInBits);
5913 };
5914 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchOutOfRange))
5915 return DAG.getNode(ISD::SRA, DL, VT, N0.getOperand(0),
5916 DAG.getConstant(OpSizeInBits - 1, DL, ShiftVT));
5917
5918 auto MatchInRange = [OpSizeInBits](ConstantSDNode *LHS,
5919 ConstantSDNode *RHS) {
5920 APInt c1 = LHS->getAPIntValue();
5921 APInt c2 = RHS->getAPIntValue();
5922 zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
5923 return (c1 + c2).ult(OpSizeInBits);
5924 };
5925 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchInRange)) {
5926 SDValue Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, N1, N0.getOperand(1));
5927 return DAG.getNode(ISD::SRA, DL, VT, N0.getOperand(0), Sum);
5928 }
5929 }
5930
5931 // fold (sra (shl X, m), (sub result_size, n))