Bug Summary

File:lib/CodeGen/SelectionDAG/DAGCombiner.cpp
Warning:line 154, column 17
The left operand of '!=' is a garbage value

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name DAGCombiner.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-eagerly-assume -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -mrelocation-model pic -pic-level 2 -mthread-model posix -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -momit-leaf-frame-pointer -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-7/lib/clang/7.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-7~svn329677/build-llvm/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-7~svn329677/build-llvm/include -I /build/llvm-toolchain-snapshot-7~svn329677/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/7.3.0/../../../../include/c++/7.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/7.3.0/../../../../include/x86_64-linux-gnu/c++/7.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/7.3.0/../../../../include/x86_64-linux-gnu/c++/7.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/7.3.0/../../../../include/c++/7.3.0/backward -internal-isystem /usr/include/clang/7.0.0/include/ -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-7/lib/clang/7.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++11 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-7~svn329677/build-llvm/lib/CodeGen/SelectionDAG -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-checker optin.performance.Padding -analyzer-output=html -analyzer-config stable-report-filename=true -o /tmp/scan-build-2018-04-11-031539-24776-1 -x c++ /build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

1//===- DAGCombiner.cpp - Implement a DAG node combiner --------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This pass combines dag nodes to form fewer, simpler DAG nodes. It can be run
11// both before and after the DAG is legalized.
12//
13// This pass is not a substitute for the LLVM IR instcombine pass. This pass is
14// primarily intended to handle simplification opportunities that are implicit
15// in the LLVM IR and exposed by the various codegen lowering phases.
16//
17//===----------------------------------------------------------------------===//
18
19#include "llvm/ADT/APFloat.h"
20#include "llvm/ADT/APInt.h"
21#include "llvm/ADT/ArrayRef.h"
22#include "llvm/ADT/DenseMap.h"
23#include "llvm/ADT/None.h"
24#include "llvm/ADT/Optional.h"
25#include "llvm/ADT/STLExtras.h"
26#include "llvm/ADT/SetVector.h"
27#include "llvm/ADT/SmallBitVector.h"
28#include "llvm/ADT/SmallPtrSet.h"
29#include "llvm/ADT/SmallSet.h"
30#include "llvm/ADT/SmallVector.h"
31#include "llvm/ADT/Statistic.h"
32#include "llvm/Analysis/AliasAnalysis.h"
33#include "llvm/Analysis/MemoryLocation.h"
34#include "llvm/CodeGen/DAGCombine.h"
35#include "llvm/CodeGen/ISDOpcodes.h"
36#include "llvm/CodeGen/MachineFrameInfo.h"
37#include "llvm/CodeGen/MachineFunction.h"
38#include "llvm/CodeGen/MachineMemOperand.h"
39#include "llvm/CodeGen/RuntimeLibcalls.h"
40#include "llvm/CodeGen/SelectionDAG.h"
41#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
42#include "llvm/CodeGen/SelectionDAGNodes.h"
43#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
44#include "llvm/CodeGen/TargetLowering.h"
45#include "llvm/CodeGen/TargetRegisterInfo.h"
46#include "llvm/CodeGen/TargetSubtargetInfo.h"
47#include "llvm/CodeGen/ValueTypes.h"
48#include "llvm/IR/Attributes.h"
49#include "llvm/IR/Constant.h"
50#include "llvm/IR/DataLayout.h"
51#include "llvm/IR/DerivedTypes.h"
52#include "llvm/IR/Function.h"
53#include "llvm/IR/LLVMContext.h"
54#include "llvm/IR/Metadata.h"
55#include "llvm/Support/Casting.h"
56#include "llvm/Support/CodeGen.h"
57#include "llvm/Support/CommandLine.h"
58#include "llvm/Support/Compiler.h"
59#include "llvm/Support/Debug.h"
60#include "llvm/Support/ErrorHandling.h"
61#include "llvm/Support/KnownBits.h"
62#include "llvm/Support/MachineValueType.h"
63#include "llvm/Support/MathExtras.h"
64#include "llvm/Support/raw_ostream.h"
65#include "llvm/Target/TargetMachine.h"
66#include "llvm/Target/TargetOptions.h"
67#include <algorithm>
68#include <cassert>
69#include <cstdint>
70#include <functional>
71#include <iterator>
72#include <string>
73#include <tuple>
74#include <utility>
75#include <vector>
76
77using namespace llvm;
78
79#define DEBUG_TYPE"dagcombine" "dagcombine"
80
81STATISTIC(NodesCombined , "Number of dag nodes combined")static llvm::Statistic NodesCombined = {"dagcombine", "NodesCombined"
, "Number of dag nodes combined", {0}, {false}}
;
82STATISTIC(PreIndexedNodes , "Number of pre-indexed nodes created")static llvm::Statistic PreIndexedNodes = {"dagcombine", "PreIndexedNodes"
, "Number of pre-indexed nodes created", {0}, {false}}
;
83STATISTIC(PostIndexedNodes, "Number of post-indexed nodes created")static llvm::Statistic PostIndexedNodes = {"dagcombine", "PostIndexedNodes"
, "Number of post-indexed nodes created", {0}, {false}}
;
84STATISTIC(OpsNarrowed , "Number of load/op/store narrowed")static llvm::Statistic OpsNarrowed = {"dagcombine", "OpsNarrowed"
, "Number of load/op/store narrowed", {0}, {false}}
;
85STATISTIC(LdStFP2Int , "Number of fp load/store pairs transformed to int")static llvm::Statistic LdStFP2Int = {"dagcombine", "LdStFP2Int"
, "Number of fp load/store pairs transformed to int", {0}, {false
}}
;
86STATISTIC(SlicedLoads, "Number of load sliced")static llvm::Statistic SlicedLoads = {"dagcombine", "SlicedLoads"
, "Number of load sliced", {0}, {false}}
;
87
88static cl::opt<bool>
89CombinerGlobalAA("combiner-global-alias-analysis", cl::Hidden,
90 cl::desc("Enable DAG combiner's use of IR alias analysis"));
91
92static cl::opt<bool>
93UseTBAA("combiner-use-tbaa", cl::Hidden, cl::init(true),
94 cl::desc("Enable DAG combiner's use of TBAA"));
95
96#ifndef NDEBUG
97static cl::opt<std::string>
98CombinerAAOnlyFunc("combiner-aa-only-func", cl::Hidden,
99 cl::desc("Only use DAG-combiner alias analysis in this"
100 " function"));
101#endif
102
103/// Hidden option to stress test load slicing, i.e., when this option
104/// is enabled, load slicing bypasses most of its profitability guards.
105static cl::opt<bool>
106StressLoadSlicing("combiner-stress-load-slicing", cl::Hidden,
107 cl::desc("Bypass the profitability model of load slicing"),
108 cl::init(false));
109
110static cl::opt<bool>
111 MaySplitLoadIndex("combiner-split-load-index", cl::Hidden, cl::init(true),
112 cl::desc("DAG combiner may split indexing from loads"));
113
114namespace {
115
116 class DAGCombiner {
117 SelectionDAG &DAG;
118 const TargetLowering &TLI;
119 CombineLevel Level;
120 CodeGenOpt::Level OptLevel;
121 bool LegalOperations = false;
122 bool LegalTypes = false;
123 bool ForCodeSize;
124
125 /// \brief Worklist of all of the nodes that need to be simplified.
126 ///
127 /// This must behave as a stack -- new nodes to process are pushed onto the
128 /// back and when processing we pop off of the back.
129 ///
130 /// The worklist will not contain duplicates but may contain null entries
131 /// due to nodes being deleted from the underlying DAG.
132 SmallVector<SDNode *, 64> Worklist;
133
134 /// \brief Mapping from an SDNode to its position on the worklist.
135 ///
136 /// This is used to find and remove nodes from the worklist (by nulling
137 /// them) when they are deleted from the underlying DAG. It relies on
138 /// stable indices of nodes within the worklist.
139 DenseMap<SDNode *, unsigned> WorklistMap;
140
141 /// \brief Set of nodes which have been combined (at least once).
142 ///
143 /// This is used to allow us to reliably add any operands of a DAG node
144 /// which have not yet been combined to the worklist.
145 SmallPtrSet<SDNode *, 32> CombinedNodes;
146
147 // AA - Used for DAG load/store alias analysis.
148 AliasAnalysis *AA;
149
150 /// When an instruction is simplified, add all users of the instruction to
151 /// the work lists because they might get more simplified now.
152 void AddUsersToWorklist(SDNode *N) {
153 for (SDNode *Node : N->uses())
154 AddToWorklist(Node);
155 }
156
157 /// Call the node-specific routine that folds each particular type of node.
158 SDValue visit(SDNode *N);
159
160 public:
161 DAGCombiner(SelectionDAG &D, AliasAnalysis *AA, CodeGenOpt::Level OL)
162 : DAG(D), TLI(D.getTargetLoweringInfo()), Level(BeforeLegalizeTypes),
163 OptLevel(OL), AA(AA) {
164 ForCodeSize = DAG.getMachineFunction().getFunction().optForSize();
165
166 MaximumLegalStoreInBits = 0;
167 for (MVT VT : MVT::all_valuetypes())
168 if (EVT(VT).isSimple() && VT != MVT::Other &&
169 TLI.isTypeLegal(EVT(VT)) &&
170 VT.getSizeInBits() >= MaximumLegalStoreInBits)
171 MaximumLegalStoreInBits = VT.getSizeInBits();
172 }
173
174 /// Add to the worklist making sure its instance is at the back (next to be
175 /// processed.)
176 void AddToWorklist(SDNode *N) {
177 assert(N->getOpcode() != ISD::DELETED_NODE &&(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& "Deleted Node added to Worklist") ? void (0) : __assert_fail
("N->getOpcode() != ISD::DELETED_NODE && \"Deleted Node added to Worklist\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 178, __extension__ __PRETTY_FUNCTION__))
178 "Deleted Node added to Worklist")(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& "Deleted Node added to Worklist") ? void (0) : __assert_fail
("N->getOpcode() != ISD::DELETED_NODE && \"Deleted Node added to Worklist\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 178, __extension__ __PRETTY_FUNCTION__))
;
179
180 // Skip handle nodes as they can't usefully be combined and confuse the
181 // zero-use deletion strategy.
182 if (N->getOpcode() == ISD::HANDLENODE)
183 return;
184
185 if (WorklistMap.insert(std::make_pair(N, Worklist.size())).second)
186 Worklist.push_back(N);
187 }
188
189 /// Remove all instances of N from the worklist.
190 void removeFromWorklist(SDNode *N) {
191 CombinedNodes.erase(N);
192
193 auto It = WorklistMap.find(N);
194 if (It == WorklistMap.end())
195 return; // Not in the worklist.
196
197 // Null out the entry rather than erasing it to avoid a linear operation.
198 Worklist[It->second] = nullptr;
199 WorklistMap.erase(It);
200 }
201
202 void deleteAndRecombine(SDNode *N);
203 bool recursivelyDeleteUnusedNodes(SDNode *N);
204
205 /// Replaces all uses of the results of one DAG node with new values.
206 SDValue CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
207 bool AddTo = true);
208
209 /// Replaces all uses of the results of one DAG node with new values.
210 SDValue CombineTo(SDNode *N, SDValue Res, bool AddTo = true) {
211 return CombineTo(N, &Res, 1, AddTo);
212 }
213
214 /// Replaces all uses of the results of one DAG node with new values.
215 SDValue CombineTo(SDNode *N, SDValue Res0, SDValue Res1,
216 bool AddTo = true) {
217 SDValue To[] = { Res0, Res1 };
218 return CombineTo(N, To, 2, AddTo);
219 }
220
221 void CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO);
222
223 private:
224 unsigned MaximumLegalStoreInBits;
225
226 /// Check the specified integer node value to see if it can be simplified or
227 /// if things it uses can be simplified by bit propagation.
228 /// If so, return true.
229 bool SimplifyDemandedBits(SDValue Op) {
230 unsigned BitWidth = Op.getScalarValueSizeInBits();
231 APInt Demanded = APInt::getAllOnesValue(BitWidth);
232 return SimplifyDemandedBits(Op, Demanded);
233 }
234
235 /// Check the specified vector node value to see if it can be simplified or
236 /// if things it uses can be simplified as it only uses some of the
237 /// elements. If so, return true.
238 bool SimplifyDemandedVectorElts(SDValue Op) {
239 unsigned NumElts = Op.getValueType().getVectorNumElements();
240 APInt Demanded = APInt::getAllOnesValue(NumElts);
241 return SimplifyDemandedVectorElts(Op, Demanded);
242 }
243
244 bool SimplifyDemandedBits(SDValue Op, const APInt &Demanded);
245 bool SimplifyDemandedVectorElts(SDValue Op, const APInt &Demanded);
246
247 bool CombineToPreIndexedLoadStore(SDNode *N);
248 bool CombineToPostIndexedLoadStore(SDNode *N);
249 SDValue SplitIndexingFromLoad(LoadSDNode *LD);
250 bool SliceUpLoad(SDNode *N);
251
252 /// \brief Replace an ISD::EXTRACT_VECTOR_ELT of a load with a narrowed
253 /// load.
254 ///
255 /// \param EVE ISD::EXTRACT_VECTOR_ELT to be replaced.
256 /// \param InVecVT type of the input vector to EVE with bitcasts resolved.
257 /// \param EltNo index of the vector element to load.
258 /// \param OriginalLoad load that EVE came from to be replaced.
259 /// \returns EVE on success SDValue() on failure.
260 SDValue ReplaceExtractVectorEltOfLoadWithNarrowedLoad(
261 SDNode *EVE, EVT InVecVT, SDValue EltNo, LoadSDNode *OriginalLoad);
262 void ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad);
263 SDValue PromoteOperand(SDValue Op, EVT PVT, bool &Replace);
264 SDValue SExtPromoteOperand(SDValue Op, EVT PVT);
265 SDValue ZExtPromoteOperand(SDValue Op, EVT PVT);
266 SDValue PromoteIntBinOp(SDValue Op);
267 SDValue PromoteIntShiftOp(SDValue Op);
268 SDValue PromoteExtend(SDValue Op);
269 bool PromoteLoad(SDValue Op);
270
271 void ExtendSetCCUses(const SmallVectorImpl<SDNode *> &SetCCs,
272 SDValue OrigLoad, SDValue ExtLoad,
273 const SDLoc &DL,
274 ISD::NodeType ExtType);
275
276 /// Call the node-specific routine that knows how to fold each
277 /// particular type of node. If that doesn't do anything, try the
278 /// target-specific DAG combines.
279 SDValue combine(SDNode *N);
280
281 // Visitation implementation - Implement dag node combining for different
282 // node types. The semantics are as follows:
283 // Return Value:
284 // SDValue.getNode() == 0 - No change was made
285 // SDValue.getNode() == N - N was replaced, is dead and has been handled.
286 // otherwise - N should be replaced by the returned Operand.
287 //
288 SDValue visitTokenFactor(SDNode *N);
289 SDValue visitMERGE_VALUES(SDNode *N);
290 SDValue visitADD(SDNode *N);
291 SDValue visitADDLike(SDValue N0, SDValue N1, SDNode *LocReference);
292 SDValue visitSUB(SDNode *N);
293 SDValue visitADDC(SDNode *N);
294 SDValue visitUADDO(SDNode *N);
295 SDValue visitUADDOLike(SDValue N0, SDValue N1, SDNode *N);
296 SDValue visitSUBC(SDNode *N);
297 SDValue visitUSUBO(SDNode *N);
298 SDValue visitADDE(SDNode *N);
299 SDValue visitADDCARRY(SDNode *N);
300 SDValue visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn, SDNode *N);
301 SDValue visitSUBE(SDNode *N);
302 SDValue visitSUBCARRY(SDNode *N);
303 SDValue visitMUL(SDNode *N);
304 SDValue useDivRem(SDNode *N);
305 SDValue visitSDIV(SDNode *N);
306 SDValue visitUDIV(SDNode *N);
307 SDValue visitREM(SDNode *N);
308 SDValue visitMULHU(SDNode *N);
309 SDValue visitMULHS(SDNode *N);
310 SDValue visitSMUL_LOHI(SDNode *N);
311 SDValue visitUMUL_LOHI(SDNode *N);
312 SDValue visitSMULO(SDNode *N);
313 SDValue visitUMULO(SDNode *N);
314 SDValue visitIMINMAX(SDNode *N);
315 SDValue visitAND(SDNode *N);
316 SDValue visitANDLike(SDValue N0, SDValue N1, SDNode *LocReference);
317 SDValue visitOR(SDNode *N);
318 SDValue visitORLike(SDValue N0, SDValue N1, SDNode *LocReference);
319 SDValue visitXOR(SDNode *N);
320 SDValue SimplifyVBinOp(SDNode *N);
321 SDValue visitSHL(SDNode *N);
322 SDValue visitSRA(SDNode *N);
323 SDValue visitSRL(SDNode *N);
324 SDValue visitRotate(SDNode *N);
325 SDValue visitABS(SDNode *N);
326 SDValue visitBSWAP(SDNode *N);
327 SDValue visitBITREVERSE(SDNode *N);
328 SDValue visitCTLZ(SDNode *N);
329 SDValue visitCTLZ_ZERO_UNDEF(SDNode *N);
330 SDValue visitCTTZ(SDNode *N);
331 SDValue visitCTTZ_ZERO_UNDEF(SDNode *N);
332 SDValue visitCTPOP(SDNode *N);
333 SDValue visitSELECT(SDNode *N);
334 SDValue visitVSELECT(SDNode *N);
335 SDValue visitSELECT_CC(SDNode *N);
336 SDValue visitSETCC(SDNode *N);
337 SDValue visitSETCCE(SDNode *N);
338 SDValue visitSETCCCARRY(SDNode *N);
339 SDValue visitSIGN_EXTEND(SDNode *N);
340 SDValue visitZERO_EXTEND(SDNode *N);
341 SDValue visitANY_EXTEND(SDNode *N);
342 SDValue visitAssertExt(SDNode *N);
343 SDValue visitSIGN_EXTEND_INREG(SDNode *N);
344 SDValue visitSIGN_EXTEND_VECTOR_INREG(SDNode *N);
345 SDValue visitZERO_EXTEND_VECTOR_INREG(SDNode *N);
346 SDValue visitTRUNCATE(SDNode *N);
347 SDValue visitBITCAST(SDNode *N);
348 SDValue visitBUILD_PAIR(SDNode *N);
349 SDValue visitFADD(SDNode *N);
350 SDValue visitFSUB(SDNode *N);
351 SDValue visitFMUL(SDNode *N);
352 SDValue visitFMA(SDNode *N);
353 SDValue visitFDIV(SDNode *N);
354 SDValue visitFREM(SDNode *N);
355 SDValue visitFSQRT(SDNode *N);
356 SDValue visitFCOPYSIGN(SDNode *N);
357 SDValue visitSINT_TO_FP(SDNode *N);
358 SDValue visitUINT_TO_FP(SDNode *N);
359 SDValue visitFP_TO_SINT(SDNode *N);
360 SDValue visitFP_TO_UINT(SDNode *N);
361 SDValue visitFP_ROUND(SDNode *N);
362 SDValue visitFP_ROUND_INREG(SDNode *N);
363 SDValue visitFP_EXTEND(SDNode *N);
364 SDValue visitFNEG(SDNode *N);
365 SDValue visitFABS(SDNode *N);
366 SDValue visitFCEIL(SDNode *N);
367 SDValue visitFTRUNC(SDNode *N);
368 SDValue visitFFLOOR(SDNode *N);
369 SDValue visitFMINNUM(SDNode *N);
370 SDValue visitFMAXNUM(SDNode *N);
371 SDValue visitBRCOND(SDNode *N);
372 SDValue visitBR_CC(SDNode *N);
373 SDValue visitLOAD(SDNode *N);
374
375 SDValue replaceStoreChain(StoreSDNode *ST, SDValue BetterChain);
376 SDValue replaceStoreOfFPConstant(StoreSDNode *ST);
377
378 SDValue visitSTORE(SDNode *N);
379 SDValue visitINSERT_VECTOR_ELT(SDNode *N);
380 SDValue visitEXTRACT_VECTOR_ELT(SDNode *N);
381 SDValue visitBUILD_VECTOR(SDNode *N);
382 SDValue visitCONCAT_VECTORS(SDNode *N);
383 SDValue visitEXTRACT_SUBVECTOR(SDNode *N);
384 SDValue visitVECTOR_SHUFFLE(SDNode *N);
385 SDValue visitSCALAR_TO_VECTOR(SDNode *N);
386 SDValue visitINSERT_SUBVECTOR(SDNode *N);
387 SDValue visitMLOAD(SDNode *N);
388 SDValue visitMSTORE(SDNode *N);
389 SDValue visitMGATHER(SDNode *N);
390 SDValue visitMSCATTER(SDNode *N);
391 SDValue visitFP_TO_FP16(SDNode *N);
392 SDValue visitFP16_TO_FP(SDNode *N);
393
394 SDValue visitFADDForFMACombine(SDNode *N);
395 SDValue visitFSUBForFMACombine(SDNode *N);
396 SDValue visitFMULForFMADistributiveCombine(SDNode *N);
397
398 SDValue XformToShuffleWithZero(SDNode *N);
399 SDValue ReassociateOps(unsigned Opc, const SDLoc &DL, SDValue LHS,
400 SDValue RHS);
401
402 SDValue visitShiftByConstant(SDNode *N, ConstantSDNode *Amt);
403
404 SDValue foldSelectOfConstants(SDNode *N);
405 SDValue foldVSelectOfConstants(SDNode *N);
406 SDValue foldBinOpIntoSelect(SDNode *BO);
407 bool SimplifySelectOps(SDNode *SELECT, SDValue LHS, SDValue RHS);
408 SDValue SimplifyBinOpWithSameOpcodeHands(SDNode *N);
409 SDValue SimplifySelect(const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2);
410 SDValue SimplifySelectCC(const SDLoc &DL, SDValue N0, SDValue N1,
411 SDValue N2, SDValue N3, ISD::CondCode CC,
412 bool NotExtCompare = false);
413 SDValue foldSelectCCToShiftAnd(const SDLoc &DL, SDValue N0, SDValue N1,
414 SDValue N2, SDValue N3, ISD::CondCode CC);
415 SDValue foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1,
416 const SDLoc &DL);
417 SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
418 const SDLoc &DL, bool foldBooleans);
419 SDValue rebuildSetCC(SDValue N);
420
421 bool isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
422 SDValue &CC) const;
423 bool isOneUseSetCC(SDValue N) const;
424
425 SDValue SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
426 unsigned HiOp);
427 SDValue CombineConsecutiveLoads(SDNode *N, EVT VT);
428 SDValue CombineExtLoad(SDNode *N);
429 SDValue CombineZExtLogicopShiftLoad(SDNode *N);
430 SDValue combineRepeatedFPDivisors(SDNode *N);
431 SDValue combineInsertEltToShuffle(SDNode *N, unsigned InsIndex);
432 SDValue ConstantFoldBITCASTofBUILD_VECTOR(SDNode *, EVT);
433 SDValue BuildSDIV(SDNode *N);
434 SDValue BuildSDIVPow2(SDNode *N);
435 SDValue BuildUDIV(SDNode *N);
436 SDValue BuildLogBase2(SDValue Op, const SDLoc &DL);
437 SDValue BuildReciprocalEstimate(SDValue Op, SDNodeFlags Flags);
438 SDValue buildRsqrtEstimate(SDValue Op, SDNodeFlags Flags);
439 SDValue buildSqrtEstimate(SDValue Op, SDNodeFlags Flags);
440 SDValue buildSqrtEstimateImpl(SDValue Op, SDNodeFlags Flags, bool Recip);
441 SDValue buildSqrtNROneConst(SDValue Op, SDValue Est, unsigned Iterations,
442 SDNodeFlags Flags, bool Reciprocal);
443 SDValue buildSqrtNRTwoConst(SDValue Op, SDValue Est, unsigned Iterations,
444 SDNodeFlags Flags, bool Reciprocal);
445 SDValue MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
446 bool DemandHighBits = true);
447 SDValue MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1);
448 SDNode *MatchRotatePosNeg(SDValue Shifted, SDValue Pos, SDValue Neg,
449 SDValue InnerPos, SDValue InnerNeg,
450 unsigned PosOpcode, unsigned NegOpcode,
451 const SDLoc &DL);
452 SDNode *MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL);
453 SDValue MatchLoadCombine(SDNode *N);
454 SDValue ReduceLoadWidth(SDNode *N);
455 SDValue ReduceLoadOpStoreWidth(SDNode *N);
456 SDValue splitMergedValStore(StoreSDNode *ST);
457 SDValue TransformFPLoadStorePair(SDNode *N);
458 SDValue convertBuildVecZextToZext(SDNode *N);
459 SDValue reduceBuildVecExtToExtBuildVec(SDNode *N);
460 SDValue reduceBuildVecConvertToConvertBuildVec(SDNode *N);
461 SDValue reduceBuildVecToShuffle(SDNode *N);
462 SDValue createBuildVecShuffle(const SDLoc &DL, SDNode *N,
463 ArrayRef<int> VectorMask, SDValue VecIn1,
464 SDValue VecIn2, unsigned LeftIdx);
465 SDValue matchVSelectOpSizesWithSetCC(SDNode *N);
466
467 /// Walk up chain skipping non-aliasing memory nodes,
468 /// looking for aliasing nodes and adding them to the Aliases vector.
469 void GatherAllAliases(SDNode *N, SDValue OriginalChain,
470 SmallVectorImpl<SDValue> &Aliases);
471
472 /// Return true if there is any possibility that the two addresses overlap.
473 bool isAlias(LSBaseSDNode *Op0, LSBaseSDNode *Op1) const;
474
475 /// Walk up chain skipping non-aliasing memory nodes, looking for a better
476 /// chain (aliasing node.)
477 SDValue FindBetterChain(SDNode *N, SDValue Chain);
478
479 /// Try to replace a store and any possibly adjacent stores on
480 /// consecutive chains with better chains. Return true only if St is
481 /// replaced.
482 ///
483 /// Notice that other chains may still be replaced even if the function
484 /// returns false.
485 bool findBetterNeighborChains(StoreSDNode *St);
486
487 /// Match "(X shl/srl V1) & V2" where V2 may not be present.
488 bool MatchRotateHalf(SDValue Op, SDValue &Shift, SDValue &Mask);
489
490 /// Holds a pointer to an LSBaseSDNode as well as information on where it
491 /// is located in a sequence of memory operations connected by a chain.
492 struct MemOpLink {
493 // Ptr to the mem node.
494 LSBaseSDNode *MemNode;
495
496 // Offset from the base ptr.
497 int64_t OffsetFromBase;
498
499 MemOpLink(LSBaseSDNode *N, int64_t Offset)
500 : MemNode(N), OffsetFromBase(Offset) {}
501 };
502
503 /// This is a helper function for visitMUL to check the profitability
504 /// of folding (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2).
505 /// MulNode is the original multiply, AddNode is (add x, c1),
506 /// and ConstNode is c2.
507 bool isMulAddWithConstProfitable(SDNode *MulNode,
508 SDValue &AddNode,
509 SDValue &ConstNode);
510
511 /// This is a helper function for visitAND and visitZERO_EXTEND. Returns
512 /// true if the (and (load x) c) pattern matches an extload. ExtVT returns
513 /// the type of the loaded value to be extended.
514 bool isAndLoadExtLoad(ConstantSDNode *AndC, LoadSDNode *LoadN,
515 EVT LoadResultTy, EVT &ExtVT);
516
517 /// Helper function to calculate whether the given Load can have its
518 /// width reduced to ExtVT.
519 bool isLegalNarrowLoad(LoadSDNode *LoadN, ISD::LoadExtType ExtType,
520 EVT &ExtVT, unsigned ShAmt = 0);
521
522 /// Used by BackwardsPropagateMask to find suitable loads.
523 bool SearchForAndLoads(SDNode *N, SmallPtrSetImpl<LoadSDNode*> &Loads,
524 SmallPtrSetImpl<SDNode*> &NodeWithConsts,
525 ConstantSDNode *Mask, SDNode *&UncombinedNode);
526 /// Attempt to propagate a given AND node back to load leaves so that they
527 /// can be combined into narrow loads.
528 bool BackwardsPropagateMask(SDNode *N, SelectionDAG &DAG);
529
530 /// Helper function for MergeConsecutiveStores which merges the
531 /// component store chains.
532 SDValue getMergeStoreChains(SmallVectorImpl<MemOpLink> &StoreNodes,
533 unsigned NumStores);
534
535 /// This is a helper function for MergeConsecutiveStores. When the
536 /// source elements of the consecutive stores are all constants or
537 /// all extracted vector elements, try to merge them into one
538 /// larger store introducing bitcasts if necessary. \return True
539 /// if a merged store was created.
540 bool MergeStoresOfConstantsOrVecElts(SmallVectorImpl<MemOpLink> &StoreNodes,
541 EVT MemVT, unsigned NumStores,
542 bool IsConstantSrc, bool UseVector,
543 bool UseTrunc);
544
545 /// This is a helper function for MergeConsecutiveStores. Stores
546 /// that potentially may be merged with St are placed in
547 /// StoreNodes.
548 void getStoreMergeCandidates(StoreSDNode *St,
549 SmallVectorImpl<MemOpLink> &StoreNodes);
550
551 /// Helper function for MergeConsecutiveStores. Checks if
552 /// candidate stores have indirect dependency through their
553 /// operands. \return True if safe to merge.
554 bool checkMergeStoreCandidatesForDependencies(
555 SmallVectorImpl<MemOpLink> &StoreNodes, unsigned NumStores);
556
557 /// Merge consecutive store operations into a wide store.
558 /// This optimization uses wide integers or vectors when possible.
559 /// \return number of stores that were merged into a merged store (the
560 /// affected nodes are stored as a prefix in \p StoreNodes).
561 bool MergeConsecutiveStores(StoreSDNode *N);
562
563 /// \brief Try to transform a truncation where C is a constant:
564 /// (trunc (and X, C)) -> (and (trunc X), (trunc C))
565 ///
566 /// \p N needs to be a truncation and its first operand an AND. Other
567 /// requirements are checked by the function (e.g. that trunc is
568 /// single-use) and if missed an empty SDValue is returned.
569 SDValue distributeTruncateThroughAnd(SDNode *N);
570
571 public:
572 /// Runs the dag combiner on all nodes in the work list
573 void Run(CombineLevel AtLevel);
574
575 SelectionDAG &getDAG() const { return DAG; }
576
577 /// Returns a type large enough to hold any valid shift amount - before type
578 /// legalization these can be huge.
579 EVT getShiftAmountTy(EVT LHSTy) {
580 assert(LHSTy.isInteger() && "Shift amount is not an integer type!")(static_cast <bool> (LHSTy.isInteger() && "Shift amount is not an integer type!"
) ? void (0) : __assert_fail ("LHSTy.isInteger() && \"Shift amount is not an integer type!\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 580, __extension__ __PRETTY_FUNCTION__))
;
581 return TLI.getShiftAmountTy(LHSTy, DAG.getDataLayout(), LegalTypes);
582 }
583
584 /// This method returns true if we are running before type legalization or
585 /// if the specified VT is legal.
586 bool isTypeLegal(const EVT &VT) {
587 if (!LegalTypes) return true;
588 return TLI.isTypeLegal(VT);
589 }
590
591 /// Convenience wrapper around TargetLowering::getSetCCResultType
592 EVT getSetCCResultType(EVT VT) const {
593 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
594 }
595 };
596
597/// This class is a DAGUpdateListener that removes any deleted
598/// nodes from the worklist.
599class WorklistRemover : public SelectionDAG::DAGUpdateListener {
600 DAGCombiner &DC;
601
602public:
603 explicit WorklistRemover(DAGCombiner &dc)
604 : SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
605
606 void NodeDeleted(SDNode *N, SDNode *E) override {
607 DC.removeFromWorklist(N);
608 }
609};
610
611} // end anonymous namespace
612
613//===----------------------------------------------------------------------===//
614// TargetLowering::DAGCombinerInfo implementation
615//===----------------------------------------------------------------------===//
616
617void TargetLowering::DAGCombinerInfo::AddToWorklist(SDNode *N) {
618 ((DAGCombiner*)DC)->AddToWorklist(N);
619}
620
621SDValue TargetLowering::DAGCombinerInfo::
622CombineTo(SDNode *N, ArrayRef<SDValue> To, bool AddTo) {
623 return ((DAGCombiner*)DC)->CombineTo(N, &To[0], To.size(), AddTo);
624}
625
626SDValue TargetLowering::DAGCombinerInfo::
627CombineTo(SDNode *N, SDValue Res, bool AddTo) {
628 return ((DAGCombiner*)DC)->CombineTo(N, Res, AddTo);
629}
630
631SDValue TargetLowering::DAGCombinerInfo::
632CombineTo(SDNode *N, SDValue Res0, SDValue Res1, bool AddTo) {
633 return ((DAGCombiner*)DC)->CombineTo(N, Res0, Res1, AddTo);
634}
635
636void TargetLowering::DAGCombinerInfo::
637CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
638 return ((DAGCombiner*)DC)->CommitTargetLoweringOpt(TLO);
639}
640
641//===----------------------------------------------------------------------===//
642// Helper Functions
643//===----------------------------------------------------------------------===//
644
645void DAGCombiner::deleteAndRecombine(SDNode *N) {
646 removeFromWorklist(N);
647
648 // If the operands of this node are only used by the node, they will now be
649 // dead. Make sure to re-visit them and recursively delete dead nodes.
650 for (const SDValue &Op : N->ops())
651 // For an operand generating multiple values, one of the values may
652 // become dead allowing further simplification (e.g. split index
653 // arithmetic from an indexed load).
654 if (Op->hasOneUse() || Op->getNumValues() > 1)
655 AddToWorklist(Op.getNode());
656
657 DAG.DeleteNode(N);
658}
659
660/// Return 1 if we can compute the negated form of the specified expression for
661/// the same cost as the expression itself, or 2 if we can compute the negated
662/// form more cheaply than the expression itself.
663static char isNegatibleForFree(SDValue Op, bool LegalOperations,
664 const TargetLowering &TLI,
665 const TargetOptions *Options,
666 unsigned Depth = 0) {
667 // fneg is removable even if it has multiple uses.
668 if (Op.getOpcode() == ISD::FNEG) return 2;
669
670 // Don't allow anything with multiple uses.
671 if (!Op.hasOneUse()) return 0;
672
673 // Don't recurse exponentially.
674 if (Depth > 6) return 0;
675
676 switch (Op.getOpcode()) {
677 default: return false;
678 case ISD::ConstantFP: {
679 if (!LegalOperations)
680 return 1;
681
682 // Don't invert constant FP values after legalization unless the target says
683 // the negated constant is legal.
684 EVT VT = Op.getValueType();
685 return TLI.isOperationLegal(ISD::ConstantFP, VT) ||
686 TLI.isFPImmLegal(neg(cast<ConstantFPSDNode>(Op)->getValueAPF()), VT);
687 }
688 case ISD::FADD:
689 // FIXME: determine better conditions for this xform.
690 if (!Options->UnsafeFPMath) return 0;
691
692 // After operation legalization, it might not be legal to create new FSUBs.
693 if (LegalOperations &&
694 !TLI.isOperationLegalOrCustom(ISD::FSUB, Op.getValueType()))
695 return 0;
696
697 // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
698 if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI,
699 Options, Depth + 1))
700 return V;
701 // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
702 return isNegatibleForFree(Op.getOperand(1), LegalOperations, TLI, Options,
703 Depth + 1);
704 case ISD::FSUB:
705 // We can't turn -(A-B) into B-A when we honor signed zeros.
706 if (!Options->NoSignedZerosFPMath &&
707 !Op.getNode()->getFlags().hasNoSignedZeros())
708 return 0;
709
710 // fold (fneg (fsub A, B)) -> (fsub B, A)
711 return 1;
712
713 case ISD::FMUL:
714 case ISD::FDIV:
715 if (Options->HonorSignDependentRoundingFPMath()) return 0;
716
717 // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y) or (fmul X, (fneg Y))
718 if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI,
719 Options, Depth + 1))
720 return V;
721
722 return isNegatibleForFree(Op.getOperand(1), LegalOperations, TLI, Options,
723 Depth + 1);
724
725 case ISD::FP_EXTEND:
726 case ISD::FP_ROUND:
727 case ISD::FSIN:
728 return isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI, Options,
729 Depth + 1);
730 }
731}
732
733/// If isNegatibleForFree returns true, return the newly negated expression.
734static SDValue GetNegatedExpression(SDValue Op, SelectionDAG &DAG,
735 bool LegalOperations, unsigned Depth = 0) {
736 const TargetOptions &Options = DAG.getTarget().Options;
737 // fneg is removable even if it has multiple uses.
738 if (Op.getOpcode() == ISD::FNEG) return Op.getOperand(0);
739
740 // Don't allow anything with multiple uses.
741 assert(Op.hasOneUse() && "Unknown reuse!")(static_cast <bool> (Op.hasOneUse() && "Unknown reuse!"
) ? void (0) : __assert_fail ("Op.hasOneUse() && \"Unknown reuse!\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 741, __extension__ __PRETTY_FUNCTION__))
;
742
743 assert(Depth <= 6 && "GetNegatedExpression doesn't match isNegatibleForFree")(static_cast <bool> (Depth <= 6 && "GetNegatedExpression doesn't match isNegatibleForFree"
) ? void (0) : __assert_fail ("Depth <= 6 && \"GetNegatedExpression doesn't match isNegatibleForFree\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 743, __extension__ __PRETTY_FUNCTION__))
;
744
745 const SDNodeFlags Flags = Op.getNode()->getFlags();
746
747 switch (Op.getOpcode()) {
748 default: llvm_unreachable("Unknown code")::llvm::llvm_unreachable_internal("Unknown code", "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 748)
;
749 case ISD::ConstantFP: {
750 APFloat V = cast<ConstantFPSDNode>(Op)->getValueAPF();
751 V.changeSign();
752 return DAG.getConstantFP(V, SDLoc(Op), Op.getValueType());
753 }
754 case ISD::FADD:
755 // FIXME: determine better conditions for this xform.
756 assert(Options.UnsafeFPMath)(static_cast <bool> (Options.UnsafeFPMath) ? void (0) :
__assert_fail ("Options.UnsafeFPMath", "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 756, __extension__ __PRETTY_FUNCTION__))
;
757
758 // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
759 if (isNegatibleForFree(Op.getOperand(0), LegalOperations,
760 DAG.getTargetLoweringInfo(), &Options, Depth+1))
761 return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
762 GetNegatedExpression(Op.getOperand(0), DAG,
763 LegalOperations, Depth+1),
764 Op.getOperand(1), Flags);
765 // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
766 return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
767 GetNegatedExpression(Op.getOperand(1), DAG,
768 LegalOperations, Depth+1),
769 Op.getOperand(0), Flags);
770 case ISD::FSUB:
771 // fold (fneg (fsub 0, B)) -> B
772 if (ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(Op.getOperand(0)))
773 if (N0CFP->isZero())
774 return Op.getOperand(1);
775
776 // fold (fneg (fsub A, B)) -> (fsub B, A)
777 return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
778 Op.getOperand(1), Op.getOperand(0), Flags);
779
780 case ISD::FMUL:
781 case ISD::FDIV:
782 assert(!Options.HonorSignDependentRoundingFPMath())(static_cast <bool> (!Options.HonorSignDependentRoundingFPMath
()) ? void (0) : __assert_fail ("!Options.HonorSignDependentRoundingFPMath()"
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 782, __extension__ __PRETTY_FUNCTION__))
;
783
784 // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y)
785 if (isNegatibleForFree(Op.getOperand(0), LegalOperations,
786 DAG.getTargetLoweringInfo(), &Options, Depth+1))
787 return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
788 GetNegatedExpression(Op.getOperand(0), DAG,
789 LegalOperations, Depth+1),
790 Op.getOperand(1), Flags);
791
792 // fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y))
793 return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
794 Op.getOperand(0),
795 GetNegatedExpression(Op.getOperand(1), DAG,
796 LegalOperations, Depth+1), Flags);
797
798 case ISD::FP_EXTEND:
799 case ISD::FSIN:
800 return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
801 GetNegatedExpression(Op.getOperand(0), DAG,
802 LegalOperations, Depth+1));
803 case ISD::FP_ROUND:
804 return DAG.getNode(ISD::FP_ROUND, SDLoc(Op), Op.getValueType(),
805 GetNegatedExpression(Op.getOperand(0), DAG,
806 LegalOperations, Depth+1),
807 Op.getOperand(1));
808 }
809}
810
811// APInts must be the same size for most operations, this helper
812// function zero extends the shorter of the pair so that they match.
813// We provide an Offset so that we can create bitwidths that won't overflow.
814static void zeroExtendToMatch(APInt &LHS, APInt &RHS, unsigned Offset = 0) {
815 unsigned Bits = Offset + std::max(LHS.getBitWidth(), RHS.getBitWidth());
816 LHS = LHS.zextOrSelf(Bits);
817 RHS = RHS.zextOrSelf(Bits);
818}
819
820// Return true if this node is a setcc, or is a select_cc
821// that selects between the target values used for true and false, making it
822// equivalent to a setcc. Also, set the incoming LHS, RHS, and CC references to
823// the appropriate nodes based on the type of node we are checking. This
824// simplifies life a bit for the callers.
825bool DAGCombiner::isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
826 SDValue &CC) const {
827 if (N.getOpcode() == ISD::SETCC) {
828 LHS = N.getOperand(0);
829 RHS = N.getOperand(1);
830 CC = N.getOperand(2);
831 return true;
832 }
833
834 if (N.getOpcode() != ISD::SELECT_CC ||
835 !TLI.isConstTrueVal(N.getOperand(2).getNode()) ||
836 !TLI.isConstFalseVal(N.getOperand(3).getNode()))
837 return false;
838
839 if (TLI.getBooleanContents(N.getValueType()) ==
840 TargetLowering::UndefinedBooleanContent)
841 return false;
842
843 LHS = N.getOperand(0);
844 RHS = N.getOperand(1);
845 CC = N.getOperand(4);
846 return true;
847}
848
849/// Return true if this is a SetCC-equivalent operation with only one use.
850/// If this is true, it allows the users to invert the operation for free when
851/// it is profitable to do so.
852bool DAGCombiner::isOneUseSetCC(SDValue N) const {
853 SDValue N0, N1, N2;
854 if (isSetCCEquivalent(N, N0, N1, N2) && N.getNode()->hasOneUse())
855 return true;
856 return false;
857}
858
859// \brief Returns the SDNode if it is a constant float BuildVector
860// or constant float.
861static SDNode *isConstantFPBuildVectorOrConstantFP(SDValue N) {
862 if (isa<ConstantFPSDNode>(N))
863 return N.getNode();
864 if (ISD::isBuildVectorOfConstantFPSDNodes(N.getNode()))
865 return N.getNode();
866 return nullptr;
867}
868
869// Determines if it is a constant integer or a build vector of constant
870// integers (and undefs).
871// Do not permit build vector implicit truncation.
872static bool isConstantOrConstantVector(SDValue N, bool NoOpaques = false) {
873 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N))
874 return !(Const->isOpaque() && NoOpaques);
875 if (N.getOpcode() != ISD::BUILD_VECTOR)
876 return false;
877 unsigned BitWidth = N.getScalarValueSizeInBits();
878 for (const SDValue &Op : N->op_values()) {
879 if (Op.isUndef())
880 continue;
881 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Op);
882 if (!Const || Const->getAPIntValue().getBitWidth() != BitWidth ||
883 (Const->isOpaque() && NoOpaques))
884 return false;
885 }
886 return true;
887}
888
889// Determines if it is a constant null integer or a splatted vector of a
890// constant null integer (with no undefs).
891// Build vector implicit truncation is not an issue for null values.
892static bool isNullConstantOrNullSplatConstant(SDValue N) {
893 if (ConstantSDNode *Splat = isConstOrConstSplat(N))
894 return Splat->isNullValue();
895 return false;
896}
897
898// Determines if it is a constant integer of one or a splatted vector of a
899// constant integer of one (with no undefs).
900// Do not permit build vector implicit truncation.
901static bool isOneConstantOrOneSplatConstant(SDValue N) {
902 unsigned BitWidth = N.getScalarValueSizeInBits();
903 if (ConstantSDNode *Splat = isConstOrConstSplat(N))
904 return Splat->isOne() && Splat->getAPIntValue().getBitWidth() == BitWidth;
905 return false;
906}
907
908// Determines if it is a constant integer of all ones or a splatted vector of a
909// constant integer of all ones (with no undefs).
910// Do not permit build vector implicit truncation.
911static bool isAllOnesConstantOrAllOnesSplatConstant(SDValue N) {
912 unsigned BitWidth = N.getScalarValueSizeInBits();
913 if (ConstantSDNode *Splat = isConstOrConstSplat(N))
914 return Splat->isAllOnesValue() &&
915 Splat->getAPIntValue().getBitWidth() == BitWidth;
916 return false;
917}
918
919// Determines if a BUILD_VECTOR is composed of all-constants possibly mixed with
920// undef's.
921static bool isAnyConstantBuildVector(const SDNode *N) {
922 return ISD::isBuildVectorOfConstantSDNodes(N) ||
923 ISD::isBuildVectorOfConstantFPSDNodes(N);
924}
925
926SDValue DAGCombiner::ReassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0,
927 SDValue N1) {
928 EVT VT = N0.getValueType();
929 if (N0.getOpcode() == Opc) {
930 if (SDNode *L = DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1))) {
931 if (SDNode *R = DAG.isConstantIntBuildVectorOrConstantInt(N1)) {
932 // reassoc. (op (op x, c1), c2) -> (op x, (op c1, c2))
933 if (SDValue OpNode = DAG.FoldConstantArithmetic(Opc, DL, VT, L, R))
934 return DAG.getNode(Opc, DL, VT, N0.getOperand(0), OpNode);
935 return SDValue();
936 }
937 if (N0.hasOneUse()) {
938 // reassoc. (op (op x, c1), y) -> (op (op x, y), c1) iff x+c1 has one
939 // use
940 SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N0.getOperand(0), N1);
941 if (!OpNode.getNode())
942 return SDValue();
943 AddToWorklist(OpNode.getNode());
944 return DAG.getNode(Opc, DL, VT, OpNode, N0.getOperand(1));
945 }
946 }
947 }
948
949 if (N1.getOpcode() == Opc) {
950 if (SDNode *R = DAG.isConstantIntBuildVectorOrConstantInt(N1.getOperand(1))) {
951 if (SDNode *L = DAG.isConstantIntBuildVectorOrConstantInt(N0)) {
952 // reassoc. (op c2, (op x, c1)) -> (op x, (op c1, c2))
953 if (SDValue OpNode = DAG.FoldConstantArithmetic(Opc, DL, VT, R, L))
954 return DAG.getNode(Opc, DL, VT, N1.getOperand(0), OpNode);
955 return SDValue();
956 }
957 if (N1.hasOneUse()) {
958 // reassoc. (op x, (op y, c1)) -> (op (op x, y), c1) iff x+c1 has one
959 // use
960 SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N0, N1.getOperand(0));
961 if (!OpNode.getNode())
962 return SDValue();
963 AddToWorklist(OpNode.getNode());
964 return DAG.getNode(Opc, DL, VT, OpNode, N1.getOperand(1));
965 }
966 }
967 }
968
969 return SDValue();
970}
971
972SDValue DAGCombiner::CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
973 bool AddTo) {
974 assert(N->getNumValues() == NumTo && "Broken CombineTo call!")(static_cast <bool> (N->getNumValues() == NumTo &&
"Broken CombineTo call!") ? void (0) : __assert_fail ("N->getNumValues() == NumTo && \"Broken CombineTo call!\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 974, __extension__ __PRETTY_FUNCTION__))
;
975 ++NodesCombined;
976 DEBUG(dbgs() << "\nReplacing.1 ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.1 "; N->dump
(&DAG); dbgs() << "\nWith: "; To[0].getNode()->dump
(&DAG); dbgs() << " and " << NumTo-1 <<
" other values\n"; } } while (false)
977 N->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.1 "; N->dump
(&DAG); dbgs() << "\nWith: "; To[0].getNode()->dump
(&DAG); dbgs() << " and " << NumTo-1 <<
" other values\n"; } } while (false)
978 dbgs() << "\nWith: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.1 "; N->dump
(&DAG); dbgs() << "\nWith: "; To[0].getNode()->dump
(&DAG); dbgs() << " and " << NumTo-1 <<
" other values\n"; } } while (false)
979 To[0].getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.1 "; N->dump
(&DAG); dbgs() << "\nWith: "; To[0].getNode()->dump
(&DAG); dbgs() << " and " << NumTo-1 <<
" other values\n"; } } while (false)
980 dbgs() << " and " << NumTo-1 << " other values\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.1 "; N->dump
(&DAG); dbgs() << "\nWith: "; To[0].getNode()->dump
(&DAG); dbgs() << " and " << NumTo-1 <<
" other values\n"; } } while (false)
;
981 for (unsigned i = 0, e = NumTo; i != e; ++i)
982 assert((!To[i].getNode() ||(static_cast <bool> ((!To[i].getNode() || N->getValueType
(i) == To[i].getValueType()) && "Cannot combine value to value of different type!"
) ? void (0) : __assert_fail ("(!To[i].getNode() || N->getValueType(i) == To[i].getValueType()) && \"Cannot combine value to value of different type!\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 984, __extension__ __PRETTY_FUNCTION__))
983 N->getValueType(i) == To[i].getValueType()) &&(static_cast <bool> ((!To[i].getNode() || N->getValueType
(i) == To[i].getValueType()) && "Cannot combine value to value of different type!"
) ? void (0) : __assert_fail ("(!To[i].getNode() || N->getValueType(i) == To[i].getValueType()) && \"Cannot combine value to value of different type!\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 984, __extension__ __PRETTY_FUNCTION__))
984 "Cannot combine value to value of different type!")(static_cast <bool> ((!To[i].getNode() || N->getValueType
(i) == To[i].getValueType()) && "Cannot combine value to value of different type!"
) ? void (0) : __assert_fail ("(!To[i].getNode() || N->getValueType(i) == To[i].getValueType()) && \"Cannot combine value to value of different type!\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 984, __extension__ __PRETTY_FUNCTION__))
;
985
986 WorklistRemover DeadNodes(*this);
987 DAG.ReplaceAllUsesWith(N, To);
988 if (AddTo) {
989 // Push the new nodes and any users onto the worklist
990 for (unsigned i = 0, e = NumTo; i != e; ++i) {
991 if (To[i].getNode()) {
992 AddToWorklist(To[i].getNode());
993 AddUsersToWorklist(To[i].getNode());
994 }
995 }
996 }
997
998 // Finally, if the node is now dead, remove it from the graph. The node
999 // may not be dead if the replacement process recursively simplified to
1000 // something else needing this node.
1001 if (N->use_empty())
1002 deleteAndRecombine(N);
1003 return SDValue(N, 0);
1004}
1005
1006void DAGCombiner::
1007CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
1008 // Replace all uses. If any nodes become isomorphic to other nodes and
1009 // are deleted, make sure to remove them from our worklist.
1010 WorklistRemover DeadNodes(*this);
1011 DAG.ReplaceAllUsesOfValueWith(TLO.Old, TLO.New);
1012
1013 // Push the new node and any (possibly new) users onto the worklist.
1014 AddToWorklist(TLO.New.getNode());
1015 AddUsersToWorklist(TLO.New.getNode());
1016
1017 // Finally, if the node is now dead, remove it from the graph. The node
1018 // may not be dead if the replacement process recursively simplified to
1019 // something else needing this node.
1020 if (TLO.Old.getNode()->use_empty())
1021 deleteAndRecombine(TLO.Old.getNode());
1022}
1023
1024/// Check the specified integer node value to see if it can be simplified or if
1025/// things it uses can be simplified by bit propagation. If so, return true.
1026bool DAGCombiner::SimplifyDemandedBits(SDValue Op, const APInt &Demanded) {
1027 TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations);
1028 KnownBits Known;
1029 if (!TLI.SimplifyDemandedBits(Op, Demanded, Known, TLO))
1030 return false;
1031
1032 // Revisit the node.
1033 AddToWorklist(Op.getNode());
1034
1035 // Replace the old value with the new one.
1036 ++NodesCombined;
1037 DEBUG(dbgs() << "\nReplacing.2 ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.getNode
()->dump(&DAG); dbgs() << "\nWith: "; TLO.New.getNode
()->dump(&DAG); dbgs() << '\n'; } } while (false
)
1038 TLO.Old.getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.getNode
()->dump(&DAG); dbgs() << "\nWith: "; TLO.New.getNode
()->dump(&DAG); dbgs() << '\n'; } } while (false
)
1039 dbgs() << "\nWith: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.getNode
()->dump(&DAG); dbgs() << "\nWith: "; TLO.New.getNode
()->dump(&DAG); dbgs() << '\n'; } } while (false
)
1040 TLO.New.getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.getNode
()->dump(&DAG); dbgs() << "\nWith: "; TLO.New.getNode
()->dump(&DAG); dbgs() << '\n'; } } while (false
)
1041 dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.getNode
()->dump(&DAG); dbgs() << "\nWith: "; TLO.New.getNode
()->dump(&DAG); dbgs() << '\n'; } } while (false
)
;
1042
1043 CommitTargetLoweringOpt(TLO);
1044 return true;
1045}
1046
1047/// Check the specified vector node value to see if it can be simplified or
1048/// if things it uses can be simplified as it only uses some of the elements.
1049/// If so, return true.
1050bool DAGCombiner::SimplifyDemandedVectorElts(SDValue Op,
1051 const APInt &Demanded) {
1052 TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations);
1053 APInt KnownUndef, KnownZero;
1054 if (!TLI.SimplifyDemandedVectorElts(Op, Demanded, KnownUndef, KnownZero, TLO))
1055 return false;
1056
1057 // Revisit the node.
1058 AddToWorklist(Op.getNode());
1059
1060 // Replace the old value with the new one.
1061 ++NodesCombined;
1062 DEBUG(dbgs() << "\nReplacing.2 "; TLO.Old.getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.getNode
()->dump(&DAG); dbgs() << "\nWith: "; TLO.New.getNode
()->dump(&DAG); dbgs() << '\n'; } } while (false
)
1063 dbgs() << "\nWith: "; TLO.New.getNode()->dump(&DAG); dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.getNode
()->dump(&DAG); dbgs() << "\nWith: "; TLO.New.getNode
()->dump(&DAG); dbgs() << '\n'; } } while (false
)
;
1064
1065 CommitTargetLoweringOpt(TLO);
1066 return true;
1067}
1068
1069void DAGCombiner::ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad) {
1070 SDLoc DL(Load);
1071 EVT VT = Load->getValueType(0);
1072 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, VT, SDValue(ExtLoad, 0));
1073
1074 DEBUG(dbgs() << "\nReplacing.9 ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.9 "; Load->
dump(&DAG); dbgs() << "\nWith: "; Trunc.getNode()->
dump(&DAG); dbgs() << '\n'; } } while (false)
1075 Load->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.9 "; Load->
dump(&DAG); dbgs() << "\nWith: "; Trunc.getNode()->
dump(&DAG); dbgs() << '\n'; } } while (false)
1076 dbgs() << "\nWith: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.9 "; Load->
dump(&DAG); dbgs() << "\nWith: "; Trunc.getNode()->
dump(&DAG); dbgs() << '\n'; } } while (false)
1077 Trunc.getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.9 "; Load->
dump(&DAG); dbgs() << "\nWith: "; Trunc.getNode()->
dump(&DAG); dbgs() << '\n'; } } while (false)
1078 dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.9 "; Load->
dump(&DAG); dbgs() << "\nWith: "; Trunc.getNode()->
dump(&DAG); dbgs() << '\n'; } } while (false)
;
1079 WorklistRemover DeadNodes(*this);
1080 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), Trunc);
1081 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), SDValue(ExtLoad, 1));
1082 deleteAndRecombine(Load);
1083 AddToWorklist(Trunc.getNode());
1084}
1085
1086SDValue DAGCombiner::PromoteOperand(SDValue Op, EVT PVT, bool &Replace) {
1087 Replace = false;
1088 SDLoc DL(Op);
1089 if (ISD::isUNINDEXEDLoad(Op.getNode())) {
1090 LoadSDNode *LD = cast<LoadSDNode>(Op);
1091 EVT MemVT = LD->getMemoryVT();
1092 ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD)
1093 ? (TLI.isLoadExtLegal(ISD::ZEXTLOAD, PVT, MemVT) ? ISD::ZEXTLOAD
1094 : ISD::EXTLOAD)
1095 : LD->getExtensionType();
1096 Replace = true;
1097 return DAG.getExtLoad(ExtType, DL, PVT,
1098 LD->getChain(), LD->getBasePtr(),
1099 MemVT, LD->getMemOperand());
1100 }
1101
1102 unsigned Opc = Op.getOpcode();
1103 switch (Opc) {
1104 default: break;
1105 case ISD::AssertSext:
1106 if (SDValue Op0 = SExtPromoteOperand(Op.getOperand(0), PVT))
1107 return DAG.getNode(ISD::AssertSext, DL, PVT, Op0, Op.getOperand(1));
1108 break;
1109 case ISD::AssertZext:
1110 if (SDValue Op0 = ZExtPromoteOperand(Op.getOperand(0), PVT))
1111 return DAG.getNode(ISD::AssertZext, DL, PVT, Op0, Op.getOperand(1));
1112 break;
1113 case ISD::Constant: {
1114 unsigned ExtOpc =
1115 Op.getValueType().isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
1116 return DAG.getNode(ExtOpc, DL, PVT, Op);
1117 }
1118 }
1119
1120 if (!TLI.isOperationLegal(ISD::ANY_EXTEND, PVT))
1121 return SDValue();
1122 return DAG.getNode(ISD::ANY_EXTEND, DL, PVT, Op);
1123}
1124
1125SDValue DAGCombiner::SExtPromoteOperand(SDValue Op, EVT PVT) {
1126 if (!TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, PVT))
1127 return SDValue();
1128 EVT OldVT = Op.getValueType();
1129 SDLoc DL(Op);
1130 bool Replace = false;
1131 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
1132 if (!NewOp.getNode())
1133 return SDValue();
1134 AddToWorklist(NewOp.getNode());
1135
1136 if (Replace)
1137 ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
1138 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, NewOp.getValueType(), NewOp,
1139 DAG.getValueType(OldVT));
1140}
1141
1142SDValue DAGCombiner::ZExtPromoteOperand(SDValue Op, EVT PVT) {
1143 EVT OldVT = Op.getValueType();
1144 SDLoc DL(Op);
1145 bool Replace = false;
1146 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
1147 if (!NewOp.getNode())
1148 return SDValue();
1149 AddToWorklist(NewOp.getNode());
1150
1151 if (Replace)
1152 ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
1153 return DAG.getZeroExtendInReg(NewOp, DL, OldVT);
1154}
1155
1156/// Promote the specified integer binary operation if the target indicates it is
1157/// beneficial. e.g. On x86, it's usually better to promote i16 operations to
1158/// i32 since i16 instructions are longer.
1159SDValue DAGCombiner::PromoteIntBinOp(SDValue Op) {
1160 if (!LegalOperations)
1161 return SDValue();
1162
1163 EVT VT = Op.getValueType();
1164 if (VT.isVector() || !VT.isInteger())
1165 return SDValue();
1166
1167 // If operation type is 'undesirable', e.g. i16 on x86, consider
1168 // promoting it.
1169 unsigned Opc = Op.getOpcode();
1170 if (TLI.isTypeDesirableForOp(Opc, VT))
1171 return SDValue();
1172
1173 EVT PVT = VT;
1174 // Consult target whether it is a good idea to promote this operation and
1175 // what's the right type to promote it to.
1176 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1177 assert(PVT != VT && "Don't know what type to promote to!")(static_cast <bool> (PVT != VT && "Don't know what type to promote to!"
) ? void (0) : __assert_fail ("PVT != VT && \"Don't know what type to promote to!\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1177, __extension__ __PRETTY_FUNCTION__))
;
1178
1179 DEBUG(dbgs() << "\nPromoting "; Op.getNode()->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; Op.getNode(
)->dump(&DAG); } } while (false)
;
1180
1181 bool Replace0 = false;
1182 SDValue N0 = Op.getOperand(0);
1183 SDValue NN0 = PromoteOperand(N0, PVT, Replace0);
1184
1185 bool Replace1 = false;
1186 SDValue N1 = Op.getOperand(1);
1187 SDValue NN1 = PromoteOperand(N1, PVT, Replace1);
1188 SDLoc DL(Op);
1189
1190 SDValue RV =
1191 DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, NN0, NN1));
1192
1193 // We are always replacing N0/N1's use in N and only need
1194 // additional replacements if there are additional uses.
1195 Replace0 &= !N0->hasOneUse();
1196 Replace1 &= (N0 != N1) && !N1->hasOneUse();
1197
1198 // Combine Op here so it is preserved past replacements.
1199 CombineTo(Op.getNode(), RV);
1200
1201 // If operands have a use ordering, make sure we deal with
1202 // predecessor first.
1203 if (Replace0 && Replace1 && N0.getNode()->isPredecessorOf(N1.getNode())) {
1204 std::swap(N0, N1);
1205 std::swap(NN0, NN1);
1206 }
1207
1208 if (Replace0) {
1209 AddToWorklist(NN0.getNode());
1210 ReplaceLoadWithPromotedLoad(N0.getNode(), NN0.getNode());
1211 }
1212 if (Replace1) {
1213 AddToWorklist(NN1.getNode());
1214 ReplaceLoadWithPromotedLoad(N1.getNode(), NN1.getNode());
1215 }
1216 return Op;
1217 }
1218 return SDValue();
1219}
1220
1221/// Promote the specified integer shift operation if the target indicates it is
1222/// beneficial. e.g. On x86, it's usually better to promote i16 operations to
1223/// i32 since i16 instructions are longer.
1224SDValue DAGCombiner::PromoteIntShiftOp(SDValue Op) {
1225 if (!LegalOperations)
1226 return SDValue();
1227
1228 EVT VT = Op.getValueType();
1229 if (VT.isVector() || !VT.isInteger())
1230 return SDValue();
1231
1232 // If operation type is 'undesirable', e.g. i16 on x86, consider
1233 // promoting it.
1234 unsigned Opc = Op.getOpcode();
1235 if (TLI.isTypeDesirableForOp(Opc, VT))
1236 return SDValue();
1237
1238 EVT PVT = VT;
1239 // Consult target whether it is a good idea to promote this operation and
1240 // what's the right type to promote it to.
1241 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1242 assert(PVT != VT && "Don't know what type to promote to!")(static_cast <bool> (PVT != VT && "Don't know what type to promote to!"
) ? void (0) : __assert_fail ("PVT != VT && \"Don't know what type to promote to!\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1242, __extension__ __PRETTY_FUNCTION__))
;
1243
1244 DEBUG(dbgs() << "\nPromoting "; Op.getNode()->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; Op.getNode(
)->dump(&DAG); } } while (false)
;
1245
1246 bool Replace = false;
1247 SDValue N0 = Op.getOperand(0);
1248 SDValue N1 = Op.getOperand(1);
1249 if (Opc == ISD::SRA)
1250 N0 = SExtPromoteOperand(N0, PVT);
1251 else if (Opc == ISD::SRL)
1252 N0 = ZExtPromoteOperand(N0, PVT);
1253 else
1254 N0 = PromoteOperand(N0, PVT, Replace);
1255
1256 if (!N0.getNode())
1257 return SDValue();
1258
1259 SDLoc DL(Op);
1260 SDValue RV =
1261 DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, N0, N1));
1262
1263 AddToWorklist(N0.getNode());
1264 if (Replace)
1265 ReplaceLoadWithPromotedLoad(Op.getOperand(0).getNode(), N0.getNode());
1266
1267 // Deal with Op being deleted.
1268 if (Op && Op.getOpcode() != ISD::DELETED_NODE)
1269 return RV;
1270 }
1271 return SDValue();
1272}
1273
1274SDValue DAGCombiner::PromoteExtend(SDValue Op) {
1275 if (!LegalOperations)
1276 return SDValue();
1277
1278 EVT VT = Op.getValueType();
1279 if (VT.isVector() || !VT.isInteger())
1280 return SDValue();
1281
1282 // If operation type is 'undesirable', e.g. i16 on x86, consider
1283 // promoting it.
1284 unsigned Opc = Op.getOpcode();
1285 if (TLI.isTypeDesirableForOp(Opc, VT))
1286 return SDValue();
1287
1288 EVT PVT = VT;
1289 // Consult target whether it is a good idea to promote this operation and
1290 // what's the right type to promote it to.
1291 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1292 assert(PVT != VT && "Don't know what type to promote to!")(static_cast <bool> (PVT != VT && "Don't know what type to promote to!"
) ? void (0) : __assert_fail ("PVT != VT && \"Don't know what type to promote to!\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1292, __extension__ __PRETTY_FUNCTION__))
;
1293 // fold (aext (aext x)) -> (aext x)
1294 // fold (aext (zext x)) -> (zext x)
1295 // fold (aext (sext x)) -> (sext x)
1296 DEBUG(dbgs() << "\nPromoting ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; Op.getNode(
)->dump(&DAG); } } while (false)
1297 Op.getNode()->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; Op.getNode(
)->dump(&DAG); } } while (false)
;
1298 return DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, Op.getOperand(0));
1299 }
1300 return SDValue();
1301}
1302
1303bool DAGCombiner::PromoteLoad(SDValue Op) {
1304 if (!LegalOperations)
1305 return false;
1306
1307 if (!ISD::isUNINDEXEDLoad(Op.getNode()))
1308 return false;
1309
1310 EVT VT = Op.getValueType();
1311 if (VT.isVector() || !VT.isInteger())
1312 return false;
1313
1314 // If operation type is 'undesirable', e.g. i16 on x86, consider
1315 // promoting it.
1316 unsigned Opc = Op.getOpcode();
1317 if (TLI.isTypeDesirableForOp(Opc, VT))
1318 return false;
1319
1320 EVT PVT = VT;
1321 // Consult target whether it is a good idea to promote this operation and
1322 // what's the right type to promote it to.
1323 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1324 assert(PVT != VT && "Don't know what type to promote to!")(static_cast <bool> (PVT != VT && "Don't know what type to promote to!"
) ? void (0) : __assert_fail ("PVT != VT && \"Don't know what type to promote to!\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1324, __extension__ __PRETTY_FUNCTION__))
;
1325
1326 SDLoc DL(Op);
1327 SDNode *N = Op.getNode();
1328 LoadSDNode *LD = cast<LoadSDNode>(N);
1329 EVT MemVT = LD->getMemoryVT();
1330 ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD)
1331 ? (TLI.isLoadExtLegal(ISD::ZEXTLOAD, PVT, MemVT) ? ISD::ZEXTLOAD
1332 : ISD::EXTLOAD)
1333 : LD->getExtensionType();
1334 SDValue NewLD = DAG.getExtLoad(ExtType, DL, PVT,
1335 LD->getChain(), LD->getBasePtr(),
1336 MemVT, LD->getMemOperand());
1337 SDValue Result = DAG.getNode(ISD::TRUNCATE, DL, VT, NewLD);
1338
1339 DEBUG(dbgs() << "\nPromoting ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; N->dump(
&DAG); dbgs() << "\nTo: "; Result.getNode()->dump
(&DAG); dbgs() << '\n'; } } while (false)
1340 N->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; N->dump(
&DAG); dbgs() << "\nTo: "; Result.getNode()->dump
(&DAG); dbgs() << '\n'; } } while (false)
1341 dbgs() << "\nTo: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; N->dump(
&DAG); dbgs() << "\nTo: "; Result.getNode()->dump
(&DAG); dbgs() << '\n'; } } while (false)
1342 Result.getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; N->dump(
&DAG); dbgs() << "\nTo: "; Result.getNode()->dump
(&DAG); dbgs() << '\n'; } } while (false)
1343 dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; N->dump(
&DAG); dbgs() << "\nTo: "; Result.getNode()->dump
(&DAG); dbgs() << '\n'; } } while (false)
;
1344 WorklistRemover DeadNodes(*this);
1345 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result);
1346 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), NewLD.getValue(1));
1347 deleteAndRecombine(N);
1348 AddToWorklist(Result.getNode());
1349 return true;
1350 }
1351 return false;
1352}
1353
1354/// \brief Recursively delete a node which has no uses and any operands for
1355/// which it is the only use.
1356///
1357/// Note that this both deletes the nodes and removes them from the worklist.
1358/// It also adds any nodes who have had a user deleted to the worklist as they
1359/// may now have only one use and subject to other combines.
1360bool DAGCombiner::recursivelyDeleteUnusedNodes(SDNode *N) {
1361 if (!N->use_empty())
1362 return false;
1363
1364 SmallSetVector<SDNode *, 16> Nodes;
1365 Nodes.insert(N);
1366 do {
1367 N = Nodes.pop_back_val();
1368 if (!N)
1369 continue;
1370
1371 if (N->use_empty()) {
1372 for (const SDValue &ChildN : N->op_values())
1373 Nodes.insert(ChildN.getNode());
1374
1375 removeFromWorklist(N);
1376 DAG.DeleteNode(N);
1377 } else {
1378 AddToWorklist(N);
1379 }
1380 } while (!Nodes.empty());
1381 return true;
1382}
1383
1384//===----------------------------------------------------------------------===//
1385// Main DAG Combiner implementation
1386//===----------------------------------------------------------------------===//
1387
1388void DAGCombiner::Run(CombineLevel AtLevel) {
1389 // set the instance variables, so that the various visit routines may use it.
1390 Level = AtLevel;
1391 LegalOperations = Level >= AfterLegalizeVectorOps;
1392 LegalTypes = Level >= AfterLegalizeTypes;
1393
1394 // Add all the dag nodes to the worklist.
1395 for (SDNode &Node : DAG.allnodes())
1396 AddToWorklist(&Node);
1397
1398 // Create a dummy node (which is not added to allnodes), that adds a reference
1399 // to the root node, preventing it from being deleted, and tracking any
1400 // changes of the root.
1401 HandleSDNode Dummy(DAG.getRoot());
1402
1403 // While the worklist isn't empty, find a node and try to combine it.
1404 while (!WorklistMap.empty()) {
1405 SDNode *N;
1406 // The Worklist holds the SDNodes in order, but it may contain null entries.
1407 do {
1408 N = Worklist.pop_back_val();
1409 } while (!N);
1410
1411 bool GoodWorklistEntry = WorklistMap.erase(N);
1412 (void)GoodWorklistEntry;
1413 assert(GoodWorklistEntry &&(static_cast <bool> (GoodWorklistEntry && "Found a worklist entry without a corresponding map entry!"
) ? void (0) : __assert_fail ("GoodWorklistEntry && \"Found a worklist entry without a corresponding map entry!\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1414, __extension__ __PRETTY_FUNCTION__))
1414 "Found a worklist entry without a corresponding map entry!")(static_cast <bool> (GoodWorklistEntry && "Found a worklist entry without a corresponding map entry!"
) ? void (0) : __assert_fail ("GoodWorklistEntry && \"Found a worklist entry without a corresponding map entry!\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1414, __extension__ __PRETTY_FUNCTION__))
;
1415
1416 // If N has no uses, it is dead. Make sure to revisit all N's operands once
1417 // N is deleted from the DAG, since they too may now be dead or may have a
1418 // reduced number of uses, allowing other xforms.
1419 if (recursivelyDeleteUnusedNodes(N))
1420 continue;
1421
1422 WorklistRemover DeadNodes(*this);
1423
1424 // If this combine is running after legalizing the DAG, re-legalize any
1425 // nodes pulled off the worklist.
1426 if (Level == AfterLegalizeDAG) {
1427 SmallSetVector<SDNode *, 16> UpdatedNodes;
1428 bool NIsValid = DAG.LegalizeOp(N, UpdatedNodes);
1429
1430 for (SDNode *LN : UpdatedNodes) {
1431 AddToWorklist(LN);
1432 AddUsersToWorklist(LN);
1433 }
1434 if (!NIsValid)
1435 continue;
1436 }
1437
1438 DEBUG(dbgs() << "\nCombining: "; N->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nCombining: "; N->dump
(&DAG); } } while (false)
;
1439
1440 // Add any operands of the new node which have not yet been combined to the
1441 // worklist as well. Because the worklist uniques things already, this
1442 // won't repeatedly process the same operand.
1443 CombinedNodes.insert(N);
1444 for (const SDValue &ChildN : N->op_values())
1445 if (!CombinedNodes.count(ChildN.getNode()))
1446 AddToWorklist(ChildN.getNode());
1447
1448 SDValue RV = combine(N);
1449
1450 if (!RV.getNode())
1451 continue;
1452
1453 ++NodesCombined;
1454
1455 // If we get back the same node we passed in, rather than a new node or
1456 // zero, we know that the node must have defined multiple values and
1457 // CombineTo was used. Since CombineTo takes care of the worklist
1458 // mechanics for us, we have no work to do in this case.
1459 if (RV.getNode() == N)
1460 continue;
1461
1462 assert(N->getOpcode() != ISD::DELETED_NODE &&(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& RV.getOpcode() != ISD::DELETED_NODE && "Node was deleted but visit returned new node!"
) ? void (0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && RV.getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned new node!\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1464, __extension__ __PRETTY_FUNCTION__))
1463 RV.getOpcode() != ISD::DELETED_NODE &&(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& RV.getOpcode() != ISD::DELETED_NODE && "Node was deleted but visit returned new node!"
) ? void (0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && RV.getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned new node!\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1464, __extension__ __PRETTY_FUNCTION__))
1464 "Node was deleted but visit returned new node!")(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& RV.getOpcode() != ISD::DELETED_NODE && "Node was deleted but visit returned new node!"
) ? void (0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && RV.getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned new node!\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1464, __extension__ __PRETTY_FUNCTION__))
;
1465
1466 DEBUG(dbgs() << " ... into: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << " ... into: "; RV.getNode()
->dump(&DAG); } } while (false)
1467 RV.getNode()->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << " ... into: "; RV.getNode()
->dump(&DAG); } } while (false)
;
1468
1469 if (N->getNumValues() == RV.getNode()->getNumValues())
1470 DAG.ReplaceAllUsesWith(N, RV.getNode());
1471 else {
1472 assert(N->getValueType(0) == RV.getValueType() &&(static_cast <bool> (N->getValueType(0) == RV.getValueType
() && N->getNumValues() == 1 && "Type mismatch"
) ? void (0) : __assert_fail ("N->getValueType(0) == RV.getValueType() && N->getNumValues() == 1 && \"Type mismatch\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1473, __extension__ __PRETTY_FUNCTION__))
1473 N->getNumValues() == 1 && "Type mismatch")(static_cast <bool> (N->getValueType(0) == RV.getValueType
() && N->getNumValues() == 1 && "Type mismatch"
) ? void (0) : __assert_fail ("N->getValueType(0) == RV.getValueType() && N->getNumValues() == 1 && \"Type mismatch\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1473, __extension__ __PRETTY_FUNCTION__))
;
1474 DAG.ReplaceAllUsesWith(N, &RV);
1475 }
1476
1477 // Push the new node and any users onto the worklist
1478 AddToWorklist(RV.getNode());
1479 AddUsersToWorklist(RV.getNode());
1480
1481 // Finally, if the node is now dead, remove it from the graph. The node
1482 // may not be dead if the replacement process recursively simplified to
1483 // something else needing this node. This will also take care of adding any
1484 // operands which have lost a user to the worklist.
1485 recursivelyDeleteUnusedNodes(N);
1486 }
1487
1488 // If the root changed (e.g. it was a dead load, update the root).
1489 DAG.setRoot(Dummy.getValue());
1490 DAG.RemoveDeadNodes();
1491}
1492
1493SDValue DAGCombiner::visit(SDNode *N) {
1494 switch (N->getOpcode()) {
1495 default: break;
1496 case ISD::TokenFactor: return visitTokenFactor(N);
1497 case ISD::MERGE_VALUES: return visitMERGE_VALUES(N);
1498 case ISD::ADD: return visitADD(N);
1499 case ISD::SUB: return visitSUB(N);
1500 case ISD::ADDC: return visitADDC(N);
1501 case ISD::UADDO: return visitUADDO(N);
1502 case ISD::SUBC: return visitSUBC(N);
1503 case ISD::USUBO: return visitUSUBO(N);
1504 case ISD::ADDE: return visitADDE(N);
1505 case ISD::ADDCARRY: return visitADDCARRY(N);
1506 case ISD::SUBE: return visitSUBE(N);
1507 case ISD::SUBCARRY: return visitSUBCARRY(N);
1508 case ISD::MUL: return visitMUL(N);
1509 case ISD::SDIV: return visitSDIV(N);
1510 case ISD::UDIV: return visitUDIV(N);
1511 case ISD::SREM:
1512 case ISD::UREM: return visitREM(N);
1513 case ISD::MULHU: return visitMULHU(N);
1514 case ISD::MULHS: return visitMULHS(N);
1515 case ISD::SMUL_LOHI: return visitSMUL_LOHI(N);
1516 case ISD::UMUL_LOHI: return visitUMUL_LOHI(N);
1517 case ISD::SMULO: return visitSMULO(N);
1518 case ISD::UMULO: return visitUMULO(N);
1519 case ISD::SMIN:
1520 case ISD::SMAX:
1521 case ISD::UMIN:
1522 case ISD::UMAX: return visitIMINMAX(N);
1523 case ISD::AND: return visitAND(N);
1524 case ISD::OR: return visitOR(N);
1525 case ISD::XOR: return visitXOR(N);
1526 case ISD::SHL: return visitSHL(N);
1527 case ISD::SRA: return visitSRA(N);
1528 case ISD::SRL: return visitSRL(N);
1529 case ISD::ROTR:
1530 case ISD::ROTL: return visitRotate(N);
1531 case ISD::ABS: return visitABS(N);
1532 case ISD::BSWAP: return visitBSWAP(N);
1533 case ISD::BITREVERSE: return visitBITREVERSE(N);
1534 case ISD::CTLZ: return visitCTLZ(N);
1535 case ISD::CTLZ_ZERO_UNDEF: return visitCTLZ_ZERO_UNDEF(N);
1536 case ISD::CTTZ: return visitCTTZ(N);
1537 case ISD::CTTZ_ZERO_UNDEF: return visitCTTZ_ZERO_UNDEF(N);
1538 case ISD::CTPOP: return visitCTPOP(N);
1539 case ISD::SELECT: return visitSELECT(N);
1540 case ISD::VSELECT: return visitVSELECT(N);
1541 case ISD::SELECT_CC: return visitSELECT_CC(N);
1542 case ISD::SETCC: return visitSETCC(N);
1543 case ISD::SETCCE: return visitSETCCE(N);
1544 case ISD::SETCCCARRY: return visitSETCCCARRY(N);
1545 case ISD::SIGN_EXTEND: return visitSIGN_EXTEND(N);
1546 case ISD::ZERO_EXTEND: return visitZERO_EXTEND(N);
1547 case ISD::ANY_EXTEND: return visitANY_EXTEND(N);
1548 case ISD::AssertSext:
1549 case ISD::AssertZext: return visitAssertExt(N);
1550 case ISD::SIGN_EXTEND_INREG: return visitSIGN_EXTEND_INREG(N);
1551 case ISD::SIGN_EXTEND_VECTOR_INREG: return visitSIGN_EXTEND_VECTOR_INREG(N);
1552 case ISD::ZERO_EXTEND_VECTOR_INREG: return visitZERO_EXTEND_VECTOR_INREG(N);
1553 case ISD::TRUNCATE: return visitTRUNCATE(N);
1554 case ISD::BITCAST: return visitBITCAST(N);
1555 case ISD::BUILD_PAIR: return visitBUILD_PAIR(N);
1556 case ISD::FADD: return visitFADD(N);
1557 case ISD::FSUB: return visitFSUB(N);
1558 case ISD::FMUL: return visitFMUL(N);
1559 case ISD::FMA: return visitFMA(N);
1560 case ISD::FDIV: return visitFDIV(N);
1561 case ISD::FREM: return visitFREM(N);
1562 case ISD::FSQRT: return visitFSQRT(N);
1563 case ISD::FCOPYSIGN: return visitFCOPYSIGN(N);
1564 case ISD::SINT_TO_FP: return visitSINT_TO_FP(N);
1565 case ISD::UINT_TO_FP: return visitUINT_TO_FP(N);
1566 case ISD::FP_TO_SINT: return visitFP_TO_SINT(N);
1567 case ISD::FP_TO_UINT: return visitFP_TO_UINT(N);
1568 case ISD::FP_ROUND: return visitFP_ROUND(N);
1569 case ISD::FP_ROUND_INREG: return visitFP_ROUND_INREG(N);
1570 case ISD::FP_EXTEND: return visitFP_EXTEND(N);
1571 case ISD::FNEG: return visitFNEG(N);
1572 case ISD::FABS: return visitFABS(N);
1573 case ISD::FFLOOR: return visitFFLOOR(N);
1574 case ISD::FMINNUM: return visitFMINNUM(N);
1575 case ISD::FMAXNUM: return visitFMAXNUM(N);
1576 case ISD::FCEIL: return visitFCEIL(N);
1577 case ISD::FTRUNC: return visitFTRUNC(N);
1578 case ISD::BRCOND: return visitBRCOND(N);
1579 case ISD::BR_CC: return visitBR_CC(N);
1580 case ISD::LOAD: return visitLOAD(N);
1581 case ISD::STORE: return visitSTORE(N);
1582 case ISD::INSERT_VECTOR_ELT: return visitINSERT_VECTOR_ELT(N);
1583 case ISD::EXTRACT_VECTOR_ELT: return visitEXTRACT_VECTOR_ELT(N);
1584 case ISD::BUILD_VECTOR: return visitBUILD_VECTOR(N);
1585 case ISD::CONCAT_VECTORS: return visitCONCAT_VECTORS(N);
1586 case ISD::EXTRACT_SUBVECTOR: return visitEXTRACT_SUBVECTOR(N);
1587 case ISD::VECTOR_SHUFFLE: return visitVECTOR_SHUFFLE(N);
1588 case ISD::SCALAR_TO_VECTOR: return visitSCALAR_TO_VECTOR(N);
1589 case ISD::INSERT_SUBVECTOR: return visitINSERT_SUBVECTOR(N);
1590 case ISD::MGATHER: return visitMGATHER(N);
1591 case ISD::MLOAD: return visitMLOAD(N);
1592 case ISD::MSCATTER: return visitMSCATTER(N);
1593 case ISD::MSTORE: return visitMSTORE(N);
1594 case ISD::FP_TO_FP16: return visitFP_TO_FP16(N);
1595 case ISD::FP16_TO_FP: return visitFP16_TO_FP(N);
1596 }
1597 return SDValue();
1598}
1599
1600SDValue DAGCombiner::combine(SDNode *N) {
1601 SDValue RV = visit(N);
1602
1603 // If nothing happened, try a target-specific DAG combine.
1604 if (!RV.getNode()) {
1605 assert(N->getOpcode() != ISD::DELETED_NODE &&(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& "Node was deleted but visit returned NULL!") ? void
(0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned NULL!\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1606, __extension__ __PRETTY_FUNCTION__))
1606 "Node was deleted but visit returned NULL!")(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& "Node was deleted but visit returned NULL!") ? void
(0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned NULL!\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1606, __extension__ __PRETTY_FUNCTION__))
;
1607
1608 if (N->getOpcode() >= ISD::BUILTIN_OP_END ||
1609 TLI.hasTargetDAGCombine((ISD::NodeType)N->getOpcode())) {
1610
1611 // Expose the DAG combiner to the target combiner impls.
1612 TargetLowering::DAGCombinerInfo
1613 DagCombineInfo(DAG, Level, false, this);
1614
1615 RV = TLI.PerformDAGCombine(N, DagCombineInfo);
1616 }
1617 }
1618
1619 // If nothing happened still, try promoting the operation.
1620 if (!RV.getNode()) {
1621 switch (N->getOpcode()) {
1622 default: break;
1623 case ISD::ADD:
1624 case ISD::SUB:
1625 case ISD::MUL:
1626 case ISD::AND:
1627 case ISD::OR:
1628 case ISD::XOR:
1629 RV = PromoteIntBinOp(SDValue(N, 0));
1630 break;
1631 case ISD::SHL:
1632 case ISD::SRA:
1633 case ISD::SRL:
1634 RV = PromoteIntShiftOp(SDValue(N, 0));
1635 break;
1636 case ISD::SIGN_EXTEND:
1637 case ISD::ZERO_EXTEND:
1638 case ISD::ANY_EXTEND:
1639 RV = PromoteExtend(SDValue(N, 0));
1640 break;
1641 case ISD::LOAD:
1642 if (PromoteLoad(SDValue(N, 0)))
1643 RV = SDValue(N, 0);
1644 break;
1645 }
1646 }
1647
1648 // If N is a commutative binary node, try eliminate it if the commuted
1649 // version is already present in the DAG.
1650 if (!RV.getNode() && TLI.isCommutativeBinOp(N->getOpcode()) &&
1651 N->getNumValues() == 1) {
1652 SDValue N0 = N->getOperand(0);
1653 SDValue N1 = N->getOperand(1);
1654
1655 // Constant operands are canonicalized to RHS.
1656 if (N0 != N1 && (isa<ConstantSDNode>(N0) || !isa<ConstantSDNode>(N1))) {
1657 SDValue Ops[] = {N1, N0};
1658 SDNode *CSENode = DAG.getNodeIfExists(N->getOpcode(), N->getVTList(), Ops,
1659 N->getFlags());
1660 if (CSENode)
1661 return SDValue(CSENode, 0);
1662 }
1663 }
1664
1665 return RV;
1666}
1667
1668/// Given a node, return its input chain if it has one, otherwise return a null
1669/// sd operand.
1670static SDValue getInputChainForNode(SDNode *N) {
1671 if (unsigned NumOps = N->getNumOperands()) {
1672 if (N->getOperand(0).getValueType() == MVT::Other)
1673 return N->getOperand(0);
1674 if (N->getOperand(NumOps-1).getValueType() == MVT::Other)
1675 return N->getOperand(NumOps-1);
1676 for (unsigned i = 1; i < NumOps-1; ++i)
1677 if (N->getOperand(i).getValueType() == MVT::Other)
1678 return N->getOperand(i);
1679 }
1680 return SDValue();
1681}
1682
1683SDValue DAGCombiner::visitTokenFactor(SDNode *N) {
1684 // If N has two operands, where one has an input chain equal to the other,
1685 // the 'other' chain is redundant.
1686 if (N->getNumOperands() == 2) {
1687 if (getInputChainForNode(N->getOperand(0).getNode()) == N->getOperand(1))
1688 return N->getOperand(0);
1689 if (getInputChainForNode(N->getOperand(1).getNode()) == N->getOperand(0))
1690 return N->getOperand(1);
1691 }
1692
1693 SmallVector<SDNode *, 8> TFs; // List of token factors to visit.
1694 SmallVector<SDValue, 8> Ops; // Ops for replacing token factor.
1695 SmallPtrSet<SDNode*, 16> SeenOps;
1696 bool Changed = false; // If we should replace this token factor.
1697
1698 // Start out with this token factor.
1699 TFs.push_back(N);
1700
1701 // Iterate through token factors. The TFs grows when new token factors are
1702 // encountered.
1703 for (unsigned i = 0; i < TFs.size(); ++i) {
1704 SDNode *TF = TFs[i];
1705
1706 // Check each of the operands.
1707 for (const SDValue &Op : TF->op_values()) {
1708 switch (Op.getOpcode()) {
1709 case ISD::EntryToken:
1710 // Entry tokens don't need to be added to the list. They are
1711 // redundant.
1712 Changed = true;
1713 break;
1714
1715 case ISD::TokenFactor:
1716 if (Op.hasOneUse() && !is_contained(TFs, Op.getNode())) {
1717 // Queue up for processing.
1718 TFs.push_back(Op.getNode());
1719 // Clean up in case the token factor is removed.
1720 AddToWorklist(Op.getNode());
1721 Changed = true;
1722 break;
1723 }
1724 LLVM_FALLTHROUGH[[clang::fallthrough]];
1725
1726 default:
1727 // Only add if it isn't already in the list.
1728 if (SeenOps.insert(Op.getNode()).second)
1729 Ops.push_back(Op);
1730 else
1731 Changed = true;
1732 break;
1733 }
1734 }
1735 }
1736
1737 // Remove Nodes that are chained to another node in the list. Do so
1738 // by walking up chains breath-first stopping when we've seen
1739 // another operand. In general we must climb to the EntryNode, but we can exit
1740 // early if we find all remaining work is associated with just one operand as
1741 // no further pruning is possible.
1742
1743 // List of nodes to search through and original Ops from which they originate.
1744 SmallVector<std::pair<SDNode *, unsigned>, 8> Worklist;
1745 SmallVector<unsigned, 8> OpWorkCount; // Count of work for each Op.
1746 SmallPtrSet<SDNode *, 16> SeenChains;
1747 bool DidPruneOps = false;
1748
1749 unsigned NumLeftToConsider = 0;
1750 for (const SDValue &Op : Ops) {
1751 Worklist.push_back(std::make_pair(Op.getNode(), NumLeftToConsider++));
1752 OpWorkCount.push_back(1);
1753 }
1754
1755 auto AddToWorklist = [&](unsigned CurIdx, SDNode *Op, unsigned OpNumber) {
1756 // If this is an Op, we can remove the op from the list. Remark any
1757 // search associated with it as from the current OpNumber.
1758 if (SeenOps.count(Op) != 0) {
1759 Changed = true;
1760 DidPruneOps = true;
1761 unsigned OrigOpNumber = 0;
1762 while (OrigOpNumber < Ops.size() && Ops[OrigOpNumber].getNode() != Op)
1763 OrigOpNumber++;
1764 assert((OrigOpNumber != Ops.size()) &&(static_cast <bool> ((OrigOpNumber != Ops.size()) &&
"expected to find TokenFactor Operand") ? void (0) : __assert_fail
("(OrigOpNumber != Ops.size()) && \"expected to find TokenFactor Operand\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1765, __extension__ __PRETTY_FUNCTION__))
1765 "expected to find TokenFactor Operand")(static_cast <bool> ((OrigOpNumber != Ops.size()) &&
"expected to find TokenFactor Operand") ? void (0) : __assert_fail
("(OrigOpNumber != Ops.size()) && \"expected to find TokenFactor Operand\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1765, __extension__ __PRETTY_FUNCTION__))
;
1766 // Re-mark worklist from OrigOpNumber to OpNumber
1767 for (unsigned i = CurIdx + 1; i < Worklist.size(); ++i) {
1768 if (Worklist[i].second == OrigOpNumber) {
1769 Worklist[i].second = OpNumber;
1770 }
1771 }
1772 OpWorkCount[OpNumber] += OpWorkCount[OrigOpNumber];
1773 OpWorkCount[OrigOpNumber] = 0;
1774 NumLeftToConsider--;
1775 }
1776 // Add if it's a new chain
1777 if (SeenChains.insert(Op).second) {
1778 OpWorkCount[OpNumber]++;
1779 Worklist.push_back(std::make_pair(Op, OpNumber));
1780 }
1781 };
1782
1783 for (unsigned i = 0; i < Worklist.size() && i < 1024; ++i) {
1784 // We need at least be consider at least 2 Ops to prune.
1785 if (NumLeftToConsider <= 1)
1786 break;
1787 auto CurNode = Worklist[i].first;
1788 auto CurOpNumber = Worklist[i].second;
1789 assert((OpWorkCount[CurOpNumber] > 0) &&(static_cast <bool> ((OpWorkCount[CurOpNumber] > 0) &&
"Node should not appear in worklist") ? void (0) : __assert_fail
("(OpWorkCount[CurOpNumber] > 0) && \"Node should not appear in worklist\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1790, __extension__ __PRETTY_FUNCTION__))
1790 "Node should not appear in worklist")(static_cast <bool> ((OpWorkCount[CurOpNumber] > 0) &&
"Node should not appear in worklist") ? void (0) : __assert_fail
("(OpWorkCount[CurOpNumber] > 0) && \"Node should not appear in worklist\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1790, __extension__ __PRETTY_FUNCTION__))
;
1791 switch (CurNode->getOpcode()) {
1792 case ISD::EntryToken:
1793 // Hitting EntryToken is the only way for the search to terminate without
1794 // hitting
1795 // another operand's search. Prevent us from marking this operand
1796 // considered.
1797 NumLeftToConsider++;
1798 break;
1799 case ISD::TokenFactor:
1800 for (const SDValue &Op : CurNode->op_values())
1801 AddToWorklist(i, Op.getNode(), CurOpNumber);
1802 break;
1803 case ISD::CopyFromReg:
1804 case ISD::CopyToReg:
1805 AddToWorklist(i, CurNode->getOperand(0).getNode(), CurOpNumber);
1806 break;
1807 default:
1808 if (auto *MemNode = dyn_cast<MemSDNode>(CurNode))
1809 AddToWorklist(i, MemNode->getChain().getNode(), CurOpNumber);
1810 break;
1811 }
1812 OpWorkCount[CurOpNumber]--;
1813 if (OpWorkCount[CurOpNumber] == 0)
1814 NumLeftToConsider--;
1815 }
1816
1817 // If we've changed things around then replace token factor.
1818 if (Changed) {
1819 SDValue Result;
1820 if (Ops.empty()) {
1821 // The entry token is the only possible outcome.
1822 Result = DAG.getEntryNode();
1823 } else {
1824 if (DidPruneOps) {
1825 SmallVector<SDValue, 8> PrunedOps;
1826 //
1827 for (const SDValue &Op : Ops) {
1828 if (SeenChains.count(Op.getNode()) == 0)
1829 PrunedOps.push_back(Op);
1830 }
1831 Result = DAG.getNode(ISD::TokenFactor, SDLoc(N), MVT::Other, PrunedOps);
1832 } else {
1833 Result = DAG.getNode(ISD::TokenFactor, SDLoc(N), MVT::Other, Ops);
1834 }
1835 }
1836 return Result;
1837 }
1838 return SDValue();
1839}
1840
1841/// MERGE_VALUES can always be eliminated.
1842SDValue DAGCombiner::visitMERGE_VALUES(SDNode *N) {
1843 WorklistRemover DeadNodes(*this);
1844 // Replacing results may cause a different MERGE_VALUES to suddenly
1845 // be CSE'd with N, and carry its uses with it. Iterate until no
1846 // uses remain, to ensure that the node can be safely deleted.
1847 // First add the users of this node to the work list so that they
1848 // can be tried again once they have new operands.
1849 AddUsersToWorklist(N);
1850 do {
1851 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
1852 DAG.ReplaceAllUsesOfValueWith(SDValue(N, i), N->getOperand(i));
1853 } while (!N->use_empty());
1854 deleteAndRecombine(N);
1855 return SDValue(N, 0); // Return N so it doesn't get rechecked!
1856}
1857
1858/// If \p N is a ConstantSDNode with isOpaque() == false return it casted to a
1859/// ConstantSDNode pointer else nullptr.
1860static ConstantSDNode *getAsNonOpaqueConstant(SDValue N) {
1861 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N);
1862 return Const != nullptr && !Const->isOpaque() ? Const : nullptr;
1863}
1864
1865SDValue DAGCombiner::foldBinOpIntoSelect(SDNode *BO) {
1866 auto BinOpcode = BO->getOpcode();
1867 assert((BinOpcode == ISD::ADD || BinOpcode == ISD::SUB ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1876, __extension__ __PRETTY_FUNCTION__))
1868 BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1876, __extension__ __PRETTY_FUNCTION__))
1869 BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1876, __extension__ __PRETTY_FUNCTION__))
1870 BinOpcode == ISD::UREM || BinOpcode == ISD::AND ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1876, __extension__ __PRETTY_FUNCTION__))
1871 BinOpcode == ISD::OR || BinOpcode == ISD::XOR ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1876, __extension__ __PRETTY_FUNCTION__))
1872 BinOpcode == ISD::SHL || BinOpcode == ISD::SRL ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1876, __extension__ __PRETTY_FUNCTION__))
1873 BinOpcode == ISD::SRA || BinOpcode == ISD::FADD ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1876, __extension__ __PRETTY_FUNCTION__))
1874 BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1876, __extension__ __PRETTY_FUNCTION__))
1875 BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) &&(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1876, __extension__ __PRETTY_FUNCTION__))
1876 "Unexpected binary operator")(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1876, __extension__ __PRETTY_FUNCTION__))
;
1877
1878 // Bail out if any constants are opaque because we can't constant fold those.
1879 SDValue C1 = BO->getOperand(1);
1880 if (!isConstantOrConstantVector(C1, true) &&
1881 !isConstantFPBuildVectorOrConstantFP(C1))
1882 return SDValue();
1883
1884 // Don't do this unless the old select is going away. We want to eliminate the
1885 // binary operator, not replace a binop with a select.
1886 // TODO: Handle ISD::SELECT_CC.
1887 SDValue Sel = BO->getOperand(0);
1888 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse())
1889 return SDValue();
1890
1891 SDValue CT = Sel.getOperand(1);
1892 if (!isConstantOrConstantVector(CT, true) &&
1893 !isConstantFPBuildVectorOrConstantFP(CT))
1894 return SDValue();
1895
1896 SDValue CF = Sel.getOperand(2);
1897 if (!isConstantOrConstantVector(CF, true) &&
1898 !isConstantFPBuildVectorOrConstantFP(CF))
1899 return SDValue();
1900
1901 // We have a select-of-constants followed by a binary operator with a
1902 // constant. Eliminate the binop by pulling the constant math into the select.
1903 // Example: add (select Cond, CT, CF), C1 --> select Cond, CT + C1, CF + C1
1904 EVT VT = Sel.getValueType();
1905 SDLoc DL(Sel);
1906 SDValue NewCT = DAG.getNode(BinOpcode, DL, VT, CT, C1);
1907 if (!NewCT.isUndef() &&
1908 !isConstantOrConstantVector(NewCT, true) &&
1909 !isConstantFPBuildVectorOrConstantFP(NewCT))
1910 return SDValue();
1911
1912 SDValue NewCF = DAG.getNode(BinOpcode, DL, VT, CF, C1);
1913 if (!NewCF.isUndef() &&
1914 !isConstantOrConstantVector(NewCF, true) &&
1915 !isConstantFPBuildVectorOrConstantFP(NewCF))
1916 return SDValue();
1917
1918 return DAG.getSelect(DL, VT, Sel.getOperand(0), NewCT, NewCF);
1919}
1920
1921SDValue DAGCombiner::visitADD(SDNode *N) {
1922 SDValue N0 = N->getOperand(0);
1923 SDValue N1 = N->getOperand(1);
1924 EVT VT = N0.getValueType();
1925 SDLoc DL(N);
1926
1927 // fold vector ops
1928 if (VT.isVector()) {
1929 if (SDValue FoldedVOp = SimplifyVBinOp(N))
1930 return FoldedVOp;
1931
1932 // fold (add x, 0) -> x, vector edition
1933 if (ISD::isBuildVectorAllZeros(N1.getNode()))
1934 return N0;
1935 if (ISD::isBuildVectorAllZeros(N0.getNode()))
1936 return N1;
1937 }
1938
1939 // fold (add x, undef) -> undef
1940 if (N0.isUndef())
1941 return N0;
1942
1943 if (N1.isUndef())
1944 return N1;
1945
1946 if (DAG.isConstantIntBuildVectorOrConstantInt(N0)) {
1947 // canonicalize constant to RHS
1948 if (!DAG.isConstantIntBuildVectorOrConstantInt(N1))
1949 return DAG.getNode(ISD::ADD, DL, VT, N1, N0);
1950 // fold (add c1, c2) -> c1+c2
1951 return DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, N0.getNode(),
1952 N1.getNode());
1953 }
1954
1955 // fold (add x, 0) -> x
1956 if (isNullConstant(N1))
1957 return N0;
1958
1959 if (isConstantOrConstantVector(N1, /* NoOpaque */ true)) {
1960 // fold ((c1-A)+c2) -> (c1+c2)-A
1961 if (N0.getOpcode() == ISD::SUB &&
1962 isConstantOrConstantVector(N0.getOperand(0), /* NoOpaque */ true)) {
1963 // FIXME: Adding 2 constants should be handled by FoldConstantArithmetic.
1964 return DAG.getNode(ISD::SUB, DL, VT,
1965 DAG.getNode(ISD::ADD, DL, VT, N1, N0.getOperand(0)),
1966 N0.getOperand(1));
1967 }
1968
1969 // add (sext i1 X), 1 -> zext (not i1 X)
1970 // We don't transform this pattern:
1971 // add (zext i1 X), -1 -> sext (not i1 X)
1972 // because most (?) targets generate better code for the zext form.
1973 if (N0.getOpcode() == ISD::SIGN_EXTEND && N0.hasOneUse() &&
1974 isOneConstantOrOneSplatConstant(N1)) {
1975 SDValue X = N0.getOperand(0);
1976 if ((!LegalOperations ||
1977 (TLI.isOperationLegal(ISD::XOR, X.getValueType()) &&
1978 TLI.isOperationLegal(ISD::ZERO_EXTEND, VT))) &&
1979 X.getScalarValueSizeInBits() == 1) {
1980 SDValue Not = DAG.getNOT(DL, X, X.getValueType());
1981 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Not);
1982 }
1983 }
1984
1985 // Undo the add -> or combine to merge constant offsets from a frame index.
1986 if (N0.getOpcode() == ISD::OR &&
1987 isa<FrameIndexSDNode>(N0.getOperand(0)) &&
1988 isa<ConstantSDNode>(N0.getOperand(1)) &&
1989 DAG.haveNoCommonBitsSet(N0.getOperand(0), N0.getOperand(1))) {
1990 SDValue Add0 = DAG.getNode(ISD::ADD, DL, VT, N1, N0.getOperand(1));
1991 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), Add0);
1992 }
1993 }
1994
1995 if (SDValue NewSel = foldBinOpIntoSelect(N))
1996 return NewSel;
1997
1998 // reassociate add
1999 if (SDValue RADD = ReassociateOps(ISD::ADD, DL, N0, N1))
2000 return RADD;
2001
2002 // fold ((0-A) + B) -> B-A
2003 if (N0.getOpcode() == ISD::SUB &&
2004 isNullConstantOrNullSplatConstant(N0.getOperand(0)))
2005 return DAG.getNode(ISD::SUB, DL, VT, N1, N0.getOperand(1));
2006
2007 // fold (A + (0-B)) -> A-B
2008 if (N1.getOpcode() == ISD::SUB &&
2009 isNullConstantOrNullSplatConstant(N1.getOperand(0)))
2010 return DAG.getNode(ISD::SUB, DL, VT, N0, N1.getOperand(1));
2011
2012 // fold (A+(B-A)) -> B
2013 if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(1))
2014 return N1.getOperand(0);
2015
2016 // fold ((B-A)+A) -> B
2017 if (N0.getOpcode() == ISD::SUB && N1 == N0.getOperand(1))
2018 return N0.getOperand(0);
2019
2020 // fold (A+(B-(A+C))) to (B-C)
2021 if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD &&
2022 N0 == N1.getOperand(1).getOperand(0))
2023 return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0),
2024 N1.getOperand(1).getOperand(1));
2025
2026 // fold (A+(B-(C+A))) to (B-C)
2027 if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD &&
2028 N0 == N1.getOperand(1).getOperand(1))
2029 return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0),
2030 N1.getOperand(1).getOperand(0));
2031
2032 // fold (A+((B-A)+or-C)) to (B+or-C)
2033 if ((N1.getOpcode() == ISD::SUB || N1.getOpcode() == ISD::ADD) &&
2034 N1.getOperand(0).getOpcode() == ISD::SUB &&
2035 N0 == N1.getOperand(0).getOperand(1))
2036 return DAG.getNode(N1.getOpcode(), DL, VT, N1.getOperand(0).getOperand(0),
2037 N1.getOperand(1));
2038
2039 // fold (A-B)+(C-D) to (A+C)-(B+D) when A or C is constant
2040 if (N0.getOpcode() == ISD::SUB && N1.getOpcode() == ISD::SUB) {
2041 SDValue N00 = N0.getOperand(0);
2042 SDValue N01 = N0.getOperand(1);
2043 SDValue N10 = N1.getOperand(0);
2044 SDValue N11 = N1.getOperand(1);
2045
2046 if (isConstantOrConstantVector(N00) || isConstantOrConstantVector(N10))
2047 return DAG.getNode(ISD::SUB, DL, VT,
2048 DAG.getNode(ISD::ADD, SDLoc(N0), VT, N00, N10),
2049 DAG.getNode(ISD::ADD, SDLoc(N1), VT, N01, N11));
2050 }
2051
2052 if (SimplifyDemandedBits(SDValue(N, 0)))
2053 return SDValue(N, 0);
2054
2055 // fold (a+b) -> (a|b) iff a and b share no bits.
2056 if ((!LegalOperations || TLI.isOperationLegal(ISD::OR, VT)) &&
2057 DAG.haveNoCommonBitsSet(N0, N1))
2058 return DAG.getNode(ISD::OR, DL, VT, N0, N1);
2059
2060 if (SDValue Combined = visitADDLike(N0, N1, N))
2061 return Combined;
2062
2063 if (SDValue Combined = visitADDLike(N1, N0, N))
2064 return Combined;
2065
2066 return SDValue();
2067}
2068
2069static SDValue getAsCarry(const TargetLowering &TLI, SDValue V) {
2070 bool Masked = false;
2071
2072 // First, peel away TRUNCATE/ZERO_EXTEND/AND nodes due to legalization.
2073 while (true) {
2074 if (V.getOpcode() == ISD::TRUNCATE || V.getOpcode() == ISD::ZERO_EXTEND) {
2075 V = V.getOperand(0);
2076 continue;
2077 }
2078
2079 if (V.getOpcode() == ISD::AND && isOneConstant(V.getOperand(1))) {
2080 Masked = true;
2081 V = V.getOperand(0);
2082 continue;
2083 }
2084
2085 break;
2086 }
2087
2088 // If this is not a carry, return.
2089 if (V.getResNo() != 1)
2090 return SDValue();
2091
2092 if (V.getOpcode() != ISD::ADDCARRY && V.getOpcode() != ISD::SUBCARRY &&
2093 V.getOpcode() != ISD::UADDO && V.getOpcode() != ISD::USUBO)
2094 return SDValue();
2095
2096 // If the result is masked, then no matter what kind of bool it is we can
2097 // return. If it isn't, then we need to make sure the bool type is either 0 or
2098 // 1 and not other values.
2099 if (Masked ||
2100 TLI.getBooleanContents(V.getValueType()) ==
2101 TargetLoweringBase::ZeroOrOneBooleanContent)
2102 return V;
2103
2104 return SDValue();
2105}
2106
2107SDValue DAGCombiner::visitADDLike(SDValue N0, SDValue N1, SDNode *LocReference) {
2108 EVT VT = N0.getValueType();
2109 SDLoc DL(LocReference);
2110
2111 // fold (add x, shl(0 - y, n)) -> sub(x, shl(y, n))
2112 if (N1.getOpcode() == ISD::SHL && N1.getOperand(0).getOpcode() == ISD::SUB &&
2113 isNullConstantOrNullSplatConstant(N1.getOperand(0).getOperand(0)))
2114 return DAG.getNode(ISD::SUB, DL, VT, N0,
2115 DAG.getNode(ISD::SHL, DL, VT,
2116 N1.getOperand(0).getOperand(1),
2117 N1.getOperand(1)));
2118
2119 if (N1.getOpcode() == ISD::AND) {
2120 SDValue AndOp0 = N1.getOperand(0);
2121 unsigned NumSignBits = DAG.ComputeNumSignBits(AndOp0);
2122 unsigned DestBits = VT.getScalarSizeInBits();
2123
2124 // (add z, (and (sbbl x, x), 1)) -> (sub z, (sbbl x, x))
2125 // and similar xforms where the inner op is either ~0 or 0.
2126 if (NumSignBits == DestBits &&
2127 isOneConstantOrOneSplatConstant(N1->getOperand(1)))
2128 return DAG.getNode(ISD::SUB, DL, VT, N0, AndOp0);
2129 }
2130
2131 // add (sext i1), X -> sub X, (zext i1)
2132 if (N0.getOpcode() == ISD::SIGN_EXTEND &&
2133 N0.getOperand(0).getValueType() == MVT::i1 &&
2134 !TLI.isOperationLegal(ISD::SIGN_EXTEND, MVT::i1)) {
2135 SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0));
2136 return DAG.getNode(ISD::SUB, DL, VT, N1, ZExt);
2137 }
2138
2139 // add X, (sextinreg Y i1) -> sub X, (and Y 1)
2140 if (N1.getOpcode() == ISD::SIGN_EXTEND_INREG) {
2141 VTSDNode *TN = cast<VTSDNode>(N1.getOperand(1));
2142 if (TN->getVT() == MVT::i1) {
2143 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0),
2144 DAG.getConstant(1, DL, VT));
2145 return DAG.getNode(ISD::SUB, DL, VT, N0, ZExt);
2146 }
2147 }
2148
2149 // (add X, (addcarry Y, 0, Carry)) -> (addcarry X, Y, Carry)
2150 if (N1.getOpcode() == ISD::ADDCARRY && isNullConstant(N1.getOperand(1)) &&
2151 N1.getResNo() == 0)
2152 return DAG.getNode(ISD::ADDCARRY, DL, N1->getVTList(),
2153 N0, N1.getOperand(0), N1.getOperand(2));
2154
2155 // (add X, Carry) -> (addcarry X, 0, Carry)
2156 if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT))
2157 if (SDValue Carry = getAsCarry(TLI, N1))
2158 return DAG.getNode(ISD::ADDCARRY, DL,
2159 DAG.getVTList(VT, Carry.getValueType()), N0,
2160 DAG.getConstant(0, DL, VT), Carry);
2161
2162 return SDValue();
2163}
2164
2165SDValue DAGCombiner::visitADDC(SDNode *N) {
2166 SDValue N0 = N->getOperand(0);
2167 SDValue N1 = N->getOperand(1);
2168 EVT VT = N0.getValueType();
2169 SDLoc DL(N);
2170
2171 // If the flag result is dead, turn this into an ADD.
2172 if (!N->hasAnyUseOfValue(1))
2173 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2174 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2175
2176 // canonicalize constant to RHS.
2177 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
2178 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2179 if (N0C && !N1C)
2180 return DAG.getNode(ISD::ADDC, DL, N->getVTList(), N1, N0);
2181
2182 // fold (addc x, 0) -> x + no carry out
2183 if (isNullConstant(N1))
2184 return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE,
2185 DL, MVT::Glue));
2186
2187 // If it cannot overflow, transform into an add.
2188 if (DAG.computeOverflowKind(N0, N1) == SelectionDAG::OFK_Never)
2189 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2190 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2191
2192 return SDValue();
2193}
2194
2195SDValue DAGCombiner::visitUADDO(SDNode *N) {
2196 SDValue N0 = N->getOperand(0);
2197 SDValue N1 = N->getOperand(1);
2198 EVT VT = N0.getValueType();
2199 if (VT.isVector())
2200 return SDValue();
2201
2202 EVT CarryVT = N->getValueType(1);
2203 SDLoc DL(N);
2204
2205 // If the flag result is dead, turn this into an ADD.
2206 if (!N->hasAnyUseOfValue(1))
2207 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2208 DAG.getUNDEF(CarryVT));
2209
2210 // canonicalize constant to RHS.
2211 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
2212 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2213 if (N0C && !N1C)
2214 return DAG.getNode(ISD::UADDO, DL, N->getVTList(), N1, N0);
2215
2216 // fold (uaddo x, 0) -> x + no carry out
2217 if (isNullConstant(N1))
2218 return CombineTo(N, N0, DAG.getConstant(0, DL, CarryVT));
2219
2220 // If it cannot overflow, transform into an add.
2221 if (DAG.computeOverflowKind(N0, N1) == SelectionDAG::OFK_Never)
2222 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2223 DAG.getConstant(0, DL, CarryVT));
2224
2225 if (SDValue Combined = visitUADDOLike(N0, N1, N))
2226 return Combined;
2227
2228 if (SDValue Combined = visitUADDOLike(N1, N0, N))
2229 return Combined;
2230
2231 return SDValue();
2232}
2233
2234SDValue DAGCombiner::visitUADDOLike(SDValue N0, SDValue N1, SDNode *N) {
2235 auto VT = N0.getValueType();
2236
2237 // (uaddo X, (addcarry Y, 0, Carry)) -> (addcarry X, Y, Carry)
2238 // If Y + 1 cannot overflow.
2239 if (N1.getOpcode() == ISD::ADDCARRY && isNullConstant(N1.getOperand(1))) {
2240 SDValue Y = N1.getOperand(0);
2241 SDValue One = DAG.getConstant(1, SDLoc(N), Y.getValueType());
2242 if (DAG.computeOverflowKind(Y, One) == SelectionDAG::OFK_Never)
2243 return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0, Y,
2244 N1.getOperand(2));
2245 }
2246
2247 // (uaddo X, Carry) -> (addcarry X, 0, Carry)
2248 if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT))
2249 if (SDValue Carry = getAsCarry(TLI, N1))
2250 return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0,
2251 DAG.getConstant(0, SDLoc(N), VT), Carry);
2252
2253 return SDValue();
2254}
2255
2256SDValue DAGCombiner::visitADDE(SDNode *N) {
2257 SDValue N0 = N->getOperand(0);
2258 SDValue N1 = N->getOperand(1);
2259 SDValue CarryIn = N->getOperand(2);
2260
2261 // canonicalize constant to RHS
2262 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
2263 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2264 if (N0C && !N1C)
2265 return DAG.getNode(ISD::ADDE, SDLoc(N), N->getVTList(),
2266 N1, N0, CarryIn);
2267
2268 // fold (adde x, y, false) -> (addc x, y)
2269 if (CarryIn.getOpcode() == ISD::CARRY_FALSE)
2270 return DAG.getNode(ISD::ADDC, SDLoc(N), N->getVTList(), N0, N1);
2271
2272 return SDValue();
2273}
2274
2275SDValue DAGCombiner::visitADDCARRY(SDNode *N) {
2276 SDValue N0 = N->getOperand(0);
2277 SDValue N1 = N->getOperand(1);
2278 SDValue CarryIn = N->getOperand(2);
2279 SDLoc DL(N);
2280
2281 // canonicalize constant to RHS
2282 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
2283 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2284 if (N0C && !N1C)
2285 return DAG.getNode(ISD::ADDCARRY, DL, N->getVTList(), N1, N0, CarryIn);
2286
2287 // fold (addcarry x, y, false) -> (uaddo x, y)
2288 if (isNullConstant(CarryIn))
2289 return DAG.getNode(ISD::UADDO, DL, N->getVTList(), N0, N1);
2290
2291 // fold (addcarry 0, 0, X) -> (and (ext/trunc X), 1) and no carry.
2292 if (isNullConstant(N0) && isNullConstant(N1)) {
2293 EVT VT = N0.getValueType();
2294 EVT CarryVT = CarryIn.getValueType();
2295 SDValue CarryExt = DAG.getBoolExtOrTrunc(CarryIn, DL, VT, CarryVT);
2296 AddToWorklist(CarryExt.getNode());
2297 return CombineTo(N, DAG.getNode(ISD::AND, DL, VT, CarryExt,
2298 DAG.getConstant(1, DL, VT)),
2299 DAG.getConstant(0, DL, CarryVT));
2300 }
2301
2302 if (SDValue Combined = visitADDCARRYLike(N0, N1, CarryIn, N))
2303 return Combined;
2304
2305 if (SDValue Combined = visitADDCARRYLike(N1, N0, CarryIn, N))
2306 return Combined;
2307
2308 return SDValue();
2309}
2310
2311SDValue DAGCombiner::visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn,
2312 SDNode *N) {
2313 // Iff the flag result is dead:
2314 // (addcarry (add|uaddo X, Y), 0, Carry) -> (addcarry X, Y, Carry)
2315 if ((N0.getOpcode() == ISD::ADD ||
2316 (N0.getOpcode() == ISD::UADDO && N0.getResNo() == 0)) &&
2317 isNullConstant(N1) && !N->hasAnyUseOfValue(1))
2318 return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(),
2319 N0.getOperand(0), N0.getOperand(1), CarryIn);
2320
2321 /**
2322 * When one of the addcarry argument is itself a carry, we may be facing
2323 * a diamond carry propagation. In which case we try to transform the DAG
2324 * to ensure linear carry propagation if that is possible.
2325 *
2326 * We are trying to get:
2327 * (addcarry X, 0, (addcarry A, B, Z):Carry)
2328 */
2329 if (auto Y = getAsCarry(TLI, N1)) {
2330 /**
2331 * (uaddo A, B)
2332 * / \
2333 * Carry Sum
2334 * | \
2335 * | (addcarry *, 0, Z)
2336 * | /
2337 * \ Carry
2338 * | /
2339 * (addcarry X, *, *)
2340 */
2341 if (Y.getOpcode() == ISD::UADDO &&
2342 CarryIn.getResNo() == 1 &&
2343 CarryIn.getOpcode() == ISD::ADDCARRY &&
2344 isNullConstant(CarryIn.getOperand(1)) &&
2345 CarryIn.getOperand(0) == Y.getValue(0)) {
2346 auto NewY = DAG.getNode(ISD::ADDCARRY, SDLoc(N), Y->getVTList(),
2347 Y.getOperand(0), Y.getOperand(1),
2348 CarryIn.getOperand(2));
2349 AddToWorklist(NewY.getNode());
2350 return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0,
2351 DAG.getConstant(0, SDLoc(N), N0.getValueType()),
2352 NewY.getValue(1));
2353 }
2354 }
2355
2356 return SDValue();
2357}
2358
2359// Since it may not be valid to emit a fold to zero for vector initializers
2360// check if we can before folding.
2361static SDValue tryFoldToZero(const SDLoc &DL, const TargetLowering &TLI, EVT VT,
2362 SelectionDAG &DAG, bool LegalOperations,
2363 bool LegalTypes) {
2364 if (!VT.isVector())
2365 return DAG.getConstant(0, DL, VT);
2366 if (!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT))
2367 return DAG.getConstant(0, DL, VT);
2368 return SDValue();
2369}
2370
2371SDValue DAGCombiner::visitSUB(SDNode *N) {
2372 SDValue N0 = N->getOperand(0);
2373 SDValue N1 = N->getOperand(1);
2374 EVT VT = N0.getValueType();
2375 SDLoc DL(N);
2376
2377 // fold vector ops
2378 if (VT.isVector()) {
2379 if (SDValue FoldedVOp = SimplifyVBinOp(N))
2380 return FoldedVOp;
2381
2382 // fold (sub x, 0) -> x, vector edition
2383 if (ISD::isBuildVectorAllZeros(N1.getNode()))
2384 return N0;
2385 }
2386
2387 // fold (sub x, x) -> 0
2388 // FIXME: Refactor this and xor and other similar operations together.
2389 if (N0 == N1)
2390 return tryFoldToZero(DL, TLI, VT, DAG, LegalOperations, LegalTypes);
2391 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
2392 DAG.isConstantIntBuildVectorOrConstantInt(N1)) {
2393 // fold (sub c1, c2) -> c1-c2
2394 return DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, N0.getNode(),
2395 N1.getNode());
2396 }
2397
2398 if (SDValue NewSel = foldBinOpIntoSelect(N))
2399 return NewSel;
2400
2401 ConstantSDNode *N1C = getAsNonOpaqueConstant(N1);
2402
2403 // fold (sub x, c) -> (add x, -c)
2404 if (N1C) {
2405 return DAG.getNode(ISD::ADD, DL, VT, N0,
2406 DAG.getConstant(-N1C->getAPIntValue(), DL, VT));
2407 }
2408
2409 if (isNullConstantOrNullSplatConstant(N0)) {
2410 unsigned BitWidth = VT.getScalarSizeInBits();
2411 // Right-shifting everything out but the sign bit followed by negation is
2412 // the same as flipping arithmetic/logical shift type without the negation:
2413 // -(X >>u 31) -> (X >>s 31)
2414 // -(X >>s 31) -> (X >>u 31)
2415 if (N1->getOpcode() == ISD::SRA || N1->getOpcode() == ISD::SRL) {
2416 ConstantSDNode *ShiftAmt = isConstOrConstSplat(N1.getOperand(1));
2417 if (ShiftAmt && ShiftAmt->getZExtValue() == BitWidth - 1) {
2418 auto NewSh = N1->getOpcode() == ISD::SRA ? ISD::SRL : ISD::SRA;
2419 if (!LegalOperations || TLI.isOperationLegal(NewSh, VT))
2420 return DAG.getNode(NewSh, DL, VT, N1.getOperand(0), N1.getOperand(1));
2421 }
2422 }
2423
2424 // 0 - X --> 0 if the sub is NUW.
2425 if (N->getFlags().hasNoUnsignedWrap())
2426 return N0;
2427
2428 if (DAG.MaskedValueIsZero(N1, ~APInt::getSignMask(BitWidth))) {
2429 // N1 is either 0 or the minimum signed value. If the sub is NSW, then
2430 // N1 must be 0 because negating the minimum signed value is undefined.
2431 if (N->getFlags().hasNoSignedWrap())
2432 return N0;
2433
2434 // 0 - X --> X if X is 0 or the minimum signed value.
2435 return N1;
2436 }
2437 }
2438
2439 // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1)
2440 if (isAllOnesConstantOrAllOnesSplatConstant(N0))
2441 return DAG.getNode(ISD::XOR, DL, VT, N1, N0);
2442
2443 // fold A-(A-B) -> B
2444 if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(0))
2445 return N1.getOperand(1);
2446
2447 // fold (A+B)-A -> B
2448 if (N0.getOpcode() == ISD::ADD && N0.getOperand(0) == N1)
2449 return N0.getOperand(1);
2450
2451 // fold (A+B)-B -> A
2452 if (N0.getOpcode() == ISD::ADD && N0.getOperand(1) == N1)
2453 return N0.getOperand(0);
2454
2455 // fold C2-(A+C1) -> (C2-C1)-A
2456 if (N1.getOpcode() == ISD::ADD) {
2457 SDValue N11 = N1.getOperand(1);
2458 if (isConstantOrConstantVector(N0, /* NoOpaques */ true) &&
2459 isConstantOrConstantVector(N11, /* NoOpaques */ true)) {
2460 SDValue NewC = DAG.getNode(ISD::SUB, DL, VT, N0, N11);
2461 return DAG.getNode(ISD::SUB, DL, VT, NewC, N1.getOperand(0));
2462 }
2463 }
2464
2465 // fold ((A+(B+or-C))-B) -> A+or-C
2466 if (N0.getOpcode() == ISD::ADD &&
2467 (N0.getOperand(1).getOpcode() == ISD::SUB ||
2468 N0.getOperand(1).getOpcode() == ISD::ADD) &&
2469 N0.getOperand(1).getOperand(0) == N1)
2470 return DAG.getNode(N0.getOperand(1).getOpcode(), DL, VT, N0.getOperand(0),
2471 N0.getOperand(1).getOperand(1));
2472
2473 // fold ((A+(C+B))-B) -> A+C
2474 if (N0.getOpcode() == ISD::ADD && N0.getOperand(1).getOpcode() == ISD::ADD &&
2475 N0.getOperand(1).getOperand(1) == N1)
2476 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0),
2477 N0.getOperand(1).getOperand(0));
2478
2479 // fold ((A-(B-C))-C) -> A-B
2480 if (N0.getOpcode() == ISD::SUB && N0.getOperand(1).getOpcode() == ISD::SUB &&
2481 N0.getOperand(1).getOperand(1) == N1)
2482 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0),
2483 N0.getOperand(1).getOperand(0));
2484
2485 // If either operand of a sub is undef, the result is undef
2486 if (N0.isUndef())
2487 return N0;
2488 if (N1.isUndef())
2489 return N1;
2490
2491 // If the relocation model supports it, consider symbol offsets.
2492 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N0))
2493 if (!LegalOperations && TLI.isOffsetFoldingLegal(GA)) {
2494 // fold (sub Sym, c) -> Sym-c
2495 if (N1C && GA->getOpcode() == ISD::GlobalAddress)
2496 return DAG.getGlobalAddress(GA->getGlobal(), SDLoc(N1C), VT,
2497 GA->getOffset() -
2498 (uint64_t)N1C->getSExtValue());
2499 // fold (sub Sym+c1, Sym+c2) -> c1-c2
2500 if (GlobalAddressSDNode *GB = dyn_cast<GlobalAddressSDNode>(N1))
2501 if (GA->getGlobal() == GB->getGlobal())
2502 return DAG.getConstant((uint64_t)GA->getOffset() - GB->getOffset(),
2503 DL, VT);
2504 }
2505
2506 // sub X, (sextinreg Y i1) -> add X, (and Y 1)
2507 if (N1.getOpcode() == ISD::SIGN_EXTEND_INREG) {
2508 VTSDNode *TN = cast<VTSDNode>(N1.getOperand(1));
2509 if (TN->getVT() == MVT::i1) {
2510 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0),
2511 DAG.getConstant(1, DL, VT));
2512 return DAG.getNode(ISD::ADD, DL, VT, N0, ZExt);
2513 }
2514 }
2515
2516 return SDValue();
2517}
2518
2519SDValue DAGCombiner::visitSUBC(SDNode *N) {
2520 SDValue N0 = N->getOperand(0);
2521 SDValue N1 = N->getOperand(1);
2522 EVT VT = N0.getValueType();
2523 SDLoc DL(N);
2524
2525 // If the flag result is dead, turn this into an SUB.
2526 if (!N->hasAnyUseOfValue(1))
2527 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1),
2528 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2529
2530 // fold (subc x, x) -> 0 + no borrow
2531 if (N0 == N1)
2532 return CombineTo(N, DAG.getConstant(0, DL, VT),
2533 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2534
2535 // fold (subc x, 0) -> x + no borrow
2536 if (isNullConstant(N1))
2537 return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2538
2539 // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1) + no borrow
2540 if (isAllOnesConstant(N0))
2541 return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0),
2542 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2543
2544 return SDValue();
2545}
2546
2547SDValue DAGCombiner::visitUSUBO(SDNode *N) {
2548 SDValue N0 = N->getOperand(0);
2549 SDValue N1 = N->getOperand(1);
2550 EVT VT = N0.getValueType();
2551 if (VT.isVector())
2552 return SDValue();
2553
2554 EVT CarryVT = N->getValueType(1);
2555 SDLoc DL(N);
2556
2557 // If the flag result is dead, turn this into an SUB.
2558 if (!N->hasAnyUseOfValue(1))
2559 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1),
2560 DAG.getUNDEF(CarryVT));
2561
2562 // fold (usubo x, x) -> 0 + no borrow
2563 if (N0 == N1)
2564 return CombineTo(N, DAG.getConstant(0, DL, VT),
2565 DAG.getConstant(0, DL, CarryVT));
2566
2567 // fold (usubo x, 0) -> x + no borrow
2568 if (isNullConstant(N1))
2569 return CombineTo(N, N0, DAG.getConstant(0, DL, CarryVT));
2570
2571 // Canonicalize (usubo -1, x) -> ~x, i.e. (xor x, -1) + no borrow
2572 if (isAllOnesConstant(N0))
2573 return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0),
2574 DAG.getConstant(0, DL, CarryVT));
2575
2576 return SDValue();
2577}
2578
2579SDValue DAGCombiner::visitSUBE(SDNode *N) {
2580 SDValue N0 = N->getOperand(0);
2581 SDValue N1 = N->getOperand(1);
2582 SDValue CarryIn = N->getOperand(2);
2583
2584 // fold (sube x, y, false) -> (subc x, y)
2585 if (CarryIn.getOpcode() == ISD::CARRY_FALSE)
2586 return DAG.getNode(ISD::SUBC, SDLoc(N), N->getVTList(), N0, N1);
2587
2588 return SDValue();
2589}
2590
2591SDValue DAGCombiner::visitSUBCARRY(SDNode *N) {
2592 SDValue N0 = N->getOperand(0);
2593 SDValue N1 = N->getOperand(1);
2594 SDValue CarryIn = N->getOperand(2);
2595
2596 // fold (subcarry x, y, false) -> (usubo x, y)
2597 if (isNullConstant(CarryIn))
2598 return DAG.getNode(ISD::USUBO, SDLoc(N), N->getVTList(), N0, N1);
2599
2600 return SDValue();
2601}
2602
2603SDValue DAGCombiner::visitMUL(SDNode *N) {
2604 SDValue N0 = N->getOperand(0);
2605 SDValue N1 = N->getOperand(1);
2606 EVT VT = N0.getValueType();
2607
2608 // fold (mul x, undef) -> 0
2609 if (N0.isUndef() || N1.isUndef())
2610 return DAG.getConstant(0, SDLoc(N), VT);
2611
2612 bool N0IsConst = false;
2613 bool N1IsConst = false;
2614 bool N1IsOpaqueConst = false;
2615 bool N0IsOpaqueConst = false;
2616 APInt ConstValue0, ConstValue1;
2617 // fold vector ops
2618 if (VT.isVector()) {
2619 if (SDValue FoldedVOp = SimplifyVBinOp(N))
2620 return FoldedVOp;
2621
2622 N0IsConst = ISD::isConstantSplatVector(N0.getNode(), ConstValue0);
2623 N1IsConst = ISD::isConstantSplatVector(N1.getNode(), ConstValue1);
2624 assert((!N0IsConst ||(static_cast <bool> ((!N0IsConst || ConstValue0.getBitWidth
() == VT.getScalarSizeInBits()) && "Splat APInt should be element width"
) ? void (0) : __assert_fail ("(!N0IsConst || ConstValue0.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2626, __extension__ __PRETTY_FUNCTION__))
2625 ConstValue0.getBitWidth() == VT.getScalarSizeInBits()) &&(static_cast <bool> ((!N0IsConst || ConstValue0.getBitWidth
() == VT.getScalarSizeInBits()) && "Splat APInt should be element width"
) ? void (0) : __assert_fail ("(!N0IsConst || ConstValue0.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2626, __extension__ __PRETTY_FUNCTION__))
2626 "Splat APInt should be element width")(static_cast <bool> ((!N0IsConst || ConstValue0.getBitWidth
() == VT.getScalarSizeInBits()) && "Splat APInt should be element width"
) ? void (0) : __assert_fail ("(!N0IsConst || ConstValue0.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2626, __extension__ __PRETTY_FUNCTION__))
;
2627 assert((!N1IsConst ||(static_cast <bool> ((!N1IsConst || ConstValue1.getBitWidth
() == VT.getScalarSizeInBits()) && "Splat APInt should be element width"
) ? void (0) : __assert_fail ("(!N1IsConst || ConstValue1.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2629, __extension__ __PRETTY_FUNCTION__))
2628 ConstValue1.getBitWidth() == VT.getScalarSizeInBits()) &&(static_cast <bool> ((!N1IsConst || ConstValue1.getBitWidth
() == VT.getScalarSizeInBits()) && "Splat APInt should be element width"
) ? void (0) : __assert_fail ("(!N1IsConst || ConstValue1.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2629, __extension__ __PRETTY_FUNCTION__))
2629 "Splat APInt should be element width")(static_cast <bool> ((!N1IsConst || ConstValue1.getBitWidth
() == VT.getScalarSizeInBits()) && "Splat APInt should be element width"
) ? void (0) : __assert_fail ("(!N1IsConst || ConstValue1.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2629, __extension__ __PRETTY_FUNCTION__))
;
2630 } else {
2631 N0IsConst = isa<ConstantSDNode>(N0);
2632 if (N0IsConst) {
2633 ConstValue0 = cast<ConstantSDNode>(N0)->getAPIntValue();
2634 N0IsOpaqueConst = cast<ConstantSDNode>(N0)->isOpaque();
2635 }
2636 N1IsConst = isa<ConstantSDNode>(N1);
2637 if (N1IsConst) {
2638 ConstValue1 = cast<ConstantSDNode>(N1)->getAPIntValue();
2639 N1IsOpaqueConst = cast<ConstantSDNode>(N1)->isOpaque();
2640 }
2641 }
2642
2643 // fold (mul c1, c2) -> c1*c2
2644 if (N0IsConst && N1IsConst && !N0IsOpaqueConst && !N1IsOpaqueConst)
2645 return DAG.FoldConstantArithmetic(ISD::MUL, SDLoc(N), VT,
2646 N0.getNode(), N1.getNode());
2647
2648 // canonicalize constant to RHS (vector doesn't have to splat)
2649 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
2650 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
2651 return DAG.getNode(ISD::MUL, SDLoc(N), VT, N1, N0);
2652 // fold (mul x, 0) -> 0
2653 if (N1IsConst && ConstValue1.isNullValue())
2654 return N1;
2655 // fold (mul x, 1) -> x
2656 if (N1IsConst && ConstValue1.isOneValue())
2657 return N0;
2658
2659 if (SDValue NewSel = foldBinOpIntoSelect(N))
2660 return NewSel;
2661
2662 // fold (mul x, -1) -> 0-x
2663 if (N1IsConst && ConstValue1.isAllOnesValue()) {
2664 SDLoc DL(N);
2665 return DAG.getNode(ISD::SUB, DL, VT,
2666 DAG.getConstant(0, DL, VT), N0);
2667 }
2668 // fold (mul x, (1 << c)) -> x << c
2669 if (isConstantOrConstantVector(N1, /*NoOpaques*/ true) &&
2670 DAG.isKnownToBeAPowerOfTwo(N1) &&
2671 (!VT.isVector() || Level <= AfterLegalizeVectorOps)) {
2672 SDLoc DL(N);
2673 SDValue LogBase2 = BuildLogBase2(N1, DL);
2674 AddToWorklist(LogBase2.getNode());
2675
2676 EVT ShiftVT = getShiftAmountTy(N0.getValueType());
2677 SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ShiftVT);
2678 AddToWorklist(Trunc.getNode());
2679 return DAG.getNode(ISD::SHL, DL, VT, N0, Trunc);
2680 }
2681 // fold (mul x, -(1 << c)) -> -(x << c) or (-x) << c
2682 if (N1IsConst && !N1IsOpaqueConst && (-ConstValue1).isPowerOf2()) {
2683 unsigned Log2Val = (-ConstValue1).logBase2();
2684 SDLoc DL(N);
2685 // FIXME: If the input is something that is easily negated (e.g. a
2686 // single-use add), we should put the negate there.
2687 return DAG.getNode(ISD::SUB, DL, VT,
2688 DAG.getConstant(0, DL, VT),
2689 DAG.getNode(ISD::SHL, DL, VT, N0,
2690 DAG.getConstant(Log2Val, DL,
2691 getShiftAmountTy(N0.getValueType()))));
2692 }
2693
2694 // (mul (shl X, c1), c2) -> (mul X, c2 << c1)
2695 if (N0.getOpcode() == ISD::SHL &&
2696 isConstantOrConstantVector(N1, /* NoOpaques */ true) &&
2697 isConstantOrConstantVector(N0.getOperand(1), /* NoOpaques */ true)) {
2698 SDValue C3 = DAG.getNode(ISD::SHL, SDLoc(N), VT, N1, N0.getOperand(1));
2699 if (isConstantOrConstantVector(C3))
2700 return DAG.getNode(ISD::MUL, SDLoc(N), VT, N0.getOperand(0), C3);
2701 }
2702
2703 // Change (mul (shl X, C), Y) -> (shl (mul X, Y), C) when the shift has one
2704 // use.
2705 {
2706 SDValue Sh(nullptr, 0), Y(nullptr, 0);
2707
2708 // Check for both (mul (shl X, C), Y) and (mul Y, (shl X, C)).
2709 if (N0.getOpcode() == ISD::SHL &&
2710 isConstantOrConstantVector(N0.getOperand(1)) &&
2711 N0.getNode()->hasOneUse()) {
2712 Sh = N0; Y = N1;
2713 } else if (N1.getOpcode() == ISD::SHL &&
2714 isConstantOrConstantVector(N1.getOperand(1)) &&
2715 N1.getNode()->hasOneUse()) {
2716 Sh = N1; Y = N0;
2717 }
2718
2719 if (Sh.getNode()) {
2720 SDValue Mul = DAG.getNode(ISD::MUL, SDLoc(N), VT, Sh.getOperand(0), Y);
2721 return DAG.getNode(ISD::SHL, SDLoc(N), VT, Mul, Sh.getOperand(1));
2722 }
2723 }
2724
2725 // fold (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2)
2726 if (DAG.isConstantIntBuildVectorOrConstantInt(N1) &&
2727 N0.getOpcode() == ISD::ADD &&
2728 DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1)) &&
2729 isMulAddWithConstProfitable(N, N0, N1))
2730 return DAG.getNode(ISD::ADD, SDLoc(N), VT,
2731 DAG.getNode(ISD::MUL, SDLoc(N0), VT,
2732 N0.getOperand(0), N1),
2733 DAG.getNode(ISD::MUL, SDLoc(N1), VT,
2734 N0.getOperand(1), N1));
2735
2736 // reassociate mul
2737 if (SDValue RMUL = ReassociateOps(ISD::MUL, SDLoc(N), N0, N1))
2738 return RMUL;
2739
2740 return SDValue();
2741}
2742
2743/// Return true if divmod libcall is available.
2744static bool isDivRemLibcallAvailable(SDNode *Node, bool isSigned,
2745 const TargetLowering &TLI) {
2746 RTLIB::Libcall LC;
2747 EVT NodeType = Node->getValueType(0);
2748 if (!NodeType.isSimple())
2749 return false;
2750 switch (NodeType.getSimpleVT().SimpleTy) {
2751 default: return false; // No libcall for vector types.
2752 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2753 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2754 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2755 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2756 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2757 }
2758
2759 return TLI.getLibcallName(LC) != nullptr;
2760}
2761
2762/// Issue divrem if both quotient and remainder are needed.
2763SDValue DAGCombiner::useDivRem(SDNode *Node) {
2764 if (Node->use_empty())
2765 return SDValue(); // This is a dead node, leave it alone.
2766
2767 unsigned Opcode = Node->getOpcode();
2768 bool isSigned = (Opcode == ISD::SDIV) || (Opcode == ISD::SREM);
2769 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
2770
2771 // DivMod lib calls can still work on non-legal types if using lib-calls.
2772 EVT VT = Node->getValueType(0);
2773 if (VT.isVector() || !VT.isInteger())
2774 return SDValue();
2775
2776 if (!TLI.isTypeLegal(VT) && !TLI.isOperationCustom(DivRemOpc, VT))
2777 return SDValue();
2778
2779 // If DIVREM is going to get expanded into a libcall,
2780 // but there is no libcall available, then don't combine.
2781 if (!TLI.isOperationLegalOrCustom(DivRemOpc, VT) &&
2782 !isDivRemLibcallAvailable(Node, isSigned, TLI))
2783 return SDValue();
2784
2785 // If div is legal, it's better to do the normal expansion
2786 unsigned OtherOpcode = 0;
2787 if ((Opcode == ISD::SDIV) || (Opcode == ISD::UDIV)) {
2788 OtherOpcode = isSigned ? ISD::SREM : ISD::UREM;
2789 if (TLI.isOperationLegalOrCustom(Opcode, VT))
2790 return SDValue();
2791 } else {
2792 OtherOpcode = isSigned ? ISD::SDIV : ISD::UDIV;
2793 if (TLI.isOperationLegalOrCustom(OtherOpcode, VT))
2794 return SDValue();
2795 }
2796
2797 SDValue Op0 = Node->getOperand(0);
2798 SDValue Op1 = Node->getOperand(1);
2799 SDValue combined;
2800 for (SDNode::use_iterator UI = Op0.getNode()->use_begin(),
2801 UE = Op0.getNode()->use_end(); UI != UE; ++UI) {
2802 SDNode *User = *UI;
2803 if (User == Node || User->use_empty())
2804 continue;
2805 // Convert the other matching node(s), too;
2806 // otherwise, the DIVREM may get target-legalized into something
2807 // target-specific that we won't be able to recognize.
2808 unsigned UserOpc = User->getOpcode();
2809 if ((UserOpc == Opcode || UserOpc == OtherOpcode || UserOpc == DivRemOpc) &&
2810 User->getOperand(0) == Op0 &&
2811 User->getOperand(1) == Op1) {
2812 if (!combined) {
2813 if (UserOpc == OtherOpcode) {
2814 SDVTList VTs = DAG.getVTList(VT, VT);
2815 combined = DAG.getNode(DivRemOpc, SDLoc(Node), VTs, Op0, Op1);
2816 } else if (UserOpc == DivRemOpc) {
2817 combined = SDValue(User, 0);
2818 } else {
2819 assert(UserOpc == Opcode)(static_cast <bool> (UserOpc == Opcode) ? void (0) : __assert_fail
("UserOpc == Opcode", "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2819, __extension__ __PRETTY_FUNCTION__))
;
2820 continue;
2821 }
2822 }
2823 if (UserOpc == ISD::SDIV || UserOpc == ISD::UDIV)
2824 CombineTo(User, combined);
2825 else if (UserOpc == ISD::SREM || UserOpc == ISD::UREM)
2826 CombineTo(User, combined.getValue(1));
2827 }
2828 }
2829 return combined;
2830}
2831
2832static SDValue simplifyDivRem(SDNode *N, SelectionDAG &DAG) {
2833 SDValue N0 = N->getOperand(0);
2834 SDValue N1 = N->getOperand(1);
2835 EVT VT = N->getValueType(0);
2836 SDLoc DL(N);
2837
2838 if (DAG.isUndef(N->getOpcode(), {N0, N1}))
2839 return DAG.getUNDEF(VT);
2840
2841 // undef / X -> 0
2842 // undef % X -> 0
2843 if (N0.isUndef())
2844 return DAG.getConstant(0, DL, VT);
2845
2846 return SDValue();
2847}
2848
2849SDValue DAGCombiner::visitSDIV(SDNode *N) {
2850 SDValue N0 = N->getOperand(0);
2851 SDValue N1 = N->getOperand(1);
2852 EVT VT = N->getValueType(0);
2853
2854 // fold vector ops
2855 if (VT.isVector())
2856 if (SDValue FoldedVOp = SimplifyVBinOp(N))
2857 return FoldedVOp;
2858
2859 SDLoc DL(N);
2860
2861 // fold (sdiv c1, c2) -> c1/c2
2862 ConstantSDNode *N0C = isConstOrConstSplat(N0);
2863 ConstantSDNode *N1C = isConstOrConstSplat(N1);
2864 if (N0C && N1C && !N0C->isOpaque() && !N1C->isOpaque())
2865 return DAG.FoldConstantArithmetic(ISD::SDIV, DL, VT, N0C, N1C);
2866 // fold (sdiv X, 1) -> X
2867 if (N1C && N1C->isOne())
2868 return N0;
2869 // fold (sdiv X, -1) -> 0-X
2870 if (N1C && N1C->isAllOnesValue())
2871 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), N0);
2872
2873 if (SDValue V = simplifyDivRem(N, DAG))
2874 return V;
2875
2876 if (SDValue NewSel = foldBinOpIntoSelect(N))
2877 return NewSel;
2878
2879 // If we know the sign bits of both operands are zero, strength reduce to a
2880 // udiv instead. Handles (X&15) /s 4 -> X&15 >> 2
2881 if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
2882 return DAG.getNode(ISD::UDIV, DL, N1.getValueType(), N0, N1);
2883
2884 // Helper for determining whether a value is a power-2 constant scalar or a
2885 // vector of such elements.
2886 SmallBitVector KnownNegatives(
2887 (N1C || !VT.isVector()) ? 1 : VT.getVectorNumElements(), false);
2888 unsigned EltIndex = 0;
2889 auto IsPowerOfTwo = [&KnownNegatives, &EltIndex](ConstantSDNode *C) {
2890 unsigned Idx = EltIndex++;
2891 if (C->isNullValue() || C->isOpaque())
2892 return false;
2893 if (C->getAPIntValue().isPowerOf2())
2894 return true;
2895 if ((-C->getAPIntValue()).isPowerOf2()) {
2896 KnownNegatives.set(Idx);
2897 return true;
2898 }
2899 return false;
2900 };
2901
2902 // fold (sdiv X, pow2) -> simple ops after legalize
2903 // FIXME: We check for the exact bit here because the generic lowering gives
2904 // better results in that case. The target-specific lowering should learn how
2905 // to handle exact sdivs efficiently.
2906 if (!N->getFlags().hasExact() &&
2907 ISD::matchUnaryPredicate(N1C ? SDValue(N1C, 0) : N1, IsPowerOfTwo)) {
2908 // Target-specific implementation of sdiv x, pow2.
2909 if (SDValue Res = BuildSDIVPow2(N))
2910 return Res;
2911
2912 // Create constants that are functions of the shift amount value.
2913 EVT ShiftAmtTy = getShiftAmountTy(N0.getValueType());
2914 SDValue Bits = DAG.getConstant(VT.getScalarSizeInBits(), DL, ShiftAmtTy);
2915 SDValue C1 = DAG.getNode(ISD::CTTZ, DL, VT, N1);
2916 C1 = DAG.getZExtOrTrunc(C1, DL, ShiftAmtTy);
2917 SDValue Inexact = DAG.getNode(ISD::SUB, DL, ShiftAmtTy, Bits, C1);
2918 if (!isConstantOrConstantVector(Inexact))
2919 return SDValue();
2920 // Splat the sign bit into the register
2921 SDValue Sign = DAG.getNode(
2922 ISD::SRA, DL, VT, N0,
2923 DAG.getConstant(VT.getScalarSizeInBits() - 1, DL, ShiftAmtTy));
2924 AddToWorklist(Sign.getNode());
2925
2926 // Add (N0 < 0) ? abs2 - 1 : 0;
2927 SDValue Srl = DAG.getNode(ISD::SRL, DL, VT, Sign, Inexact);
2928 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Srl);
2929 AddToWorklist(Srl.getNode());
2930 AddToWorklist(Add.getNode()); // Divide by pow2
2931 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Add, C1);
2932
2933 // If dividing by a positive value, we're done. Otherwise, the result must
2934 // be negated.
2935 if (KnownNegatives.none())
2936 return Sra;
2937
2938 AddToWorklist(Sra.getNode());
2939 SDValue Sub =
2940 DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Sra);
2941 // If all shift amount elements are negative, we're done.
2942 if (KnownNegatives.all())
2943 return Sub;
2944
2945 // Shift amount has both positive and negative elements.
2946 assert(VT.isVector() && !N0C &&(static_cast <bool> (VT.isVector() && !N0C &&
"Expecting a non-splat vector shift amount") ? void (0) : __assert_fail
("VT.isVector() && !N0C && \"Expecting a non-splat vector shift amount\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2947, __extension__ __PRETTY_FUNCTION__))
2947 "Expecting a non-splat vector shift amount")(static_cast <bool> (VT.isVector() && !N0C &&
"Expecting a non-splat vector shift amount") ? void (0) : __assert_fail
("VT.isVector() && !N0C && \"Expecting a non-splat vector shift amount\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2947, __extension__ __PRETTY_FUNCTION__))
;
2948
2949 SmallVector<SDValue, 64> VSelectMask;
2950 for (int i = 0, e = VT.getVectorNumElements(); i < e; ++i)
2951 VSelectMask.push_back(
2952 DAG.getConstant(KnownNegatives[i] ? -1 : 0, DL, MVT::i1));
2953
2954 SDValue Mask =
2955 DAG.getBuildVector(EVT::getVectorVT(*DAG.getContext(), MVT::i1,
2956 VT.getVectorElementCount()),
2957 DL, VSelectMask);
2958 return DAG.getNode(ISD::VSELECT, DL, VT, Mask, Sub, Sra);
2959 }
2960
2961 // If integer divide is expensive and we satisfy the requirements, emit an
2962 // alternate sequence. Targets may check function attributes for size/speed
2963 // trade-offs.
2964 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
2965 if (N1C && !TLI.isIntDivCheap(N->getValueType(0), Attr))
2966 if (SDValue Op = BuildSDIV(N))
2967 return Op;
2968
2969 // sdiv, srem -> sdivrem
2970 // If the divisor is constant, then return DIVREM only if isIntDivCheap() is
2971 // true. Otherwise, we break the simplification logic in visitREM().
2972 if (!N1C || TLI.isIntDivCheap(N->getValueType(0), Attr))
2973 if (SDValue DivRem = useDivRem(N))
2974 return DivRem;
2975
2976 return SDValue();
2977}
2978
2979SDValue DAGCombiner::visitUDIV(SDNode *N) {
2980 SDValue N0 = N->getOperand(0);
2981 SDValue N1 = N->getOperand(1);
2982 EVT VT = N->getValueType(0);
2983
2984 // fold vector ops
2985 if (VT.isVector())
2986 if (SDValue FoldedVOp = SimplifyVBinOp(N))
2987 return FoldedVOp;
2988
2989 SDLoc DL(N);
2990
2991 // fold (udiv c1, c2) -> c1/c2
2992 ConstantSDNode *N0C = isConstOrConstSplat(N0);
2993 ConstantSDNode *N1C = isConstOrConstSplat(N1);
2994 if (N0C && N1C)
2995 if (SDValue Folded = DAG.FoldConstantArithmetic(ISD::UDIV, DL, VT,
2996 N0C, N1C))
2997 return Folded;
2998
2999 if (SDValue V = simplifyDivRem(N, DAG))
3000 return V;
3001
3002 if (SDValue NewSel = foldBinOpIntoSelect(N))
3003 return NewSel;
3004
3005 // fold (udiv x, (1 << c)) -> x >>u c
3006 if (isConstantOrConstantVector(N1, /*NoOpaques*/ true) &&
3007 DAG.isKnownToBeAPowerOfTwo(N1)) {
3008 SDValue LogBase2 = BuildLogBase2(N1, DL);
3009 AddToWorklist(LogBase2.getNode());
3010
3011 EVT ShiftVT = getShiftAmountTy(N0.getValueType());
3012 SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ShiftVT);
3013 AddToWorklist(Trunc.getNode());
3014 return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc);
3015 }
3016
3017 // fold (udiv x, (shl c, y)) -> x >>u (log2(c)+y) iff c is power of 2
3018 if (N1.getOpcode() == ISD::SHL) {
3019 SDValue N10 = N1.getOperand(0);
3020 if (isConstantOrConstantVector(N10, /*NoOpaques*/ true) &&
3021 DAG.isKnownToBeAPowerOfTwo(N10)) {
3022 SDValue LogBase2 = BuildLogBase2(N10, DL);
3023 AddToWorklist(LogBase2.getNode());
3024
3025 EVT ADDVT = N1.getOperand(1).getValueType();
3026 SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ADDVT);
3027 AddToWorklist(Trunc.getNode());
3028 SDValue Add = DAG.getNode(ISD::ADD, DL, ADDVT, N1.getOperand(1), Trunc);
3029 AddToWorklist(Add.getNode());
3030 return DAG.getNode(ISD::SRL, DL, VT, N0, Add);
3031 }
3032 }
3033
3034 // fold (udiv x, c) -> alternate
3035 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
3036 if (N1C && !TLI.isIntDivCheap(N->getValueType(0), Attr))
3037 if (SDValue Op = BuildUDIV(N))
3038 return Op;
3039
3040 // sdiv, srem -> sdivrem
3041 // If the divisor is constant, then return DIVREM only if isIntDivCheap() is
3042 // true. Otherwise, we break the simplification logic in visitREM().
3043 if (!N1C || TLI.isIntDivCheap(N->getValueType(0), Attr))
3044 if (SDValue DivRem = useDivRem(N))
3045 return DivRem;
3046
3047 return SDValue();
3048}
3049
3050// handles ISD::SREM and ISD::UREM
3051SDValue DAGCombiner::visitREM(SDNode *N) {
3052 unsigned Opcode = N->getOpcode();
3053 SDValue N0 = N->getOperand(0);
3054 SDValue N1 = N->getOperand(1);
3055 EVT VT = N->getValueType(0);
3056 bool isSigned = (Opcode == ISD::SREM);
3057 SDLoc DL(N);
3058
3059 // fold (rem c1, c2) -> c1%c2
3060 ConstantSDNode *N0C = isConstOrConstSplat(N0);
3061 ConstantSDNode *N1C = isConstOrConstSplat(N1);
3062 if (N0C && N1C)
3063 if (SDValue Folded = DAG.FoldConstantArithmetic(Opcode, DL, VT, N0C, N1C))
3064 return Folded;
3065
3066 if (SDValue V = simplifyDivRem(N, DAG))
3067 return V;
3068
3069 if (SDValue NewSel = foldBinOpIntoSelect(N))
3070 return NewSel;
3071
3072 if (isSigned) {
3073 // If we know the sign bits of both operands are zero, strength reduce to a
3074 // urem instead. Handles (X & 0x0FFFFFFF) %s 16 -> X&15
3075 if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
3076 return DAG.getNode(ISD::UREM, DL, VT, N0, N1);
3077 } else {
3078 SDValue NegOne = DAG.getAllOnesConstant(DL, VT);
3079 if (DAG.isKnownToBeAPowerOfTwo(N1)) {
3080 // fold (urem x, pow2) -> (and x, pow2-1)
3081 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne);
3082 AddToWorklist(Add.getNode());
3083 return DAG.getNode(ISD::AND, DL, VT, N0, Add);
3084 }
3085 if (N1.getOpcode() == ISD::SHL &&
3086 DAG.isKnownToBeAPowerOfTwo(N1.getOperand(0))) {
3087 // fold (urem x, (shl pow2, y)) -> (and x, (add (shl pow2, y), -1))
3088 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne);
3089 AddToWorklist(Add.getNode());
3090 return DAG.getNode(ISD::AND, DL, VT, N0, Add);
3091 }
3092 }
3093
3094 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
3095
3096 // If X/C can be simplified by the division-by-constant logic, lower
3097 // X%C to the equivalent of X-X/C*C.
3098 // To avoid mangling nodes, this simplification requires that the combine()
3099 // call for the speculative DIV must not cause a DIVREM conversion. We guard
3100 // against this by skipping the simplification if isIntDivCheap(). When
3101 // div is not cheap, combine will not return a DIVREM. Regardless,
3102 // checking cheapness here makes sense since the simplification results in
3103 // fatter code.
3104 if (N1C && !N1C->isNullValue() && !TLI.isIntDivCheap(VT, Attr)) {
3105 unsigned DivOpcode = isSigned ? ISD::SDIV : ISD::UDIV;
3106 SDValue Div = DAG.getNode(DivOpcode, DL, VT, N0, N1);
3107 AddToWorklist(Div.getNode());
3108 SDValue OptimizedDiv = combine(Div.getNode());
3109 if (OptimizedDiv.getNode() && OptimizedDiv.getNode() != Div.getNode() &&
3110 OptimizedDiv.getOpcode() != ISD::UDIVREM &&
3111 OptimizedDiv.getOpcode() != ISD::SDIVREM) {
3112 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, OptimizedDiv, N1);
3113 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
3114 AddToWorklist(Mul.getNode());
3115 return Sub;
3116 }
3117 }
3118
3119 // sdiv, srem -> sdivrem
3120 if (SDValue DivRem = useDivRem(N))
3121 return DivRem.getValue(1);
3122
3123 return SDValue();
3124}
3125
3126SDValue DAGCombiner::visitMULHS(SDNode *N) {
3127 SDValue N0 = N->getOperand(0);
3128 SDValue N1 = N->getOperand(1);
3129 EVT VT = N->getValueType(0);
3130 SDLoc DL(N);
3131
3132 if (VT.isVector()) {
3133 // fold (mulhs x, 0) -> 0
3134 if (ISD::isBuildVectorAllZeros(N1.getNode()))
3135 return N1;
3136 if (ISD::isBuildVectorAllZeros(N0.getNode()))
3137 return N0;
3138 }
3139
3140 // fold (mulhs x, 0) -> 0
3141 if (isNullConstant(N1))
3142 return N1;
3143 // fold (mulhs x, 1) -> (sra x, size(x)-1)
3144 if (isOneConstant(N1))
3145 return DAG.getNode(ISD::SRA, DL, N0.getValueType(), N0,
3146 DAG.getConstant(N0.getValueSizeInBits() - 1, DL,
3147 getShiftAmountTy(N0.getValueType())));
3148
3149 // fold (mulhs x, undef) -> 0
3150 if (N0.isUndef() || N1.isUndef())
3151 return DAG.getConstant(0, DL, VT);
3152
3153 // If the type twice as wide is legal, transform the mulhs to a wider multiply
3154 // plus a shift.
3155 if (VT.isSimple() && !VT.isVector()) {
3156 MVT Simple = VT.getSimpleVT();
3157 unsigned SimpleSize = Simple.getSizeInBits();
3158 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
3159 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
3160 N0 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N0);
3161 N1 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N1);
3162 N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
3163 N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
3164 DAG.getConstant(SimpleSize, DL,
3165 getShiftAmountTy(N1.getValueType())));
3166 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
3167 }
3168 }
3169
3170 return SDValue();
3171}
3172
3173SDValue DAGCombiner::visitMULHU(SDNode *N) {
3174 SDValue N0 = N->getOperand(0);
3175 SDValue N1 = N->getOperand(1);
3176 EVT VT = N->getValueType(0);
3177 SDLoc DL(N);
3178
3179 if (VT.isVector()) {
3180 // fold (mulhu x, 0) -> 0
3181 if (ISD::isBuildVectorAllZeros(N1.getNode()))
3182 return N1;
3183 if (ISD::isBuildVectorAllZeros(N0.getNode()))
3184 return N0;
3185 }
3186
3187 // fold (mulhu x, 0) -> 0
3188 if (isNullConstant(N1))
3189 return N1;
3190 // fold (mulhu x, 1) -> 0
3191 if (isOneConstant(N1))
3192 return DAG.getConstant(0, DL, N0.getValueType());
3193 // fold (mulhu x, undef) -> 0
3194 if (N0.isUndef() || N1.isUndef())
3195 return DAG.getConstant(0, DL, VT);
3196
3197 // If the type twice as wide is legal, transform the mulhu to a wider multiply
3198 // plus a shift.
3199 if (VT.isSimple() && !VT.isVector()) {
3200 MVT Simple = VT.getSimpleVT();
3201 unsigned SimpleSize = Simple.getSizeInBits();
3202 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
3203 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
3204 N0 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N0);
3205 N1 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N1);
3206 N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
3207 N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
3208 DAG.getConstant(SimpleSize, DL,
3209 getShiftAmountTy(N1.getValueType())));
3210 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
3211 }
3212 }
3213
3214 return SDValue();
3215}
3216
3217/// Perform optimizations common to nodes that compute two values. LoOp and HiOp
3218/// give the opcodes for the two computations that are being performed. Return
3219/// true if a simplification was made.
3220SDValue DAGCombiner::SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
3221 unsigned HiOp) {
3222 // If the high half is not needed, just compute the low half.
3223 bool HiExists = N->hasAnyUseOfValue(1);
3224 if (!HiExists &&
3225 (!LegalOperations ||
3226 TLI.isOperationLegalOrCustom(LoOp, N->getValueType(0)))) {
3227 SDValue Res = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0), N->ops());
3228 return CombineTo(N, Res, Res);
3229 }
3230
3231 // If the low half is not needed, just compute the high half.
3232 bool LoExists = N->hasAnyUseOfValue(0);
3233 if (!LoExists &&
3234 (!LegalOperations ||
3235 TLI.isOperationLegal(HiOp, N->getValueType(1)))) {
3236 SDValue Res = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1), N->ops());
3237 return CombineTo(N, Res, Res);
3238 }
3239
3240 // If both halves are used, return as it is.
3241 if (LoExists && HiExists)
3242 return SDValue();
3243
3244 // If the two computed results can be simplified separately, separate them.
3245 if (LoExists) {
3246 SDValue Lo = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0), N->ops());
3247 AddToWorklist(Lo.getNode());
3248 SDValue LoOpt = combine(Lo.getNode());
3249 if (LoOpt.getNode() && LoOpt.getNode() != Lo.getNode() &&
3250 (!LegalOperations ||
3251 TLI.isOperationLegal(LoOpt.getOpcode(), LoOpt.getValueType())))
3252 return CombineTo(N, LoOpt, LoOpt);
3253 }
3254
3255 if (HiExists) {
3256 SDValue Hi = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1), N->ops());
3257 AddToWorklist(Hi.getNode());
3258 SDValue HiOpt = combine(Hi.getNode());
3259 if (HiOpt.getNode() && HiOpt != Hi &&
3260 (!LegalOperations ||
3261 TLI.isOperationLegal(HiOpt.getOpcode(), HiOpt.getValueType())))
3262 return CombineTo(N, HiOpt, HiOpt);
3263 }
3264
3265 return SDValue();
3266}
3267
3268SDValue DAGCombiner::visitSMUL_LOHI(SDNode *N) {
3269 if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHS))
3270 return Res;
3271
3272 EVT VT = N->getValueType(0);
3273 SDLoc DL(N);
3274
3275 // If the type is twice as wide is legal, transform the mulhu to a wider
3276 // multiply plus a shift.
3277 if (VT.isSimple() && !VT.isVector()) {
3278 MVT Simple = VT.getSimpleVT();
3279 unsigned SimpleSize = Simple.getSizeInBits();
3280 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
3281 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
3282 SDValue Lo = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(0));
3283 SDValue Hi = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(1));
3284 Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
3285 // Compute the high part as N1.
3286 Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
3287 DAG.getConstant(SimpleSize, DL,
3288 getShiftAmountTy(Lo.getValueType())));
3289 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
3290 // Compute the low part as N0.
3291 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
3292 return CombineTo(N, Lo, Hi);
3293 }
3294 }
3295
3296 return SDValue();
3297}
3298
3299SDValue DAGCombiner::visitUMUL_LOHI(SDNode *N) {
3300 if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHU))
3301 return Res;
3302
3303 EVT VT = N->getValueType(0);
3304 SDLoc DL(N);
3305
3306 // If the type is twice as wide is legal, transform the mulhu to a wider
3307 // multiply plus a shift.
3308 if (VT.isSimple() && !VT.isVector()) {
3309 MVT Simple = VT.getSimpleVT();
3310 unsigned SimpleSize = Simple.getSizeInBits();
3311 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
3312 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
3313 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(0));
3314 SDValue Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(1));
3315 Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
3316 // Compute the high part as N1.
3317 Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
3318 DAG.getConstant(SimpleSize, DL,
3319 getShiftAmountTy(Lo.getValueType())));
3320 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
3321 // Compute the low part as N0.
3322 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
3323 return CombineTo(N, Lo, Hi);
3324 }
3325 }
3326
3327 return SDValue();
3328}
3329
3330SDValue DAGCombiner::visitSMULO(SDNode *N) {
3331 // (smulo x, 2) -> (saddo x, x)
3332 if (ConstantSDNode *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1)))
3333 if (C2->getAPIntValue() == 2)
3334 return DAG.getNode(ISD::SADDO, SDLoc(N), N->getVTList(),
3335 N->getOperand(0), N->getOperand(0));
3336
3337 return SDValue();
3338}
3339
3340SDValue DAGCombiner::visitUMULO(SDNode *N) {
3341 // (umulo x, 2) -> (uaddo x, x)
3342 if (ConstantSDNode *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1)))
3343 if (C2->getAPIntValue() == 2)
3344 return DAG.getNode(ISD::UADDO, SDLoc(N), N->getVTList(),
3345 N->getOperand(0), N->getOperand(0));
3346
3347 return SDValue();
3348}
3349
3350SDValue DAGCombiner::visitIMINMAX(SDNode *N) {
3351 SDValue N0 = N->getOperand(0);
3352 SDValue N1 = N->getOperand(1);
3353 EVT VT = N0.getValueType();
3354
3355 // fold vector ops
3356 if (VT.isVector())
3357 if (SDValue FoldedVOp = SimplifyVBinOp(N))
3358 return FoldedVOp;
3359
3360 // fold operation with constant operands.
3361 ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
3362 ConstantSDNode *N1C = getAsNonOpaqueConstant(N1);
3363 if (N0C && N1C)
3364 return DAG.FoldConstantArithmetic(N->getOpcode(), SDLoc(N), VT, N0C, N1C);
3365
3366 // canonicalize constant to RHS
3367 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
3368 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
3369 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N1, N0);
3370
3371 // Is sign bits are zero, flip between UMIN/UMAX and SMIN/SMAX.
3372 // Only do this if the current op isn't legal and the flipped is.
3373 unsigned Opcode = N->getOpcode();
3374 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3375 if (!TLI.isOperationLegal(Opcode, VT) &&
3376 (N0.isUndef() || DAG.SignBitIsZero(N0)) &&
3377 (N1.isUndef() || DAG.SignBitIsZero(N1))) {
3378 unsigned AltOpcode;
3379 switch (Opcode) {
3380 case ISD::SMIN: AltOpcode = ISD::UMIN; break;
3381 case ISD::SMAX: AltOpcode = ISD::UMAX; break;
3382 case ISD::UMIN: AltOpcode = ISD::SMIN; break;
3383 case ISD::UMAX: AltOpcode = ISD::SMAX; break;
3384 default: llvm_unreachable("Unknown MINMAX opcode")::llvm::llvm_unreachable_internal("Unknown MINMAX opcode", "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3384)
;
3385 }
3386 if (TLI.isOperationLegal(AltOpcode, VT))
3387 return DAG.getNode(AltOpcode, SDLoc(N), VT, N0, N1);
3388 }
3389
3390 return SDValue();
3391}
3392
3393/// If this is a binary operator with two operands of the same opcode, try to
3394/// simplify it.
3395SDValue DAGCombiner::SimplifyBinOpWithSameOpcodeHands(SDNode *N) {
3396 SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
3397 EVT VT = N0.getValueType();
3398 assert(N0.getOpcode() == N1.getOpcode() && "Bad input!")(static_cast <bool> (N0.getOpcode() == N1.getOpcode() &&
"Bad input!") ? void (0) : __assert_fail ("N0.getOpcode() == N1.getOpcode() && \"Bad input!\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3398, __extension__ __PRETTY_FUNCTION__))
;
3399
3400 // Bail early if none of these transforms apply.
3401 if (N0.getNumOperands() == 0) return SDValue();
3402
3403 // For each of OP in AND/OR/XOR:
3404 // fold (OP (zext x), (zext y)) -> (zext (OP x, y))
3405 // fold (OP (sext x), (sext y)) -> (sext (OP x, y))
3406 // fold (OP (aext x), (aext y)) -> (aext (OP x, y))
3407 // fold (OP (bswap x), (bswap y)) -> (bswap (OP x, y))
3408 // fold (OP (trunc x), (trunc y)) -> (trunc (OP x, y)) (if trunc isn't free)
3409 //
3410 // do not sink logical op inside of a vector extend, since it may combine
3411 // into a vsetcc.
3412 EVT Op0VT = N0.getOperand(0).getValueType();
3413 if ((N0.getOpcode() == ISD::ZERO_EXTEND ||
3414 N0.getOpcode() == ISD::SIGN_EXTEND ||
3415 N0.getOpcode() == ISD::BSWAP ||
3416 // Avoid infinite looping with PromoteIntBinOp.
3417 (N0.getOpcode() == ISD::ANY_EXTEND &&
3418 (!LegalTypes || TLI.isTypeDesirableForOp(N->getOpcode(), Op0VT))) ||
3419 (N0.getOpcode() == ISD::TRUNCATE &&
3420 (!TLI.isZExtFree(VT, Op0VT) ||
3421 !TLI.isTruncateFree(Op0VT, VT)) &&
3422 TLI.isTypeLegal(Op0VT))) &&
3423 !VT.isVector() &&
3424 Op0VT == N1.getOperand(0).getValueType() &&
3425 (!LegalOperations || TLI.isOperationLegal(N->getOpcode(), Op0VT))) {
3426 SDValue ORNode = DAG.getNode(N->getOpcode(), SDLoc(N0),
3427 N0.getOperand(0).getValueType(),
3428 N0.getOperand(0), N1.getOperand(0));
3429 AddToWorklist(ORNode.getNode());
3430 return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, ORNode);
3431 }
3432
3433 // For each of OP in SHL/SRL/SRA/AND...
3434 // fold (and (OP x, z), (OP y, z)) -> (OP (and x, y), z)
3435 // fold (or (OP x, z), (OP y, z)) -> (OP (or x, y), z)
3436 // fold (xor (OP x, z), (OP y, z)) -> (OP (xor x, y), z)
3437 if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL ||
3438 N0.getOpcode() == ISD::SRA || N0.getOpcode() == ISD::AND) &&
3439 N0.getOperand(1) == N1.getOperand(1)) {
3440 SDValue ORNode = DAG.getNode(N->getOpcode(), SDLoc(N0),
3441 N0.getOperand(0).getValueType(),
3442 N0.getOperand(0), N1.getOperand(0));
3443 AddToWorklist(ORNode.getNode());
3444 return DAG.getNode(N0.getOpcode(), SDLoc(N), VT,
3445 ORNode, N0.getOperand(1));
3446 }
3447
3448 // Simplify xor/and/or (bitcast(A), bitcast(B)) -> bitcast(op (A,B))
3449 // Only perform this optimization up until type legalization, before
3450 // LegalizeVectorOprs. LegalizeVectorOprs promotes vector operations by
3451 // adding bitcasts. For example (xor v4i32) is promoted to (v2i64), and
3452 // we don't want to undo this promotion.
3453 // We also handle SCALAR_TO_VECTOR because xor/or/and operations are cheaper
3454 // on scalars.
3455 if ((N0.getOpcode() == ISD::BITCAST ||
3456 N0.getOpcode() == ISD::SCALAR_TO_VECTOR) &&
3457 Level <= AfterLegalizeTypes) {
3458 SDValue In0 = N0.getOperand(0);
3459 SDValue In1 = N1.getOperand(0);
3460 EVT In0Ty = In0.getValueType();
3461 EVT In1Ty = In1.getValueType();
3462 SDLoc DL(N);
3463 // If both incoming values are integers, and the original types are the
3464 // same.
3465 if (In0Ty.isInteger() && In1Ty.isInteger() && In0Ty == In1Ty) {
3466 SDValue Op = DAG.getNode(N->getOpcode(), DL, In0Ty, In0, In1);
3467 SDValue BC = DAG.getNode(N0.getOpcode(), DL, VT, Op);
3468 AddToWorklist(Op.getNode());
3469 return BC;
3470 }
3471 }
3472
3473 // Xor/and/or are indifferent to the swizzle operation (shuffle of one value).
3474 // Simplify xor/and/or (shuff(A), shuff(B)) -> shuff(op (A,B))
3475 // If both shuffles use the same mask, and both shuffle within a single
3476 // vector, then it is worthwhile to move the swizzle after the operation.
3477 // The type-legalizer generates this pattern when loading illegal
3478 // vector types from memory. In many cases this allows additional shuffle
3479 // optimizations.
3480 // There are other cases where moving the shuffle after the xor/and/or
3481 // is profitable even if shuffles don't perform a swizzle.
3482 // If both shuffles use the same mask, and both shuffles have the same first
3483 // or second operand, then it might still be profitable to move the shuffle
3484 // after the xor/and/or operation.
3485 if (N0.getOpcode() == ISD::VECTOR_SHUFFLE && Level < AfterLegalizeDAG) {
3486 ShuffleVectorSDNode *SVN0 = cast<ShuffleVectorSDNode>(N0);
3487 ShuffleVectorSDNode *SVN1 = cast<ShuffleVectorSDNode>(N1);
3488
3489 assert(N0.getOperand(0).getValueType() == N1.getOperand(0).getValueType() &&(static_cast <bool> (N0.getOperand(0).getValueType() ==
N1.getOperand(0).getValueType() && "Inputs to shuffles are not the same type"
) ? void (0) : __assert_fail ("N0.getOperand(0).getValueType() == N1.getOperand(0).getValueType() && \"Inputs to shuffles are not the same type\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3490, __extension__ __PRETTY_FUNCTION__))
3490 "Inputs to shuffles are not the same type")(static_cast <bool> (N0.getOperand(0).getValueType() ==
N1.getOperand(0).getValueType() && "Inputs to shuffles are not the same type"
) ? void (0) : __assert_fail ("N0.getOperand(0).getValueType() == N1.getOperand(0).getValueType() && \"Inputs to shuffles are not the same type\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3490, __extension__ __PRETTY_FUNCTION__))
;
3491
3492 // Check that both shuffles use the same mask. The masks are known to be of
3493 // the same length because the result vector type is the same.
3494 // Check also that shuffles have only one use to avoid introducing extra
3495 // instructions.
3496 if (SVN0->hasOneUse() && SVN1->hasOneUse() &&
3497 SVN0->getMask().equals(SVN1->getMask())) {
3498 SDValue ShOp = N0->getOperand(1);
3499
3500 // Don't try to fold this node if it requires introducing a
3501 // build vector of all zeros that might be illegal at this stage.
3502 if (N->getOpcode() == ISD::XOR && !ShOp.isUndef()) {
3503 if (!LegalTypes)
3504 ShOp = DAG.getConstant(0, SDLoc(N), VT);
3505 else
3506 ShOp = SDValue();
3507 }
3508
3509 // (AND (shuf (A, C), shuf (B, C)) -> shuf (AND (A, B), C)
3510 // (OR (shuf (A, C), shuf (B, C)) -> shuf (OR (A, B), C)
3511 // (XOR (shuf (A, C), shuf (B, C)) -> shuf (XOR (A, B), V_0)
3512 if (N0.getOperand(1) == N1.getOperand(1) && ShOp.getNode()) {
3513 SDValue NewNode = DAG.getNode(N->getOpcode(), SDLoc(N), VT,
3514 N0->getOperand(0), N1->getOperand(0));
3515 AddToWorklist(NewNode.getNode());
3516 return DAG.getVectorShuffle(VT, SDLoc(N), NewNode, ShOp,
3517 SVN0->getMask());
3518 }
3519
3520 // Don't try to fold this node if it requires introducing a
3521 // build vector of all zeros that might be illegal at this stage.
3522 ShOp = N0->getOperand(0);
3523 if (N->getOpcode() == ISD::XOR && !ShOp.isUndef()) {
3524 if (!LegalTypes)
3525 ShOp = DAG.getConstant(0, SDLoc(N), VT);
3526 else
3527 ShOp = SDValue();
3528 }
3529
3530 // (AND (shuf (C, A), shuf (C, B)) -> shuf (C, AND (A, B))
3531 // (OR (shuf (C, A), shuf (C, B)) -> shuf (C, OR (A, B))
3532 // (XOR (shuf (C, A), shuf (C, B)) -> shuf (V_0, XOR (A, B))
3533 if (N0->getOperand(0) == N1->getOperand(0) && ShOp.getNode()) {
3534 SDValue NewNode = DAG.getNode(N->getOpcode(), SDLoc(N), VT,
3535 N0->getOperand(1), N1->getOperand(1));
3536 AddToWorklist(NewNode.getNode());
3537 return DAG.getVectorShuffle(VT, SDLoc(N), ShOp, NewNode,
3538 SVN0->getMask());
3539 }
3540 }
3541 }
3542
3543 return SDValue();
3544}
3545
3546/// Try to make (and/or setcc (LL, LR), setcc (RL, RR)) more efficient.
3547SDValue DAGCombiner::foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1,
3548 const SDLoc &DL) {
3549 SDValue LL, LR, RL, RR, N0CC, N1CC;
3550 if (!isSetCCEquivalent(N0, LL, LR, N0CC) ||
3551 !isSetCCEquivalent(N1, RL, RR, N1CC))
3552 return SDValue();
3553
3554 assert(N0.getValueType() == N1.getValueType() &&(static_cast <bool> (N0.getValueType() == N1.getValueType
() && "Unexpected operand types for bitwise logic op"
) ? void (0) : __assert_fail ("N0.getValueType() == N1.getValueType() && \"Unexpected operand types for bitwise logic op\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3555, __extension__ __PRETTY_FUNCTION__))
3555 "Unexpected operand types for bitwise logic op")(static_cast <bool> (N0.getValueType() == N1.getValueType
() && "Unexpected operand types for bitwise logic op"
) ? void (0) : __assert_fail ("N0.getValueType() == N1.getValueType() && \"Unexpected operand types for bitwise logic op\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3555, __extension__ __PRETTY_FUNCTION__))
;
3556 assert(LL.getValueType() == LR.getValueType() &&(static_cast <bool> (LL.getValueType() == LR.getValueType
() && RL.getValueType() == RR.getValueType() &&
"Unexpected operand types for setcc") ? void (0) : __assert_fail
("LL.getValueType() == LR.getValueType() && RL.getValueType() == RR.getValueType() && \"Unexpected operand types for setcc\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3558, __extension__ __PRETTY_FUNCTION__))
3557 RL.getValueType() == RR.getValueType() &&(static_cast <bool> (LL.getValueType() == LR.getValueType
() && RL.getValueType() == RR.getValueType() &&
"Unexpected operand types for setcc") ? void (0) : __assert_fail
("LL.getValueType() == LR.getValueType() && RL.getValueType() == RR.getValueType() && \"Unexpected operand types for setcc\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3558, __extension__ __PRETTY_FUNCTION__))
3558 "Unexpected operand types for setcc")(static_cast <bool> (LL.getValueType() == LR.getValueType
() && RL.getValueType() == RR.getValueType() &&
"Unexpected operand types for setcc") ? void (0) : __assert_fail
("LL.getValueType() == LR.getValueType() && RL.getValueType() == RR.getValueType() && \"Unexpected operand types for setcc\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3558, __extension__ __PRETTY_FUNCTION__))
;
3559
3560 // If we're here post-legalization or the logic op type is not i1, the logic
3561 // op type must match a setcc result type. Also, all folds require new
3562 // operations on the left and right operands, so those types must match.
3563 EVT VT = N0.getValueType();
3564 EVT OpVT = LL.getValueType();
3565 if (LegalOperations || VT.getScalarType() != MVT::i1)
3566 if (VT != getSetCCResultType(OpVT))
3567 return SDValue();
3568 if (OpVT != RL.getValueType())
3569 return SDValue();
3570
3571 ISD::CondCode CC0 = cast<CondCodeSDNode>(N0CC)->get();
3572 ISD::CondCode CC1 = cast<CondCodeSDNode>(N1CC)->get();
3573 bool IsInteger = OpVT.isInteger();
3574 if (LR == RR && CC0 == CC1 && IsInteger) {
3575 bool IsZero = isNullConstantOrNullSplatConstant(LR);
3576 bool IsNeg1 = isAllOnesConstantOrAllOnesSplatConstant(LR);
3577
3578 // All bits clear?
3579 bool AndEqZero = IsAnd && CC1 == ISD::SETEQ && IsZero;
3580 // All sign bits clear?
3581 bool AndGtNeg1 = IsAnd && CC1 == ISD::SETGT && IsNeg1;
3582 // Any bits set?
3583 bool OrNeZero = !IsAnd && CC1 == ISD::SETNE && IsZero;
3584 // Any sign bits set?
3585 bool OrLtZero = !IsAnd && CC1 == ISD::SETLT && IsZero;
3586
3587 // (and (seteq X, 0), (seteq Y, 0)) --> (seteq (or X, Y), 0)
3588 // (and (setgt X, -1), (setgt Y, -1)) --> (setgt (or X, Y), -1)
3589 // (or (setne X, 0), (setne Y, 0)) --> (setne (or X, Y), 0)
3590 // (or (setlt X, 0), (setlt Y, 0)) --> (setlt (or X, Y), 0)
3591 if (AndEqZero || AndGtNeg1 || OrNeZero || OrLtZero) {
3592 SDValue Or = DAG.getNode(ISD::OR, SDLoc(N0), OpVT, LL, RL);
3593 AddToWorklist(Or.getNode());
3594 return DAG.getSetCC(DL, VT, Or, LR, CC1);
3595 }
3596
3597 // All bits set?
3598 bool AndEqNeg1 = IsAnd && CC1 == ISD::SETEQ && IsNeg1;
3599 // All sign bits set?
3600 bool AndLtZero = IsAnd && CC1 == ISD::SETLT && IsZero;
3601 // Any bits clear?
3602 bool OrNeNeg1 = !IsAnd && CC1 == ISD::SETNE && IsNeg1;
3603 // Any sign bits clear?
3604 bool OrGtNeg1 = !IsAnd && CC1 == ISD::SETGT && IsNeg1;
3605
3606 // (and (seteq X, -1), (seteq Y, -1)) --> (seteq (and X, Y), -1)
3607 // (and (setlt X, 0), (setlt Y, 0)) --> (setlt (and X, Y), 0)
3608 // (or (setne X, -1), (setne Y, -1)) --> (setne (and X, Y), -1)
3609 // (or (setgt X, -1), (setgt Y -1)) --> (setgt (and X, Y), -1)
3610 if (AndEqNeg1 || AndLtZero || OrNeNeg1 || OrGtNeg1) {
3611 SDValue And = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, LL, RL);
3612 AddToWorklist(And.getNode());
3613 return DAG.getSetCC(DL, VT, And, LR, CC1);
3614 }
3615 }
3616
3617 // TODO: What is the 'or' equivalent of this fold?
3618 // (and (setne X, 0), (setne X, -1)) --> (setuge (add X, 1), 2)
3619 if (IsAnd && LL == RL && CC0 == CC1 && OpVT.getScalarSizeInBits() > 1 &&
3620 IsInteger && CC0 == ISD::SETNE &&
3621 ((isNullConstant(LR) && isAllOnesConstant(RR)) ||
3622 (isAllOnesConstant(LR) && isNullConstant(RR)))) {
3623 SDValue One = DAG.getConstant(1, DL, OpVT);
3624 SDValue Two = DAG.getConstant(2, DL, OpVT);
3625 SDValue Add = DAG.getNode(ISD::ADD, SDLoc(N0), OpVT, LL, One);
3626 AddToWorklist(Add.getNode());
3627 return DAG.getSetCC(DL, VT, Add, Two, ISD::SETUGE);
3628 }
3629
3630 // Try more general transforms if the predicates match and the only user of
3631 // the compares is the 'and' or 'or'.
3632 if (IsInteger && TLI.convertSetCCLogicToBitwiseLogic(OpVT) && CC0 == CC1 &&
3633 N0.hasOneUse() && N1.hasOneUse()) {
3634 // and (seteq A, B), (seteq C, D) --> seteq (or (xor A, B), (xor C, D)), 0
3635 // or (setne A, B), (setne C, D) --> setne (or (xor A, B), (xor C, D)), 0
3636 if ((IsAnd && CC1 == ISD::SETEQ) || (!IsAnd && CC1 == ISD::SETNE)) {
3637 SDValue XorL = DAG.getNode(ISD::XOR, SDLoc(N0), OpVT, LL, LR);
3638 SDValue XorR = DAG.getNode(ISD::XOR, SDLoc(N1), OpVT, RL, RR);
3639 SDValue Or = DAG.getNode(ISD::OR, DL, OpVT, XorL, XorR);
3640 SDValue Zero = DAG.getConstant(0, DL, OpVT);
3641 return DAG.getSetCC(DL, VT, Or, Zero, CC1);
3642 }
3643 }
3644
3645 // Canonicalize equivalent operands to LL == RL.
3646 if (LL == RR && LR == RL) {
3647 CC1 = ISD::getSetCCSwappedOperands(CC1);
3648 std::swap(RL, RR);
3649 }
3650
3651 // (and (setcc X, Y, CC0), (setcc X, Y, CC1)) --> (setcc X, Y, NewCC)
3652 // (or (setcc X, Y, CC0), (setcc X, Y, CC1)) --> (setcc X, Y, NewCC)
3653 if (LL == RL && LR == RR) {
3654 ISD::CondCode NewCC = IsAnd ? ISD::getSetCCAndOperation(CC0, CC1, IsInteger)
3655 : ISD::getSetCCOrOperation(CC0, CC1, IsInteger);
3656 if (NewCC != ISD::SETCC_INVALID &&
3657 (!LegalOperations ||
3658 (TLI.isCondCodeLegal(NewCC, LL.getSimpleValueType()) &&
3659 TLI.isOperationLegal(ISD::SETCC, OpVT))))
3660 return DAG.getSetCC(DL, VT, LL, LR, NewCC);
3661 }
3662
3663 return SDValue();
3664}
3665
3666/// This contains all DAGCombine rules which reduce two values combined by
3667/// an And operation to a single value. This makes them reusable in the context
3668/// of visitSELECT(). Rules involving constants are not included as
3669/// visitSELECT() already handles those cases.
3670SDValue DAGCombiner::visitANDLike(SDValue N0, SDValue N1, SDNode *N) {
3671 EVT VT = N1.getValueType();
3672 SDLoc DL(N);
3673
3674 // fold (and x, undef) -> 0
3675 if (N0.isUndef() || N1.isUndef())
3676 return DAG.getConstant(0, DL, VT);
3677
3678 if (SDValue V = foldLogicOfSetCCs(true, N0, N1, DL))
3679 return V;
3680
3681 if (N0.getOpcode() == ISD::ADD && N1.getOpcode() == ISD::SRL &&
3682 VT.getSizeInBits() <= 64) {
3683 if (ConstantSDNode *ADDI = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
3684 if (ConstantSDNode *SRLI = dyn_cast<ConstantSDNode>(N1.getOperand(1))) {
3685 // Look for (and (add x, c1), (lshr y, c2)). If C1 wasn't a legal
3686 // immediate for an add, but it is legal if its top c2 bits are set,
3687 // transform the ADD so the immediate doesn't need to be materialized
3688 // in a register.
3689 APInt ADDC = ADDI->getAPIntValue();
3690 APInt SRLC = SRLI->getAPIntValue();
3691 if (ADDC.getMinSignedBits() <= 64 &&
3692 SRLC.ult(VT.getSizeInBits()) &&
3693 !TLI.isLegalAddImmediate(ADDC.getSExtValue())) {
3694 APInt Mask = APInt::getHighBitsSet(VT.getSizeInBits(),
3695 SRLC.getZExtValue());
3696 if (DAG.MaskedValueIsZero(N0.getOperand(1), Mask)) {
3697 ADDC |= Mask;
3698 if (TLI.isLegalAddImmediate(ADDC.getSExtValue())) {
3699 SDLoc DL0(N0);
3700 SDValue NewAdd =
3701 DAG.getNode(ISD::ADD, DL0, VT,
3702 N0.getOperand(0), DAG.getConstant(ADDC, DL, VT));
3703 CombineTo(N0.getNode(), NewAdd);
3704 // Return N so it doesn't get rechecked!
3705 return SDValue(N, 0);
3706 }
3707 }
3708 }
3709 }
3710 }
3711 }
3712
3713 // Reduce bit extract of low half of an integer to the narrower type.
3714 // (and (srl i64:x, K), KMask) ->
3715 // (i64 zero_extend (and (srl (i32 (trunc i64:x)), K)), KMask)
3716 if (N0.getOpcode() == ISD::SRL && N0.hasOneUse()) {
3717 if (ConstantSDNode *CAnd = dyn_cast<ConstantSDNode>(N1)) {
3718 if (ConstantSDNode *CShift = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
3719 unsigned Size = VT.getSizeInBits();
3720 const APInt &AndMask = CAnd->getAPIntValue();
3721 unsigned ShiftBits = CShift->getZExtValue();
3722
3723 // Bail out, this node will probably disappear anyway.
3724 if (ShiftBits == 0)
3725 return SDValue();
3726
3727 unsigned MaskBits = AndMask.countTrailingOnes();
3728 EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), Size / 2);
3729
3730 if (AndMask.isMask() &&
3731 // Required bits must not span the two halves of the integer and
3732 // must fit in the half size type.
3733 (ShiftBits + MaskBits <= Size / 2) &&
3734 TLI.isNarrowingProfitable(VT, HalfVT) &&
3735 TLI.isTypeDesirableForOp(ISD::AND, HalfVT) &&
3736 TLI.isTypeDesirableForOp(ISD::SRL, HalfVT) &&
3737 TLI.isTruncateFree(VT, HalfVT) &&
3738 TLI.isZExtFree(HalfVT, VT)) {
3739 // The isNarrowingProfitable is to avoid regressions on PPC and
3740 // AArch64 which match a few 64-bit bit insert / bit extract patterns
3741 // on downstream users of this. Those patterns could probably be
3742 // extended to handle extensions mixed in.
3743
3744 SDValue SL(N0);
3745 assert(MaskBits <= Size)(static_cast <bool> (MaskBits <= Size) ? void (0) : __assert_fail
("MaskBits <= Size", "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3745, __extension__ __PRETTY_FUNCTION__))
;
3746
3747 // Extracting the highest bit of the low half.
3748 EVT ShiftVT = TLI.getShiftAmountTy(HalfVT, DAG.getDataLayout());
3749 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, HalfVT,
3750 N0.getOperand(0));
3751
3752 SDValue NewMask = DAG.getConstant(AndMask.trunc(Size / 2), SL, HalfVT);
3753 SDValue ShiftK = DAG.getConstant(ShiftBits, SL, ShiftVT);
3754 SDValue Shift = DAG.getNode(ISD::SRL, SL, HalfVT, Trunc, ShiftK);
3755 SDValue And = DAG.getNode(ISD::AND, SL, HalfVT, Shift, NewMask);
3756 return DAG.getNode(ISD::ZERO_EXTEND, SL, VT, And);
3757 }
3758 }
3759 }
3760 }
3761
3762 return SDValue();
3763}
3764
3765bool DAGCombiner::isAndLoadExtLoad(ConstantSDNode *AndC, LoadSDNode *LoadN,
3766 EVT LoadResultTy, EVT &ExtVT) {
3767 if (!AndC->getAPIntValue().isMask())
3768 return false;
3769
3770 unsigned ActiveBits = AndC->getAPIntValue().countTrailingOnes();
3771
3772 ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
3773 EVT LoadedVT = LoadN->getMemoryVT();
3774
3775 if (ExtVT == LoadedVT &&
3776 (!LegalOperations ||
3777 TLI.isLoadExtLegal(ISD::ZEXTLOAD, LoadResultTy, ExtVT))) {
3778 // ZEXTLOAD will match without needing to change the size of the value being
3779 // loaded.
3780 return true;
3781 }
3782
3783 // Do not change the width of a volatile load.
3784 if (LoadN->isVolatile())
3785 return false;
3786
3787 // Do not generate loads of non-round integer types since these can
3788 // be expensive (and would be wrong if the type is not byte sized).
3789 if (!LoadedVT.bitsGT(ExtVT) || !ExtVT.isRound())
3790 return false;
3791
3792 if (LegalOperations &&
3793 !TLI.isLoadExtLegal(ISD::ZEXTLOAD, LoadResultTy, ExtVT))
3794 return false;
3795
3796 if (!TLI.shouldReduceLoadWidth(LoadN, ISD::ZEXTLOAD, ExtVT))
3797 return false;
3798
3799 return true;
3800}
3801
3802bool DAGCombiner::isLegalNarrowLoad(LoadSDNode *LoadN, ISD::LoadExtType ExtType,
3803 EVT &ExtVT, unsigned ShAmt) {
3804 // Don't transform one with multiple uses, this would require adding a new
3805 // load.
3806 if (!SDValue(LoadN, 0).hasOneUse())
3807 return false;
3808
3809 if (LegalOperations &&
3810 !TLI.isLoadExtLegal(ExtType, LoadN->getValueType(0), ExtVT))
3811 return false;
3812
3813 // Do not generate loads of non-round integer types since these can
3814 // be expensive (and would be wrong if the type is not byte sized).
3815 if (!ExtVT.isRound())
3816 return false;
3817
3818 // Don't change the width of a volatile load.
3819 if (LoadN->isVolatile())
3820 return false;
3821
3822 // Verify that we are actually reducing a load width here.
3823 if (LoadN->getMemoryVT().getSizeInBits() < ExtVT.getSizeInBits())
3824 return false;
3825
3826 // For the transform to be legal, the load must produce only two values
3827 // (the value loaded and the chain). Don't transform a pre-increment
3828 // load, for example, which produces an extra value. Otherwise the
3829 // transformation is not equivalent, and the downstream logic to replace
3830 // uses gets things wrong.
3831 if (LoadN->getNumValues() > 2)
3832 return false;
3833
3834 // Only allow byte offsets.
3835 if (ShAmt % 8)
3836 return false;
3837
3838 // Ensure that this isn't going to produce an unsupported unaligned access.
3839 if (ShAmt && !TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(),
3840 ExtVT, LoadN->getAddressSpace(),
3841 ShAmt / 8))
3842 return false;
3843
3844
3845 // If the load that we're shrinking is an extload and we're not just
3846 // discarding the extension we can't simply shrink the load. Bail.
3847 // TODO: It would be possible to merge the extensions in some cases.
3848 if (LoadN->getExtensionType() != ISD::NON_EXTLOAD &&
3849 LoadN->getMemoryVT().getSizeInBits() < ExtVT.getSizeInBits() + ShAmt)
3850 return false;
3851
3852 if (!TLI.shouldReduceLoadWidth(LoadN, ExtType, ExtVT))
3853 return false;
3854
3855 // It's not possible to generate a constant of extended or untyped type.
3856 EVT PtrType = LoadN->getOperand(1).getValueType();
3857 if (PtrType == MVT::Untyped || PtrType.isExtended())
3858 return false;
3859
3860 return true;
3861}
3862
3863bool DAGCombiner::SearchForAndLoads(SDNode *N,
3864 SmallPtrSetImpl<LoadSDNode*> &Loads,
3865 SmallPtrSetImpl<SDNode*> &NodesWithConsts,
3866 ConstantSDNode *Mask,
3867 SDNode *&NodeToMask) {
3868 // Recursively search for the operands, looking for loads which can be
3869 // narrowed.
3870 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) {
3871 SDValue Op = N->getOperand(i);
3872
3873 if (Op.getValueType().isVector())
3874 return false;
3875
3876 // Some constants may need fixing up later if they are too large.
3877 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
3878 if ((N->getOpcode() == ISD::OR || N->getOpcode() == ISD::XOR) &&
3879 (Mask->getAPIntValue() & C->getAPIntValue()) != C->getAPIntValue())
3880 NodesWithConsts.insert(N);
3881 continue;
3882 }
3883
3884 if (!Op.hasOneUse())
3885 return false;
3886
3887 switch(Op.getOpcode()) {
3888 case ISD::LOAD: {
3889 auto *Load = cast<LoadSDNode>(Op);
3890 EVT ExtVT;
3891 if (isAndLoadExtLoad(Mask, Load, Load->getValueType(0), ExtVT) &&
3892 isLegalNarrowLoad(Load, ISD::ZEXTLOAD, ExtVT)) {
3893
3894 // ZEXTLOAD is already small enough.
3895 if (Load->getExtensionType() == ISD::ZEXTLOAD &&
3896 ExtVT.bitsGE(Load->getMemoryVT()))
3897 continue;
3898
3899 // Use LE to convert equal sized loads to zext.
3900 if (ExtVT.bitsLE(Load->getMemoryVT()))
3901 Loads.insert(Load);
3902
3903 continue;
3904 }
3905 return false;
3906 }
3907 case ISD::ZERO_EXTEND:
3908 case ISD::AssertZext: {
3909 unsigned ActiveBits = Mask->getAPIntValue().countTrailingOnes();
3910 EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
3911 EVT VT = Op.getOpcode() == ISD::AssertZext ?
3912 cast<VTSDNode>(Op.getOperand(1))->getVT() :
3913 Op.getOperand(0).getValueType();
3914
3915 // We can accept extending nodes if the mask is wider or an equal
3916 // width to the original type.
3917 if (ExtVT.bitsGE(VT))
3918 continue;
3919 break;
3920 }
3921 case ISD::OR:
3922 case ISD::XOR:
3923 case ISD::AND:
3924 if (!SearchForAndLoads(Op.getNode(), Loads, NodesWithConsts, Mask,
3925 NodeToMask))
3926 return false;
3927 continue;
3928 }
3929
3930 // Allow one node which will masked along with any loads found.
3931 if (NodeToMask)
3932 return false;
3933 NodeToMask = Op.getNode();
3934 }
3935 return true;
3936}
3937
3938bool DAGCombiner::BackwardsPropagateMask(SDNode *N, SelectionDAG &DAG) {
3939 auto *Mask = dyn_cast<ConstantSDNode>(N->getOperand(1));
3940 if (!Mask)
3941 return false;
3942
3943 if (!Mask->getAPIntValue().isMask())
3944 return false;
3945
3946 // No need to do anything if the and directly uses a load.
3947 if (isa<LoadSDNode>(N->getOperand(0)))
3948 return false;
3949
3950 SmallPtrSet<LoadSDNode*, 8> Loads;
3951 SmallPtrSet<SDNode*, 2> NodesWithConsts;
3952 SDNode *FixupNode = nullptr;
3953 if (SearchForAndLoads(N, Loads, NodesWithConsts, Mask, FixupNode)) {
3954 if (Loads.size() == 0)
3955 return false;
3956
3957 DEBUG(dbgs() << "Backwards propagate AND: "; N->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "Backwards propagate AND: "
; N->dump(); } } while (false)
;
3958 SDValue MaskOp = N->getOperand(1);
3959
3960 // If it exists, fixup the single node we allow in the tree that needs
3961 // masking.
3962 if (FixupNode) {
3963 DEBUG(dbgs() << "First, need to fix up: "; FixupNode->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "First, need to fix up: "; FixupNode
->dump(); } } while (false)
;
3964 SDValue And = DAG.getNode(ISD::AND, SDLoc(FixupNode),
3965 FixupNode->getValueType(0),
3966 SDValue(FixupNode, 0), MaskOp);
3967 DAG.ReplaceAllUsesOfValueWith(SDValue(FixupNode, 0), And);
3968 DAG.UpdateNodeOperands(And.getNode(), SDValue(FixupNode, 0),
3969 MaskOp);
3970 }
3971
3972 // Narrow any constants that need it.
3973 for (auto *LogicN : NodesWithConsts) {
3974 SDValue Op0 = LogicN->getOperand(0);
3975 SDValue Op1 = LogicN->getOperand(1);
3976
3977 if (isa<ConstantSDNode>(Op0))
3978 std::swap(Op0, Op1);
3979
3980 SDValue And = DAG.getNode(ISD::AND, SDLoc(Op1), Op1.getValueType(),
3981 Op1, MaskOp);
3982
3983 DAG.UpdateNodeOperands(LogicN, Op0, And);
3984 }
3985
3986 // Create narrow loads.
3987 for (auto *Load : Loads) {
3988 DEBUG(dbgs() << "Propagate AND back to: "; Load->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "Propagate AND back to: "; Load
->dump(); } } while (false)
;
3989 SDValue And = DAG.getNode(ISD::AND, SDLoc(Load), Load->getValueType(0),
3990 SDValue(Load, 0), MaskOp);
3991 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), And);
3992 DAG.UpdateNodeOperands(And.getNode(), SDValue(Load, 0), MaskOp);
3993 SDValue NewLoad = ReduceLoadWidth(And.getNode());
3994 assert(NewLoad &&(static_cast <bool> (NewLoad && "Shouldn't be masking the load if it can't be narrowed"
) ? void (0) : __assert_fail ("NewLoad && \"Shouldn't be masking the load if it can't be narrowed\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3995, __extension__ __PRETTY_FUNCTION__))
3995 "Shouldn't be masking the load if it can't be narrowed")(static_cast <bool> (NewLoad && "Shouldn't be masking the load if it can't be narrowed"
) ? void (0) : __assert_fail ("NewLoad && \"Shouldn't be masking the load if it can't be narrowed\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3995, __extension__ __PRETTY_FUNCTION__))
;
3996 CombineTo(Load, NewLoad, NewLoad.getValue(1));
3997 }
3998 DAG.ReplaceAllUsesWith(N, N->getOperand(0).getNode());
3999 return true;
4000 }
4001 return false;
4002}
4003
4004SDValue DAGCombiner::visitAND(SDNode *N) {
4005 SDValue N0 = N->getOperand(0);
4006 SDValue N1 = N->getOperand(1);
4007 EVT VT = N1.getValueType();
4008
4009 // x & x --> x
4010 if (N0 == N1)
4011 return N0;
4012
4013 // fold vector ops
4014 if (VT.isVector()) {
4015 if (SDValue FoldedVOp = SimplifyVBinOp(N))
4016 return FoldedVOp;
4017
4018 // fold (and x, 0) -> 0, vector edition
4019 if (ISD::isBuildVectorAllZeros(N0.getNode()))
4020 // do not return N0, because undef node may exist in N0
4021 return DAG.getConstant(APInt::getNullValue(N0.getScalarValueSizeInBits()),
4022 SDLoc(N), N0.getValueType());
4023 if (ISD::isBuildVectorAllZeros(N1.getNode()))
4024 // do not return N1, because undef node may exist in N1
4025 return DAG.getConstant(APInt::getNullValue(N1.getScalarValueSizeInBits()),
4026 SDLoc(N), N1.getValueType());
4027
4028 // fold (and x, -1) -> x, vector edition
4029 if (ISD::isBuildVectorAllOnes(N0.getNode()))
4030 return N1;
4031 if (ISD::isBuildVectorAllOnes(N1.getNode()))
4032 return N0;
4033 }
4034
4035 // fold (and c1, c2) -> c1&c2
4036 ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
4037 ConstantSDNode *N1C = isConstOrConstSplat(N1);
4038 if (N0C && N1C && !N1C->isOpaque())
4039 return DAG.FoldConstantArithmetic(ISD::AND, SDLoc(N), VT, N0C, N1C);
4040 // canonicalize constant to RHS
4041 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
4042 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
4043 return DAG.getNode(ISD::AND, SDLoc(N), VT, N1, N0);
4044 // fold (and x, -1) -> x
4045 if (isAllOnesConstant(N1))
4046 return N0;
4047 // if (and x, c) is known to be zero, return 0
4048 unsigned BitWidth = VT.getScalarSizeInBits();
4049 if (N1C && DAG.MaskedValueIsZero(SDValue(N, 0),
4050 APInt::getAllOnesValue(BitWidth)))
4051 return DAG.getConstant(0, SDLoc(N), VT);
4052
4053 if (SDValue NewSel = foldBinOpIntoSelect(N))
4054 return NewSel;
4055
4056 // reassociate and
4057 if (SDValue RAND = ReassociateOps(ISD::AND, SDLoc(N), N0, N1))
4058 return RAND;
4059
4060 // Try to convert a constant mask AND into a shuffle clear mask.
4061 if (VT.isVector())
4062 if (SDValue Shuffle = XformToShuffleWithZero(N))
4063 return Shuffle;
4064
4065 // fold (and (or x, C), D) -> D if (C & D) == D
4066 auto MatchSubset = [](ConstantSDNode *LHS, ConstantSDNode *RHS) {
4067 return RHS->getAPIntValue().isSubsetOf(LHS->getAPIntValue());
4068 };
4069 if (N0.getOpcode() == ISD::OR &&
4070 ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchSubset))
4071 return N1;
4072 // fold (and (any_ext V), c) -> (zero_ext V) if 'and' only clears top bits.
4073 if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) {
4074 SDValue N0Op0 = N0.getOperand(0);
4075 APInt Mask = ~N1C->getAPIntValue();
4076 Mask = Mask.trunc(N0Op0.getScalarValueSizeInBits());
4077 if (DAG.MaskedValueIsZero(N0Op0, Mask)) {
4078 SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N),
4079 N0.getValueType(), N0Op0);
4080
4081 // Replace uses of the AND with uses of the Zero extend node.
4082 CombineTo(N, Zext);
4083
4084 // We actually want to replace all uses of the any_extend with the
4085 // zero_extend, to avoid duplicating things. This will later cause this
4086 // AND to be folded.
4087 CombineTo(N0.getNode(), Zext);
4088 return SDValue(N, 0); // Return N so it doesn't get rechecked!
4089 }
4090 }
4091 // similarly fold (and (X (load ([non_ext|any_ext|zero_ext] V))), c) ->
4092 // (X (load ([non_ext|zero_ext] V))) if 'and' only clears top bits which must
4093 // already be zero by virtue of the width of the base type of the load.
4094 //
4095 // the 'X' node here can either be nothing or an extract_vector_elt to catch
4096 // more cases.
4097 if ((N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
4098 N0.getValueSizeInBits() == N0.getOperand(0).getScalarValueSizeInBits() &&
4099 N0.getOperand(0).getOpcode() == ISD::LOAD &&
4100 N0.getOperand(0).getResNo() == 0) ||
4101 (N0.getOpcode() == ISD::LOAD && N0.getResNo() == 0)) {
4102 LoadSDNode *Load = cast<LoadSDNode>( (N0.getOpcode() == ISD::LOAD) ?
4103 N0 : N0.getOperand(0) );
4104
4105 // Get the constant (if applicable) the zero'th operand is being ANDed with.
4106 // This can be a pure constant or a vector splat, in which case we treat the
4107 // vector as a scalar and use the splat value.
4108 APInt Constant = APInt::getNullValue(1);
4109 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) {
4110 Constant = C->getAPIntValue();
4111 } else if (BuildVectorSDNode *Vector = dyn_cast<BuildVectorSDNode>(N1)) {
4112 APInt SplatValue, SplatUndef;
4113 unsigned SplatBitSize;
4114 bool HasAnyUndefs;
4115 bool IsSplat = Vector->isConstantSplat(SplatValue, SplatUndef,
4116 SplatBitSize, HasAnyUndefs);
4117 if (IsSplat) {
4118 // Undef bits can contribute to a possible optimisation if set, so
4119 // set them.
4120 SplatValue |= SplatUndef;
4121
4122 // The splat value may be something like "0x00FFFFFF", which means 0 for
4123 // the first vector value and FF for the rest, repeating. We need a mask
4124 // that will apply equally to all members of the vector, so AND all the
4125 // lanes of the constant together.
4126 EVT VT = Vector->getValueType(0);
4127 unsigned BitWidth = VT.getScalarSizeInBits();
4128
4129 // If the splat value has been compressed to a bitlength lower
4130 // than the size of the vector lane, we need to re-expand it to
4131 // the lane size.
4132 if (BitWidth > SplatBitSize)
4133 for (SplatValue = SplatValue.zextOrTrunc(BitWidth);
4134 SplatBitSize < BitWidth;
4135 SplatBitSize = SplatBitSize * 2)
4136 SplatValue |= SplatValue.shl(SplatBitSize);
4137
4138 // Make sure that variable 'Constant' is only set if 'SplatBitSize' is a
4139 // multiple of 'BitWidth'. Otherwise, we could propagate a wrong value.
4140 if (SplatBitSize % BitWidth == 0) {
4141 Constant = APInt::getAllOnesValue(BitWidth);
4142 for (unsigned i = 0, n = SplatBitSize/BitWidth; i < n; ++i)
4143 Constant &= SplatValue.lshr(i*BitWidth).zextOrTrunc(BitWidth);
4144 }
4145 }
4146 }
4147
4148 // If we want to change an EXTLOAD to a ZEXTLOAD, ensure a ZEXTLOAD is
4149 // actually legal and isn't going to get expanded, else this is a false
4150 // optimisation.
4151 bool CanZextLoadProfitably = TLI.isLoadExtLegal(ISD::ZEXTLOAD,
4152 Load->getValueType(0),
4153 Load->getMemoryVT());
4154
4155 // Resize the constant to the same size as the original memory access before
4156 // extension. If it is still the AllOnesValue then this AND is completely
4157 // unneeded.
4158 Constant = Constant.zextOrTrunc(Load->getMemoryVT().getScalarSizeInBits());
4159
4160 bool B;
4161 switch (Load->getExtensionType()) {
4162 default: B = false; break;
4163 case ISD::EXTLOAD: B = CanZextLoadProfitably; break;
4164 case ISD::ZEXTLOAD:
4165 case ISD::NON_EXTLOAD: B = true; break;
4166 }
4167
4168 if (B && Constant.isAllOnesValue()) {
4169 // If the load type was an EXTLOAD, convert to ZEXTLOAD in order to
4170 // preserve semantics once we get rid of the AND.
4171 SDValue NewLoad(Load, 0);
4172
4173 // Fold the AND away. NewLoad may get replaced immediately.
4174 CombineTo(N, (N0.getNode() == Load) ? NewLoad : N0);
4175
4176 if (Load->getExtensionType() == ISD::EXTLOAD) {
4177 NewLoad = DAG.getLoad(Load->getAddressingMode(), ISD::ZEXTLOAD,
4178 Load->getValueType(0), SDLoc(Load),
4179 Load->getChain(), Load->getBasePtr(),
4180 Load->getOffset(), Load->getMemoryVT(),
4181 Load->getMemOperand());
4182 // Replace uses of the EXTLOAD with the new ZEXTLOAD.
4183 if (Load->getNumValues() == 3) {
4184 // PRE/POST_INC loads have 3 values.
4185 SDValue To[] = { NewLoad.getValue(0), NewLoad.getValue(1),
4186 NewLoad.getValue(2) };
4187 CombineTo(Load, To, 3, true);
4188 } else {
4189 CombineTo(Load, NewLoad.getValue(0), NewLoad.getValue(1));
4190 }
4191 }
4192
4193 return SDValue(N, 0); // Return N so it doesn't get rechecked!
4194 }
4195 }
4196
4197 // fold (and (load x), 255) -> (zextload x, i8)
4198 // fold (and (extload x, i16), 255) -> (zextload x, i8)
4199 // fold (and (any_ext (extload x, i16)), 255) -> (zextload x, i8)
4200 if (!VT.isVector() && N1C && (N0.getOpcode() == ISD::LOAD ||
4201 (N0.getOpcode() == ISD::ANY_EXTEND &&
4202 N0.getOperand(0).getOpcode() == ISD::LOAD))) {
4203 if (SDValue Res = ReduceLoadWidth(N)) {
4204 LoadSDNode *LN0 = N0->getOpcode() == ISD::ANY_EXTEND
4205 ? cast<LoadSDNode>(N0.getOperand(0)) : cast<LoadSDNode>(N0);
4206
4207 AddToWorklist(N);
4208 CombineTo(LN0, Res, Res.getValue(1));
4209 return SDValue(N, 0);
4210 }
4211 }
4212
4213 if (Level >= AfterLegalizeTypes) {
4214 // Attempt to propagate the AND back up to the leaves which, if they're
4215 // loads, can be combined to narrow loads and the AND node can be removed.
4216 // Perform after legalization so that extend nodes will already be
4217 // combined into the loads.
4218 if (BackwardsPropagateMask(N, DAG)) {
4219 return SDValue(N, 0);
4220 }
4221 }
4222
4223 if (SDValue Combined = visitANDLike(N0, N1, N))
4224 return Combined;
4225
4226 // Simplify: (and (op x...), (op y...)) -> (op (and x, y))
4227 if (N0.getOpcode() == N1.getOpcode())
4228 if (SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N))
4229 return Tmp;
4230
4231 // Masking the negated extension of a boolean is just the zero-extended
4232 // boolean:
4233 // and (sub 0, zext(bool X)), 1 --> zext(bool X)
4234 // and (sub 0, sext(bool X)), 1 --> zext(bool X)
4235 //
4236 // Note: the SimplifyDemandedBits fold below can make an information-losing
4237 // transform, and then we have no way to find this better fold.
4238 if (N1C && N1C->isOne() && N0.getOpcode() == ISD::SUB) {
4239 if (isNullConstantOrNullSplatConstant(N0.getOperand(0))) {
4240 SDValue SubRHS = N0.getOperand(1);
4241 if (SubRHS.getOpcode() == ISD::ZERO_EXTEND &&
4242 SubRHS.getOperand(0).getScalarValueSizeInBits() == 1)
4243 return SubRHS;
4244 if (SubRHS.getOpcode() == ISD::SIGN_EXTEND &&
4245 SubRHS.getOperand(0).getScalarValueSizeInBits() == 1)
4246 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, SubRHS.getOperand(0));
4247 }
4248 }
4249
4250 // fold (and (sign_extend_inreg x, i16 to i32), 1) -> (and x, 1)
4251 // fold (and (sra)) -> (and (srl)) when possible.
4252 if (SimplifyDemandedBits(SDValue(N, 0)))
4253 return SDValue(N, 0);
4254
4255 // fold (zext_inreg (extload x)) -> (zextload x)
4256 if (ISD::isEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode())) {
4257 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
4258 EVT MemVT = LN0->getMemoryVT();
4259 // If we zero all the possible extended bits, then we can turn this into
4260 // a zextload if we are running before legalize or the operation is legal.
4261 unsigned BitWidth = N1.getScalarValueSizeInBits();
4262 if (DAG.MaskedValueIsZero(N1, APInt::getHighBitsSet(BitWidth,
4263 BitWidth - MemVT.getScalarSizeInBits())) &&
4264 ((!LegalOperations && !LN0->isVolatile()) ||
4265 TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT))) {
4266 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT,
4267 LN0->getChain(), LN0->getBasePtr(),
4268 MemVT, LN0->getMemOperand());
4269 AddToWorklist(N);
4270 CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
4271 return SDValue(N, 0); // Return N so it doesn't get rechecked!
4272 }
4273 }
4274 // fold (zext_inreg (sextload x)) -> (zextload x) iff load has one use
4275 if (ISD::isSEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) &&
4276 N0.hasOneUse()) {
4277 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
4278 EVT MemVT = LN0->getMemoryVT();
4279 // If we zero all the possible extended bits, then we can turn this into
4280 // a zextload if we are running before legalize or the operation is legal.
4281 unsigned BitWidth = N1.getScalarValueSizeInBits();
4282 if (DAG.MaskedValueIsZero(N1, APInt::getHighBitsSet(BitWidth,
4283 BitWidth - MemVT.getScalarSizeInBits())) &&
4284 ((!LegalOperations && !LN0->isVolatile()) ||
4285 TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT))) {
4286 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT,
4287 LN0->getChain(), LN0->getBasePtr(),
4288 MemVT, LN0->getMemOperand());
4289 AddToWorklist(N);
4290 CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
4291 return SDValue(N, 0); // Return N so it doesn't get rechecked!
4292 }
4293 }
4294 // fold (and (or (srl N, 8), (shl N, 8)), 0xffff) -> (srl (bswap N), const)
4295 if (N1C && N1C->getAPIntValue() == 0xffff && N0.getOpcode() == ISD::OR) {
4296 if (SDValue BSwap = MatchBSwapHWordLow(N0.getNode(), N0.getOperand(0),
4297 N0.getOperand(1), false))
4298 return BSwap;
4299 }
4300
4301 return SDValue();
4302}
4303
4304/// Match (a >> 8) | (a << 8) as (bswap a) >> 16.
4305SDValue DAGCombiner::MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
4306 bool DemandHighBits) {
4307 if (!LegalOperations)
4308 return SDValue();
4309
4310 EVT VT = N->getValueType(0);
4311 if (VT != MVT::i64 && VT != MVT::i32 && VT != MVT::i16)
4312 return SDValue();
4313 if (!TLI.isOperationLegalOrCustom(ISD::BSWAP, VT))
4314 return SDValue();
4315
4316 // Recognize (and (shl a, 8), 0xff00), (and (srl a, 8), 0xff)
4317 bool LookPassAnd0 = false;
4318 bool LookPassAnd1 = false;
4319 if (N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::SRL)
4320 std::swap(N0, N1);
4321 if (N1.getOpcode() == ISD::AND && N1.getOperand(0).getOpcode() == ISD::SHL)
4322 std::swap(N0, N1);
4323 if (N0.getOpcode() == ISD::AND) {
4324 if (!N0.getNode()->hasOneUse())
4325 return SDValue();
4326 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
4327 // Also handle 0xffff since the LHS is guaranteed to have zeros there.
4328 // This is needed for X86.
4329 if (!N01C || (N01C->getZExtValue() != 0xFF00 &&
4330 N01C->getZExtValue() != 0xFFFF))
4331 return SDValue();
4332 N0 = N0.getOperand(0);
4333 LookPassAnd0 = true;
4334 }
4335
4336 if (N1.getOpcode() == ISD::AND) {
4337 if (!N1.getNode()->hasOneUse())
4338 return SDValue();
4339 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
4340 if (!N11C || N11C->getZExtValue() != 0xFF)
4341 return SDValue();
4342 N1 = N1.getOperand(0);
4343 LookPassAnd1 = true;
4344 }
4345
4346 if (N0.getOpcode() == ISD::SRL && N1.getOpcode() == ISD::SHL)
4347 std::swap(N0, N1);
4348 if (N0.getOpcode() != ISD::SHL || N1.getOpcode() != ISD::SRL)
4349 return SDValue();
4350 if (!N0.getNode()->hasOneUse() || !N1.getNode()->hasOneUse())
4351 return SDValue();
4352
4353 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
4354 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
4355 if (!N01C || !N11C)
4356 return SDValue();
4357 if (N01C->getZExtValue() != 8 || N11C->getZExtValue() != 8)
4358 return SDValue();
4359
4360 // Look for (shl (and a, 0xff), 8), (srl (and a, 0xff00), 8)
4361 SDValue N00 = N0->getOperand(0);
4362 if (!LookPassAnd0 && N00.getOpcode() == ISD::AND) {
4363 if (!N00.getNode()->hasOneUse())
4364 return SDValue();
4365 ConstantSDNode *N001C = dyn_cast<ConstantSDNode>(N00.getOperand(1));
4366 if (!N001C || N001C->getZExtValue() != 0xFF)
4367 return SDValue();
4368 N00 = N00.getOperand(0);
4369 LookPassAnd0 = true;
4370 }
4371
4372 SDValue N10 = N1->getOperand(0);
4373 if (!LookPassAnd1 && N10.getOpcode() == ISD::AND) {
4374 if (!N10.getNode()->hasOneUse())
4375 return SDValue();
4376 ConstantSDNode *N101C = dyn_cast<ConstantSDNode>(N10.getOperand(1));
4377 // Also allow 0xFFFF since the bits will be shifted out. This is needed
4378 // for X86.
4379 if (!N101C || (N101C->getZExtValue() != 0xFF00 &&
4380 N101C->getZExtValue() != 0xFFFF))
4381 return SDValue();
4382 N10 = N10.getOperand(0);
4383 LookPassAnd1 = true;
4384 }
4385
4386 if (N00 != N10)
4387 return SDValue();
4388
4389 // Make sure everything beyond the low halfword gets set to zero since the SRL
4390 // 16 will clear the top bits.
4391 unsigned OpSizeInBits = VT.getSizeInBits();
4392 if (DemandHighBits && OpSizeInBits > 16) {
4393 // If the left-shift isn't masked out then the only way this is a bswap is
4394 // if all bits beyond the low 8 are 0. In that case the entire pattern
4395 // reduces to a left shift anyway: leave it for other parts of the combiner.
4396 if (!LookPassAnd0)
4397 return SDValue();
4398
4399 // However, if the right shift isn't masked out then it might be because
4400 // it's not needed. See if we can spot that too.
4401 if (!LookPassAnd1 &&
4402 !DAG.MaskedValueIsZero(
4403 N10, APInt::getHighBitsSet(OpSizeInBits, OpSizeInBits - 16)))
4404 return SDValue();
4405 }
4406
4407 SDValue Res = DAG.getNode(ISD::BSWAP, SDLoc(N), VT, N00);
4408 if (OpSizeInBits > 16) {
4409 SDLoc DL(N);
4410 Res = DAG.getNode(ISD::SRL, DL, VT, Res,
4411 DAG.getConstant(OpSizeInBits - 16, DL,
4412 getShiftAmountTy(VT)));
4413 }
4414 return Res;
4415}
4416
4417/// Return true if the specified node is an element that makes up a 32-bit
4418/// packed halfword byteswap.
4419/// ((x & 0x000000ff) << 8) |
4420/// ((x & 0x0000ff00) >> 8) |
4421/// ((x & 0x00ff0000) << 8) |
4422/// ((x & 0xff000000) >> 8)
4423static bool isBSwapHWordElement(SDValue N, MutableArrayRef<SDNode *> Parts) {
4424 if (!N.getNode()->hasOneUse())
4425 return false;
4426
4427 unsigned Opc = N.getOpcode();
4428 if (Opc != ISD::AND && Opc != ISD::SHL && Opc != ISD::SRL)
4429 return false;
4430
4431 SDValue N0 = N.getOperand(0);
4432 unsigned Opc0 = N0.getOpcode();
4433 if (Opc0 != ISD::AND && Opc0 != ISD::SHL && Opc0 != ISD::SRL)
4434 return false;
4435
4436 ConstantSDNode *N1C = nullptr;
4437 // SHL or SRL: look upstream for AND mask operand
4438 if (Opc == ISD::AND)
4439 N1C = dyn_cast<ConstantSDNode>(N.getOperand(1));
4440 else if (Opc0 == ISD::AND)
4441 N1C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
4442 if (!N1C)
4443 return false;
4444
4445 unsigned MaskByteOffset;
4446 switch (N1C->getZExtValue()) {
4447 default:
4448 return false;
4449 case 0xFF: MaskByteOffset = 0; break;
4450 case 0xFF00: MaskByteOffset = 1; break;
4451 case 0xFFFF:
4452 // In case demanded bits didn't clear the bits that will be shifted out.
4453 // This is needed for X86.
4454 if (Opc == ISD::SRL || (Opc == ISD::AND && Opc0 == ISD::SHL)) {
4455 MaskByteOffset = 1;
4456 break;
4457 }
4458 return false;
4459 case 0xFF0000: MaskByteOffset = 2; break;
4460 case 0xFF000000: MaskByteOffset = 3; break;
4461 }
4462
4463 // Look for (x & 0xff) << 8 as well as ((x << 8) & 0xff00).
4464 if (Opc == ISD::AND) {
4465 if (MaskByteOffset == 0 || MaskByteOffset == 2) {
4466 // (x >> 8) & 0xff
4467 // (x >> 8) & 0xff0000
4468 if (Opc0 != ISD::SRL)
4469 return false;
4470 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
4471 if (!C || C->getZExtValue() != 8)
4472 return false;
4473 } else {
4474 // (x << 8) & 0xff00
4475 // (x << 8) & 0xff000000
4476 if (Opc0 != ISD::SHL)
4477 return false;
4478 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
4479 if (!C || C->getZExtValue() != 8)
4480 return false;
4481 }
4482 } else if (Opc == ISD::SHL) {
4483 // (x & 0xff) << 8
4484 // (x & 0xff0000) << 8
4485 if (MaskByteOffset != 0 && MaskByteOffset != 2)
4486 return false;
4487 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
4488 if (!C || C->getZExtValue() != 8)
4489 return false;
4490 } else { // Opc == ISD::SRL
4491 // (x & 0xff00) >> 8
4492 // (x & 0xff000000) >> 8
4493 if (MaskByteOffset != 1 && MaskByteOffset != 3)
4494 return false;
4495 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
4496 if (!C || C->getZExtValue() != 8)
4497 return false;
4498 }
4499
4500 if (Parts[MaskByteOffset])
4501 return false;
4502
4503 Parts[MaskByteOffset] = N0.getOperand(0).getNode();
4504 return true;
4505}
4506
4507/// Match a 32-bit packed halfword bswap. That is
4508/// ((x & 0x000000ff) << 8) |
4509/// ((x & 0x0000ff00) >> 8) |
4510/// ((x & 0x00ff0000) << 8) |
4511/// ((x & 0xff000000) >> 8)
4512/// => (rotl (bswap x), 16)
4513SDValue DAGCombiner::MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1) {
4514 if (!LegalOperations)
4515 return SDValue();
4516
4517 EVT VT = N->getValueType(0);
4518 if (VT != MVT::i32)
4519 return SDValue();
4520 if (!TLI.isOperationLegalOrCustom(ISD::BSWAP, VT))
4521 return SDValue();
4522
4523 // Look for either
4524 // (or (or (and), (and)), (or (and), (and)))
4525 // (or (or (or (and), (and)), (and)), (and))
4526 if (N0.getOpcode() != ISD::OR)
4527 return SDValue();
4528 SDValue N00 = N0.getOperand(0);
4529 SDValue N01 = N0.getOperand(1);
4530 SDNode *Parts[4] = {};
4531
4532 if (N1.getOpcode() == ISD::OR &&
4533 N00.getNumOperands() == 2 && N01.getNumOperands() == 2) {
4534 // (or (or (and), (and)), (or (and), (and)))
4535 if (!isBSwapHWordElement(N00, Parts))
4536 return SDValue();
4537
4538 if (!isBSwapHWordElement(N01, Parts))
4539 return SDValue();
4540 SDValue N10 = N1.getOperand(0);
4541 if (!isBSwapHWordElement(N10, Parts))
4542 return SDValue();
4543 SDValue N11 = N1.getOperand(1);
4544 if (!isBSwapHWordElement(N11, Parts))
4545 return SDValue();
4546 } else {
4547 // (or (or (or (and), (and)), (and)), (and))
4548 if (!isBSwapHWordElement(N1, Parts))
4549 return SDValue();
4550 if (!isBSwapHWordElement(N01, Parts))
4551 return SDValue();
4552 if (N00.getOpcode() != ISD::OR)
4553 return SDValue();
4554 SDValue N000 = N00.getOperand(0);
4555 if (!isBSwapHWordElement(N000, Parts))
4556 return SDValue();
4557 SDValue N001 = N00.getOperand(1);
4558 if (!isBSwapHWordElement(N001, Parts))
4559 return SDValue();
4560 }
4561
4562 // Make sure the parts are all coming from the same node.
4563 if (Parts[0] != Parts[1] || Parts[0] != Parts[2] || Parts[0] != Parts[3])
4564 return SDValue();
4565
4566 SDLoc DL(N);
4567 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT,
4568 SDValue(Parts[0], 0));
4569
4570 // Result of the bswap should be rotated by 16. If it's not legal, then
4571 // do (x << 16) | (x >> 16).
4572 SDValue ShAmt = DAG.getConstant(16, DL, getShiftAmountTy(VT));
4573 if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT))
4574 return DAG.getNode(ISD::ROTL, DL, VT, BSwap, ShAmt);
4575 if (TLI.isOperationLegalOrCustom(ISD::ROTR, VT))
4576 return DAG.getNode(ISD::ROTR, DL, VT, BSwap, ShAmt);
4577 return DAG.getNode(ISD::OR, DL, VT,
4578 DAG.getNode(ISD::SHL, DL, VT, BSwap, ShAmt),
4579 DAG.getNode(ISD::SRL, DL, VT, BSwap, ShAmt));
4580}
4581
4582/// This contains all DAGCombine rules which reduce two values combined by
4583/// an Or operation to a single value \see visitANDLike().
4584SDValue DAGCombiner::visitORLike(SDValue N0, SDValue N1, SDNode *N) {
4585 EVT VT = N1.getValueType();
4586 SDLoc DL(N);
4587
4588 // fold (or x, undef) -> -1
4589 if (!LegalOperations && (N0.isUndef() || N1.isUndef()))
4590 return DAG.getAllOnesConstant(DL, VT);
4591
4592 if (SDValue V = foldLogicOfSetCCs(false, N0, N1, DL))
4593 return V;
4594
4595 // (or (and X, C1), (and Y, C2)) -> (and (or X, Y), C3) if possible.
4596 if (N0.getOpcode() == ISD::AND && N1.getOpcode() == ISD::AND &&
4597 // Don't increase # computations.
4598 (N0.getNode()->hasOneUse() || N1.getNode()->hasOneUse())) {
4599 // We can only do this xform if we know that bits from X that are set in C2
4600 // but not in C1 are already zero. Likewise for Y.
4601 if (const ConstantSDNode *N0O1C =
4602 getAsNonOpaqueConstant(N0.getOperand(1))) {
4603 if (const ConstantSDNode *N1O1C =
4604 getAsNonOpaqueConstant(N1.getOperand(1))) {
4605 // We can only do this xform if we know that bits from X that are set in
4606 // C2 but not in C1 are already zero. Likewise for Y.
4607 const APInt &LHSMask = N0O1C->getAPIntValue();
4608 const APInt &RHSMask = N1O1C->getAPIntValue();
4609
4610 if (DAG.MaskedValueIsZero(N0.getOperand(0), RHSMask&~LHSMask) &&
4611 DAG.MaskedValueIsZero(N1.getOperand(0), LHSMask&~RHSMask)) {
4612 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT,
4613 N0.getOperand(0), N1.getOperand(0));
4614 return DAG.getNode(ISD::AND, DL, VT, X,
4615 DAG.getConstant(LHSMask | RHSMask, DL, VT));
4616 }
4617 }
4618 }
4619 }
4620
4621 // (or (and X, M), (and X, N)) -> (and X, (or M, N))
4622 if (N0.getOpcode() == ISD::AND &&
4623 N1.getOpcode() == ISD::AND &&
4624 N0.getOperand(0) == N1.getOperand(0) &&
4625 // Don't increase # computations.
4626 (N0.getNode()->hasOneUse() || N1.getNode()->hasOneUse())) {
4627 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT,
4628 N0.getOperand(1), N1.getOperand(1));
4629 return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), X);
4630 }
4631
4632 return SDValue();
4633}
4634
4635SDValue DAGCombiner::visitOR(SDNode *N) {
4636 SDValue N0 = N->getOperand(0);
4637 SDValue N1 = N->getOperand(1);
4638 EVT VT = N1.getValueType();
4639
4640 // x | x --> x
4641 if (N0 == N1)
4642 return N0;
4643
4644 // fold vector ops
4645 if (VT.isVector()) {
4646 if (SDValue FoldedVOp = SimplifyVBinOp(N))
4647 return FoldedVOp;
4648
4649 // fold (or x, 0) -> x, vector edition
4650 if (ISD::isBuildVectorAllZeros(N0.getNode()))
4651 return N1;
4652 if (ISD::isBuildVectorAllZeros(N1.getNode()))
4653 return N0;
4654
4655 // fold (or x, -1) -> -1, vector edition
4656 if (ISD::isBuildVectorAllOnes(N0.getNode()))
4657 // do not return N0, because undef node may exist in N0
4658 return DAG.getAllOnesConstant(SDLoc(N), N0.getValueType());
4659 if (ISD::isBuildVectorAllOnes(N1.getNode()))
4660 // do not return N1, because undef node may exist in N1
4661 return DAG.getAllOnesConstant(SDLoc(N), N1.getValueType());
4662
4663 // fold (or (shuf A, V_0, MA), (shuf B, V_0, MB)) -> (shuf A, B, Mask)
4664 // Do this only if the resulting shuffle is legal.
4665 if (isa<ShuffleVectorSDNode>(N0) &&
4666 isa<ShuffleVectorSDNode>(N1) &&
4667 // Avoid folding a node with illegal type.
4668 TLI.isTypeLegal(VT)) {
4669 bool ZeroN00 = ISD::isBuildVectorAllZeros(N0.getOperand(0).getNode());
4670 bool ZeroN01 = ISD::isBuildVectorAllZeros(N0.getOperand(1).getNode());
4671 bool ZeroN10 = ISD::isBuildVectorAllZeros(N1.getOperand(0).getNode());
4672 bool ZeroN11 = ISD::isBuildVectorAllZeros(N1.getOperand(1).getNode());
4673 // Ensure both shuffles have a zero input.
4674 if ((ZeroN00 != ZeroN01) && (ZeroN10 != ZeroN11)) {
4675 assert((!ZeroN00 || !ZeroN01) && "Both inputs zero!")(static_cast <bool> ((!ZeroN00 || !ZeroN01) && "Both inputs zero!"
) ? void (0) : __assert_fail ("(!ZeroN00 || !ZeroN01) && \"Both inputs zero!\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4675, __extension__ __PRETTY_FUNCTION__))
;
4676 assert((!ZeroN10 || !ZeroN11) && "Both inputs zero!")(static_cast <bool> ((!ZeroN10 || !ZeroN11) && "Both inputs zero!"
) ? void (0) : __assert_fail ("(!ZeroN10 || !ZeroN11) && \"Both inputs zero!\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4676, __extension__ __PRETTY_FUNCTION__))
;
4677 const ShuffleVectorSDNode *SV0 = cast<ShuffleVectorSDNode>(N0);
4678 const ShuffleVectorSDNode *SV1 = cast<ShuffleVectorSDNode>(N1);
4679 bool CanFold = true;
4680 int NumElts = VT.getVectorNumElements();
4681 SmallVector<int, 4> Mask(NumElts);
4682
4683 for (int i = 0; i != NumElts; ++i) {
4684 int M0 = SV0->getMaskElt(i);
4685 int M1 = SV1->getMaskElt(i);
4686
4687 // Determine if either index is pointing to a zero vector.
4688 bool M0Zero = M0 < 0 || (ZeroN00 == (M0 < NumElts));
4689 bool M1Zero = M1 < 0 || (ZeroN10 == (M1 < NumElts));
4690
4691 // If one element is zero and the otherside is undef, keep undef.
4692 // This also handles the case that both are undef.
4693 if ((M0Zero && M1 < 0) || (M1Zero && M0 < 0)) {
4694 Mask[i] = -1;
4695 continue;
4696 }
4697
4698 // Make sure only one of the elements is zero.
4699 if (M0Zero == M1Zero) {
4700 CanFold = false;
4701 break;
4702 }
4703
4704 assert((M0 >= 0 || M1 >= 0) && "Undef index!")(static_cast <bool> ((M0 >= 0 || M1 >= 0) &&
"Undef index!") ? void (0) : __assert_fail ("(M0 >= 0 || M1 >= 0) && \"Undef index!\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4704, __extension__ __PRETTY_FUNCTION__))
;
4705
4706 // We have a zero and non-zero element. If the non-zero came from
4707 // SV0 make the index a LHS index. If it came from SV1, make it
4708 // a RHS index. We need to mod by NumElts because we don't care
4709 // which operand it came from in the original shuffles.
4710 Mask[i] = M1Zero ? M0 % NumElts : (M1 % NumElts) + NumElts;
4711 }
4712
4713 if (CanFold) {
4714 SDValue NewLHS = ZeroN00 ? N0.getOperand(1) : N0.getOperand(0);
4715 SDValue NewRHS = ZeroN10 ? N1.getOperand(1) : N1.getOperand(0);
4716
4717 bool LegalMask = TLI.isShuffleMaskLegal(Mask, VT);
4718 if (!LegalMask) {
4719 std::swap(NewLHS, NewRHS);
4720 ShuffleVectorSDNode::commuteMask(Mask);
4721 LegalMask = TLI.isShuffleMaskLegal(Mask, VT);
4722 }
4723
4724 if (LegalMask)
4725 return DAG.getVectorShuffle(VT, SDLoc(N), NewLHS, NewRHS, Mask);
4726 }
4727 }
4728 }
4729 }
4730
4731 // fold (or c1, c2) -> c1|c2
4732 ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
4733 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
4734 if (N0C && N1C && !N1C->isOpaque())
4735 return DAG.FoldConstantArithmetic(ISD::OR, SDLoc(N), VT, N0C, N1C);
4736 // canonicalize constant to RHS
4737 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
4738 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
4739 return DAG.getNode(ISD::OR, SDLoc(N), VT, N1, N0);
4740 // fold (or x, 0) -> x
4741 if (isNullConstant(N1))
4742 return N0;
4743 // fold (or x, -1) -> -1
4744 if (isAllOnesConstant(N1))
4745 return N1;
4746
4747 if (SDValue NewSel = foldBinOpIntoSelect(N))
4748 return NewSel;
4749
4750 // fold (or x, c) -> c iff (x & ~c) == 0
4751 if (N1C && DAG.MaskedValueIsZero(N0, ~N1C->getAPIntValue()))
4752 return N1;
4753
4754 if (SDValue Combined = visitORLike(N0, N1, N))
4755 return Combined;
4756
4757 // Recognize halfword bswaps as (bswap + rotl 16) or (bswap + shl 16)
4758 if (SDValue BSwap = MatchBSwapHWord(N, N0, N1))
4759 return BSwap;
4760 if (SDValue BSwap = MatchBSwapHWordLow(N, N0, N1))
4761 return BSwap;
4762
4763 // reassociate or
4764 if (SDValue ROR = ReassociateOps(ISD::OR, SDLoc(N), N0, N1))
4765 return ROR;
4766
4767 // Canonicalize (or (and X, c1), c2) -> (and (or X, c2), c1|c2)
4768 // iff (c1 & c2) != 0.
4769 auto MatchIntersect = [](ConstantSDNode *LHS, ConstantSDNode *RHS) {
4770 return LHS->getAPIntValue().intersects(RHS->getAPIntValue());
4771 };
4772 if (N0.getOpcode() == ISD::AND && N0.getNode()->hasOneUse() &&
4773 ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchIntersect)) {
4774 if (SDValue COR = DAG.FoldConstantArithmetic(
4775 ISD::OR, SDLoc(N1), VT, N1.getNode(), N0.getOperand(1).getNode())) {
4776 SDValue IOR = DAG.getNode(ISD::OR, SDLoc(N0), VT, N0.getOperand(0), N1);
4777 AddToWorklist(IOR.getNode());
4778 return DAG.getNode(ISD::AND, SDLoc(N), VT, COR, IOR);
4779 }
4780 }
4781
4782 // Simplify: (or (op x...), (op y...)) -> (op (or x, y))
4783 if (N0.getOpcode() == N1.getOpcode())
4784 if (SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N))
4785 return Tmp;
4786
4787 // See if this is some rotate idiom.
4788 if (SDNode *Rot = MatchRotate(N0, N1, SDLoc(N)))
4789 return SDValue(Rot, 0);
4790
4791 if (SDValue Load = MatchLoadCombine(N))
4792 return Load;
4793
4794 // Simplify the operands using demanded-bits information.
4795 if (SimplifyDemandedBits(SDValue(N, 0)))
4796 return SDValue(N, 0);
4797
4798 return SDValue();
4799}
4800
4801/// Match "(X shl/srl V1) & V2" where V2 may not be present.
4802bool DAGCombiner::MatchRotateHalf(SDValue Op, SDValue &Shift, SDValue &Mask) {
4803 if (Op.getOpcode() == ISD::AND) {
4804 if (DAG.isConstantIntBuildVectorOrConstantInt(Op.getOperand(1))) {
4805 Mask = Op.getOperand(1);
4806 Op = Op.getOperand(0);
4807 } else {
4808 return false;
4809 }
4810 }
4811
4812 if (Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SHL) {
4813 Shift = Op;
4814 return true;
4815 }
4816
4817 return false;
4818}
4819
4820// Return true if we can prove that, whenever Neg and Pos are both in the
4821// range [0, EltSize), Neg == (Pos == 0 ? 0 : EltSize - Pos). This means that
4822// for two opposing shifts shift1 and shift2 and a value X with OpBits bits:
4823//
4824// (or (shift1 X, Neg), (shift2 X, Pos))
4825//
4826// reduces to a rotate in direction shift2 by Pos or (equivalently) a rotate
4827// in direction shift1 by Neg. The range [0, EltSize) means that we only need
4828// to consider shift amounts with defined behavior.
4829static bool matchRotateSub(SDValue Pos, SDValue Neg, unsigned EltSize) {
4830 // If EltSize is a power of 2 then:
4831 //
4832 // (a) (Pos == 0 ? 0 : EltSize - Pos) == (EltSize - Pos) & (EltSize - 1)
4833 // (b) Neg == Neg & (EltSize - 1) whenever Neg is in [0, EltSize).
4834 //
4835 // So if EltSize is a power of 2 and Neg is (and Neg', EltSize-1), we check
4836 // for the stronger condition:
4837 //
4838 // Neg & (EltSize - 1) == (EltSize - Pos) & (EltSize - 1) [A]
4839 //
4840 // for all Neg and Pos. Since Neg & (EltSize - 1) == Neg' & (EltSize - 1)
4841 // we can just replace Neg with Neg' for the rest of the function.
4842 //
4843 // In other cases we check for the even stronger condition:
4844 //
4845 // Neg == EltSize - Pos [B]
4846 //
4847 // for all Neg and Pos. Note that the (or ...) then invokes undefined
4848 // behavior if Pos == 0 (and consequently Neg == EltSize).
4849 //
4850 // We could actually use [A] whenever EltSize is a power of 2, but the
4851 // only extra cases that it would match are those uninteresting ones
4852 // where Neg and Pos are never in range at the same time. E.g. for
4853 // EltSize == 32, using [A] would allow a Neg of the form (sub 64, Pos)
4854 // as well as (sub 32, Pos), but:
4855 //
4856 // (or (shift1 X, (sub 64, Pos)), (shift2 X, Pos))
4857 //
4858 // always invokes undefined behavior for 32-bit X.
4859 //
4860 // Below, Mask == EltSize - 1 when using [A] and is all-ones otherwise.
4861 unsigned MaskLoBits = 0;
4862 if (Neg.getOpcode() == ISD::AND && isPowerOf2_64(EltSize)) {
4863 if (ConstantSDNode *NegC = isConstOrConstSplat(Neg.getOperand(1))) {
4864 if (NegC->getAPIntValue() == EltSize - 1) {
4865 Neg = Neg.getOperand(0);
4866 MaskLoBits = Log2_64(EltSize);
4867 }
4868 }
4869 }
4870
4871 // Check whether Neg has the form (sub NegC, NegOp1) for some NegC and NegOp1.
4872 if (Neg.getOpcode() != ISD::SUB)
4873 return false;
4874 ConstantSDNode *NegC = isConstOrConstSplat(Neg.getOperand(0));
4875 if (!NegC)
4876 return false;
4877 SDValue NegOp1 = Neg.getOperand(1);
4878
4879 // On the RHS of [A], if Pos is Pos' & (EltSize - 1), just replace Pos with
4880 // Pos'. The truncation is redundant for the purpose of the equality.
4881 if (MaskLoBits && Pos.getOpcode() == ISD::AND)
4882 if (ConstantSDNode *PosC = isConstOrConstSplat(Pos.getOperand(1)))
4883 if (PosC->getAPIntValue() == EltSize - 1)
4884 Pos = Pos.getOperand(0);
4885
4886 // The condition we need is now:
4887 //
4888 // (NegC - NegOp1) & Mask == (EltSize - Pos) & Mask
4889 //
4890 // If NegOp1 == Pos then we need:
4891 //
4892 // EltSize & Mask == NegC & Mask
4893 //
4894 // (because "x & Mask" is a truncation and distributes through subtraction).
4895 APInt Width;
4896 if (Pos == NegOp1)
4897 Width = NegC->getAPIntValue();
4898
4899 // Check for cases where Pos has the form (add NegOp1, PosC) for some PosC.
4900 // Then the condition we want to prove becomes:
4901 //
4902 // (NegC - NegOp1) & Mask == (EltSize - (NegOp1 + PosC)) & Mask
4903 //
4904 // which, again because "x & Mask" is a truncation, becomes:
4905 //
4906 // NegC & Mask == (EltSize - PosC) & Mask
4907 // EltSize & Mask == (NegC + PosC) & Mask
4908 else if (Pos.getOpcode() == ISD::ADD && Pos.getOperand(0) == NegOp1) {
4909 if (ConstantSDNode *PosC = isConstOrConstSplat(Pos.getOperand(1)))
4910 Width = PosC->getAPIntValue() + NegC->getAPIntValue();
4911 else
4912 return false;
4913 } else
4914 return false;
4915
4916 // Now we just need to check that EltSize & Mask == Width & Mask.
4917 if (MaskLoBits)
4918 // EltSize & Mask is 0 since Mask is EltSize - 1.
4919 return Width.getLoBits(MaskLoBits) == 0;
4920 return Width == EltSize;
4921}
4922
4923// A subroutine of MatchRotate used once we have found an OR of two opposite
4924// shifts of Shifted. If Neg == <operand size> - Pos then the OR reduces
4925// to both (PosOpcode Shifted, Pos) and (NegOpcode Shifted, Neg), with the
4926// former being preferred if supported. InnerPos and InnerNeg are Pos and
4927// Neg with outer conversions stripped away.
4928SDNode *DAGCombiner::MatchRotatePosNeg(SDValue Shifted, SDValue Pos,
4929 SDValue Neg, SDValue InnerPos,
4930 SDValue InnerNeg, unsigned PosOpcode,
4931 unsigned NegOpcode, const SDLoc &DL) {
4932 // fold (or (shl x, (*ext y)),
4933 // (srl x, (*ext (sub 32, y)))) ->
4934 // (rotl x, y) or (rotr x, (sub 32, y))
4935 //
4936 // fold (or (shl x, (*ext (sub 32, y))),
4937 // (srl x, (*ext y))) ->
4938 // (rotr x, y) or (rotl x, (sub 32, y))
4939 EVT VT = Shifted.getValueType();
4940 if (matchRotateSub(InnerPos, InnerNeg, VT.getScalarSizeInBits())) {
4941 bool HasPos = TLI.isOperationLegalOrCustom(PosOpcode, VT);
4942 return DAG.getNode(HasPos ? PosOpcode : NegOpcode, DL, VT, Shifted,
4943 HasPos ? Pos : Neg).getNode();
4944 }
4945
4946 return nullptr;
4947}
4948
4949// MatchRotate - Handle an 'or' of two operands. If this is one of the many
4950// idioms for rotate, and if the target supports rotation instructions, generate
4951// a rot[lr].
4952SDNode *DAGCombiner::MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL) {
4953 // Must be a legal type. Expanded 'n promoted things won't work with rotates.
4954 EVT VT = LHS.getValueType();
4955 if (!TLI.isTypeLegal(VT)) return nullptr;
4956
4957 // The target must have at least one rotate flavor.
4958 bool HasROTL = TLI.isOperationLegalOrCustom(ISD::ROTL, VT);
4959 bool HasROTR = TLI.isOperationLegalOrCustom(ISD::ROTR, VT);
4960 if (!HasROTL && !HasROTR) return nullptr;
4961
4962 // Check for truncated rotate.
4963 if (LHS.getOpcode() == ISD::TRUNCATE && RHS.getOpcode() == ISD::TRUNCATE &&
4964 LHS.getOperand(0).getValueType() == RHS.getOperand(0).getValueType()) {
4965 assert(LHS.getValueType() == RHS.getValueType())(static_cast <bool> (LHS.getValueType() == RHS.getValueType
()) ? void (0) : __assert_fail ("LHS.getValueType() == RHS.getValueType()"
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4965, __extension__ __PRETTY_FUNCTION__))
;
4966 if (SDNode *Rot = MatchRotate(LHS.getOperand(0), RHS.getOperand(0), DL)) {
4967 return DAG.getNode(ISD::TRUNCATE, SDLoc(LHS), LHS.getValueType(),
4968 SDValue(Rot, 0)).getNode();
4969 }
4970 }
4971
4972 // Match "(X shl/srl V1) & V2" where V2 may not be present.
4973 SDValue LHSShift; // The shift.
4974 SDValue LHSMask; // AND value if any.
4975 if (!MatchRotateHalf(LHS, LHSShift, LHSMask))
4976 return nullptr; // Not part of a rotate.
4977
4978 SDValue RHSShift; // The shift.
4979 SDValue RHSMask; // AND value if any.
4980 if (!MatchRotateHalf(RHS, RHSShift, RHSMask))
4981 return nullptr; // Not part of a rotate.
4982
4983 if (LHSShift.getOperand(0) != RHSShift.getOperand(0))
4984 return nullptr; // Not shifting the same value.
4985
4986 if (LHSShift.getOpcode() == RHSShift.getOpcode())
4987 return nullptr; // Shifts must disagree.
4988
4989 // Canonicalize shl to left side in a shl/srl pair.
4990 if (RHSShift.getOpcode() == ISD::SHL) {
4991 std::swap(LHS, RHS);
4992 std::swap(LHSShift, RHSShift);
4993 std::swap(LHSMask, RHSMask);
4994 }
4995
4996 unsigned EltSizeInBits = VT.getScalarSizeInBits();
4997 SDValue LHSShiftArg = LHSShift.getOperand(0);
4998 SDValue LHSShiftAmt = LHSShift.getOperand(1);
4999 SDValue RHSShiftArg = RHSShift.getOperand(0);
5000 SDValue RHSShiftAmt = RHSShift.getOperand(1);
5001
5002 // fold (or (shl x, C1), (srl x, C2)) -> (rotl x, C1)
5003 // fold (or (shl x, C1), (srl x, C2)) -> (rotr x, C2)
5004 auto MatchRotateSum = [EltSizeInBits](ConstantSDNode *LHS,
5005 ConstantSDNode *RHS) {
5006 return (LHS->getAPIntValue() + RHS->getAPIntValue()) == EltSizeInBits;
5007 };
5008 if (ISD::matchBinaryPredicate(LHSShiftAmt, RHSShiftAmt, MatchRotateSum)) {
5009 SDValue Rot = DAG.getNode(HasROTL ? ISD::ROTL : ISD::ROTR, DL, VT,
5010 LHSShiftArg, HasROTL ? LHSShiftAmt : RHSShiftAmt);
5011
5012 // If there is an AND of either shifted operand, apply it to the result.
5013 if (LHSMask.getNode() || RHSMask.getNode()) {
5014 SDValue AllOnes = DAG.getAllOnesConstant(DL, VT);
5015 SDValue Mask = AllOnes;
5016
5017 if (LHSMask.getNode()) {
5018 SDValue RHSBits = DAG.getNode(ISD::SRL, DL, VT, AllOnes, RHSShiftAmt);
5019 Mask = DAG.getNode(ISD::AND, DL, VT, Mask,
5020 DAG.getNode(ISD::OR, DL, VT, LHSMask, RHSBits));
5021 }
5022 if (RHSMask.getNode()) {
5023 SDValue LHSBits = DAG.getNode(ISD::SHL, DL, VT, AllOnes, LHSShiftAmt);
5024 Mask = DAG.getNode(ISD::AND, DL, VT, Mask,
5025 DAG.getNode(ISD::OR, DL, VT, RHSMask, LHSBits));
5026 }
5027
5028 Rot = DAG.getNode(ISD::AND, DL, VT, Rot, Mask);
5029 }
5030
5031 return Rot.getNode();
5032 }
5033
5034 // If there is a mask here, and we have a variable shift, we can't be sure
5035 // that we're masking out the right stuff.
5036 if (LHSMask.getNode() || RHSMask.getNode())
5037 return nullptr;
5038
5039 // If the shift amount is sign/zext/any-extended just peel it off.
5040 SDValue LExtOp0 = LHSShiftAmt;
5041 SDValue RExtOp0 = RHSShiftAmt;
5042 if ((LHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND ||
5043 LHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND ||
5044 LHSShiftAmt.getOpcode() == ISD::ANY_EXTEND ||
5045 LHSShiftAmt.getOpcode() == ISD::TRUNCATE) &&
5046 (RHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND ||
5047 RHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND ||
5048 RHSShiftAmt.getOpcode() == ISD::ANY_EXTEND ||
5049 RHSShiftAmt.getOpcode() == ISD::TRUNCATE)) {
5050 LExtOp0 = LHSShiftAmt.getOperand(0);
5051 RExtOp0 = RHSShiftAmt.getOperand(0);
5052 }
5053
5054 SDNode *TryL = MatchRotatePosNeg(LHSShiftArg, LHSShiftAmt, RHSShiftAmt,
5055 LExtOp0, RExtOp0, ISD::ROTL, ISD::ROTR, DL);
5056 if (TryL)
5057 return TryL;
5058
5059 SDNode *TryR = MatchRotatePosNeg(RHSShiftArg, RHSShiftAmt, LHSShiftAmt,
5060 RExtOp0, LExtOp0, ISD::ROTR, ISD::ROTL, DL);
5061 if (TryR)
5062 return TryR;
5063
5064 return nullptr;
5065}
5066
5067namespace {
5068
5069/// Represents known origin of an individual byte in load combine pattern. The
5070/// value of the byte is either constant zero or comes from memory.
5071struct ByteProvider {
5072 // For constant zero providers Load is set to nullptr. For memory providers
5073 // Load represents the node which loads the byte from memory.
5074 // ByteOffset is the offset of the byte in the value produced by the load.
5075 LoadSDNode *Load = nullptr;
5076 unsigned ByteOffset = 0;
5077
5078 ByteProvider() = default;
5079
5080 static ByteProvider getMemory(LoadSDNode *Load, unsigned ByteOffset) {
5081 return ByteProvider(Load, ByteOffset);
5082 }
5083
5084 static ByteProvider getConstantZero() { return ByteProvider(nullptr, 0); }
5085
5086 bool isConstantZero() const { return !Load; }
5087 bool isMemory() const { return Load; }
5088
5089 bool operator==(const ByteProvider &Other) const {
5090 return Other.Load == Load && Other.ByteOffset == ByteOffset;
5091 }
5092
5093private:
5094 ByteProvider(LoadSDNode *Load, unsigned ByteOffset)
5095 : Load(Load), ByteOffset(ByteOffset) {}
5096};
5097
5098} // end anonymous namespace
5099
5100/// Recursively traverses the expression calculating the origin of the requested
5101/// byte of the given value. Returns None if the provider can't be calculated.
5102///
5103/// For all the values except the root of the expression verifies that the value
5104/// has exactly one use and if it's not true return None. This way if the origin
5105/// of the byte is returned it's guaranteed that the values which contribute to
5106/// the byte are not used outside of this expression.
5107///
5108/// Because the parts of the expression are not allowed to have more than one
5109/// use this function iterates over trees, not DAGs. So it never visits the same
5110/// node more than once.
5111static const Optional<ByteProvider>
5112calculateByteProvider(SDValue Op, unsigned Index, unsigned Depth,
5113 bool Root = false) {
5114 // Typical i64 by i8 pattern requires recursion up to 8 calls depth
5115 if (Depth == 10)
5116 return None;
5117
5118 if (!Root && !Op.hasOneUse())
5119 return None;
5120
5121 assert(Op.getValueType().isScalarInteger() && "can't handle other types")(static_cast <bool> (Op.getValueType().isScalarInteger(
) && "can't handle other types") ? void (0) : __assert_fail
("Op.getValueType().isScalarInteger() && \"can't handle other types\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5121, __extension__ __PRETTY_FUNCTION__))
;
5122 unsigned BitWidth = Op.getValueSizeInBits();
5123 if (BitWidth % 8 != 0)
5124 return None;
5125 unsigned ByteWidth = BitWidth / 8;
5126 assert(Index < ByteWidth && "invalid index requested")(static_cast <bool> (Index < ByteWidth && "invalid index requested"
) ? void (0) : __assert_fail ("Index < ByteWidth && \"invalid index requested\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5126, __extension__ __PRETTY_FUNCTION__))
;
5127 (void) ByteWidth;
5128
5129 switch (Op.getOpcode()) {
5130 case ISD::OR: {
5131 auto LHS = calculateByteProvider(Op->getOperand(0), Index, Depth + 1);
5132 if (!LHS)
5133 return None;
5134 auto RHS = calculateByteProvider(Op->getOperand(1), Index, Depth + 1);
5135 if (!RHS)
5136 return None;
5137
5138 if (LHS->isConstantZero())
5139 return RHS;
5140 if (RHS->isConstantZero())
5141 return LHS;
5142 return None;
5143 }
5144 case ISD::SHL: {
5145 auto ShiftOp = dyn_cast<ConstantSDNode>(Op->getOperand(1));
5146 if (!ShiftOp)
5147 return None;
5148
5149 uint64_t BitShift = ShiftOp->getZExtValue();
5150 if (BitShift % 8 != 0)
5151 return None;
5152 uint64_t ByteShift = BitShift / 8;
5153
5154 return Index < ByteShift
5155 ? ByteProvider::getConstantZero()
5156 : calculateByteProvider(Op->getOperand(0), Index - ByteShift,
5157 Depth + 1);
5158 }
5159 case ISD::ANY_EXTEND:
5160 case ISD::SIGN_EXTEND:
5161 case ISD::ZERO_EXTEND: {
5162 SDValue NarrowOp = Op->getOperand(0);
5163 unsigned NarrowBitWidth = NarrowOp.getScalarValueSizeInBits();
5164 if (NarrowBitWidth % 8 != 0)
5165 return None;
5166 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
5167
5168 if (Index >= NarrowByteWidth)
5169 return Op.getOpcode() == ISD::ZERO_EXTEND
5170 ? Optional<ByteProvider>(ByteProvider::getConstantZero())
5171 : None;
5172 return calculateByteProvider(NarrowOp, Index, Depth + 1);
5173 }
5174 case ISD::BSWAP:
5175 return calculateByteProvider(Op->getOperand(0), ByteWidth - Index - 1,
5176 Depth + 1);
5177 case ISD::LOAD: {
5178 auto L = cast<LoadSDNode>(Op.getNode());
5179 if (L->isVolatile() || L->isIndexed())
5180 return None;
5181
5182 unsigned NarrowBitWidth = L->getMemoryVT().getSizeInBits();
5183 if (NarrowBitWidth % 8 != 0)
5184 return None;
5185 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
5186
5187 if (Index >= NarrowByteWidth)
5188 return L->getExtensionType() == ISD::ZEXTLOAD
5189 ? Optional<ByteProvider>(ByteProvider::getConstantZero())
5190 : None;
5191 return ByteProvider::getMemory(L, Index);
5192 }
5193 }
5194
5195 return None;
5196}
5197
5198/// Match a pattern where a wide type scalar value is loaded by several narrow
5199/// loads and combined by shifts and ors. Fold it into a single load or a load
5200/// and a BSWAP if the targets supports it.
5201///
5202/// Assuming little endian target:
5203/// i8 *a = ...
5204/// i32 val = a[0] | (a[1] << 8) | (a[2] << 16) | (a[3] << 24)
5205/// =>
5206/// i32 val = *((i32)a)
5207///
5208/// i8 *a = ...
5209/// i32 val = (a[0] << 24) | (a[1] << 16) | (a[2] << 8) | a[3]
5210/// =>
5211/// i32 val = BSWAP(*((i32)a))
5212///
5213/// TODO: This rule matches complex patterns with OR node roots and doesn't
5214/// interact well with the worklist mechanism. When a part of the pattern is
5215/// updated (e.g. one of the loads) its direct users are put into the worklist,
5216/// but the root node of the pattern which triggers the load combine is not
5217/// necessarily a direct user of the changed node. For example, once the address
5218/// of t28 load is reassociated load combine won't be triggered:
5219/// t25: i32 = add t4, Constant:i32<2>
5220/// t26: i64 = sign_extend t25
5221/// t27: i64 = add t2, t26
5222/// t28: i8,ch = load<LD1[%tmp9]> t0, t27, undef:i64
5223/// t29: i32 = zero_extend t28
5224/// t32: i32 = shl t29, Constant:i8<8>
5225/// t33: i32 = or t23, t32
5226/// As a possible fix visitLoad can check if the load can be a part of a load
5227/// combine pattern and add corresponding OR roots to the worklist.
5228SDValue DAGCombiner::MatchLoadCombine(SDNode *N) {
5229 assert(N->getOpcode() == ISD::OR &&(static_cast <bool> (N->getOpcode() == ISD::OR &&
"Can only match load combining against OR nodes") ? void (0)
: __assert_fail ("N->getOpcode() == ISD::OR && \"Can only match load combining against OR nodes\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5230, __extension__ __PRETTY_FUNCTION__))
5230 "Can only match load combining against OR nodes")(static_cast <bool> (N->getOpcode() == ISD::OR &&
"Can only match load combining against OR nodes") ? void (0)
: __assert_fail ("N->getOpcode() == ISD::OR && \"Can only match load combining against OR nodes\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5230, __extension__ __PRETTY_FUNCTION__))
;
5231
5232 // Handles simple types only
5233 EVT VT = N->getValueType(0);
5234 if (VT != MVT::i16 && VT != MVT::i32 && VT != MVT::i64)
5235 return SDValue();
5236 unsigned ByteWidth = VT.getSizeInBits() / 8;
5237
5238 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5239 // Before legalize we can introduce too wide illegal loads which will be later
5240 // split into legal sized loads. This enables us to combine i64 load by i8
5241 // patterns to a couple of i32 loads on 32 bit targets.
5242 if (LegalOperations && !TLI.isOperationLegal(ISD::LOAD, VT))
5243 return SDValue();
5244
5245 std::function<unsigned(unsigned, unsigned)> LittleEndianByteAt = [](
5246 unsigned BW, unsigned i) { return i; };
5247 std::function<unsigned(unsigned, unsigned)> BigEndianByteAt = [](
5248 unsigned BW, unsigned i) { return BW - i - 1; };
5249
5250 bool IsBigEndianTarget = DAG.getDataLayout().isBigEndian();
5251 auto MemoryByteOffset = [&] (ByteProvider P) {
5252 assert(P.isMemory() && "Must be a memory byte provider")(static_cast <bool> (P.isMemory() && "Must be a memory byte provider"
) ? void (0) : __assert_fail ("P.isMemory() && \"Must be a memory byte provider\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5252, __extension__ __PRETTY_FUNCTION__))
;
5253 unsigned LoadBitWidth = P.Load->getMemoryVT().getSizeInBits();
5254 assert(LoadBitWidth % 8 == 0 &&(static_cast <bool> (LoadBitWidth % 8 == 0 && "can only analyze providers for individual bytes not bit"
) ? void (0) : __assert_fail ("LoadBitWidth % 8 == 0 && \"can only analyze providers for individual bytes not bit\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5255, __extension__ __PRETTY_FUNCTION__))
5255 "can only analyze providers for individual bytes not bit")(static_cast <bool> (LoadBitWidth % 8 == 0 && "can only analyze providers for individual bytes not bit"
) ? void (0) : __assert_fail ("LoadBitWidth % 8 == 0 && \"can only analyze providers for individual bytes not bit\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5255, __extension__ __PRETTY_FUNCTION__))
;
5256 unsigned LoadByteWidth = LoadBitWidth / 8;
5257 return IsBigEndianTarget
5258 ? BigEndianByteAt(LoadByteWidth, P.ByteOffset)
5259 : LittleEndianByteAt(LoadByteWidth, P.ByteOffset);
5260 };
5261
5262 Optional<BaseIndexOffset> Base;
5263 SDValue Chain;
5264
5265 SmallSet<LoadSDNode *, 8> Loads;
5266 Optional<ByteProvider> FirstByteProvider;
5267 int64_t FirstOffset = INT64_MAX(9223372036854775807L);
5268
5269 // Check if all the bytes of the OR we are looking at are loaded from the same
5270 // base address. Collect bytes offsets from Base address in ByteOffsets.
5271 SmallVector<int64_t, 4> ByteOffsets(ByteWidth);
5272 for (unsigned i = 0; i < ByteWidth; i++) {
5273 auto P = calculateByteProvider(SDValue(N, 0), i, 0, /*Root=*/true);
5274 if (!P || !P->isMemory()) // All the bytes must be loaded from memory
5275 return SDValue();
5276
5277 LoadSDNode *L = P->Load;
5278 assert(L->hasNUsesOfValue(1, 0) && !L->isVolatile() && !L->isIndexed() &&(static_cast <bool> (L->hasNUsesOfValue(1, 0) &&
!L->isVolatile() && !L->isIndexed() &&
"Must be enforced by calculateByteProvider") ? void (0) : __assert_fail
("L->hasNUsesOfValue(1, 0) && !L->isVolatile() && !L->isIndexed() && \"Must be enforced by calculateByteProvider\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5279, __extension__ __PRETTY_FUNCTION__))
5279 "Must be enforced by calculateByteProvider")(static_cast <bool> (L->hasNUsesOfValue(1, 0) &&
!L->isVolatile() && !L->isIndexed() &&
"Must be enforced by calculateByteProvider") ? void (0) : __assert_fail
("L->hasNUsesOfValue(1, 0) && !L->isVolatile() && !L->isIndexed() && \"Must be enforced by calculateByteProvider\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5279, __extension__ __PRETTY_FUNCTION__))
;
5280 assert(L->getOffset().isUndef() && "Unindexed load must have undef offset")(static_cast <bool> (L->getOffset().isUndef() &&
"Unindexed load must have undef offset") ? void (0) : __assert_fail
("L->getOffset().isUndef() && \"Unindexed load must have undef offset\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5280, __extension__ __PRETTY_FUNCTION__))
;
5281
5282 // All loads must share the same chain
5283 SDValue LChain = L->getChain();
5284 if (!Chain)
5285 Chain = LChain;
5286 else if (Chain != LChain)
5287 return SDValue();
5288
5289 // Loads must share the same base address
5290 BaseIndexOffset Ptr = BaseIndexOffset::match(L, DAG);
5291 int64_t ByteOffsetFromBase = 0;
5292 if (!Base)
5293 Base = Ptr;
5294 else if (!Base->equalBaseIndex(Ptr, DAG, ByteOffsetFromBase))
5295 return SDValue();
5296
5297 // Calculate the offset of the current byte from the base address
5298 ByteOffsetFromBase += MemoryByteOffset(*P);
5299 ByteOffsets[i] = ByteOffsetFromBase;
5300
5301 // Remember the first byte load
5302 if (ByteOffsetFromBase < FirstOffset) {
5303 FirstByteProvider = P;
5304 FirstOffset = ByteOffsetFromBase;
5305 }
5306
5307 Loads.insert(L);
5308 }
5309 assert(!Loads.empty() && "All the bytes of the value must be loaded from "(static_cast <bool> (!Loads.empty() && "All the bytes of the value must be loaded from "
"memory, so there must be at least one load which produces the value"
) ? void (0) : __assert_fail ("!Loads.empty() && \"All the bytes of the value must be loaded from \" \"memory, so there must be at least one load which produces the value\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5310, __extension__ __PRETTY_FUNCTION__))
5310 "memory, so there must be at least one load which produces the value")(static_cast <bool> (!Loads.empty() && "All the bytes of the value must be loaded from "
"memory, so there must be at least one load which produces the value"
) ? void (0) : __assert_fail ("!Loads.empty() && \"All the bytes of the value must be loaded from \" \"memory, so there must be at least one load which produces the value\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5310, __extension__ __PRETTY_FUNCTION__))
;
5311 assert(Base && "Base address of the accessed memory location must be set")(static_cast <bool> (Base && "Base address of the accessed memory location must be set"
) ? void (0) : __assert_fail ("Base && \"Base address of the accessed memory location must be set\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5311, __extension__ __PRETTY_FUNCTION__))
;
5312 assert(FirstOffset != INT64_MAX && "First byte offset must be set")(static_cast <bool> (FirstOffset != (9223372036854775807L
) && "First byte offset must be set") ? void (0) : __assert_fail
("FirstOffset != INT64_MAX && \"First byte offset must be set\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5312, __extension__ __PRETTY_FUNCTION__))
;
5313
5314 // Check if the bytes of the OR we are looking at match with either big or
5315 // little endian value load
5316 bool BigEndian = true, LittleEndian = true;
5317 for (unsigned i = 0; i < ByteWidth; i++) {
5318 int64_t CurrentByteOffset = ByteOffsets[i] - FirstOffset;
5319 LittleEndian &= CurrentByteOffset == LittleEndianByteAt(ByteWidth, i);
5320 BigEndian &= CurrentByteOffset == BigEndianByteAt(ByteWidth, i);
5321 if (!BigEndian && !LittleEndian)
5322 return SDValue();
5323 }
5324 assert((BigEndian != LittleEndian) && "should be either or")(static_cast <bool> ((BigEndian != LittleEndian) &&
"should be either or") ? void (0) : __assert_fail ("(BigEndian != LittleEndian) && \"should be either or\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5324, __extension__ __PRETTY_FUNCTION__))
;
5325 assert(FirstByteProvider && "must be set")(static_cast <bool> (FirstByteProvider && "must be set"
) ? void (0) : __assert_fail ("FirstByteProvider && \"must be set\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5325, __extension__ __PRETTY_FUNCTION__))
;
5326
5327 // Ensure that the first byte is loaded from zero offset of the first load.
5328 // So the combined value can be loaded from the first load address.
5329 if (MemoryByteOffset(*FirstByteProvider) != 0)
5330 return SDValue();
5331 LoadSDNode *FirstLoad = FirstByteProvider->Load;
5332
5333 // The node we are looking at matches with the pattern, check if we can
5334 // replace it with a single load and bswap if needed.
5335
5336 // If the load needs byte swap check if the target supports it
5337 bool NeedsBswap = IsBigEndianTarget != BigEndian;
5338
5339 // Before legalize we can introduce illegal bswaps which will be later
5340 // converted to an explicit bswap sequence. This way we end up with a single
5341 // load and byte shuffling instead of several loads and byte shuffling.
5342 if (NeedsBswap && LegalOperations && !TLI.isOperationLegal(ISD::BSWAP, VT))
5343 return SDValue();
5344
5345 // Check that a load of the wide type is both allowed and fast on the target
5346 bool Fast = false;
5347 bool Allowed = TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(),
5348 VT, FirstLoad->getAddressSpace(),
5349 FirstLoad->getAlignment(), &Fast);
5350 if (!Allowed || !Fast)
5351 return SDValue();
5352
5353 SDValue NewLoad =
5354 DAG.getLoad(VT, SDLoc(N), Chain, FirstLoad->getBasePtr(),
5355 FirstLoad->getPointerInfo(), FirstLoad->getAlignment());
5356
5357 // Transfer chain users from old loads to the new load.
5358 for (LoadSDNode *L : Loads)
5359 DAG.ReplaceAllUsesOfValueWith(SDValue(L, 1), SDValue(NewLoad.getNode(), 1));
5360
5361 return NeedsBswap ? DAG.getNode(ISD::BSWAP, SDLoc(N), VT, NewLoad) : NewLoad;
5362}
5363
5364SDValue DAGCombiner::visitXOR(SDNode *N) {
5365 SDValue N0 = N->getOperand(0);
5366 SDValue N1 = N->getOperand(1);
5367 EVT VT = N0.getValueType();
5368
5369 // fold vector ops
5370 if (VT.isVector()) {
5371 if (SDValue FoldedVOp = SimplifyVBinOp(N))
5372 return FoldedVOp;
5373
5374 // fold (xor x, 0) -> x, vector edition
5375 if (ISD::isBuildVectorAllZeros(N0.getNode()))
5376 return N1;
5377 if (ISD::isBuildVectorAllZeros(N1.getNode()))
5378 return N0;
5379 }
5380
5381 // fold (xor undef, undef) -> 0. This is a common idiom (misuse).
5382 if (N0.isUndef() && N1.isUndef())
5383 return DAG.getConstant(0, SDLoc(N), VT);
5384 // fold (xor x, undef) -> undef
5385 if (N0.isUndef())
5386 return N0;
5387 if (N1.isUndef())
5388 return N1;
5389 // fold (xor c1, c2) -> c1^c2
5390 ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
5391 ConstantSDNode *N1C = getAsNonOpaqueConstant(N1);
5392 if (N0C && N1C)
5393 return DAG.FoldConstantArithmetic(ISD::XOR, SDLoc(N), VT, N0C, N1C);
5394 // canonicalize constant to RHS
5395 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
5396 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
5397 return DAG.getNode(ISD::XOR, SDLoc(N), VT, N1, N0);
5398 // fold (xor x, 0) -> x
5399 if (isNullConstant(N1))
5400 return N0;
5401
5402 if (SDValue NewSel = foldBinOpIntoSelect(N))
5403 return NewSel;
5404
5405 // reassociate xor
5406 if (SDValue RXOR = ReassociateOps(ISD::XOR, SDLoc(N), N0, N1))
5407 return RXOR;
5408
5409 // fold !(x cc y) -> (x !cc y)
5410 SDValue LHS, RHS, CC;
5411 if (TLI.isConstTrueVal(N1.getNode()) && isSetCCEquivalent(N0, LHS, RHS, CC)) {
5412 bool isInt = LHS.getValueType().isInteger();
5413 ISD::CondCode NotCC = ISD::getSetCCInverse(cast<CondCodeSDNode>(CC)->get(),
5414 isInt);
5415
5416 if (!LegalOperations ||
5417 TLI.isCondCodeLegal(NotCC, LHS.getSimpleValueType())) {
5418 switch (N0.getOpcode()) {
5419 default:
5420 llvm_unreachable("Unhandled SetCC Equivalent!")::llvm::llvm_unreachable_internal("Unhandled SetCC Equivalent!"
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5420)
;
5421 case ISD::SETCC:
5422 return DAG.getSetCC(SDLoc(N0), VT, LHS, RHS, NotCC);
5423 case ISD::SELECT_CC:
5424 return DAG.getSelectCC(SDLoc(N0), LHS, RHS, N0.getOperand(2),
5425 N0.getOperand(3), NotCC);
5426 }
5427 }
5428 }
5429
5430 // fold (not (zext (setcc x, y))) -> (zext (not (setcc x, y)))
5431 if (isOneConstant(N1) && N0.getOpcode() == ISD::ZERO_EXTEND &&
5432 N0.getNode()->hasOneUse() &&
5433 isSetCCEquivalent(N0.getOperand(0), LHS, RHS, CC)){
5434 SDValue V = N0.getOperand(0);
5435 SDLoc DL(N0);
5436 V = DAG.getNode(ISD::XOR, DL, V.getValueType(), V,
5437 DAG.getConstant(1, DL, V.getValueType()));
5438 AddToWorklist(V.getNode());
5439 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, V);
5440 }
5441
5442 // fold (not (or x, y)) -> (and (not x), (not y)) iff x or y are setcc
5443 if (isOneConstant(N1) && VT == MVT::i1 && N0.hasOneUse() &&
5444 (N0.getOpcode() == ISD::OR || N0.getOpcode() == ISD::AND)) {
5445 SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1);
5446 if (isOneUseSetCC(RHS) || isOneUseSetCC(LHS)) {
5447 unsigned NewOpcode = N0.getOpcode() == ISD::AND ? ISD::OR : ISD::AND;
5448 LHS = DAG.getNode(ISD::XOR, SDLoc(LHS), VT, LHS, N1); // LHS = ~LHS
5449 RHS = DAG.getNode(ISD::XOR, SDLoc(RHS), VT, RHS, N1); // RHS = ~RHS
5450 AddToWorklist(LHS.getNode()); AddToWorklist(RHS.getNode());
5451 return DAG.getNode(NewOpcode, SDLoc(N), VT, LHS, RHS);
5452 }
5453 }
5454 // fold (not (or x, y)) -> (and (not x), (not y)) iff x or y are constants
5455 if (isAllOnesConstant(N1) && N0.hasOneUse() &&
5456 (N0.getOpcode() == ISD::OR || N0.getOpcode() == ISD::AND)) {
5457 SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1);
5458 if (isa<ConstantSDNode>(RHS) || isa<ConstantSDNode>(LHS)) {
5459 unsigned NewOpcode = N0.getOpcode() == ISD::AND ? ISD::OR : ISD::AND;
5460 LHS = DAG.getNode(ISD::XOR, SDLoc(LHS), VT, LHS, N1); // LHS = ~LHS
5461 RHS = DAG.getNode(ISD::XOR, SDLoc(RHS), VT, RHS, N1); // RHS = ~RHS
5462 AddToWorklist(LHS.getNode()); AddToWorklist(RHS.getNode());
5463 return DAG.getNode(NewOpcode, SDLoc(N), VT, LHS, RHS);
5464 }
5465 }
5466 // fold (xor (and x, y), y) -> (and (not x), y)
5467 if (N0.getOpcode() == ISD::AND && N0.getNode()->hasOneUse() &&
5468 N0->getOperand(1) == N1) {
5469 SDValue X = N0->getOperand(0);
5470 SDValue NotX = DAG.getNOT(SDLoc(X), X, VT);
5471 AddToWorklist(NotX.getNode());
5472 return DAG.getNode(ISD::AND, SDLoc(N), VT, NotX, N1);
5473 }
5474
5475 // fold Y = sra (X, size(X)-1); xor (add (X, Y), Y) -> (abs X)
5476 unsigned OpSizeInBits = VT.getScalarSizeInBits();
5477 if (N0.getOpcode() == ISD::ADD && N0.getOperand(1) == N1 &&
5478 N1.getOpcode() == ISD::SRA && N1.getOperand(0) == N0.getOperand(0) &&
5479 TLI.isOperationLegalOrCustom(ISD::ABS, VT)) {
5480 if (ConstantSDNode *C = isConstOrConstSplat(N1.getOperand(1)))
5481 if (C->getAPIntValue() == (OpSizeInBits - 1))
5482 return DAG.getNode(ISD::ABS, SDLoc(N), VT, N0.getOperand(0));
5483 }
5484
5485 // fold (xor x, x) -> 0
5486 if (N0 == N1)
5487 return tryFoldToZero(SDLoc(N), TLI, VT, DAG, LegalOperations, LegalTypes);
5488
5489 // fold (xor (shl 1, x), -1) -> (rotl ~1, x)
5490 // Here is a concrete example of this equivalence:
5491 // i16 x == 14
5492 // i16 shl == 1 << 14 == 16384 == 0b0100000000000000
5493 // i16 xor == ~(1 << 14) == 49151 == 0b1011111111111111
5494 //
5495 // =>
5496 //
5497 // i16 ~1 == 0b1111111111111110
5498 // i16 rol(~1, 14) == 0b1011111111111111
5499 //
5500 // Some additional tips to help conceptualize this transform:
5501 // - Try to see the operation as placing a single zero in a value of all ones.
5502 // - There exists no value for x which would allow the result to contain zero.
5503 // - Values of x larger than the bitwidth are undefined and do not require a
5504 // consistent result.
5505 // - Pushing the zero left requires shifting one bits in from the right.
5506 // A rotate left of ~1 is a nice way of achieving the desired result.
5507 if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT) && N0.getOpcode() == ISD::SHL
5508 && isAllOnesConstant(N1) && isOneConstant(N0.getOperand(0))) {
5509 SDLoc DL(N);
5510 return DAG.getNode(ISD::ROTL, DL, VT, DAG.getConstant(~1, DL, VT),
5511 N0.getOperand(1));
5512 }
5513
5514 // Simplify: xor (op x...), (op y...) -> (op (xor x, y))
5515 if (N0.getOpcode() == N1.getOpcode())
5516 if (SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N))
5517 return Tmp;
5518
5519 // Simplify the expression using non-local knowledge.
5520 if (SimplifyDemandedBits(SDValue(N, 0)))
5521 return SDValue(N, 0);
5522
5523 return SDValue();
5524}
5525
5526/// Handle transforms common to the three shifts, when the shift amount is a
5527/// constant.
5528SDValue DAGCombiner::visitShiftByConstant(SDNode *N, ConstantSDNode *Amt) {
5529 SDNode *LHS = N->getOperand(0).getNode();
5530 if (!LHS->hasOneUse()) return SDValue();
5531
5532 // We want to pull some binops through shifts, so that we have (and (shift))
5533 // instead of (shift (and)), likewise for add, or, xor, etc. This sort of
5534 // thing happens with address calculations, so it's important to canonicalize
5535 // it.
5536 bool HighBitSet = false; // Can we transform this if the high bit is set?
5537
5538 switch (LHS->getOpcode()) {
5539 default: return SDValue();
5540 case ISD::OR:
5541 case ISD::XOR:
5542 HighBitSet = false; // We can only transform sra if the high bit is clear.
5543 break;
5544 case ISD::AND:
5545 HighBitSet = true; // We can only transform sra if the high bit is set.
5546 break;
5547 case ISD::ADD:
5548 if (N->getOpcode() != ISD::SHL)
5549 return SDValue(); // only shl(add) not sr[al](add).
5550 HighBitSet = false; // We can only transform sra if the high bit is clear.
5551 break;
5552 }
5553
5554 // We require the RHS of the binop to be a constant and not opaque as well.
5555 ConstantSDNode *BinOpCst = getAsNonOpaqueConstant(LHS->getOperand(1));
5556 if (!BinOpCst) return SDValue();
5557
5558 // FIXME: disable this unless the input to the binop is a shift by a constant
5559 // or is copy/select.Enable this in other cases when figure out it's exactly profitable.
5560 SDNode *BinOpLHSVal = LHS->getOperand(0).getNode();
5561 bool isShift = BinOpLHSVal->getOpcode() == ISD::SHL ||
5562 BinOpLHSVal->getOpcode() == ISD::SRA ||
5563 BinOpLHSVal->getOpcode() == ISD::SRL;
5564 bool isCopyOrSelect = BinOpLHSVal->getOpcode() == ISD::CopyFromReg ||
5565 BinOpLHSVal->getOpcode() == ISD::SELECT;
5566
5567 if ((!isShift || !isa<ConstantSDNode>(BinOpLHSVal->getOperand(1))) &&
5568 !isCopyOrSelect)
5569 return SDValue();
5570
5571 if (isCopyOrSelect && N->hasOneUse())
5572 return SDValue();
5573
5574 EVT VT = N->getValueType(0);
5575
5576 // If this is a signed shift right, and the high bit is modified by the
5577 // logical operation, do not perform the transformation. The highBitSet
5578 // boolean indicates the value of the high bit of the constant which would
5579 // cause it to be modified for this operation.
5580 if (N->getOpcode() == ISD::SRA) {
5581 bool BinOpRHSSignSet = BinOpCst->getAPIntValue().isNegative();
5582 if (BinOpRHSSignSet != HighBitSet)
5583 return SDValue();
5584 }
5585
5586 if (!TLI.isDesirableToCommuteWithShift(LHS))
5587 return SDValue();
5588
5589 // Fold the constants, shifting the binop RHS by the shift amount.
5590 SDValue NewRHS = DAG.getNode(N->getOpcode(), SDLoc(LHS->getOperand(1)),
5591 N->getValueType(0),
5592 LHS->getOperand(1), N->getOperand(1));
5593 assert(isa<ConstantSDNode>(NewRHS) && "Folding was not successful!")(static_cast <bool> (isa<ConstantSDNode>(NewRHS) &&
"Folding was not successful!") ? void (0) : __assert_fail ("isa<ConstantSDNode>(NewRHS) && \"Folding was not successful!\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5593, __extension__ __PRETTY_FUNCTION__))
;
5594
5595 // Create the new shift.
5596 SDValue NewShift = DAG.getNode(N->getOpcode(),
5597 SDLoc(LHS->getOperand(0)),
5598 VT, LHS->getOperand(0), N->getOperand(1));
5599
5600 // Create the new binop.
5601 return DAG.getNode(LHS->getOpcode(), SDLoc(N), VT, NewShift, NewRHS);
5602}
5603
5604SDValue DAGCombiner::distributeTruncateThroughAnd(SDNode *N) {
5605 assert(N->getOpcode() == ISD::TRUNCATE)(static_cast <bool> (N->getOpcode() == ISD::TRUNCATE
) ? void (0) : __assert_fail ("N->getOpcode() == ISD::TRUNCATE"
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5605, __extension__ __PRETTY_FUNCTION__))
;
5606 assert(N->getOperand(0).getOpcode() == ISD::AND)(static_cast <bool> (N->getOperand(0).getOpcode() ==
ISD::AND) ? void (0) : __assert_fail ("N->getOperand(0).getOpcode() == ISD::AND"
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5606, __extension__ __PRETTY_FUNCTION__))
;
5607
5608 // (truncate:TruncVT (and N00, N01C)) -> (and (truncate:TruncVT N00), TruncC)
5609 if (N->hasOneUse() && N->getOperand(0).hasOneUse()) {
5610 SDValue N01 = N->getOperand(0).getOperand(1);
5611 if (isConstantOrConstantVector(N01, /* NoOpaques */ true)) {
5612 SDLoc DL(N);
5613 EVT TruncVT = N->getValueType(0);
5614 SDValue N00 = N->getOperand(0).getOperand(0);
5615 SDValue Trunc00 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N00);
5616 SDValue Trunc01 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N01);
5617 AddToWorklist(Trunc00.getNode());
5618 AddToWorklist(Trunc01.getNode());
5619 return DAG.getNode(ISD::AND, DL, TruncVT, Trunc00, Trunc01);
5620 }
5621 }
5622
5623 return SDValue();
5624}
5625
5626SDValue DAGCombiner::visitRotate(SDNode *N) {
5627 SDLoc dl(N);
5628 SDValue N0 = N->getOperand(0);
5629 SDValue N1 = N->getOperand(1);
5630 EVT VT = N->getValueType(0);
5631 unsigned Bitsize = VT.getScalarSizeInBits();
5632
5633 // fold (rot x, 0) -> x
5634 if (isNullConstantOrNullSplatConstant(N1))
5635 return N0;
5636
5637 // fold (rot x, c) -> (rot x, c % BitSize)
5638 if (ConstantSDNode *Cst = isConstOrConstSplat(N1)) {
5639 if (Cst->getAPIntValue().uge(Bitsize)) {
5640 uint64_t RotAmt = Cst->getAPIntValue().urem(Bitsize);
5641 return DAG.getNode(N->getOpcode(), dl, VT, N0,
5642 DAG.getConstant(RotAmt, dl, N1.getValueType()));
5643 }
5644 }
5645
5646 // fold (rot* x, (trunc (and y, c))) -> (rot* x, (and (trunc y), (trunc c))).
5647 if (N1.getOpcode() == ISD::TRUNCATE &&
5648 N1.getOperand(0).getOpcode() == ISD::AND) {
5649 if (SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode()))
5650 return DAG.getNode(N->getOpcode(), dl, VT, N0, NewOp1);
5651 }
5652
5653 unsigned NextOp = N0.getOpcode();
5654 // fold (rot* (rot* x, c2), c1) -> (rot* x, c1 +- c2 % bitsize)
5655 if (NextOp == ISD::ROTL || NextOp == ISD::ROTR) {
5656 SDNode *C1 = DAG.isConstantIntBuildVectorOrConstantInt(N1);
5657 SDNode *C2 = DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1));
5658 if (C1 && C2 && C1->getValueType(0) == C2->getValueType(0)) {
5659 EVT ShiftVT = C1->getValueType(0);
5660 bool SameSide = (N->getOpcode() == NextOp);
5661 unsigned CombineOp = SameSide ? ISD::ADD : ISD::SUB;
5662 if (SDValue CombinedShift =
5663 DAG.FoldConstantArithmetic(CombineOp, dl, ShiftVT, C1, C2)) {
5664 SDValue BitsizeC = DAG.getConstant(Bitsize, dl, ShiftVT);
5665 SDValue CombinedShiftNorm = DAG.FoldConstantArithmetic(
5666 ISD::SREM, dl, ShiftVT, CombinedShift.getNode(),
5667 BitsizeC.getNode());
5668 return DAG.getNode(N->getOpcode(), dl, VT, N0->getOperand(0),
5669 CombinedShiftNorm);
5670 }
5671 }
5672 }
5673 return SDValue();
5674}
5675
5676SDValue DAGCombiner::visitSHL(SDNode *N) {
5677 SDValue N0 = N->getOperand(0);
5678 SDValue N1 = N->getOperand(1);
5679 EVT VT = N0.getValueType();
5680 unsigned OpSizeInBits = VT.getScalarSizeInBits();
5681
5682 // fold vector ops
5683 if (VT.isVector()) {
5684 if (SDValue FoldedVOp = SimplifyVBinOp(N))
5685 return FoldedVOp;
5686
5687 BuildVectorSDNode *N1CV = dyn_cast<BuildVectorSDNode>(N1);
5688 // If setcc produces all-one true value then:
5689 // (shl (and (setcc) N01CV) N1CV) -> (and (setcc) N01CV<<N1CV)
5690 if (N1CV && N1CV->isConstant()) {
5691 if (N0.getOpcode() == ISD::AND) {
5692 SDValue N00 = N0->getOperand(0);
5693 SDValue N01 = N0->getOperand(1);
5694 BuildVectorSDNode *N01CV = dyn_cast<BuildVectorSDNode>(N01);
5695
5696 if (N01CV && N01CV->isConstant() && N00.getOpcode() == ISD::SETCC &&
5697 TLI.getBooleanContents(N00.getOperand(0).getValueType()) ==
5698 TargetLowering::ZeroOrNegativeOneBooleanContent) {
5699 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT,
5700 N01CV, N1CV))
5701 return DAG.getNode(ISD::AND, SDLoc(N), VT, N00, C);
5702 }
5703 }
5704 }
5705 }
5706
5707 ConstantSDNode *N1C = isConstOrConstSplat(N1);
5708
5709 // fold (shl c1, c2) -> c1<<c2
5710 ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
5711 if (N0C && N1C && !N1C->isOpaque())
5712 return DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT, N0C, N1C);
5713 // fold (shl 0, x) -> 0
5714 if (isNullConstantOrNullSplatConstant(N0))
5715 return N0;
5716 // fold (shl x, c >= size(x)) -> undef
5717 // NOTE: ALL vector elements must be too big to avoid partial UNDEFs.
5718 auto MatchShiftTooBig = [OpSizeInBits](ConstantSDNode *Val) {
5719 return Val->getAPIntValue().uge(OpSizeInBits);
5720 };
5721 if (ISD::matchUnaryPredicate(N1, MatchShiftTooBig))
5722 return DAG.getUNDEF(VT);
5723 // fold (shl x, 0) -> x
5724 if (N1C && N1C->isNullValue())
5725 return N0;
5726 // fold (shl undef, x) -> 0
5727 if (N0.isUndef())
5728 return DAG.getConstant(0, SDLoc(N), VT);
5729
5730 if (SDValue NewSel = foldBinOpIntoSelect(N))
5731 return NewSel;
5732
5733 // if (shl x, c) is known to be zero, return 0
5734 if (DAG.MaskedValueIsZero(SDValue(N, 0),
5735 APInt::getAllOnesValue(OpSizeInBits)))
5736 return DAG.getConstant(0, SDLoc(N), VT);
5737 // fold (shl x, (trunc (and y, c))) -> (shl x, (and (trunc y), (trunc c))).
5738 if (N1.getOpcode() == ISD::TRUNCATE &&
5739 N1.getOperand(0).getOpcode() == ISD::AND) {
5740 if (SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode()))
5741 return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, NewOp1);
5742 }
5743
5744 if (N1C && SimplifyDemandedBits(SDValue(N, 0)))
5745 return SDValue(N, 0);
5746
5747 // fold (shl (shl x, c1), c2) -> 0 or (shl x, (add c1, c2))
5748 if (N0.getOpcode() == ISD::SHL) {
5749 auto MatchOutOfRange = [OpSizeInBits](ConstantSDNode *LHS,
5750 ConstantSDNode *RHS) {
5751 APInt c1 = LHS->getAPIntValue();
5752 APInt c2 = RHS->getAPIntValue();
5753 zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
5754 return (c1 + c2).uge(OpSizeInBits);
5755 };
5756 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchOutOfRange))
5757 return DAG.getConstant(0, SDLoc(N), VT);
5758
5759 auto MatchInRange = [OpSizeInBits](ConstantSDNode *LHS,
5760 ConstantSDNode *RHS) {
5761 APInt c1 = LHS->getAPIntValue();
5762 APInt c2 = RHS->getAPIntValue();
5763 zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
5764 return (c1 + c2).ult(OpSizeInBits);
5765 };
5766 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchInRange)) {
5767 SDLoc DL(N);
5768 EVT ShiftVT = N1.getValueType();
5769 SDValue Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, N1, N0.getOperand(1));
5770 return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Sum);
5771 }
5772 }
5773
5774 // fold (shl (ext (shl x, c1)), c2) -> (ext (shl x, (add c1, c2)))
5775 // For this to be valid, the second form must not preserve any of the bits
5776 // that are shifted out by the inner shift in the first form. This means
5777 // the outer shift size must be >= the number of bits added by the ext.
5778 // As a corollary, we don't care what kind of ext it is.
5779 if (N1C && (N0.getOpcode() == ISD::ZERO_EXTEND ||
5780 N0.getOpcode() == ISD::ANY_EXTEND ||
5781 N0.getOpcode() == ISD::SIGN_EXTEND) &&
5782 N0.getOperand(0).getOpcode() == ISD::SHL) {
5783 SDValue N0Op0 = N0.getOperand(0);
5784 if (ConstantSDNode *N0Op0C1 = isConstOrConstSplat(N0Op0.getOperand(1))) {
5785 APInt c1 = N0Op0C1->getAPIntValue();
5786 APInt c2 = N1C->getAPIntValue();
5787 zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
5788
5789 EVT InnerShiftVT = N0Op0.getValueType();
5790 uint64_t InnerShiftSize = InnerShiftVT.getScalarSizeInBits();
5791 if (c2.uge(OpSizeInBits - InnerShiftSize)) {
5792 SDLoc DL(N0);
5793 APInt Sum = c1 + c2;
5794 if (Sum.uge(OpSizeInBits))
5795 return DAG.getConstant(0, DL, VT);
5796
5797 return DAG.getNode(
5798 ISD::SHL, DL, VT,
5799 DAG.getNode(N0.getOpcode(), DL, VT, N0Op0->getOperand(0)),
5800 DAG.getConstant(Sum.getZExtValue(), DL, N1.getValueType()));
5801 }
5802 }
5803 }
5804
5805 // fold (shl (zext (srl x, C)), C) -> (zext (shl (srl x, C), C))
5806 // Only fold this if the inner zext has no other uses to avoid increasing
5807 // the total number of instructions.
5808 if (N1C && N0.getOpcode() == ISD::ZERO_EXTEND && N0.hasOneUse() &&
5809 N0.getOperand(0).getOpcode() == ISD::SRL) {
5810 SDValue N0Op0 = N0.getOperand(0);
5811 if (ConstantSDNode *N0Op0C1 = isConstOrConstSplat(N0Op0.getOperand(1))) {
5812 if (N0Op0C1->getAPIntValue().ult(VT.getScalarSizeInBits())) {
5813 uint64_t c1 = N0Op0C1->getZExtValue();
5814 uint64_t c2 = N1C->getZExtValue();
5815 if (c1 == c2) {
5816 SDValue NewOp0 = N0.getOperand(0);
5817 EVT CountVT = NewOp0.getOperand(1).getValueType();
5818 SDLoc DL(N);
5819 SDValue NewSHL = DAG.getNode(ISD::SHL, DL, NewOp0.getValueType(),
5820 NewOp0,
5821 DAG.getConstant(c2, DL, CountVT));
5822 AddToWorklist(NewSHL.getNode());
5823 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N0), VT, NewSHL);
5824 }
5825 }
5826 }
5827 }
5828
5829 // fold (shl (sr[la] exact X, C1), C2) -> (shl X, (C2-C1)) if C1 <= C2
5830 // fold (shl (sr[la] exact X, C1), C2) -> (sr[la] X, (C2-C1)) if C1 > C2
5831 if (N1C && (N0.getOpcode() == ISD::SRL || N0.getOpcode() == ISD::SRA) &&
5832 N0->getFlags().hasExact()) {
5833 if (ConstantSDNode *N0C1 = isConstOrConstSplat(N0.getOperand(1))) {
5834 uint64_t C1 = N0C1->getZExtValue();
5835 uint64_t C2 = N1C->getZExtValue();
5836 SDLoc DL(N);
5837 if (C1 <= C2)
5838 return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0),
5839 DAG.getConstant(C2 - C1, DL, N1.getValueType()));
5840 return DAG.getNode(N0.getOpcode(), DL, VT, N0.getOperand(0),
5841 DAG.getConstant(C1 - C2, DL, N1.getValueType()));
5842 }
5843 }
5844
5845 // fold (shl (srl x, c1), c2) -> (and (shl x, (sub c2, c1), MASK) or
5846 // (and (srl x, (sub c1, c2), MASK)
5847 // Only fold this if the inner shift has no other uses -- if it does, folding
5848 // this will increase the total number of instructions.
5849 if (N1C && N0.getOpcode() == ISD::SRL && N0.hasOneUse()) {
5850 if (ConstantSDNode *N0C1 = isConstOrConstSplat(N0.getOperand(1))) {
5851 uint64_t c1 = N0C1->getZExtValue();
5852 if (c1 < OpSizeInBits) {
5853 uint64_t c2 = N1C->getZExtValue();
5854 APInt Mask = APInt::getHighBitsSet(OpSizeInBits, OpSizeInBits - c1);
5855 SDValue Shift;
5856 if (c2 > c1) {
5857 Mask <<= c2 - c1;
5858 SDLoc DL(N);
5859 Shift = DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0),
5860 DAG.getConstant(c2 - c1, DL, N1.getValueType()));
5861 } else {
5862 Mask.lshrInPlace(c1 - c2);
5863 SDLoc DL(N);
5864 Shift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0),
5865 DAG.getConstant(c1 - c2, DL, N1.getValueType()));
5866 }
5867 SDLoc DL(N0);
5868 return DAG.getNode(ISD::AND, DL, VT, Shift,
5869 DAG.getConstant(Mask, DL, VT));
5870 }
5871 }
5872 }
5873
5874 // fold (shl (sra x, c1), c1) -> (and x, (shl -1, c1))
5875 if (N0.getOpcode() == ISD::SRA && N1 == N0.getOperand(1) &&
5876 isConstantOrConstantVector(N1, /* No Opaques */ true)) {
5877 SDLoc DL(N);
5878 SDValue AllBits = DAG.getAllOnesConstant(DL, VT);
5879 SDValue HiBitsMask = DAG.getNode(ISD::SHL, DL, VT, AllBits, N1);
5880 return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), HiBitsMask);
5881 }
5882
5883 // fold (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2)
5884 // fold (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2)
5885 // Variant of version done on multiply, except mul by a power of 2 is turned
5886 // into a shift.
5887 if ((N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::OR) &&
5888 N0.getNode()->hasOneUse() &&
5889 isConstantOrConstantVector(N1, /* No Opaques */ true) &&
5890 isConstantOrConstantVector(N0.getOperand(1), /* No Opaques */ true)) {
5891 SDValue Shl0 = DAG.getNode(ISD::SHL, SDLoc(N0), VT, N0.getOperand(0), N1);
5892 SDValue Shl1 = DAG.getNode(ISD::SHL, SDLoc(N1), VT, N0.getOperand(1), N1);
5893 AddToWorklist(Shl0.getNode());
5894 AddToWorklist(Shl1.getNode());
5895 return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, Shl0, Shl1);
5896 }
5897
5898 // fold (shl (mul x, c1), c2) -> (mul x, c1 << c2)
5899 if (N0.getOpcode() == ISD::MUL && N0.getNode()->hasOneUse() &&
5900 isConstantOrConstantVector(N1, /* No Opaques */ true) &&
5901 isConstantOrConstantVector(N0.getOperand(1), /* No Opaques */ true)) {
5902 SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(N1), VT, N0.getOperand(1), N1);
5903 if (isConstantOrConstantVector(Shl))
5904 return DAG.getNode(ISD::MUL, SDLoc(N), VT, N0.getOperand(0), Shl);
5905 }
5906
5907 if (N1C && !N1C->isOpaque())
5908 if (SDValue NewSHL = visitShiftByConstant(N, N1C))
5909 return NewSHL;
5910
5911 return SDValue();
5912}
5913
5914SDValue DAGCombiner::visitSRA(SDNode *N) {
5915 SDValue N0 = N->getOperand(0);
5916 SDValue N1 = N->getOperand(1);
5917 EVT VT = N0.getValueType();
5918 unsigned OpSizeInBits = VT.getScalarSizeInBits();
5919
5920 // Arithmetic shifting an all-sign-bit value is a no-op.
5921 // fold (sra 0, x) -> 0
5922 // fold (sra -1, x) -> -1
5923 if (DAG.ComputeNumSignBits(N0) == OpSizeInBits)
5924 return N0;
5925
5926 // fold vector ops
5927 if (VT.isVector())
5928 if (SDValue FoldedVOp = SimplifyVBinOp(N))
5929 return FoldedVOp;
5930
5931 ConstantSDNode *N1C = isConstOrConstSplat(N1);
5932
5933 // fold (sra c1, c2) -> (sra c1, c2)
5934 ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
5935 if (N0C && N1C && !N1C->isOpaque())
5936 return DAG.FoldConstantArithmetic(ISD::SRA, SDLoc(N), VT, N0C, N1C);
5937 // fold (sra x, c >=