Bug Summary

File:lib/CodeGen/SelectionDAG/DAGCombiner.cpp
Warning:line 15644, column 1
Potential leak of memory pointed to by 'N0UsedElements.X'

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name DAGCombiner.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-eagerly-assume -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -mrelocation-model pic -pic-level 2 -mthread-model posix -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -momit-leaf-frame-pointer -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-7/lib/clang/7.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-7~svn325118/build-llvm/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-7~svn325118/build-llvm/include -I /build/llvm-toolchain-snapshot-7~svn325118/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/7.3.0/../../../../include/c++/7.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/7.3.0/../../../../include/x86_64-linux-gnu/c++/7.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/7.3.0/../../../../include/x86_64-linux-gnu/c++/7.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/7.3.0/../../../../include/c++/7.3.0/backward -internal-isystem /usr/include/clang/7.0.0/include/ -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-7/lib/clang/7.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++11 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-7~svn325118/build-llvm/lib/CodeGen/SelectionDAG -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-checker optin.performance.Padding -analyzer-output=html -analyzer-config stable-report-filename=true -o /tmp/scan-build-2018-02-14-150435-17243-1 -x c++ /build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

1//===- DAGCombiner.cpp - Implement a DAG node combiner --------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This pass combines dag nodes to form fewer, simpler DAG nodes. It can be run
11// both before and after the DAG is legalized.
12//
13// This pass is not a substitute for the LLVM IR instcombine pass. This pass is
14// primarily intended to handle simplification opportunities that are implicit
15// in the LLVM IR and exposed by the various codegen lowering phases.
16//
17//===----------------------------------------------------------------------===//
18
19#include "llvm/ADT/APFloat.h"
20#include "llvm/ADT/APInt.h"
21#include "llvm/ADT/ArrayRef.h"
22#include "llvm/ADT/DenseMap.h"
23#include "llvm/ADT/None.h"
24#include "llvm/ADT/Optional.h"
25#include "llvm/ADT/STLExtras.h"
26#include "llvm/ADT/SetVector.h"
27#include "llvm/ADT/SmallBitVector.h"
28#include "llvm/ADT/SmallPtrSet.h"
29#include "llvm/ADT/SmallSet.h"
30#include "llvm/ADT/SmallVector.h"
31#include "llvm/ADT/Statistic.h"
32#include "llvm/Analysis/AliasAnalysis.h"
33#include "llvm/Analysis/MemoryLocation.h"
34#include "llvm/CodeGen/DAGCombine.h"
35#include "llvm/CodeGen/ISDOpcodes.h"
36#include "llvm/CodeGen/MachineFrameInfo.h"
37#include "llvm/CodeGen/MachineFunction.h"
38#include "llvm/CodeGen/MachineMemOperand.h"
39#include "llvm/CodeGen/MachineValueType.h"
40#include "llvm/CodeGen/RuntimeLibcalls.h"
41#include "llvm/CodeGen/SelectionDAG.h"
42#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
43#include "llvm/CodeGen/SelectionDAGNodes.h"
44#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
45#include "llvm/CodeGen/TargetLowering.h"
46#include "llvm/CodeGen/TargetRegisterInfo.h"
47#include "llvm/CodeGen/TargetSubtargetInfo.h"
48#include "llvm/CodeGen/ValueTypes.h"
49#include "llvm/IR/Attributes.h"
50#include "llvm/IR/Constant.h"
51#include "llvm/IR/DataLayout.h"
52#include "llvm/IR/DerivedTypes.h"
53#include "llvm/IR/Function.h"
54#include "llvm/IR/LLVMContext.h"
55#include "llvm/IR/Metadata.h"
56#include "llvm/Support/Casting.h"
57#include "llvm/Support/CodeGen.h"
58#include "llvm/Support/CommandLine.h"
59#include "llvm/Support/Compiler.h"
60#include "llvm/Support/Debug.h"
61#include "llvm/Support/ErrorHandling.h"
62#include "llvm/Support/KnownBits.h"
63#include "llvm/Support/MathExtras.h"
64#include "llvm/Support/raw_ostream.h"
65#include "llvm/Target/TargetMachine.h"
66#include "llvm/Target/TargetOptions.h"
67#include <algorithm>
68#include <cassert>
69#include <cstdint>
70#include <functional>
71#include <iterator>
72#include <string>
73#include <tuple>
74#include <utility>
75#include <vector>
76
77using namespace llvm;
78
79#define DEBUG_TYPE"dagcombine" "dagcombine"
80
81STATISTIC(NodesCombined , "Number of dag nodes combined")static llvm::Statistic NodesCombined = {"dagcombine", "NodesCombined"
, "Number of dag nodes combined", {0}, {false}}
;
82STATISTIC(PreIndexedNodes , "Number of pre-indexed nodes created")static llvm::Statistic PreIndexedNodes = {"dagcombine", "PreIndexedNodes"
, "Number of pre-indexed nodes created", {0}, {false}}
;
83STATISTIC(PostIndexedNodes, "Number of post-indexed nodes created")static llvm::Statistic PostIndexedNodes = {"dagcombine", "PostIndexedNodes"
, "Number of post-indexed nodes created", {0}, {false}}
;
84STATISTIC(OpsNarrowed , "Number of load/op/store narrowed")static llvm::Statistic OpsNarrowed = {"dagcombine", "OpsNarrowed"
, "Number of load/op/store narrowed", {0}, {false}}
;
85STATISTIC(LdStFP2Int , "Number of fp load/store pairs transformed to int")static llvm::Statistic LdStFP2Int = {"dagcombine", "LdStFP2Int"
, "Number of fp load/store pairs transformed to int", {0}, {false
}}
;
86STATISTIC(SlicedLoads, "Number of load sliced")static llvm::Statistic SlicedLoads = {"dagcombine", "SlicedLoads"
, "Number of load sliced", {0}, {false}}
;
87
88static cl::opt<bool>
89CombinerGlobalAA("combiner-global-alias-analysis", cl::Hidden,
90 cl::desc("Enable DAG combiner's use of IR alias analysis"));
91
92static cl::opt<bool>
93UseTBAA("combiner-use-tbaa", cl::Hidden, cl::init(true),
94 cl::desc("Enable DAG combiner's use of TBAA"));
95
96#ifndef NDEBUG
97static cl::opt<std::string>
98CombinerAAOnlyFunc("combiner-aa-only-func", cl::Hidden,
99 cl::desc("Only use DAG-combiner alias analysis in this"
100 " function"));
101#endif
102
103/// Hidden option to stress test load slicing, i.e., when this option
104/// is enabled, load slicing bypasses most of its profitability guards.
105static cl::opt<bool>
106StressLoadSlicing("combiner-stress-load-slicing", cl::Hidden,
107 cl::desc("Bypass the profitability model of load slicing"),
108 cl::init(false));
109
110static cl::opt<bool>
111 MaySplitLoadIndex("combiner-split-load-index", cl::Hidden, cl::init(true),
112 cl::desc("DAG combiner may split indexing from loads"));
113
114namespace {
115
116 class DAGCombiner {
117 SelectionDAG &DAG;
118 const TargetLowering &TLI;
119 CombineLevel Level;
120 CodeGenOpt::Level OptLevel;
121 bool LegalOperations = false;
122 bool LegalTypes = false;
123 bool ForCodeSize;
124
125 /// \brief Worklist of all of the nodes that need to be simplified.
126 ///
127 /// This must behave as a stack -- new nodes to process are pushed onto the
128 /// back and when processing we pop off of the back.
129 ///
130 /// The worklist will not contain duplicates but may contain null entries
131 /// due to nodes being deleted from the underlying DAG.
132 SmallVector<SDNode *, 64> Worklist;
133
134 /// \brief Mapping from an SDNode to its position on the worklist.
135 ///
136 /// This is used to find and remove nodes from the worklist (by nulling
137 /// them) when they are deleted from the underlying DAG. It relies on
138 /// stable indices of nodes within the worklist.
139 DenseMap<SDNode *, unsigned> WorklistMap;
140
141 /// \brief Set of nodes which have been combined (at least once).
142 ///
143 /// This is used to allow us to reliably add any operands of a DAG node
144 /// which have not yet been combined to the worklist.
145 SmallPtrSet<SDNode *, 32> CombinedNodes;
146
147 // AA - Used for DAG load/store alias analysis.
148 AliasAnalysis *AA;
149
150 /// When an instruction is simplified, add all users of the instruction to
151 /// the work lists because they might get more simplified now.
152 void AddUsersToWorklist(SDNode *N) {
153 for (SDNode *Node : N->uses())
154 AddToWorklist(Node);
155 }
156
157 /// Call the node-specific routine that folds each particular type of node.
158 SDValue visit(SDNode *N);
159
160 public:
161 DAGCombiner(SelectionDAG &D, AliasAnalysis *AA, CodeGenOpt::Level OL)
162 : DAG(D), TLI(D.getTargetLoweringInfo()), Level(BeforeLegalizeTypes),
163 OptLevel(OL), AA(AA) {
164 ForCodeSize = DAG.getMachineFunction().getFunction().optForSize();
165
166 MaximumLegalStoreInBits = 0;
167 for (MVT VT : MVT::all_valuetypes())
168 if (EVT(VT).isSimple() && VT != MVT::Other &&
169 TLI.isTypeLegal(EVT(VT)) &&
170 VT.getSizeInBits() >= MaximumLegalStoreInBits)
171 MaximumLegalStoreInBits = VT.getSizeInBits();
172 }
173
174 /// Add to the worklist making sure its instance is at the back (next to be
175 /// processed.)
176 void AddToWorklist(SDNode *N) {
177 assert(N->getOpcode() != ISD::DELETED_NODE &&(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& "Deleted Node added to Worklist") ? void (0) : __assert_fail
("N->getOpcode() != ISD::DELETED_NODE && \"Deleted Node added to Worklist\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 178, __extension__ __PRETTY_FUNCTION__))
178 "Deleted Node added to Worklist")(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& "Deleted Node added to Worklist") ? void (0) : __assert_fail
("N->getOpcode() != ISD::DELETED_NODE && \"Deleted Node added to Worklist\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 178, __extension__ __PRETTY_FUNCTION__))
;
179
180 // Skip handle nodes as they can't usefully be combined and confuse the
181 // zero-use deletion strategy.
182 if (N->getOpcode() == ISD::HANDLENODE)
183 return;
184
185 if (WorklistMap.insert(std::make_pair(N, Worklist.size())).second)
186 Worklist.push_back(N);
187 }
188
189 /// Remove all instances of N from the worklist.
190 void removeFromWorklist(SDNode *N) {
191 CombinedNodes.erase(N);
192
193 auto It = WorklistMap.find(N);
194 if (It == WorklistMap.end())
195 return; // Not in the worklist.
196
197 // Null out the entry rather than erasing it to avoid a linear operation.
198 Worklist[It->second] = nullptr;
199 WorklistMap.erase(It);
200 }
201
202 void deleteAndRecombine(SDNode *N);
203 bool recursivelyDeleteUnusedNodes(SDNode *N);
204
205 /// Replaces all uses of the results of one DAG node with new values.
206 SDValue CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
207 bool AddTo = true);
208
209 /// Replaces all uses of the results of one DAG node with new values.
210 SDValue CombineTo(SDNode *N, SDValue Res, bool AddTo = true) {
211 return CombineTo(N, &Res, 1, AddTo);
212 }
213
214 /// Replaces all uses of the results of one DAG node with new values.
215 SDValue CombineTo(SDNode *N, SDValue Res0, SDValue Res1,
216 bool AddTo = true) {
217 SDValue To[] = { Res0, Res1 };
218 return CombineTo(N, To, 2, AddTo);
219 }
220
221 void CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO);
222
223 private:
224 unsigned MaximumLegalStoreInBits;
225
226 /// Check the specified integer node value to see if it can be simplified or
227 /// if things it uses can be simplified by bit propagation.
228 /// If so, return true.
229 bool SimplifyDemandedBits(SDValue Op) {
230 unsigned BitWidth = Op.getScalarValueSizeInBits();
231 APInt Demanded = APInt::getAllOnesValue(BitWidth);
232 return SimplifyDemandedBits(Op, Demanded);
233 }
234
235 bool SimplifyDemandedBits(SDValue Op, const APInt &Demanded);
236
237 bool CombineToPreIndexedLoadStore(SDNode *N);
238 bool CombineToPostIndexedLoadStore(SDNode *N);
239 SDValue SplitIndexingFromLoad(LoadSDNode *LD);
240 bool SliceUpLoad(SDNode *N);
241
242 /// \brief Replace an ISD::EXTRACT_VECTOR_ELT of a load with a narrowed
243 /// load.
244 ///
245 /// \param EVE ISD::EXTRACT_VECTOR_ELT to be replaced.
246 /// \param InVecVT type of the input vector to EVE with bitcasts resolved.
247 /// \param EltNo index of the vector element to load.
248 /// \param OriginalLoad load that EVE came from to be replaced.
249 /// \returns EVE on success SDValue() on failure.
250 SDValue ReplaceExtractVectorEltOfLoadWithNarrowedLoad(
251 SDNode *EVE, EVT InVecVT, SDValue EltNo, LoadSDNode *OriginalLoad);
252 void ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad);
253 SDValue PromoteOperand(SDValue Op, EVT PVT, bool &Replace);
254 SDValue SExtPromoteOperand(SDValue Op, EVT PVT);
255 SDValue ZExtPromoteOperand(SDValue Op, EVT PVT);
256 SDValue PromoteIntBinOp(SDValue Op);
257 SDValue PromoteIntShiftOp(SDValue Op);
258 SDValue PromoteExtend(SDValue Op);
259 bool PromoteLoad(SDValue Op);
260
261 void ExtendSetCCUses(const SmallVectorImpl<SDNode *> &SetCCs,
262 SDValue OrigLoad, SDValue ExtLoad,
263 const SDLoc &DL,
264 ISD::NodeType ExtType);
265
266 /// Call the node-specific routine that knows how to fold each
267 /// particular type of node. If that doesn't do anything, try the
268 /// target-specific DAG combines.
269 SDValue combine(SDNode *N);
270
271 // Visitation implementation - Implement dag node combining for different
272 // node types. The semantics are as follows:
273 // Return Value:
274 // SDValue.getNode() == 0 - No change was made
275 // SDValue.getNode() == N - N was replaced, is dead and has been handled.
276 // otherwise - N should be replaced by the returned Operand.
277 //
278 SDValue visitTokenFactor(SDNode *N);
279 SDValue visitMERGE_VALUES(SDNode *N);
280 SDValue visitADD(SDNode *N);
281 SDValue visitADDLike(SDValue N0, SDValue N1, SDNode *LocReference);
282 SDValue visitSUB(SDNode *N);
283 SDValue visitADDC(SDNode *N);
284 SDValue visitUADDO(SDNode *N);
285 SDValue visitUADDOLike(SDValue N0, SDValue N1, SDNode *N);
286 SDValue visitSUBC(SDNode *N);
287 SDValue visitUSUBO(SDNode *N);
288 SDValue visitADDE(SDNode *N);
289 SDValue visitADDCARRY(SDNode *N);
290 SDValue visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn, SDNode *N);
291 SDValue visitSUBE(SDNode *N);
292 SDValue visitSUBCARRY(SDNode *N);
293 SDValue visitMUL(SDNode *N);
294 SDValue useDivRem(SDNode *N);
295 SDValue visitSDIV(SDNode *N);
296 SDValue visitUDIV(SDNode *N);
297 SDValue visitREM(SDNode *N);
298 SDValue visitMULHU(SDNode *N);
299 SDValue visitMULHS(SDNode *N);
300 SDValue visitSMUL_LOHI(SDNode *N);
301 SDValue visitUMUL_LOHI(SDNode *N);
302 SDValue visitSMULO(SDNode *N);
303 SDValue visitUMULO(SDNode *N);
304 SDValue visitIMINMAX(SDNode *N);
305 SDValue visitAND(SDNode *N);
306 SDValue visitANDLike(SDValue N0, SDValue N1, SDNode *LocReference);
307 SDValue visitOR(SDNode *N);
308 SDValue visitORLike(SDValue N0, SDValue N1, SDNode *LocReference);
309 SDValue visitXOR(SDNode *N);
310 SDValue SimplifyVBinOp(SDNode *N);
311 SDValue visitSHL(SDNode *N);
312 SDValue visitSRA(SDNode *N);
313 SDValue visitSRL(SDNode *N);
314 SDValue visitRotate(SDNode *N);
315 SDValue visitABS(SDNode *N);
316 SDValue visitBSWAP(SDNode *N);
317 SDValue visitBITREVERSE(SDNode *N);
318 SDValue visitCTLZ(SDNode *N);
319 SDValue visitCTLZ_ZERO_UNDEF(SDNode *N);
320 SDValue visitCTTZ(SDNode *N);
321 SDValue visitCTTZ_ZERO_UNDEF(SDNode *N);
322 SDValue visitCTPOP(SDNode *N);
323 SDValue visitSELECT(SDNode *N);
324 SDValue visitVSELECT(SDNode *N);
325 SDValue visitSELECT_CC(SDNode *N);
326 SDValue visitSETCC(SDNode *N);
327 SDValue visitSETCCE(SDNode *N);
328 SDValue visitSETCCCARRY(SDNode *N);
329 SDValue visitSIGN_EXTEND(SDNode *N);
330 SDValue visitZERO_EXTEND(SDNode *N);
331 SDValue visitANY_EXTEND(SDNode *N);
332 SDValue visitAssertExt(SDNode *N);
333 SDValue visitSIGN_EXTEND_INREG(SDNode *N);
334 SDValue visitSIGN_EXTEND_VECTOR_INREG(SDNode *N);
335 SDValue visitZERO_EXTEND_VECTOR_INREG(SDNode *N);
336 SDValue visitTRUNCATE(SDNode *N);
337 SDValue visitBITCAST(SDNode *N);
338 SDValue visitBUILD_PAIR(SDNode *N);
339 SDValue visitFADD(SDNode *N);
340 SDValue visitFSUB(SDNode *N);
341 SDValue visitFMUL(SDNode *N);
342 SDValue visitFMA(SDNode *N);
343 SDValue visitFDIV(SDNode *N);
344 SDValue visitFREM(SDNode *N);
345 SDValue visitFSQRT(SDNode *N);
346 SDValue visitFCOPYSIGN(SDNode *N);
347 SDValue visitSINT_TO_FP(SDNode *N);
348 SDValue visitUINT_TO_FP(SDNode *N);
349 SDValue visitFP_TO_SINT(SDNode *N);
350 SDValue visitFP_TO_UINT(SDNode *N);
351 SDValue visitFP_ROUND(SDNode *N);
352 SDValue visitFP_ROUND_INREG(SDNode *N);
353 SDValue visitFP_EXTEND(SDNode *N);
354 SDValue visitFNEG(SDNode *N);
355 SDValue visitFABS(SDNode *N);
356 SDValue visitFCEIL(SDNode *N);
357 SDValue visitFTRUNC(SDNode *N);
358 SDValue visitFFLOOR(SDNode *N);
359 SDValue visitFMINNUM(SDNode *N);
360 SDValue visitFMAXNUM(SDNode *N);
361 SDValue visitBRCOND(SDNode *N);
362 SDValue visitBR_CC(SDNode *N);
363 SDValue visitLOAD(SDNode *N);
364
365 SDValue replaceStoreChain(StoreSDNode *ST, SDValue BetterChain);
366 SDValue replaceStoreOfFPConstant(StoreSDNode *ST);
367
368 SDValue visitSTORE(SDNode *N);
369 SDValue visitINSERT_VECTOR_ELT(SDNode *N);
370 SDValue visitEXTRACT_VECTOR_ELT(SDNode *N);
371 SDValue visitBUILD_VECTOR(SDNode *N);
372 SDValue visitCONCAT_VECTORS(SDNode *N);
373 SDValue visitEXTRACT_SUBVECTOR(SDNode *N);
374 SDValue visitVECTOR_SHUFFLE(SDNode *N);
375 SDValue visitSCALAR_TO_VECTOR(SDNode *N);
376 SDValue visitINSERT_SUBVECTOR(SDNode *N);
377 SDValue visitMLOAD(SDNode *N);
378 SDValue visitMSTORE(SDNode *N);
379 SDValue visitMGATHER(SDNode *N);
380 SDValue visitMSCATTER(SDNode *N);
381 SDValue visitFP_TO_FP16(SDNode *N);
382 SDValue visitFP16_TO_FP(SDNode *N);
383
384 SDValue visitFADDForFMACombine(SDNode *N);
385 SDValue visitFSUBForFMACombine(SDNode *N);
386 SDValue visitFMULForFMADistributiveCombine(SDNode *N);
387
388 SDValue XformToShuffleWithZero(SDNode *N);
389 SDValue ReassociateOps(unsigned Opc, const SDLoc &DL, SDValue LHS,
390 SDValue RHS);
391
392 SDValue visitShiftByConstant(SDNode *N, ConstantSDNode *Amt);
393
394 SDValue foldSelectOfConstants(SDNode *N);
395 SDValue foldVSelectOfConstants(SDNode *N);
396 SDValue foldBinOpIntoSelect(SDNode *BO);
397 bool SimplifySelectOps(SDNode *SELECT, SDValue LHS, SDValue RHS);
398 SDValue SimplifyBinOpWithSameOpcodeHands(SDNode *N);
399 SDValue SimplifySelect(const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2);
400 SDValue SimplifySelectCC(const SDLoc &DL, SDValue N0, SDValue N1,
401 SDValue N2, SDValue N3, ISD::CondCode CC,
402 bool NotExtCompare = false);
403 SDValue foldSelectCCToShiftAnd(const SDLoc &DL, SDValue N0, SDValue N1,
404 SDValue N2, SDValue N3, ISD::CondCode CC);
405 SDValue foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1,
406 const SDLoc &DL);
407 SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
408 const SDLoc &DL, bool foldBooleans = true);
409
410 bool isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
411 SDValue &CC) const;
412 bool isOneUseSetCC(SDValue N) const;
413
414 SDValue SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
415 unsigned HiOp);
416 SDValue CombineConsecutiveLoads(SDNode *N, EVT VT);
417 SDValue CombineExtLoad(SDNode *N);
418 SDValue combineRepeatedFPDivisors(SDNode *N);
419 SDValue combineInsertEltToShuffle(SDNode *N, unsigned InsIndex);
420 SDValue ConstantFoldBITCASTofBUILD_VECTOR(SDNode *, EVT);
421 SDValue BuildSDIV(SDNode *N);
422 SDValue BuildSDIVPow2(SDNode *N);
423 SDValue BuildUDIV(SDNode *N);
424 SDValue BuildLogBase2(SDValue Op, const SDLoc &DL);
425 SDValue BuildReciprocalEstimate(SDValue Op, SDNodeFlags Flags);
426 SDValue buildRsqrtEstimate(SDValue Op, SDNodeFlags Flags);
427 SDValue buildSqrtEstimate(SDValue Op, SDNodeFlags Flags);
428 SDValue buildSqrtEstimateImpl(SDValue Op, SDNodeFlags Flags, bool Recip);
429 SDValue buildSqrtNROneConst(SDValue Op, SDValue Est, unsigned Iterations,
430 SDNodeFlags Flags, bool Reciprocal);
431 SDValue buildSqrtNRTwoConst(SDValue Op, SDValue Est, unsigned Iterations,
432 SDNodeFlags Flags, bool Reciprocal);
433 SDValue MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
434 bool DemandHighBits = true);
435 SDValue MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1);
436 SDNode *MatchRotatePosNeg(SDValue Shifted, SDValue Pos, SDValue Neg,
437 SDValue InnerPos, SDValue InnerNeg,
438 unsigned PosOpcode, unsigned NegOpcode,
439 const SDLoc &DL);
440 SDNode *MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL);
441 SDValue MatchLoadCombine(SDNode *N);
442 SDValue ReduceLoadWidth(SDNode *N);
443 SDValue ReduceLoadOpStoreWidth(SDNode *N);
444 SDValue splitMergedValStore(StoreSDNode *ST);
445 SDValue TransformFPLoadStorePair(SDNode *N);
446 SDValue reduceBuildVecExtToExtBuildVec(SDNode *N);
447 SDValue reduceBuildVecConvertToConvertBuildVec(SDNode *N);
448 SDValue reduceBuildVecToShuffle(SDNode *N);
449 SDValue createBuildVecShuffle(const SDLoc &DL, SDNode *N,
450 ArrayRef<int> VectorMask, SDValue VecIn1,
451 SDValue VecIn2, unsigned LeftIdx);
452 SDValue matchVSelectOpSizesWithSetCC(SDNode *N);
453
454 /// Walk up chain skipping non-aliasing memory nodes,
455 /// looking for aliasing nodes and adding them to the Aliases vector.
456 void GatherAllAliases(SDNode *N, SDValue OriginalChain,
457 SmallVectorImpl<SDValue> &Aliases);
458
459 /// Return true if there is any possibility that the two addresses overlap.
460 bool isAlias(LSBaseSDNode *Op0, LSBaseSDNode *Op1) const;
461
462 /// Walk up chain skipping non-aliasing memory nodes, looking for a better
463 /// chain (aliasing node.)
464 SDValue FindBetterChain(SDNode *N, SDValue Chain);
465
466 /// Try to replace a store and any possibly adjacent stores on
467 /// consecutive chains with better chains. Return true only if St is
468 /// replaced.
469 ///
470 /// Notice that other chains may still be replaced even if the function
471 /// returns false.
472 bool findBetterNeighborChains(StoreSDNode *St);
473
474 /// Match "(X shl/srl V1) & V2" where V2 may not be present.
475 bool MatchRotateHalf(SDValue Op, SDValue &Shift, SDValue &Mask);
476
477 /// Holds a pointer to an LSBaseSDNode as well as information on where it
478 /// is located in a sequence of memory operations connected by a chain.
479 struct MemOpLink {
480 // Ptr to the mem node.
481 LSBaseSDNode *MemNode;
482
483 // Offset from the base ptr.
484 int64_t OffsetFromBase;
485
486 MemOpLink(LSBaseSDNode *N, int64_t Offset)
487 : MemNode(N), OffsetFromBase(Offset) {}
488 };
489
490 /// This is a helper function for visitMUL to check the profitability
491 /// of folding (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2).
492 /// MulNode is the original multiply, AddNode is (add x, c1),
493 /// and ConstNode is c2.
494 bool isMulAddWithConstProfitable(SDNode *MulNode,
495 SDValue &AddNode,
496 SDValue &ConstNode);
497
498 /// This is a helper function for visitAND and visitZERO_EXTEND. Returns
499 /// true if the (and (load x) c) pattern matches an extload. ExtVT returns
500 /// the type of the loaded value to be extended.
501 bool isAndLoadExtLoad(ConstantSDNode *AndC, LoadSDNode *LoadN,
502 EVT LoadResultTy, EVT &ExtVT);
503
504 /// Helper function to calculate whether the given Load can have its
505 /// width reduced to ExtVT.
506 bool isLegalNarrowLoad(LoadSDNode *LoadN, ISD::LoadExtType ExtType,
507 EVT &ExtVT, unsigned ShAmt = 0);
508
509 /// Used by BackwardsPropagateMask to find suitable loads.
510 bool SearchForAndLoads(SDNode *N, SmallPtrSetImpl<LoadSDNode*> &Loads,
511 SmallPtrSetImpl<SDNode*> &NodeWithConsts,
512 ConstantSDNode *Mask, SDNode *&UncombinedNode);
513 /// Attempt to propagate a given AND node back to load leaves so that they
514 /// can be combined into narrow loads.
515 bool BackwardsPropagateMask(SDNode *N, SelectionDAG &DAG);
516
517 /// Helper function for MergeConsecutiveStores which merges the
518 /// component store chains.
519 SDValue getMergeStoreChains(SmallVectorImpl<MemOpLink> &StoreNodes,
520 unsigned NumStores);
521
522 /// This is a helper function for MergeConsecutiveStores. When the
523 /// source elements of the consecutive stores are all constants or
524 /// all extracted vector elements, try to merge them into one
525 /// larger store introducing bitcasts if necessary. \return True
526 /// if a merged store was created.
527 bool MergeStoresOfConstantsOrVecElts(SmallVectorImpl<MemOpLink> &StoreNodes,
528 EVT MemVT, unsigned NumStores,
529 bool IsConstantSrc, bool UseVector,
530 bool UseTrunc);
531
532 /// This is a helper function for MergeConsecutiveStores. Stores
533 /// that potentially may be merged with St are placed in
534 /// StoreNodes.
535 void getStoreMergeCandidates(StoreSDNode *St,
536 SmallVectorImpl<MemOpLink> &StoreNodes);
537
538 /// Helper function for MergeConsecutiveStores. Checks if
539 /// candidate stores have indirect dependency through their
540 /// operands. \return True if safe to merge.
541 bool checkMergeStoreCandidatesForDependencies(
542 SmallVectorImpl<MemOpLink> &StoreNodes, unsigned NumStores);
543
544 /// Merge consecutive store operations into a wide store.
545 /// This optimization uses wide integers or vectors when possible.
546 /// \return number of stores that were merged into a merged store (the
547 /// affected nodes are stored as a prefix in \p StoreNodes).
548 bool MergeConsecutiveStores(StoreSDNode *N);
549
550 /// \brief Try to transform a truncation where C is a constant:
551 /// (trunc (and X, C)) -> (and (trunc X), (trunc C))
552 ///
553 /// \p N needs to be a truncation and its first operand an AND. Other
554 /// requirements are checked by the function (e.g. that trunc is
555 /// single-use) and if missed an empty SDValue is returned.
556 SDValue distributeTruncateThroughAnd(SDNode *N);
557
558 public:
559 /// Runs the dag combiner on all nodes in the work list
560 void Run(CombineLevel AtLevel);
561
562 SelectionDAG &getDAG() const { return DAG; }
563
564 /// Returns a type large enough to hold any valid shift amount - before type
565 /// legalization these can be huge.
566 EVT getShiftAmountTy(EVT LHSTy) {
567 assert(LHSTy.isInteger() && "Shift amount is not an integer type!")(static_cast <bool> (LHSTy.isInteger() && "Shift amount is not an integer type!"
) ? void (0) : __assert_fail ("LHSTy.isInteger() && \"Shift amount is not an integer type!\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 567, __extension__ __PRETTY_FUNCTION__))
;
568 if (LHSTy.isVector())
569 return LHSTy;
570 auto &DL = DAG.getDataLayout();
571 return LegalTypes ? TLI.getScalarShiftAmountTy(DL, LHSTy)
572 : TLI.getPointerTy(DL);
573 }
574
575 /// This method returns true if we are running before type legalization or
576 /// if the specified VT is legal.
577 bool isTypeLegal(const EVT &VT) {
578 if (!LegalTypes) return true;
579 return TLI.isTypeLegal(VT);
580 }
581
582 /// Convenience wrapper around TargetLowering::getSetCCResultType
583 EVT getSetCCResultType(EVT VT) const {
584 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
585 }
586 };
587
588/// This class is a DAGUpdateListener that removes any deleted
589/// nodes from the worklist.
590class WorklistRemover : public SelectionDAG::DAGUpdateListener {
591 DAGCombiner &DC;
592
593public:
594 explicit WorklistRemover(DAGCombiner &dc)
595 : SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
596
597 void NodeDeleted(SDNode *N, SDNode *E) override {
598 DC.removeFromWorklist(N);
599 }
600};
601
602} // end anonymous namespace
603
604//===----------------------------------------------------------------------===//
605// TargetLowering::DAGCombinerInfo implementation
606//===----------------------------------------------------------------------===//
607
608void TargetLowering::DAGCombinerInfo::AddToWorklist(SDNode *N) {
609 ((DAGCombiner*)DC)->AddToWorklist(N);
610}
611
612SDValue TargetLowering::DAGCombinerInfo::
613CombineTo(SDNode *N, ArrayRef<SDValue> To, bool AddTo) {
614 return ((DAGCombiner*)DC)->CombineTo(N, &To[0], To.size(), AddTo);
615}
616
617SDValue TargetLowering::DAGCombinerInfo::
618CombineTo(SDNode *N, SDValue Res, bool AddTo) {
619 return ((DAGCombiner*)DC)->CombineTo(N, Res, AddTo);
620}
621
622SDValue TargetLowering::DAGCombinerInfo::
623CombineTo(SDNode *N, SDValue Res0, SDValue Res1, bool AddTo) {
624 return ((DAGCombiner*)DC)->CombineTo(N, Res0, Res1, AddTo);
625}
626
627void TargetLowering::DAGCombinerInfo::
628CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
629 return ((DAGCombiner*)DC)->CommitTargetLoweringOpt(TLO);
630}
631
632//===----------------------------------------------------------------------===//
633// Helper Functions
634//===----------------------------------------------------------------------===//
635
636void DAGCombiner::deleteAndRecombine(SDNode *N) {
637 removeFromWorklist(N);
638
639 // If the operands of this node are only used by the node, they will now be
640 // dead. Make sure to re-visit them and recursively delete dead nodes.
641 for (const SDValue &Op : N->ops())
642 // For an operand generating multiple values, one of the values may
643 // become dead allowing further simplification (e.g. split index
644 // arithmetic from an indexed load).
645 if (Op->hasOneUse() || Op->getNumValues() > 1)
646 AddToWorklist(Op.getNode());
647
648 DAG.DeleteNode(N);
649}
650
651/// Return 1 if we can compute the negated form of the specified expression for
652/// the same cost as the expression itself, or 2 if we can compute the negated
653/// form more cheaply than the expression itself.
654static char isNegatibleForFree(SDValue Op, bool LegalOperations,
655 const TargetLowering &TLI,
656 const TargetOptions *Options,
657 unsigned Depth = 0) {
658 // fneg is removable even if it has multiple uses.
659 if (Op.getOpcode() == ISD::FNEG) return 2;
660
661 // Don't allow anything with multiple uses.
662 if (!Op.hasOneUse()) return 0;
663
664 // Don't recurse exponentially.
665 if (Depth > 6) return 0;
666
667 switch (Op.getOpcode()) {
668 default: return false;
669 case ISD::ConstantFP: {
670 if (!LegalOperations)
671 return 1;
672
673 // Don't invert constant FP values after legalization unless the target says
674 // the negated constant is legal.
675 EVT VT = Op.getValueType();
676 return TLI.isOperationLegal(ISD::ConstantFP, VT) ||
677 TLI.isFPImmLegal(neg(cast<ConstantFPSDNode>(Op)->getValueAPF()), VT);
678 }
679 case ISD::FADD:
680 // FIXME: determine better conditions for this xform.
681 if (!Options->UnsafeFPMath) return 0;
682
683 // After operation legalization, it might not be legal to create new FSUBs.
684 if (LegalOperations &&
685 !TLI.isOperationLegalOrCustom(ISD::FSUB, Op.getValueType()))
686 return 0;
687
688 // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
689 if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI,
690 Options, Depth + 1))
691 return V;
692 // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
693 return isNegatibleForFree(Op.getOperand(1), LegalOperations, TLI, Options,
694 Depth + 1);
695 case ISD::FSUB:
696 // We can't turn -(A-B) into B-A when we honor signed zeros.
697 if (!Options->NoSignedZerosFPMath &&
698 !Op.getNode()->getFlags().hasNoSignedZeros())
699 return 0;
700
701 // fold (fneg (fsub A, B)) -> (fsub B, A)
702 return 1;
703
704 case ISD::FMUL:
705 case ISD::FDIV:
706 if (Options->HonorSignDependentRoundingFPMath()) return 0;
707
708 // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y) or (fmul X, (fneg Y))
709 if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI,
710 Options, Depth + 1))
711 return V;
712
713 return isNegatibleForFree(Op.getOperand(1), LegalOperations, TLI, Options,
714 Depth + 1);
715
716 case ISD::FP_EXTEND:
717 case ISD::FP_ROUND:
718 case ISD::FSIN:
719 return isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI, Options,
720 Depth + 1);
721 }
722}
723
724/// If isNegatibleForFree returns true, return the newly negated expression.
725static SDValue GetNegatedExpression(SDValue Op, SelectionDAG &DAG,
726 bool LegalOperations, unsigned Depth = 0) {
727 const TargetOptions &Options = DAG.getTarget().Options;
728 // fneg is removable even if it has multiple uses.
729 if (Op.getOpcode() == ISD::FNEG) return Op.getOperand(0);
730
731 // Don't allow anything with multiple uses.
732 assert(Op.hasOneUse() && "Unknown reuse!")(static_cast <bool> (Op.hasOneUse() && "Unknown reuse!"
) ? void (0) : __assert_fail ("Op.hasOneUse() && \"Unknown reuse!\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 732, __extension__ __PRETTY_FUNCTION__))
;
733
734 assert(Depth <= 6 && "GetNegatedExpression doesn't match isNegatibleForFree")(static_cast <bool> (Depth <= 6 && "GetNegatedExpression doesn't match isNegatibleForFree"
) ? void (0) : __assert_fail ("Depth <= 6 && \"GetNegatedExpression doesn't match isNegatibleForFree\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 734, __extension__ __PRETTY_FUNCTION__))
;
735
736 const SDNodeFlags Flags = Op.getNode()->getFlags();
737
738 switch (Op.getOpcode()) {
739 default: llvm_unreachable("Unknown code")::llvm::llvm_unreachable_internal("Unknown code", "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 739)
;
740 case ISD::ConstantFP: {
741 APFloat V = cast<ConstantFPSDNode>(Op)->getValueAPF();
742 V.changeSign();
743 return DAG.getConstantFP(V, SDLoc(Op), Op.getValueType());
744 }
745 case ISD::FADD:
746 // FIXME: determine better conditions for this xform.
747 assert(Options.UnsafeFPMath)(static_cast <bool> (Options.UnsafeFPMath) ? void (0) :
__assert_fail ("Options.UnsafeFPMath", "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 747, __extension__ __PRETTY_FUNCTION__))
;
748
749 // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
750 if (isNegatibleForFree(Op.getOperand(0), LegalOperations,
751 DAG.getTargetLoweringInfo(), &Options, Depth+1))
752 return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
753 GetNegatedExpression(Op.getOperand(0), DAG,
754 LegalOperations, Depth+1),
755 Op.getOperand(1), Flags);
756 // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
757 return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
758 GetNegatedExpression(Op.getOperand(1), DAG,
759 LegalOperations, Depth+1),
760 Op.getOperand(0), Flags);
761 case ISD::FSUB:
762 // fold (fneg (fsub 0, B)) -> B
763 if (ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(Op.getOperand(0)))
764 if (N0CFP->isZero())
765 return Op.getOperand(1);
766
767 // fold (fneg (fsub A, B)) -> (fsub B, A)
768 return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
769 Op.getOperand(1), Op.getOperand(0), Flags);
770
771 case ISD::FMUL:
772 case ISD::FDIV:
773 assert(!Options.HonorSignDependentRoundingFPMath())(static_cast <bool> (!Options.HonorSignDependentRoundingFPMath
()) ? void (0) : __assert_fail ("!Options.HonorSignDependentRoundingFPMath()"
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 773, __extension__ __PRETTY_FUNCTION__))
;
774
775 // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y)
776 if (isNegatibleForFree(Op.getOperand(0), LegalOperations,
777 DAG.getTargetLoweringInfo(), &Options, Depth+1))
778 return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
779 GetNegatedExpression(Op.getOperand(0), DAG,
780 LegalOperations, Depth+1),
781 Op.getOperand(1), Flags);
782
783 // fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y))
784 return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
785 Op.getOperand(0),
786 GetNegatedExpression(Op.getOperand(1), DAG,
787 LegalOperations, Depth+1), Flags);
788
789 case ISD::FP_EXTEND:
790 case ISD::FSIN:
791 return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
792 GetNegatedExpression(Op.getOperand(0), DAG,
793 LegalOperations, Depth+1));
794 case ISD::FP_ROUND:
795 return DAG.getNode(ISD::FP_ROUND, SDLoc(Op), Op.getValueType(),
796 GetNegatedExpression(Op.getOperand(0), DAG,
797 LegalOperations, Depth+1),
798 Op.getOperand(1));
799 }
800}
801
802// APInts must be the same size for most operations, this helper
803// function zero extends the shorter of the pair so that they match.
804// We provide an Offset so that we can create bitwidths that won't overflow.
805static void zeroExtendToMatch(APInt &LHS, APInt &RHS, unsigned Offset = 0) {
806 unsigned Bits = Offset + std::max(LHS.getBitWidth(), RHS.getBitWidth());
807 LHS = LHS.zextOrSelf(Bits);
808 RHS = RHS.zextOrSelf(Bits);
809}
810
811// Return true if this node is a setcc, or is a select_cc
812// that selects between the target values used for true and false, making it
813// equivalent to a setcc. Also, set the incoming LHS, RHS, and CC references to
814// the appropriate nodes based on the type of node we are checking. This
815// simplifies life a bit for the callers.
816bool DAGCombiner::isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
817 SDValue &CC) const {
818 if (N.getOpcode() == ISD::SETCC) {
819 LHS = N.getOperand(0);
820 RHS = N.getOperand(1);
821 CC = N.getOperand(2);
822 return true;
823 }
824
825 if (N.getOpcode() != ISD::SELECT_CC ||
826 !TLI.isConstTrueVal(N.getOperand(2).getNode()) ||
827 !TLI.isConstFalseVal(N.getOperand(3).getNode()))
828 return false;
829
830 if (TLI.getBooleanContents(N.getValueType()) ==
831 TargetLowering::UndefinedBooleanContent)
832 return false;
833
834 LHS = N.getOperand(0);
835 RHS = N.getOperand(1);
836 CC = N.getOperand(4);
837 return true;
838}
839
840/// Return true if this is a SetCC-equivalent operation with only one use.
841/// If this is true, it allows the users to invert the operation for free when
842/// it is profitable to do so.
843bool DAGCombiner::isOneUseSetCC(SDValue N) const {
844 SDValue N0, N1, N2;
845 if (isSetCCEquivalent(N, N0, N1, N2) && N.getNode()->hasOneUse())
846 return true;
847 return false;
848}
849
850// \brief Returns the SDNode if it is a constant float BuildVector
851// or constant float.
852static SDNode *isConstantFPBuildVectorOrConstantFP(SDValue N) {
853 if (isa<ConstantFPSDNode>(N))
854 return N.getNode();
855 if (ISD::isBuildVectorOfConstantFPSDNodes(N.getNode()))
856 return N.getNode();
857 return nullptr;
858}
859
860// Determines if it is a constant integer or a build vector of constant
861// integers (and undefs).
862// Do not permit build vector implicit truncation.
863static bool isConstantOrConstantVector(SDValue N, bool NoOpaques = false) {
864 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N))
865 return !(Const->isOpaque() && NoOpaques);
866 if (N.getOpcode() != ISD::BUILD_VECTOR)
867 return false;
868 unsigned BitWidth = N.getScalarValueSizeInBits();
869 for (const SDValue &Op : N->op_values()) {
870 if (Op.isUndef())
871 continue;
872 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Op);
873 if (!Const || Const->getAPIntValue().getBitWidth() != BitWidth ||
874 (Const->isOpaque() && NoOpaques))
875 return false;
876 }
877 return true;
878}
879
880// Determines if it is a constant null integer or a splatted vector of a
881// constant null integer (with no undefs).
882// Build vector implicit truncation is not an issue for null values.
883static bool isNullConstantOrNullSplatConstant(SDValue N) {
884 if (ConstantSDNode *Splat = isConstOrConstSplat(N))
885 return Splat->isNullValue();
886 return false;
887}
888
889// Determines if it is a constant integer of one or a splatted vector of a
890// constant integer of one (with no undefs).
891// Do not permit build vector implicit truncation.
892static bool isOneConstantOrOneSplatConstant(SDValue N) {
893 unsigned BitWidth = N.getScalarValueSizeInBits();
894 if (ConstantSDNode *Splat = isConstOrConstSplat(N))
895 return Splat->isOne() && Splat->getAPIntValue().getBitWidth() == BitWidth;
896 return false;
897}
898
899// Determines if it is a constant integer of all ones or a splatted vector of a
900// constant integer of all ones (with no undefs).
901// Do not permit build vector implicit truncation.
902static bool isAllOnesConstantOrAllOnesSplatConstant(SDValue N) {
903 unsigned BitWidth = N.getScalarValueSizeInBits();
904 if (ConstantSDNode *Splat = isConstOrConstSplat(N))
905 return Splat->isAllOnesValue() &&
906 Splat->getAPIntValue().getBitWidth() == BitWidth;
907 return false;
908}
909
910// Determines if a BUILD_VECTOR is composed of all-constants possibly mixed with
911// undef's.
912static bool isAnyConstantBuildVector(const SDNode *N) {
913 return ISD::isBuildVectorOfConstantSDNodes(N) ||
914 ISD::isBuildVectorOfConstantFPSDNodes(N);
915}
916
917// Attempt to match a unary predicate against a scalar/splat constant or
918// every element of a constant BUILD_VECTOR.
919static bool matchUnaryPredicate(SDValue Op,
920 std::function<bool(ConstantSDNode *)> Match) {
921 if (auto *Cst = dyn_cast<ConstantSDNode>(Op))
922 return Match(Cst);
923
924 if (ISD::BUILD_VECTOR != Op.getOpcode())
925 return false;
926
927 EVT SVT = Op.getValueType().getScalarType();
928 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
929 auto *Cst = dyn_cast<ConstantSDNode>(Op.getOperand(i));
930 if (!Cst || Cst->getValueType(0) != SVT || !Match(Cst))
931 return false;
932 }
933 return true;
934}
935
936// Attempt to match a binary predicate against a pair of scalar/splat constants
937// or every element of a pair of constant BUILD_VECTORs.
938static bool matchBinaryPredicate(
939 SDValue LHS, SDValue RHS,
940 std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match) {
941 if (LHS.getValueType() != RHS.getValueType())
942 return false;
943
944 if (auto *LHSCst = dyn_cast<ConstantSDNode>(LHS))
945 if (auto *RHSCst = dyn_cast<ConstantSDNode>(RHS))
946 return Match(LHSCst, RHSCst);
947
948 if (ISD::BUILD_VECTOR != LHS.getOpcode() ||
949 ISD::BUILD_VECTOR != RHS.getOpcode())
950 return false;
951
952 EVT SVT = LHS.getValueType().getScalarType();
953 for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) {
954 auto *LHSCst = dyn_cast<ConstantSDNode>(LHS.getOperand(i));
955 auto *RHSCst = dyn_cast<ConstantSDNode>(RHS.getOperand(i));
956 if (!LHSCst || !RHSCst)
957 return false;
958 if (LHSCst->getValueType(0) != SVT ||
959 LHSCst->getValueType(0) != RHSCst->getValueType(0))
960 return false;
961 if (!Match(LHSCst, RHSCst))
962 return false;
963 }
964 return true;
965}
966
967SDValue DAGCombiner::ReassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0,
968 SDValue N1) {
969 EVT VT = N0.getValueType();
970 if (N0.getOpcode() == Opc) {
971 if (SDNode *L = DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1))) {
972 if (SDNode *R = DAG.isConstantIntBuildVectorOrConstantInt(N1)) {
973 // reassoc. (op (op x, c1), c2) -> (op x, (op c1, c2))
974 if (SDValue OpNode = DAG.FoldConstantArithmetic(Opc, DL, VT, L, R))
975 return DAG.getNode(Opc, DL, VT, N0.getOperand(0), OpNode);
976 return SDValue();
977 }
978 if (N0.hasOneUse()) {
979 // reassoc. (op (op x, c1), y) -> (op (op x, y), c1) iff x+c1 has one
980 // use
981 SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N0.getOperand(0), N1);
982 if (!OpNode.getNode())
983 return SDValue();
984 AddToWorklist(OpNode.getNode());
985 return DAG.getNode(Opc, DL, VT, OpNode, N0.getOperand(1));
986 }
987 }
988 }
989
990 if (N1.getOpcode() == Opc) {
991 if (SDNode *R = DAG.isConstantIntBuildVectorOrConstantInt(N1.getOperand(1))) {
992 if (SDNode *L = DAG.isConstantIntBuildVectorOrConstantInt(N0)) {
993 // reassoc. (op c2, (op x, c1)) -> (op x, (op c1, c2))
994 if (SDValue OpNode = DAG.FoldConstantArithmetic(Opc, DL, VT, R, L))
995 return DAG.getNode(Opc, DL, VT, N1.getOperand(0), OpNode);
996 return SDValue();
997 }
998 if (N1.hasOneUse()) {
999 // reassoc. (op x, (op y, c1)) -> (op (op x, y), c1) iff x+c1 has one
1000 // use
1001 SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N0, N1.getOperand(0));
1002 if (!OpNode.getNode())
1003 return SDValue();
1004 AddToWorklist(OpNode.getNode());
1005 return DAG.getNode(Opc, DL, VT, OpNode, N1.getOperand(1));
1006 }
1007 }
1008 }
1009
1010 return SDValue();
1011}
1012
1013SDValue DAGCombiner::CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
1014 bool AddTo) {
1015 assert(N->getNumValues() == NumTo && "Broken CombineTo call!")(static_cast <bool> (N->getNumValues() == NumTo &&
"Broken CombineTo call!") ? void (0) : __assert_fail ("N->getNumValues() == NumTo && \"Broken CombineTo call!\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1015, __extension__ __PRETTY_FUNCTION__))
;
1016 ++NodesCombined;
1017 DEBUG(dbgs() << "\nReplacing.1 ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.1 "; N->dump
(&DAG); dbgs() << "\nWith: "; To[0].getNode()->dump
(&DAG); dbgs() << " and " << NumTo-1 <<
" other values\n"; } } while (false)
1018 N->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.1 "; N->dump
(&DAG); dbgs() << "\nWith: "; To[0].getNode()->dump
(&DAG); dbgs() << " and " << NumTo-1 <<
" other values\n"; } } while (false)
1019 dbgs() << "\nWith: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.1 "; N->dump
(&DAG); dbgs() << "\nWith: "; To[0].getNode()->dump
(&DAG); dbgs() << " and " << NumTo-1 <<
" other values\n"; } } while (false)
1020 To[0].getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.1 "; N->dump
(&DAG); dbgs() << "\nWith: "; To[0].getNode()->dump
(&DAG); dbgs() << " and " << NumTo-1 <<
" other values\n"; } } while (false)
1021 dbgs() << " and " << NumTo-1 << " other values\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.1 "; N->dump
(&DAG); dbgs() << "\nWith: "; To[0].getNode()->dump
(&DAG); dbgs() << " and " << NumTo-1 <<
" other values\n"; } } while (false)
;
1022 for (unsigned i = 0, e = NumTo; i != e; ++i)
1023 assert((!To[i].getNode() ||(static_cast <bool> ((!To[i].getNode() || N->getValueType
(i) == To[i].getValueType()) && "Cannot combine value to value of different type!"
) ? void (0) : __assert_fail ("(!To[i].getNode() || N->getValueType(i) == To[i].getValueType()) && \"Cannot combine value to value of different type!\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1025, __extension__ __PRETTY_FUNCTION__))
1024 N->getValueType(i) == To[i].getValueType()) &&(static_cast <bool> ((!To[i].getNode() || N->getValueType
(i) == To[i].getValueType()) && "Cannot combine value to value of different type!"
) ? void (0) : __assert_fail ("(!To[i].getNode() || N->getValueType(i) == To[i].getValueType()) && \"Cannot combine value to value of different type!\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1025, __extension__ __PRETTY_FUNCTION__))
1025 "Cannot combine value to value of different type!")(static_cast <bool> ((!To[i].getNode() || N->getValueType
(i) == To[i].getValueType()) && "Cannot combine value to value of different type!"
) ? void (0) : __assert_fail ("(!To[i].getNode() || N->getValueType(i) == To[i].getValueType()) && \"Cannot combine value to value of different type!\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1025, __extension__ __PRETTY_FUNCTION__))
;
1026
1027 WorklistRemover DeadNodes(*this);
1028 DAG.ReplaceAllUsesWith(N, To);
1029 if (AddTo) {
1030 // Push the new nodes and any users onto the worklist
1031 for (unsigned i = 0, e = NumTo; i != e; ++i) {
1032 if (To[i].getNode()) {
1033 AddToWorklist(To[i].getNode());
1034 AddUsersToWorklist(To[i].getNode());
1035 }
1036 }
1037 }
1038
1039 // Finally, if the node is now dead, remove it from the graph. The node
1040 // may not be dead if the replacement process recursively simplified to
1041 // something else needing this node.
1042 if (N->use_empty())
1043 deleteAndRecombine(N);
1044 return SDValue(N, 0);
1045}
1046
1047void DAGCombiner::
1048CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
1049 // Replace all uses. If any nodes become isomorphic to other nodes and
1050 // are deleted, make sure to remove them from our worklist.
1051 WorklistRemover DeadNodes(*this);
1052 DAG.ReplaceAllUsesOfValueWith(TLO.Old, TLO.New);
1053
1054 // Push the new node and any (possibly new) users onto the worklist.
1055 AddToWorklist(TLO.New.getNode());
1056 AddUsersToWorklist(TLO.New.getNode());
1057
1058 // Finally, if the node is now dead, remove it from the graph. The node
1059 // may not be dead if the replacement process recursively simplified to
1060 // something else needing this node.
1061 if (TLO.Old.getNode()->use_empty())
1062 deleteAndRecombine(TLO.Old.getNode());
1063}
1064
1065/// Check the specified integer node value to see if it can be simplified or if
1066/// things it uses can be simplified by bit propagation. If so, return true.
1067bool DAGCombiner::SimplifyDemandedBits(SDValue Op, const APInt &Demanded) {
1068 TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations);
1069 KnownBits Known;
1070 if (!TLI.SimplifyDemandedBits(Op, Demanded, Known, TLO))
1071 return false;
1072
1073 // Revisit the node.
1074 AddToWorklist(Op.getNode());
1075
1076 // Replace the old value with the new one.
1077 ++NodesCombined;
1078 DEBUG(dbgs() << "\nReplacing.2 ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.getNode
()->dump(&DAG); dbgs() << "\nWith: "; TLO.New.getNode
()->dump(&DAG); dbgs() << '\n'; } } while (false
)
1079 TLO.Old.getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.getNode
()->dump(&DAG); dbgs() << "\nWith: "; TLO.New.getNode
()->dump(&DAG); dbgs() << '\n'; } } while (false
)
1080 dbgs() << "\nWith: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.getNode
()->dump(&DAG); dbgs() << "\nWith: "; TLO.New.getNode
()->dump(&DAG); dbgs() << '\n'; } } while (false
)
1081 TLO.New.getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.getNode
()->dump(&DAG); dbgs() << "\nWith: "; TLO.New.getNode
()->dump(&DAG); dbgs() << '\n'; } } while (false
)
1082 dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.getNode
()->dump(&DAG); dbgs() << "\nWith: "; TLO.New.getNode
()->dump(&DAG); dbgs() << '\n'; } } while (false
)
;
1083
1084 CommitTargetLoweringOpt(TLO);
1085 return true;
1086}
1087
1088void DAGCombiner::ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad) {
1089 SDLoc DL(Load);
1090 EVT VT = Load->getValueType(0);
1091 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, VT, SDValue(ExtLoad, 0));
1092
1093 DEBUG(dbgs() << "\nReplacing.9 ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.9 "; Load->
dump(&DAG); dbgs() << "\nWith: "; Trunc.getNode()->
dump(&DAG); dbgs() << '\n'; } } while (false)
1094 Load->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.9 "; Load->
dump(&DAG); dbgs() << "\nWith: "; Trunc.getNode()->
dump(&DAG); dbgs() << '\n'; } } while (false)
1095 dbgs() << "\nWith: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.9 "; Load->
dump(&DAG); dbgs() << "\nWith: "; Trunc.getNode()->
dump(&DAG); dbgs() << '\n'; } } while (false)
1096 Trunc.getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.9 "; Load->
dump(&DAG); dbgs() << "\nWith: "; Trunc.getNode()->
dump(&DAG); dbgs() << '\n'; } } while (false)
1097 dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.9 "; Load->
dump(&DAG); dbgs() << "\nWith: "; Trunc.getNode()->
dump(&DAG); dbgs() << '\n'; } } while (false)
;
1098 WorklistRemover DeadNodes(*this);
1099 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), Trunc);
1100 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), SDValue(ExtLoad, 1));
1101 deleteAndRecombine(Load);
1102 AddToWorklist(Trunc.getNode());
1103}
1104
1105SDValue DAGCombiner::PromoteOperand(SDValue Op, EVT PVT, bool &Replace) {
1106 Replace = false;
1107 SDLoc DL(Op);
1108 if (ISD::isUNINDEXEDLoad(Op.getNode())) {
1109 LoadSDNode *LD = cast<LoadSDNode>(Op);
1110 EVT MemVT = LD->getMemoryVT();
1111 ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD)
1112 ? (TLI.isLoadExtLegal(ISD::ZEXTLOAD, PVT, MemVT) ? ISD::ZEXTLOAD
1113 : ISD::EXTLOAD)
1114 : LD->getExtensionType();
1115 Replace = true;
1116 return DAG.getExtLoad(ExtType, DL, PVT,
1117 LD->getChain(), LD->getBasePtr(),
1118 MemVT, LD->getMemOperand());
1119 }
1120
1121 unsigned Opc = Op.getOpcode();
1122 switch (Opc) {
1123 default: break;
1124 case ISD::AssertSext:
1125 if (SDValue Op0 = SExtPromoteOperand(Op.getOperand(0), PVT))
1126 return DAG.getNode(ISD::AssertSext, DL, PVT, Op0, Op.getOperand(1));
1127 break;
1128 case ISD::AssertZext:
1129 if (SDValue Op0 = ZExtPromoteOperand(Op.getOperand(0), PVT))
1130 return DAG.getNode(ISD::AssertZext, DL, PVT, Op0, Op.getOperand(1));
1131 break;
1132 case ISD::Constant: {
1133 unsigned ExtOpc =
1134 Op.getValueType().isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
1135 return DAG.getNode(ExtOpc, DL, PVT, Op);
1136 }
1137 }
1138
1139 if (!TLI.isOperationLegal(ISD::ANY_EXTEND, PVT))
1140 return SDValue();
1141 return DAG.getNode(ISD::ANY_EXTEND, DL, PVT, Op);
1142}
1143
1144SDValue DAGCombiner::SExtPromoteOperand(SDValue Op, EVT PVT) {
1145 if (!TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, PVT))
1146 return SDValue();
1147 EVT OldVT = Op.getValueType();
1148 SDLoc DL(Op);
1149 bool Replace = false;
1150 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
1151 if (!NewOp.getNode())
1152 return SDValue();
1153 AddToWorklist(NewOp.getNode());
1154
1155 if (Replace)
1156 ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
1157 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, NewOp.getValueType(), NewOp,
1158 DAG.getValueType(OldVT));
1159}
1160
1161SDValue DAGCombiner::ZExtPromoteOperand(SDValue Op, EVT PVT) {
1162 EVT OldVT = Op.getValueType();
1163 SDLoc DL(Op);
1164 bool Replace = false;
1165 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
1166 if (!NewOp.getNode())
1167 return SDValue();
1168 AddToWorklist(NewOp.getNode());
1169
1170 if (Replace)
1171 ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
1172 return DAG.getZeroExtendInReg(NewOp, DL, OldVT);
1173}
1174
1175/// Promote the specified integer binary operation if the target indicates it is
1176/// beneficial. e.g. On x86, it's usually better to promote i16 operations to
1177/// i32 since i16 instructions are longer.
1178SDValue DAGCombiner::PromoteIntBinOp(SDValue Op) {
1179 if (!LegalOperations)
1180 return SDValue();
1181
1182 EVT VT = Op.getValueType();
1183 if (VT.isVector() || !VT.isInteger())
1184 return SDValue();
1185
1186 // If operation type is 'undesirable', e.g. i16 on x86, consider
1187 // promoting it.
1188 unsigned Opc = Op.getOpcode();
1189 if (TLI.isTypeDesirableForOp(Opc, VT))
1190 return SDValue();
1191
1192 EVT PVT = VT;
1193 // Consult target whether it is a good idea to promote this operation and
1194 // what's the right type to promote it to.
1195 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1196 assert(PVT != VT && "Don't know what type to promote to!")(static_cast <bool> (PVT != VT && "Don't know what type to promote to!"
) ? void (0) : __assert_fail ("PVT != VT && \"Don't know what type to promote to!\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1196, __extension__ __PRETTY_FUNCTION__))
;
1197
1198 DEBUG(dbgs() << "\nPromoting "; Op.getNode()->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; Op.getNode(
)->dump(&DAG); } } while (false)
;
1199
1200 bool Replace0 = false;
1201 SDValue N0 = Op.getOperand(0);
1202 SDValue NN0 = PromoteOperand(N0, PVT, Replace0);
1203
1204 bool Replace1 = false;
1205 SDValue N1 = Op.getOperand(1);
1206 SDValue NN1 = PromoteOperand(N1, PVT, Replace1);
1207 SDLoc DL(Op);
1208
1209 SDValue RV =
1210 DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, NN0, NN1));
1211
1212 // We are always replacing N0/N1's use in N and only need
1213 // additional replacements if there are additional uses.
1214 Replace0 &= !N0->hasOneUse();
1215 Replace1 &= (N0 != N1) && !N1->hasOneUse();
1216
1217 // Combine Op here so it is preserved past replacements.
1218 CombineTo(Op.getNode(), RV);
1219
1220 // If operands have a use ordering, make sure we deal with
1221 // predecessor first.
1222 if (Replace0 && Replace1 && N0.getNode()->isPredecessorOf(N1.getNode())) {
1223 std::swap(N0, N1);
1224 std::swap(NN0, NN1);
1225 }
1226
1227 if (Replace0) {
1228 AddToWorklist(NN0.getNode());
1229 ReplaceLoadWithPromotedLoad(N0.getNode(), NN0.getNode());
1230 }
1231 if (Replace1) {
1232 AddToWorklist(NN1.getNode());
1233 ReplaceLoadWithPromotedLoad(N1.getNode(), NN1.getNode());
1234 }
1235 return Op;
1236 }
1237 return SDValue();
1238}
1239
1240/// Promote the specified integer shift operation if the target indicates it is
1241/// beneficial. e.g. On x86, it's usually better to promote i16 operations to
1242/// i32 since i16 instructions are longer.
1243SDValue DAGCombiner::PromoteIntShiftOp(SDValue Op) {
1244 if (!LegalOperations)
1245 return SDValue();
1246
1247 EVT VT = Op.getValueType();
1248 if (VT.isVector() || !VT.isInteger())
1249 return SDValue();
1250
1251 // If operation type is 'undesirable', e.g. i16 on x86, consider
1252 // promoting it.
1253 unsigned Opc = Op.getOpcode();
1254 if (TLI.isTypeDesirableForOp(Opc, VT))
1255 return SDValue();
1256
1257 EVT PVT = VT;
1258 // Consult target whether it is a good idea to promote this operation and
1259 // what's the right type to promote it to.
1260 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1261 assert(PVT != VT && "Don't know what type to promote to!")(static_cast <bool> (PVT != VT && "Don't know what type to promote to!"
) ? void (0) : __assert_fail ("PVT != VT && \"Don't know what type to promote to!\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1261, __extension__ __PRETTY_FUNCTION__))
;
1262
1263 DEBUG(dbgs() << "\nPromoting "; Op.getNode()->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; Op.getNode(
)->dump(&DAG); } } while (false)
;
1264
1265 bool Replace = false;
1266 SDValue N0 = Op.getOperand(0);
1267 SDValue N1 = Op.getOperand(1);
1268 if (Opc == ISD::SRA)
1269 N0 = SExtPromoteOperand(N0, PVT);
1270 else if (Opc == ISD::SRL)
1271 N0 = ZExtPromoteOperand(N0, PVT);
1272 else
1273 N0 = PromoteOperand(N0, PVT, Replace);
1274
1275 if (!N0.getNode())
1276 return SDValue();
1277
1278 SDLoc DL(Op);
1279 SDValue RV =
1280 DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, N0, N1));
1281
1282 AddToWorklist(N0.getNode());
1283 if (Replace)
1284 ReplaceLoadWithPromotedLoad(Op.getOperand(0).getNode(), N0.getNode());
1285
1286 // Deal with Op being deleted.
1287 if (Op && Op.getOpcode() != ISD::DELETED_NODE)
1288 return RV;
1289 }
1290 return SDValue();
1291}
1292
1293SDValue DAGCombiner::PromoteExtend(SDValue Op) {
1294 if (!LegalOperations)
1295 return SDValue();
1296
1297 EVT VT = Op.getValueType();
1298 if (VT.isVector() || !VT.isInteger())
1299 return SDValue();
1300
1301 // If operation type is 'undesirable', e.g. i16 on x86, consider
1302 // promoting it.
1303 unsigned Opc = Op.getOpcode();
1304 if (TLI.isTypeDesirableForOp(Opc, VT))
1305 return SDValue();
1306
1307 EVT PVT = VT;
1308 // Consult target whether it is a good idea to promote this operation and
1309 // what's the right type to promote it to.
1310 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1311 assert(PVT != VT && "Don't know what type to promote to!")(static_cast <bool> (PVT != VT && "Don't know what type to promote to!"
) ? void (0) : __assert_fail ("PVT != VT && \"Don't know what type to promote to!\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1311, __extension__ __PRETTY_FUNCTION__))
;
1312 // fold (aext (aext x)) -> (aext x)
1313 // fold (aext (zext x)) -> (zext x)
1314 // fold (aext (sext x)) -> (sext x)
1315 DEBUG(dbgs() << "\nPromoting ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; Op.getNode(
)->dump(&DAG); } } while (false)
1316 Op.getNode()->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; Op.getNode(
)->dump(&DAG); } } while (false)
;
1317 return DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, Op.getOperand(0));
1318 }
1319 return SDValue();
1320}
1321
1322bool DAGCombiner::PromoteLoad(SDValue Op) {
1323 if (!LegalOperations)
1324 return false;
1325
1326 if (!ISD::isUNINDEXEDLoad(Op.getNode()))
1327 return false;
1328
1329 EVT VT = Op.getValueType();
1330 if (VT.isVector() || !VT.isInteger())
1331 return false;
1332
1333 // If operation type is 'undesirable', e.g. i16 on x86, consider
1334 // promoting it.
1335 unsigned Opc = Op.getOpcode();
1336 if (TLI.isTypeDesirableForOp(Opc, VT))
1337 return false;
1338
1339 EVT PVT = VT;
1340 // Consult target whether it is a good idea to promote this operation and
1341 // what's the right type to promote it to.
1342 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1343 assert(PVT != VT && "Don't know what type to promote to!")(static_cast <bool> (PVT != VT && "Don't know what type to promote to!"
) ? void (0) : __assert_fail ("PVT != VT && \"Don't know what type to promote to!\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1343, __extension__ __PRETTY_FUNCTION__))
;
1344
1345 SDLoc DL(Op);
1346 SDNode *N = Op.getNode();
1347 LoadSDNode *LD = cast<LoadSDNode>(N);
1348 EVT MemVT = LD->getMemoryVT();
1349 ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD)
1350 ? (TLI.isLoadExtLegal(ISD::ZEXTLOAD, PVT, MemVT) ? ISD::ZEXTLOAD
1351 : ISD::EXTLOAD)
1352 : LD->getExtensionType();
1353 SDValue NewLD = DAG.getExtLoad(ExtType, DL, PVT,
1354 LD->getChain(), LD->getBasePtr(),
1355 MemVT, LD->getMemOperand());
1356 SDValue Result = DAG.getNode(ISD::TRUNCATE, DL, VT, NewLD);
1357
1358 DEBUG(dbgs() << "\nPromoting ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; N->dump(
&DAG); dbgs() << "\nTo: "; Result.getNode()->dump
(&DAG); dbgs() << '\n'; } } while (false)
1359 N->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; N->dump(
&DAG); dbgs() << "\nTo: "; Result.getNode()->dump
(&DAG); dbgs() << '\n'; } } while (false)
1360 dbgs() << "\nTo: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; N->dump(
&DAG); dbgs() << "\nTo: "; Result.getNode()->dump
(&DAG); dbgs() << '\n'; } } while (false)
1361 Result.getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; N->dump(
&DAG); dbgs() << "\nTo: "; Result.getNode()->dump
(&DAG); dbgs() << '\n'; } } while (false)
1362 dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; N->dump(
&DAG); dbgs() << "\nTo: "; Result.getNode()->dump
(&DAG); dbgs() << '\n'; } } while (false)
;
1363 WorklistRemover DeadNodes(*this);
1364 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result);
1365 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), NewLD.getValue(1));
1366 deleteAndRecombine(N);
1367 AddToWorklist(Result.getNode());
1368 return true;
1369 }
1370 return false;
1371}
1372
1373/// \brief Recursively delete a node which has no uses and any operands for
1374/// which it is the only use.
1375///
1376/// Note that this both deletes the nodes and removes them from the worklist.
1377/// It also adds any nodes who have had a user deleted to the worklist as they
1378/// may now have only one use and subject to other combines.
1379bool DAGCombiner::recursivelyDeleteUnusedNodes(SDNode *N) {
1380 if (!N->use_empty())
1381 return false;
1382
1383 SmallSetVector<SDNode *, 16> Nodes;
1384 Nodes.insert(N);
1385 do {
1386 N = Nodes.pop_back_val();
1387 if (!N)
1388 continue;
1389
1390 if (N->use_empty()) {
1391 for (const SDValue &ChildN : N->op_values())
1392 Nodes.insert(ChildN.getNode());
1393
1394 removeFromWorklist(N);
1395 DAG.DeleteNode(N);
1396 } else {
1397 AddToWorklist(N);
1398 }
1399 } while (!Nodes.empty());
1400 return true;
1401}
1402
1403//===----------------------------------------------------------------------===//
1404// Main DAG Combiner implementation
1405//===----------------------------------------------------------------------===//
1406
1407void DAGCombiner::Run(CombineLevel AtLevel) {
1408 // set the instance variables, so that the various visit routines may use it.
1409 Level = AtLevel;
1410 LegalOperations = Level >= AfterLegalizeVectorOps;
1411 LegalTypes = Level >= AfterLegalizeTypes;
1412
1413 // Add all the dag nodes to the worklist.
1414 for (SDNode &Node : DAG.allnodes())
1415 AddToWorklist(&Node);
1416
1417 // Create a dummy node (which is not added to allnodes), that adds a reference
1418 // to the root node, preventing it from being deleted, and tracking any
1419 // changes of the root.
1420 HandleSDNode Dummy(DAG.getRoot());
1421
1422 // While the worklist isn't empty, find a node and try to combine it.
1423 while (!WorklistMap.empty()) {
1424 SDNode *N;
1425 // The Worklist holds the SDNodes in order, but it may contain null entries.
1426 do {
1427 N = Worklist.pop_back_val();
1428 } while (!N);
1429
1430 bool GoodWorklistEntry = WorklistMap.erase(N);
1431 (void)GoodWorklistEntry;
1432 assert(GoodWorklistEntry &&(static_cast <bool> (GoodWorklistEntry && "Found a worklist entry without a corresponding map entry!"
) ? void (0) : __assert_fail ("GoodWorklistEntry && \"Found a worklist entry without a corresponding map entry!\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1433, __extension__ __PRETTY_FUNCTION__))
1433 "Found a worklist entry without a corresponding map entry!")(static_cast <bool> (GoodWorklistEntry && "Found a worklist entry without a corresponding map entry!"
) ? void (0) : __assert_fail ("GoodWorklistEntry && \"Found a worklist entry without a corresponding map entry!\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1433, __extension__ __PRETTY_FUNCTION__))
;
1434
1435 // If N has no uses, it is dead. Make sure to revisit all N's operands once
1436 // N is deleted from the DAG, since they too may now be dead or may have a
1437 // reduced number of uses, allowing other xforms.
1438 if (recursivelyDeleteUnusedNodes(N))
1439 continue;
1440
1441 WorklistRemover DeadNodes(*this);
1442
1443 // If this combine is running after legalizing the DAG, re-legalize any
1444 // nodes pulled off the worklist.
1445 if (Level == AfterLegalizeDAG) {
1446 SmallSetVector<SDNode *, 16> UpdatedNodes;
1447 bool NIsValid = DAG.LegalizeOp(N, UpdatedNodes);
1448
1449 for (SDNode *LN : UpdatedNodes) {
1450 AddToWorklist(LN);
1451 AddUsersToWorklist(LN);
1452 }
1453 if (!NIsValid)
1454 continue;
1455 }
1456
1457 DEBUG(dbgs() << "\nCombining: "; N->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nCombining: "; N->dump
(&DAG); } } while (false)
;
1458
1459 // Add any operands of the new node which have not yet been combined to the
1460 // worklist as well. Because the worklist uniques things already, this
1461 // won't repeatedly process the same operand.
1462 CombinedNodes.insert(N);
1463 for (const SDValue &ChildN : N->op_values())
1464 if (!CombinedNodes.count(ChildN.getNode()))
1465 AddToWorklist(ChildN.getNode());
1466
1467 SDValue RV = combine(N);
1468
1469 if (!RV.getNode())
1470 continue;
1471
1472 ++NodesCombined;
1473
1474 // If we get back the same node we passed in, rather than a new node or
1475 // zero, we know that the node must have defined multiple values and
1476 // CombineTo was used. Since CombineTo takes care of the worklist
1477 // mechanics for us, we have no work to do in this case.
1478 if (RV.getNode() == N)
1479 continue;
1480
1481 assert(N->getOpcode() != ISD::DELETED_NODE &&(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& RV.getOpcode() != ISD::DELETED_NODE && "Node was deleted but visit returned new node!"
) ? void (0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && RV.getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned new node!\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1483, __extension__ __PRETTY_FUNCTION__))
1482 RV.getOpcode() != ISD::DELETED_NODE &&(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& RV.getOpcode() != ISD::DELETED_NODE && "Node was deleted but visit returned new node!"
) ? void (0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && RV.getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned new node!\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1483, __extension__ __PRETTY_FUNCTION__))
1483 "Node was deleted but visit returned new node!")(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& RV.getOpcode() != ISD::DELETED_NODE && "Node was deleted but visit returned new node!"
) ? void (0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && RV.getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned new node!\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1483, __extension__ __PRETTY_FUNCTION__))
;
1484
1485 DEBUG(dbgs() << " ... into: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << " ... into: "; RV.getNode()
->dump(&DAG); } } while (false)
1486 RV.getNode()->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << " ... into: "; RV.getNode()
->dump(&DAG); } } while (false)
;
1487
1488 if (N->getNumValues() == RV.getNode()->getNumValues())
1489 DAG.ReplaceAllUsesWith(N, RV.getNode());
1490 else {
1491 assert(N->getValueType(0) == RV.getValueType() &&(static_cast <bool> (N->getValueType(0) == RV.getValueType
() && N->getNumValues() == 1 && "Type mismatch"
) ? void (0) : __assert_fail ("N->getValueType(0) == RV.getValueType() && N->getNumValues() == 1 && \"Type mismatch\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1492, __extension__ __PRETTY_FUNCTION__))
1492 N->getNumValues() == 1 && "Type mismatch")(static_cast <bool> (N->getValueType(0) == RV.getValueType
() && N->getNumValues() == 1 && "Type mismatch"
) ? void (0) : __assert_fail ("N->getValueType(0) == RV.getValueType() && N->getNumValues() == 1 && \"Type mismatch\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1492, __extension__ __PRETTY_FUNCTION__))
;
1493 DAG.ReplaceAllUsesWith(N, &RV);
1494 }
1495
1496 // Push the new node and any users onto the worklist
1497 AddToWorklist(RV.getNode());
1498 AddUsersToWorklist(RV.getNode());
1499
1500 // Finally, if the node is now dead, remove it from the graph. The node
1501 // may not be dead if the replacement process recursively simplified to
1502 // something else needing this node. This will also take care of adding any
1503 // operands which have lost a user to the worklist.
1504 recursivelyDeleteUnusedNodes(N);
1505 }
1506
1507 // If the root changed (e.g. it was a dead load, update the root).
1508 DAG.setRoot(Dummy.getValue());
1509 DAG.RemoveDeadNodes();
1510}
1511
1512SDValue DAGCombiner::visit(SDNode *N) {
1513 switch (N->getOpcode()) {
1514 default: break;
1515 case ISD::TokenFactor: return visitTokenFactor(N);
1516 case ISD::MERGE_VALUES: return visitMERGE_VALUES(N);
1517 case ISD::ADD: return visitADD(N);
1518 case ISD::SUB: return visitSUB(N);
1519 case ISD::ADDC: return visitADDC(N);
1520 case ISD::UADDO: return visitUADDO(N);
1521 case ISD::SUBC: return visitSUBC(N);
1522 case ISD::USUBO: return visitUSUBO(N);
1523 case ISD::ADDE: return visitADDE(N);
1524 case ISD::ADDCARRY: return visitADDCARRY(N);
1525 case ISD::SUBE: return visitSUBE(N);
1526 case ISD::SUBCARRY: return visitSUBCARRY(N);
1527 case ISD::MUL: return visitMUL(N);
1528 case ISD::SDIV: return visitSDIV(N);
1529 case ISD::UDIV: return visitUDIV(N);
1530 case ISD::SREM:
1531 case ISD::UREM: return visitREM(N);
1532 case ISD::MULHU: return visitMULHU(N);
1533 case ISD::MULHS: return visitMULHS(N);
1534 case ISD::SMUL_LOHI: return visitSMUL_LOHI(N);
1535 case ISD::UMUL_LOHI: return visitUMUL_LOHI(N);
1536 case ISD::SMULO: return visitSMULO(N);
1537 case ISD::UMULO: return visitUMULO(N);
1538 case ISD::SMIN:
1539 case ISD::SMAX:
1540 case ISD::UMIN:
1541 case ISD::UMAX: return visitIMINMAX(N);
1542 case ISD::AND: return visitAND(N);
1543 case ISD::OR: return visitOR(N);
1544 case ISD::XOR: return visitXOR(N);
1545 case ISD::SHL: return visitSHL(N);
1546 case ISD::SRA: return visitSRA(N);
1547 case ISD::SRL: return visitSRL(N);
1548 case ISD::ROTR:
1549 case ISD::ROTL: return visitRotate(N);
1550 case ISD::ABS: return visitABS(N);
1551 case ISD::BSWAP: return visitBSWAP(N);
1552 case ISD::BITREVERSE: return visitBITREVERSE(N);
1553 case ISD::CTLZ: return visitCTLZ(N);
1554 case ISD::CTLZ_ZERO_UNDEF: return visitCTLZ_ZERO_UNDEF(N);
1555 case ISD::CTTZ: return visitCTTZ(N);
1556 case ISD::CTTZ_ZERO_UNDEF: return visitCTTZ_ZERO_UNDEF(N);
1557 case ISD::CTPOP: return visitCTPOP(N);
1558 case ISD::SELECT: return visitSELECT(N);
1559 case ISD::VSELECT: return visitVSELECT(N);
1560 case ISD::SELECT_CC: return visitSELECT_CC(N);
1561 case ISD::SETCC: return visitSETCC(N);
1562 case ISD::SETCCE: return visitSETCCE(N);
1563 case ISD::SETCCCARRY: return visitSETCCCARRY(N);
1564 case ISD::SIGN_EXTEND: return visitSIGN_EXTEND(N);
1565 case ISD::ZERO_EXTEND: return visitZERO_EXTEND(N);
1566 case ISD::ANY_EXTEND: return visitANY_EXTEND(N);
1567 case ISD::AssertSext:
1568 case ISD::AssertZext: return visitAssertExt(N);
1569 case ISD::SIGN_EXTEND_INREG: return visitSIGN_EXTEND_INREG(N);
1570 case ISD::SIGN_EXTEND_VECTOR_INREG: return visitSIGN_EXTEND_VECTOR_INREG(N);
1571 case ISD::ZERO_EXTEND_VECTOR_INREG: return visitZERO_EXTEND_VECTOR_INREG(N);
1572 case ISD::TRUNCATE: return visitTRUNCATE(N);
1573 case ISD::BITCAST: return visitBITCAST(N);
1574 case ISD::BUILD_PAIR: return visitBUILD_PAIR(N);
1575 case ISD::FADD: return visitFADD(N);
1576 case ISD::FSUB: return visitFSUB(N);
1577 case ISD::FMUL: return visitFMUL(N);
1578 case ISD::FMA: return visitFMA(N);
1579 case ISD::FDIV: return visitFDIV(N);
1580 case ISD::FREM: return visitFREM(N);
1581 case ISD::FSQRT: return visitFSQRT(N);
1582 case ISD::FCOPYSIGN: return visitFCOPYSIGN(N);
1583 case ISD::SINT_TO_FP: return visitSINT_TO_FP(N);
1584 case ISD::UINT_TO_FP: return visitUINT_TO_FP(N);
1585 case ISD::FP_TO_SINT: return visitFP_TO_SINT(N);
1586 case ISD::FP_TO_UINT: return visitFP_TO_UINT(N);
1587 case ISD::FP_ROUND: return visitFP_ROUND(N);
1588 case ISD::FP_ROUND_INREG: return visitFP_ROUND_INREG(N);
1589 case ISD::FP_EXTEND: return visitFP_EXTEND(N);
1590 case ISD::FNEG: return visitFNEG(N);
1591 case ISD::FABS: return visitFABS(N);
1592 case ISD::FFLOOR: return visitFFLOOR(N);
1593 case ISD::FMINNUM: return visitFMINNUM(N);
1594 case ISD::FMAXNUM: return visitFMAXNUM(N);
1595 case ISD::FCEIL: return visitFCEIL(N);
1596 case ISD::FTRUNC: return visitFTRUNC(N);
1597 case ISD::BRCOND: return visitBRCOND(N);
1598 case ISD::BR_CC: return visitBR_CC(N);
1599 case ISD::LOAD: return visitLOAD(N);
1600 case ISD::STORE: return visitSTORE(N);
1601 case ISD::INSERT_VECTOR_ELT: return visitINSERT_VECTOR_ELT(N);
1602 case ISD::EXTRACT_VECTOR_ELT: return visitEXTRACT_VECTOR_ELT(N);
1603 case ISD::BUILD_VECTOR: return visitBUILD_VECTOR(N);
1604 case ISD::CONCAT_VECTORS: return visitCONCAT_VECTORS(N);
1605 case ISD::EXTRACT_SUBVECTOR: return visitEXTRACT_SUBVECTOR(N);
1606 case ISD::VECTOR_SHUFFLE: return visitVECTOR_SHUFFLE(N);
1607 case ISD::SCALAR_TO_VECTOR: return visitSCALAR_TO_VECTOR(N);
1608 case ISD::INSERT_SUBVECTOR: return visitINSERT_SUBVECTOR(N);
1609 case ISD::MGATHER: return visitMGATHER(N);
1610 case ISD::MLOAD: return visitMLOAD(N);
1611 case ISD::MSCATTER: return visitMSCATTER(N);
1612 case ISD::MSTORE: return visitMSTORE(N);
1613 case ISD::FP_TO_FP16: return visitFP_TO_FP16(N);
1614 case ISD::FP16_TO_FP: return visitFP16_TO_FP(N);
1615 }
1616 return SDValue();
1617}
1618
1619SDValue DAGCombiner::combine(SDNode *N) {
1620 SDValue RV = visit(N);
1621
1622 // If nothing happened, try a target-specific DAG combine.
1623 if (!RV.getNode()) {
1624 assert(N->getOpcode() != ISD::DELETED_NODE &&(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& "Node was deleted but visit returned NULL!") ? void
(0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned NULL!\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1625, __extension__ __PRETTY_FUNCTION__))
1625 "Node was deleted but visit returned NULL!")(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& "Node was deleted but visit returned NULL!") ? void
(0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned NULL!\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1625, __extension__ __PRETTY_FUNCTION__))
;
1626
1627 if (N->getOpcode() >= ISD::BUILTIN_OP_END ||
1628 TLI.hasTargetDAGCombine((ISD::NodeType)N->getOpcode())) {
1629
1630 // Expose the DAG combiner to the target combiner impls.
1631 TargetLowering::DAGCombinerInfo
1632 DagCombineInfo(DAG, Level, false, this);
1633
1634 RV = TLI.PerformDAGCombine(N, DagCombineInfo);
1635 }
1636 }
1637
1638 // If nothing happened still, try promoting the operation.
1639 if (!RV.getNode()) {
1640 switch (N->getOpcode()) {
1641 default: break;
1642 case ISD::ADD:
1643 case ISD::SUB:
1644 case ISD::MUL:
1645 case ISD::AND:
1646 case ISD::OR:
1647 case ISD::XOR:
1648 RV = PromoteIntBinOp(SDValue(N, 0));
1649 break;
1650 case ISD::SHL:
1651 case ISD::SRA:
1652 case ISD::SRL:
1653 RV = PromoteIntShiftOp(SDValue(N, 0));
1654 break;
1655 case ISD::SIGN_EXTEND:
1656 case ISD::ZERO_EXTEND:
1657 case ISD::ANY_EXTEND:
1658 RV = PromoteExtend(SDValue(N, 0));
1659 break;
1660 case ISD::LOAD:
1661 if (PromoteLoad(SDValue(N, 0)))
1662 RV = SDValue(N, 0);
1663 break;
1664 }
1665 }
1666
1667 // If N is a commutative binary node, try eliminate it if the commuted
1668 // version is already present in the DAG.
1669 if (!RV.getNode() && TLI.isCommutativeBinOp(N->getOpcode()) &&
1670 N->getNumValues() == 1) {
1671 SDValue N0 = N->getOperand(0);
1672 SDValue N1 = N->getOperand(1);
1673
1674 // Constant operands are canonicalized to RHS.
1675 if (N0 != N1 && (isa<ConstantSDNode>(N0) || !isa<ConstantSDNode>(N1))) {
1676 SDValue Ops[] = {N1, N0};
1677 SDNode *CSENode = DAG.getNodeIfExists(N->getOpcode(), N->getVTList(), Ops,
1678 N->getFlags());
1679 if (CSENode)
1680 return SDValue(CSENode, 0);
1681 }
1682 }
1683
1684 return RV;
1685}
1686
1687/// Given a node, return its input chain if it has one, otherwise return a null
1688/// sd operand.
1689static SDValue getInputChainForNode(SDNode *N) {
1690 if (unsigned NumOps = N->getNumOperands()) {
1691 if (N->getOperand(0).getValueType() == MVT::Other)
1692 return N->getOperand(0);
1693 if (N->getOperand(NumOps-1).getValueType() == MVT::Other)
1694 return N->getOperand(NumOps-1);
1695 for (unsigned i = 1; i < NumOps-1; ++i)
1696 if (N->getOperand(i).getValueType() == MVT::Other)
1697 return N->getOperand(i);
1698 }
1699 return SDValue();
1700}
1701
1702SDValue DAGCombiner::visitTokenFactor(SDNode *N) {
1703 // If N has two operands, where one has an input chain equal to the other,
1704 // the 'other' chain is redundant.
1705 if (N->getNumOperands() == 2) {
1706 if (getInputChainForNode(N->getOperand(0).getNode()) == N->getOperand(1))
1707 return N->getOperand(0);
1708 if (getInputChainForNode(N->getOperand(1).getNode()) == N->getOperand(0))
1709 return N->getOperand(1);
1710 }
1711
1712 SmallVector<SDNode *, 8> TFs; // List of token factors to visit.
1713 SmallVector<SDValue, 8> Ops; // Ops for replacing token factor.
1714 SmallPtrSet<SDNode*, 16> SeenOps;
1715 bool Changed = false; // If we should replace this token factor.
1716
1717 // Start out with this token factor.
1718 TFs.push_back(N);
1719
1720 // Iterate through token factors. The TFs grows when new token factors are
1721 // encountered.
1722 for (unsigned i = 0; i < TFs.size(); ++i) {
1723 SDNode *TF = TFs[i];
1724
1725 // Check each of the operands.
1726 for (const SDValue &Op : TF->op_values()) {
1727 switch (Op.getOpcode()) {
1728 case ISD::EntryToken:
1729 // Entry tokens don't need to be added to the list. They are
1730 // redundant.
1731 Changed = true;
1732 break;
1733
1734 case ISD::TokenFactor:
1735 if (Op.hasOneUse() && !is_contained(TFs, Op.getNode())) {
1736 // Queue up for processing.
1737 TFs.push_back(Op.getNode());
1738 // Clean up in case the token factor is removed.
1739 AddToWorklist(Op.getNode());
1740 Changed = true;
1741 break;
1742 }
1743 LLVM_FALLTHROUGH[[clang::fallthrough]];
1744
1745 default:
1746 // Only add if it isn't already in the list.
1747 if (SeenOps.insert(Op.getNode()).second)
1748 Ops.push_back(Op);
1749 else
1750 Changed = true;
1751 break;
1752 }
1753 }
1754 }
1755
1756 // Remove Nodes that are chained to another node in the list. Do so
1757 // by walking up chains breath-first stopping when we've seen
1758 // another operand. In general we must climb to the EntryNode, but we can exit
1759 // early if we find all remaining work is associated with just one operand as
1760 // no further pruning is possible.
1761
1762 // List of nodes to search through and original Ops from which they originate.
1763 SmallVector<std::pair<SDNode *, unsigned>, 8> Worklist;
1764 SmallVector<unsigned, 8> OpWorkCount; // Count of work for each Op.
1765 SmallPtrSet<SDNode *, 16> SeenChains;
1766 bool DidPruneOps = false;
1767
1768 unsigned NumLeftToConsider = 0;
1769 for (const SDValue &Op : Ops) {
1770 Worklist.push_back(std::make_pair(Op.getNode(), NumLeftToConsider++));
1771 OpWorkCount.push_back(1);
1772 }
1773
1774 auto AddToWorklist = [&](unsigned CurIdx, SDNode *Op, unsigned OpNumber) {
1775 // If this is an Op, we can remove the op from the list. Remark any
1776 // search associated with it as from the current OpNumber.
1777 if (SeenOps.count(Op) != 0) {
1778 Changed = true;
1779 DidPruneOps = true;
1780 unsigned OrigOpNumber = 0;
1781 while (OrigOpNumber < Ops.size() && Ops[OrigOpNumber].getNode() != Op)
1782 OrigOpNumber++;
1783 assert((OrigOpNumber != Ops.size()) &&(static_cast <bool> ((OrigOpNumber != Ops.size()) &&
"expected to find TokenFactor Operand") ? void (0) : __assert_fail
("(OrigOpNumber != Ops.size()) && \"expected to find TokenFactor Operand\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1784, __extension__ __PRETTY_FUNCTION__))
1784 "expected to find TokenFactor Operand")(static_cast <bool> ((OrigOpNumber != Ops.size()) &&
"expected to find TokenFactor Operand") ? void (0) : __assert_fail
("(OrigOpNumber != Ops.size()) && \"expected to find TokenFactor Operand\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1784, __extension__ __PRETTY_FUNCTION__))
;
1785 // Re-mark worklist from OrigOpNumber to OpNumber
1786 for (unsigned i = CurIdx + 1; i < Worklist.size(); ++i) {
1787 if (Worklist[i].second == OrigOpNumber) {
1788 Worklist[i].second = OpNumber;
1789 }
1790 }
1791 OpWorkCount[OpNumber] += OpWorkCount[OrigOpNumber];
1792 OpWorkCount[OrigOpNumber] = 0;
1793 NumLeftToConsider--;
1794 }
1795 // Add if it's a new chain
1796 if (SeenChains.insert(Op).second) {
1797 OpWorkCount[OpNumber]++;
1798 Worklist.push_back(std::make_pair(Op, OpNumber));
1799 }
1800 };
1801
1802 for (unsigned i = 0; i < Worklist.size() && i < 1024; ++i) {
1803 // We need at least be consider at least 2 Ops to prune.
1804 if (NumLeftToConsider <= 1)
1805 break;
1806 auto CurNode = Worklist[i].first;
1807 auto CurOpNumber = Worklist[i].second;
1808 assert((OpWorkCount[CurOpNumber] > 0) &&(static_cast <bool> ((OpWorkCount[CurOpNumber] > 0) &&
"Node should not appear in worklist") ? void (0) : __assert_fail
("(OpWorkCount[CurOpNumber] > 0) && \"Node should not appear in worklist\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1809, __extension__ __PRETTY_FUNCTION__))
1809 "Node should not appear in worklist")(static_cast <bool> ((OpWorkCount[CurOpNumber] > 0) &&
"Node should not appear in worklist") ? void (0) : __assert_fail
("(OpWorkCount[CurOpNumber] > 0) && \"Node should not appear in worklist\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1809, __extension__ __PRETTY_FUNCTION__))
;
1810 switch (CurNode->getOpcode()) {
1811 case ISD::EntryToken:
1812 // Hitting EntryToken is the only way for the search to terminate without
1813 // hitting
1814 // another operand's search. Prevent us from marking this operand
1815 // considered.
1816 NumLeftToConsider++;
1817 break;
1818 case ISD::TokenFactor:
1819 for (const SDValue &Op : CurNode->op_values())
1820 AddToWorklist(i, Op.getNode(), CurOpNumber);
1821 break;
1822 case ISD::CopyFromReg:
1823 case ISD::CopyToReg:
1824 AddToWorklist(i, CurNode->getOperand(0).getNode(), CurOpNumber);
1825 break;
1826 default:
1827 if (auto *MemNode = dyn_cast<MemSDNode>(CurNode))
1828 AddToWorklist(i, MemNode->getChain().getNode(), CurOpNumber);
1829 break;
1830 }
1831 OpWorkCount[CurOpNumber]--;
1832 if (OpWorkCount[CurOpNumber] == 0)
1833 NumLeftToConsider--;
1834 }
1835
1836 // If we've changed things around then replace token factor.
1837 if (Changed) {
1838 SDValue Result;
1839 if (Ops.empty()) {
1840 // The entry token is the only possible outcome.
1841 Result = DAG.getEntryNode();
1842 } else {
1843 if (DidPruneOps) {
1844 SmallVector<SDValue, 8> PrunedOps;
1845 //
1846 for (const SDValue &Op : Ops) {
1847 if (SeenChains.count(Op.getNode()) == 0)
1848 PrunedOps.push_back(Op);
1849 }
1850 Result = DAG.getNode(ISD::TokenFactor, SDLoc(N), MVT::Other, PrunedOps);
1851 } else {
1852 Result = DAG.getNode(ISD::TokenFactor, SDLoc(N), MVT::Other, Ops);
1853 }
1854 }
1855 return Result;
1856 }
1857 return SDValue();
1858}
1859
1860/// MERGE_VALUES can always be eliminated.
1861SDValue DAGCombiner::visitMERGE_VALUES(SDNode *N) {
1862 WorklistRemover DeadNodes(*this);
1863 // Replacing results may cause a different MERGE_VALUES to suddenly
1864 // be CSE'd with N, and carry its uses with it. Iterate until no
1865 // uses remain, to ensure that the node can be safely deleted.
1866 // First add the users of this node to the work list so that they
1867 // can be tried again once they have new operands.
1868 AddUsersToWorklist(N);
1869 do {
1870 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
1871 DAG.ReplaceAllUsesOfValueWith(SDValue(N, i), N->getOperand(i));
1872 } while (!N->use_empty());
1873 deleteAndRecombine(N);
1874 return SDValue(N, 0); // Return N so it doesn't get rechecked!
1875}
1876
1877/// If \p N is a ConstantSDNode with isOpaque() == false return it casted to a
1878/// ConstantSDNode pointer else nullptr.
1879static ConstantSDNode *getAsNonOpaqueConstant(SDValue N) {
1880 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N);
1881 return Const != nullptr && !Const->isOpaque() ? Const : nullptr;
1882}
1883
1884SDValue DAGCombiner::foldBinOpIntoSelect(SDNode *BO) {
1885 auto BinOpcode = BO->getOpcode();
1886 assert((BinOpcode == ISD::ADD || BinOpcode == ISD::SUB ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1895, __extension__ __PRETTY_FUNCTION__))
1887 BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1895, __extension__ __PRETTY_FUNCTION__))
1888 BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1895, __extension__ __PRETTY_FUNCTION__))
1889 BinOpcode == ISD::UREM || BinOpcode == ISD::AND ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1895, __extension__ __PRETTY_FUNCTION__))
1890 BinOpcode == ISD::OR || BinOpcode == ISD::XOR ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1895, __extension__ __PRETTY_FUNCTION__))
1891 BinOpcode == ISD::SHL || BinOpcode == ISD::SRL ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1895, __extension__ __PRETTY_FUNCTION__))
1892 BinOpcode == ISD::SRA || BinOpcode == ISD::FADD ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1895, __extension__ __PRETTY_FUNCTION__))
1893 BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1895, __extension__ __PRETTY_FUNCTION__))
1894 BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) &&(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1895, __extension__ __PRETTY_FUNCTION__))
1895 "Unexpected binary operator")(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1895, __extension__ __PRETTY_FUNCTION__))
;
1896
1897 // Bail out if any constants are opaque because we can't constant fold those.
1898 SDValue C1 = BO->getOperand(1);
1899 if (!isConstantOrConstantVector(C1, true) &&
1900 !isConstantFPBuildVectorOrConstantFP(C1))
1901 return SDValue();
1902
1903 // Don't do this unless the old select is going away. We want to eliminate the
1904 // binary operator, not replace a binop with a select.
1905 // TODO: Handle ISD::SELECT_CC.
1906 SDValue Sel = BO->getOperand(0);
1907 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse())
1908 return SDValue();
1909
1910 SDValue CT = Sel.getOperand(1);
1911 if (!isConstantOrConstantVector(CT, true) &&
1912 !isConstantFPBuildVectorOrConstantFP(CT))
1913 return SDValue();
1914
1915 SDValue CF = Sel.getOperand(2);
1916 if (!isConstantOrConstantVector(CF, true) &&
1917 !isConstantFPBuildVectorOrConstantFP(CF))
1918 return SDValue();
1919
1920 // We have a select-of-constants followed by a binary operator with a
1921 // constant. Eliminate the binop by pulling the constant math into the select.
1922 // Example: add (select Cond, CT, CF), C1 --> select Cond, CT + C1, CF + C1
1923 EVT VT = Sel.getValueType();
1924 SDLoc DL(Sel);
1925 SDValue NewCT = DAG.getNode(BinOpcode, DL, VT, CT, C1);
1926 if (!NewCT.isUndef() &&
1927 !isConstantOrConstantVector(NewCT, true) &&
1928 !isConstantFPBuildVectorOrConstantFP(NewCT))
1929 return SDValue();
1930
1931 SDValue NewCF = DAG.getNode(BinOpcode, DL, VT, CF, C1);
1932 if (!NewCF.isUndef() &&
1933 !isConstantOrConstantVector(NewCF, true) &&
1934 !isConstantFPBuildVectorOrConstantFP(NewCF))
1935 return SDValue();
1936
1937 return DAG.getSelect(DL, VT, Sel.getOperand(0), NewCT, NewCF);
1938}
1939
1940SDValue DAGCombiner::visitADD(SDNode *N) {
1941 SDValue N0 = N->getOperand(0);
1942 SDValue N1 = N->getOperand(1);
1943 EVT VT = N0.getValueType();
1944 SDLoc DL(N);
1945
1946 // fold vector ops
1947 if (VT.isVector()) {
1948 if (SDValue FoldedVOp = SimplifyVBinOp(N))
1949 return FoldedVOp;
1950
1951 // fold (add x, 0) -> x, vector edition
1952 if (ISD::isBuildVectorAllZeros(N1.getNode()))
1953 return N0;
1954 if (ISD::isBuildVectorAllZeros(N0.getNode()))
1955 return N1;
1956 }
1957
1958 // fold (add x, undef) -> undef
1959 if (N0.isUndef())
1960 return N0;
1961
1962 if (N1.isUndef())
1963 return N1;
1964
1965 if (DAG.isConstantIntBuildVectorOrConstantInt(N0)) {
1966 // canonicalize constant to RHS
1967 if (!DAG.isConstantIntBuildVectorOrConstantInt(N1))
1968 return DAG.getNode(ISD::ADD, DL, VT, N1, N0);
1969 // fold (add c1, c2) -> c1+c2
1970 return DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, N0.getNode(),
1971 N1.getNode());
1972 }
1973
1974 // fold (add x, 0) -> x
1975 if (isNullConstant(N1))
1976 return N0;
1977
1978 if (isConstantOrConstantVector(N1, /* NoOpaque */ true)) {
1979 // fold ((c1-A)+c2) -> (c1+c2)-A
1980 if (N0.getOpcode() == ISD::SUB &&
1981 isConstantOrConstantVector(N0.getOperand(0), /* NoOpaque */ true)) {
1982 // FIXME: Adding 2 constants should be handled by FoldConstantArithmetic.
1983 return DAG.getNode(ISD::SUB, DL, VT,
1984 DAG.getNode(ISD::ADD, DL, VT, N1, N0.getOperand(0)),
1985 N0.getOperand(1));
1986 }
1987
1988 // add (sext i1 X), 1 -> zext (not i1 X)
1989 // We don't transform this pattern:
1990 // add (zext i1 X), -1 -> sext (not i1 X)
1991 // because most (?) targets generate better code for the zext form.
1992 if (N0.getOpcode() == ISD::SIGN_EXTEND && N0.hasOneUse() &&
1993 isOneConstantOrOneSplatConstant(N1)) {
1994 SDValue X = N0.getOperand(0);
1995 if ((!LegalOperations ||
1996 (TLI.isOperationLegal(ISD::XOR, X.getValueType()) &&
1997 TLI.isOperationLegal(ISD::ZERO_EXTEND, VT))) &&
1998 X.getScalarValueSizeInBits() == 1) {
1999 SDValue Not = DAG.getNOT(DL, X, X.getValueType());
2000 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Not);
2001 }
2002 }
2003
2004 // Undo the add -> or combine to merge constant offsets from a frame index.
2005 if (N0.getOpcode() == ISD::OR &&
2006 isa<FrameIndexSDNode>(N0.getOperand(0)) &&
2007 isa<ConstantSDNode>(N0.getOperand(1)) &&
2008 DAG.haveNoCommonBitsSet(N0.getOperand(0), N0.getOperand(1))) {
2009 SDValue Add0 = DAG.getNode(ISD::ADD, DL, VT, N1, N0.getOperand(1));
2010 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), Add0);
2011 }
2012 }
2013
2014 if (SDValue NewSel = foldBinOpIntoSelect(N))
2015 return NewSel;
2016
2017 // reassociate add
2018 if (SDValue RADD = ReassociateOps(ISD::ADD, DL, N0, N1))
2019 return RADD;
2020
2021 // fold ((0-A) + B) -> B-A
2022 if (N0.getOpcode() == ISD::SUB &&
2023 isNullConstantOrNullSplatConstant(N0.getOperand(0)))
2024 return DAG.getNode(ISD::SUB, DL, VT, N1, N0.getOperand(1));
2025
2026 // fold (A + (0-B)) -> A-B
2027 if (N1.getOpcode() == ISD::SUB &&
2028 isNullConstantOrNullSplatConstant(N1.getOperand(0)))
2029 return DAG.getNode(ISD::SUB, DL, VT, N0, N1.getOperand(1));
2030
2031 // fold (A+(B-A)) -> B
2032 if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(1))
2033 return N1.getOperand(0);
2034
2035 // fold ((B-A)+A) -> B
2036 if (N0.getOpcode() == ISD::SUB && N1 == N0.getOperand(1))
2037 return N0.getOperand(0);
2038
2039 // fold (A+(B-(A+C))) to (B-C)
2040 if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD &&
2041 N0 == N1.getOperand(1).getOperand(0))
2042 return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0),
2043 N1.getOperand(1).getOperand(1));
2044
2045 // fold (A+(B-(C+A))) to (B-C)
2046 if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD &&
2047 N0 == N1.getOperand(1).getOperand(1))
2048 return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0),
2049 N1.getOperand(1).getOperand(0));
2050
2051 // fold (A+((B-A)+or-C)) to (B+or-C)
2052 if ((N1.getOpcode() == ISD::SUB || N1.getOpcode() == ISD::ADD) &&
2053 N1.getOperand(0).getOpcode() == ISD::SUB &&
2054 N0 == N1.getOperand(0).getOperand(1))
2055 return DAG.getNode(N1.getOpcode(), DL, VT, N1.getOperand(0).getOperand(0),
2056 N1.getOperand(1));
2057
2058 // fold (A-B)+(C-D) to (A+C)-(B+D) when A or C is constant
2059 if (N0.getOpcode() == ISD::SUB && N1.getOpcode() == ISD::SUB) {
2060 SDValue N00 = N0.getOperand(0);
2061 SDValue N01 = N0.getOperand(1);
2062 SDValue N10 = N1.getOperand(0);
2063 SDValue N11 = N1.getOperand(1);
2064
2065 if (isConstantOrConstantVector(N00) || isConstantOrConstantVector(N10))
2066 return DAG.getNode(ISD::SUB, DL, VT,
2067 DAG.getNode(ISD::ADD, SDLoc(N0), VT, N00, N10),
2068 DAG.getNode(ISD::ADD, SDLoc(N1), VT, N01, N11));
2069 }
2070
2071 if (SimplifyDemandedBits(SDValue(N, 0)))
2072 return SDValue(N, 0);
2073
2074 // fold (a+b) -> (a|b) iff a and b share no bits.
2075 if ((!LegalOperations || TLI.isOperationLegal(ISD::OR, VT)) &&
2076 DAG.haveNoCommonBitsSet(N0, N1))
2077 return DAG.getNode(ISD::OR, DL, VT, N0, N1);
2078
2079 if (SDValue Combined = visitADDLike(N0, N1, N))
2080 return Combined;
2081
2082 if (SDValue Combined = visitADDLike(N1, N0, N))
2083 return Combined;
2084
2085 return SDValue();
2086}
2087
2088static SDValue getAsCarry(const TargetLowering &TLI, SDValue V) {
2089 bool Masked = false;
2090
2091 // First, peel away TRUNCATE/ZERO_EXTEND/AND nodes due to legalization.
2092 while (true) {
2093 if (V.getOpcode() == ISD::TRUNCATE || V.getOpcode() == ISD::ZERO_EXTEND) {
2094 V = V.getOperand(0);
2095 continue;
2096 }
2097
2098 if (V.getOpcode() == ISD::AND && isOneConstant(V.getOperand(1))) {
2099 Masked = true;
2100 V = V.getOperand(0);
2101 continue;
2102 }
2103
2104 break;
2105 }
2106
2107 // If this is not a carry, return.
2108 if (V.getResNo() != 1)
2109 return SDValue();
2110
2111 if (V.getOpcode() != ISD::ADDCARRY && V.getOpcode() != ISD::SUBCARRY &&
2112 V.getOpcode() != ISD::UADDO && V.getOpcode() != ISD::USUBO)
2113 return SDValue();
2114
2115 // If the result is masked, then no matter what kind of bool it is we can
2116 // return. If it isn't, then we need to make sure the bool type is either 0 or
2117 // 1 and not other values.
2118 if (Masked ||
2119 TLI.getBooleanContents(V.getValueType()) ==
2120 TargetLoweringBase::ZeroOrOneBooleanContent)
2121 return V;
2122
2123 return SDValue();
2124}
2125
2126SDValue DAGCombiner::visitADDLike(SDValue N0, SDValue N1, SDNode *LocReference) {
2127 EVT VT = N0.getValueType();
2128 SDLoc DL(LocReference);
2129
2130 // fold (add x, shl(0 - y, n)) -> sub(x, shl(y, n))
2131 if (N1.getOpcode() == ISD::SHL && N1.getOperand(0).getOpcode() == ISD::SUB &&
2132 isNullConstantOrNullSplatConstant(N1.getOperand(0).getOperand(0)))
2133 return DAG.getNode(ISD::SUB, DL, VT, N0,
2134 DAG.getNode(ISD::SHL, DL, VT,
2135 N1.getOperand(0).getOperand(1),
2136 N1.getOperand(1)));
2137
2138 if (N1.getOpcode() == ISD::AND) {
2139 SDValue AndOp0 = N1.getOperand(0);
2140 unsigned NumSignBits = DAG.ComputeNumSignBits(AndOp0);
2141 unsigned DestBits = VT.getScalarSizeInBits();
2142
2143 // (add z, (and (sbbl x, x), 1)) -> (sub z, (sbbl x, x))
2144 // and similar xforms where the inner op is either ~0 or 0.
2145 if (NumSignBits == DestBits &&
2146 isOneConstantOrOneSplatConstant(N1->getOperand(1)))
2147 return DAG.getNode(ISD::SUB, DL, VT, N0, AndOp0);
2148 }
2149
2150 // add (sext i1), X -> sub X, (zext i1)
2151 if (N0.getOpcode() == ISD::SIGN_EXTEND &&
2152 N0.getOperand(0).getValueType() == MVT::i1 &&
2153 !TLI.isOperationLegal(ISD::SIGN_EXTEND, MVT::i1)) {
2154 SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0));
2155 return DAG.getNode(ISD::SUB, DL, VT, N1, ZExt);
2156 }
2157
2158 // add X, (sextinreg Y i1) -> sub X, (and Y 1)
2159 if (N1.getOpcode() == ISD::SIGN_EXTEND_INREG) {
2160 VTSDNode *TN = cast<VTSDNode>(N1.getOperand(1));
2161 if (TN->getVT() == MVT::i1) {
2162 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0),
2163 DAG.getConstant(1, DL, VT));
2164 return DAG.getNode(ISD::SUB, DL, VT, N0, ZExt);
2165 }
2166 }
2167
2168 // (add X, (addcarry Y, 0, Carry)) -> (addcarry X, Y, Carry)
2169 if (N1.getOpcode() == ISD::ADDCARRY && isNullConstant(N1.getOperand(1)) &&
2170 N1.getResNo() == 0)
2171 return DAG.getNode(ISD::ADDCARRY, DL, N1->getVTList(),
2172 N0, N1.getOperand(0), N1.getOperand(2));
2173
2174 // (add X, Carry) -> (addcarry X, 0, Carry)
2175 if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT))
2176 if (SDValue Carry = getAsCarry(TLI, N1))
2177 return DAG.getNode(ISD::ADDCARRY, DL,
2178 DAG.getVTList(VT, Carry.getValueType()), N0,
2179 DAG.getConstant(0, DL, VT), Carry);
2180
2181 return SDValue();
2182}
2183
2184SDValue DAGCombiner::visitADDC(SDNode *N) {
2185 SDValue N0 = N->getOperand(0);
2186 SDValue N1 = N->getOperand(1);
2187 EVT VT = N0.getValueType();
2188 SDLoc DL(N);
2189
2190 // If the flag result is dead, turn this into an ADD.
2191 if (!N->hasAnyUseOfValue(1))
2192 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2193 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2194
2195 // canonicalize constant to RHS.
2196 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
2197 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2198 if (N0C && !N1C)
2199 return DAG.getNode(ISD::ADDC, DL, N->getVTList(), N1, N0);
2200
2201 // fold (addc x, 0) -> x + no carry out
2202 if (isNullConstant(N1))
2203 return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE,
2204 DL, MVT::Glue));
2205
2206 // If it cannot overflow, transform into an add.
2207 if (DAG.computeOverflowKind(N0, N1) == SelectionDAG::OFK_Never)
2208 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2209 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2210
2211 return SDValue();
2212}
2213
2214SDValue DAGCombiner::visitUADDO(SDNode *N) {
2215 SDValue N0 = N->getOperand(0);
2216 SDValue N1 = N->getOperand(1);
2217 EVT VT = N0.getValueType();
2218 if (VT.isVector())
2219 return SDValue();
2220
2221 EVT CarryVT = N->getValueType(1);
2222 SDLoc DL(N);
2223
2224 // If the flag result is dead, turn this into an ADD.
2225 if (!N->hasAnyUseOfValue(1))
2226 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2227 DAG.getUNDEF(CarryVT));
2228
2229 // canonicalize constant to RHS.
2230 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
2231 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2232 if (N0C && !N1C)
2233 return DAG.getNode(ISD::UADDO, DL, N->getVTList(), N1, N0);
2234
2235 // fold (uaddo x, 0) -> x + no carry out
2236 if (isNullConstant(N1))
2237 return CombineTo(N, N0, DAG.getConstant(0, DL, CarryVT));
2238
2239 // If it cannot overflow, transform into an add.
2240 if (DAG.computeOverflowKind(N0, N1) == SelectionDAG::OFK_Never)
2241 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2242 DAG.getConstant(0, DL, CarryVT));
2243
2244 if (SDValue Combined = visitUADDOLike(N0, N1, N))
2245 return Combined;
2246
2247 if (SDValue Combined = visitUADDOLike(N1, N0, N))
2248 return Combined;
2249
2250 return SDValue();
2251}
2252
2253SDValue DAGCombiner::visitUADDOLike(SDValue N0, SDValue N1, SDNode *N) {
2254 auto VT = N0.getValueType();
2255
2256 // (uaddo X, (addcarry Y, 0, Carry)) -> (addcarry X, Y, Carry)
2257 // If Y + 1 cannot overflow.
2258 if (N1.getOpcode() == ISD::ADDCARRY && isNullConstant(N1.getOperand(1))) {
2259 SDValue Y = N1.getOperand(0);
2260 SDValue One = DAG.getConstant(1, SDLoc(N), Y.getValueType());
2261 if (DAG.computeOverflowKind(Y, One) == SelectionDAG::OFK_Never)
2262 return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0, Y,
2263 N1.getOperand(2));
2264 }
2265
2266 // (uaddo X, Carry) -> (addcarry X, 0, Carry)
2267 if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT))
2268 if (SDValue Carry = getAsCarry(TLI, N1))
2269 return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0,
2270 DAG.getConstant(0, SDLoc(N), VT), Carry);
2271
2272 return SDValue();
2273}
2274
2275SDValue DAGCombiner::visitADDE(SDNode *N) {
2276 SDValue N0 = N->getOperand(0);
2277 SDValue N1 = N->getOperand(1);
2278 SDValue CarryIn = N->getOperand(2);
2279
2280 // canonicalize constant to RHS
2281 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
2282 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2283 if (N0C && !N1C)
2284 return DAG.getNode(ISD::ADDE, SDLoc(N), N->getVTList(),
2285 N1, N0, CarryIn);
2286
2287 // fold (adde x, y, false) -> (addc x, y)
2288 if (CarryIn.getOpcode() == ISD::CARRY_FALSE)
2289 return DAG.getNode(ISD::ADDC, SDLoc(N), N->getVTList(), N0, N1);
2290
2291 return SDValue();
2292}
2293
2294SDValue DAGCombiner::visitADDCARRY(SDNode *N) {
2295 SDValue N0 = N->getOperand(0);
2296 SDValue N1 = N->getOperand(1);
2297 SDValue CarryIn = N->getOperand(2);
2298 SDLoc DL(N);
2299
2300 // canonicalize constant to RHS
2301 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
2302 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2303 if (N0C && !N1C)
2304 return DAG.getNode(ISD::ADDCARRY, DL, N->getVTList(), N1, N0, CarryIn);
2305
2306 // fold (addcarry x, y, false) -> (uaddo x, y)
2307 if (isNullConstant(CarryIn))
2308 return DAG.getNode(ISD::UADDO, DL, N->getVTList(), N0, N1);
2309
2310 // fold (addcarry 0, 0, X) -> (and (ext/trunc X), 1) and no carry.
2311 if (isNullConstant(N0) && isNullConstant(N1)) {
2312 EVT VT = N0.getValueType();
2313 EVT CarryVT = CarryIn.getValueType();
2314 SDValue CarryExt = DAG.getBoolExtOrTrunc(CarryIn, DL, VT, CarryVT);
2315 AddToWorklist(CarryExt.getNode());
2316 return CombineTo(N, DAG.getNode(ISD::AND, DL, VT, CarryExt,
2317 DAG.getConstant(1, DL, VT)),
2318 DAG.getConstant(0, DL, CarryVT));
2319 }
2320
2321 if (SDValue Combined = visitADDCARRYLike(N0, N1, CarryIn, N))
2322 return Combined;
2323
2324 if (SDValue Combined = visitADDCARRYLike(N1, N0, CarryIn, N))
2325 return Combined;
2326
2327 return SDValue();
2328}
2329
2330SDValue DAGCombiner::visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn,
2331 SDNode *N) {
2332 // Iff the flag result is dead:
2333 // (addcarry (add|uaddo X, Y), 0, Carry) -> (addcarry X, Y, Carry)
2334 if ((N0.getOpcode() == ISD::ADD ||
2335 (N0.getOpcode() == ISD::UADDO && N0.getResNo() == 0)) &&
2336 isNullConstant(N1) && !N->hasAnyUseOfValue(1))
2337 return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(),
2338 N0.getOperand(0), N0.getOperand(1), CarryIn);
2339
2340 /**
2341 * When one of the addcarry argument is itself a carry, we may be facing
2342 * a diamond carry propagation. In which case we try to transform the DAG
2343 * to ensure linear carry propagation if that is possible.
2344 *
2345 * We are trying to get:
2346 * (addcarry X, 0, (addcarry A, B, Z):Carry)
2347 */
2348 if (auto Y = getAsCarry(TLI, N1)) {
2349 /**
2350 * (uaddo A, B)
2351 * / \
2352 * Carry Sum
2353 * | \
2354 * | (addcarry *, 0, Z)
2355 * | /
2356 * \ Carry
2357 * | /
2358 * (addcarry X, *, *)
2359 */
2360 if (Y.getOpcode() == ISD::UADDO &&
2361 CarryIn.getResNo() == 1 &&
2362 CarryIn.getOpcode() == ISD::ADDCARRY &&
2363 isNullConstant(CarryIn.getOperand(1)) &&
2364 CarryIn.getOperand(0) == Y.getValue(0)) {
2365 auto NewY = DAG.getNode(ISD::ADDCARRY, SDLoc(N), Y->getVTList(),
2366 Y.getOperand(0), Y.getOperand(1),
2367 CarryIn.getOperand(2));
2368 AddToWorklist(NewY.getNode());
2369 return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0,
2370 DAG.getConstant(0, SDLoc(N), N0.getValueType()),
2371 NewY.getValue(1));
2372 }
2373 }
2374
2375 return SDValue();
2376}
2377
2378// Since it may not be valid to emit a fold to zero for vector initializers
2379// check if we can before folding.
2380static SDValue tryFoldToZero(const SDLoc &DL, const TargetLowering &TLI, EVT VT,
2381 SelectionDAG &DAG, bool LegalOperations,
2382 bool LegalTypes) {
2383 if (!VT.isVector())
2384 return DAG.getConstant(0, DL, VT);
2385 if (!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT))
2386 return DAG.getConstant(0, DL, VT);
2387 return SDValue();
2388}
2389
2390SDValue DAGCombiner::visitSUB(SDNode *N) {
2391 SDValue N0 = N->getOperand(0);
2392 SDValue N1 = N->getOperand(1);
2393 EVT VT = N0.getValueType();
2394 SDLoc DL(N);
2395
2396 // fold vector ops
2397 if (VT.isVector()) {
2398 if (SDValue FoldedVOp = SimplifyVBinOp(N))
2399 return FoldedVOp;
2400
2401 // fold (sub x, 0) -> x, vector edition
2402 if (ISD::isBuildVectorAllZeros(N1.getNode()))
2403 return N0;
2404 }
2405
2406 // fold (sub x, x) -> 0
2407 // FIXME: Refactor this and xor and other similar operations together.
2408 if (N0 == N1)
2409 return tryFoldToZero(DL, TLI, VT, DAG, LegalOperations, LegalTypes);
2410 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
2411 DAG.isConstantIntBuildVectorOrConstantInt(N1)) {
2412 // fold (sub c1, c2) -> c1-c2
2413 return DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, N0.getNode(),
2414 N1.getNode());
2415 }
2416
2417 if (SDValue NewSel = foldBinOpIntoSelect(N))
2418 return NewSel;
2419
2420 ConstantSDNode *N1C = getAsNonOpaqueConstant(N1);
2421
2422 // fold (sub x, c) -> (add x, -c)
2423 if (N1C) {
2424 return DAG.getNode(ISD::ADD, DL, VT, N0,
2425 DAG.getConstant(-N1C->getAPIntValue(), DL, VT));
2426 }
2427
2428 if (isNullConstantOrNullSplatConstant(N0)) {
2429 unsigned BitWidth = VT.getScalarSizeInBits();
2430 // Right-shifting everything out but the sign bit followed by negation is
2431 // the same as flipping arithmetic/logical shift type without the negation:
2432 // -(X >>u 31) -> (X >>s 31)
2433 // -(X >>s 31) -> (X >>u 31)
2434 if (N1->getOpcode() == ISD::SRA || N1->getOpcode() == ISD::SRL) {
2435 ConstantSDNode *ShiftAmt = isConstOrConstSplat(N1.getOperand(1));
2436 if (ShiftAmt && ShiftAmt->getZExtValue() == BitWidth - 1) {
2437 auto NewSh = N1->getOpcode() == ISD::SRA ? ISD::SRL : ISD::SRA;
2438 if (!LegalOperations || TLI.isOperationLegal(NewSh, VT))
2439 return DAG.getNode(NewSh, DL, VT, N1.getOperand(0), N1.getOperand(1));
2440 }
2441 }
2442
2443 // 0 - X --> 0 if the sub is NUW.
2444 if (N->getFlags().hasNoUnsignedWrap())
2445 return N0;
2446
2447 if (DAG.MaskedValueIsZero(N1, ~APInt::getSignMask(BitWidth))) {
2448 // N1 is either 0 or the minimum signed value. If the sub is NSW, then
2449 // N1 must be 0 because negating the minimum signed value is undefined.
2450 if (N->getFlags().hasNoSignedWrap())
2451 return N0;
2452
2453 // 0 - X --> X if X is 0 or the minimum signed value.
2454 return N1;
2455 }
2456 }
2457
2458 // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1)
2459 if (isAllOnesConstantOrAllOnesSplatConstant(N0))
2460 return DAG.getNode(ISD::XOR, DL, VT, N1, N0);
2461
2462 // fold A-(A-B) -> B
2463 if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(0))
2464 return N1.getOperand(1);
2465
2466 // fold (A+B)-A -> B
2467 if (N0.getOpcode() == ISD::ADD && N0.getOperand(0) == N1)
2468 return N0.getOperand(1);
2469
2470 // fold (A+B)-B -> A
2471 if (N0.getOpcode() == ISD::ADD && N0.getOperand(1) == N1)
2472 return N0.getOperand(0);
2473
2474 // fold C2-(A+C1) -> (C2-C1)-A
2475 if (N1.getOpcode() == ISD::ADD) {
2476 SDValue N11 = N1.getOperand(1);
2477 if (isConstantOrConstantVector(N0, /* NoOpaques */ true) &&
2478 isConstantOrConstantVector(N11, /* NoOpaques */ true)) {
2479 SDValue NewC = DAG.getNode(ISD::SUB, DL, VT, N0, N11);
2480 return DAG.getNode(ISD::SUB, DL, VT, NewC, N1.getOperand(0));
2481 }
2482 }
2483
2484 // fold ((A+(B+or-C))-B) -> A+or-C
2485 if (N0.getOpcode() == ISD::ADD &&
2486 (N0.getOperand(1).getOpcode() == ISD::SUB ||
2487 N0.getOperand(1).getOpcode() == ISD::ADD) &&
2488 N0.getOperand(1).getOperand(0) == N1)
2489 return DAG.getNode(N0.getOperand(1).getOpcode(), DL, VT, N0.getOperand(0),
2490 N0.getOperand(1).getOperand(1));
2491
2492 // fold ((A+(C+B))-B) -> A+C
2493 if (N0.getOpcode() == ISD::ADD && N0.getOperand(1).getOpcode() == ISD::ADD &&
2494 N0.getOperand(1).getOperand(1) == N1)
2495 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0),
2496 N0.getOperand(1).getOperand(0));
2497
2498 // fold ((A-(B-C))-C) -> A-B
2499 if (N0.getOpcode() == ISD::SUB && N0.getOperand(1).getOpcode() == ISD::SUB &&
2500 N0.getOperand(1).getOperand(1) == N1)
2501 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0),
2502 N0.getOperand(1).getOperand(0));
2503
2504 // If either operand of a sub is undef, the result is undef
2505 if (N0.isUndef())
2506 return N0;
2507 if (N1.isUndef())
2508 return N1;
2509
2510 // If the relocation model supports it, consider symbol offsets.
2511 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N0))
2512 if (!LegalOperations && TLI.isOffsetFoldingLegal(GA)) {
2513 // fold (sub Sym, c) -> Sym-c
2514 if (N1C && GA->getOpcode() == ISD::GlobalAddress)
2515 return DAG.getGlobalAddress(GA->getGlobal(), SDLoc(N1C), VT,
2516 GA->getOffset() -
2517 (uint64_t)N1C->getSExtValue());
2518 // fold (sub Sym+c1, Sym+c2) -> c1-c2
2519 if (GlobalAddressSDNode *GB = dyn_cast<GlobalAddressSDNode>(N1))
2520 if (GA->getGlobal() == GB->getGlobal())
2521 return DAG.getConstant((uint64_t)GA->getOffset() - GB->getOffset(),
2522 DL, VT);
2523 }
2524
2525 // sub X, (sextinreg Y i1) -> add X, (and Y 1)
2526 if (N1.getOpcode() == ISD::SIGN_EXTEND_INREG) {
2527 VTSDNode *TN = cast<VTSDNode>(N1.getOperand(1));
2528 if (TN->getVT() == MVT::i1) {
2529 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0),
2530 DAG.getConstant(1, DL, VT));
2531 return DAG.getNode(ISD::ADD, DL, VT, N0, ZExt);
2532 }
2533 }
2534
2535 return SDValue();
2536}
2537
2538SDValue DAGCombiner::visitSUBC(SDNode *N) {
2539 SDValue N0 = N->getOperand(0);
2540 SDValue N1 = N->getOperand(1);
2541 EVT VT = N0.getValueType();
2542 SDLoc DL(N);
2543
2544 // If the flag result is dead, turn this into an SUB.
2545 if (!N->hasAnyUseOfValue(1))
2546 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1),
2547 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2548
2549 // fold (subc x, x) -> 0 + no borrow
2550 if (N0 == N1)
2551 return CombineTo(N, DAG.getConstant(0, DL, VT),
2552 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2553
2554 // fold (subc x, 0) -> x + no borrow
2555 if (isNullConstant(N1))
2556 return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2557
2558 // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1) + no borrow
2559 if (isAllOnesConstant(N0))
2560 return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0),
2561 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2562
2563 return SDValue();
2564}
2565
2566SDValue DAGCombiner::visitUSUBO(SDNode *N) {
2567 SDValue N0 = N->getOperand(0);
2568 SDValue N1 = N->getOperand(1);
2569 EVT VT = N0.getValueType();
2570 if (VT.isVector())
2571 return SDValue();
2572
2573 EVT CarryVT = N->getValueType(1);
2574 SDLoc DL(N);
2575
2576 // If the flag result is dead, turn this into an SUB.
2577 if (!N->hasAnyUseOfValue(1))
2578 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1),
2579 DAG.getUNDEF(CarryVT));
2580
2581 // fold (usubo x, x) -> 0 + no borrow
2582 if (N0 == N1)
2583 return CombineTo(N, DAG.getConstant(0, DL, VT),
2584 DAG.getConstant(0, DL, CarryVT));
2585
2586 // fold (usubo x, 0) -> x + no borrow
2587 if (isNullConstant(N1))
2588 return CombineTo(N, N0, DAG.getConstant(0, DL, CarryVT));
2589
2590 // Canonicalize (usubo -1, x) -> ~x, i.e. (xor x, -1) + no borrow
2591 if (isAllOnesConstant(N0))
2592 return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0),
2593 DAG.getConstant(0, DL, CarryVT));
2594
2595 return SDValue();
2596}
2597
2598SDValue DAGCombiner::visitSUBE(SDNode *N) {
2599 SDValue N0 = N->getOperand(0);
2600 SDValue N1 = N->getOperand(1);
2601 SDValue CarryIn = N->getOperand(2);
2602
2603 // fold (sube x, y, false) -> (subc x, y)
2604 if (CarryIn.getOpcode() == ISD::CARRY_FALSE)
2605 return DAG.getNode(ISD::SUBC, SDLoc(N), N->getVTList(), N0, N1);
2606
2607 return SDValue();
2608}
2609
2610SDValue DAGCombiner::visitSUBCARRY(SDNode *N) {
2611 SDValue N0 = N->getOperand(0);
2612 SDValue N1 = N->getOperand(1);
2613 SDValue CarryIn = N->getOperand(2);
2614
2615 // fold (subcarry x, y, false) -> (usubo x, y)
2616 if (isNullConstant(CarryIn))
2617 return DAG.getNode(ISD::USUBO, SDLoc(N), N->getVTList(), N0, N1);
2618
2619 return SDValue();
2620}
2621
2622SDValue DAGCombiner::visitMUL(SDNode *N) {
2623 SDValue N0 = N->getOperand(0);
2624 SDValue N1 = N->getOperand(1);
2625 EVT VT = N0.getValueType();
2626
2627 // fold (mul x, undef) -> 0
2628 if (N0.isUndef() || N1.isUndef())
2629 return DAG.getConstant(0, SDLoc(N), VT);
2630
2631 bool N0IsConst = false;
2632 bool N1IsConst = false;
2633 bool N1IsOpaqueConst = false;
2634 bool N0IsOpaqueConst = false;
2635 APInt ConstValue0, ConstValue1;
2636 // fold vector ops
2637 if (VT.isVector()) {
2638 if (SDValue FoldedVOp = SimplifyVBinOp(N))
2639 return FoldedVOp;
2640
2641 N0IsConst = ISD::isConstantSplatVector(N0.getNode(), ConstValue0);
2642 N1IsConst = ISD::isConstantSplatVector(N1.getNode(), ConstValue1);
2643 assert((!N0IsConst ||(static_cast <bool> ((!N0IsConst || ConstValue0.getBitWidth
() == VT.getScalarSizeInBits()) && "Splat APInt should be element width"
) ? void (0) : __assert_fail ("(!N0IsConst || ConstValue0.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2645, __extension__ __PRETTY_FUNCTION__))
2644 ConstValue0.getBitWidth() == VT.getScalarSizeInBits()) &&(static_cast <bool> ((!N0IsConst || ConstValue0.getBitWidth
() == VT.getScalarSizeInBits()) && "Splat APInt should be element width"
) ? void (0) : __assert_fail ("(!N0IsConst || ConstValue0.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2645, __extension__ __PRETTY_FUNCTION__))
2645 "Splat APInt should be element width")(static_cast <bool> ((!N0IsConst || ConstValue0.getBitWidth
() == VT.getScalarSizeInBits()) && "Splat APInt should be element width"
) ? void (0) : __assert_fail ("(!N0IsConst || ConstValue0.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2645, __extension__ __PRETTY_FUNCTION__))
;
2646 assert((!N1IsConst ||(static_cast <bool> ((!N1IsConst || ConstValue1.getBitWidth
() == VT.getScalarSizeInBits()) && "Splat APInt should be element width"
) ? void (0) : __assert_fail ("(!N1IsConst || ConstValue1.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2648, __extension__ __PRETTY_FUNCTION__))
2647 ConstValue1.getBitWidth() == VT.getScalarSizeInBits()) &&(static_cast <bool> ((!N1IsConst || ConstValue1.getBitWidth
() == VT.getScalarSizeInBits()) && "Splat APInt should be element width"
) ? void (0) : __assert_fail ("(!N1IsConst || ConstValue1.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2648, __extension__ __PRETTY_FUNCTION__))
2648 "Splat APInt should be element width")(static_cast <bool> ((!N1IsConst || ConstValue1.getBitWidth
() == VT.getScalarSizeInBits()) && "Splat APInt should be element width"
) ? void (0) : __assert_fail ("(!N1IsConst || ConstValue1.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2648, __extension__ __PRETTY_FUNCTION__))
;
2649 } else {
2650 N0IsConst = isa<ConstantSDNode>(N0);
2651 if (N0IsConst) {
2652 ConstValue0 = cast<ConstantSDNode>(N0)->getAPIntValue();
2653 N0IsOpaqueConst = cast<ConstantSDNode>(N0)->isOpaque();
2654 }
2655 N1IsConst = isa<ConstantSDNode>(N1);
2656 if (N1IsConst) {
2657 ConstValue1 = cast<ConstantSDNode>(N1)->getAPIntValue();
2658 N1IsOpaqueConst = cast<ConstantSDNode>(N1)->isOpaque();
2659 }
2660 }
2661
2662 // fold (mul c1, c2) -> c1*c2
2663 if (N0IsConst && N1IsConst && !N0IsOpaqueConst && !N1IsOpaqueConst)
2664 return DAG.FoldConstantArithmetic(ISD::MUL, SDLoc(N), VT,
2665 N0.getNode(), N1.getNode());
2666
2667 // canonicalize constant to RHS (vector doesn't have to splat)
2668 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
2669 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
2670 return DAG.getNode(ISD::MUL, SDLoc(N), VT, N1, N0);
2671 // fold (mul x, 0) -> 0
2672 if (N1IsConst && ConstValue1.isNullValue())
2673 return N1;
2674 // fold (mul x, 1) -> x
2675 if (N1IsConst && ConstValue1.isOneValue())
2676 return N0;
2677
2678 if (SDValue NewSel = foldBinOpIntoSelect(N))
2679 return NewSel;
2680
2681 // fold (mul x, -1) -> 0-x
2682 if (N1IsConst && ConstValue1.isAllOnesValue()) {
2683 SDLoc DL(N);
2684 return DAG.getNode(ISD::SUB, DL, VT,
2685 DAG.getConstant(0, DL, VT), N0);
2686 }
2687 // fold (mul x, (1 << c)) -> x << c
2688 if (isConstantOrConstantVector(N1, /*NoOpaques*/ true) &&
2689 DAG.isKnownToBeAPowerOfTwo(N1) &&
2690 (!VT.isVector() || Level <= AfterLegalizeVectorOps)) {
2691 SDLoc DL(N);
2692 SDValue LogBase2 = BuildLogBase2(N1, DL);
2693 AddToWorklist(LogBase2.getNode());
2694
2695 EVT ShiftVT = getShiftAmountTy(N0.getValueType());
2696 SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ShiftVT);
2697 AddToWorklist(Trunc.getNode());
2698 return DAG.getNode(ISD::SHL, DL, VT, N0, Trunc);
2699 }
2700 // fold (mul x, -(1 << c)) -> -(x << c) or (-x) << c
2701 if (N1IsConst && !N1IsOpaqueConst && (-ConstValue1).isPowerOf2()) {
2702 unsigned Log2Val = (-ConstValue1).logBase2();
2703 SDLoc DL(N);
2704 // FIXME: If the input is something that is easily negated (e.g. a
2705 // single-use add), we should put the negate there.
2706 return DAG.getNode(ISD::SUB, DL, VT,
2707 DAG.getConstant(0, DL, VT),
2708 DAG.getNode(ISD::SHL, DL, VT, N0,
2709 DAG.getConstant(Log2Val, DL,
2710 getShiftAmountTy(N0.getValueType()))));
2711 }
2712
2713 // (mul (shl X, c1), c2) -> (mul X, c2 << c1)
2714 if (N0.getOpcode() == ISD::SHL &&
2715 isConstantOrConstantVector(N1, /* NoOpaques */ true) &&
2716 isConstantOrConstantVector(N0.getOperand(1), /* NoOpaques */ true)) {
2717 SDValue C3 = DAG.getNode(ISD::SHL, SDLoc(N), VT, N1, N0.getOperand(1));
2718 if (isConstantOrConstantVector(C3))
2719 return DAG.getNode(ISD::MUL, SDLoc(N), VT, N0.getOperand(0), C3);
2720 }
2721
2722 // Change (mul (shl X, C), Y) -> (shl (mul X, Y), C) when the shift has one
2723 // use.
2724 {
2725 SDValue Sh(nullptr, 0), Y(nullptr, 0);
2726
2727 // Check for both (mul (shl X, C), Y) and (mul Y, (shl X, C)).
2728 if (N0.getOpcode() == ISD::SHL &&
2729 isConstantOrConstantVector(N0.getOperand(1)) &&
2730 N0.getNode()->hasOneUse()) {
2731 Sh = N0; Y = N1;
2732 } else if (N1.getOpcode() == ISD::SHL &&
2733 isConstantOrConstantVector(N1.getOperand(1)) &&
2734 N1.getNode()->hasOneUse()) {
2735 Sh = N1; Y = N0;
2736 }
2737
2738 if (Sh.getNode()) {
2739 SDValue Mul = DAG.getNode(ISD::MUL, SDLoc(N), VT, Sh.getOperand(0), Y);
2740 return DAG.getNode(ISD::SHL, SDLoc(N), VT, Mul, Sh.getOperand(1));
2741 }
2742 }
2743
2744 // fold (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2)
2745 if (DAG.isConstantIntBuildVectorOrConstantInt(N1) &&
2746 N0.getOpcode() == ISD::ADD &&
2747 DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1)) &&
2748 isMulAddWithConstProfitable(N, N0, N1))
2749 return DAG.getNode(ISD::ADD, SDLoc(N), VT,
2750 DAG.getNode(ISD::MUL, SDLoc(N0), VT,
2751 N0.getOperand(0), N1),
2752 DAG.getNode(ISD::MUL, SDLoc(N1), VT,
2753 N0.getOperand(1), N1));
2754
2755 // reassociate mul
2756 if (SDValue RMUL = ReassociateOps(ISD::MUL, SDLoc(N), N0, N1))
2757 return RMUL;
2758
2759 return SDValue();
2760}
2761
2762/// Return true if divmod libcall is available.
2763static bool isDivRemLibcallAvailable(SDNode *Node, bool isSigned,
2764 const TargetLowering &TLI) {
2765 RTLIB::Libcall LC;
2766 EVT NodeType = Node->getValueType(0);
2767 if (!NodeType.isSimple())
2768 return false;
2769 switch (NodeType.getSimpleVT().SimpleTy) {
2770 default: return false; // No libcall for vector types.
2771 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2772 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2773 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2774 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2775 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2776 }
2777
2778 return TLI.getLibcallName(LC) != nullptr;
2779}
2780
2781/// Issue divrem if both quotient and remainder are needed.
2782SDValue DAGCombiner::useDivRem(SDNode *Node) {
2783 if (Node->use_empty())
2784 return SDValue(); // This is a dead node, leave it alone.
2785
2786 unsigned Opcode = Node->getOpcode();
2787 bool isSigned = (Opcode == ISD::SDIV) || (Opcode == ISD::SREM);
2788 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
2789
2790 // DivMod lib calls can still work on non-legal types if using lib-calls.
2791 EVT VT = Node->getValueType(0);
2792 if (VT.isVector() || !VT.isInteger())
2793 return SDValue();
2794
2795 if (!TLI.isTypeLegal(VT) && !TLI.isOperationCustom(DivRemOpc, VT))
2796 return SDValue();
2797
2798 // If DIVREM is going to get expanded into a libcall,
2799 // but there is no libcall available, then don't combine.
2800 if (!TLI.isOperationLegalOrCustom(DivRemOpc, VT) &&
2801 !isDivRemLibcallAvailable(Node, isSigned, TLI))
2802 return SDValue();
2803
2804 // If div is legal, it's better to do the normal expansion
2805 unsigned OtherOpcode = 0;
2806 if ((Opcode == ISD::SDIV) || (Opcode == ISD::UDIV)) {
2807 OtherOpcode = isSigned ? ISD::SREM : ISD::UREM;
2808 if (TLI.isOperationLegalOrCustom(Opcode, VT))
2809 return SDValue();
2810 } else {
2811 OtherOpcode = isSigned ? ISD::SDIV : ISD::UDIV;
2812 if (TLI.isOperationLegalOrCustom(OtherOpcode, VT))
2813 return SDValue();
2814 }
2815
2816 SDValue Op0 = Node->getOperand(0);
2817 SDValue Op1 = Node->getOperand(1);
2818 SDValue combined;
2819 for (SDNode::use_iterator UI = Op0.getNode()->use_begin(),
2820 UE = Op0.getNode()->use_end(); UI != UE; ++UI) {
2821 SDNode *User = *UI;
2822 if (User == Node || User->use_empty())
2823 continue;
2824 // Convert the other matching node(s), too;
2825 // otherwise, the DIVREM may get target-legalized into something
2826 // target-specific that we won't be able to recognize.
2827 unsigned UserOpc = User->getOpcode();
2828 if ((UserOpc == Opcode || UserOpc == OtherOpcode || UserOpc == DivRemOpc) &&
2829 User->getOperand(0) == Op0 &&
2830 User->getOperand(1) == Op1) {
2831 if (!combined) {
2832 if (UserOpc == OtherOpcode) {
2833 SDVTList VTs = DAG.getVTList(VT, VT);
2834 combined = DAG.getNode(DivRemOpc, SDLoc(Node), VTs, Op0, Op1);
2835 } else if (UserOpc == DivRemOpc) {
2836 combined = SDValue(User, 0);
2837 } else {
2838 assert(UserOpc == Opcode)(static_cast <bool> (UserOpc == Opcode) ? void (0) : __assert_fail
("UserOpc == Opcode", "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2838, __extension__ __PRETTY_FUNCTION__))
;
2839 continue;
2840 }
2841 }
2842 if (UserOpc == ISD::SDIV || UserOpc == ISD::UDIV)
2843 CombineTo(User, combined);
2844 else if (UserOpc == ISD::SREM || UserOpc == ISD::UREM)
2845 CombineTo(User, combined.getValue(1));
2846 }
2847 }
2848 return combined;
2849}
2850
2851static SDValue simplifyDivRem(SDNode *N, SelectionDAG &DAG) {
2852 SDValue N0 = N->getOperand(0);
2853 SDValue N1 = N->getOperand(1);
2854 EVT VT = N->getValueType(0);
2855 SDLoc DL(N);
2856
2857 if (DAG.isUndef(N->getOpcode(), {N0, N1}))
2858 return DAG.getUNDEF(VT);
2859
2860 // undef / X -> 0
2861 // undef % X -> 0
2862 if (N0.isUndef())
2863 return DAG.getConstant(0, DL, VT);
2864
2865 return SDValue();
2866}
2867
2868SDValue DAGCombiner::visitSDIV(SDNode *N) {
2869 SDValue N0 = N->getOperand(0);
2870 SDValue N1 = N->getOperand(1);
2871 EVT VT = N->getValueType(0);
2872
2873 // fold vector ops
2874 if (VT.isVector())
2875 if (SDValue FoldedVOp = SimplifyVBinOp(N))
2876 return FoldedVOp;
2877
2878 SDLoc DL(N);
2879
2880 // fold (sdiv c1, c2) -> c1/c2
2881 ConstantSDNode *N0C = isConstOrConstSplat(N0);
2882 ConstantSDNode *N1C = isConstOrConstSplat(N1);
2883 if (N0C && N1C && !N0C->isOpaque() && !N1C->isOpaque())
2884 return DAG.FoldConstantArithmetic(ISD::SDIV, DL, VT, N0C, N1C);
2885 // fold (sdiv X, 1) -> X
2886 if (N1C && N1C->isOne())
2887 return N0;
2888 // fold (sdiv X, -1) -> 0-X
2889 if (N1C && N1C->isAllOnesValue())
2890 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), N0);
2891
2892 if (SDValue V = simplifyDivRem(N, DAG))
2893 return V;
2894
2895 if (SDValue NewSel = foldBinOpIntoSelect(N))
2896 return NewSel;
2897
2898 // If we know the sign bits of both operands are zero, strength reduce to a
2899 // udiv instead. Handles (X&15) /s 4 -> X&15 >> 2
2900 if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
2901 return DAG.getNode(ISD::UDIV, DL, N1.getValueType(), N0, N1);
2902
2903 // fold (sdiv X, pow2) -> simple ops after legalize
2904 // FIXME: We check for the exact bit here because the generic lowering gives
2905 // better results in that case. The target-specific lowering should learn how
2906 // to handle exact sdivs efficiently.
2907 if (N1C && !N1C->isNullValue() && !N1C->isOpaque() &&
2908 !N->getFlags().hasExact() && (N1C->getAPIntValue().isPowerOf2() ||
2909 (-N1C->getAPIntValue()).isPowerOf2())) {
2910 // Target-specific implementation of sdiv x, pow2.
2911 if (SDValue Res = BuildSDIVPow2(N))
2912 return Res;
2913
2914 unsigned lg2 = N1C->getAPIntValue().countTrailingZeros();
2915
2916 // Splat the sign bit into the register
2917 SDValue SGN =
2918 DAG.getNode(ISD::SRA, DL, VT, N0,
2919 DAG.getConstant(VT.getScalarSizeInBits() - 1, DL,
2920 getShiftAmountTy(N0.getValueType())));
2921 AddToWorklist(SGN.getNode());
2922
2923 // Add (N0 < 0) ? abs2 - 1 : 0;
2924 SDValue SRL =
2925 DAG.getNode(ISD::SRL, DL, VT, SGN,
2926 DAG.getConstant(VT.getScalarSizeInBits() - lg2, DL,
2927 getShiftAmountTy(SGN.getValueType())));
2928 SDValue ADD = DAG.getNode(ISD::ADD, DL, VT, N0, SRL);
2929 AddToWorklist(SRL.getNode());
2930 AddToWorklist(ADD.getNode()); // Divide by pow2
2931 SDValue SRA = DAG.getNode(ISD::SRA, DL, VT, ADD,
2932 DAG.getConstant(lg2, DL,
2933 getShiftAmountTy(ADD.getValueType())));
2934
2935 // If we're dividing by a positive value, we're done. Otherwise, we must
2936 // negate the result.
2937 if (N1C->getAPIntValue().isNonNegative())
2938 return SRA;
2939
2940 AddToWorklist(SRA.getNode());
2941 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), SRA);
2942 }
2943
2944 // If integer divide is expensive and we satisfy the requirements, emit an
2945 // alternate sequence. Targets may check function attributes for size/speed
2946 // trade-offs.
2947 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
2948 if (N1C && !TLI.isIntDivCheap(N->getValueType(0), Attr))
2949 if (SDValue Op = BuildSDIV(N))
2950 return Op;
2951
2952 // sdiv, srem -> sdivrem
2953 // If the divisor is constant, then return DIVREM only if isIntDivCheap() is
2954 // true. Otherwise, we break the simplification logic in visitREM().
2955 if (!N1C || TLI.isIntDivCheap(N->getValueType(0), Attr))
2956 if (SDValue DivRem = useDivRem(N))
2957 return DivRem;
2958
2959 return SDValue();
2960}
2961
2962SDValue DAGCombiner::visitUDIV(SDNode *N) {
2963 SDValue N0 = N->getOperand(0);
2964 SDValue N1 = N->getOperand(1);
2965 EVT VT = N->getValueType(0);
2966
2967 // fold vector ops
2968 if (VT.isVector())
2969 if (SDValue FoldedVOp = SimplifyVBinOp(N))
2970 return FoldedVOp;
2971
2972 SDLoc DL(N);
2973
2974 // fold (udiv c1, c2) -> c1/c2
2975 ConstantSDNode *N0C = isConstOrConstSplat(N0);
2976 ConstantSDNode *N1C = isConstOrConstSplat(N1);
2977 if (N0C && N1C)
2978 if (SDValue Folded = DAG.FoldConstantArithmetic(ISD::UDIV, DL, VT,
2979 N0C, N1C))
2980 return Folded;
2981
2982 if (SDValue V = simplifyDivRem(N, DAG))
2983 return V;
2984
2985 if (SDValue NewSel = foldBinOpIntoSelect(N))
2986 return NewSel;
2987
2988 // fold (udiv x, (1 << c)) -> x >>u c
2989 if (isConstantOrConstantVector(N1, /*NoOpaques*/ true) &&
2990 DAG.isKnownToBeAPowerOfTwo(N1)) {
2991 SDValue LogBase2 = BuildLogBase2(N1, DL);
2992 AddToWorklist(LogBase2.getNode());
2993
2994 EVT ShiftVT = getShiftAmountTy(N0.getValueType());
2995 SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ShiftVT);
2996 AddToWorklist(Trunc.getNode());
2997 return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc);
2998 }
2999
3000 // fold (udiv x, (shl c, y)) -> x >>u (log2(c)+y) iff c is power of 2
3001 if (N1.getOpcode() == ISD::SHL) {
3002 SDValue N10 = N1.getOperand(0);
3003 if (isConstantOrConstantVector(N10, /*NoOpaques*/ true) &&
3004 DAG.isKnownToBeAPowerOfTwo(N10)) {
3005 SDValue LogBase2 = BuildLogBase2(N10, DL);
3006 AddToWorklist(LogBase2.getNode());
3007
3008 EVT ADDVT = N1.getOperand(1).getValueType();
3009 SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ADDVT);
3010 AddToWorklist(Trunc.getNode());
3011 SDValue Add = DAG.getNode(ISD::ADD, DL, ADDVT, N1.getOperand(1), Trunc);
3012 AddToWorklist(Add.getNode());
3013 return DAG.getNode(ISD::SRL, DL, VT, N0, Add);
3014 }
3015 }
3016
3017 // fold (udiv x, c) -> alternate
3018 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
3019 if (N1C && !TLI.isIntDivCheap(N->getValueType(0), Attr))
3020 if (SDValue Op = BuildUDIV(N))
3021 return Op;
3022
3023 // sdiv, srem -> sdivrem
3024 // If the divisor is constant, then return DIVREM only if isIntDivCheap() is
3025 // true. Otherwise, we break the simplification logic in visitREM().
3026 if (!N1C || TLI.isIntDivCheap(N->getValueType(0), Attr))
3027 if (SDValue DivRem = useDivRem(N))
3028 return DivRem;
3029
3030 return SDValue();
3031}
3032
3033// handles ISD::SREM and ISD::UREM
3034SDValue DAGCombiner::visitREM(SDNode *N) {
3035 unsigned Opcode = N->getOpcode();
3036 SDValue N0 = N->getOperand(0);
3037 SDValue N1 = N->getOperand(1);
3038 EVT VT = N->getValueType(0);
3039 bool isSigned = (Opcode == ISD::SREM);
3040 SDLoc DL(N);
3041
3042 // fold (rem c1, c2) -> c1%c2
3043 ConstantSDNode *N0C = isConstOrConstSplat(N0);
3044 ConstantSDNode *N1C = isConstOrConstSplat(N1);
3045 if (N0C && N1C)
3046 if (SDValue Folded = DAG.FoldConstantArithmetic(Opcode, DL, VT, N0C, N1C))
3047 return Folded;
3048
3049 if (SDValue V = simplifyDivRem(N, DAG))
3050 return V;
3051
3052 if (SDValue NewSel = foldBinOpIntoSelect(N))
3053 return NewSel;
3054
3055 if (isSigned) {
3056 // If we know the sign bits of both operands are zero, strength reduce to a
3057 // urem instead. Handles (X & 0x0FFFFFFF) %s 16 -> X&15
3058 if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
3059 return DAG.getNode(ISD::UREM, DL, VT, N0, N1);
3060 } else {
3061 SDValue NegOne = DAG.getAllOnesConstant(DL, VT);
3062 if (DAG.isKnownToBeAPowerOfTwo(N1)) {
3063 // fold (urem x, pow2) -> (and x, pow2-1)
3064 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne);
3065 AddToWorklist(Add.getNode());
3066 return DAG.getNode(ISD::AND, DL, VT, N0, Add);
3067 }
3068 if (N1.getOpcode() == ISD::SHL &&
3069 DAG.isKnownToBeAPowerOfTwo(N1.getOperand(0))) {
3070 // fold (urem x, (shl pow2, y)) -> (and x, (add (shl pow2, y), -1))
3071 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne);
3072 AddToWorklist(Add.getNode());
3073 return DAG.getNode(ISD::AND, DL, VT, N0, Add);
3074 }
3075 }
3076
3077 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
3078
3079 // If X/C can be simplified by the division-by-constant logic, lower
3080 // X%C to the equivalent of X-X/C*C.
3081 // To avoid mangling nodes, this simplification requires that the combine()
3082 // call for the speculative DIV must not cause a DIVREM conversion. We guard
3083 // against this by skipping the simplification if isIntDivCheap(). When
3084 // div is not cheap, combine will not return a DIVREM. Regardless,
3085 // checking cheapness here makes sense since the simplification results in
3086 // fatter code.
3087 if (N1C && !N1C->isNullValue() && !TLI.isIntDivCheap(VT, Attr)) {
3088 unsigned DivOpcode = isSigned ? ISD::SDIV : ISD::UDIV;
3089 SDValue Div = DAG.getNode(DivOpcode, DL, VT, N0, N1);
3090 AddToWorklist(Div.getNode());
3091 SDValue OptimizedDiv = combine(Div.getNode());
3092 if (OptimizedDiv.getNode() && OptimizedDiv.getNode() != Div.getNode()) {
3093 assert((OptimizedDiv.getOpcode() != ISD::UDIVREM) &&(static_cast <bool> ((OptimizedDiv.getOpcode() != ISD::
UDIVREM) && (OptimizedDiv.getOpcode() != ISD::SDIVREM
)) ? void (0) : __assert_fail ("(OptimizedDiv.getOpcode() != ISD::UDIVREM) && (OptimizedDiv.getOpcode() != ISD::SDIVREM)"
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3094, __extension__ __PRETTY_FUNCTION__))
3094 (OptimizedDiv.getOpcode() != ISD::SDIVREM))(static_cast <bool> ((OptimizedDiv.getOpcode() != ISD::
UDIVREM) && (OptimizedDiv.getOpcode() != ISD::SDIVREM
)) ? void (0) : __assert_fail ("(OptimizedDiv.getOpcode() != ISD::UDIVREM) && (OptimizedDiv.getOpcode() != ISD::SDIVREM)"
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3094, __extension__ __PRETTY_FUNCTION__))
;
3095 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, OptimizedDiv, N1);
3096 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
3097 AddToWorklist(Mul.getNode());
3098 return Sub;
3099 }
3100 }
3101
3102 // sdiv, srem -> sdivrem
3103 if (SDValue DivRem = useDivRem(N))
3104 return DivRem.getValue(1);
3105
3106 return SDValue();
3107}
3108
3109SDValue DAGCombiner::visitMULHS(SDNode *N) {
3110 SDValue N0 = N->getOperand(0);
3111 SDValue N1 = N->getOperand(1);
3112 EVT VT = N->getValueType(0);
3113 SDLoc DL(N);
3114
3115 if (VT.isVector()) {
3116 // fold (mulhs x, 0) -> 0
3117 if (ISD::isBuildVectorAllZeros(N1.getNode()))
3118 return N1;
3119 if (ISD::isBuildVectorAllZeros(N0.getNode()))
3120 return N0;
3121 }
3122
3123 // fold (mulhs x, 0) -> 0
3124 if (isNullConstant(N1))
3125 return N1;
3126 // fold (mulhs x, 1) -> (sra x, size(x)-1)
3127 if (isOneConstant(N1))
3128 return DAG.getNode(ISD::SRA, DL, N0.getValueType(), N0,
3129 DAG.getConstant(N0.getValueSizeInBits() - 1, DL,
3130 getShiftAmountTy(N0.getValueType())));
3131
3132 // fold (mulhs x, undef) -> 0
3133 if (N0.isUndef() || N1.isUndef())
3134 return DAG.getConstant(0, DL, VT);
3135
3136 // If the type twice as wide is legal, transform the mulhs to a wider multiply
3137 // plus a shift.
3138 if (VT.isSimple() && !VT.isVector()) {
3139 MVT Simple = VT.getSimpleVT();
3140 unsigned SimpleSize = Simple.getSizeInBits();
3141 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
3142 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
3143 N0 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N0);
3144 N1 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N1);
3145 N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
3146 N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
3147 DAG.getConstant(SimpleSize, DL,
3148 getShiftAmountTy(N1.getValueType())));
3149 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
3150 }
3151 }
3152
3153 return SDValue();
3154}
3155
3156SDValue DAGCombiner::visitMULHU(SDNode *N) {
3157 SDValue N0 = N->getOperand(0);
3158 SDValue N1 = N->getOperand(1);
3159 EVT VT = N->getValueType(0);
3160 SDLoc DL(N);
3161
3162 if (VT.isVector()) {
3163 // fold (mulhu x, 0) -> 0
3164 if (ISD::isBuildVectorAllZeros(N1.getNode()))
3165 return N1;
3166 if (ISD::isBuildVectorAllZeros(N0.getNode()))
3167 return N0;
3168 }
3169
3170 // fold (mulhu x, 0) -> 0
3171 if (isNullConstant(N1))
3172 return N1;
3173 // fold (mulhu x, 1) -> 0
3174 if (isOneConstant(N1))
3175 return DAG.getConstant(0, DL, N0.getValueType());
3176 // fold (mulhu x, undef) -> 0
3177 if (N0.isUndef() || N1.isUndef())
3178 return DAG.getConstant(0, DL, VT);
3179
3180 // If the type twice as wide is legal, transform the mulhu to a wider multiply
3181 // plus a shift.
3182 if (VT.isSimple() && !VT.isVector()) {
3183 MVT Simple = VT.getSimpleVT();
3184 unsigned SimpleSize = Simple.getSizeInBits();
3185 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
3186 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
3187 N0 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N0);
3188 N1 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N1);
3189 N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
3190 N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
3191 DAG.getConstant(SimpleSize, DL,
3192 getShiftAmountTy(N1.getValueType())));
3193 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
3194 }
3195 }
3196
3197 return SDValue();
3198}
3199
3200/// Perform optimizations common to nodes that compute two values. LoOp and HiOp
3201/// give the opcodes for the two computations that are being performed. Return
3202/// true if a simplification was made.
3203SDValue DAGCombiner::SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
3204 unsigned HiOp) {
3205 // If the high half is not needed, just compute the low half.
3206 bool HiExists = N->hasAnyUseOfValue(1);
3207 if (!HiExists &&
3208 (!LegalOperations ||
3209 TLI.isOperationLegalOrCustom(LoOp, N->getValueType(0)))) {
3210 SDValue Res = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0), N->ops());
3211 return CombineTo(N, Res, Res);
3212 }
3213
3214 // If the low half is not needed, just compute the high half.
3215 bool LoExists = N->hasAnyUseOfValue(0);
3216 if (!LoExists &&
3217 (!LegalOperations ||
3218 TLI.isOperationLegal(HiOp, N->getValueType(1)))) {
3219 SDValue Res = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1), N->ops());
3220 return CombineTo(N, Res, Res);
3221 }
3222
3223 // If both halves are used, return as it is.
3224 if (LoExists && HiExists)
3225 return SDValue();
3226
3227 // If the two computed results can be simplified separately, separate them.
3228 if (LoExists) {
3229 SDValue Lo = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0), N->ops());
3230 AddToWorklist(Lo.getNode());
3231 SDValue LoOpt = combine(Lo.getNode());
3232 if (LoOpt.getNode() && LoOpt.getNode() != Lo.getNode() &&
3233 (!LegalOperations ||
3234 TLI.isOperationLegal(LoOpt.getOpcode(), LoOpt.getValueType())))
3235 return CombineTo(N, LoOpt, LoOpt);
3236 }
3237
3238 if (HiExists) {
3239 SDValue Hi = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1), N->ops());
3240 AddToWorklist(Hi.getNode());
3241 SDValue HiOpt = combine(Hi.getNode());
3242 if (HiOpt.getNode() && HiOpt != Hi &&
3243 (!LegalOperations ||
3244 TLI.isOperationLegal(HiOpt.getOpcode(), HiOpt.getValueType())))
3245 return CombineTo(N, HiOpt, HiOpt);
3246 }
3247
3248 return SDValue();
3249}
3250
3251SDValue DAGCombiner::visitSMUL_LOHI(SDNode *N) {
3252 if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHS))
3253 return Res;
3254
3255 EVT VT = N->getValueType(0);
3256 SDLoc DL(N);
3257
3258 // If the type is twice as wide is legal, transform the mulhu to a wider
3259 // multiply plus a shift.
3260 if (VT.isSimple() && !VT.isVector()) {
3261 MVT Simple = VT.getSimpleVT();
3262 unsigned SimpleSize = Simple.getSizeInBits();
3263 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
3264 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
3265 SDValue Lo = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(0));
3266 SDValue Hi = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(1));
3267 Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
3268 // Compute the high part as N1.
3269 Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
3270 DAG.getConstant(SimpleSize, DL,
3271 getShiftAmountTy(Lo.getValueType())));
3272 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
3273 // Compute the low part as N0.
3274 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
3275 return CombineTo(N, Lo, Hi);
3276 }
3277 }
3278
3279 return SDValue();
3280}
3281
3282SDValue DAGCombiner::visitUMUL_LOHI(SDNode *N) {
3283 if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHU))
3284 return Res;
3285
3286 EVT VT = N->getValueType(0);
3287 SDLoc DL(N);
3288
3289 // If the type is twice as wide is legal, transform the mulhu to a wider
3290 // multiply plus a shift.
3291 if (VT.isSimple() && !VT.isVector()) {
3292 MVT Simple = VT.getSimpleVT();
3293 unsigned SimpleSize = Simple.getSizeInBits();
3294 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
3295 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
3296 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(0));
3297 SDValue Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(1));
3298 Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
3299 // Compute the high part as N1.
3300 Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
3301 DAG.getConstant(SimpleSize, DL,
3302 getShiftAmountTy(Lo.getValueType())));
3303 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
3304 // Compute the low part as N0.
3305 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
3306 return CombineTo(N, Lo, Hi);
3307 }
3308 }
3309
3310 return SDValue();
3311}
3312
3313SDValue DAGCombiner::visitSMULO(SDNode *N) {
3314 // (smulo x, 2) -> (saddo x, x)
3315 if (ConstantSDNode *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1)))
3316 if (C2->getAPIntValue() == 2)
3317 return DAG.getNode(ISD::SADDO, SDLoc(N), N->getVTList(),
3318 N->getOperand(0), N->getOperand(0));
3319
3320 return SDValue();
3321}
3322
3323SDValue DAGCombiner::visitUMULO(SDNode *N) {
3324 // (umulo x, 2) -> (uaddo x, x)
3325 if (ConstantSDNode *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1)))
3326 if (C2->getAPIntValue() == 2)
3327 return DAG.getNode(ISD::UADDO, SDLoc(N), N->getVTList(),
3328 N->getOperand(0), N->getOperand(0));
3329
3330 return SDValue();
3331}
3332
3333SDValue DAGCombiner::visitIMINMAX(SDNode *N) {
3334 SDValue N0 = N->getOperand(0);
3335 SDValue N1 = N->getOperand(1);
3336 EVT VT = N0.getValueType();
3337
3338 // fold vector ops
3339 if (VT.isVector())
3340 if (SDValue FoldedVOp = SimplifyVBinOp(N))
3341 return FoldedVOp;
3342
3343 // fold operation with constant operands.
3344 ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
3345 ConstantSDNode *N1C = getAsNonOpaqueConstant(N1);
3346 if (N0C && N1C)
3347 return DAG.FoldConstantArithmetic(N->getOpcode(), SDLoc(N), VT, N0C, N1C);
3348
3349 // canonicalize constant to RHS
3350 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
3351 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
3352 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N1, N0);
3353
3354 // Is sign bits are zero, flip between UMIN/UMAX and SMIN/SMAX.
3355 // Only do this if the current op isn't legal and the flipped is.
3356 unsigned Opcode = N->getOpcode();
3357 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3358 if (!TLI.isOperationLegal(Opcode, VT) &&
3359 (N0.isUndef() || DAG.SignBitIsZero(N0)) &&
3360 (N1.isUndef() || DAG.SignBitIsZero(N1))) {
3361 unsigned AltOpcode;
3362 switch (Opcode) {
3363 case ISD::SMIN: AltOpcode = ISD::UMIN; break;
3364 case ISD::SMAX: AltOpcode = ISD::UMAX; break;
3365 case ISD::UMIN: AltOpcode = ISD::SMIN; break;
3366 case ISD::UMAX: AltOpcode = ISD::SMAX; break;
3367 default: llvm_unreachable("Unknown MINMAX opcode")::llvm::llvm_unreachable_internal("Unknown MINMAX opcode", "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3367)
;
3368 }
3369 if (TLI.isOperationLegal(AltOpcode, VT))
3370 return DAG.getNode(AltOpcode, SDLoc(N), VT, N0, N1);
3371 }
3372
3373 return SDValue();
3374}
3375
3376/// If this is a binary operator with two operands of the same opcode, try to
3377/// simplify it.
3378SDValue DAGCombiner::SimplifyBinOpWithSameOpcodeHands(SDNode *N) {
3379 SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
3380 EVT VT = N0.getValueType();
3381 assert(N0.getOpcode() == N1.getOpcode() && "Bad input!")(static_cast <bool> (N0.getOpcode() == N1.getOpcode() &&
"Bad input!") ? void (0) : __assert_fail ("N0.getOpcode() == N1.getOpcode() && \"Bad input!\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3381, __extension__ __PRETTY_FUNCTION__))
;
3382
3383 // Bail early if none of these transforms apply.
3384 if (N0.getNumOperands() == 0) return SDValue();
3385
3386 // For each of OP in AND/OR/XOR:
3387 // fold (OP (zext x), (zext y)) -> (zext (OP x, y))
3388 // fold (OP (sext x), (sext y)) -> (sext (OP x, y))
3389 // fold (OP (aext x), (aext y)) -> (aext (OP x, y))
3390 // fold (OP (bswap x), (bswap y)) -> (bswap (OP x, y))
3391 // fold (OP (trunc x), (trunc y)) -> (trunc (OP x, y)) (if trunc isn't free)
3392 //
3393 // do not sink logical op inside of a vector extend, since it may combine
3394 // into a vsetcc.
3395 EVT Op0VT = N0.getOperand(0).getValueType();
3396 if ((N0.getOpcode() == ISD::ZERO_EXTEND ||
3397 N0.getOpcode() == ISD::SIGN_EXTEND ||
3398 N0.getOpcode() == ISD::BSWAP ||
3399 // Avoid infinite looping with PromoteIntBinOp.
3400 (N0.getOpcode() == ISD::ANY_EXTEND &&
3401 (!LegalTypes || TLI.isTypeDesirableForOp(N->getOpcode(), Op0VT))) ||
3402 (N0.getOpcode() == ISD::TRUNCATE &&
3403 (!TLI.isZExtFree(VT, Op0VT) ||
3404 !TLI.isTruncateFree(Op0VT, VT)) &&
3405 TLI.isTypeLegal(Op0VT))) &&
3406 !VT.isVector() &&
3407 Op0VT == N1.getOperand(0).getValueType() &&
3408 (!LegalOperations || TLI.isOperationLegal(N->getOpcode(), Op0VT))) {
3409 SDValue ORNode = DAG.getNode(N->getOpcode(), SDLoc(N0),
3410 N0.getOperand(0).getValueType(),
3411 N0.getOperand(0), N1.getOperand(0));
3412 AddToWorklist(ORNode.getNode());
3413 return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, ORNode);
3414 }
3415
3416 // For each of OP in SHL/SRL/SRA/AND...
3417 // fold (and (OP x, z), (OP y, z)) -> (OP (and x, y), z)
3418 // fold (or (OP x, z), (OP y, z)) -> (OP (or x, y), z)
3419 // fold (xor (OP x, z), (OP y, z)) -> (OP (xor x, y), z)
3420 if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL ||
3421 N0.getOpcode() == ISD::SRA || N0.getOpcode() == ISD::AND) &&
3422 N0.getOperand(1) == N1.getOperand(1)) {
3423 SDValue ORNode = DAG.getNode(N->getOpcode(), SDLoc(N0),
3424 N0.getOperand(0).getValueType(),
3425 N0.getOperand(0), N1.getOperand(0));
3426 AddToWorklist(ORNode.getNode());
3427 return DAG.getNode(N0.getOpcode(), SDLoc(N), VT,
3428 ORNode, N0.getOperand(1));
3429 }
3430
3431 // Simplify xor/and/or (bitcast(A), bitcast(B)) -> bitcast(op (A,B))
3432 // Only perform this optimization up until type legalization, before
3433 // LegalizeVectorOprs. LegalizeVectorOprs promotes vector operations by
3434 // adding bitcasts. For example (xor v4i32) is promoted to (v2i64), and
3435 // we don't want to undo this promotion.
3436 // We also handle SCALAR_TO_VECTOR because xor/or/and operations are cheaper
3437 // on scalars.
3438 if ((N0.getOpcode() == ISD::BITCAST ||
3439 N0.getOpcode() == ISD::SCALAR_TO_VECTOR) &&
3440 Level <= AfterLegalizeTypes) {
3441 SDValue In0 = N0.getOperand(0);
3442 SDValue In1 = N1.getOperand(0);
3443 EVT In0Ty = In0.getValueType();
3444 EVT In1Ty = In1.getValueType();
3445 SDLoc DL(N);
3446 // If both incoming values are integers, and the original types are the
3447 // same.
3448 if (In0Ty.isInteger() && In1Ty.isInteger() && In0Ty == In1Ty) {
3449 SDValue Op = DAG.getNode(N->getOpcode(), DL, In0Ty, In0, In1);
3450 SDValue BC = DAG.getNode(N0.getOpcode(), DL, VT, Op);
3451 AddToWorklist(Op.getNode());
3452 return BC;
3453 }
3454 }
3455
3456 // Xor/and/or are indifferent to the swizzle operation (shuffle of one value).
3457 // Simplify xor/and/or (shuff(A), shuff(B)) -> shuff(op (A,B))
3458 // If both shuffles use the same mask, and both shuffle within a single
3459 // vector, then it is worthwhile to move the swizzle after the operation.
3460 // The type-legalizer generates this pattern when loading illegal
3461 // vector types from memory. In many cases this allows additional shuffle
3462 // optimizations.
3463 // There are other cases where moving the shuffle after the xor/and/or
3464 // is profitable even if shuffles don't perform a swizzle.
3465 // If both shuffles use the same mask, and both shuffles have the same first
3466 // or second operand, then it might still be profitable to move the shuffle
3467 // after the xor/and/or operation.
3468 if (N0.getOpcode() == ISD::VECTOR_SHUFFLE && Level < AfterLegalizeDAG) {
3469 ShuffleVectorSDNode *SVN0 = cast<ShuffleVectorSDNode>(N0);
3470 ShuffleVectorSDNode *SVN1 = cast<ShuffleVectorSDNode>(N1);
3471
3472 assert(N0.getOperand(0).getValueType() == N1.getOperand(0).getValueType() &&(static_cast <bool> (N0.getOperand(0).getValueType() ==
N1.getOperand(0).getValueType() && "Inputs to shuffles are not the same type"
) ? void (0) : __assert_fail ("N0.getOperand(0).getValueType() == N1.getOperand(0).getValueType() && \"Inputs to shuffles are not the same type\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3473, __extension__ __PRETTY_FUNCTION__))
3473 "Inputs to shuffles are not the same type")(static_cast <bool> (N0.getOperand(0).getValueType() ==
N1.getOperand(0).getValueType() && "Inputs to shuffles are not the same type"
) ? void (0) : __assert_fail ("N0.getOperand(0).getValueType() == N1.getOperand(0).getValueType() && \"Inputs to shuffles are not the same type\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3473, __extension__ __PRETTY_FUNCTION__))
;
3474
3475 // Check that both shuffles use the same mask. The masks are known to be of
3476 // the same length because the result vector type is the same.
3477 // Check also that shuffles have only one use to avoid introducing extra
3478 // instructions.
3479 if (SVN0->hasOneUse() && SVN1->hasOneUse() &&
3480 SVN0->getMask().equals(SVN1->getMask())) {
3481 SDValue ShOp = N0->getOperand(1);
3482
3483 // Don't try to fold this node if it requires introducing a
3484 // build vector of all zeros that might be illegal at this stage.
3485 if (N->getOpcode() == ISD::XOR && !ShOp.isUndef()) {
3486 if (!LegalTypes)
3487 ShOp = DAG.getConstant(0, SDLoc(N), VT);
3488 else
3489 ShOp = SDValue();
3490 }
3491
3492 // (AND (shuf (A, C), shuf (B, C)) -> shuf (AND (A, B), C)
3493 // (OR (shuf (A, C), shuf (B, C)) -> shuf (OR (A, B), C)
3494 // (XOR (shuf (A, C), shuf (B, C)) -> shuf (XOR (A, B), V_0)
3495 if (N0.getOperand(1) == N1.getOperand(1) && ShOp.getNode()) {
3496 SDValue NewNode = DAG.getNode(N->getOpcode(), SDLoc(N), VT,
3497 N0->getOperand(0), N1->getOperand(0));
3498 AddToWorklist(NewNode.getNode());
3499 return DAG.getVectorShuffle(VT, SDLoc(N), NewNode, ShOp,
3500 SVN0->getMask());
3501 }
3502
3503 // Don't try to fold this node if it requires introducing a
3504 // build vector of all zeros that might be illegal at this stage.
3505 ShOp = N0->getOperand(0);
3506 if (N->getOpcode() == ISD::XOR && !ShOp.isUndef()) {
3507 if (!LegalTypes)
3508 ShOp = DAG.getConstant(0, SDLoc(N), VT);
3509 else
3510 ShOp = SDValue();
3511 }
3512
3513 // (AND (shuf (C, A), shuf (C, B)) -> shuf (C, AND (A, B))
3514 // (OR (shuf (C, A), shuf (C, B)) -> shuf (C, OR (A, B))
3515 // (XOR (shuf (C, A), shuf (C, B)) -> shuf (V_0, XOR (A, B))
3516 if (N0->getOperand(0) == N1->getOperand(0) && ShOp.getNode()) {
3517 SDValue NewNode = DAG.getNode(N->getOpcode(), SDLoc(N), VT,
3518 N0->getOperand(1), N1->getOperand(1));
3519 AddToWorklist(NewNode.getNode());
3520 return DAG.getVectorShuffle(VT, SDLoc(N), ShOp, NewNode,
3521 SVN0->getMask());
3522 }
3523 }
3524 }
3525
3526 return SDValue();
3527}
3528
3529/// Try to make (and/or setcc (LL, LR), setcc (RL, RR)) more efficient.
3530SDValue DAGCombiner::foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1,
3531 const SDLoc &DL) {
3532 SDValue LL, LR, RL, RR, N0CC, N1CC;
3533 if (!isSetCCEquivalent(N0, LL, LR, N0CC) ||
3534 !isSetCCEquivalent(N1, RL, RR, N1CC))
3535 return SDValue();
3536
3537 assert(N0.getValueType() == N1.getValueType() &&(static_cast <bool> (N0.getValueType() == N1.getValueType
() && "Unexpected operand types for bitwise logic op"
) ? void (0) : __assert_fail ("N0.getValueType() == N1.getValueType() && \"Unexpected operand types for bitwise logic op\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3538, __extension__ __PRETTY_FUNCTION__))
3538 "Unexpected operand types for bitwise logic op")(static_cast <bool> (N0.getValueType() == N1.getValueType
() && "Unexpected operand types for bitwise logic op"
) ? void (0) : __assert_fail ("N0.getValueType() == N1.getValueType() && \"Unexpected operand types for bitwise logic op\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3538, __extension__ __PRETTY_FUNCTION__))
;
3539 assert(LL.getValueType() == LR.getValueType() &&(static_cast <bool> (LL.getValueType() == LR.getValueType
() && RL.getValueType() == RR.getValueType() &&
"Unexpected operand types for setcc") ? void (0) : __assert_fail
("LL.getValueType() == LR.getValueType() && RL.getValueType() == RR.getValueType() && \"Unexpected operand types for setcc\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3541, __extension__ __PRETTY_FUNCTION__))
3540 RL.getValueType() == RR.getValueType() &&(static_cast <bool> (LL.getValueType() == LR.getValueType
() && RL.getValueType() == RR.getValueType() &&
"Unexpected operand types for setcc") ? void (0) : __assert_fail
("LL.getValueType() == LR.getValueType() && RL.getValueType() == RR.getValueType() && \"Unexpected operand types for setcc\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3541, __extension__ __PRETTY_FUNCTION__))
3541 "Unexpected operand types for setcc")(static_cast <bool> (LL.getValueType() == LR.getValueType
() && RL.getValueType() == RR.getValueType() &&
"Unexpected operand types for setcc") ? void (0) : __assert_fail
("LL.getValueType() == LR.getValueType() && RL.getValueType() == RR.getValueType() && \"Unexpected operand types for setcc\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3541, __extension__ __PRETTY_FUNCTION__))
;
3542
3543 // If we're here post-legalization or the logic op type is not i1, the logic
3544 // op type must match a setcc result type. Also, all folds require new
3545 // operations on the left and right operands, so those types must match.
3546 EVT VT = N0.getValueType();
3547 EVT OpVT = LL.getValueType();
3548 if (LegalOperations || VT.getScalarType() != MVT::i1)
3549 if (VT != getSetCCResultType(OpVT))
3550 return SDValue();
3551 if (OpVT != RL.getValueType())
3552 return SDValue();
3553
3554 ISD::CondCode CC0 = cast<CondCodeSDNode>(N0CC)->get();
3555 ISD::CondCode CC1 = cast<CondCodeSDNode>(N1CC)->get();
3556 bool IsInteger = OpVT.isInteger();
3557 if (LR == RR && CC0 == CC1 && IsInteger) {
3558 bool IsZero = isNullConstantOrNullSplatConstant(LR);
3559 bool IsNeg1 = isAllOnesConstantOrAllOnesSplatConstant(LR);
3560
3561 // All bits clear?
3562 bool AndEqZero = IsAnd && CC1 == ISD::SETEQ && IsZero;
3563 // All sign bits clear?
3564 bool AndGtNeg1 = IsAnd && CC1 == ISD::SETGT && IsNeg1;
3565 // Any bits set?
3566 bool OrNeZero = !IsAnd && CC1 == ISD::SETNE && IsZero;
3567 // Any sign bits set?
3568 bool OrLtZero = !IsAnd && CC1 == ISD::SETLT && IsZero;
3569
3570 // (and (seteq X, 0), (seteq Y, 0)) --> (seteq (or X, Y), 0)
3571 // (and (setgt X, -1), (setgt Y, -1)) --> (setgt (or X, Y), -1)
3572 // (or (setne X, 0), (setne Y, 0)) --> (setne (or X, Y), 0)
3573 // (or (setlt X, 0), (setlt Y, 0)) --> (setlt (or X, Y), 0)
3574 if (AndEqZero || AndGtNeg1 || OrNeZero || OrLtZero) {
3575 SDValue Or = DAG.getNode(ISD::OR, SDLoc(N0), OpVT, LL, RL);
3576 AddToWorklist(Or.getNode());
3577 return DAG.getSetCC(DL, VT, Or, LR, CC1);
3578 }
3579
3580 // All bits set?
3581 bool AndEqNeg1 = IsAnd && CC1 == ISD::SETEQ && IsNeg1;
3582 // All sign bits set?
3583 bool AndLtZero = IsAnd && CC1 == ISD::SETLT && IsZero;
3584 // Any bits clear?
3585 bool OrNeNeg1 = !IsAnd && CC1 == ISD::SETNE && IsNeg1;
3586 // Any sign bits clear?
3587 bool OrGtNeg1 = !IsAnd && CC1 == ISD::SETGT && IsNeg1;
3588
3589 // (and (seteq X, -1), (seteq Y, -1)) --> (seteq (and X, Y), -1)
3590 // (and (setlt X, 0), (setlt Y, 0)) --> (setlt (and X, Y), 0)
3591 // (or (setne X, -1), (setne Y, -1)) --> (setne (and X, Y), -1)
3592 // (or (setgt X, -1), (setgt Y -1)) --> (setgt (and X, Y), -1)
3593 if (AndEqNeg1 || AndLtZero || OrNeNeg1 || OrGtNeg1) {
3594 SDValue And = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, LL, RL);
3595 AddToWorklist(And.getNode());
3596 return DAG.getSetCC(DL, VT, And, LR, CC1);
3597 }
3598 }
3599
3600 // TODO: What is the 'or' equivalent of this fold?
3601 // (and (setne X, 0), (setne X, -1)) --> (setuge (add X, 1), 2)
3602 if (IsAnd && LL == RL && CC0 == CC1 && OpVT.getScalarSizeInBits() > 1 &&
3603 IsInteger && CC0 == ISD::SETNE &&
3604 ((isNullConstant(LR) && isAllOnesConstant(RR)) ||
3605 (isAllOnesConstant(LR) && isNullConstant(RR)))) {
3606 SDValue One = DAG.getConstant(1, DL, OpVT);
3607 SDValue Two = DAG.getConstant(2, DL, OpVT);
3608 SDValue Add = DAG.getNode(ISD::ADD, SDLoc(N0), OpVT, LL, One);
3609 AddToWorklist(Add.getNode());
3610 return DAG.getSetCC(DL, VT, Add, Two, ISD::SETUGE);
3611 }
3612
3613 // Try more general transforms if the predicates match and the only user of
3614 // the compares is the 'and' or 'or'.
3615 if (IsInteger && TLI.convertSetCCLogicToBitwiseLogic(OpVT) && CC0 == CC1 &&
3616 N0.hasOneUse() && N1.hasOneUse()) {
3617 // and (seteq A, B), (seteq C, D) --> seteq (or (xor A, B), (xor C, D)), 0
3618 // or (setne A, B), (setne C, D) --> setne (or (xor A, B), (xor C, D)), 0
3619 if ((IsAnd && CC1 == ISD::SETEQ) || (!IsAnd && CC1 == ISD::SETNE)) {
3620 SDValue XorL = DAG.getNode(ISD::XOR, SDLoc(N0), OpVT, LL, LR);
3621 SDValue XorR = DAG.getNode(ISD::XOR, SDLoc(N1), OpVT, RL, RR);
3622 SDValue Or = DAG.getNode(ISD::OR, DL, OpVT, XorL, XorR);
3623 SDValue Zero = DAG.getConstant(0, DL, OpVT);
3624 return DAG.getSetCC(DL, VT, Or, Zero, CC1);
3625 }
3626 }
3627
3628 // Canonicalize equivalent operands to LL == RL.
3629 if (LL == RR && LR == RL) {
3630 CC1 = ISD::getSetCCSwappedOperands(CC1);
3631 std::swap(RL, RR);
3632 }
3633
3634 // (and (setcc X, Y, CC0), (setcc X, Y, CC1)) --> (setcc X, Y, NewCC)
3635 // (or (setcc X, Y, CC0), (setcc X, Y, CC1)) --> (setcc X, Y, NewCC)
3636 if (LL == RL && LR == RR) {
3637 ISD::CondCode NewCC = IsAnd ? ISD::getSetCCAndOperation(CC0, CC1, IsInteger)
3638 : ISD::getSetCCOrOperation(CC0, CC1, IsInteger);
3639 if (NewCC != ISD::SETCC_INVALID &&
3640 (!LegalOperations ||
3641 (TLI.isCondCodeLegal(NewCC, LL.getSimpleValueType()) &&
3642 TLI.isOperationLegal(ISD::SETCC, OpVT))))
3643 return DAG.getSetCC(DL, VT, LL, LR, NewCC);
3644 }
3645
3646 return SDValue();
3647}
3648
3649/// This contains all DAGCombine rules which reduce two values combined by
3650/// an And operation to a single value. This makes them reusable in the context
3651/// of visitSELECT(). Rules involving constants are not included as
3652/// visitSELECT() already handles those cases.
3653SDValue DAGCombiner::visitANDLike(SDValue N0, SDValue N1, SDNode *N) {
3654 EVT VT = N1.getValueType();
3655 SDLoc DL(N);
3656
3657 // fold (and x, undef) -> 0
3658 if (N0.isUndef() || N1.isUndef())
3659 return DAG.getConstant(0, DL, VT);
3660
3661 if (SDValue V = foldLogicOfSetCCs(true, N0, N1, DL))
3662 return V;
3663
3664 if (N0.getOpcode() == ISD::ADD && N1.getOpcode() == ISD::SRL &&
3665 VT.getSizeInBits() <= 64) {
3666 if (ConstantSDNode *ADDI = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
3667 if (ConstantSDNode *SRLI = dyn_cast<ConstantSDNode>(N1.getOperand(1))) {
3668 // Look for (and (add x, c1), (lshr y, c2)). If C1 wasn't a legal
3669 // immediate for an add, but it is legal if its top c2 bits are set,
3670 // transform the ADD so the immediate doesn't need to be materialized
3671 // in a register.
3672 APInt ADDC = ADDI->getAPIntValue();
3673 APInt SRLC = SRLI->getAPIntValue();
3674 if (ADDC.getMinSignedBits() <= 64 &&
3675 SRLC.ult(VT.getSizeInBits()) &&
3676 !TLI.isLegalAddImmediate(ADDC.getSExtValue())) {
3677 APInt Mask = APInt::getHighBitsSet(VT.getSizeInBits(),
3678 SRLC.getZExtValue());
3679 if (DAG.MaskedValueIsZero(N0.getOperand(1), Mask)) {
3680 ADDC |= Mask;
3681 if (TLI.isLegalAddImmediate(ADDC.getSExtValue())) {
3682 SDLoc DL0(N0);
3683 SDValue NewAdd =
3684 DAG.getNode(ISD::ADD, DL0, VT,
3685 N0.getOperand(0), DAG.getConstant(ADDC, DL, VT));
3686 CombineTo(N0.getNode(), NewAdd);
3687 // Return N so it doesn't get rechecked!
3688 return SDValue(N, 0);
3689 }
3690 }
3691 }
3692 }
3693 }
3694 }
3695
3696 // Reduce bit extract of low half of an integer to the narrower type.
3697 // (and (srl i64:x, K), KMask) ->
3698 // (i64 zero_extend (and (srl (i32 (trunc i64:x)), K)), KMask)
3699 if (N0.getOpcode() == ISD::SRL && N0.hasOneUse()) {
3700 if (ConstantSDNode *CAnd = dyn_cast<ConstantSDNode>(N1)) {
3701 if (ConstantSDNode *CShift = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
3702 unsigned Size = VT.getSizeInBits();
3703 const APInt &AndMask = CAnd->getAPIntValue();
3704 unsigned ShiftBits = CShift->getZExtValue();
3705
3706 // Bail out, this node will probably disappear anyway.
3707 if (ShiftBits == 0)
3708 return SDValue();
3709
3710 unsigned MaskBits = AndMask.countTrailingOnes();
3711 EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), Size / 2);
3712
3713 if (AndMask.isMask() &&
3714 // Required bits must not span the two halves of the integer and
3715 // must fit in the half size type.
3716 (ShiftBits + MaskBits <= Size / 2) &&
3717 TLI.isNarrowingProfitable(VT, HalfVT) &&
3718 TLI.isTypeDesirableForOp(ISD::AND, HalfVT) &&
3719 TLI.isTypeDesirableForOp(ISD::SRL, HalfVT) &&
3720 TLI.isTruncateFree(VT, HalfVT) &&
3721 TLI.isZExtFree(HalfVT, VT)) {
3722 // The isNarrowingProfitable is to avoid regressions on PPC and
3723 // AArch64 which match a few 64-bit bit insert / bit extract patterns
3724 // on downstream users of this. Those patterns could probably be
3725 // extended to handle extensions mixed in.
3726
3727 SDValue SL(N0);
3728 assert(MaskBits <= Size)(static_cast <bool> (MaskBits <= Size) ? void (0) : __assert_fail
("MaskBits <= Size", "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3728, __extension__ __PRETTY_FUNCTION__))
;
3729
3730 // Extracting the highest bit of the low half.
3731 EVT ShiftVT = TLI.getShiftAmountTy(HalfVT, DAG.getDataLayout());
3732 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, HalfVT,
3733 N0.getOperand(0));
3734
3735 SDValue NewMask = DAG.getConstant(AndMask.trunc(Size / 2), SL, HalfVT);
3736 SDValue ShiftK = DAG.getConstant(ShiftBits, SL, ShiftVT);
3737 SDValue Shift = DAG.getNode(ISD::SRL, SL, HalfVT, Trunc, ShiftK);
3738 SDValue And = DAG.getNode(ISD::AND, SL, HalfVT, Shift, NewMask);
3739 return DAG.getNode(ISD::ZERO_EXTEND, SL, VT, And);
3740 }
3741 }
3742 }
3743 }
3744
3745 return SDValue();
3746}
3747
3748bool DAGCombiner::isAndLoadExtLoad(ConstantSDNode *AndC, LoadSDNode *LoadN,
3749 EVT LoadResultTy, EVT &ExtVT) {
3750 if (!AndC->getAPIntValue().isMask())
3751 return false;
3752
3753 unsigned ActiveBits = AndC->getAPIntValue().countTrailingOnes();
3754
3755 ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
3756 EVT LoadedVT = LoadN->getMemoryVT();
3757
3758 if (ExtVT == LoadedVT &&
3759 (!LegalOperations ||
3760 TLI.isLoadExtLegal(ISD::ZEXTLOAD, LoadResultTy, ExtVT))) {
3761 // ZEXTLOAD will match without needing to change the size of the value being
3762 // loaded.
3763 return true;
3764 }
3765
3766 // Do not change the width of a volatile load.
3767 if (LoadN->isVolatile())
3768 return false;
3769
3770 // Do not generate loads of non-round integer types since these can
3771 // be expensive (and would be wrong if the type is not byte sized).
3772 if (!LoadedVT.bitsGT(ExtVT) || !ExtVT.isRound())
3773 return false;
3774
3775 if (LegalOperations &&
3776 !TLI.isLoadExtLegal(ISD::ZEXTLOAD, LoadResultTy, ExtVT))
3777 return false;
3778
3779 if (!TLI.shouldReduceLoadWidth(LoadN, ISD::ZEXTLOAD, ExtVT))
3780 return false;
3781
3782 return true;
3783}
3784
3785bool DAGCombiner::isLegalNarrowLoad(LoadSDNode *LoadN, ISD::LoadExtType ExtType,
3786 EVT &ExtVT, unsigned ShAmt) {
3787 // Don't transform one with multiple uses, this would require adding a new
3788 // load.
3789 if (!SDValue(LoadN, 0).hasOneUse())
3790 return false;
3791
3792 if (LegalOperations &&
3793 !TLI.isLoadExtLegal(ExtType, LoadN->getValueType(0), ExtVT))
3794 return false;
3795
3796 // Do not generate loads of non-round integer types since these can
3797 // be expensive (and would be wrong if the type is not byte sized).
3798 if (!ExtVT.isRound())
3799 return false;
3800
3801 // Don't change the width of a volatile load.
3802 if (LoadN->isVolatile())
3803 return false;
3804
3805 // Verify that we are actually reducing a load width here.
3806 if (LoadN->getMemoryVT().getSizeInBits() < ExtVT.getSizeInBits())
3807 return false;
3808
3809 // For the transform to be legal, the load must produce only two values
3810 // (the value loaded and the chain). Don't transform a pre-increment
3811 // load, for example, which produces an extra value. Otherwise the
3812 // transformation is not equivalent, and the downstream logic to replace
3813 // uses gets things wrong.
3814 if (LoadN->getNumValues() > 2)
3815 return false;
3816
3817 // If the load that we're shrinking is an extload and we're not just
3818 // discarding the extension we can't simply shrink the load. Bail.
3819 // TODO: It would be possible to merge the extensions in some cases.
3820 if (LoadN->getExtensionType() != ISD::NON_EXTLOAD &&
3821 LoadN->getMemoryVT().getSizeInBits() < ExtVT.getSizeInBits() + ShAmt)
3822 return false;
3823
3824 if (!TLI.shouldReduceLoadWidth(LoadN, ExtType, ExtVT))
3825 return false;
3826
3827 // It's not possible to generate a constant of extended or untyped type.
3828 EVT PtrType = LoadN->getOperand(1).getValueType();
3829 if (PtrType == MVT::Untyped || PtrType.isExtended())
3830 return false;
3831
3832 return true;
3833}
3834
3835bool DAGCombiner::SearchForAndLoads(SDNode *N,
3836 SmallPtrSetImpl<LoadSDNode*> &Loads,
3837 SmallPtrSetImpl<SDNode*> &NodesWithConsts,
3838 ConstantSDNode *Mask,
3839 SDNode *&NodeToMask) {
3840 // Recursively search for the operands, looking for loads which can be
3841 // narrowed.
3842 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) {
3843 SDValue Op = N->getOperand(i);
3844
3845 if (Op.getValueType().isVector())
3846 return false;
3847
3848 // Some constants may need fixing up later if they are too large.
3849 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
3850 if ((N->getOpcode() == ISD::OR || N->getOpcode() == ISD::XOR) &&
3851 (Mask->getAPIntValue() & C->getAPIntValue()) != C->getAPIntValue())
3852 NodesWithConsts.insert(N);
3853 continue;
3854 }
3855
3856 if (!Op.hasOneUse())
3857 return false;
3858
3859 switch(Op.getOpcode()) {
3860 case ISD::LOAD: {
3861 auto *Load = cast<LoadSDNode>(Op);
3862 EVT ExtVT;
3863 if (isAndLoadExtLoad(Mask, Load, Load->getValueType(0), ExtVT) &&
3864 isLegalNarrowLoad(Load, ISD::ZEXTLOAD, ExtVT)) {
3865
3866 // ZEXTLOAD is already small enough.
3867 if (Load->getExtensionType() == ISD::ZEXTLOAD &&
3868 ExtVT.bitsGE(Load->getMemoryVT()))
3869 continue;
3870
3871 // Use LE to convert equal sized loads to zext.
3872 if (ExtVT.bitsLE(Load->getMemoryVT()))
3873 Loads.insert(Load);
3874
3875 continue;
3876 }
3877 return false;
3878 }
3879 case ISD::ZERO_EXTEND:
3880 case ISD::AssertZext: {
3881 unsigned ActiveBits = Mask->getAPIntValue().countTrailingOnes();
3882 EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
3883 EVT VT = Op.getOpcode() == ISD::AssertZext ?
3884 cast<VTSDNode>(Op.getOperand(1))->getVT() :
3885 Op.getOperand(0).getValueType();
3886
3887 // We can accept extending nodes if the mask is wider or an equal
3888 // width to the original type.
3889 if (ExtVT.bitsGE(VT))
3890 continue;
3891 break;
3892 }
3893 case ISD::OR:
3894 case ISD::XOR:
3895 case ISD::AND:
3896 if (!SearchForAndLoads(Op.getNode(), Loads, NodesWithConsts, Mask,
3897 NodeToMask))
3898 return false;
3899 continue;
3900 }
3901
3902 // Allow one node which will masked along with any loads found.
3903 if (NodeToMask)
3904 return false;
3905 NodeToMask = Op.getNode();
3906 }
3907 return true;
3908}
3909
3910bool DAGCombiner::BackwardsPropagateMask(SDNode *N, SelectionDAG &DAG) {
3911 auto *Mask = dyn_cast<ConstantSDNode>(N->getOperand(1));
3912 if (!Mask)
3913 return false;
3914
3915 if (!Mask->getAPIntValue().isMask())
3916 return false;
3917
3918 // No need to do anything if the and directly uses a load.
3919 if (isa<LoadSDNode>(N->getOperand(0)))
3920 return false;
3921
3922 SmallPtrSet<LoadSDNode*, 8> Loads;
3923 SmallPtrSet<SDNode*, 2> NodesWithConsts;
3924 SDNode *FixupNode = nullptr;
3925 if (SearchForAndLoads(N, Loads, NodesWithConsts, Mask, FixupNode)) {
3926 if (Loads.size() == 0)
3927 return false;
3928
3929 DEBUG(dbgs() << "Backwards propagate AND: "; N->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "Backwards propagate AND: "
; N->dump(); } } while (false)
;
3930 SDValue MaskOp = N->getOperand(1);
3931
3932 // If it exists, fixup the single node we allow in the tree that needs
3933 // masking.
3934 if (FixupNode) {
3935 DEBUG(dbgs() << "First, need to fix up: "; FixupNode->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "First, need to fix up: "; FixupNode
->dump(); } } while (false)
;
3936 SDValue And = DAG.getNode(ISD::AND, SDLoc(FixupNode),
3937 FixupNode->getValueType(0),
3938 SDValue(FixupNode, 0), MaskOp);
3939 DAG.ReplaceAllUsesOfValueWith(SDValue(FixupNode, 0), And);
3940 DAG.UpdateNodeOperands(And.getNode(), SDValue(FixupNode, 0),
3941 MaskOp);
3942 }
3943
3944 // Narrow any constants that need it.
3945 for (auto *LogicN : NodesWithConsts) {
3946 SDValue Op0 = LogicN->getOperand(0);
3947 SDValue Op1 = LogicN->getOperand(1);
3948
3949 if (isa<ConstantSDNode>(Op0))
3950 std::swap(Op0, Op1);
3951
3952 SDValue And = DAG.getNode(ISD::AND, SDLoc(Op1), Op1.getValueType(),
3953 Op1, MaskOp);
3954
3955 DAG.UpdateNodeOperands(LogicN, Op0, And);
3956 }
3957
3958 // Create narrow loads.
3959 for (auto *Load : Loads) {
3960 DEBUG(dbgs() << "Propagate AND back to: "; Load->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "Propagate AND back to: "; Load
->dump(); } } while (false)
;
3961 SDValue And = DAG.getNode(ISD::AND, SDLoc(Load), Load->getValueType(0),
3962 SDValue(Load, 0), MaskOp);
3963 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), And);
3964 DAG.UpdateNodeOperands(And.getNode(), SDValue(Load, 0), MaskOp);
3965 SDValue NewLoad = ReduceLoadWidth(And.getNode());
3966 assert(NewLoad &&(static_cast <bool> (NewLoad && "Shouldn't be masking the load if it can't be narrowed"
) ? void (0) : __assert_fail ("NewLoad && \"Shouldn't be masking the load if it can't be narrowed\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3967, __extension__ __PRETTY_FUNCTION__))
3967 "Shouldn't be masking the load if it can't be narrowed")(static_cast <bool> (NewLoad && "Shouldn't be masking the load if it can't be narrowed"
) ? void (0) : __assert_fail ("NewLoad && \"Shouldn't be masking the load if it can't be narrowed\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3967, __extension__ __PRETTY_FUNCTION__))
;
3968 CombineTo(Load, NewLoad, NewLoad.getValue(1));
3969 }
3970 DAG.ReplaceAllUsesWith(N, N->getOperand(0).getNode());
3971 return true;
3972 }
3973 return false;
3974}
3975
3976SDValue DAGCombiner::visitAND(SDNode *N) {
3977 SDValue N0 = N->getOperand(0);
3978 SDValue N1 = N->getOperand(1);
3979 EVT VT = N1.getValueType();
3980
3981 // x & x --> x
3982 if (N0 == N1)
3983 return N0;
3984
3985 // fold vector ops
3986 if (VT.isVector()) {
3987 if (SDValue FoldedVOp = SimplifyVBinOp(N))
3988 return FoldedVOp;
3989
3990 // fold (and x, 0) -> 0, vector edition
3991 if (ISD::isBuildVectorAllZeros(N0.getNode()))
3992 // do not return N0, because undef node may exist in N0
3993 return DAG.getConstant(APInt::getNullValue(N0.getScalarValueSizeInBits()),
3994 SDLoc(N), N0.getValueType());
3995 if (ISD::isBuildVectorAllZeros(N1.getNode()))
3996 // do not return N1, because undef node may exist in N1
3997 return DAG.getConstant(APInt::getNullValue(N1.getScalarValueSizeInBits()),
3998 SDLoc(N), N1.getValueType());
3999
4000 // fold (and x, -1) -> x, vector edition
4001 if (ISD::isBuildVectorAllOnes(N0.getNode()))
4002 return N1;
4003 if (ISD::isBuildVectorAllOnes(N1.getNode()))
4004 return N0;
4005 }
4006
4007 // fold (and c1, c2) -> c1&c2
4008 ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
4009 ConstantSDNode *N1C = isConstOrConstSplat(N1);
4010 if (N0C && N1C && !N1C->isOpaque())
4011 return DAG.FoldConstantArithmetic(ISD::AND, SDLoc(N), VT, N0C, N1C);
4012 // canonicalize constant to RHS
4013 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
4014 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
4015 return DAG.getNode(ISD::AND, SDLoc(N), VT, N1, N0);
4016 // fold (and x, -1) -> x
4017 if (isAllOnesConstant(N1))
4018 return N0;
4019 // if (and x, c) is known to be zero, return 0
4020 unsigned BitWidth = VT.getScalarSizeInBits();
4021 if (N1C && DAG.MaskedValueIsZero(SDValue(N, 0),
4022 APInt::getAllOnesValue(BitWidth)))
4023 return DAG.getConstant(0, SDLoc(N), VT);
4024
4025 if (SDValue NewSel = foldBinOpIntoSelect(N))
4026 return NewSel;
4027
4028 // reassociate and
4029 if (SDValue RAND = ReassociateOps(ISD::AND, SDLoc(N), N0, N1))
4030 return RAND;
4031
4032 // Try to convert a constant mask AND into a shuffle clear mask.
4033 if (VT.isVector())
4034 if (SDValue Shuffle = XformToShuffleWithZero(N))
4035 return Shuffle;
4036
4037 // fold (and (or x, C), D) -> D if (C & D) == D
4038 auto MatchSubset = [](ConstantSDNode *LHS, ConstantSDNode *RHS) {
4039 return RHS->getAPIntValue().isSubsetOf(LHS->getAPIntValue());
4040 };
4041 if (N0.getOpcode() == ISD::OR &&
4042 matchBinaryPredicate(N0.getOperand(1), N1, MatchSubset))
4043 return N1;
4044 // fold (and (any_ext V), c) -> (zero_ext V) if 'and' only clears top bits.
4045 if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) {
4046 SDValue N0Op0 = N0.getOperand(0);
4047 APInt Mask = ~N1C->getAPIntValue();
4048 Mask = Mask.trunc(N0Op0.getScalarValueSizeInBits());
4049 if (DAG.MaskedValueIsZero(N0Op0, Mask)) {
4050 SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N),
4051 N0.getValueType(), N0Op0);
4052
4053 // Replace uses of the AND with uses of the Zero extend node.
4054 CombineTo(N, Zext);
4055
4056 // We actually want to replace all uses of the any_extend with the
4057 // zero_extend, to avoid duplicating things. This will later cause this
4058 // AND to be folded.
4059 CombineTo(N0.getNode(), Zext);
4060 return SDValue(N, 0); // Return N so it doesn't get rechecked!
4061 }
4062 }
4063 // similarly fold (and (X (load ([non_ext|any_ext|zero_ext] V))), c) ->
4064 // (X (load ([non_ext|zero_ext] V))) if 'and' only clears top bits which must
4065 // already be zero by virtue of the width of the base type of the load.
4066 //
4067 // the 'X' node here can either be nothing or an extract_vector_elt to catch
4068 // more cases.
4069 if ((N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
4070 N0.getValueSizeInBits() == N0.getOperand(0).getScalarValueSizeInBits() &&
4071 N0.getOperand(0).getOpcode() == ISD::LOAD &&
4072 N0.getOperand(0).getResNo() == 0) ||
4073 (N0.getOpcode() == ISD::LOAD && N0.getResNo() == 0)) {
4074 LoadSDNode *Load = cast<LoadSDNode>( (N0.getOpcode() == ISD::LOAD) ?
4075 N0 : N0.getOperand(0) );
4076
4077 // Get the constant (if applicable) the zero'th operand is being ANDed with.
4078 // This can be a pure constant or a vector splat, in which case we treat the
4079 // vector as a scalar and use the splat value.
4080 APInt Constant = APInt::getNullValue(1);
4081 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) {
4082 Constant = C->getAPIntValue();
4083 } else if (BuildVectorSDNode *Vector = dyn_cast<BuildVectorSDNode>(N1)) {
4084 APInt SplatValue, SplatUndef;
4085 unsigned SplatBitSize;
4086 bool HasAnyUndefs;
4087 bool IsSplat = Vector->isConstantSplat(SplatValue, SplatUndef,
4088 SplatBitSize, HasAnyUndefs);
4089 if (IsSplat) {
4090 // Undef bits can contribute to a possible optimisation if set, so
4091 // set them.
4092 SplatValue |= SplatUndef;
4093
4094 // The splat value may be something like "0x00FFFFFF", which means 0 for
4095 // the first vector value and FF for the rest, repeating. We need a mask
4096 // that will apply equally to all members of the vector, so AND all the
4097 // lanes of the constant together.
4098 EVT VT = Vector->getValueType(0);
4099 unsigned BitWidth = VT.getScalarSizeInBits();
4100
4101 // If the splat value has been compressed to a bitlength lower
4102 // than the size of the vector lane, we need to re-expand it to
4103 // the lane size.
4104 if (BitWidth > SplatBitSize)
4105 for (SplatValue = SplatValue.zextOrTrunc(BitWidth);
4106 SplatBitSize < BitWidth;
4107 SplatBitSize = SplatBitSize * 2)
4108 SplatValue |= SplatValue.shl(SplatBitSize);
4109
4110 // Make sure that variable 'Constant' is only set if 'SplatBitSize' is a
4111 // multiple of 'BitWidth'. Otherwise, we could propagate a wrong value.
4112 if (SplatBitSize % BitWidth == 0) {
4113 Constant = APInt::getAllOnesValue(BitWidth);
4114 for (unsigned i = 0, n = SplatBitSize/BitWidth; i < n; ++i)
4115 Constant &= SplatValue.lshr(i*BitWidth).zextOrTrunc(BitWidth);
4116 }
4117 }
4118 }
4119
4120 // If we want to change an EXTLOAD to a ZEXTLOAD, ensure a ZEXTLOAD is
4121 // actually legal and isn't going to get expanded, else this is a false
4122 // optimisation.
4123 bool CanZextLoadProfitably = TLI.isLoadExtLegal(ISD::ZEXTLOAD,
4124 Load->getValueType(0),
4125 Load->getMemoryVT());
4126
4127 // Resize the constant to the same size as the original memory access before
4128 // extension. If it is still the AllOnesValue then this AND is completely
4129 // unneeded.
4130 Constant = Constant.zextOrTrunc(Load->getMemoryVT().getScalarSizeInBits());
4131
4132 bool B;
4133 switch (Load->getExtensionType()) {
4134 default: B = false; break;
4135 case ISD::EXTLOAD: B = CanZextLoadProfitably; break;
4136 case ISD::ZEXTLOAD:
4137 case ISD::NON_EXTLOAD: B = true; break;
4138 }
4139
4140 if (B && Constant.isAllOnesValue()) {
4141 // If the load type was an EXTLOAD, convert to ZEXTLOAD in order to
4142 // preserve semantics once we get rid of the AND.
4143 SDValue NewLoad(Load, 0);
4144
4145 // Fold the AND away. NewLoad may get replaced immediately.
4146 CombineTo(N, (N0.getNode() == Load) ? NewLoad : N0);
4147
4148 if (Load->getExtensionType() == ISD::EXTLOAD) {
4149 NewLoad = DAG.getLoad(Load->getAddressingMode(), ISD::ZEXTLOAD,
4150 Load->getValueType(0), SDLoc(Load),
4151 Load->getChain(), Load->getBasePtr(),
4152 Load->getOffset(), Load->getMemoryVT(),
4153 Load->getMemOperand());
4154 // Replace uses of the EXTLOAD with the new ZEXTLOAD.
4155 if (Load->getNumValues() == 3) {
4156 // PRE/POST_INC loads have 3 values.
4157 SDValue To[] = { NewLoad.getValue(0), NewLoad.getValue(1),
4158 NewLoad.getValue(2) };
4159 CombineTo(Load, To, 3, true);
4160 } else {
4161 CombineTo(Load, NewLoad.getValue(0), NewLoad.getValue(1));
4162 }
4163 }
4164
4165 return SDValue(N, 0); // Return N so it doesn't get rechecked!
4166 }
4167 }
4168
4169 // fold (and (load x), 255) -> (zextload x, i8)
4170 // fold (and (extload x, i16), 255) -> (zextload x, i8)
4171 // fold (and (any_ext (extload x, i16)), 255) -> (zextload x, i8)
4172 if (!VT.isVector() && N1C && (N0.getOpcode() == ISD::LOAD ||
4173 (N0.getOpcode() == ISD::ANY_EXTEND &&
4174 N0.getOperand(0).getOpcode() == ISD::LOAD))) {
4175 if (SDValue Res = ReduceLoadWidth(N)) {
4176 LoadSDNode *LN0 = N0->getOpcode() == ISD::ANY_EXTEND
4177 ? cast<LoadSDNode>(N0.getOperand(0)) : cast<LoadSDNode>(N0);
4178
4179 AddToWorklist(N);
4180 CombineTo(LN0, Res, Res.getValue(1));
4181 return SDValue(N, 0);
4182 }
4183 }
4184
4185 if (Level >= AfterLegalizeTypes) {
4186 // Attempt to propagate the AND back up to the leaves which, if they're
4187 // loads, can be combined to narrow loads and the AND node can be removed.
4188 // Perform after legalization so that extend nodes will already be
4189 // combined into the loads.
4190 if (BackwardsPropagateMask(N, DAG)) {
4191 return SDValue(N, 0);
4192 }
4193 }
4194
4195 if (SDValue Combined = visitANDLike(N0, N1, N))
4196 return Combined;
4197
4198 // Simplify: (and (op x...), (op y...)) -> (op (and x, y))
4199 if (N0.getOpcode() == N1.getOpcode())
4200 if (SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N))
4201 return Tmp;
4202
4203 // Masking the negated extension of a boolean is just the zero-extended
4204 // boolean:
4205 // and (sub 0, zext(bool X)), 1 --> zext(bool X)
4206 // and (sub 0, sext(bool X)), 1 --> zext(bool X)
4207 //
4208 // Note: the SimplifyDemandedBits fold below can make an information-losing
4209 // transform, and then we have no way to find this better fold.
4210 if (N1C && N1C->isOne() && N0.getOpcode() == ISD::SUB) {
4211 if (isNullConstantOrNullSplatConstant(N0.getOperand(0))) {
4212 SDValue SubRHS = N0.getOperand(1);
4213 if (SubRHS.getOpcode() == ISD::ZERO_EXTEND &&
4214 SubRHS.getOperand(0).getScalarValueSizeInBits() == 1)
4215 return SubRHS;
4216 if (SubRHS.getOpcode() == ISD::SIGN_EXTEND &&
4217 SubRHS.getOperand(0).getScalarValueSizeInBits() == 1)
4218 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, SubRHS.getOperand(0));
4219 }
4220 }
4221
4222 // fold (and (sign_extend_inreg x, i16 to i32), 1) -> (and x, 1)
4223 // fold (and (sra)) -> (and (srl)) when possible.
4224 if (SimplifyDemandedBits(SDValue(N, 0)))
4225 return SDValue(N, 0);
4226
4227 // fold (zext_inreg (extload x)) -> (zextload x)
4228 if (ISD::isEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode())) {
4229 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
4230 EVT MemVT = LN0->getMemoryVT();
4231 // If we zero all the possible extended bits, then we can turn this into
4232 // a zextload if we are running before legalize or the operation is legal.
4233 unsigned BitWidth = N1.getScalarValueSizeInBits();
4234 if (DAG.MaskedValueIsZero(N1, APInt::getHighBitsSet(BitWidth,
4235 BitWidth - MemVT.getScalarSizeInBits())) &&
4236 ((!LegalOperations && !LN0->isVolatile()) ||
4237 TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT))) {
4238 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT,
4239 LN0->getChain(), LN0->getBasePtr(),
4240 MemVT, LN0->getMemOperand());
4241 AddToWorklist(N);
4242 CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
4243 return SDValue(N, 0); // Return N so it doesn't get rechecked!
4244 }
4245 }
4246 // fold (zext_inreg (sextload x)) -> (zextload x) iff load has one use
4247 if (ISD::isSEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) &&
4248 N0.hasOneUse()) {
4249 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
4250 EVT MemVT = LN0->getMemoryVT();
4251 // If we zero all the possible extended bits, then we can turn this into
4252 // a zextload if we are running before legalize or the operation is legal.
4253 unsigned BitWidth = N1.getScalarValueSizeInBits();
4254 if (DAG.MaskedValueIsZero(N1, APInt::getHighBitsSet(BitWidth,
4255 BitWidth - MemVT.getScalarSizeInBits())) &&
4256 ((!LegalOperations && !LN0->isVolatile()) ||
4257 TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT))) {
4258 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT,
4259 LN0->getChain(), LN0->getBasePtr(),
4260 MemVT, LN0->getMemOperand());
4261 AddToWorklist(N);
4262 CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
4263 return SDValue(N, 0); // Return N so it doesn't get rechecked!
4264 }
4265 }
4266 // fold (and (or (srl N, 8), (shl N, 8)), 0xffff) -> (srl (bswap N), const)
4267 if (N1C && N1C->getAPIntValue() == 0xffff && N0.getOpcode() == ISD::OR) {
4268 if (SDValue BSwap = MatchBSwapHWordLow(N0.getNode(), N0.getOperand(0),
4269 N0.getOperand(1), false))
4270 return BSwap;
4271 }
4272
4273 return SDValue();
4274}
4275
4276/// Match (a >> 8) | (a << 8) as (bswap a) >> 16.
4277SDValue DAGCombiner::MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
4278 bool DemandHighBits) {
4279 if (!LegalOperations)
4280 return SDValue();
4281
4282 EVT VT = N->getValueType(0);
4283 if (VT != MVT::i64 && VT != MVT::i32 && VT != MVT::i16)
4284 return SDValue();
4285 if (!TLI.isOperationLegalOrCustom(ISD::BSWAP, VT))
4286 return SDValue();
4287
4288 // Recognize (and (shl a, 8), 0xff00), (and (srl a, 8), 0xff)
4289 bool LookPassAnd0 = false;
4290 bool LookPassAnd1 = false;
4291 if (N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::SRL)
4292 std::swap(N0, N1);
4293 if (N1.getOpcode() == ISD::AND && N1.getOperand(0).getOpcode() == ISD::SHL)
4294 std::swap(N0, N1);
4295 if (N0.getOpcode() == ISD::AND) {
4296 if (!N0.getNode()->hasOneUse())
4297 return SDValue();
4298 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
4299 if (!N01C || N01C->getZExtValue() != 0xFF00)
4300 return SDValue();
4301 N0 = N0.getOperand(0);
4302 LookPassAnd0 = true;
4303 }
4304
4305 if (N1.getOpcode() == ISD::AND) {
4306 if (!N1.getNode()->hasOneUse())
4307 return SDValue();
4308 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
4309 if (!N11C || N11C->getZExtValue() != 0xFF)
4310 return SDValue();
4311 N1 = N1.getOperand(0);
4312 LookPassAnd1 = true;
4313 }
4314
4315 if (N0.getOpcode() == ISD::SRL && N1.getOpcode() == ISD::SHL)
4316 std::swap(N0, N1);
4317 if (N0.getOpcode() != ISD::SHL || N1.getOpcode() != ISD::SRL)
4318 return SDValue();
4319 if (!N0.getNode()->hasOneUse() || !N1.getNode()->hasOneUse())
4320 return SDValue();
4321
4322 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
4323 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
4324 if (!N01C || !N11C)
4325 return SDValue();
4326 if (N01C->getZExtValue() != 8 || N11C->getZExtValue() != 8)
4327 return SDValue();
4328
4329 // Look for (shl (and a, 0xff), 8), (srl (and a, 0xff00), 8)
4330 SDValue N00 = N0->getOperand(0);
4331 if (!LookPassAnd0 && N00.getOpcode() == ISD::AND) {
4332 if (!N00.getNode()->hasOneUse())
4333 return SDValue();
4334 ConstantSDNode *N001C = dyn_cast<ConstantSDNode>(N00.getOperand(1));
4335 if (!N001C || N001C->getZExtValue() != 0xFF)
4336 return SDValue();
4337 N00 = N00.getOperand(0);
4338 LookPassAnd0 = true;
4339 }
4340
4341 SDValue N10 = N1->getOperand(0);
4342 if (!LookPassAnd1 && N10.getOpcode() == ISD::AND) {
4343 if (!N10.getNode()->hasOneUse())
4344 return SDValue();
4345 ConstantSDNode *N101C = dyn_cast<ConstantSDNode>(N10.getOperand(1));
4346 if (!N101C || N101C->getZExtValue() != 0xFF00)
4347 return SDValue();
4348 N10 = N10.getOperand(0);
4349 LookPassAnd1 = true;
4350 }
4351
4352 if (N00 != N10)
4353 return SDValue();
4354
4355 // Make sure everything beyond the low halfword gets set to zero since the SRL
4356 // 16 will clear the top bits.
4357 unsigned OpSizeInBits = VT.getSizeInBits();
4358 if (DemandHighBits && OpSizeInBits > 16) {
4359 // If the left-shift isn't masked out then the only way this is a bswap is
4360 // if all bits beyond the low 8 are 0. In that case the entire pattern
4361 // reduces to a left shift anyway: leave it for other parts of the combiner.
4362 if (!LookPassAnd0)
4363 return SDValue();
4364
4365 // However, if the right shift isn't masked out then it might be because
4366 // it's not needed. See if we can spot that too.
4367 if (!LookPassAnd1 &&
4368 !DAG.MaskedValueIsZero(
4369 N10, APInt::getHighBitsSet(OpSizeInBits, OpSizeInBits - 16)))
4370 return SDValue();
4371 }
4372
4373 SDValue Res = DAG.getNode(ISD::BSWAP, SDLoc(N), VT, N00);
4374 if (OpSizeInBits > 16) {
4375 SDLoc DL(N);
4376 Res = DAG.getNode(ISD::SRL, DL, VT, Res,
4377 DAG.getConstant(OpSizeInBits - 16, DL,
4378 getShiftAmountTy(VT)));
4379 }
4380 return Res;
4381}
4382
4383/// Return true if the specified node is an element that makes up a 32-bit
4384/// packed halfword byteswap.
4385/// ((x & 0x000000ff) << 8) |
4386/// ((x & 0x0000ff00) >> 8) |
4387/// ((x & 0x00ff0000) << 8) |
4388/// ((x & 0xff000000) >> 8)
4389static bool isBSwapHWordElement(SDValue N, MutableArrayRef<SDNode *> Parts) {
4390 if (!N.getNode()->hasOneUse())
4391 return false;
4392
4393 unsigned Opc = N.getOpcode();
4394 if (Opc != ISD::AND && Opc != ISD::SHL && Opc != ISD::SRL)
4395 return false;
4396
4397 SDValue N0 = N.getOperand(0);
4398 unsigned Opc0 = N0.getOpcode();
4399 if (Opc0 != ISD::AND && Opc0 != ISD::SHL && Opc0 != ISD::SRL)
4400 return false;
4401
4402 ConstantSDNode *N1C = nullptr;
4403 // SHL or SRL: look upstream for AND mask operand
4404 if (Opc == ISD::AND)
4405 N1C = dyn_cast<ConstantSDNode>(N.getOperand(1));
4406 else if (Opc0 == ISD::AND)
4407 N1C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
4408 if (!N1C)
4409 return false;
4410
4411 unsigned MaskByteOffset;
4412 switch (N1C->getZExtValue()) {
4413 default:
4414 return false;
4415 case 0xFF: MaskByteOffset = 0; break;
4416 case 0xFF00: MaskByteOffset = 1; break;
4417 case 0xFF0000: MaskByteOffset = 2; break;
4418 case 0xFF000000: MaskByteOffset = 3; break;
4419 }
4420
4421 // Look for (x & 0xff) << 8 as well as ((x << 8) & 0xff00).
4422 if (Opc == ISD::AND) {
4423 if (MaskByteOffset == 0 || MaskByteOffset == 2) {
4424 // (x >> 8) & 0xff
4425 // (x >> 8) & 0xff0000
4426 if (Opc0 != ISD::SRL)
4427 return false;
4428 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
4429 if (!C || C->getZExtValue() != 8)
4430 return false;
4431 } else {
4432 // (x << 8) & 0xff00
4433 // (x << 8) & 0xff000000
4434 if (Opc0 != ISD::SHL)
4435 return false;
4436 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
4437 if (!C || C->getZExtValue() != 8)
4438 return false;
4439 }
4440 } else if (Opc == ISD::SHL) {
4441 // (x & 0xff) << 8
4442 // (x & 0xff0000) << 8
4443 if (MaskByteOffset != 0 && MaskByteOffset != 2)
4444 return false;
4445 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
4446 if (!C || C->getZExtValue() != 8)
4447 return false;
4448 } else { // Opc == ISD::SRL
4449 // (x & 0xff00) >> 8
4450 // (x & 0xff000000) >> 8
4451 if (MaskByteOffset != 1 && MaskByteOffset != 3)
4452 return false;
4453 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
4454 if (!C || C->getZExtValue() != 8)
4455 return false;
4456 }
4457
4458 if (Parts[MaskByteOffset])
4459 return false;
4460
4461 Parts[MaskByteOffset] = N0.getOperand(0).getNode();
4462 return true;
4463}
4464
4465/// Match a 32-bit packed halfword bswap. That is
4466/// ((x & 0x000000ff) << 8) |
4467/// ((x & 0x0000ff00) >> 8) |
4468/// ((x & 0x00ff0000) << 8) |
4469/// ((x & 0xff000000) >> 8)
4470/// => (rotl (bswap x), 16)
4471SDValue DAGCombiner::MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1) {
4472 if (!LegalOperations)
4473 return SDValue();
4474
4475 EVT VT = N->getValueType(0);
4476 if (VT != MVT::i32)
4477 return SDValue();
4478 if (!TLI.isOperationLegalOrCustom(ISD::BSWAP, VT))
4479 return SDValue();
4480
4481 // Look for either
4482 // (or (or (and), (and)), (or (and), (and)))
4483 // (or (or (or (and), (and)), (and)), (and))
4484 if (N0.getOpcode() != ISD::OR)
4485 return SDValue();
4486 SDValue N00 = N0.getOperand(0);
4487 SDValue N01 = N0.getOperand(1);
4488 SDNode *Parts[4] = {};
4489
4490 if (N1.getOpcode() == ISD::OR &&
4491 N00.getNumOperands() == 2 && N01.getNumOperands() == 2) {
4492 // (or (or (and), (and)), (or (and), (and)))
4493 if (!isBSwapHWordElement(N00, Parts))
4494 return SDValue();
4495
4496 if (!isBSwapHWordElement(N01, Parts))
4497 return SDValue();
4498 SDValue N10 = N1.getOperand(0);
4499 if (!isBSwapHWordElement(N10, Parts))
4500 return SDValue();
4501 SDValue N11 = N1.getOperand(1);
4502 if (!isBSwapHWordElement(N11, Parts))
4503 return SDValue();
4504 } else {
4505 // (or (or (or (and), (and)), (and)), (and))
4506 if (!isBSwapHWordElement(N1, Parts))
4507 return SDValue();
4508 if (!isBSwapHWordElement(N01, Parts))
4509 return SDValue();
4510 if (N00.getOpcode() != ISD::OR)
4511 return SDValue();
4512 SDValue N000 = N00.getOperand(0);
4513 if (!isBSwapHWordElement(N000, Parts))
4514 return SDValue();
4515 SDValue N001 = N00.getOperand(1);
4516 if (!isBSwapHWordElement(N001, Parts))
4517 return SDValue();
4518 }
4519
4520 // Make sure the parts are all coming from the same node.
4521 if (Parts[0] != Parts[1] || Parts[0] != Parts[2] || Parts[0] != Parts[3])
4522 return SDValue();
4523
4524 SDLoc DL(N);
4525 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT,
4526 SDValue(Parts[0], 0));
4527
4528 // Result of the bswap should be rotated by 16. If it's not legal, then
4529 // do (x << 16) | (x >> 16).
4530 SDValue ShAmt = DAG.getConstant(16, DL, getShiftAmountTy(VT));
4531 if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT))
4532 return DAG.getNode(ISD::ROTL, DL, VT, BSwap, ShAmt);
4533 if (TLI.isOperationLegalOrCustom(ISD::ROTR, VT))
4534 return DAG.getNode(ISD::ROTR, DL, VT, BSwap, ShAmt);
4535 return DAG.getNode(ISD::OR, DL, VT,
4536 DAG.getNode(ISD::SHL, DL, VT, BSwap, ShAmt),
4537 DAG.getNode(ISD::SRL, DL, VT, BSwap, ShAmt));
4538}
4539
4540/// This contains all DAGCombine rules which reduce two values combined by
4541/// an Or operation to a single value \see visitANDLike().
4542SDValue DAGCombiner::visitORLike(SDValue N0, SDValue N1, SDNode *N) {
4543 EVT VT = N1.getValueType();
4544 SDLoc DL(N);
4545
4546 // fold (or x, undef) -> -1
4547 if (!LegalOperations && (N0.isUndef() || N1.isUndef()))
4548 return DAG.getAllOnesConstant(DL, VT);
4549
4550 if (SDValue V = foldLogicOfSetCCs(false, N0, N1, DL))
4551 return V;
4552
4553 // (or (and X, C1), (and Y, C2)) -> (and (or X, Y), C3) if possible.
4554 if (N0.getOpcode() == ISD::AND && N1.getOpcode() == ISD::AND &&
4555 // Don't increase # computations.
4556 (N0.getNode()->hasOneUse() || N1.getNode()->hasOneUse())) {
4557 // We can only do this xform if we know that bits from X that are set in C2
4558 // but not in C1 are already zero. Likewise for Y.
4559 if (const ConstantSDNode *N0O1C =
4560 getAsNonOpaqueConstant(N0.getOperand(1))) {
4561 if (const ConstantSDNode *N1O1C =
4562 getAsNonOpaqueConstant(N1.getOperand(1))) {
4563 // We can only do this xform if we know that bits from X that are set in
4564 // C2 but not in C1 are already zero. Likewise for Y.
4565 const APInt &LHSMask = N0O1C->getAPIntValue();
4566 const APInt &RHSMask = N1O1C->getAPIntValue();
4567
4568 if (DAG.MaskedValueIsZero(N0.getOperand(0), RHSMask&~LHSMask) &&
4569 DAG.MaskedValueIsZero(N1.getOperand(0), LHSMask&~RHSMask)) {
4570 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT,
4571 N0.getOperand(0), N1.getOperand(0));
4572 return DAG.getNode(ISD::AND, DL, VT, X,
4573 DAG.getConstant(LHSMask | RHSMask, DL, VT));
4574 }
4575 }
4576 }
4577 }
4578
4579 // (or (and X, M), (and X, N)) -> (and X, (or M, N))
4580 if (N0.getOpcode() == ISD::AND &&
4581 N1.getOpcode() == ISD::AND &&
4582 N0.getOperand(0) == N1.getOperand(0) &&
4583 // Don't increase # computations.
4584 (N0.getNode()->hasOneUse() || N1.getNode()->hasOneUse())) {
4585 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT,
4586 N0.getOperand(1), N1.getOperand(1));
4587 return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), X);
4588 }
4589
4590 return SDValue();
4591}
4592
4593SDValue DAGCombiner::visitOR(SDNode *N) {
4594 SDValue N0 = N->getOperand(0);
4595 SDValue N1 = N->getOperand(1);
4596 EVT VT = N1.getValueType();
4597
4598 // x | x --> x
4599 if (N0 == N1)
4600 return N0;
4601
4602 // fold vector ops
4603 if (VT.isVector()) {
4604 if (SDValue FoldedVOp = SimplifyVBinOp(N))
4605 return FoldedVOp;
4606
4607 // fold (or x, 0) -> x, vector edition
4608 if (ISD::isBuildVectorAllZeros(N0.getNode()))
4609 return N1;
4610 if (ISD::isBuildVectorAllZeros(N1.getNode()))
4611 return N0;
4612
4613 // fold (or x, -1) -> -1, vector edition
4614 if (ISD::isBuildVectorAllOnes(N0.getNode()))
4615 // do not return N0, because undef node may exist in N0
4616 return DAG.getAllOnesConstant(SDLoc(N), N0.getValueType());
4617 if (ISD::isBuildVectorAllOnes(N1.getNode()))
4618 // do not return N1, because undef node may exist in N1
4619 return DAG.getAllOnesConstant(SDLoc(N), N1.getValueType());
4620
4621 // fold (or (shuf A, V_0, MA), (shuf B, V_0, MB)) -> (shuf A, B, Mask)
4622 // Do this only if the resulting shuffle is legal.
4623 if (isa<ShuffleVectorSDNode>(N0) &&
4624 isa<ShuffleVectorSDNode>(N1) &&
4625 // Avoid folding a node with illegal type.
4626 TLI.isTypeLegal(VT)) {
4627 bool ZeroN00 = ISD::isBuildVectorAllZeros(N0.getOperand(0).getNode());
4628 bool ZeroN01 = ISD::isBuildVectorAllZeros(N0.getOperand(1).getNode());
4629 bool ZeroN10 = ISD::isBuildVectorAllZeros(N1.getOperand(0).getNode());
4630 bool ZeroN11 = ISD::isBuildVectorAllZeros(N1.getOperand(1).getNode());
4631 // Ensure both shuffles have a zero input.
4632 if ((ZeroN00 != ZeroN01) && (ZeroN10 != ZeroN11)) {
4633 assert((!ZeroN00 || !ZeroN01) && "Both inputs zero!")(static_cast <bool> ((!ZeroN00 || !ZeroN01) && "Both inputs zero!"
) ? void (0) : __assert_fail ("(!ZeroN00 || !ZeroN01) && \"Both inputs zero!\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4633, __extension__ __PRETTY_FUNCTION__))
;
4634 assert((!ZeroN10 || !ZeroN11) && "Both inputs zero!")(static_cast <bool> ((!ZeroN10 || !ZeroN11) && "Both inputs zero!"
) ? void (0) : __assert_fail ("(!ZeroN10 || !ZeroN11) && \"Both inputs zero!\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4634, __extension__ __PRETTY_FUNCTION__))
;
4635 const ShuffleVectorSDNode *SV0 = cast<ShuffleVectorSDNode>(N0);
4636 const ShuffleVectorSDNode *SV1 = cast<ShuffleVectorSDNode>(N1);
4637 bool CanFold = true;
4638 int NumElts = VT.getVectorNumElements();
4639 SmallVector<int, 4> Mask(NumElts);
4640
4641 for (int i = 0; i != NumElts; ++i) {
4642 int M0 = SV0->getMaskElt(i);
4643 int M1 = SV1->getMaskElt(i);
4644
4645 // Determine if either index is pointing to a zero vector.
4646 bool M0Zero = M0 < 0 || (ZeroN00 == (M0 < NumElts));
4647 bool M1Zero = M1 < 0 || (ZeroN10 == (M1 < NumElts));
4648
4649 // If one element is zero and the otherside is undef, keep undef.
4650 // This also handles the case that both are undef.
4651 if ((M0Zero && M1 < 0) || (M1Zero && M0 < 0)) {
4652 Mask[i] = -1;
4653 continue;
4654 }
4655
4656 // Make sure only one of the elements is zero.
4657 if (M0Zero == M1Zero) {
4658 CanFold = false;
4659 break;
4660 }
4661
4662 assert((M0 >= 0 || M1 >= 0) && "Undef index!")(static_cast <bool> ((M0 >= 0 || M1 >= 0) &&
"Undef index!") ? void (0) : __assert_fail ("(M0 >= 0 || M1 >= 0) && \"Undef index!\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4662, __extension__ __PRETTY_FUNCTION__))
;
4663
4664 // We have a zero and non-zero element. If the non-zero came from
4665 // SV0 make the index a LHS index. If it came from SV1, make it
4666 // a RHS index. We need to mod by NumElts because we don't care
4667 // which operand it came from in the original shuffles.
4668 Mask[i] = M1Zero ? M0 % NumElts : (M1 % NumElts) + NumElts;
4669 }
4670
4671 if (CanFold) {
4672 SDValue NewLHS = ZeroN00 ? N0.getOperand(1) : N0.getOperand(0);
4673 SDValue NewRHS = ZeroN10 ? N1.getOperand(1) : N1.getOperand(0);
4674
4675 bool LegalMask = TLI.isShuffleMaskLegal(Mask, VT);
4676 if (!LegalMask) {
4677 std::swap(NewLHS, NewRHS);
4678 ShuffleVectorSDNode::commuteMask(Mask);
4679 LegalMask = TLI.isShuffleMaskLegal(Mask, VT);
4680 }
4681
4682 if (LegalMask)
4683 return DAG.getVectorShuffle(VT, SDLoc(N), NewLHS, NewRHS, Mask);
4684 }
4685 }
4686 }
4687 }
4688
4689 // fold (or c1, c2) -> c1|c2
4690 ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
4691 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
4692 if (N0C && N1C && !N1C->isOpaque())
4693 return DAG.FoldConstantArithmetic(ISD::OR, SDLoc(N), VT, N0C, N1C);
4694 // canonicalize constant to RHS
4695 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
4696 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
4697 return DAG.getNode(ISD::OR, SDLoc(N), VT, N1, N0);
4698 // fold (or x, 0) -> x
4699 if (isNullConstant(N1))
4700 return N0;
4701 // fold (or x, -1) -> -1
4702 if (isAllOnesConstant(N1))
4703 return N1;
4704
4705 if (SDValue NewSel = foldBinOpIntoSelect(N))
4706 return NewSel;
4707
4708 // fold (or x, c) -> c iff (x & ~c) == 0
4709 if (N1C && DAG.MaskedValueIsZero(N0, ~N1C->getAPIntValue()))
4710 return N1;
4711
4712 if (SDValue Combined = visitORLike(N0, N1, N))
4713 return Combined;
4714
4715 // Recognize halfword bswaps as (bswap + rotl 16) or (bswap + shl 16)
4716 if (SDValue BSwap = MatchBSwapHWord(N, N0, N1))
4717 return BSwap;
4718 if (SDValue BSwap = MatchBSwapHWordLow(N, N0, N1))
4719 return BSwap;
4720
4721 // reassociate or
4722 if (SDValue ROR = ReassociateOps(ISD::OR, SDLoc(N), N0, N1))
4723 return ROR;
4724
4725 // Canonicalize (or (and X, c1), c2) -> (and (or X, c2), c1|c2)
4726 // iff (c1 & c2) != 0.
4727 auto MatchIntersect = [](ConstantSDNode *LHS, ConstantSDNode *RHS) {
4728 return LHS->getAPIntValue().intersects(RHS->getAPIntValue());
4729 };
4730 if (N0.getOpcode() == ISD::AND && N0.getNode()->hasOneUse() &&
4731 matchBinaryPredicate(N0.getOperand(1), N1, MatchIntersect)) {
4732 if (SDValue COR = DAG.FoldConstantArithmetic(
4733 ISD::OR, SDLoc(N1), VT, N1.getNode(), N0.getOperand(1).getNode())) {
4734 SDValue IOR = DAG.getNode(ISD::OR, SDLoc(N0), VT, N0.getOperand(0), N1);
4735 AddToWorklist(IOR.getNode());
4736 return DAG.getNode(ISD::AND, SDLoc(N), VT, COR, IOR);
4737 }
4738 }
4739
4740 // Simplify: (or (op x...), (op y...)) -> (op (or x, y))
4741 if (N0.getOpcode() == N1.getOpcode())
4742 if (SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N))
4743 return Tmp;
4744
4745 // See if this is some rotate idiom.
4746 if (SDNode *Rot = MatchRotate(N0, N1, SDLoc(N)))
4747 return SDValue(Rot, 0);
4748
4749 if (SDValue Load = MatchLoadCombine(N))
4750 return Load;
4751
4752 // Simplify the operands using demanded-bits information.
4753 if (SimplifyDemandedBits(SDValue(N, 0)))
4754 return SDValue(N, 0);
4755
4756 return SDValue();
4757}
4758
4759/// Match "(X shl/srl V1) & V2" where V2 may not be present.
4760bool DAGCombiner::MatchRotateHalf(SDValue Op, SDValue &Shift, SDValue &Mask) {
4761 if (Op.getOpcode() == ISD::AND) {
4762 if (DAG.isConstantIntBuildVectorOrConstantInt(Op.getOperand(1))) {
4763 Mask = Op.getOperand(1);
4764 Op = Op.getOperand(0);
4765 } else {
4766 return false;
4767 }
4768 }
4769
4770 if (Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SHL) {
4771 Shift = Op;
4772 return true;
4773 }
4774
4775 return false;
4776}
4777
4778// Return true if we can prove that, whenever Neg and Pos are both in the
4779// range [0, EltSize), Neg == (Pos == 0 ? 0 : EltSize - Pos). This means that
4780// for two opposing shifts shift1 and shift2 and a value X with OpBits bits:
4781//
4782// (or (shift1 X, Neg), (shift2 X, Pos))
4783//
4784// reduces to a rotate in direction shift2 by Pos or (equivalently) a rotate
4785// in direction shift1 by Neg. The range [0, EltSize) means that we only need
4786// to consider shift amounts with defined behavior.
4787static bool matchRotateSub(SDValue Pos, SDValue Neg, unsigned EltSize) {
4788 // If EltSize is a power of 2 then:
4789 //
4790 // (a) (Pos == 0 ? 0 : EltSize - Pos) == (EltSize - Pos) & (EltSize - 1)
4791 // (b) Neg == Neg & (EltSize - 1) whenever Neg is in [0, EltSize).
4792 //
4793 // So if EltSize is a power of 2 and Neg is (and Neg', EltSize-1), we check
4794 // for the stronger condition:
4795 //
4796 // Neg & (EltSize - 1) == (EltSize - Pos) & (EltSize - 1) [A]
4797 //
4798 // for all Neg and Pos. Since Neg & (EltSize - 1) == Neg' & (EltSize - 1)
4799 // we can just replace Neg with Neg' for the rest of the function.
4800 //
4801 // In other cases we check for the even stronger condition:
4802 //
4803 // Neg == EltSize - Pos [B]
4804 //
4805 // for all Neg and Pos. Note that the (or ...) then invokes undefined
4806 // behavior if Pos == 0 (and consequently Neg == EltSize).
4807 //
4808 // We could actually use [A] whenever EltSize is a power of 2, but the
4809 // only extra cases that it would match are those uninteresting ones
4810 // where Neg and Pos are never in range at the same time. E.g. for
4811 // EltSize == 32, using [A] would allow a Neg of the form (sub 64, Pos)
4812 // as well as (sub 32, Pos), but:
4813 //
4814 // (or (shift1 X, (sub 64, Pos)), (shift2 X, Pos))
4815 //
4816 // always invokes undefined behavior for 32-bit X.
4817 //
4818 // Below, Mask == EltSize - 1 when using [A] and is all-ones otherwise.
4819 unsigned MaskLoBits = 0;
4820 if (Neg.getOpcode() == ISD::AND && isPowerOf2_64(EltSize)) {
4821 if (ConstantSDNode *NegC = isConstOrConstSplat(Neg.getOperand(1))) {
4822 if (NegC->getAPIntValue() == EltSize - 1) {
4823 Neg = Neg.getOperand(0);
4824 MaskLoBits = Log2_64(EltSize);
4825 }
4826 }
4827 }
4828
4829 // Check whether Neg has the form (sub NegC, NegOp1) for some NegC and NegOp1.
4830 if (Neg.getOpcode() != ISD::SUB)
4831 return false;
4832 ConstantSDNode *NegC = isConstOrConstSplat(Neg.getOperand(0));
4833 if (!NegC)
4834 return false;
4835 SDValue NegOp1 = Neg.getOperand(1);
4836
4837 // On the RHS of [A], if Pos is Pos' & (EltSize - 1), just replace Pos with
4838 // Pos'. The truncation is redundant for the purpose of the equality.
4839 if (MaskLoBits && Pos.getOpcode() == ISD::AND)
4840 if (ConstantSDNode *PosC = isConstOrConstSplat(Pos.getOperand(1)))
4841 if (PosC->getAPIntValue() == EltSize - 1)
4842 Pos = Pos.getOperand(0);
4843
4844 // The condition we need is now:
4845 //
4846 // (NegC - NegOp1) & Mask == (EltSize - Pos) & Mask
4847 //
4848 // If NegOp1 == Pos then we need:
4849 //
4850 // EltSize & Mask == NegC & Mask
4851 //
4852 // (because "x & Mask" is a truncation and distributes through subtraction).
4853 APInt Width;
4854 if (Pos == NegOp1)
4855 Width = NegC->getAPIntValue();
4856
4857 // Check for cases where Pos has the form (add NegOp1, PosC) for some PosC.
4858 // Then the condition we want to prove becomes:
4859 //
4860 // (NegC - NegOp1) & Mask == (EltSize - (NegOp1 + PosC)) & Mask
4861 //
4862 // which, again because "x & Mask" is a truncation, becomes:
4863 //
4864 // NegC & Mask == (EltSize - PosC) & Mask
4865 // EltSize & Mask == (NegC + PosC) & Mask
4866 else if (Pos.getOpcode() == ISD::ADD && Pos.getOperand(0) == NegOp1) {
4867 if (ConstantSDNode *PosC = isConstOrConstSplat(Pos.getOperand(1)))
4868 Width = PosC->getAPIntValue() + NegC->getAPIntValue();
4869 else
4870 return false;
4871 } else
4872 return false;
4873
4874 // Now we just need to check that EltSize & Mask == Width & Mask.
4875 if (MaskLoBits)
4876 // EltSize & Mask is 0 since Mask is EltSize - 1.
4877 return Width.getLoBits(MaskLoBits) == 0;
4878 return Width == EltSize;
4879}
4880
4881// A subroutine of MatchRotate used once we have found an OR of two opposite
4882// shifts of Shifted. If Neg == <operand size> - Pos then the OR reduces
4883// to both (PosOpcode Shifted, Pos) and (NegOpcode Shifted, Neg), with the
4884// former being preferred if supported. InnerPos and InnerNeg are Pos and
4885// Neg with outer conversions stripped away.
4886SDNode *DAGCombiner::MatchRotatePosNeg(SDValue Shifted, SDValue Pos,
4887 SDValue Neg, SDValue InnerPos,
4888 SDValue InnerNeg, unsigned PosOpcode,
4889 unsigned NegOpcode, const SDLoc &DL) {
4890 // fold (or (shl x, (*ext y)),
4891 // (srl x, (*ext (sub 32, y)))) ->
4892 // (rotl x, y) or (rotr x, (sub 32, y))
4893 //
4894 // fold (or (shl x, (*ext (sub 32, y))),
4895 // (srl x, (*ext y))) ->
4896 // (rotr x, y) or (rotl x, (sub 32, y))
4897 EVT VT = Shifted.getValueType();
4898 if (matchRotateSub(InnerPos, InnerNeg, VT.getScalarSizeInBits())) {
4899 bool HasPos = TLI.isOperationLegalOrCustom(PosOpcode, VT);
4900 return DAG.getNode(HasPos ? PosOpcode : NegOpcode, DL, VT, Shifted,
4901 HasPos ? Pos : Neg).getNode();
4902 }
4903
4904 return nullptr;
4905}
4906
4907// MatchRotate - Handle an 'or' of two operands. If this is one of the many
4908// idioms for rotate, and if the target supports rotation instructions, generate
4909// a rot[lr].
4910SDNode *DAGCombiner::MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL) {
4911 // Must be a legal type. Expanded 'n promoted things won't work with rotates.
4912 EVT VT = LHS.getValueType();
4913 if (!TLI.isTypeLegal(VT)) return nullptr;
4914
4915 // The target must have at least one rotate flavor.
4916 bool HasROTL = TLI.isOperationLegalOrCustom(ISD::ROTL, VT);
4917 bool HasROTR = TLI.isOperationLegalOrCustom(ISD::ROTR, VT);
4918 if (!HasROTL && !HasROTR) return nullptr;
4919
4920 // Check for truncated rotate.
4921 if (LHS.getOpcode() == ISD::TRUNCATE && RHS.getOpcode() == ISD::TRUNCATE &&
4922 LHS.getOperand(0).getValueType() == RHS.getOperand(0).getValueType()) {
4923 assert(LHS.getValueType() == RHS.getValueType())(static_cast <bool> (LHS.getValueType() == RHS.getValueType
()) ? void (0) : __assert_fail ("LHS.getValueType() == RHS.getValueType()"
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4923, __extension__ __PRETTY_FUNCTION__))
;
4924 if (SDNode *Rot = MatchRotate(LHS.getOperand(0), RHS.getOperand(0), DL)) {
4925 return DAG.getNode(ISD::TRUNCATE, SDLoc(LHS), LHS.getValueType(),
4926 SDValue(Rot, 0)).getNode();
4927 }
4928 }
4929
4930 // Match "(X shl/srl V1) & V2" where V2 may not be present.
4931 SDValue LHSShift; // The shift.
4932 SDValue LHSMask; // AND value if any.
4933 if (!MatchRotateHalf(LHS, LHSShift, LHSMask))
4934 return nullptr; // Not part of a rotate.
4935
4936 SDValue RHSShift; // The shift.
4937 SDValue RHSMask; // AND value if any.
4938 if (!MatchRotateHalf(RHS, RHSShift, RHSMask))
4939 return nullptr; // Not part of a rotate.
4940
4941 if (LHSShift.getOperand(0) != RHSShift.getOperand(0))
4942 return nullptr; // Not shifting the same value.
4943
4944 if (LHSShift.getOpcode() == RHSShift.getOpcode())
4945 return nullptr; // Shifts must disagree.
4946
4947 // Canonicalize shl to left side in a shl/srl pair.
4948 if (RHSShift.getOpcode() == ISD::SHL) {
4949 std::swap(LHS, RHS);
4950 std::swap(LHSShift, RHSShift);
4951 std::swap(LHSMask, RHSMask);
4952 }
4953
4954 unsigned EltSizeInBits = VT.getScalarSizeInBits();
4955 SDValue LHSShiftArg = LHSShift.getOperand(0);
4956 SDValue LHSShiftAmt = LHSShift.getOperand(1);
4957 SDValue RHSShiftArg = RHSShift.getOperand(0);
4958 SDValue RHSShiftAmt = RHSShift.getOperand(1);
4959
4960 // fold (or (shl x, C1), (srl x, C2)) -> (rotl x, C1)
4961 // fold (or (shl x, C1), (srl x, C2)) -> (rotr x, C2)
4962 auto MatchRotateSum = [EltSizeInBits](ConstantSDNode *LHS,
4963 ConstantSDNode *RHS) {
4964 return (LHS->getAPIntValue() + RHS->getAPIntValue()) == EltSizeInBits;
4965 };
4966 if (matchBinaryPredicate(LHSShiftAmt, RHSShiftAmt, MatchRotateSum)) {
4967 SDValue Rot = DAG.getNode(HasROTL ? ISD::ROTL : ISD::ROTR, DL, VT,
4968 LHSShiftArg, HasROTL ? LHSShiftAmt : RHSShiftAmt);
4969
4970 // If there is an AND of either shifted operand, apply it to the result.
4971 if (LHSMask.getNode() || RHSMask.getNode()) {
4972 SDValue AllOnes = DAG.getAllOnesConstant(DL, VT);
4973 SDValue Mask = AllOnes;
4974
4975 if (LHSMask.getNode()) {
4976 SDValue RHSBits = DAG.getNode(ISD::SRL, DL, VT, AllOnes, RHSShiftAmt);
4977 Mask = DAG.getNode(ISD::AND, DL, VT, Mask,
4978 DAG.getNode(ISD::OR, DL, VT, LHSMask, RHSBits));
4979 }
4980 if (RHSMask.getNode()) {
4981 SDValue LHSBits = DAG.getNode(ISD::SHL, DL, VT, AllOnes, LHSShiftAmt);
4982 Mask = DAG.getNode(ISD::AND, DL, VT, Mask,
4983 DAG.getNode(ISD::OR, DL, VT, RHSMask, LHSBits));
4984 }
4985
4986 Rot = DAG.getNode(ISD::AND, DL, VT, Rot, Mask);
4987 }
4988
4989 return Rot.getNode();
4990 }
4991
4992 // If there is a mask here, and we have a variable shift, we can't be sure
4993 // that we're masking out the right stuff.
4994 if (LHSMask.getNode() || RHSMask.getNode())
4995 return nullptr;
4996
4997 // If the shift amount is sign/zext/any-extended just peel it off.
4998 SDValue LExtOp0 = LHSShiftAmt;
4999 SDValue RExtOp0 = RHSShiftAmt;
5000 if ((LHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND ||
5001 LHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND ||
5002 LHSShiftAmt.getOpcode() == ISD::ANY_EXTEND ||
5003 LHSShiftAmt.getOpcode() == ISD::TRUNCATE) &&
5004 (RHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND ||
5005 RHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND ||
5006 RHSShiftAmt.getOpcode() == ISD::ANY_EXTEND ||
5007 RHSShiftAmt.getOpcode() == ISD::TRUNCATE)) {
5008 LExtOp0 = LHSShiftAmt.getOperand(0);
5009 RExtOp0 = RHSShiftAmt.getOperand(0);
5010 }
5011
5012 SDNode *TryL = MatchRotatePosNeg(LHSShiftArg, LHSShiftAmt, RHSShiftAmt,
5013 LExtOp0, RExtOp0, ISD::ROTL, ISD::ROTR, DL);
5014 if (TryL)
5015 return TryL;
5016
5017 SDNode *TryR = MatchRotatePosNeg(RHSShiftArg, RHSShiftAmt, LHSShiftAmt,
5018 RExtOp0, LExtOp0, ISD::ROTR, ISD::ROTL, DL);
5019 if (TryR)
5020 return TryR;
5021
5022 return nullptr;
5023}
5024
5025namespace {
5026
5027/// Represents known origin of an individual byte in load combine pattern. The
5028/// value of the byte is either constant zero or comes from memory.
5029struct ByteProvider {
5030 // For constant zero providers Load is set to nullptr. For memory providers
5031 // Load represents the node which loads the byte from memory.
5032 // ByteOffset is the offset of the byte in the value produced by the load.
5033 LoadSDNode *Load = nullptr;
5034 unsigned ByteOffset = 0;
5035
5036 ByteProvider() = default;
5037
5038 static ByteProvider getMemory(LoadSDNode *Load, unsigned ByteOffset) {
5039 return ByteProvider(Load, ByteOffset);
5040 }
5041
5042 static ByteProvider getConstantZero() { return ByteProvider(nullptr, 0); }
5043
5044 bool isConstantZero() const { return !Load; }
5045 bool isMemory() const { return Load; }
5046
5047 bool operator==(const ByteProvider &Other) const {
5048 return Other.Load == Load && Other.ByteOffset == ByteOffset;
5049 }
5050
5051private:
5052 ByteProvider(LoadSDNode *Load, unsigned ByteOffset)
5053 : Load(Load), ByteOffset(ByteOffset) {}
5054};
5055
5056} // end anonymous namespace
5057
5058/// Recursively traverses the expression calculating the origin of the requested
5059/// byte of the given value. Returns None if the provider can't be calculated.
5060///
5061/// For all the values except the root of the expression verifies that the value
5062/// has exactly one use and if it's not true return None. This way if the origin
5063/// of the byte is returned it's guaranteed that the values which contribute to
5064/// the byte are not used outside of this expression.
5065///
5066/// Because the parts of the expression are not allowed to have more than one
5067/// use this function iterates over trees, not DAGs. So it never visits the same
5068/// node more than once.
5069static const Optional<ByteProvider>
5070calculateByteProvider(SDValue Op, unsigned Index, unsigned Depth,
5071 bool Root = false) {
5072 // Typical i64 by i8 pattern requires recursion up to 8 calls depth
5073 if (Depth == 10)
5074 return None;
5075
5076 if (!Root && !Op.hasOneUse())
5077 return None;
5078
5079 assert(Op.getValueType().isScalarInteger() && "can't handle other types")(static_cast <bool> (Op.getValueType().isScalarInteger(
) && "can't handle other types") ? void (0) : __assert_fail
("Op.getValueType().isScalarInteger() && \"can't handle other types\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5079, __extension__ __PRETTY_FUNCTION__))
;
5080 unsigned BitWidth = Op.getValueSizeInBits();
5081 if (BitWidth % 8 != 0)
5082 return None;
5083 unsigned ByteWidth = BitWidth / 8;
5084 assert(Index < ByteWidth && "invalid index requested")(static_cast <bool> (Index < ByteWidth && "invalid index requested"
) ? void (0) : __assert_fail ("Index < ByteWidth && \"invalid index requested\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5084, __extension__ __PRETTY_FUNCTION__))
;
5085 (void) ByteWidth;
5086
5087 switch (Op.getOpcode()) {
5088 case ISD::OR: {
5089 auto LHS = calculateByteProvider(Op->getOperand(0), Index, Depth + 1);
5090 if (!LHS)
5091 return None;
5092 auto RHS = calculateByteProvider(Op->getOperand(1), Index, Depth + 1);
5093 if (!RHS)
5094 return None;
5095
5096 if (LHS->isConstantZero())
5097 return RHS;
5098 if (RHS->isConstantZero())
5099 return LHS;
5100 return None;
5101 }
5102 case ISD::SHL: {
5103 auto ShiftOp = dyn_cast<ConstantSDNode>(Op->getOperand(1));
5104 if (!ShiftOp)
5105 return None;
5106
5107 uint64_t BitShift = ShiftOp->getZExtValue();
5108 if (BitShift % 8 != 0)
5109 return None;
5110 uint64_t ByteShift = BitShift / 8;
5111
5112 return Index < ByteShift
5113 ? ByteProvider::getConstantZero()
5114 : calculateByteProvider(Op->getOperand(0), Index - ByteShift,
5115 Depth + 1);
5116 }
5117 case ISD::ANY_EXTEND:
5118 case ISD::SIGN_EXTEND:
5119 case ISD::ZERO_EXTEND: {
5120 SDValue NarrowOp = Op->getOperand(0);
5121 unsigned NarrowBitWidth = NarrowOp.getScalarValueSizeInBits();
5122 if (NarrowBitWidth % 8 != 0)
5123 return None;
5124 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
5125
5126 if (Index >= NarrowByteWidth)
5127 return Op.getOpcode() == ISD::ZERO_EXTEND
5128 ? Optional<ByteProvider>(ByteProvider::getConstantZero())
5129 : None;
5130 return calculateByteProvider(NarrowOp, Index, Depth + 1);
5131 }
5132 case ISD::BSWAP:
5133 return calculateByteProvider(Op->getOperand(0), ByteWidth - Index - 1,
5134 Depth + 1);
5135 case ISD::LOAD: {
5136 auto L = cast<LoadSDNode>(Op.getNode());
5137 if (L->isVolatile() || L->isIndexed())
5138 return None;
5139
5140 unsigned NarrowBitWidth = L->getMemoryVT().getSizeInBits();
5141 if (NarrowBitWidth % 8 != 0)
5142 return None;
5143 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
5144
5145 if (Index >= NarrowByteWidth)
5146 return L->getExtensionType() == ISD::ZEXTLOAD
5147 ? Optional<ByteProvider>(ByteProvider::getConstantZero())
5148 : None;
5149 return ByteProvider::getMemory(L, Index);
5150 }
5151 }
5152
5153 return None;
5154}
5155
5156/// Match a pattern where a wide type scalar value is loaded by several narrow
5157/// loads and combined by shifts and ors. Fold it into a single load or a load
5158/// and a BSWAP if the targets supports it.
5159///
5160/// Assuming little endian target:
5161/// i8 *a = ...
5162/// i32 val = a[0] | (a[1] << 8) | (a[2] << 16) | (a[3] << 24)
5163/// =>
5164/// i32 val = *((i32)a)
5165///
5166/// i8 *a = ...
5167/// i32 val = (a[0] << 24) | (a[1] << 16) | (a[2] << 8) | a[3]
5168/// =>
5169/// i32 val = BSWAP(*((i32)a))
5170///
5171/// TODO: This rule matches complex patterns with OR node roots and doesn't
5172/// interact well with the worklist mechanism. When a part of the pattern is
5173/// updated (e.g. one of the loads) its direct users are put into the worklist,
5174/// but the root node of the pattern which triggers the load combine is not
5175/// necessarily a direct user of the changed node. For example, once the address
5176/// of t28 load is reassociated load combine won't be triggered:
5177/// t25: i32 = add t4, Constant:i32<2>
5178/// t26: i64 = sign_extend t25
5179/// t27: i64 = add t2, t26
5180/// t28: i8,ch = load<LD1[%tmp9]> t0, t27, undef:i64
5181/// t29: i32 = zero_extend t28
5182/// t32: i32 = shl t29, Constant:i8<8>
5183/// t33: i32 = or t23, t32
5184/// As a possible fix visitLoad can check if the load can be a part of a load
5185/// combine pattern and add corresponding OR roots to the worklist.
5186SDValue DAGCombiner::MatchLoadCombine(SDNode *N) {
5187 assert(N->getOpcode() == ISD::OR &&(static_cast <bool> (N->getOpcode() == ISD::OR &&
"Can only match load combining against OR nodes") ? void (0)
: __assert_fail ("N->getOpcode() == ISD::OR && \"Can only match load combining against OR nodes\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5188, __extension__ __PRETTY_FUNCTION__))
5188 "Can only match load combining against OR nodes")(static_cast <bool> (N->getOpcode() == ISD::OR &&
"Can only match load combining against OR nodes") ? void (0)
: __assert_fail ("N->getOpcode() == ISD::OR && \"Can only match load combining against OR nodes\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5188, __extension__ __PRETTY_FUNCTION__))
;
5189
5190 // Handles simple types only
5191 EVT VT = N->getValueType(0);
5192 if (VT != MVT::i16 && VT != MVT::i32 && VT != MVT::i64)
5193 return SDValue();
5194 unsigned ByteWidth = VT.getSizeInBits() / 8;
5195
5196 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5197 // Before legalize we can introduce too wide illegal loads which will be later
5198 // split into legal sized loads. This enables us to combine i64 load by i8
5199 // patterns to a couple of i32 loads on 32 bit targets.
5200 if (LegalOperations && !TLI.isOperationLegal(ISD::LOAD, VT))
5201 return SDValue();
5202
5203 std::function<unsigned(unsigned, unsigned)> LittleEndianByteAt = [](
5204 unsigned BW, unsigned i) { return i; };
5205 std::function<unsigned(unsigned, unsigned)> BigEndianByteAt = [](
5206 unsigned BW, unsigned i) { return BW - i - 1; };
5207
5208 bool IsBigEndianTarget = DAG.getDataLayout().isBigEndian();
5209 auto MemoryByteOffset = [&] (ByteProvider P) {
5210 assert(P.isMemory() && "Must be a memory byte provider")(static_cast <bool> (P.isMemory() && "Must be a memory byte provider"
) ? void (0) : __assert_fail ("P.isMemory() && \"Must be a memory byte provider\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5210, __extension__ __PRETTY_FUNCTION__))
;
5211 unsigned LoadBitWidth = P.Load->getMemoryVT().getSizeInBits();
5212 assert(LoadBitWidth % 8 == 0 &&(static_cast <bool> (LoadBitWidth % 8 == 0 && "can only analyze providers for individual bytes not bit"
) ? void (0) : __assert_fail ("LoadBitWidth % 8 == 0 && \"can only analyze providers for individual bytes not bit\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5213, __extension__ __PRETTY_FUNCTION__))
5213 "can only analyze providers for individual bytes not bit")(static_cast <bool> (LoadBitWidth % 8 == 0 && "can only analyze providers for individual bytes not bit"
) ? void (0) : __assert_fail ("LoadBitWidth % 8 == 0 && \"can only analyze providers for individual bytes not bit\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5213, __extension__ __PRETTY_FUNCTION__))
;
5214 unsigned LoadByteWidth = LoadBitWidth / 8;
5215 return IsBigEndianTarget
5216 ? BigEndianByteAt(LoadByteWidth, P.ByteOffset)
5217 : LittleEndianByteAt(LoadByteWidth, P.ByteOffset);
5218 };
5219
5220 Optional<BaseIndexOffset> Base;
5221 SDValue Chain;
5222
5223 SmallSet<LoadSDNode *, 8> Loads;
5224 Optional<ByteProvider> FirstByteProvider;
5225 int64_t FirstOffset = INT64_MAX(9223372036854775807L);
5226
5227 // Check if all the bytes of the OR we are looking at are loaded from the same
5228 // base address. Collect bytes offsets from Base address in ByteOffsets.
5229 SmallVector<int64_t, 4> ByteOffsets(ByteWidth);
5230 for (unsigned i = 0; i < ByteWidth; i++) {
5231 auto P = calculateByteProvider(SDValue(N, 0), i, 0, /*Root=*/true);
5232 if (!P || !P->isMemory()) // All the bytes must be loaded from memory
5233 return SDValue();
5234
5235 LoadSDNode *L = P->Load;
5236 assert(L->hasNUsesOfValue(1, 0) && !L->isVolatile() && !L->isIndexed() &&(static_cast <bool> (L->hasNUsesOfValue(1, 0) &&
!L->isVolatile() && !L->isIndexed() &&
"Must be enforced by calculateByteProvider") ? void (0) : __assert_fail
("L->hasNUsesOfValue(1, 0) && !L->isVolatile() && !L->isIndexed() && \"Must be enforced by calculateByteProvider\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5237, __extension__ __PRETTY_FUNCTION__))
5237 "Must be enforced by calculateByteProvider")(static_cast <bool> (L->hasNUsesOfValue(1, 0) &&
!L->isVolatile() && !L->isIndexed() &&
"Must be enforced by calculateByteProvider") ? void (0) : __assert_fail
("L->hasNUsesOfValue(1, 0) && !L->isVolatile() && !L->isIndexed() && \"Must be enforced by calculateByteProvider\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5237, __extension__ __PRETTY_FUNCTION__))
;
5238 assert(L->getOffset().isUndef() && "Unindexed load must have undef offset")(static_cast <bool> (L->getOffset().isUndef() &&
"Unindexed load must have undef offset") ? void (0) : __assert_fail
("L->getOffset().isUndef() && \"Unindexed load must have undef offset\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5238, __extension__ __PRETTY_FUNCTION__))
;
5239
5240 // All loads must share the same chain
5241 SDValue LChain = L->getChain();
5242 if (!Chain)
5243 Chain = LChain;
5244 else if (Chain != LChain)
5245 return SDValue();
5246
5247 // Loads must share the same base address
5248 BaseIndexOffset Ptr = BaseIndexOffset::match(L, DAG);
5249 int64_t ByteOffsetFromBase = 0;
5250 if (!Base)
5251 Base = Ptr;
5252 else if (!Base->equalBaseIndex(Ptr, DAG, ByteOffsetFromBase))
5253 return SDValue();
5254
5255 // Calculate the offset of the current byte from the base address
5256 ByteOffsetFromBase += MemoryByteOffset(*P);
5257 ByteOffsets[i] = ByteOffsetFromBase;
5258
5259 // Remember the first byte load
5260 if (ByteOffsetFromBase < FirstOffset) {
5261 FirstByteProvider = P;
5262 FirstOffset = ByteOffsetFromBase;
5263 }
5264
5265 Loads.insert(L);
5266 }
5267 assert(!Loads.empty() && "All the bytes of the value must be loaded from "(static_cast <bool> (!Loads.empty() && "All the bytes of the value must be loaded from "
"memory, so there must be at least one load which produces the value"
) ? void (0) : __assert_fail ("!Loads.empty() && \"All the bytes of the value must be loaded from \" \"memory, so there must be at least one load which produces the value\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5268, __extension__ __PRETTY_FUNCTION__))
5268 "memory, so there must be at least one load which produces the value")(static_cast <bool> (!Loads.empty() && "All the bytes of the value must be loaded from "
"memory, so there must be at least one load which produces the value"
) ? void (0) : __assert_fail ("!Loads.empty() && \"All the bytes of the value must be loaded from \" \"memory, so there must be at least one load which produces the value\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5268, __extension__ __PRETTY_FUNCTION__))
;
5269 assert(Base && "Base address of the accessed memory location must be set")(static_cast <bool> (Base && "Base address of the accessed memory location must be set"
) ? void (0) : __assert_fail ("Base && \"Base address of the accessed memory location must be set\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5269, __extension__ __PRETTY_FUNCTION__))
;
5270 assert(FirstOffset != INT64_MAX && "First byte offset must be set")(static_cast <bool> (FirstOffset != (9223372036854775807L
) && "First byte offset must be set") ? void (0) : __assert_fail
("FirstOffset != INT64_MAX && \"First byte offset must be set\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5270, __extension__ __PRETTY_FUNCTION__))
;
5271
5272 // Check if the bytes of the OR we are looking at match with either big or
5273 // little endian value load
5274 bool BigEndian = true, LittleEndian = true;
5275 for (unsigned i = 0; i < ByteWidth; i++) {
5276 int64_t CurrentByteOffset = ByteOffsets[i] - FirstOffset;
5277 LittleEndian &= CurrentByteOffset == LittleEndianByteAt(ByteWidth, i);
5278 BigEndian &= CurrentByteOffset == BigEndianByteAt(ByteWidth, i);
5279 if (!BigEndian && !LittleEndian)
5280 return SDValue();
5281 }
5282 assert((BigEndian != LittleEndian) && "should be either or")(static_cast <bool> ((BigEndian != LittleEndian) &&
"should be either or") ? void (0) : __assert_fail ("(BigEndian != LittleEndian) && \"should be either or\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5282, __extension__ __PRETTY_FUNCTION__))
;
5283 assert(FirstByteProvider && "must be set")(static_cast <bool> (FirstByteProvider && "must be set"
) ? void (0) : __assert_fail ("FirstByteProvider && \"must be set\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5283, __extension__ __PRETTY_FUNCTION__))
;
5284
5285 // Ensure that the first byte is loaded from zero offset of the first load.
5286 // So the combined value can be loaded from the first load address.
5287 if (MemoryByteOffset(*FirstByteProvider) != 0)
5288 return SDValue();
5289 LoadSDNode *FirstLoad = FirstByteProvider->Load;
5290
5291 // The node we are looking at matches with the pattern, check if we can
5292 // replace it with a single load and bswap if needed.
5293
5294 // If the load needs byte swap check if the target supports it
5295 bool NeedsBswap = IsBigEndianTarget != BigEndian;
5296
5297 // Before legalize we can introduce illegal bswaps which will be later
5298 // converted to an explicit bswap sequence. This way we end up with a single
5299 // load and byte shuffling instead of several loads and byte shuffling.
5300 if (NeedsBswap && LegalOperations && !TLI.isOperationLegal(ISD::BSWAP, VT))
5301 return SDValue();
5302
5303 // Check that a load of the wide type is both allowed and fast on the target
5304 bool Fast = false;
5305 bool Allowed = TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(),
5306 VT, FirstLoad->getAddressSpace(),
5307 FirstLoad->getAlignment(), &Fast);
5308 if (!Allowed || !Fast)
5309 return SDValue();
5310
5311 SDValue NewLoad =
5312 DAG.getLoad(VT, SDLoc(N), Chain, FirstLoad->getBasePtr(),
5313 FirstLoad->getPointerInfo(), FirstLoad->getAlignment());
5314
5315 // Transfer chain users from old loads to the new load.
5316 for (LoadSDNode *L : Loads)
5317 DAG.ReplaceAllUsesOfValueWith(SDValue(L, 1), SDValue(NewLoad.getNode(), 1));
5318
5319 return NeedsBswap ? DAG.getNode(ISD::BSWAP, SDLoc(N), VT, NewLoad) : NewLoad;
5320}
5321
5322SDValue DAGCombiner::visitXOR(SDNode *N) {
5323 SDValue N0 = N->getOperand(0);
5324 SDValue N1 = N->getOperand(1);
5325 EVT VT = N0.getValueType();
5326
5327 // fold vector ops
5328 if (VT.isVector()) {
5329 if (SDValue FoldedVOp = SimplifyVBinOp(N))
5330 return FoldedVOp;
5331
5332 // fold (xor x, 0) -> x, vector edition
5333 if (ISD::isBuildVectorAllZeros(N0.getNode()))
5334 return N1;
5335 if (ISD::isBuildVectorAllZeros(N1.getNode()))
5336 return N0;
5337 }
5338
5339 // fold (xor undef, undef) -> 0. This is a common idiom (misuse).
5340 if (N0.isUndef() && N1.isUndef())
5341 return DAG.getConstant(0, SDLoc(N), VT);
5342 // fold (xor x, undef) -> undef
5343 if (N0.isUndef())
5344 return N0;
5345 if (N1.isUndef())
5346 return N1;
5347 // fold (xor c1, c2) -> c1^c2
5348 ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
5349 ConstantSDNode *N1C = getAsNonOpaqueConstant(N1);
5350 if (N0C && N1C)
5351 return DAG.FoldConstantArithmetic(ISD::XOR, SDLoc(N), VT, N0C, N1C);
5352 // canonicalize constant to RHS
5353 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
5354 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
5355 return DAG.getNode(ISD::XOR, SDLoc(N), VT, N1, N0);
5356 // fold (xor x, 0) -> x
5357 if (isNullConstant(N1))
5358 return N0;
5359
5360 if (SDValue NewSel = foldBinOpIntoSelect(N))
5361 return NewSel;
5362
5363 // reassociate xor
5364 if (SDValue RXOR = ReassociateOps(ISD::XOR, SDLoc(N), N0, N1))
5365 return RXOR;
5366
5367 // fold !(x cc y) -> (x !cc y)
5368 SDValue LHS, RHS, CC;
5369 if (TLI.isConstTrueVal(N1.getNode()) && isSetCCEquivalent(N0, LHS, RHS, CC)) {
5370 bool isInt = LHS.getValueType().isInteger();
5371 ISD::CondCode NotCC = ISD::getSetCCInverse(cast<CondCodeSDNode>(CC)->get(),
5372 isInt);
5373
5374 if (!LegalOperations ||
5375 TLI.isCondCodeLegal(NotCC, LHS.getSimpleValueType())) {
5376 switch (N0.getOpcode()) {
5377 default:
5378 llvm_unreachable("Unhandled SetCC Equivalent!")::llvm::llvm_unreachable_internal("Unhandled SetCC Equivalent!"
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5378)
;
5379 case ISD::SETCC:
5380 return DAG.getSetCC(SDLoc(N0), VT, LHS, RHS, NotCC);
5381 case ISD::SELECT_CC:
5382 return DAG.getSelectCC(SDLoc(N0), LHS, RHS, N0.getOperand(2),
5383 N0.getOperand(3), NotCC);
5384 }
5385 }
5386 }
5387
5388 // fold (not (zext (setcc x, y))) -> (zext (not (setcc x, y)))
5389 if (isOneConstant(N1) && N0.getOpcode() == ISD::ZERO_EXTEND &&
5390 N0.getNode()->hasOneUse() &&
5391 isSetCCEquivalent(N0.getOperand(0), LHS, RHS, CC)){
5392 SDValue V = N0.getOperand(0);
5393 SDLoc DL(N0);
5394 V = DAG.getNode(ISD::XOR, DL, V.getValueType(), V,
5395 DAG.getConstant(1, DL, V.getValueType()));
5396 AddToWorklist(V.getNode());
5397 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, V);
5398 }
5399
5400 // fold (not (or x, y)) -> (and (not x), (not y)) iff x or y are setcc
5401 if (isOneConstant(N1) && VT == MVT::i1 && N0.hasOneUse() &&
5402 (N0.getOpcode() == ISD::OR || N0.getOpcode() == ISD::AND)) {
5403 SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1);
5404 if (isOneUseSetCC(RHS) || isOneUseSetCC(LHS)) {
5405 unsigned NewOpcode = N0.getOpcode() == ISD::AND ? ISD::OR : ISD::AND;
5406 LHS = DAG.getNode(ISD::XOR, SDLoc(LHS), VT, LHS, N1); // LHS = ~LHS
5407 RHS = DAG.getNode(ISD::XOR, SDLoc(RHS), VT, RHS, N1); // RHS = ~RHS
5408 AddToWorklist(LHS.getNode()); AddToWorklist(RHS.getNode());
5409 return DAG.getNode(NewOpcode, SDLoc(N), VT, LHS, RHS);
5410 }
5411 }
5412 // fold (not (or x, y)) -> (and (not x), (not y)) iff x or y are constants
5413 if (isAllOnesConstant(N1) && N0.hasOneUse() &&
5414 (N0.getOpcode() == ISD::OR || N0.getOpcode() == ISD::AND)) {
5415 SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1);
5416 if (isa<ConstantSDNode>(RHS) || isa<ConstantSDNode>(LHS)) {
5417 unsigned NewOpcode = N0.getOpcode() == ISD::AND ? ISD::OR : ISD::AND;
5418 LHS = DAG.getNode(ISD::XOR, SDLoc(LHS), VT, LHS, N1); // LHS = ~LHS
5419 RHS = DAG.getNode(ISD::XOR, SDLoc(RHS), VT, RHS, N1); // RHS = ~RHS
5420 AddToWorklist(LHS.getNode()); AddToWorklist(RHS.getNode());
5421 return DAG.getNode(NewOpcode, SDLoc(N), VT, LHS, RHS);
5422 }
5423 }
5424 // fold (xor (and x, y), y) -> (and (not x), y)
5425 if (N0.getOpcode() == ISD::AND && N0.getNode()->hasOneUse() &&
5426 N0->getOperand(1) == N1) {
5427 SDValue X = N0->getOperand(0);
5428 SDValue NotX = DAG.getNOT(SDLoc(X), X, VT);
5429 AddToWorklist(NotX.getNode());
5430 return DAG.getNode(ISD::AND, SDLoc(N), VT, NotX, N1);
5431 }
5432
5433 // fold Y = sra (X, size(X)-1); xor (add (X, Y), Y) -> (abs X)
5434 unsigned OpSizeInBits = VT.getScalarSizeInBits();
5435 if (N0.getOpcode() == ISD::ADD && N0.getOperand(1) == N1 &&
5436 N1.getOpcode() == ISD::SRA && N1.getOperand(0) == N0.getOperand(0) &&
5437 TLI.isOperationLegalOrCustom(ISD::ABS, VT)) {
5438 if (ConstantSDNode *C = isConstOrConstSplat(N1.getOperand(1)))
5439 if (C->getAPIntValue() == (OpSizeInBits - 1))
5440 return DAG.getNode(ISD::ABS, SDLoc(N), VT, N0.getOperand(0));
5441 }
5442
5443 // fold (xor x, x) -> 0
5444 if (N0 == N1)
5445 return tryFoldToZero(SDLoc(N), TLI, VT, DAG, LegalOperations, LegalTypes);
5446
5447 // fold (xor (shl 1, x), -1) -> (rotl ~1, x)
5448 // Here is a concrete example of this equivalence:
5449 // i16 x == 14
5450 // i16 shl == 1 << 14 == 16384 == 0b0100000000000000
5451 // i16 xor == ~(1 << 14) == 49151 == 0b1011111111111111
5452 //
5453 // =>
5454 //
5455 // i16 ~1 == 0b1111111111111110
5456 // i16 rol(~1, 14) == 0b1011111111111111
5457 //
5458 // Some additional tips to help conceptualize this transform:
5459 // - Try to see the operation as placing a single zero in a value of all ones.
5460 // - There exists no value for x which would allow the result to contain zero.
5461 // - Values of x larger than the bitwidth are undefined and do not require a
5462 // consistent result.
5463 // - Pushing the zero left requires shifting one bits in from the right.
5464 // A rotate left of ~1 is a nice way of achieving the desired result.
5465 if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT) && N0.getOpcode() == ISD::SHL
5466 && isAllOnesConstant(N1) && isOneConstant(N0.getOperand(0))) {
5467 SDLoc DL(N);
5468 return DAG.getNode(ISD::ROTL, DL, VT, DAG.getConstant(~1, DL, VT),
5469 N0.getOperand(1));
5470 }
5471
5472 // Simplify: xor (op x...), (op y...) -> (op (xor x, y))
5473 if (N0.getOpcode() == N1.getOpcode())
5474 if (SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N))
5475 return Tmp;
5476
5477 // Simplify the expression using non-local knowledge.
5478 if (SimplifyDemandedBits(SDValue(N, 0)))
5479 return SDValue(N, 0);
5480
5481 return SDValue();
5482}
5483
5484/// Handle transforms common to the three shifts, when the shift amount is a
5485/// constant.
5486SDValue DAGCombiner::visitShiftByConstant(SDNode *N, ConstantSDNode *Amt) {
5487 SDNode *LHS = N->getOperand(0).getNode();
5488 if (!LHS->hasOneUse()) return SDValue();
5489
5490 // We want to pull some binops through shifts, so that we have (and (shift))
5491 // instead of (shift (and)), likewise for add, or, xor, etc. This sort of
5492 // thing happens with address calculations, so it's important to canonicalize
5493 // it.
5494 bool HighBitSet = false; // Can we transform this if the high bit is set?
5495
5496 switch (LHS->getOpcode()) {
5497 default: return SDValue();
5498 case ISD::OR:
5499 case ISD::XOR:
5500 HighBitSet = false; // We can only transform sra if the high bit is clear.
5501 break;
5502 case ISD::AND:
5503 HighBitSet = true; // We can only transform sra if the high bit is set.
5504 break;
5505 case ISD::ADD:
5506 if (N->getOpcode() != ISD::SHL)
5507 return SDValue(); // only shl(add) not sr[al](add).
5508 HighBitSet = false; // We can only transform sra if the high bit is clear.
5509 break;
5510 }
5511
5512 // We require the RHS of the binop to be a constant and not opaque as well.
5513 ConstantSDNode *BinOpCst = getAsNonOpaqueConstant(LHS->getOperand(1));
5514 if (!BinOpCst) return SDValue();
5515
5516 // FIXME: disable this unless the input to the binop is a shift by a constant
5517 // or is copy/select.Enable this in other cases when figure out it's exactly profitable.
5518 SDNode *BinOpLHSVal = LHS->getOperand(0).getNode();
5519 bool isShift = BinOpLHSVal->getOpcode() == ISD::SHL ||
5520 BinOpLHSVal->getOpcode() == ISD::SRA ||
5521 BinOpLHSVal->getOpcode() == ISD::SRL;
5522 bool isCopyOrSelect = BinOpLHSVal->getOpcode() == ISD::CopyFromReg ||
5523 BinOpLHSVal->getOpcode() == ISD::SELECT;
5524
5525 if ((!isShift || !isa<ConstantSDNode>(BinOpLHSVal->getOperand(1))) &&
5526 !isCopyOrSelect)
5527 return SDValue();
5528
5529 if (isCopyOrSelect && N->hasOneUse())
5530 return SDValue();
5531
5532 EVT VT = N->getValueType(0);
5533
5534 // If this is a signed shift right, and the high bit is modified by the
5535 // logical operation, do not perform the transformation. The highBitSet
5536 // boolean indicates the value of the high bit of the constant which would
5537 // cause it to be modified for this operation.
5538 if (N->getOpcode() == ISD::SRA) {
5539 bool BinOpRHSSignSet = BinOpCst->getAPIntValue().isNegative();
5540 if (BinOpRHSSignSet != HighBitSet)
5541 return SDValue();
5542 }
5543
5544 if (!TLI.isDesirableToCommuteWithShift(LHS))
5545 return SDValue();
5546
5547 // Fold the constants, shifting the binop RHS by the shift amount.
5548 SDValue NewRHS = DAG.getNode(N->getOpcode(), SDLoc(LHS->getOperand(1)),
5549 N->getValueType(0),
5550 LHS->getOperand(1), N->getOperand(1));
5551 assert(isa<ConstantSDNode>(NewRHS) && "Folding was not successful!")(static_cast <bool> (isa<ConstantSDNode>(NewRHS) &&
"Folding was not successful!") ? void (0) : __assert_fail ("isa<ConstantSDNode>(NewRHS) && \"Folding was not successful!\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5551, __extension__ __PRETTY_FUNCTION__))
;
5552
5553 // Create the new shift.
5554 SDValue NewShift = DAG.getNode(N->getOpcode(),
5555 SDLoc(LHS->getOperand(0)),
5556 VT, LHS->getOperand(0), N->getOperand(1));
5557
5558 // Create the new binop.
5559 return DAG.getNode(LHS->getOpcode(), SDLoc(N), VT, NewShift, NewRHS);
5560}
5561
5562SDValue DAGCombiner::distributeTruncateThroughAnd(SDNode *N) {
5563 assert(N->getOpcode() == ISD::TRUNCATE)(static_cast <bool> (N->getOpcode() == ISD::TRUNCATE
) ? void (0) : __assert_fail ("N->getOpcode() == ISD::TRUNCATE"
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5563, __extension__ __PRETTY_FUNCTION__))
;
5564 assert(N->getOperand(0).getOpcode() == ISD::AND)(static_cast <bool> (N->getOperand(0).getOpcode() ==
ISD::AND) ? void (0) : __assert_fail ("N->getOperand(0).getOpcode() == ISD::AND"
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5564, __extension__ __PRETTY_FUNCTION__))
;
5565
5566 // (truncate:TruncVT (and N00, N01C)) -> (and (truncate:TruncVT N00), TruncC)
5567 if (N->hasOneUse() && N->getOperand(0).hasOneUse()) {
5568 SDValue N01 = N->getOperand(0).getOperand(1);
5569 if (isConstantOrConstantVector(N01, /* NoOpaques */ true)) {
5570 SDLoc DL(N);
5571 EVT TruncVT = N->getValueType(0);
5572 SDValue N00 = N->getOperand(0).getOperand(0);
5573 SDValue Trunc00 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N00);
5574 SDValue Trunc01 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N01);
5575 AddToWorklist(Trunc00.getNode());
5576 AddToWorklist(Trunc01.getNode());
5577 return DAG.getNode(ISD::AND, DL, TruncVT, Trunc00, Trunc01);
5578 }
5579 }
5580
5581 return SDValue();
5582}
5583
5584SDValue DAGCombiner::visitRotate(SDNode *N) {
5585 SDLoc dl(N);
5586 SDValue N0 = N->getOperand(0);
5587 SDValue N1 = N->getOperand(1);
5588 EVT VT = N->getValueType(0);
5589 unsigned Bitsize = VT.getScalarSizeInBits();
5590
5591 // fold (rot x, 0) -> x
5592 if (isNullConstantOrNullSplatConstant(N1))
5593 return N0;
5594
5595 // fold (rot x, c) -> (rot x, c % BitSize)
5596 if (ConstantSDNode *Cst = isConstOrConstSplat(N1)) {
5597 if (Cst->getAPIntValue().uge(Bitsize)) {
5598 uint64_t RotAmt = Cst->getAPIntValue().urem(Bitsize);
5599 return DAG.getNode(N->getOpcode(), dl, VT, N0,
5600 DAG.getConstant(RotAmt, dl, N1.getValueType()));
5601 }
5602 }
5603
5604 // fold (rot* x, (trunc (and y, c))) -> (rot* x, (and (trunc y), (trunc c))).
5605 if (N1.getOpcode() == ISD::TRUNCATE &&
5606 N1.getOperand(0).getOpcode() == ISD::AND) {
5607 if (SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode()))
5608 return DAG.getNode(N->getOpcode(), dl, VT, N0, NewOp1);
5609 }
5610
5611 unsigned NextOp = N0.getOpcode();
5612 // fold (rot* (rot* x, c2), c1) -> (rot* x, c1 +- c2 % bitsize)
5613 if (NextOp == ISD::ROTL || NextOp == ISD::ROTR) {
5614 SDNode *C1 = DAG.isConstantIntBuildVectorOrConstantInt(N1);
5615 SDNode *C2 = DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1));
5616 if (C1 && C2 && C1->getValueType(0) == C2->getValueType(0)) {
5617 EVT ShiftVT = C1->getValueType(0);
5618 bool SameSide = (N->getOpcode() == NextOp);
5619 unsigned CombineOp = SameSide ? ISD::ADD : ISD::SUB;
5620 if (SDValue CombinedShift =
5621 DAG.FoldConstantArithmetic(CombineOp, dl, ShiftVT, C1, C2)) {
5622 SDValue BitsizeC = DAG.getConstant(Bitsize, dl, ShiftVT);
5623 SDValue CombinedShiftNorm = DAG.FoldConstantArithmetic(
5624 ISD::SREM, dl, ShiftVT, CombinedShift.getNode(),
5625 BitsizeC.getNode());
5626 return DAG.getNode(N->getOpcode(), dl, VT, N0->getOperand(0),
5627 CombinedShiftNorm);
5628 }
5629 }
5630 }
5631 return SDValue();
5632}
5633
5634SDValue DAGCombiner::visitSHL(SDNode *N) {
5635 SDValue N0 = N->getOperand(0);
5636 SDValue N1 = N->getOperand(1);
5637 EVT VT = N0.getValueType();
5638 unsigned OpSizeInBits = VT.getScalarSizeInBits();
5639
5640 // fold vector ops
5641 if (VT.isVector()) {
5642 if (SDValue FoldedVOp = SimplifyVBinOp(N))
5643 return FoldedVOp;
5644
5645 BuildVectorSDNode *N1CV = dyn_cast<BuildVectorSDNode>(N1);
5646 // If setcc produces all-one true value then:
5647 // (shl (and (setcc) N01CV) N1CV) -> (and (setcc) N01CV<<N1CV)
5648 if (N1CV && N1CV->isConstant()) {
5649 if (N0.getOpcode() == ISD::AND) {
5650 SDValue N00 = N0->getOperand(0);
5651 SDValue N01 = N0->getOperand(1);
5652 BuildVectorSDNode *N01CV = dyn_cast<BuildVectorSDNode>(N01);
5653
5654 if (N01CV && N01CV->isConstant() && N00.getOpcode() == ISD::SETCC &&
5655 TLI.getBooleanContents(N00.getOperand(0).getValueType()) ==
5656 TargetLowering::ZeroOrNegativeOneBooleanContent) {
5657 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT,
5658 N01CV, N1CV))
5659 return DAG.getNode(ISD::AND, SDLoc(N), VT, N00, C);
5660 }
5661 }
5662 }
5663 }
5664
5665 ConstantSDNode *N1C = isConstOrConstSplat(N1);
5666
5667 // fold (shl c1, c2) -> c1<<c2
5668 ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
5669 if (N0C && N1C && !N1C->isOpaque())
5670 return DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT, N0C, N1C);
5671 // fold (shl 0, x) -> 0
5672 if (isNullConstantOrNullSplatConstant(N0))
5673 return N0;
5674 // fold (shl x, c >= size(x)) -> undef
5675 // NOTE: ALL vector elements must be too big to avoid partial UNDEFs.
5676 auto MatchShiftTooBig = [OpSizeInBits](ConstantSDNode *Val) {
5677 return Val->getAPIntValue().uge(OpSizeInBits);
5678 };
5679 if (matchUnaryPredicate(N1, MatchShiftTooBig))
5680 return DAG.getUNDEF(VT);
5681 // fold (shl x, 0) -> x
5682 if (N1C && N1C->isNullValue())
5683 return N0;
5684 // fold (shl undef, x) -> 0
5685 if (N0.isUndef())
5686 return DAG.getConstant(0, SDLoc(N), VT);
5687
5688 if (SDValue NewSel = foldBinOpIntoSelect(N))
5689 return NewSel;
5690
5691 // if (shl x, c) is known to be zero, return 0
5692 if (DAG.MaskedValueIsZero(SDValue(N, 0),
5693 APInt::getAllOnesValue(OpSizeInBits)))
5694 return DAG.getConstant(0, SDLoc(N), VT);
5695 // fold (shl x, (trunc (and y, c))) -> (shl x, (and (trunc y), (trunc c))).
5696 if (N1.getOpcode() == ISD::TRUNCATE &&
5697 N1.getOperand(0).getOpcode() == ISD::AND) {
5698 if (SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode()))
5699 return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, NewOp1);
5700 }
5701
5702 if (N1C && SimplifyDemandedBits(SDValue(N, 0)))
5703 return SDValue(N, 0);
5704
5705 // fold (shl (shl x, c1), c2) -> 0 or (shl x, (add c1, c2))
5706 if (N0.getOpcode() == ISD::SHL) {
5707 auto MatchOutOfRange = [OpSizeInBits](ConstantSDNode *LHS,
5708 ConstantSDNode *RHS) {
5709 APInt c1 = LHS->getAPIntValue();
5710 APInt c2 = RHS->getAPIntValue();
5711 zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
5712 return (c1 + c2).uge(OpSizeInBits);
5713 };
5714 if (matchBinaryPredicate(N1, N0.getOperand(1), MatchOutOfRange))
5715 return DAG.getConstant(0, SDLoc(N), VT);
5716
5717 auto MatchInRange = [OpSizeInBits](ConstantSDNode *LHS,
5718 ConstantSDNode *RHS) {
5719 APInt c1 = LHS->getAPIntValue();
5720 APInt c2 = RHS->getAPIntValue();
5721 zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
5722 return (c1 + c2).ult(OpSizeInBits);
5723 };
5724 if (matchBinaryPredicate(N1, N0.getOperand(1), MatchInRange)) {
5725 SDLoc DL(N);
5726 EVT ShiftVT = N1.getValueType();
5727 SDValue Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, N1, N0.getOperand(1));
5728 return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Sum);
5729 }
5730 }
5731
5732 // fold (shl (ext (shl x, c1)), c2) -> (ext (shl x, (add c1, c2)))
5733 // For this to be valid, the second form must not preserve any of the bits
5734 // that are shifted out by the inner shift in the first form. This means
5735 // the outer shift size must be >= the number of bits added by the ext.
5736 // As a corollary, we don't care what kind of ext it is.
5737 if (N1C && (N0.getOpcode() == ISD::ZERO_EXTEND ||
5738 N0.getOpcode() == ISD::ANY_EXTEND ||
5739 N0.getOpcode() == ISD::SIGN_EXTEND) &&
5740 N0.getOperand(0).getOpcode() == ISD::SHL) {
5741 SDValue N0Op0 = N0.getOperand(0);
5742 if (ConstantSDNode *N0Op0C1 = isConstOrConstSplat(N0Op0.getOperand(1))) {
5743 APInt c1 = N0Op0C1->getAPIntValue();
5744 APInt c2 = N1C->getAPIntValue();
5745 zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
5746
5747 EVT InnerShiftVT = N0Op0.getValueType();
5748 uint64_t InnerShiftSize = InnerShiftVT.getScalarSizeInBits();
5749 if (c2.uge(OpSizeInBits - InnerShiftSize)) {
5750 SDLoc DL(N0);
5751 APInt Sum = c1 + c2;
5752 if (Sum.uge(OpSizeInBits))
5753 return DAG.getConstant(0, DL, VT);
5754
5755 return DAG.getNode(
5756 ISD::SHL, DL, VT,
5757 DAG.getNode(N0.getOpcode(), DL, VT, N0Op0->getOperand(0)),
5758 DAG.getConstant(Sum.getZExtValue(), DL, N1.getValueType()));
5759 }
5760 }
5761 }
5762
5763 // fold (shl (zext (srl x, C)), C) -> (zext (shl (srl x, C), C))
5764 // Only fold this if the inner zext has no other uses to avoid increasing
5765 // the total number of instructions.
5766 if (N1C && N0.getOpcode() == ISD::ZERO_EXTEND && N0.hasOneUse() &&
5767 N0.getOperand(0).getOpcode() == ISD::SRL) {
5768 SDValue N0Op0 = N0.getOperand(0);
5769 if (ConstantSDNode *N0Op0C1 = isConstOrConstSplat(N0Op0.getOperand(1))) {
5770 if (N0Op0C1->getAPIntValue().ult(VT.getScalarSizeInBits())) {
5771 uint64_t c1 = N0Op0C1->getZExtValue();
5772 uint64_t c2 = N1C->getZExtValue();
5773 if (c1 == c2) {
5774 SDValue NewOp0 = N0.getOperand(0);
5775 EVT CountVT = NewOp0.getOperand(1).getValueType();
5776 SDLoc DL(N);
5777 SDValue NewSHL = DAG.getNode(ISD::SHL, DL, NewOp0.getValueType(),
5778 NewOp0,
5779 DAG.getConstant(c2, DL, CountVT));
5780 AddToWorklist(NewSHL.getNode());
5781 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N0), VT, NewSHL);
5782 }
5783 }
5784 }
5785 }
5786
5787 // fold (shl (sr[la] exact X, C1), C2) -> (shl X, (C2-C1)) if C1 <= C2
5788 // fold (shl (sr[la] exact X, C1), C2) -> (sr[la] X, (C2-C1)) if C1 > C2
5789 if (N1C && (N0.getOpcode() == ISD::SRL || N0.getOpcode() == ISD::SRA) &&
5790 N0->getFlags().hasExact()) {
5791 if (ConstantSDNode *N0C1 = isConstOrConstSplat(N0.getOperand(1))) {
5792 uint64_t C1 = N0C1->getZExtValue();
5793 uint64_t C2 = N1C->getZExtValue();
5794 SDLoc DL(N);
5795 if (C1 <= C2)
5796 return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0),
5797 DAG.getConstant(C2 - C1, DL, N1.getValueType()));
5798 return DAG.getNode(N0.getOpcode(), DL, VT, N0.getOperand(0),
5799 DAG.getConstant(C1 - C2, DL, N1.getValueType()));
5800 }
5801 }
5802
5803 // fold (shl (srl x, c1), c2) -> (and (shl x, (sub c2, c1), MASK) or
5804 // (and (srl x, (sub c1, c2), MASK)
5805 // Only fold this if the inner shift has no other uses -- if it does, folding
5806 // this will increase the total number of instructions.
5807 if (N1C && N0.getOpcode() == ISD::SRL && N0.hasOneUse()) {
5808 if (ConstantSDNode *N0C1 = isConstOrConstSplat(N0.getOperand(1))) {
5809 uint64_t c1 = N0C1->getZExtValue();
5810 if (c1 < OpSizeInBits) {
5811 uint64_t c2 = N1C->getZExtValue();
5812 APInt Mask = APInt::getHighBitsSet(OpSizeInBits, OpSizeInBits - c1);
5813 SDValue Shift;
5814 if (c2 > c1) {
5815 Mask <<= c2 - c1;
5816 SDLoc DL(N);
5817 Shift = DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0),
5818 DAG.getConstant(c2 - c1, DL, N1.getValueType()));
5819 } else {
5820 Mask.lshrInPlace(c1 - c2);
5821 SDLoc DL(N);
5822 Shift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0),
5823 DAG.getConstant(c1 - c2, DL, N1.getValueType()));
5824 }
5825 SDLoc DL(N0);
5826 return DAG.getNode(ISD::AND, DL, VT, Shift,
5827 DAG.getConstant(Mask, DL, VT));
5828 }
5829 }
5830 }
5831
5832 // fold (shl (sra x, c1), c1) -> (and x, (shl -1, c1))
5833 if (N0.getOpcode() == ISD::SRA && N1 == N0.getOperand(1) &&
5834 isConstantOrConstantVector(N1, /* No Opaques */ true)) {
5835 SDLoc DL(N);
5836 SDValue AllBits = DAG.getAllOnesConstant(DL, VT);
5837 SDValue HiBitsMask = DAG.getNode(ISD::SHL, DL, VT, AllBits, N1);
5838 return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), HiBitsMask);
5839 }
5840
5841 // fold (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2)
5842 // fold (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2)
5843 // Variant of version done on multiply, except mul by a power of 2 is turned
5844 // into a shift.
5845 if ((N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::OR) &&
5846 N0.getNode()->hasOneUse() &&
5847 isConstantOrConstantVector(N1, /* No Opaques */ true) &&
5848 isConstantOrConstantVector(N0.getOperand(1), /* No Opaques */ true)) {
5849 SDValue Shl0 = DAG.getNode(ISD::SHL, SDLoc(N0), VT, N0.getOperand(0), N1);
5850 SDValue Shl1 = DAG.getNode(ISD::SHL, SDLoc(N1), VT, N0.getOperand(1), N1);
5851 AddToWorklist(Shl0.getNode());
5852 AddToWorklist(Shl1.getNode());
5853 return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, Shl0, Shl1);
5854 }
5855
5856 // fold (shl (mul x, c1), c2) -> (mul x, c1 << c2)
5857 if (N0.getOpcode() == ISD::MUL && N0.getNode()->hasOneUse() &&
5858 isConstantOrConstantVector(N1, /* No Opaques */ true) &&
5859 isConstantOrConstantVector(N0.getOperand(1), /* No Opaques */ true)) {
5860 SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(N1), VT, N0.getOperand(1), N1);
5861 if (isConstantOrConstantVector(Shl))
5862 return DAG.getNode(ISD::MUL, SDLoc(N), VT, N0.getOperand(0), Shl);
5863 }
5864
5865 if (N1C && !N1C->isOpaque())
5866 if (SDValue NewSHL = visitShiftByConstant(N, N1C))
5867 return NewSHL;
5868
5869 return SDValue();
5870}
5871
5872SDValue DAGCombiner::visitSRA(SDNode *N) {
5873 SDValue N0 = N->getOperand(0);
5874 SDValue N1 = N->getOperand(1);
5875 EVT VT = N0.getValueType();
5876 unsigned OpSizeInBits = VT.getScalarSizeInBits();
5877
5878 // Arithmetic shifting an all-sign-bit value is a no-op.
5879 // fold (sra 0, x) -> 0
5880 // fold (sra -1, x) -> -1
5881 if (DAG.ComputeNumSignBits(N0) == OpSizeInBits)
5882 return N0;
5883
5884 // fold vector ops
5885 if (VT.isVector())
5886 if (SDValue FoldedVOp = SimplifyVBinOp(N))
5887 return FoldedVOp;
5888
5889 ConstantSDNode *N1C = isConstOrConstSplat(N1);
5890
5891 // fold (sra c1, c2) -> (sra c1, c2)
5892 ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
5893 if (N0C && N1C && !N1C->isOpaque())
5894 return DAG.FoldConstantArithmetic(ISD::SRA, SDLoc(N), VT, N0C, N1C);
5895 // fold (sra x, c >= size(x)) -> undef
5896 // NOTE: ALL vector elements must be too big to avoid partial UNDEFs.
5897 auto MatchShiftTooBig = [OpSizeInBits](ConstantSDNode *Val) {
5898 return Val->getAPIntValue().uge(OpSizeInBits);
5899 };
5900 if (matchUnaryPredicate(N1, MatchShiftTooBig))
5901 return DAG.getUNDEF(VT);
5902 // fold (sra x, 0) -> x
5903 if (N1C && N1C->isNullValue())
5904 return N0;
5905
5906 if (SDValue NewSel = foldBinOpIntoSelect(N))
5907 return NewSel;
5908
5909 // fold (sra (shl x, c1), c1) -> sext_inreg for some c1 and target supports
5910 // sext_inreg.
5911 if (N1C && N0.getOpcode() == ISD::SHL && N1 == N0.getOperand(1)) {
5912 unsigned LowBits = OpSizeInBits - (unsigned)N1C->getZExtValue();
5913 EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), LowBits);
5914 if (VT.isVector())
5915 ExtVT = EVT::getVectorVT(*DAG.getContext(),
5916 ExtVT, VT.getVectorNumElements());
5917 if ((!LegalOperations ||
5918 TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, ExtVT)))
5919 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT,
5920 N0.getOperand(0), DAG.getValueType(ExtVT));
5921 }
5922
5923 // fold (sra (sra x, c1), c2) -> (sra x, (add c1, c2))
5924 if (N0.getOpcode() == ISD::SRA) {
5925 SDLoc DL(N);
5926 EVT ShiftVT = N1.getValueType();
5927
5928 auto MatchOutOfRange = [OpSizeInBits](ConstantSDNode *LHS,
5929 ConstantSDNode *RHS) {
5930 APInt c1 = LHS->getAPIntValue();
5931 APInt c2 = RHS->getAPIntValue();
5932 zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
5933 return (c1 + c2).uge(OpSizeInBits);
5934 };
5935