Bug Summary

File:include/llvm/ADT/APInt.h
Warning:line 983, column 15
Assigned value is garbage or undefined

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name DAGCombiner.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-eagerly-assume -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -mrelocation-model pic -pic-level 2 -mthread-model posix -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -momit-leaf-frame-pointer -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-7/lib/clang/7.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-7~svn338205/build-llvm/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-7~svn338205/build-llvm/include -I /build/llvm-toolchain-snapshot-7~svn338205/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/8/../../../../include/c++/8 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/8/../../../../include/x86_64-linux-gnu/c++/8 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/8/../../../../include/x86_64-linux-gnu/c++/8 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/8/../../../../include/c++/8/backward -internal-isystem /usr/include/clang/7.0.0/include/ -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-7/lib/clang/7.0.0/include -internal-externc-isystem /usr/lib/gcc/x86_64-linux-gnu/8/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-comment -std=c++11 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-7~svn338205/build-llvm/lib/CodeGen/SelectionDAG -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -o /tmp/scan-build-2018-07-29-043837-17923-1 -x c++ /build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp -faddrsig

/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

1//===- DAGCombiner.cpp - Implement a DAG node combiner --------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This pass combines dag nodes to form fewer, simpler DAG nodes. It can be run
11// both before and after the DAG is legalized.
12//
13// This pass is not a substitute for the LLVM IR instcombine pass. This pass is
14// primarily intended to handle simplification opportunities that are implicit
15// in the LLVM IR and exposed by the various codegen lowering phases.
16//
17//===----------------------------------------------------------------------===//
18
19#include "llvm/ADT/APFloat.h"
20#include "llvm/ADT/APInt.h"
21#include "llvm/ADT/ArrayRef.h"
22#include "llvm/ADT/DenseMap.h"
23#include "llvm/ADT/None.h"
24#include "llvm/ADT/Optional.h"
25#include "llvm/ADT/STLExtras.h"
26#include "llvm/ADT/SetVector.h"
27#include "llvm/ADT/SmallBitVector.h"
28#include "llvm/ADT/SmallPtrSet.h"
29#include "llvm/ADT/SmallSet.h"
30#include "llvm/ADT/SmallVector.h"
31#include "llvm/ADT/Statistic.h"
32#include "llvm/Analysis/AliasAnalysis.h"
33#include "llvm/Analysis/MemoryLocation.h"
34#include "llvm/CodeGen/DAGCombine.h"
35#include "llvm/CodeGen/ISDOpcodes.h"
36#include "llvm/CodeGen/MachineFrameInfo.h"
37#include "llvm/CodeGen/MachineFunction.h"
38#include "llvm/CodeGen/MachineMemOperand.h"
39#include "llvm/CodeGen/RuntimeLibcalls.h"
40#include "llvm/CodeGen/SelectionDAG.h"
41#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
42#include "llvm/CodeGen/SelectionDAGNodes.h"
43#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
44#include "llvm/CodeGen/TargetLowering.h"
45#include "llvm/CodeGen/TargetRegisterInfo.h"
46#include "llvm/CodeGen/TargetSubtargetInfo.h"
47#include "llvm/CodeGen/ValueTypes.h"
48#include "llvm/IR/Attributes.h"
49#include "llvm/IR/Constant.h"
50#include "llvm/IR/DataLayout.h"
51#include "llvm/IR/DerivedTypes.h"
52#include "llvm/IR/Function.h"
53#include "llvm/IR/LLVMContext.h"
54#include "llvm/IR/Metadata.h"
55#include "llvm/Support/Casting.h"
56#include "llvm/Support/CodeGen.h"
57#include "llvm/Support/CommandLine.h"
58#include "llvm/Support/Compiler.h"
59#include "llvm/Support/Debug.h"
60#include "llvm/Support/ErrorHandling.h"
61#include "llvm/Support/KnownBits.h"
62#include "llvm/Support/MachineValueType.h"
63#include "llvm/Support/MathExtras.h"
64#include "llvm/Support/raw_ostream.h"
65#include "llvm/Target/TargetMachine.h"
66#include "llvm/Target/TargetOptions.h"
67#include <algorithm>
68#include <cassert>
69#include <cstdint>
70#include <functional>
71#include <iterator>
72#include <string>
73#include <tuple>
74#include <utility>
75#include <vector>
76
77using namespace llvm;
78
79#define DEBUG_TYPE"dagcombine" "dagcombine"
80
81STATISTIC(NodesCombined , "Number of dag nodes combined")static llvm::Statistic NodesCombined = {"dagcombine", "NodesCombined"
, "Number of dag nodes combined", {0}, {false}}
;
82STATISTIC(PreIndexedNodes , "Number of pre-indexed nodes created")static llvm::Statistic PreIndexedNodes = {"dagcombine", "PreIndexedNodes"
, "Number of pre-indexed nodes created", {0}, {false}}
;
83STATISTIC(PostIndexedNodes, "Number of post-indexed nodes created")static llvm::Statistic PostIndexedNodes = {"dagcombine", "PostIndexedNodes"
, "Number of post-indexed nodes created", {0}, {false}}
;
84STATISTIC(OpsNarrowed , "Number of load/op/store narrowed")static llvm::Statistic OpsNarrowed = {"dagcombine", "OpsNarrowed"
, "Number of load/op/store narrowed", {0}, {false}}
;
85STATISTIC(LdStFP2Int , "Number of fp load/store pairs transformed to int")static llvm::Statistic LdStFP2Int = {"dagcombine", "LdStFP2Int"
, "Number of fp load/store pairs transformed to int", {0}, {false
}}
;
86STATISTIC(SlicedLoads, "Number of load sliced")static llvm::Statistic SlicedLoads = {"dagcombine", "SlicedLoads"
, "Number of load sliced", {0}, {false}}
;
87
88static cl::opt<bool>
89CombinerGlobalAA("combiner-global-alias-analysis", cl::Hidden,
90 cl::desc("Enable DAG combiner's use of IR alias analysis"));
91
92static cl::opt<bool>
93UseTBAA("combiner-use-tbaa", cl::Hidden, cl::init(true),
94 cl::desc("Enable DAG combiner's use of TBAA"));
95
96#ifndef NDEBUG
97static cl::opt<std::string>
98CombinerAAOnlyFunc("combiner-aa-only-func", cl::Hidden,
99 cl::desc("Only use DAG-combiner alias analysis in this"
100 " function"));
101#endif
102
103/// Hidden option to stress test load slicing, i.e., when this option
104/// is enabled, load slicing bypasses most of its profitability guards.
105static cl::opt<bool>
106StressLoadSlicing("combiner-stress-load-slicing", cl::Hidden,
107 cl::desc("Bypass the profitability model of load slicing"),
108 cl::init(false));
109
110static cl::opt<bool>
111 MaySplitLoadIndex("combiner-split-load-index", cl::Hidden, cl::init(true),
112 cl::desc("DAG combiner may split indexing from loads"));
113
114namespace {
115
116 class DAGCombiner {
117 SelectionDAG &DAG;
118 const TargetLowering &TLI;
119 CombineLevel Level;
120 CodeGenOpt::Level OptLevel;
121 bool LegalOperations = false;
122 bool LegalTypes = false;
123 bool ForCodeSize;
124
125 /// Worklist of all of the nodes that need to be simplified.
126 ///
127 /// This must behave as a stack -- new nodes to process are pushed onto the
128 /// back and when processing we pop off of the back.
129 ///
130 /// The worklist will not contain duplicates but may contain null entries
131 /// due to nodes being deleted from the underlying DAG.
132 SmallVector<SDNode *, 64> Worklist;
133
134 /// Mapping from an SDNode to its position on the worklist.
135 ///
136 /// This is used to find and remove nodes from the worklist (by nulling
137 /// them) when they are deleted from the underlying DAG. It relies on
138 /// stable indices of nodes within the worklist.
139 DenseMap<SDNode *, unsigned> WorklistMap;
140
141 /// Set of nodes which have been combined (at least once).
142 ///
143 /// This is used to allow us to reliably add any operands of a DAG node
144 /// which have not yet been combined to the worklist.
145 SmallPtrSet<SDNode *, 32> CombinedNodes;
146
147 // AA - Used for DAG load/store alias analysis.
148 AliasAnalysis *AA;
149
150 /// When an instruction is simplified, add all users of the instruction to
151 /// the work lists because they might get more simplified now.
152 void AddUsersToWorklist(SDNode *N) {
153 for (SDNode *Node : N->uses())
154 AddToWorklist(Node);
155 }
156
157 /// Call the node-specific routine that folds each particular type of node.
158 SDValue visit(SDNode *N);
159
160 public:
161 DAGCombiner(SelectionDAG &D, AliasAnalysis *AA, CodeGenOpt::Level OL)
162 : DAG(D), TLI(D.getTargetLoweringInfo()), Level(BeforeLegalizeTypes),
163 OptLevel(OL), AA(AA) {
164 ForCodeSize = DAG.getMachineFunction().getFunction().optForSize();
165
166 MaximumLegalStoreInBits = 0;
167 for (MVT VT : MVT::all_valuetypes())
168 if (EVT(VT).isSimple() && VT != MVT::Other &&
169 TLI.isTypeLegal(EVT(VT)) &&
170 VT.getSizeInBits() >= MaximumLegalStoreInBits)
171 MaximumLegalStoreInBits = VT.getSizeInBits();
172 }
173
174 /// Add to the worklist making sure its instance is at the back (next to be
175 /// processed.)
176 void AddToWorklist(SDNode *N) {
177 assert(N->getOpcode() != ISD::DELETED_NODE &&(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& "Deleted Node added to Worklist") ? void (0) : __assert_fail
("N->getOpcode() != ISD::DELETED_NODE && \"Deleted Node added to Worklist\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 178, __extension__ __PRETTY_FUNCTION__))
178 "Deleted Node added to Worklist")(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& "Deleted Node added to Worklist") ? void (0) : __assert_fail
("N->getOpcode() != ISD::DELETED_NODE && \"Deleted Node added to Worklist\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 178, __extension__ __PRETTY_FUNCTION__))
;
179
180 // Skip handle nodes as they can't usefully be combined and confuse the
181 // zero-use deletion strategy.
182 if (N->getOpcode() == ISD::HANDLENODE)
183 return;
184
185 if (WorklistMap.insert(std::make_pair(N, Worklist.size())).second)
186 Worklist.push_back(N);
187 }
188
189 /// Remove all instances of N from the worklist.
190 void removeFromWorklist(SDNode *N) {
191 CombinedNodes.erase(N);
192
193 auto It = WorklistMap.find(N);
194 if (It == WorklistMap.end())
195 return; // Not in the worklist.
196
197 // Null out the entry rather than erasing it to avoid a linear operation.
198 Worklist[It->second] = nullptr;
199 WorklistMap.erase(It);
200 }
201
202 void deleteAndRecombine(SDNode *N);
203 bool recursivelyDeleteUnusedNodes(SDNode *N);
204
205 /// Replaces all uses of the results of one DAG node with new values.
206 SDValue CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
207 bool AddTo = true);
208
209 /// Replaces all uses of the results of one DAG node with new values.
210 SDValue CombineTo(SDNode *N, SDValue Res, bool AddTo = true) {
211 return CombineTo(N, &Res, 1, AddTo);
212 }
213
214 /// Replaces all uses of the results of one DAG node with new values.
215 SDValue CombineTo(SDNode *N, SDValue Res0, SDValue Res1,
216 bool AddTo = true) {
217 SDValue To[] = { Res0, Res1 };
218 return CombineTo(N, To, 2, AddTo);
219 }
220
221 void CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO);
222
223 private:
224 unsigned MaximumLegalStoreInBits;
225
226 /// Check the specified integer node value to see if it can be simplified or
227 /// if things it uses can be simplified by bit propagation.
228 /// If so, return true.
229 bool SimplifyDemandedBits(SDValue Op) {
230 unsigned BitWidth = Op.getScalarValueSizeInBits();
231 APInt Demanded = APInt::getAllOnesValue(BitWidth);
232 return SimplifyDemandedBits(Op, Demanded);
233 }
234
235 /// Check the specified vector node value to see if it can be simplified or
236 /// if things it uses can be simplified as it only uses some of the
237 /// elements. If so, return true.
238 bool SimplifyDemandedVectorElts(SDValue Op) {
239 unsigned NumElts = Op.getValueType().getVectorNumElements();
240 APInt Demanded = APInt::getAllOnesValue(NumElts);
241 return SimplifyDemandedVectorElts(Op, Demanded);
242 }
243
244 bool SimplifyDemandedBits(SDValue Op, const APInt &Demanded);
245 bool SimplifyDemandedVectorElts(SDValue Op, const APInt &Demanded,
246 bool AssumeSingleUse = false);
247
248 bool CombineToPreIndexedLoadStore(SDNode *N);
249 bool CombineToPostIndexedLoadStore(SDNode *N);
250 SDValue SplitIndexingFromLoad(LoadSDNode *LD);
251 bool SliceUpLoad(SDNode *N);
252
253 /// Replace an ISD::EXTRACT_VECTOR_ELT of a load with a narrowed
254 /// load.
255 ///
256 /// \param EVE ISD::EXTRACT_VECTOR_ELT to be replaced.
257 /// \param InVecVT type of the input vector to EVE with bitcasts resolved.
258 /// \param EltNo index of the vector element to load.
259 /// \param OriginalLoad load that EVE came from to be replaced.
260 /// \returns EVE on success SDValue() on failure.
261 SDValue ReplaceExtractVectorEltOfLoadWithNarrowedLoad(
262 SDNode *EVE, EVT InVecVT, SDValue EltNo, LoadSDNode *OriginalLoad);
263 void ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad);
264 SDValue PromoteOperand(SDValue Op, EVT PVT, bool &Replace);
265 SDValue SExtPromoteOperand(SDValue Op, EVT PVT);
266 SDValue ZExtPromoteOperand(SDValue Op, EVT PVT);
267 SDValue PromoteIntBinOp(SDValue Op);
268 SDValue PromoteIntShiftOp(SDValue Op);
269 SDValue PromoteExtend(SDValue Op);
270 bool PromoteLoad(SDValue Op);
271
272 /// Call the node-specific routine that knows how to fold each
273 /// particular type of node. If that doesn't do anything, try the
274 /// target-specific DAG combines.
275 SDValue combine(SDNode *N);
276
277 // Visitation implementation - Implement dag node combining for different
278 // node types. The semantics are as follows:
279 // Return Value:
280 // SDValue.getNode() == 0 - No change was made
281 // SDValue.getNode() == N - N was replaced, is dead and has been handled.
282 // otherwise - N should be replaced by the returned Operand.
283 //
284 SDValue visitTokenFactor(SDNode *N);
285 SDValue visitMERGE_VALUES(SDNode *N);
286 SDValue visitADD(SDNode *N);
287 SDValue visitADDLike(SDValue N0, SDValue N1, SDNode *LocReference);
288 SDValue visitSUB(SDNode *N);
289 SDValue visitADDC(SDNode *N);
290 SDValue visitUADDO(SDNode *N);
291 SDValue visitUADDOLike(SDValue N0, SDValue N1, SDNode *N);
292 SDValue visitSUBC(SDNode *N);
293 SDValue visitUSUBO(SDNode *N);
294 SDValue visitADDE(SDNode *N);
295 SDValue visitADDCARRY(SDNode *N);
296 SDValue visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn, SDNode *N);
297 SDValue visitSUBE(SDNode *N);
298 SDValue visitSUBCARRY(SDNode *N);
299 SDValue visitMUL(SDNode *N);
300 SDValue useDivRem(SDNode *N);
301 SDValue visitSDIV(SDNode *N);
302 SDValue visitSDIVLike(SDValue N0, SDValue N1, SDNode *N);
303 SDValue visitUDIV(SDNode *N);
304 SDValue visitUDIVLike(SDValue N0, SDValue N1, SDNode *N);
305 SDValue visitREM(SDNode *N);
306 SDValue visitMULHU(SDNode *N);
307 SDValue visitMULHS(SDNode *N);
308 SDValue visitSMUL_LOHI(SDNode *N);
309 SDValue visitUMUL_LOHI(SDNode *N);
310 SDValue visitSMULO(SDNode *N);
311 SDValue visitUMULO(SDNode *N);
312 SDValue visitIMINMAX(SDNode *N);
313 SDValue visitAND(SDNode *N);
314 SDValue visitANDLike(SDValue N0, SDValue N1, SDNode *N);
315 SDValue visitOR(SDNode *N);
316 SDValue visitORLike(SDValue N0, SDValue N1, SDNode *N);
317 SDValue visitXOR(SDNode *N);
318 SDValue SimplifyVBinOp(SDNode *N);
319 SDValue visitSHL(SDNode *N);
320 SDValue visitSRA(SDNode *N);
321 SDValue visitSRL(SDNode *N);
322 SDValue visitRotate(SDNode *N);
323 SDValue visitABS(SDNode *N);
324 SDValue visitBSWAP(SDNode *N);
325 SDValue visitBITREVERSE(SDNode *N);
326 SDValue visitCTLZ(SDNode *N);
327 SDValue visitCTLZ_ZERO_UNDEF(SDNode *N);
328 SDValue visitCTTZ(SDNode *N);
329 SDValue visitCTTZ_ZERO_UNDEF(SDNode *N);
330 SDValue visitCTPOP(SDNode *N);
331 SDValue visitSELECT(SDNode *N);
332 SDValue visitVSELECT(SDNode *N);
333 SDValue visitSELECT_CC(SDNode *N);
334 SDValue visitSETCC(SDNode *N);
335 SDValue visitSETCCCARRY(SDNode *N);
336 SDValue visitSIGN_EXTEND(SDNode *N);
337 SDValue visitZERO_EXTEND(SDNode *N);
338 SDValue visitANY_EXTEND(SDNode *N);
339 SDValue visitAssertExt(SDNode *N);
340 SDValue visitSIGN_EXTEND_INREG(SDNode *N);
341 SDValue visitSIGN_EXTEND_VECTOR_INREG(SDNode *N);
342 SDValue visitZERO_EXTEND_VECTOR_INREG(SDNode *N);
343 SDValue visitTRUNCATE(SDNode *N);
344 SDValue visitBITCAST(SDNode *N);
345 SDValue visitBUILD_PAIR(SDNode *N);
346 SDValue visitFADD(SDNode *N);
347 SDValue visitFSUB(SDNode *N);
348 SDValue visitFMUL(SDNode *N);
349 SDValue visitFMA(SDNode *N);
350 SDValue visitFDIV(SDNode *N);
351 SDValue visitFREM(SDNode *N);
352 SDValue visitFSQRT(SDNode *N);
353 SDValue visitFCOPYSIGN(SDNode *N);
354 SDValue visitSINT_TO_FP(SDNode *N);
355 SDValue visitUINT_TO_FP(SDNode *N);
356 SDValue visitFP_TO_SINT(SDNode *N);
357 SDValue visitFP_TO_UINT(SDNode *N);
358 SDValue visitFP_ROUND(SDNode *N);
359 SDValue visitFP_ROUND_INREG(SDNode *N);
360 SDValue visitFP_EXTEND(SDNode *N);
361 SDValue visitFNEG(SDNode *N);
362 SDValue visitFABS(SDNode *N);
363 SDValue visitFCEIL(SDNode *N);
364 SDValue visitFTRUNC(SDNode *N);
365 SDValue visitFFLOOR(SDNode *N);
366 SDValue visitFMINNUM(SDNode *N);
367 SDValue visitFMAXNUM(SDNode *N);
368 SDValue visitBRCOND(SDNode *N);
369 SDValue visitBR_CC(SDNode *N);
370 SDValue visitLOAD(SDNode *N);
371
372 SDValue replaceStoreChain(StoreSDNode *ST, SDValue BetterChain);
373 SDValue replaceStoreOfFPConstant(StoreSDNode *ST);
374
375 SDValue visitSTORE(SDNode *N);
376 SDValue visitINSERT_VECTOR_ELT(SDNode *N);
377 SDValue visitEXTRACT_VECTOR_ELT(SDNode *N);
378 SDValue visitBUILD_VECTOR(SDNode *N);
379 SDValue visitCONCAT_VECTORS(SDNode *N);
380 SDValue visitEXTRACT_SUBVECTOR(SDNode *N);
381 SDValue visitVECTOR_SHUFFLE(SDNode *N);
382 SDValue visitSCALAR_TO_VECTOR(SDNode *N);
383 SDValue visitINSERT_SUBVECTOR(SDNode *N);
384 SDValue visitMLOAD(SDNode *N);
385 SDValue visitMSTORE(SDNode *N);
386 SDValue visitMGATHER(SDNode *N);
387 SDValue visitMSCATTER(SDNode *N);
388 SDValue visitFP_TO_FP16(SDNode *N);
389 SDValue visitFP16_TO_FP(SDNode *N);
390
391 SDValue visitFADDForFMACombine(SDNode *N);
392 SDValue visitFSUBForFMACombine(SDNode *N);
393 SDValue visitFMULForFMADistributiveCombine(SDNode *N);
394
395 SDValue XformToShuffleWithZero(SDNode *N);
396 SDValue ReassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0,
397 SDValue N1);
398
399 SDValue visitShiftByConstant(SDNode *N, ConstantSDNode *Amt);
400
401 SDValue foldSelectOfConstants(SDNode *N);
402 SDValue foldVSelectOfConstants(SDNode *N);
403 SDValue foldBinOpIntoSelect(SDNode *BO);
404 bool SimplifySelectOps(SDNode *SELECT, SDValue LHS, SDValue RHS);
405 SDValue SimplifyBinOpWithSameOpcodeHands(SDNode *N);
406 SDValue SimplifySelect(const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2);
407 SDValue SimplifySelectCC(const SDLoc &DL, SDValue N0, SDValue N1,
408 SDValue N2, SDValue N3, ISD::CondCode CC,
409 bool NotExtCompare = false);
410 SDValue foldSelectCCToShiftAnd(const SDLoc &DL, SDValue N0, SDValue N1,
411 SDValue N2, SDValue N3, ISD::CondCode CC);
412 SDValue foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1,
413 const SDLoc &DL);
414 SDValue unfoldMaskedMerge(SDNode *N);
415 SDValue unfoldExtremeBitClearingToShifts(SDNode *N);
416 SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
417 const SDLoc &DL, bool foldBooleans);
418 SDValue rebuildSetCC(SDValue N);
419
420 bool isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
421 SDValue &CC) const;
422 bool isOneUseSetCC(SDValue N) const;
423
424 SDValue SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
425 unsigned HiOp);
426 SDValue CombineConsecutiveLoads(SDNode *N, EVT VT);
427 SDValue CombineExtLoad(SDNode *N);
428 SDValue CombineZExtLogicopShiftLoad(SDNode *N);
429 SDValue combineRepeatedFPDivisors(SDNode *N);
430 SDValue combineInsertEltToShuffle(SDNode *N, unsigned InsIndex);
431 SDValue ConstantFoldBITCASTofBUILD_VECTOR(SDNode *, EVT);
432 SDValue BuildSDIV(SDNode *N);
433 SDValue BuildSDIVPow2(SDNode *N);
434 SDValue BuildUDIV(SDNode *N);
435 SDValue BuildLogBase2(SDValue V, const SDLoc &DL);
436 SDValue BuildReciprocalEstimate(SDValue Op, SDNodeFlags Flags);
437 SDValue buildRsqrtEstimate(SDValue Op, SDNodeFlags Flags);
438 SDValue buildSqrtEstimate(SDValue Op, SDNodeFlags Flags);
439 SDValue buildSqrtEstimateImpl(SDValue Op, SDNodeFlags Flags, bool Recip);
440 SDValue buildSqrtNROneConst(SDValue Arg, SDValue Est, unsigned Iterations,
441 SDNodeFlags Flags, bool Reciprocal);
442 SDValue buildSqrtNRTwoConst(SDValue Arg, SDValue Est, unsigned Iterations,
443 SDNodeFlags Flags, bool Reciprocal);
444 SDValue MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
445 bool DemandHighBits = true);
446 SDValue MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1);
447 SDNode *MatchRotatePosNeg(SDValue Shifted, SDValue Pos, SDValue Neg,
448 SDValue InnerPos, SDValue InnerNeg,
449 unsigned PosOpcode, unsigned NegOpcode,
450 const SDLoc &DL);
451 SDNode *MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL);
452 SDValue MatchLoadCombine(SDNode *N);
453 SDValue ReduceLoadWidth(SDNode *N);
454 SDValue ReduceLoadOpStoreWidth(SDNode *N);
455 SDValue splitMergedValStore(StoreSDNode *ST);
456 SDValue TransformFPLoadStorePair(SDNode *N);
457 SDValue convertBuildVecZextToZext(SDNode *N);
458 SDValue reduceBuildVecExtToExtBuildVec(SDNode *N);
459 SDValue reduceBuildVecConvertToConvertBuildVec(SDNode *N);
460 SDValue reduceBuildVecToShuffle(SDNode *N);
461 SDValue createBuildVecShuffle(const SDLoc &DL, SDNode *N,
462 ArrayRef<int> VectorMask, SDValue VecIn1,
463 SDValue VecIn2, unsigned LeftIdx);
464 SDValue matchVSelectOpSizesWithSetCC(SDNode *Cast);
465
466 /// Walk up chain skipping non-aliasing memory nodes,
467 /// looking for aliasing nodes and adding them to the Aliases vector.
468 void GatherAllAliases(SDNode *N, SDValue OriginalChain,
469 SmallVectorImpl<SDValue> &Aliases);
470
471 /// Return true if there is any possibility that the two addresses overlap.
472 bool isAlias(LSBaseSDNode *Op0, LSBaseSDNode *Op1) const;
473
474 /// Walk up chain skipping non-aliasing memory nodes, looking for a better
475 /// chain (aliasing node.)
476 SDValue FindBetterChain(SDNode *N, SDValue Chain);
477
478 /// Try to replace a store and any possibly adjacent stores on
479 /// consecutive chains with better chains. Return true only if St is
480 /// replaced.
481 ///
482 /// Notice that other chains may still be replaced even if the function
483 /// returns false.
484 bool findBetterNeighborChains(StoreSDNode *St);
485
486 /// Match "(X shl/srl V1) & V2" where V2 may not be present.
487 bool MatchRotateHalf(SDValue Op, SDValue &Shift, SDValue &Mask);
488
489 /// Holds a pointer to an LSBaseSDNode as well as information on where it
490 /// is located in a sequence of memory operations connected by a chain.
491 struct MemOpLink {
492 // Ptr to the mem node.
493 LSBaseSDNode *MemNode;
494
495 // Offset from the base ptr.
496 int64_t OffsetFromBase;
497
498 MemOpLink(LSBaseSDNode *N, int64_t Offset)
499 : MemNode(N), OffsetFromBase(Offset) {}
500 };
501
502 /// This is a helper function for visitMUL to check the profitability
503 /// of folding (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2).
504 /// MulNode is the original multiply, AddNode is (add x, c1),
505 /// and ConstNode is c2.
506 bool isMulAddWithConstProfitable(SDNode *MulNode,
507 SDValue &AddNode,
508 SDValue &ConstNode);
509
510 /// This is a helper function for visitAND and visitZERO_EXTEND. Returns
511 /// true if the (and (load x) c) pattern matches an extload. ExtVT returns
512 /// the type of the loaded value to be extended.
513 bool isAndLoadExtLoad(ConstantSDNode *AndC, LoadSDNode *LoadN,
514 EVT LoadResultTy, EVT &ExtVT);
515
516 /// Helper function to calculate whether the given Load/Store can have its
517 /// width reduced to ExtVT.
518 bool isLegalNarrowLdSt(LSBaseSDNode *LDSTN, ISD::LoadExtType ExtType,
519 EVT &MemVT, unsigned ShAmt = 0);
520
521 /// Used by BackwardsPropagateMask to find suitable loads.
522 bool SearchForAndLoads(SDNode *N, SmallPtrSetImpl<LoadSDNode*> &Loads,
523 SmallPtrSetImpl<SDNode*> &NodesWithConsts,
524 ConstantSDNode *Mask, SDNode *&NodeToMask);
525 /// Attempt to propagate a given AND node back to load leaves so that they
526 /// can be combined into narrow loads.
527 bool BackwardsPropagateMask(SDNode *N, SelectionDAG &DAG);
528
529 /// Helper function for MergeConsecutiveStores which merges the
530 /// component store chains.
531 SDValue getMergeStoreChains(SmallVectorImpl<MemOpLink> &StoreNodes,
532 unsigned NumStores);
533
534 /// This is a helper function for MergeConsecutiveStores. When the
535 /// source elements of the consecutive stores are all constants or
536 /// all extracted vector elements, try to merge them into one
537 /// larger store introducing bitcasts if necessary. \return True
538 /// if a merged store was created.
539 bool MergeStoresOfConstantsOrVecElts(SmallVectorImpl<MemOpLink> &StoreNodes,
540 EVT MemVT, unsigned NumStores,
541 bool IsConstantSrc, bool UseVector,
542 bool UseTrunc);
543
544 /// This is a helper function for MergeConsecutiveStores. Stores
545 /// that potentially may be merged with St are placed in
546 /// StoreNodes. RootNode is a chain predecessor to all store
547 /// candidates.
548 void getStoreMergeCandidates(StoreSDNode *St,
549 SmallVectorImpl<MemOpLink> &StoreNodes,
550 SDNode *&Root);
551
552 /// Helper function for MergeConsecutiveStores. Checks if
553 /// candidate stores have indirect dependency through their
554 /// operands. RootNode is the predecessor to all stores calculated
555 /// by getStoreMergeCandidates and is used to prune the dependency check.
556 /// \return True if safe to merge.
557 bool checkMergeStoreCandidatesForDependencies(
558 SmallVectorImpl<MemOpLink> &StoreNodes, unsigned NumStores,
559 SDNode *RootNode);
560
561 /// Merge consecutive store operations into a wide store.
562 /// This optimization uses wide integers or vectors when possible.
563 /// \return number of stores that were merged into a merged store (the
564 /// affected nodes are stored as a prefix in \p StoreNodes).
565 bool MergeConsecutiveStores(StoreSDNode *St);
566
567 /// Try to transform a truncation where C is a constant:
568 /// (trunc (and X, C)) -> (and (trunc X), (trunc C))
569 ///
570 /// \p N needs to be a truncation and its first operand an AND. Other
571 /// requirements are checked by the function (e.g. that trunc is
572 /// single-use) and if missed an empty SDValue is returned.
573 SDValue distributeTruncateThroughAnd(SDNode *N);
574
575 /// Helper function to determine whether the target supports operation
576 /// given by \p Opcode for type \p VT, that is, whether the operation
577 /// is legal or custom before legalizing operations, and whether is
578 /// legal (but not custom) after legalization.
579 bool hasOperation(unsigned Opcode, EVT VT) {
580 if (LegalOperations)
581 return TLI.isOperationLegal(Opcode, VT);
582 return TLI.isOperationLegalOrCustom(Opcode, VT);
583 }
584
585 public:
586 /// Runs the dag combiner on all nodes in the work list
587 void Run(CombineLevel AtLevel);
588
589 SelectionDAG &getDAG() const { return DAG; }
590
591 /// Returns a type large enough to hold any valid shift amount - before type
592 /// legalization these can be huge.
593 EVT getShiftAmountTy(EVT LHSTy) {
594 assert(LHSTy.isInteger() && "Shift amount is not an integer type!")(static_cast <bool> (LHSTy.isInteger() && "Shift amount is not an integer type!"
) ? void (0) : __assert_fail ("LHSTy.isInteger() && \"Shift amount is not an integer type!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 594, __extension__ __PRETTY_FUNCTION__))
;
595 return TLI.getShiftAmountTy(LHSTy, DAG.getDataLayout(), LegalTypes);
596 }
597
598 /// This method returns true if we are running before type legalization or
599 /// if the specified VT is legal.
600 bool isTypeLegal(const EVT &VT) {
601 if (!LegalTypes) return true;
602 return TLI.isTypeLegal(VT);
603 }
604
605 /// Convenience wrapper around TargetLowering::getSetCCResultType
606 EVT getSetCCResultType(EVT VT) const {
607 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
608 }
609
610 void ExtendSetCCUses(const SmallVectorImpl<SDNode *> &SetCCs,
611 SDValue OrigLoad, SDValue ExtLoad,
612 ISD::NodeType ExtType);
613 };
614
615/// This class is a DAGUpdateListener that removes any deleted
616/// nodes from the worklist.
617class WorklistRemover : public SelectionDAG::DAGUpdateListener {
618 DAGCombiner &DC;
619
620public:
621 explicit WorklistRemover(DAGCombiner &dc)
622 : SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
623
624 void NodeDeleted(SDNode *N, SDNode *E) override {
625 DC.removeFromWorklist(N);
626 }
627};
628
629} // end anonymous namespace
630
631//===----------------------------------------------------------------------===//
632// TargetLowering::DAGCombinerInfo implementation
633//===----------------------------------------------------------------------===//
634
635void TargetLowering::DAGCombinerInfo::AddToWorklist(SDNode *N) {
636 ((DAGCombiner*)DC)->AddToWorklist(N);
637}
638
639SDValue TargetLowering::DAGCombinerInfo::
640CombineTo(SDNode *N, ArrayRef<SDValue> To, bool AddTo) {
641 return ((DAGCombiner*)DC)->CombineTo(N, &To[0], To.size(), AddTo);
642}
643
644SDValue TargetLowering::DAGCombinerInfo::
645CombineTo(SDNode *N, SDValue Res, bool AddTo) {
646 return ((DAGCombiner*)DC)->CombineTo(N, Res, AddTo);
647}
648
649SDValue TargetLowering::DAGCombinerInfo::
650CombineTo(SDNode *N, SDValue Res0, SDValue Res1, bool AddTo) {
651 return ((DAGCombiner*)DC)->CombineTo(N, Res0, Res1, AddTo);
652}
653
654void TargetLowering::DAGCombinerInfo::
655CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
656 return ((DAGCombiner*)DC)->CommitTargetLoweringOpt(TLO);
657}
658
659//===----------------------------------------------------------------------===//
660// Helper Functions
661//===----------------------------------------------------------------------===//
662
663void DAGCombiner::deleteAndRecombine(SDNode *N) {
664 removeFromWorklist(N);
665
666 // If the operands of this node are only used by the node, they will now be
667 // dead. Make sure to re-visit them and recursively delete dead nodes.
668 for (const SDValue &Op : N->ops())
669 // For an operand generating multiple values, one of the values may
670 // become dead allowing further simplification (e.g. split index
671 // arithmetic from an indexed load).
672 if (Op->hasOneUse() || Op->getNumValues() > 1)
673 AddToWorklist(Op.getNode());
674
675 DAG.DeleteNode(N);
676}
677
678/// Return 1 if we can compute the negated form of the specified expression for
679/// the same cost as the expression itself, or 2 if we can compute the negated
680/// form more cheaply than the expression itself.
681static char isNegatibleForFree(SDValue Op, bool LegalOperations,
682 const TargetLowering &TLI,
683 const TargetOptions *Options,
684 unsigned Depth = 0) {
685 // fneg is removable even if it has multiple uses.
686 if (Op.getOpcode() == ISD::FNEG) return 2;
687
688 // Don't allow anything with multiple uses unless we know it is free.
689 EVT VT = Op.getValueType();
690 const SDNodeFlags Flags = Op->getFlags();
691 if (!Op.hasOneUse())
692 if (!(Op.getOpcode() == ISD::FP_EXTEND &&
693 TLI.isFPExtFree(VT, Op.getOperand(0).getValueType())))
694 return 0;
695
696 // Don't recurse exponentially.
697 if (Depth > 6) return 0;
698
699 switch (Op.getOpcode()) {
700 default: return false;
701 case ISD::ConstantFP: {
702 if (!LegalOperations)
703 return 1;
704
705 // Don't invert constant FP values after legalization unless the target says
706 // the negated constant is legal.
707 return TLI.isOperationLegal(ISD::ConstantFP, VT) ||
708 TLI.isFPImmLegal(neg(cast<ConstantFPSDNode>(Op)->getValueAPF()), VT);
709 }
710 case ISD::FADD:
711 if (!Options->UnsafeFPMath && !Flags.hasNoSignedZeros())
712 return 0;
713
714 // After operation legalization, it might not be legal to create new FSUBs.
715 if (LegalOperations && !TLI.isOperationLegalOrCustom(ISD::FSUB, VT))
716 return 0;
717
718 // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
719 if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI,
720 Options, Depth + 1))
721 return V;
722 // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
723 return isNegatibleForFree(Op.getOperand(1), LegalOperations, TLI, Options,
724 Depth + 1);
725 case ISD::FSUB:
726 // We can't turn -(A-B) into B-A when we honor signed zeros.
727 if (!Options->NoSignedZerosFPMath &&
728 !Flags.hasNoSignedZeros())
729 return 0;
730
731 // fold (fneg (fsub A, B)) -> (fsub B, A)
732 return 1;
733
734 case ISD::FMUL:
735 case ISD::FDIV:
736 // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y) or (fmul X, (fneg Y))
737 if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI,
738 Options, Depth + 1))
739 return V;
740
741 return isNegatibleForFree(Op.getOperand(1), LegalOperations, TLI, Options,
742 Depth + 1);
743
744 case ISD::FP_EXTEND:
745 case ISD::FP_ROUND:
746 case ISD::FSIN:
747 return isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI, Options,
748 Depth + 1);
749 }
750}
751
752/// If isNegatibleForFree returns true, return the newly negated expression.
753static SDValue GetNegatedExpression(SDValue Op, SelectionDAG &DAG,
754 bool LegalOperations, unsigned Depth = 0) {
755 const TargetOptions &Options = DAG.getTarget().Options;
756 // fneg is removable even if it has multiple uses.
757 if (Op.getOpcode() == ISD::FNEG) return Op.getOperand(0);
758
759 assert(Depth <= 6 && "GetNegatedExpression doesn't match isNegatibleForFree")(static_cast <bool> (Depth <= 6 && "GetNegatedExpression doesn't match isNegatibleForFree"
) ? void (0) : __assert_fail ("Depth <= 6 && \"GetNegatedExpression doesn't match isNegatibleForFree\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 759, __extension__ __PRETTY_FUNCTION__))
;
760
761 const SDNodeFlags Flags = Op.getNode()->getFlags();
762
763 switch (Op.getOpcode()) {
764 default: llvm_unreachable("Unknown code")::llvm::llvm_unreachable_internal("Unknown code", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 764)
;
765 case ISD::ConstantFP: {
766 APFloat V = cast<ConstantFPSDNode>(Op)->getValueAPF();
767 V.changeSign();
768 return DAG.getConstantFP(V, SDLoc(Op), Op.getValueType());
769 }
770 case ISD::FADD:
771 assert(Options.UnsafeFPMath || Flags.hasNoSignedZeros())(static_cast <bool> (Options.UnsafeFPMath || Flags.hasNoSignedZeros
()) ? void (0) : __assert_fail ("Options.UnsafeFPMath || Flags.hasNoSignedZeros()"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 771, __extension__ __PRETTY_FUNCTION__))
;
772
773 // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
774 if (isNegatibleForFree(Op.getOperand(0), LegalOperations,
775 DAG.getTargetLoweringInfo(), &Options, Depth+1))
776 return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
777 GetNegatedExpression(Op.getOperand(0), DAG,
778 LegalOperations, Depth+1),
779 Op.getOperand(1), Flags);
780 // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
781 return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
782 GetNegatedExpression(Op.getOperand(1), DAG,
783 LegalOperations, Depth+1),
784 Op.getOperand(0), Flags);
785 case ISD::FSUB:
786 // fold (fneg (fsub 0, B)) -> B
787 if (ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(Op.getOperand(0)))
788 if (N0CFP->isZero())
789 return Op.getOperand(1);
790
791 // fold (fneg (fsub A, B)) -> (fsub B, A)
792 return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
793 Op.getOperand(1), Op.getOperand(0), Flags);
794
795 case ISD::FMUL:
796 case ISD::FDIV:
797 // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y)
798 if (isNegatibleForFree(Op.getOperand(0), LegalOperations,
799 DAG.getTargetLoweringInfo(), &Options, Depth+1))
800 return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
801 GetNegatedExpression(Op.getOperand(0), DAG,
802 LegalOperations, Depth+1),
803 Op.getOperand(1), Flags);
804
805 // fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y))
806 return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
807 Op.getOperand(0),
808 GetNegatedExpression(Op.getOperand(1), DAG,
809 LegalOperations, Depth+1), Flags);
810
811 case ISD::FP_EXTEND:
812 case ISD::FSIN:
813 return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
814 GetNegatedExpression(Op.getOperand(0), DAG,
815 LegalOperations, Depth+1));
816 case ISD::FP_ROUND:
817 return DAG.getNode(ISD::FP_ROUND, SDLoc(Op), Op.getValueType(),
818 GetNegatedExpression(Op.getOperand(0), DAG,
819 LegalOperations, Depth+1),
820 Op.getOperand(1));
821 }
822}
823
824// APInts must be the same size for most operations, this helper
825// function zero extends the shorter of the pair so that they match.
826// We provide an Offset so that we can create bitwidths that won't overflow.
827static void zeroExtendToMatch(APInt &LHS, APInt &RHS, unsigned Offset = 0) {
828 unsigned Bits = Offset + std::max(LHS.getBitWidth(), RHS.getBitWidth());
829 LHS = LHS.zextOrSelf(Bits);
830 RHS = RHS.zextOrSelf(Bits);
831}
832
833// Return true if this node is a setcc, or is a select_cc
834// that selects between the target values used for true and false, making it
835// equivalent to a setcc. Also, set the incoming LHS, RHS, and CC references to
836// the appropriate nodes based on the type of node we are checking. This
837// simplifies life a bit for the callers.
838bool DAGCombiner::isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
839 SDValue &CC) const {
840 if (N.getOpcode() == ISD::SETCC) {
841 LHS = N.getOperand(0);
842 RHS = N.getOperand(1);
843 CC = N.getOperand(2);
844 return true;
845 }
846
847 if (N.getOpcode() != ISD::SELECT_CC ||
848 !TLI.isConstTrueVal(N.getOperand(2).getNode()) ||
849 !TLI.isConstFalseVal(N.getOperand(3).getNode()))
850 return false;
851
852 if (TLI.getBooleanContents(N.getValueType()) ==
853 TargetLowering::UndefinedBooleanContent)
854 return false;
855
856 LHS = N.getOperand(0);
857 RHS = N.getOperand(1);
858 CC = N.getOperand(4);
859 return true;
860}
861
862/// Return true if this is a SetCC-equivalent operation with only one use.
863/// If this is true, it allows the users to invert the operation for free when
864/// it is profitable to do so.
865bool DAGCombiner::isOneUseSetCC(SDValue N) const {
866 SDValue N0, N1, N2;
867 if (isSetCCEquivalent(N, N0, N1, N2) && N.getNode()->hasOneUse())
868 return true;
869 return false;
870}
871
872static SDValue peekThroughBitcast(SDValue V) {
873 while (V.getOpcode() == ISD::BITCAST)
874 V = V.getOperand(0);
875 return V;
876}
877
878// Returns the SDNode if it is a constant float BuildVector
879// or constant float.
880static SDNode *isConstantFPBuildVectorOrConstantFP(SDValue N) {
881 if (isa<ConstantFPSDNode>(N))
882 return N.getNode();
883 if (ISD::isBuildVectorOfConstantFPSDNodes(N.getNode()))
884 return N.getNode();
885 return nullptr;
886}
887
888// Determines if it is a constant integer or a build vector of constant
889// integers (and undefs).
890// Do not permit build vector implicit truncation.
891static bool isConstantOrConstantVector(SDValue N, bool NoOpaques = false) {
892 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N))
893 return !(Const->isOpaque() && NoOpaques);
894 if (N.getOpcode() != ISD::BUILD_VECTOR)
895 return false;
896 unsigned BitWidth = N.getScalarValueSizeInBits();
897 for (const SDValue &Op : N->op_values()) {
898 if (Op.isUndef())
899 continue;
900 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Op);
901 if (!Const || Const->getAPIntValue().getBitWidth() != BitWidth ||
902 (Const->isOpaque() && NoOpaques))
903 return false;
904 }
905 return true;
906}
907
908// Determines if it is a constant null integer or a splatted vector of a
909// constant null integer (with no undefs).
910// Build vector implicit truncation is not an issue for null values.
911static bool isNullConstantOrNullSplatConstant(SDValue N) {
912 // TODO: may want to use peekThroughBitcast() here.
913 if (ConstantSDNode *Splat = isConstOrConstSplat(N))
914 return Splat->isNullValue();
915 return false;
916}
917
918// Determines if it is a constant integer of one or a splatted vector of a
919// constant integer of one (with no undefs).
920// Do not permit build vector implicit truncation.
921static bool isOneConstantOrOneSplatConstant(SDValue N) {
922 // TODO: may want to use peekThroughBitcast() here.
923 unsigned BitWidth = N.getScalarValueSizeInBits();
924 if (ConstantSDNode *Splat = isConstOrConstSplat(N))
925 return Splat->isOne() && Splat->getAPIntValue().getBitWidth() == BitWidth;
926 return false;
927}
928
929// Determines if it is a constant integer of all ones or a splatted vector of a
930// constant integer of all ones (with no undefs).
931// Do not permit build vector implicit truncation.
932static bool isAllOnesConstantOrAllOnesSplatConstant(SDValue N) {
933 N = peekThroughBitcast(N);
934 unsigned BitWidth = N.getScalarValueSizeInBits();
935 if (ConstantSDNode *Splat = isConstOrConstSplat(N))
936 return Splat->isAllOnesValue() &&
937 Splat->getAPIntValue().getBitWidth() == BitWidth;
938 return false;
939}
940
941// Determines if a BUILD_VECTOR is composed of all-constants possibly mixed with
942// undef's.
943static bool isAnyConstantBuildVector(const SDNode *N) {
944 return ISD::isBuildVectorOfConstantSDNodes(N) ||
945 ISD::isBuildVectorOfConstantFPSDNodes(N);
946}
947
948SDValue DAGCombiner::ReassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0,
949 SDValue N1) {
950 EVT VT = N0.getValueType();
951 if (N0.getOpcode() == Opc) {
952 if (SDNode *L = DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1))) {
953 if (SDNode *R = DAG.isConstantIntBuildVectorOrConstantInt(N1)) {
954 // reassoc. (op (op x, c1), c2) -> (op x, (op c1, c2))
955 if (SDValue OpNode = DAG.FoldConstantArithmetic(Opc, DL, VT, L, R))
956 return DAG.getNode(Opc, DL, VT, N0.getOperand(0), OpNode);
957 return SDValue();
958 }
959 if (N0.hasOneUse()) {
960 // reassoc. (op (op x, c1), y) -> (op (op x, y), c1) iff x+c1 has one
961 // use
962 SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N0.getOperand(0), N1);
963 if (!OpNode.getNode())
964 return SDValue();
965 AddToWorklist(OpNode.getNode());
966 return DAG.getNode(Opc, DL, VT, OpNode, N0.getOperand(1));
967 }
968 }
969 }
970
971 if (N1.getOpcode() == Opc) {
972 if (SDNode *R = DAG.isConstantIntBuildVectorOrConstantInt(N1.getOperand(1))) {
973 if (SDNode *L = DAG.isConstantIntBuildVectorOrConstantInt(N0)) {
974 // reassoc. (op c2, (op x, c1)) -> (op x, (op c1, c2))
975 if (SDValue OpNode = DAG.FoldConstantArithmetic(Opc, DL, VT, R, L))
976 return DAG.getNode(Opc, DL, VT, N1.getOperand(0), OpNode);
977 return SDValue();
978 }
979 if (N1.hasOneUse()) {
980 // reassoc. (op x, (op y, c1)) -> (op (op x, y), c1) iff x+c1 has one
981 // use
982 SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N0, N1.getOperand(0));
983 if (!OpNode.getNode())
984 return SDValue();
985 AddToWorklist(OpNode.getNode());
986 return DAG.getNode(Opc, DL, VT, OpNode, N1.getOperand(1));
987 }
988 }
989 }
990
991 return SDValue();
992}
993
994SDValue DAGCombiner::CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
995 bool AddTo) {
996 assert(N->getNumValues() == NumTo && "Broken CombineTo call!")(static_cast <bool> (N->getNumValues() == NumTo &&
"Broken CombineTo call!") ? void (0) : __assert_fail ("N->getNumValues() == NumTo && \"Broken CombineTo call!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 996, __extension__ __PRETTY_FUNCTION__))
;
997 ++NodesCombined;
998 LLVM_DEBUG(dbgs() << "\nReplacing.1 "; N->dump(&DAG); dbgs() << "\nWith: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.1 "; N->dump
(&DAG); dbgs() << "\nWith: "; To[0].getNode()->dump
(&DAG); dbgs() << " and " << NumTo - 1 <<
" other values\n"; } } while (false)
999 To[0].getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.1 "; N->dump
(&DAG); dbgs() << "\nWith: "; To[0].getNode()->dump
(&DAG); dbgs() << " and " << NumTo - 1 <<
" other values\n"; } } while (false)
1000 dbgs() << " and " << NumTo - 1 << " other values\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.1 "; N->dump
(&DAG); dbgs() << "\nWith: "; To[0].getNode()->dump
(&DAG); dbgs() << " and " << NumTo - 1 <<
" other values\n"; } } while (false)
;
1001 for (unsigned i = 0, e = NumTo; i != e; ++i)
1002 assert((!To[i].getNode() ||(static_cast <bool> ((!To[i].getNode() || N->getValueType
(i) == To[i].getValueType()) && "Cannot combine value to value of different type!"
) ? void (0) : __assert_fail ("(!To[i].getNode() || N->getValueType(i) == To[i].getValueType()) && \"Cannot combine value to value of different type!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1004, __extension__ __PRETTY_FUNCTION__))
1003 N->getValueType(i) == To[i].getValueType()) &&(static_cast <bool> ((!To[i].getNode() || N->getValueType
(i) == To[i].getValueType()) && "Cannot combine value to value of different type!"
) ? void (0) : __assert_fail ("(!To[i].getNode() || N->getValueType(i) == To[i].getValueType()) && \"Cannot combine value to value of different type!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1004, __extension__ __PRETTY_FUNCTION__))
1004 "Cannot combine value to value of different type!")(static_cast <bool> ((!To[i].getNode() || N->getValueType
(i) == To[i].getValueType()) && "Cannot combine value to value of different type!"
) ? void (0) : __assert_fail ("(!To[i].getNode() || N->getValueType(i) == To[i].getValueType()) && \"Cannot combine value to value of different type!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1004, __extension__ __PRETTY_FUNCTION__))
;
1005
1006 WorklistRemover DeadNodes(*this);
1007 DAG.ReplaceAllUsesWith(N, To);
1008 if (AddTo) {
1009 // Push the new nodes and any users onto the worklist
1010 for (unsigned i = 0, e = NumTo; i != e; ++i) {
1011 if (To[i].getNode()) {
1012 AddToWorklist(To[i].getNode());
1013 AddUsersToWorklist(To[i].getNode());
1014 }
1015 }
1016 }
1017
1018 // Finally, if the node is now dead, remove it from the graph. The node
1019 // may not be dead if the replacement process recursively simplified to
1020 // something else needing this node.
1021 if (N->use_empty())
1022 deleteAndRecombine(N);
1023 return SDValue(N, 0);
1024}
1025
1026void DAGCombiner::
1027CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
1028 // Replace all uses. If any nodes become isomorphic to other nodes and
1029 // are deleted, make sure to remove them from our worklist.
1030 WorklistRemover DeadNodes(*this);
1031 DAG.ReplaceAllUsesOfValueWith(TLO.Old, TLO.New);
1032
1033 // Push the new node and any (possibly new) users onto the worklist.
1034 AddToWorklist(TLO.New.getNode());
1035 AddUsersToWorklist(TLO.New.getNode());
1036
1037 // Finally, if the node is now dead, remove it from the graph. The node
1038 // may not be dead if the replacement process recursively simplified to
1039 // something else needing this node.
1040 if (TLO.Old.getNode()->use_empty())
1041 deleteAndRecombine(TLO.Old.getNode());
1042}
1043
1044/// Check the specified integer node value to see if it can be simplified or if
1045/// things it uses can be simplified by bit propagation. If so, return true.
1046bool DAGCombiner::SimplifyDemandedBits(SDValue Op, const APInt &Demanded) {
1047 TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations);
1048 KnownBits Known;
1049 if (!TLI.SimplifyDemandedBits(Op, Demanded, Known, TLO))
1050 return false;
1051
1052 // Revisit the node.
1053 AddToWorklist(Op.getNode());
1054
1055 // Replace the old value with the new one.
1056 ++NodesCombined;
1057 LLVM_DEBUG(dbgs() << "\nReplacing.2 "; TLO.Old.getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.getNode
()->dump(&DAG); dbgs() << "\nWith: "; TLO.New.getNode
()->dump(&DAG); dbgs() << '\n'; } } while (false
)
1058 dbgs() << "\nWith: "; TLO.New.getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.getNode
()->dump(&DAG); dbgs() << "\nWith: "; TLO.New.getNode
()->dump(&DAG); dbgs() << '\n'; } } while (false
)
1059 dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.getNode
()->dump(&DAG); dbgs() << "\nWith: "; TLO.New.getNode
()->dump(&DAG); dbgs() << '\n'; } } while (false
)
;
1060
1061 CommitTargetLoweringOpt(TLO);
1062 return true;
1063}
1064
1065/// Check the specified vector node value to see if it can be simplified or
1066/// if things it uses can be simplified as it only uses some of the elements.
1067/// If so, return true.
1068bool DAGCombiner::SimplifyDemandedVectorElts(SDValue Op, const APInt &Demanded,
1069 bool AssumeSingleUse) {
1070 TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations);
1071 APInt KnownUndef, KnownZero;
1072 if (!TLI.SimplifyDemandedVectorElts(Op, Demanded, KnownUndef, KnownZero, TLO,
1073 0, AssumeSingleUse))
1074 return false;
1075
1076 // Revisit the node.
1077 AddToWorklist(Op.getNode());
1078
1079 // Replace the old value with the new one.
1080 ++NodesCombined;
1081 LLVM_DEBUG(dbgs() << "\nReplacing.2 "; TLO.Old.getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.getNode
()->dump(&DAG); dbgs() << "\nWith: "; TLO.New.getNode
()->dump(&DAG); dbgs() << '\n'; } } while (false
)
1082 dbgs() << "\nWith: "; TLO.New.getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.getNode
()->dump(&DAG); dbgs() << "\nWith: "; TLO.New.getNode
()->dump(&DAG); dbgs() << '\n'; } } while (false
)
1083 dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.getNode
()->dump(&DAG); dbgs() << "\nWith: "; TLO.New.getNode
()->dump(&DAG); dbgs() << '\n'; } } while (false
)
;
1084
1085 CommitTargetLoweringOpt(TLO);
1086 return true;
1087}
1088
1089void DAGCombiner::ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad) {
1090 SDLoc DL(Load);
1091 EVT VT = Load->getValueType(0);
1092 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, VT, SDValue(ExtLoad, 0));
1093
1094 LLVM_DEBUG(dbgs() << "\nReplacing.9 "; Load->dump(&DAG); dbgs() << "\nWith: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.9 "; Load->
dump(&DAG); dbgs() << "\nWith: "; Trunc.getNode()->
dump(&DAG); dbgs() << '\n'; } } while (false)
1095 Trunc.getNode()->dump(&DAG); dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.9 "; Load->
dump(&DAG); dbgs() << "\nWith: "; Trunc.getNode()->
dump(&DAG); dbgs() << '\n'; } } while (false)
;
1096 WorklistRemover DeadNodes(*this);
1097 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), Trunc);
1098 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), SDValue(ExtLoad, 1));
1099 deleteAndRecombine(Load);
1100 AddToWorklist(Trunc.getNode());
1101}
1102
1103SDValue DAGCombiner::PromoteOperand(SDValue Op, EVT PVT, bool &Replace) {
1104 Replace = false;
1105 SDLoc DL(Op);
1106 if (ISD::isUNINDEXEDLoad(Op.getNode())) {
1107 LoadSDNode *LD = cast<LoadSDNode>(Op);
1108 EVT MemVT = LD->getMemoryVT();
1109 ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD) ? ISD::EXTLOAD
1110 : LD->getExtensionType();
1111 Replace = true;
1112 return DAG.getExtLoad(ExtType, DL, PVT,
1113 LD->getChain(), LD->getBasePtr(),
1114 MemVT, LD->getMemOperand());
1115 }
1116
1117 unsigned Opc = Op.getOpcode();
1118 switch (Opc) {
1119 default: break;
1120 case ISD::AssertSext:
1121 if (SDValue Op0 = SExtPromoteOperand(Op.getOperand(0), PVT))
1122 return DAG.getNode(ISD::AssertSext, DL, PVT, Op0, Op.getOperand(1));
1123 break;
1124 case ISD::AssertZext:
1125 if (SDValue Op0 = ZExtPromoteOperand(Op.getOperand(0), PVT))
1126 return DAG.getNode(ISD::AssertZext, DL, PVT, Op0, Op.getOperand(1));
1127 break;
1128 case ISD::Constant: {
1129 unsigned ExtOpc =
1130 Op.getValueType().isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
1131 return DAG.getNode(ExtOpc, DL, PVT, Op);
1132 }
1133 }
1134
1135 if (!TLI.isOperationLegal(ISD::ANY_EXTEND, PVT))
1136 return SDValue();
1137 return DAG.getNode(ISD::ANY_EXTEND, DL, PVT, Op);
1138}
1139
1140SDValue DAGCombiner::SExtPromoteOperand(SDValue Op, EVT PVT) {
1141 if (!TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, PVT))
1142 return SDValue();
1143 EVT OldVT = Op.getValueType();
1144 SDLoc DL(Op);
1145 bool Replace = false;
1146 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
1147 if (!NewOp.getNode())
1148 return SDValue();
1149 AddToWorklist(NewOp.getNode());
1150
1151 if (Replace)
1152 ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
1153 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, NewOp.getValueType(), NewOp,
1154 DAG.getValueType(OldVT));
1155}
1156
1157SDValue DAGCombiner::ZExtPromoteOperand(SDValue Op, EVT PVT) {
1158 EVT OldVT = Op.getValueType();
1159 SDLoc DL(Op);
1160 bool Replace = false;
1161 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
1162 if (!NewOp.getNode())
1163 return SDValue();
1164 AddToWorklist(NewOp.getNode());
1165
1166 if (Replace)
1167 ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
1168 return DAG.getZeroExtendInReg(NewOp, DL, OldVT);
1169}
1170
1171/// Promote the specified integer binary operation if the target indicates it is
1172/// beneficial. e.g. On x86, it's usually better to promote i16 operations to
1173/// i32 since i16 instructions are longer.
1174SDValue DAGCombiner::PromoteIntBinOp(SDValue Op) {
1175 if (!LegalOperations)
1176 return SDValue();
1177
1178 EVT VT = Op.getValueType();
1179 if (VT.isVector() || !VT.isInteger())
1180 return SDValue();
1181
1182 // If operation type is 'undesirable', e.g. i16 on x86, consider
1183 // promoting it.
1184 unsigned Opc = Op.getOpcode();
1185 if (TLI.isTypeDesirableForOp(Opc, VT))
1186 return SDValue();
1187
1188 EVT PVT = VT;
1189 // Consult target whether it is a good idea to promote this operation and
1190 // what's the right type to promote it to.
1191 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1192 assert(PVT != VT && "Don't know what type to promote to!")(static_cast <bool> (PVT != VT && "Don't know what type to promote to!"
) ? void (0) : __assert_fail ("PVT != VT && \"Don't know what type to promote to!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1192, __extension__ __PRETTY_FUNCTION__))
;
1193
1194 LLVM_DEBUG(dbgs() << "\nPromoting "; Op.getNode()->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; Op.getNode(
)->dump(&DAG); } } while (false)
;
1195
1196 bool Replace0 = false;
1197 SDValue N0 = Op.getOperand(0);
1198 SDValue NN0 = PromoteOperand(N0, PVT, Replace0);
1199
1200 bool Replace1 = false;
1201 SDValue N1 = Op.getOperand(1);
1202 SDValue NN1 = PromoteOperand(N1, PVT, Replace1);
1203 SDLoc DL(Op);
1204
1205 SDValue RV =
1206 DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, NN0, NN1));
1207
1208 // We are always replacing N0/N1's use in N and only need
1209 // additional replacements if there are additional uses.
1210 Replace0 &= !N0->hasOneUse();
1211 Replace1 &= (N0 != N1) && !N1->hasOneUse();
1212
1213 // Combine Op here so it is preserved past replacements.
1214 CombineTo(Op.getNode(), RV);
1215
1216 // If operands have a use ordering, make sure we deal with
1217 // predecessor first.
1218 if (Replace0 && Replace1 && N0.getNode()->isPredecessorOf(N1.getNode())) {
1219 std::swap(N0, N1);
1220 std::swap(NN0, NN1);
1221 }
1222
1223 if (Replace0) {
1224 AddToWorklist(NN0.getNode());
1225 ReplaceLoadWithPromotedLoad(N0.getNode(), NN0.getNode());
1226 }
1227 if (Replace1) {
1228 AddToWorklist(NN1.getNode());
1229 ReplaceLoadWithPromotedLoad(N1.getNode(), NN1.getNode());
1230 }
1231 return Op;
1232 }
1233 return SDValue();
1234}
1235
1236/// Promote the specified integer shift operation if the target indicates it is
1237/// beneficial. e.g. On x86, it's usually better to promote i16 operations to
1238/// i32 since i16 instructions are longer.
1239SDValue DAGCombiner::PromoteIntShiftOp(SDValue Op) {
1240 if (!LegalOperations)
1241 return SDValue();
1242
1243 EVT VT = Op.getValueType();
1244 if (VT.isVector() || !VT.isInteger())
1245 return SDValue();
1246
1247 // If operation type is 'undesirable', e.g. i16 on x86, consider
1248 // promoting it.
1249 unsigned Opc = Op.getOpcode();
1250 if (TLI.isTypeDesirableForOp(Opc, VT))
1251 return SDValue();
1252
1253 EVT PVT = VT;
1254 // Consult target whether it is a good idea to promote this operation and
1255 // what's the right type to promote it to.
1256 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1257 assert(PVT != VT && "Don't know what type to promote to!")(static_cast <bool> (PVT != VT && "Don't know what type to promote to!"
) ? void (0) : __assert_fail ("PVT != VT && \"Don't know what type to promote to!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1257, __extension__ __PRETTY_FUNCTION__))
;
1258
1259 LLVM_DEBUG(dbgs() << "\nPromoting "; Op.getNode()->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; Op.getNode(
)->dump(&DAG); } } while (false)
;
1260
1261 bool Replace = false;
1262 SDValue N0 = Op.getOperand(0);
1263 SDValue N1 = Op.getOperand(1);
1264 if (Opc == ISD::SRA)
1265 N0 = SExtPromoteOperand(N0, PVT);
1266 else if (Opc == ISD::SRL)
1267 N0 = ZExtPromoteOperand(N0, PVT);
1268 else
1269 N0 = PromoteOperand(N0, PVT, Replace);
1270
1271 if (!N0.getNode())
1272 return SDValue();
1273
1274 SDLoc DL(Op);
1275 SDValue RV =
1276 DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, N0, N1));
1277
1278 AddToWorklist(N0.getNode());
1279 if (Replace)
1280 ReplaceLoadWithPromotedLoad(Op.getOperand(0).getNode(), N0.getNode());
1281
1282 // Deal with Op being deleted.
1283 if (Op && Op.getOpcode() != ISD::DELETED_NODE)
1284 return RV;
1285 }
1286 return SDValue();
1287}
1288
1289SDValue DAGCombiner::PromoteExtend(SDValue Op) {
1290 if (!LegalOperations)
1291 return SDValue();
1292
1293 EVT VT = Op.getValueType();
1294 if (VT.isVector() || !VT.isInteger())
1295 return SDValue();
1296
1297 // If operation type is 'undesirable', e.g. i16 on x86, consider
1298 // promoting it.
1299 unsigned Opc = Op.getOpcode();
1300 if (TLI.isTypeDesirableForOp(Opc, VT))
1301 return SDValue();
1302
1303 EVT PVT = VT;
1304 // Consult target whether it is a good idea to promote this operation and
1305 // what's the right type to promote it to.
1306 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1307 assert(PVT != VT && "Don't know what type to promote to!")(static_cast <bool> (PVT != VT && "Don't know what type to promote to!"
) ? void (0) : __assert_fail ("PVT != VT && \"Don't know what type to promote to!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1307, __extension__ __PRETTY_FUNCTION__))
;
1308 // fold (aext (aext x)) -> (aext x)
1309 // fold (aext (zext x)) -> (zext x)
1310 // fold (aext (sext x)) -> (sext x)
1311 LLVM_DEBUG(dbgs() << "\nPromoting "; Op.getNode()->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; Op.getNode(
)->dump(&DAG); } } while (false)
;
1312 return DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, Op.getOperand(0));
1313 }
1314 return SDValue();
1315}
1316
1317bool DAGCombiner::PromoteLoad(SDValue Op) {
1318 if (!LegalOperations)
1319 return false;
1320
1321 if (!ISD::isUNINDEXEDLoad(Op.getNode()))
1322 return false;
1323
1324 EVT VT = Op.getValueType();
1325 if (VT.isVector() || !VT.isInteger())
1326 return false;
1327
1328 // If operation type is 'undesirable', e.g. i16 on x86, consider
1329 // promoting it.
1330 unsigned Opc = Op.getOpcode();
1331 if (TLI.isTypeDesirableForOp(Opc, VT))
1332 return false;
1333
1334 EVT PVT = VT;
1335 // Consult target whether it is a good idea to promote this operation and
1336 // what's the right type to promote it to.
1337 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1338 assert(PVT != VT && "Don't know what type to promote to!")(static_cast <bool> (PVT != VT && "Don't know what type to promote to!"
) ? void (0) : __assert_fail ("PVT != VT && \"Don't know what type to promote to!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1338, __extension__ __PRETTY_FUNCTION__))
;
1339
1340 SDLoc DL(Op);
1341 SDNode *N = Op.getNode();
1342 LoadSDNode *LD = cast<LoadSDNode>(N);
1343 EVT MemVT = LD->getMemoryVT();
1344 ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD) ? ISD::EXTLOAD
1345 : LD->getExtensionType();
1346 SDValue NewLD = DAG.getExtLoad(ExtType, DL, PVT,
1347 LD->getChain(), LD->getBasePtr(),
1348 MemVT, LD->getMemOperand());
1349 SDValue Result = DAG.getNode(ISD::TRUNCATE, DL, VT, NewLD);
1350
1351 LLVM_DEBUG(dbgs() << "\nPromoting "; N->dump(&DAG); dbgs() << "\nTo: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; N->dump(
&DAG); dbgs() << "\nTo: "; Result.getNode()->dump
(&DAG); dbgs() << '\n'; } } while (false)
1352 Result.getNode()->dump(&DAG); dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; N->dump(
&DAG); dbgs() << "\nTo: "; Result.getNode()->dump
(&DAG); dbgs() << '\n'; } } while (false)
;
1353 WorklistRemover DeadNodes(*this);
1354 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result);
1355 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), NewLD.getValue(1));
1356 deleteAndRecombine(N);
1357 AddToWorklist(Result.getNode());
1358 return true;
1359 }
1360 return false;
1361}
1362
1363/// Recursively delete a node which has no uses and any operands for
1364/// which it is the only use.
1365///
1366/// Note that this both deletes the nodes and removes them from the worklist.
1367/// It also adds any nodes who have had a user deleted to the worklist as they
1368/// may now have only one use and subject to other combines.
1369bool DAGCombiner::recursivelyDeleteUnusedNodes(SDNode *N) {
1370 if (!N->use_empty())
1371 return false;
1372
1373 SmallSetVector<SDNode *, 16> Nodes;
1374 Nodes.insert(N);
1375 do {
1376 N = Nodes.pop_back_val();
1377 if (!N)
1378 continue;
1379
1380 if (N->use_empty()) {
1381 for (const SDValue &ChildN : N->op_values())
1382 Nodes.insert(ChildN.getNode());
1383
1384 removeFromWorklist(N);
1385 DAG.DeleteNode(N);
1386 } else {
1387 AddToWorklist(N);
1388 }
1389 } while (!Nodes.empty());
1390 return true;
1391}
1392
1393//===----------------------------------------------------------------------===//
1394// Main DAG Combiner implementation
1395//===----------------------------------------------------------------------===//
1396
1397void DAGCombiner::Run(CombineLevel AtLevel) {
1398 // set the instance variables, so that the various visit routines may use it.
1399 Level = AtLevel;
1400 LegalOperations = Level >= AfterLegalizeVectorOps;
1401 LegalTypes = Level >= AfterLegalizeTypes;
1402
1403 // Add all the dag nodes to the worklist.
1404 for (SDNode &Node : DAG.allnodes())
1405 AddToWorklist(&Node);
1406
1407 // Create a dummy node (which is not added to allnodes), that adds a reference
1408 // to the root node, preventing it from being deleted, and tracking any
1409 // changes of the root.
1410 HandleSDNode Dummy(DAG.getRoot());
1411
1412 // While the worklist isn't empty, find a node and try to combine it.
1413 while (!WorklistMap.empty()) {
1414 SDNode *N;
1415 // The Worklist holds the SDNodes in order, but it may contain null entries.
1416 do {
1417 N = Worklist.pop_back_val();
1418 } while (!N);
1419
1420 bool GoodWorklistEntry = WorklistMap.erase(N);
1421 (void)GoodWorklistEntry;
1422 assert(GoodWorklistEntry &&(static_cast <bool> (GoodWorklistEntry && "Found a worklist entry without a corresponding map entry!"
) ? void (0) : __assert_fail ("GoodWorklistEntry && \"Found a worklist entry without a corresponding map entry!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1423, __extension__ __PRETTY_FUNCTION__))
1423 "Found a worklist entry without a corresponding map entry!")(static_cast <bool> (GoodWorklistEntry && "Found a worklist entry without a corresponding map entry!"
) ? void (0) : __assert_fail ("GoodWorklistEntry && \"Found a worklist entry without a corresponding map entry!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1423, __extension__ __PRETTY_FUNCTION__))
;
1424
1425 // If N has no uses, it is dead. Make sure to revisit all N's operands once
1426 // N is deleted from the DAG, since they too may now be dead or may have a
1427 // reduced number of uses, allowing other xforms.
1428 if (recursivelyDeleteUnusedNodes(N))
1429 continue;
1430
1431 WorklistRemover DeadNodes(*this);
1432
1433 // If this combine is running after legalizing the DAG, re-legalize any
1434 // nodes pulled off the worklist.
1435 if (Level == AfterLegalizeDAG) {
1436 SmallSetVector<SDNode *, 16> UpdatedNodes;
1437 bool NIsValid = DAG.LegalizeOp(N, UpdatedNodes);
1438
1439 for (SDNode *LN : UpdatedNodes) {
1440 AddToWorklist(LN);
1441 AddUsersToWorklist(LN);
1442 }
1443 if (!NIsValid)
1444 continue;
1445 }
1446
1447 LLVM_DEBUG(dbgs() << "\nCombining: "; N->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nCombining: "; N->dump
(&DAG); } } while (false)
;
1448
1449 // Add any operands of the new node which have not yet been combined to the
1450 // worklist as well. Because the worklist uniques things already, this
1451 // won't repeatedly process the same operand.
1452 CombinedNodes.insert(N);
1453 for (const SDValue &ChildN : N->op_values())
1454 if (!CombinedNodes.count(ChildN.getNode()))
1455 AddToWorklist(ChildN.getNode());
1456
1457 SDValue RV = combine(N);
1458
1459 if (!RV.getNode())
1460 continue;
1461
1462 ++NodesCombined;
1463
1464 // If we get back the same node we passed in, rather than a new node or
1465 // zero, we know that the node must have defined multiple values and
1466 // CombineTo was used. Since CombineTo takes care of the worklist
1467 // mechanics for us, we have no work to do in this case.
1468 if (RV.getNode() == N)
1469 continue;
1470
1471 assert(N->getOpcode() != ISD::DELETED_NODE &&(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& RV.getOpcode() != ISD::DELETED_NODE && "Node was deleted but visit returned new node!"
) ? void (0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && RV.getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned new node!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1473, __extension__ __PRETTY_FUNCTION__))
1472 RV.getOpcode() != ISD::DELETED_NODE &&(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& RV.getOpcode() != ISD::DELETED_NODE && "Node was deleted but visit returned new node!"
) ? void (0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && RV.getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned new node!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1473, __extension__ __PRETTY_FUNCTION__))
1473 "Node was deleted but visit returned new node!")(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& RV.getOpcode() != ISD::DELETED_NODE && "Node was deleted but visit returned new node!"
) ? void (0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && RV.getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned new node!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1473, __extension__ __PRETTY_FUNCTION__))
;
1474
1475 LLVM_DEBUG(dbgs() << " ... into: "; RV.getNode()->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << " ... into: "; RV.getNode()
->dump(&DAG); } } while (false)
;
1476
1477 if (N->getNumValues() == RV.getNode()->getNumValues())
1478 DAG.ReplaceAllUsesWith(N, RV.getNode());
1479 else {
1480 assert(N->getValueType(0) == RV.getValueType() &&(static_cast <bool> (N->getValueType(0) == RV.getValueType
() && N->getNumValues() == 1 && "Type mismatch"
) ? void (0) : __assert_fail ("N->getValueType(0) == RV.getValueType() && N->getNumValues() == 1 && \"Type mismatch\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1481, __extension__ __PRETTY_FUNCTION__))
1481 N->getNumValues() == 1 && "Type mismatch")(static_cast <bool> (N->getValueType(0) == RV.getValueType
() && N->getNumValues() == 1 && "Type mismatch"
) ? void (0) : __assert_fail ("N->getValueType(0) == RV.getValueType() && N->getNumValues() == 1 && \"Type mismatch\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1481, __extension__ __PRETTY_FUNCTION__))
;
1482 DAG.ReplaceAllUsesWith(N, &RV);
1483 }
1484
1485 // Push the new node and any users onto the worklist
1486 AddToWorklist(RV.getNode());
1487 AddUsersToWorklist(RV.getNode());
1488
1489 // Finally, if the node is now dead, remove it from the graph. The node
1490 // may not be dead if the replacement process recursively simplified to
1491 // something else needing this node. This will also take care of adding any
1492 // operands which have lost a user to the worklist.
1493 recursivelyDeleteUnusedNodes(N);
1494 }
1495
1496 // If the root changed (e.g. it was a dead load, update the root).
1497 DAG.setRoot(Dummy.getValue());
1498 DAG.RemoveDeadNodes();
1499}
1500
1501SDValue DAGCombiner::visit(SDNode *N) {
1502 switch (N->getOpcode()) {
1503 default: break;
1504 case ISD::TokenFactor: return visitTokenFactor(N);
1505 case ISD::MERGE_VALUES: return visitMERGE_VALUES(N);
1506 case ISD::ADD: return visitADD(N);
1507 case ISD::SUB: return visitSUB(N);
1508 case ISD::ADDC: return visitADDC(N);
1509 case ISD::UADDO: return visitUADDO(N);
1510 case ISD::SUBC: return visitSUBC(N);
1511 case ISD::USUBO: return visitUSUBO(N);
1512 case ISD::ADDE: return visitADDE(N);
1513 case ISD::ADDCARRY: return visitADDCARRY(N);
1514 case ISD::SUBE: return visitSUBE(N);
1515 case ISD::SUBCARRY: return visitSUBCARRY(N);
1516 case ISD::MUL: return visitMUL(N);
1517 case ISD::SDIV: return visitSDIV(N);
1518 case ISD::UDIV: return visitUDIV(N);
1519 case ISD::SREM:
1520 case ISD::UREM: return visitREM(N);
1521 case ISD::MULHU: return visitMULHU(N);
1522 case ISD::MULHS: return visitMULHS(N);
1523 case ISD::SMUL_LOHI: return visitSMUL_LOHI(N);
1524 case ISD::UMUL_LOHI: return visitUMUL_LOHI(N);
1525 case ISD::SMULO: return visitSMULO(N);
1526 case ISD::UMULO: return visitUMULO(N);
1527 case ISD::SMIN:
1528 case ISD::SMAX:
1529 case ISD::UMIN:
1530 case ISD::UMAX: return visitIMINMAX(N);
1531 case ISD::AND: return visitAND(N);
1532 case ISD::OR: return visitOR(N);
1533 case ISD::XOR: return visitXOR(N);
1534 case ISD::SHL: return visitSHL(N);
1535 case ISD::SRA: return visitSRA(N);
1536 case ISD::SRL: return visitSRL(N);
1537 case ISD::ROTR:
1538 case ISD::ROTL: return visitRotate(N);
1539 case ISD::ABS: return visitABS(N);
1540 case ISD::BSWAP: return visitBSWAP(N);
1541 case ISD::BITREVERSE: return visitBITREVERSE(N);
1542 case ISD::CTLZ: return visitCTLZ(N);
1543 case ISD::CTLZ_ZERO_UNDEF: return visitCTLZ_ZERO_UNDEF(N);
1544 case ISD::CTTZ: return visitCTTZ(N);
1545 case ISD::CTTZ_ZERO_UNDEF: return visitCTTZ_ZERO_UNDEF(N);
1546 case ISD::CTPOP: return visitCTPOP(N);
1547 case ISD::SELECT: return visitSELECT(N);
1548 case ISD::VSELECT: return visitVSELECT(N);
1549 case ISD::SELECT_CC: return visitSELECT_CC(N);
1550 case ISD::SETCC: return visitSETCC(N);
1551 case ISD::SETCCCARRY: return visitSETCCCARRY(N);
1552 case ISD::SIGN_EXTEND: return visitSIGN_EXTEND(N);
1553 case ISD::ZERO_EXTEND: return visitZERO_EXTEND(N);
1554 case ISD::ANY_EXTEND: return visitANY_EXTEND(N);
1555 case ISD::AssertSext:
1556 case ISD::AssertZext: return visitAssertExt(N);
1557 case ISD::SIGN_EXTEND_INREG: return visitSIGN_EXTEND_INREG(N);
1558 case ISD::SIGN_EXTEND_VECTOR_INREG: return visitSIGN_EXTEND_VECTOR_INREG(N);
1559 case ISD::ZERO_EXTEND_VECTOR_INREG: return visitZERO_EXTEND_VECTOR_INREG(N);
1560 case ISD::TRUNCATE: return visitTRUNCATE(N);
1561 case ISD::BITCAST: return visitBITCAST(N);
1562 case ISD::BUILD_PAIR: return visitBUILD_PAIR(N);
1563 case ISD::FADD: return visitFADD(N);
1564 case ISD::FSUB: return visitFSUB(N);
1565 case ISD::FMUL: return visitFMUL(N);
1566 case ISD::FMA: return visitFMA(N);
1567 case ISD::FDIV: return visitFDIV(N);
1568 case ISD::FREM: return visitFREM(N);
1569 case ISD::FSQRT: return visitFSQRT(N);
1570 case ISD::FCOPYSIGN: return visitFCOPYSIGN(N);
1571 case ISD::SINT_TO_FP: return visitSINT_TO_FP(N);
1572 case ISD::UINT_TO_FP: return visitUINT_TO_FP(N);
1573 case ISD::FP_TO_SINT: return visitFP_TO_SINT(N);
1574 case ISD::FP_TO_UINT: return visitFP_TO_UINT(N);
1575 case ISD::FP_ROUND: return visitFP_ROUND(N);
1576 case ISD::FP_ROUND_INREG: return visitFP_ROUND_INREG(N);
1577 case ISD::FP_EXTEND: return visitFP_EXTEND(N);
1578 case ISD::FNEG: return visitFNEG(N);
1579 case ISD::FABS: return visitFABS(N);
1580 case ISD::FFLOOR: return visitFFLOOR(N);
1581 case ISD::FMINNUM: return visitFMINNUM(N);
1582 case ISD::FMAXNUM: return visitFMAXNUM(N);
1583 case ISD::FCEIL: return visitFCEIL(N);
1584 case ISD::FTRUNC: return visitFTRUNC(N);
1585 case ISD::BRCOND: return visitBRCOND(N);
1586 case ISD::BR_CC: return visitBR_CC(N);
1587 case ISD::LOAD: return visitLOAD(N);
1588 case ISD::STORE: return visitSTORE(N);
1589 case ISD::INSERT_VECTOR_ELT: return visitINSERT_VECTOR_ELT(N);
1590 case ISD::EXTRACT_VECTOR_ELT: return visitEXTRACT_VECTOR_ELT(N);
1591 case ISD::BUILD_VECTOR: return visitBUILD_VECTOR(N);
1592 case ISD::CONCAT_VECTORS: return visitCONCAT_VECTORS(N);
1593 case ISD::EXTRACT_SUBVECTOR: return visitEXTRACT_SUBVECTOR(N);
1594 case ISD::VECTOR_SHUFFLE: return visitVECTOR_SHUFFLE(N);
1595 case ISD::SCALAR_TO_VECTOR: return visitSCALAR_TO_VECTOR(N);
1596 case ISD::INSERT_SUBVECTOR: return visitINSERT_SUBVECTOR(N);
1597 case ISD::MGATHER: return visitMGATHER(N);
1598 case ISD::MLOAD: return visitMLOAD(N);
1599 case ISD::MSCATTER: return visitMSCATTER(N);
1600 case ISD::MSTORE: return visitMSTORE(N);
1601 case ISD::FP_TO_FP16: return visitFP_TO_FP16(N);
1602 case ISD::FP16_TO_FP: return visitFP16_TO_FP(N);
1603 }
1604 return SDValue();
1605}
1606
1607SDValue DAGCombiner::combine(SDNode *N) {
1608 SDValue RV = visit(N);
1609
1610 // If nothing happened, try a target-specific DAG combine.
1611 if (!RV.getNode()) {
1612 assert(N->getOpcode() != ISD::DELETED_NODE &&(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& "Node was deleted but visit returned NULL!") ? void
(0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned NULL!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1613, __extension__ __PRETTY_FUNCTION__))
1613 "Node was deleted but visit returned NULL!")(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& "Node was deleted but visit returned NULL!") ? void
(0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned NULL!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1613, __extension__ __PRETTY_FUNCTION__))
;
1614
1615 if (N->getOpcode() >= ISD::BUILTIN_OP_END ||
1616 TLI.hasTargetDAGCombine((ISD::NodeType)N->getOpcode())) {
1617
1618 // Expose the DAG combiner to the target combiner impls.
1619 TargetLowering::DAGCombinerInfo
1620 DagCombineInfo(DAG, Level, false, this);
1621
1622 RV = TLI.PerformDAGCombine(N, DagCombineInfo);
1623 }
1624 }
1625
1626 // If nothing happened still, try promoting the operation.
1627 if (!RV.getNode()) {
1628 switch (N->getOpcode()) {
1629 default: break;
1630 case ISD::ADD:
1631 case ISD::SUB:
1632 case ISD::MUL:
1633 case ISD::AND:
1634 case ISD::OR:
1635 case ISD::XOR:
1636 RV = PromoteIntBinOp(SDValue(N, 0));
1637 break;
1638 case ISD::SHL:
1639 case ISD::SRA:
1640 case ISD::SRL:
1641 RV = PromoteIntShiftOp(SDValue(N, 0));
1642 break;
1643 case ISD::SIGN_EXTEND:
1644 case ISD::ZERO_EXTEND:
1645 case ISD::ANY_EXTEND:
1646 RV = PromoteExtend(SDValue(N, 0));
1647 break;
1648 case ISD::LOAD:
1649 if (PromoteLoad(SDValue(N, 0)))
1650 RV = SDValue(N, 0);
1651 break;
1652 }
1653 }
1654
1655 // If N is a commutative binary node, try eliminate it if the commuted
1656 // version is already present in the DAG.
1657 if (!RV.getNode() && TLI.isCommutativeBinOp(N->getOpcode()) &&
1658 N->getNumValues() == 1) {
1659 SDValue N0 = N->getOperand(0);
1660 SDValue N1 = N->getOperand(1);
1661
1662 // Constant operands are canonicalized to RHS.
1663 if (N0 != N1 && (isa<ConstantSDNode>(N0) || !isa<ConstantSDNode>(N1))) {
1664 SDValue Ops[] = {N1, N0};
1665 SDNode *CSENode = DAG.getNodeIfExists(N->getOpcode(), N->getVTList(), Ops,
1666 N->getFlags());
1667 if (CSENode)
1668 return SDValue(CSENode, 0);
1669 }
1670 }
1671
1672 return RV;
1673}
1674
1675/// Given a node, return its input chain if it has one, otherwise return a null
1676/// sd operand.
1677static SDValue getInputChainForNode(SDNode *N) {
1678 if (unsigned NumOps = N->getNumOperands()) {
1679 if (N->getOperand(0).getValueType() == MVT::Other)
1680 return N->getOperand(0);
1681 if (N->getOperand(NumOps-1).getValueType() == MVT::Other)
1682 return N->getOperand(NumOps-1);
1683 for (unsigned i = 1; i < NumOps-1; ++i)
1684 if (N->getOperand(i).getValueType() == MVT::Other)
1685 return N->getOperand(i);
1686 }
1687 return SDValue();
1688}
1689
1690SDValue DAGCombiner::visitTokenFactor(SDNode *N) {
1691 // If N has two operands, where one has an input chain equal to the other,
1692 // the 'other' chain is redundant.
1693 if (N->getNumOperands() == 2) {
1694 if (getInputChainForNode(N->getOperand(0).getNode()) == N->getOperand(1))
1695 return N->getOperand(0);
1696 if (getInputChainForNode(N->getOperand(1).getNode()) == N->getOperand(0))
1697 return N->getOperand(1);
1698 }
1699
1700 // Don't simplify token factors if optnone.
1701 if (OptLevel == CodeGenOpt::None)
1702 return SDValue();
1703
1704 SmallVector<SDNode *, 8> TFs; // List of token factors to visit.
1705 SmallVector<SDValue, 8> Ops; // Ops for replacing token factor.
1706 SmallPtrSet<SDNode*, 16> SeenOps;
1707 bool Changed = false; // If we should replace this token factor.
1708
1709 // Start out with this token factor.
1710 TFs.push_back(N);
1711
1712 // Iterate through token factors. The TFs grows when new token factors are
1713 // encountered.
1714 for (unsigned i = 0; i < TFs.size(); ++i) {
1715 SDNode *TF = TFs[i];
1716
1717 // Check each of the operands.
1718 for (const SDValue &Op : TF->op_values()) {
1719 switch (Op.getOpcode()) {
1720 case ISD::EntryToken:
1721 // Entry tokens don't need to be added to the list. They are
1722 // redundant.
1723 Changed = true;
1724 break;
1725
1726 case ISD::TokenFactor:
1727 if (Op.hasOneUse() && !is_contained(TFs, Op.getNode())) {
1728 // Queue up for processing.
1729 TFs.push_back(Op.getNode());
1730 // Clean up in case the token factor is removed.
1731 AddToWorklist(Op.getNode());
1732 Changed = true;
1733 break;
1734 }
1735 LLVM_FALLTHROUGH[[clang::fallthrough]];
1736
1737 default:
1738 // Only add if it isn't already in the list.
1739 if (SeenOps.insert(Op.getNode()).second)
1740 Ops.push_back(Op);
1741 else
1742 Changed = true;
1743 break;
1744 }
1745 }
1746 }
1747
1748 // Remove Nodes that are chained to another node in the list. Do so
1749 // by walking up chains breath-first stopping when we've seen
1750 // another operand. In general we must climb to the EntryNode, but we can exit
1751 // early if we find all remaining work is associated with just one operand as
1752 // no further pruning is possible.
1753
1754 // List of nodes to search through and original Ops from which they originate.
1755 SmallVector<std::pair<SDNode *, unsigned>, 8> Worklist;
1756 SmallVector<unsigned, 8> OpWorkCount; // Count of work for each Op.
1757 SmallPtrSet<SDNode *, 16> SeenChains;
1758 bool DidPruneOps = false;
1759
1760 unsigned NumLeftToConsider = 0;
1761 for (const SDValue &Op : Ops) {
1762 Worklist.push_back(std::make_pair(Op.getNode(), NumLeftToConsider++));
1763 OpWorkCount.push_back(1);
1764 }
1765
1766 auto AddToWorklist = [&](unsigned CurIdx, SDNode *Op, unsigned OpNumber) {
1767 // If this is an Op, we can remove the op from the list. Remark any
1768 // search associated with it as from the current OpNumber.
1769 if (SeenOps.count(Op) != 0) {
1770 Changed = true;
1771 DidPruneOps = true;
1772 unsigned OrigOpNumber = 0;
1773 while (OrigOpNumber < Ops.size() && Ops[OrigOpNumber].getNode() != Op)
1774 OrigOpNumber++;
1775 assert((OrigOpNumber != Ops.size()) &&(static_cast <bool> ((OrigOpNumber != Ops.size()) &&
"expected to find TokenFactor Operand") ? void (0) : __assert_fail
("(OrigOpNumber != Ops.size()) && \"expected to find TokenFactor Operand\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1776, __extension__ __PRETTY_FUNCTION__))
1776 "expected to find TokenFactor Operand")(static_cast <bool> ((OrigOpNumber != Ops.size()) &&
"expected to find TokenFactor Operand") ? void (0) : __assert_fail
("(OrigOpNumber != Ops.size()) && \"expected to find TokenFactor Operand\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1776, __extension__ __PRETTY_FUNCTION__))
;
1777 // Re-mark worklist from OrigOpNumber to OpNumber
1778 for (unsigned i = CurIdx + 1; i < Worklist.size(); ++i) {
1779 if (Worklist[i].second == OrigOpNumber) {
1780 Worklist[i].second = OpNumber;
1781 }
1782 }
1783 OpWorkCount[OpNumber] += OpWorkCount[OrigOpNumber];
1784 OpWorkCount[OrigOpNumber] = 0;
1785 NumLeftToConsider--;
1786 }
1787 // Add if it's a new chain
1788 if (SeenChains.insert(Op).second) {
1789 OpWorkCount[OpNumber]++;
1790 Worklist.push_back(std::make_pair(Op, OpNumber));
1791 }
1792 };
1793
1794 for (unsigned i = 0; i < Worklist.size() && i < 1024; ++i) {
1795 // We need at least be consider at least 2 Ops to prune.
1796 if (NumLeftToConsider <= 1)
1797 break;
1798 auto CurNode = Worklist[i].first;
1799 auto CurOpNumber = Worklist[i].second;
1800 assert((OpWorkCount[CurOpNumber] > 0) &&(static_cast <bool> ((OpWorkCount[CurOpNumber] > 0) &&
"Node should not appear in worklist") ? void (0) : __assert_fail
("(OpWorkCount[CurOpNumber] > 0) && \"Node should not appear in worklist\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1801, __extension__ __PRETTY_FUNCTION__))
1801 "Node should not appear in worklist")(static_cast <bool> ((OpWorkCount[CurOpNumber] > 0) &&
"Node should not appear in worklist") ? void (0) : __assert_fail
("(OpWorkCount[CurOpNumber] > 0) && \"Node should not appear in worklist\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1801, __extension__ __PRETTY_FUNCTION__))
;
1802 switch (CurNode->getOpcode()) {
1803 case ISD::EntryToken:
1804 // Hitting EntryToken is the only way for the search to terminate without
1805 // hitting
1806 // another operand's search. Prevent us from marking this operand
1807 // considered.
1808 NumLeftToConsider++;
1809 break;
1810 case ISD::TokenFactor:
1811 for (const SDValue &Op : CurNode->op_values())
1812 AddToWorklist(i, Op.getNode(), CurOpNumber);
1813 break;
1814 case ISD::CopyFromReg:
1815 case ISD::CopyToReg:
1816 AddToWorklist(i, CurNode->getOperand(0).getNode(), CurOpNumber);
1817 break;
1818 default:
1819 if (auto *MemNode = dyn_cast<MemSDNode>(CurNode))
1820 AddToWorklist(i, MemNode->getChain().getNode(), CurOpNumber);
1821 break;
1822 }
1823 OpWorkCount[CurOpNumber]--;
1824 if (OpWorkCount[CurOpNumber] == 0)
1825 NumLeftToConsider--;
1826 }
1827
1828 // If we've changed things around then replace token factor.
1829 if (Changed) {
1830 SDValue Result;
1831 if (Ops.empty()) {
1832 // The entry token is the only possible outcome.
1833 Result = DAG.getEntryNode();
1834 } else {
1835 if (DidPruneOps) {
1836 SmallVector<SDValue, 8> PrunedOps;
1837 //
1838 for (const SDValue &Op : Ops) {
1839 if (SeenChains.count(Op.getNode()) == 0)
1840 PrunedOps.push_back(Op);
1841 }
1842 Result = DAG.getNode(ISD::TokenFactor, SDLoc(N), MVT::Other, PrunedOps);
1843 } else {
1844 Result = DAG.getNode(ISD::TokenFactor, SDLoc(N), MVT::Other, Ops);
1845 }
1846 }
1847 return Result;
1848 }
1849 return SDValue();
1850}
1851
1852/// MERGE_VALUES can always be eliminated.
1853SDValue DAGCombiner::visitMERGE_VALUES(SDNode *N) {
1854 WorklistRemover DeadNodes(*this);
1855 // Replacing results may cause a different MERGE_VALUES to suddenly
1856 // be CSE'd with N, and carry its uses with it. Iterate until no
1857 // uses remain, to ensure that the node can be safely deleted.
1858 // First add the users of this node to the work list so that they
1859 // can be tried again once they have new operands.
1860 AddUsersToWorklist(N);
1861 do {
1862 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
1863 DAG.ReplaceAllUsesOfValueWith(SDValue(N, i), N->getOperand(i));
1864 } while (!N->use_empty());
1865 deleteAndRecombine(N);
1866 return SDValue(N, 0); // Return N so it doesn't get rechecked!
1867}
1868
1869/// If \p N is a ConstantSDNode with isOpaque() == false return it casted to a
1870/// ConstantSDNode pointer else nullptr.
1871static ConstantSDNode *getAsNonOpaqueConstant(SDValue N) {
1872 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N);
1873 return Const != nullptr && !Const->isOpaque() ? Const : nullptr;
1874}
1875
1876SDValue DAGCombiner::foldBinOpIntoSelect(SDNode *BO) {
1877 auto BinOpcode = BO->getOpcode();
1878 assert((BinOpcode == ISD::ADD || BinOpcode == ISD::SUB ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1887, __extension__ __PRETTY_FUNCTION__))
1879 BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1887, __extension__ __PRETTY_FUNCTION__))
1880 BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1887, __extension__ __PRETTY_FUNCTION__))
1881 BinOpcode == ISD::UREM || BinOpcode == ISD::AND ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1887, __extension__ __PRETTY_FUNCTION__))
1882 BinOpcode == ISD::OR || BinOpcode == ISD::XOR ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1887, __extension__ __PRETTY_FUNCTION__))
1883 BinOpcode == ISD::SHL || BinOpcode == ISD::SRL ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1887, __extension__ __PRETTY_FUNCTION__))
1884 BinOpcode == ISD::SRA || BinOpcode == ISD::FADD ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1887, __extension__ __PRETTY_FUNCTION__))
1885 BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1887, __extension__ __PRETTY_FUNCTION__))
1886 BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) &&(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1887, __extension__ __PRETTY_FUNCTION__))
1887 "Unexpected binary operator")(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1887, __extension__ __PRETTY_FUNCTION__))
;
1888
1889 // Don't do this unless the old select is going away. We want to eliminate the
1890 // binary operator, not replace a binop with a select.
1891 // TODO: Handle ISD::SELECT_CC.
1892 unsigned SelOpNo = 0;
1893 SDValue Sel = BO->getOperand(0);
1894 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse()) {
1895 SelOpNo = 1;
1896 Sel = BO->getOperand(1);
1897 }
1898
1899 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse())
1900 return SDValue();
1901
1902 SDValue CT = Sel.getOperand(1);
1903 if (!isConstantOrConstantVector(CT, true) &&
1904 !isConstantFPBuildVectorOrConstantFP(CT))
1905 return SDValue();
1906
1907 SDValue CF = Sel.getOperand(2);
1908 if (!isConstantOrConstantVector(CF, true) &&
1909 !isConstantFPBuildVectorOrConstantFP(CF))
1910 return SDValue();
1911
1912 // Bail out if any constants are opaque because we can't constant fold those.
1913 // The exception is "and" and "or" with either 0 or -1 in which case we can
1914 // propagate non constant operands into select. I.e.:
1915 // and (select Cond, 0, -1), X --> select Cond, 0, X
1916 // or X, (select Cond, -1, 0) --> select Cond, -1, X
1917 bool CanFoldNonConst = (BinOpcode == ISD::AND || BinOpcode == ISD::OR) &&
1918 (isNullConstantOrNullSplatConstant(CT) ||
1919 isAllOnesConstantOrAllOnesSplatConstant(CT)) &&
1920 (isNullConstantOrNullSplatConstant(CF) ||
1921 isAllOnesConstantOrAllOnesSplatConstant(CF));
1922
1923 SDValue CBO = BO->getOperand(SelOpNo ^ 1);
1924 if (!CanFoldNonConst &&
1925 !isConstantOrConstantVector(CBO, true) &&
1926 !isConstantFPBuildVectorOrConstantFP(CBO))
1927 return SDValue();
1928
1929 EVT VT = Sel.getValueType();
1930
1931 // In case of shift value and shift amount may have different VT. For instance
1932 // on x86 shift amount is i8 regardles of LHS type. Bail out if we have
1933 // swapped operands and value types do not match. NB: x86 is fine if operands
1934 // are not swapped with shift amount VT being not bigger than shifted value.
1935 // TODO: that is possible to check for a shift operation, correct VTs and
1936 // still perform optimization on x86 if needed.
1937 if (SelOpNo && VT != CBO.getValueType())
1938 return SDValue();
1939
1940 // We have a select-of-constants followed by a binary operator with a
1941 // constant. Eliminate the binop by pulling the constant math into the select.
1942 // Example: add (select Cond, CT, CF), CBO --> select Cond, CT + CBO, CF + CBO
1943 SDLoc DL(Sel);
1944 SDValue NewCT = SelOpNo ? DAG.getNode(BinOpcode, DL, VT, CBO, CT)
1945 : DAG.getNode(BinOpcode, DL, VT, CT, CBO);
1946 if (!CanFoldNonConst && !NewCT.isUndef() &&
1947 !isConstantOrConstantVector(NewCT, true) &&
1948 !isConstantFPBuildVectorOrConstantFP(NewCT))
1949 return SDValue();
1950
1951 SDValue NewCF = SelOpNo ? DAG.getNode(BinOpcode, DL, VT, CBO, CF)
1952 : DAG.getNode(BinOpcode, DL, VT, CF, CBO);
1953 if (!CanFoldNonConst && !NewCF.isUndef() &&
1954 !isConstantOrConstantVector(NewCF, true) &&
1955 !isConstantFPBuildVectorOrConstantFP(NewCF))
1956 return SDValue();
1957
1958 return DAG.getSelect(DL, VT, Sel.getOperand(0), NewCT, NewCF);
1959}
1960
1961static SDValue foldAddSubBoolOfMaskedVal(SDNode *N, SelectionDAG &DAG) {
1962 assert((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) &&(static_cast <bool> ((N->getOpcode() == ISD::ADD || N
->getOpcode() == ISD::SUB) && "Expecting add or sub"
) ? void (0) : __assert_fail ("(N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) && \"Expecting add or sub\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1963, __extension__ __PRETTY_FUNCTION__))
1963 "Expecting add or sub")(static_cast <bool> ((N->getOpcode() == ISD::ADD || N
->getOpcode() == ISD::SUB) && "Expecting add or sub"
) ? void (0) : __assert_fail ("(N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) && \"Expecting add or sub\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1963, __extension__ __PRETTY_FUNCTION__))
;
1964
1965 // Match a constant operand and a zext operand for the math instruction:
1966 // add Z, C
1967 // sub C, Z
1968 bool IsAdd = N->getOpcode() == ISD::ADD;
1969 SDValue C = IsAdd ? N->getOperand(1) : N->getOperand(0);
1970 SDValue Z = IsAdd ? N->getOperand(0) : N->getOperand(1);
1971 auto *CN = dyn_cast<ConstantSDNode>(C);
1972 if (!CN || Z.getOpcode() != ISD::ZERO_EXTEND)
1973 return SDValue();
1974
1975 // Match the zext operand as a setcc of a boolean.
1976 if (Z.getOperand(0).getOpcode() != ISD::SETCC ||
1977 Z.getOperand(0).getValueType() != MVT::i1)
1978 return SDValue();
1979
1980 // Match the compare as: setcc (X & 1), 0, eq.
1981 SDValue SetCC = Z.getOperand(0);
1982 ISD::CondCode CC = cast<CondCodeSDNode>(SetCC->getOperand(2))->get();
1983 if (CC != ISD::SETEQ || !isNullConstant(SetCC.getOperand(1)) ||
1984 SetCC.getOperand(0).getOpcode() != ISD::AND ||
1985 !isOneConstant(SetCC.getOperand(0).getOperand(1)))
1986 return SDValue();
1987
1988 // We are adding/subtracting a constant and an inverted low bit. Turn that
1989 // into a subtract/add of the low bit with incremented/decremented constant:
1990 // add (zext i1 (seteq (X & 1), 0)), C --> sub C+1, (zext (X & 1))
1991 // sub C, (zext i1 (seteq (X & 1), 0)) --> add C-1, (zext (X & 1))
1992 EVT VT = C.getValueType();
1993 SDLoc DL(N);
1994 SDValue LowBit = DAG.getZExtOrTrunc(SetCC.getOperand(0), DL, VT);
1995 SDValue C1 = IsAdd ? DAG.getConstant(CN->getAPIntValue() + 1, DL, VT) :
1996 DAG.getConstant(CN->getAPIntValue() - 1, DL, VT);
1997 return DAG.getNode(IsAdd ? ISD::SUB : ISD::ADD, DL, VT, C1, LowBit);
1998}
1999
2000/// Try to fold a 'not' shifted sign-bit with add/sub with constant operand into
2001/// a shift and add with a different constant.
2002static SDValue foldAddSubOfSignBit(SDNode *N, SelectionDAG &DAG) {
2003 assert((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) &&(static_cast <bool> ((N->getOpcode() == ISD::ADD || N
->getOpcode() == ISD::SUB) && "Expecting add or sub"
) ? void (0) : __assert_fail ("(N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) && \"Expecting add or sub\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2004, __extension__ __PRETTY_FUNCTION__))
2004 "Expecting add or sub")(static_cast <bool> ((N->getOpcode() == ISD::ADD || N
->getOpcode() == ISD::SUB) && "Expecting add or sub"
) ? void (0) : __assert_fail ("(N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) && \"Expecting add or sub\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2004, __extension__ __PRETTY_FUNCTION__))
;
2005
2006 // We need a constant operand for the add/sub, and the other operand is a
2007 // logical shift right: add (srl), C or sub C, (srl).
2008 bool IsAdd = N->getOpcode() == ISD::ADD;
2009 SDValue ConstantOp = IsAdd ? N->getOperand(1) : N->getOperand(0);
2010 SDValue ShiftOp = IsAdd ? N->getOperand(0) : N->getOperand(1);
2011 ConstantSDNode *C = isConstOrConstSplat(ConstantOp);
2012 if (!C || ShiftOp.getOpcode() != ISD::SRL)
2013 return SDValue();
2014
2015 // The shift must be of a 'not' value.
2016 // TODO: Use isBitwiseNot() if it works with vectors.
2017 SDValue Not = ShiftOp.getOperand(0);
2018 if (!Not.hasOneUse() || Not.getOpcode() != ISD::XOR ||
2019 !isAllOnesConstantOrAllOnesSplatConstant(Not.getOperand(1)))
2020 return SDValue();
2021
2022 // The shift must be moving the sign bit to the least-significant-bit.
2023 EVT VT = ShiftOp.getValueType();
2024 SDValue ShAmt = ShiftOp.getOperand(1);
2025 ConstantSDNode *ShAmtC = isConstOrConstSplat(ShAmt);
2026 if (!ShAmtC || ShAmtC->getZExtValue() != VT.getScalarSizeInBits() - 1)
2027 return SDValue();
2028
2029 // Eliminate the 'not' by adjusting the shift and add/sub constant:
2030 // add (srl (not X), 31), C --> add (sra X, 31), (C + 1)
2031 // sub C, (srl (not X), 31) --> add (srl X, 31), (C - 1)
2032 SDLoc DL(N);
2033 auto ShOpcode = IsAdd ? ISD::SRA : ISD::SRL;
2034 SDValue NewShift = DAG.getNode(ShOpcode, DL, VT, Not.getOperand(0), ShAmt);
2035 APInt NewC = IsAdd ? C->getAPIntValue() + 1 : C->getAPIntValue() - 1;
2036 return DAG.getNode(ISD::ADD, DL, VT, NewShift, DAG.getConstant(NewC, DL, VT));
2037}
2038
2039SDValue DAGCombiner::visitADD(SDNode *N) {
2040 SDValue N0 = N->getOperand(0);
2041 SDValue N1 = N->getOperand(1);
2042 EVT VT = N0.getValueType();
2043 SDLoc DL(N);
2044
2045 // fold vector ops
2046 if (VT.isVector()) {
2047 if (SDValue FoldedVOp = SimplifyVBinOp(N))
2048 return FoldedVOp;
2049
2050 // fold (add x, 0) -> x, vector edition
2051 if (ISD::isBuildVectorAllZeros(N1.getNode()))
2052 return N0;
2053 if (ISD::isBuildVectorAllZeros(N0.getNode()))
2054 return N1;
2055 }
2056
2057 // fold (add x, undef) -> undef
2058 if (N0.isUndef())
2059 return N0;
2060
2061 if (N1.isUndef())
2062 return N1;
2063
2064 if (DAG.isConstantIntBuildVectorOrConstantInt(N0)) {
2065 // canonicalize constant to RHS
2066 if (!DAG.isConstantIntBuildVectorOrConstantInt(N1))
2067 return DAG.getNode(ISD::ADD, DL, VT, N1, N0);
2068 // fold (add c1, c2) -> c1+c2
2069 return DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, N0.getNode(),
2070 N1.getNode());
2071 }
2072
2073 // fold (add x, 0) -> x
2074 if (isNullConstant(N1))
2075 return N0;
2076
2077 if (isConstantOrConstantVector(N1, /* NoOpaque */ true)) {
2078 // fold ((c1-A)+c2) -> (c1+c2)-A
2079 if (N0.getOpcode() == ISD::SUB &&
2080 isConstantOrConstantVector(N0.getOperand(0), /* NoOpaque */ true)) {
2081 // FIXME: Adding 2 constants should be handled by FoldConstantArithmetic.
2082 return DAG.getNode(ISD::SUB, DL, VT,
2083 DAG.getNode(ISD::ADD, DL, VT, N1, N0.getOperand(0)),
2084 N0.getOperand(1));
2085 }
2086
2087 // add (sext i1 X), 1 -> zext (not i1 X)
2088 // We don't transform this pattern:
2089 // add (zext i1 X), -1 -> sext (not i1 X)
2090 // because most (?) targets generate better code for the zext form.
2091 if (N0.getOpcode() == ISD::SIGN_EXTEND && N0.hasOneUse() &&
2092 isOneConstantOrOneSplatConstant(N1)) {
2093 SDValue X = N0.getOperand(0);
2094 if ((!LegalOperations ||
2095 (TLI.isOperationLegal(ISD::XOR, X.getValueType()) &&
2096 TLI.isOperationLegal(ISD::ZERO_EXTEND, VT))) &&
2097 X.getScalarValueSizeInBits() == 1) {
2098 SDValue Not = DAG.getNOT(DL, X, X.getValueType());
2099 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Not);
2100 }
2101 }
2102
2103 // Undo the add -> or combine to merge constant offsets from a frame index.
2104 if (N0.getOpcode() == ISD::OR &&
2105 isa<FrameIndexSDNode>(N0.getOperand(0)) &&
2106 isa<ConstantSDNode>(N0.getOperand(1)) &&
2107 DAG.haveNoCommonBitsSet(N0.getOperand(0), N0.getOperand(1))) {
2108 SDValue Add0 = DAG.getNode(ISD::ADD, DL, VT, N1, N0.getOperand(1));
2109 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), Add0);
2110 }
2111 }
2112
2113 if (SDValue NewSel = foldBinOpIntoSelect(N))
2114 return NewSel;
2115
2116 // reassociate add
2117 if (SDValue RADD = ReassociateOps(ISD::ADD, DL, N0, N1))
2118 return RADD;
2119
2120 // fold ((0-A) + B) -> B-A
2121 if (N0.getOpcode() == ISD::SUB &&
2122 isNullConstantOrNullSplatConstant(N0.getOperand(0)))
2123 return DAG.getNode(ISD::SUB, DL, VT, N1, N0.getOperand(1));
2124
2125 // fold (A + (0-B)) -> A-B
2126 if (N1.getOpcode() == ISD::SUB &&
2127 isNullConstantOrNullSplatConstant(N1.getOperand(0)))
2128 return DAG.getNode(ISD::SUB, DL, VT, N0, N1.getOperand(1));
2129
2130 // fold (A+(B-A)) -> B
2131 if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(1))
2132 return N1.getOperand(0);
2133
2134 // fold ((B-A)+A) -> B
2135 if (N0.getOpcode() == ISD::SUB && N1 == N0.getOperand(1))
2136 return N0.getOperand(0);
2137
2138 // fold (A+(B-(A+C))) to (B-C)
2139 if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD &&
2140 N0 == N1.getOperand(1).getOperand(0))
2141 return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0),
2142 N1.getOperand(1).getOperand(1));
2143
2144 // fold (A+(B-(C+A))) to (B-C)
2145 if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD &&
2146 N0 == N1.getOperand(1).getOperand(1))
2147 return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0),
2148 N1.getOperand(1).getOperand(0));
2149
2150 // fold (A+((B-A)+or-C)) to (B+or-C)
2151 if ((N1.getOpcode() == ISD::SUB || N1.getOpcode() == ISD::ADD) &&
2152 N1.getOperand(0).getOpcode() == ISD::SUB &&
2153 N0 == N1.getOperand(0).getOperand(1))
2154 return DAG.getNode(N1.getOpcode(), DL, VT, N1.getOperand(0).getOperand(0),
2155 N1.getOperand(1));
2156
2157 // fold (A-B)+(C-D) to (A+C)-(B+D) when A or C is constant
2158 if (N0.getOpcode() == ISD::SUB && N1.getOpcode() == ISD::SUB) {
2159 SDValue N00 = N0.getOperand(0);
2160 SDValue N01 = N0.getOperand(1);
2161 SDValue N10 = N1.getOperand(0);
2162 SDValue N11 = N1.getOperand(1);
2163
2164 if (isConstantOrConstantVector(N00) || isConstantOrConstantVector(N10))
2165 return DAG.getNode(ISD::SUB, DL, VT,
2166 DAG.getNode(ISD::ADD, SDLoc(N0), VT, N00, N10),
2167 DAG.getNode(ISD::ADD, SDLoc(N1), VT, N01, N11));
2168 }
2169
2170 if (SDValue V = foldAddSubBoolOfMaskedVal(N, DAG))
2171 return V;
2172
2173 if (SDValue V = foldAddSubOfSignBit(N, DAG))
2174 return V;
2175
2176 if (SimplifyDemandedBits(SDValue(N, 0)))
2177 return SDValue(N, 0);
2178
2179 // fold (a+b) -> (a|b) iff a and b share no bits.
2180 if ((!LegalOperations || TLI.isOperationLegal(ISD::OR, VT)) &&
2181 DAG.haveNoCommonBitsSet(N0, N1))
2182 return DAG.getNode(ISD::OR, DL, VT, N0, N1);
2183
2184 // fold (add (xor a, -1), 1) -> (sub 0, a)
2185 if (isBitwiseNot(N0) && isOneConstantOrOneSplatConstant(N1))
2186 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
2187 N0.getOperand(0));
2188
2189 if (SDValue Combined = visitADDLike(N0, N1, N))
2190 return Combined;
2191
2192 if (SDValue Combined = visitADDLike(N1, N0, N))
2193 return Combined;
2194
2195 return SDValue();
2196}
2197
2198static SDValue getAsCarry(const TargetLowering &TLI, SDValue V) {
2199 bool Masked = false;
2200
2201 // First, peel away TRUNCATE/ZERO_EXTEND/AND nodes due to legalization.
2202 while (true) {
2203 if (V.getOpcode() == ISD::TRUNCATE || V.getOpcode() == ISD::ZERO_EXTEND) {
2204 V = V.getOperand(0);
2205 continue;
2206 }
2207
2208 if (V.getOpcode() == ISD::AND && isOneConstant(V.getOperand(1))) {
2209 Masked = true;
2210 V = V.getOperand(0);
2211 continue;
2212 }
2213
2214 break;
2215 }
2216
2217 // If this is not a carry, return.
2218 if (V.getResNo() != 1)
2219 return SDValue();
2220
2221 if (V.getOpcode() != ISD::ADDCARRY && V.getOpcode() != ISD::SUBCARRY &&
2222 V.getOpcode() != ISD::UADDO && V.getOpcode() != ISD::USUBO)
2223 return SDValue();
2224
2225 // If the result is masked, then no matter what kind of bool it is we can
2226 // return. If it isn't, then we need to make sure the bool type is either 0 or
2227 // 1 and not other values.
2228 if (Masked ||
2229 TLI.getBooleanContents(V.getValueType()) ==
2230 TargetLoweringBase::ZeroOrOneBooleanContent)
2231 return V;
2232
2233 return SDValue();
2234}
2235
2236SDValue DAGCombiner::visitADDLike(SDValue N0, SDValue N1, SDNode *LocReference) {
2237 EVT VT = N0.getValueType();
2238 SDLoc DL(LocReference);
2239
2240 // fold (add x, shl(0 - y, n)) -> sub(x, shl(y, n))
2241 if (N1.getOpcode() == ISD::SHL && N1.getOperand(0).getOpcode() == ISD::SUB &&
2242 isNullConstantOrNullSplatConstant(N1.getOperand(0).getOperand(0)))
2243 return DAG.getNode(ISD::SUB, DL, VT, N0,
2244 DAG.getNode(ISD::SHL, DL, VT,
2245 N1.getOperand(0).getOperand(1),
2246 N1.getOperand(1)));
2247
2248 if (N1.getOpcode() == ISD::AND) {
2249 SDValue AndOp0 = N1.getOperand(0);
2250 unsigned NumSignBits = DAG.ComputeNumSignBits(AndOp0);
2251 unsigned DestBits = VT.getScalarSizeInBits();
2252
2253 // (add z, (and (sbbl x, x), 1)) -> (sub z, (sbbl x, x))
2254 // and similar xforms where the inner op is either ~0 or 0.
2255 if (NumSignBits == DestBits &&
2256 isOneConstantOrOneSplatConstant(N1->getOperand(1)))
2257 return DAG.getNode(ISD::SUB, DL, VT, N0, AndOp0);
2258 }
2259
2260 // add (sext i1), X -> sub X, (zext i1)
2261 if (N0.getOpcode() == ISD::SIGN_EXTEND &&
2262 N0.getOperand(0).getValueType() == MVT::i1 &&
2263 !TLI.isOperationLegal(ISD::SIGN_EXTEND, MVT::i1)) {
2264 SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0));
2265 return DAG.getNode(ISD::SUB, DL, VT, N1, ZExt);
2266 }
2267
2268 // add X, (sextinreg Y i1) -> sub X, (and Y 1)
2269 if (N1.getOpcode() == ISD::SIGN_EXTEND_INREG) {
2270 VTSDNode *TN = cast<VTSDNode>(N1.getOperand(1));
2271 if (TN->getVT() == MVT::i1) {
2272 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0),
2273 DAG.getConstant(1, DL, VT));
2274 return DAG.getNode(ISD::SUB, DL, VT, N0, ZExt);
2275 }
2276 }
2277
2278 // (add X, (addcarry Y, 0, Carry)) -> (addcarry X, Y, Carry)
2279 if (N1.getOpcode() == ISD::ADDCARRY && isNullConstant(N1.getOperand(1)) &&
2280 N1.getResNo() == 0)
2281 return DAG.getNode(ISD::ADDCARRY, DL, N1->getVTList(),
2282 N0, N1.getOperand(0), N1.getOperand(2));
2283
2284 // (add X, Carry) -> (addcarry X, 0, Carry)
2285 if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT))
2286 if (SDValue Carry = getAsCarry(TLI, N1))
2287 return DAG.getNode(ISD::ADDCARRY, DL,
2288 DAG.getVTList(VT, Carry.getValueType()), N0,
2289 DAG.getConstant(0, DL, VT), Carry);
2290
2291 return SDValue();
2292}
2293
2294SDValue DAGCombiner::visitADDC(SDNode *N) {
2295 SDValue N0 = N->getOperand(0);
2296 SDValue N1 = N->getOperand(1);
2297 EVT VT = N0.getValueType();
2298 SDLoc DL(N);
2299
2300 // If the flag result is dead, turn this into an ADD.
2301 if (!N->hasAnyUseOfValue(1))
2302 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2303 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2304
2305 // canonicalize constant to RHS.
2306 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
2307 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2308 if (N0C && !N1C)
2309 return DAG.getNode(ISD::ADDC, DL, N->getVTList(), N1, N0);
2310
2311 // fold (addc x, 0) -> x + no carry out
2312 if (isNullConstant(N1))
2313 return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE,
2314 DL, MVT::Glue));
2315
2316 // If it cannot overflow, transform into an add.
2317 if (DAG.computeOverflowKind(N0, N1) == SelectionDAG::OFK_Never)
2318 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2319 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2320
2321 return SDValue();
2322}
2323
2324static SDValue flipBoolean(SDValue V, const SDLoc &DL, EVT VT,
2325 SelectionDAG &DAG, const TargetLowering &TLI) {
2326 SDValue Cst;
2327 switch (TLI.getBooleanContents(VT)) {
2328 case TargetLowering::ZeroOrOneBooleanContent:
2329 case TargetLowering::UndefinedBooleanContent:
2330 Cst = DAG.getConstant(1, DL, VT);
2331 break;
2332 case TargetLowering::ZeroOrNegativeOneBooleanContent:
2333 Cst = DAG.getConstant(-1, DL, VT);
2334 break;
2335 }
2336
2337 return DAG.getNode(ISD::XOR, DL, VT, V, Cst);
2338}
2339
2340static bool isBooleanFlip(SDValue V, EVT VT, const TargetLowering &TLI) {
2341 if (V.getOpcode() != ISD::XOR) return false;
2342 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V.getOperand(1));
2343 if (!Const) return false;
2344
2345 switch(TLI.getBooleanContents(VT)) {
2346 case TargetLowering::ZeroOrOneBooleanContent:
2347 return Const->isOne();
2348 case TargetLowering::ZeroOrNegativeOneBooleanContent:
2349 return Const->isAllOnesValue();
2350 case TargetLowering::UndefinedBooleanContent:
2351 return (Const->getAPIntValue() & 0x01) == 1;
2352 }
2353 llvm_unreachable("Unsupported boolean content")::llvm::llvm_unreachable_internal("Unsupported boolean content"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2353)
;
2354}
2355
2356SDValue DAGCombiner::visitUADDO(SDNode *N) {
2357 SDValue N0 = N->getOperand(0);
2358 SDValue N1 = N->getOperand(1);
2359 EVT VT = N0.getValueType();
2360 if (VT.isVector())
2361 return SDValue();
2362
2363 EVT CarryVT = N->getValueType(1);
2364 SDLoc DL(N);
2365
2366 // If the flag result is dead, turn this into an ADD.
2367 if (!N->hasAnyUseOfValue(1))
2368 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2369 DAG.getUNDEF(CarryVT));
2370
2371 // canonicalize constant to RHS.
2372 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
2373 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2374 if (N0C && !N1C)
2375 return DAG.getNode(ISD::UADDO, DL, N->getVTList(), N1, N0);
2376
2377 // fold (uaddo x, 0) -> x + no carry out
2378 if (isNullConstant(N1))
2379 return CombineTo(N, N0, DAG.getConstant(0, DL, CarryVT));
2380
2381 // If it cannot overflow, transform into an add.
2382 if (DAG.computeOverflowKind(N0, N1) == SelectionDAG::OFK_Never)
2383 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2384 DAG.getConstant(0, DL, CarryVT));
2385
2386 // fold (uaddo (xor a, -1), 1) -> (usub 0, a) and flip carry.
2387 if (isBitwiseNot(N0) && isOneConstantOrOneSplatConstant(N1)) {
2388 SDValue Sub = DAG.getNode(ISD::USUBO, DL, N->getVTList(),
2389 DAG.getConstant(0, DL, VT),
2390 N0.getOperand(0));
2391 return CombineTo(N, Sub,
2392 flipBoolean(Sub.getValue(1), DL, CarryVT, DAG, TLI));
2393 }
2394
2395 if (SDValue Combined = visitUADDOLike(N0, N1, N))
2396 return Combined;
2397
2398 if (SDValue Combined = visitUADDOLike(N1, N0, N))
2399 return Combined;
2400
2401 return SDValue();
2402}
2403
2404SDValue DAGCombiner::visitUADDOLike(SDValue N0, SDValue N1, SDNode *N) {
2405 auto VT = N0.getValueType();
2406
2407 // (uaddo X, (addcarry Y, 0, Carry)) -> (addcarry X, Y, Carry)
2408 // If Y + 1 cannot overflow.
2409 if (N1.getOpcode() == ISD::ADDCARRY && isNullConstant(N1.getOperand(1))) {
2410 SDValue Y = N1.getOperand(0);
2411 SDValue One = DAG.getConstant(1, SDLoc(N), Y.getValueType());
2412 if (DAG.computeOverflowKind(Y, One) == SelectionDAG::OFK_Never)
2413 return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0, Y,
2414 N1.getOperand(2));
2415 }
2416
2417 // (uaddo X, Carry) -> (addcarry X, 0, Carry)
2418 if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT))
2419 if (SDValue Carry = getAsCarry(TLI, N1))
2420 return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0,
2421 DAG.getConstant(0, SDLoc(N), VT), Carry);
2422
2423 return SDValue();
2424}
2425
2426SDValue DAGCombiner::visitADDE(SDNode *N) {
2427 SDValue N0 = N->getOperand(0);
2428 SDValue N1 = N->getOperand(1);
2429 SDValue CarryIn = N->getOperand(2);
2430
2431 // canonicalize constant to RHS
2432 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
2433 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2434 if (N0C && !N1C)
2435 return DAG.getNode(ISD::ADDE, SDLoc(N), N->getVTList(),
2436 N1, N0, CarryIn);
2437
2438 // fold (adde x, y, false) -> (addc x, y)
2439 if (CarryIn.getOpcode() == ISD::CARRY_FALSE)
2440 return DAG.getNode(ISD::ADDC, SDLoc(N), N->getVTList(), N0, N1);
2441
2442 return SDValue();
2443}
2444
2445SDValue DAGCombiner::visitADDCARRY(SDNode *N) {
2446 SDValue N0 = N->getOperand(0);
2447 SDValue N1 = N->getOperand(1);
2448 SDValue CarryIn = N->getOperand(2);
2449 SDLoc DL(N);
2450
2451 // canonicalize constant to RHS
2452 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
2453 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2454 if (N0C && !N1C)
2455 return DAG.getNode(ISD::ADDCARRY, DL, N->getVTList(), N1, N0, CarryIn);
2456
2457 // fold (addcarry x, y, false) -> (uaddo x, y)
2458 if (isNullConstant(CarryIn)) {
2459 if (!LegalOperations ||
2460 TLI.isOperationLegalOrCustom(ISD::UADDO, N->getValueType(0)))
2461 return DAG.getNode(ISD::UADDO, DL, N->getVTList(), N0, N1);
2462 }
2463
2464 EVT CarryVT = CarryIn.getValueType();
2465
2466 // fold (addcarry 0, 0, X) -> (and (ext/trunc X), 1) and no carry.
2467 if (isNullConstant(N0) && isNullConstant(N1)) {
2468 EVT VT = N0.getValueType();
2469 SDValue CarryExt = DAG.getBoolExtOrTrunc(CarryIn, DL, VT, CarryVT);
2470 AddToWorklist(CarryExt.getNode());
2471 return CombineTo(N, DAG.getNode(ISD::AND, DL, VT, CarryExt,
2472 DAG.getConstant(1, DL, VT)),
2473 DAG.getConstant(0, DL, CarryVT));
2474 }
2475
2476 // fold (addcarry (xor a, -1), 0, !b) -> (subcarry 0, a, b) and flip carry.
2477 if (isBitwiseNot(N0) && isNullConstant(N1) &&
2478 isBooleanFlip(CarryIn, CarryVT, TLI)) {
2479 SDValue Sub = DAG.getNode(ISD::SUBCARRY, DL, N->getVTList(),
2480 DAG.getConstant(0, DL, N0.getValueType()),
2481 N0.getOperand(0), CarryIn.getOperand(0));
2482 return CombineTo(N, Sub,
2483 flipBoolean(Sub.getValue(1), DL, CarryVT, DAG, TLI));
2484 }
2485
2486 if (SDValue Combined = visitADDCARRYLike(N0, N1, CarryIn, N))
2487 return Combined;
2488
2489 if (SDValue Combined = visitADDCARRYLike(N1, N0, CarryIn, N))
2490 return Combined;
2491
2492 return SDValue();
2493}
2494
2495SDValue DAGCombiner::visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn,
2496 SDNode *N) {
2497 // Iff the flag result is dead:
2498 // (addcarry (add|uaddo X, Y), 0, Carry) -> (addcarry X, Y, Carry)
2499 if ((N0.getOpcode() == ISD::ADD ||
2500 (N0.getOpcode() == ISD::UADDO && N0.getResNo() == 0)) &&
2501 isNullConstant(N1) && !N->hasAnyUseOfValue(1))
2502 return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(),
2503 N0.getOperand(0), N0.getOperand(1), CarryIn);
2504
2505 /**
2506 * When one of the addcarry argument is itself a carry, we may be facing
2507 * a diamond carry propagation. In which case we try to transform the DAG
2508 * to ensure linear carry propagation if that is possible.
2509 *
2510 * We are trying to get:
2511 * (addcarry X, 0, (addcarry A, B, Z):Carry)
2512 */
2513 if (auto Y = getAsCarry(TLI, N1)) {
2514 /**
2515 * (uaddo A, B)
2516 * / \
2517 * Carry Sum
2518 * | \
2519 * | (addcarry *, 0, Z)
2520 * | /
2521 * \ Carry
2522 * | /
2523 * (addcarry X, *, *)
2524 */
2525 if (Y.getOpcode() == ISD::UADDO &&
2526 CarryIn.getResNo() == 1 &&
2527 CarryIn.getOpcode() == ISD::ADDCARRY &&
2528 isNullConstant(CarryIn.getOperand(1)) &&
2529 CarryIn.getOperand(0) == Y.getValue(0)) {
2530 auto NewY = DAG.getNode(ISD::ADDCARRY, SDLoc(N), Y->getVTList(),
2531 Y.getOperand(0), Y.getOperand(1),
2532 CarryIn.getOperand(2));
2533 AddToWorklist(NewY.getNode());
2534 return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0,
2535 DAG.getConstant(0, SDLoc(N), N0.getValueType()),
2536 NewY.getValue(1));
2537 }
2538 }
2539
2540 return SDValue();
2541}
2542
2543// Since it may not be valid to emit a fold to zero for vector initializers
2544// check if we can before folding.
2545static SDValue tryFoldToZero(const SDLoc &DL, const TargetLowering &TLI, EVT VT,
2546 SelectionDAG &DAG, bool LegalOperations,
2547 bool LegalTypes) {
2548 if (!VT.isVector())
2549 return DAG.getConstant(0, DL, VT);
2550 if (!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT))
2551 return DAG.getConstant(0, DL, VT);
2552 return SDValue();
2553}
2554
2555SDValue DAGCombiner::visitSUB(SDNode *N) {
2556 SDValue N0 = N->getOperand(0);
2557 SDValue N1 = N->getOperand(1);
2558 EVT VT = N0.getValueType();
2559 SDLoc DL(N);
2560
2561 // fold vector ops
2562 if (VT.isVector()) {
2563 if (SDValue FoldedVOp = SimplifyVBinOp(N))
2564 return FoldedVOp;
2565
2566 // fold (sub x, 0) -> x, vector edition
2567 if (ISD::isBuildVectorAllZeros(N1.getNode()))
2568 return N0;
2569 }
2570
2571 // fold (sub x, x) -> 0
2572 // FIXME: Refactor this and xor and other similar operations together.
2573 if (N0 == N1)
2574 return tryFoldToZero(DL, TLI, VT, DAG, LegalOperations, LegalTypes);
2575 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
2576 DAG.isConstantIntBuildVectorOrConstantInt(N1)) {
2577 // fold (sub c1, c2) -> c1-c2
2578 return DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, N0.getNode(),
2579 N1.getNode());
2580 }
2581
2582 if (SDValue NewSel = foldBinOpIntoSelect(N))
2583 return NewSel;
2584
2585 ConstantSDNode *N1C = getAsNonOpaqueConstant(N1);
2586
2587 // fold (sub x, c) -> (add x, -c)
2588 if (N1C) {
2589 return DAG.getNode(ISD::ADD, DL, VT, N0,
2590 DAG.getConstant(-N1C->getAPIntValue(), DL, VT));
2591 }
2592
2593 if (isNullConstantOrNullSplatConstant(N0)) {
2594 unsigned BitWidth = VT.getScalarSizeInBits();
2595 // Right-shifting everything out but the sign bit followed by negation is
2596 // the same as flipping arithmetic/logical shift type without the negation:
2597 // -(X >>u 31) -> (X >>s 31)
2598 // -(X >>s 31) -> (X >>u 31)
2599 if (N1->getOpcode() == ISD::SRA || N1->getOpcode() == ISD::SRL) {
2600 ConstantSDNode *ShiftAmt = isConstOrConstSplat(N1.getOperand(1));
2601 if (ShiftAmt && ShiftAmt->getZExtValue() == BitWidth - 1) {
2602 auto NewSh = N1->getOpcode() == ISD::SRA ? ISD::SRL : ISD::SRA;
2603 if (!LegalOperations || TLI.isOperationLegal(NewSh, VT))
2604 return DAG.getNode(NewSh, DL, VT, N1.getOperand(0), N1.getOperand(1));
2605 }
2606 }
2607
2608 // 0 - X --> 0 if the sub is NUW.
2609 if (N->getFlags().hasNoUnsignedWrap())
2610 return N0;
2611
2612 if (DAG.MaskedValueIsZero(N1, ~APInt::getSignMask(BitWidth))) {
2613 // N1 is either 0 or the minimum signed value. If the sub is NSW, then
2614 // N1 must be 0 because negating the minimum signed value is undefined.
2615 if (N->getFlags().hasNoSignedWrap())
2616 return N0;
2617
2618 // 0 - X --> X if X is 0 or the minimum signed value.
2619 return N1;
2620 }
2621 }
2622
2623 // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1)
2624 if (isAllOnesConstantOrAllOnesSplatConstant(N0))
2625 return DAG.getNode(ISD::XOR, DL, VT, N1, N0);
2626
2627 // fold (A - (0-B)) -> A+B
2628 if (N1.getOpcode() == ISD::SUB &&
2629 isNullConstantOrNullSplatConstant(N1.getOperand(0)))
2630 return DAG.getNode(ISD::ADD, DL, VT, N0, N1.getOperand(1));
2631
2632 // fold A-(A-B) -> B
2633 if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(0))
2634 return N1.getOperand(1);
2635
2636 // fold (A+B)-A -> B
2637 if (N0.getOpcode() == ISD::ADD && N0.getOperand(0) == N1)
2638 return N0.getOperand(1);
2639
2640 // fold (A+B)-B -> A
2641 if (N0.getOpcode() == ISD::ADD && N0.getOperand(1) == N1)
2642 return N0.getOperand(0);
2643
2644 // fold C2-(A+C1) -> (C2-C1)-A
2645 if (N1.getOpcode() == ISD::ADD) {
2646 SDValue N11 = N1.getOperand(1);
2647 if (isConstantOrConstantVector(N0, /* NoOpaques */ true) &&
2648 isConstantOrConstantVector(N11, /* NoOpaques */ true)) {
2649 SDValue NewC = DAG.getNode(ISD::SUB, DL, VT, N0, N11);
2650 return DAG.getNode(ISD::SUB, DL, VT, NewC, N1.getOperand(0));
2651 }
2652 }
2653
2654 // fold ((A+(B+or-C))-B) -> A+or-C
2655 if (N0.getOpcode() == ISD::ADD &&
2656 (N0.getOperand(1).getOpcode() == ISD::SUB ||
2657 N0.getOperand(1).getOpcode() == ISD::ADD) &&
2658 N0.getOperand(1).getOperand(0) == N1)
2659 return DAG.getNode(N0.getOperand(1).getOpcode(), DL, VT, N0.getOperand(0),
2660 N0.getOperand(1).getOperand(1));
2661
2662 // fold ((A+(C+B))-B) -> A+C
2663 if (N0.getOpcode() == ISD::ADD && N0.getOperand(1).getOpcode() == ISD::ADD &&
2664 N0.getOperand(1).getOperand(1) == N1)
2665 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0),
2666 N0.getOperand(1).getOperand(0));
2667
2668 // fold ((A-(B-C))-C) -> A-B
2669 if (N0.getOpcode() == ISD::SUB && N0.getOperand(1).getOpcode() == ISD::SUB &&
2670 N0.getOperand(1).getOperand(1) == N1)
2671 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0),
2672 N0.getOperand(1).getOperand(0));
2673
2674 // fold (A-(B-C)) -> A+(C-B)
2675 if (N1.getOpcode() == ISD::SUB && N1.hasOneUse())
2676 return DAG.getNode(ISD::ADD, DL, VT, N0,
2677 DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(1),
2678 N1.getOperand(0)));
2679
2680 // fold (X - (-Y * Z)) -> (X + (Y * Z))
2681 if (N1.getOpcode() == ISD::MUL && N1.hasOneUse()) {
2682 if (N1.getOperand(0).getOpcode() == ISD::SUB &&
2683 isNullConstantOrNullSplatConstant(N1.getOperand(0).getOperand(0))) {
2684 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT,
2685 N1.getOperand(0).getOperand(1),
2686 N1.getOperand(1));
2687 return DAG.getNode(ISD::ADD, DL, VT, N0, Mul);
2688 }
2689 if (N1.getOperand(1).getOpcode() == ISD::SUB &&
2690 isNullConstantOrNullSplatConstant(N1.getOperand(1).getOperand(0))) {
2691 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT,
2692 N1.getOperand(0),
2693 N1.getOperand(1).getOperand(1));
2694 return DAG.getNode(ISD::ADD, DL, VT, N0, Mul);
2695 }
2696 }
2697
2698 // If either operand of a sub is undef, the result is undef
2699 if (N0.isUndef())
2700 return N0;
2701 if (N1.isUndef())
2702 return N1;
2703
2704 if (SDValue V = foldAddSubBoolOfMaskedVal(N, DAG))
2705 return V;
2706
2707 if (SDValue V = foldAddSubOfSignBit(N, DAG))
2708 return V;
2709
2710 // fold Y = sra (X, size(X)-1); sub (xor (X, Y), Y) -> (abs X)
2711 if (TLI.isOperationLegalOrCustom(ISD::ABS, VT)) {
2712 if (N0.getOpcode() == ISD::XOR && N1.getOpcode() == ISD::SRA) {
2713 SDValue X0 = N0.getOperand(0), X1 = N0.getOperand(1);
2714 SDValue S0 = N1.getOperand(0);
2715 if ((X0 == S0 && X1 == N1) || (X0 == N1 && X1 == S0)) {
2716 unsigned OpSizeInBits = VT.getScalarSizeInBits();
2717 if (ConstantSDNode *C = isConstOrConstSplat(N1.getOperand(1)))
2718 if (C->getAPIntValue() == (OpSizeInBits - 1))
2719 return DAG.getNode(ISD::ABS, SDLoc(N), VT, S0);
2720 }
2721 }
2722 }
2723
2724 // If the relocation model supports it, consider symbol offsets.
2725 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N0))
2726 if (!LegalOperations && TLI.isOffsetFoldingLegal(GA)) {
2727 // fold (sub Sym, c) -> Sym-c
2728 if (N1C && GA->getOpcode() == ISD::GlobalAddress)
2729 return DAG.getGlobalAddress(GA->getGlobal(), SDLoc(N1C), VT,
2730 GA->getOffset() -
2731 (uint64_t)N1C->getSExtValue());
2732 // fold (sub Sym+c1, Sym+c2) -> c1-c2
2733 if (GlobalAddressSDNode *GB = dyn_cast<GlobalAddressSDNode>(N1))
2734 if (GA->getGlobal() == GB->getGlobal())
2735 return DAG.getConstant((uint64_t)GA->getOffset() - GB->getOffset(),
2736 DL, VT);
2737 }
2738
2739 // sub X, (sextinreg Y i1) -> add X, (and Y 1)
2740 if (N1.getOpcode() == ISD::SIGN_EXTEND_INREG) {
2741 VTSDNode *TN = cast<VTSDNode>(N1.getOperand(1));
2742 if (TN->getVT() == MVT::i1) {
2743 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0),
2744 DAG.getConstant(1, DL, VT));
2745 return DAG.getNode(ISD::ADD, DL, VT, N0, ZExt);
2746 }
2747 }
2748
2749 return SDValue();
2750}
2751
2752SDValue DAGCombiner::visitSUBC(SDNode *N) {
2753 SDValue N0 = N->getOperand(0);
2754 SDValue N1 = N->getOperand(1);
2755 EVT VT = N0.getValueType();
2756 SDLoc DL(N);
2757
2758 // If the flag result is dead, turn this into an SUB.
2759 if (!N->hasAnyUseOfValue(1))
2760 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1),
2761 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2762
2763 // fold (subc x, x) -> 0 + no borrow
2764 if (N0 == N1)
2765 return CombineTo(N, DAG.getConstant(0, DL, VT),
2766 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2767
2768 // fold (subc x, 0) -> x + no borrow
2769 if (isNullConstant(N1))
2770 return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2771
2772 // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1) + no borrow
2773 if (isAllOnesConstant(N0))
2774 return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0),
2775 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2776
2777 return SDValue();
2778}
2779
2780SDValue DAGCombiner::visitUSUBO(SDNode *N) {
2781 SDValue N0 = N->getOperand(0);
2782 SDValue N1 = N->getOperand(1);
2783 EVT VT = N0.getValueType();
2784 if (VT.isVector())
2785 return SDValue();
2786
2787 EVT CarryVT = N->getValueType(1);
2788 SDLoc DL(N);
2789
2790 // If the flag result is dead, turn this into an SUB.
2791 if (!N->hasAnyUseOfValue(1))
2792 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1),
2793 DAG.getUNDEF(CarryVT));
2794
2795 // fold (usubo x, x) -> 0 + no borrow
2796 if (N0 == N1)
2797 return CombineTo(N, DAG.getConstant(0, DL, VT),
2798 DAG.getConstant(0, DL, CarryVT));
2799
2800 // fold (usubo x, 0) -> x + no borrow
2801 if (isNullConstant(N1))
2802 return CombineTo(N, N0, DAG.getConstant(0, DL, CarryVT));
2803
2804 // Canonicalize (usubo -1, x) -> ~x, i.e. (xor x, -1) + no borrow
2805 if (isAllOnesConstant(N0))
2806 return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0),
2807 DAG.getConstant(0, DL, CarryVT));
2808
2809 return SDValue();
2810}
2811
2812SDValue DAGCombiner::visitSUBE(SDNode *N) {
2813 SDValue N0 = N->getOperand(0);
2814 SDValue N1 = N->getOperand(1);
2815 SDValue CarryIn = N->getOperand(2);
2816
2817 // fold (sube x, y, false) -> (subc x, y)
2818 if (CarryIn.getOpcode() == ISD::CARRY_FALSE)
2819 return DAG.getNode(ISD::SUBC, SDLoc(N), N->getVTList(), N0, N1);
2820
2821 return SDValue();
2822}
2823
2824SDValue DAGCombiner::visitSUBCARRY(SDNode *N) {
2825 SDValue N0 = N->getOperand(0);
2826 SDValue N1 = N->getOperand(1);
2827 SDValue CarryIn = N->getOperand(2);
2828
2829 // fold (subcarry x, y, false) -> (usubo x, y)
2830 if (isNullConstant(CarryIn)) {
2831 if (!LegalOperations ||
2832 TLI.isOperationLegalOrCustom(ISD::USUBO, N->getValueType(0)))
2833 return DAG.getNode(ISD::USUBO, SDLoc(N), N->getVTList(), N0, N1);
2834 }
2835
2836 return SDValue();
2837}
2838
2839SDValue DAGCombiner::visitMUL(SDNode *N) {
2840 SDValue N0 = N->getOperand(0);
2841 SDValue N1 = N->getOperand(1);
2842 EVT VT = N0.getValueType();
2843
2844 // fold (mul x, undef) -> 0
2845 if (N0.isUndef() || N1.isUndef())
2846 return DAG.getConstant(0, SDLoc(N), VT);
2847
2848 bool N0IsConst = false;
2849 bool N1IsConst = false;
2850 bool N1IsOpaqueConst = false;
2851 bool N0IsOpaqueConst = false;
2852 APInt ConstValue0, ConstValue1;
2853 // fold vector ops
2854 if (VT.isVector()) {
2855 if (SDValue FoldedVOp = SimplifyVBinOp(N))
2856 return FoldedVOp;
2857
2858 N0IsConst = ISD::isConstantSplatVector(N0.getNode(), ConstValue0);
2859 N1IsConst = ISD::isConstantSplatVector(N1.getNode(), ConstValue1);
2860 assert((!N0IsConst ||(static_cast <bool> ((!N0IsConst || ConstValue0.getBitWidth
() == VT.getScalarSizeInBits()) && "Splat APInt should be element width"
) ? void (0) : __assert_fail ("(!N0IsConst || ConstValue0.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2862, __extension__ __PRETTY_FUNCTION__))
2861 ConstValue0.getBitWidth() == VT.getScalarSizeInBits()) &&(static_cast <bool> ((!N0IsConst || ConstValue0.getBitWidth
() == VT.getScalarSizeInBits()) && "Splat APInt should be element width"
) ? void (0) : __assert_fail ("(!N0IsConst || ConstValue0.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2862, __extension__ __PRETTY_FUNCTION__))
2862 "Splat APInt should be element width")(static_cast <bool> ((!N0IsConst || ConstValue0.getBitWidth
() == VT.getScalarSizeInBits()) && "Splat APInt should be element width"
) ? void (0) : __assert_fail ("(!N0IsConst || ConstValue0.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2862, __extension__ __PRETTY_FUNCTION__))
;
2863 assert((!N1IsConst ||(static_cast <bool> ((!N1IsConst || ConstValue1.getBitWidth
() == VT.getScalarSizeInBits()) && "Splat APInt should be element width"
) ? void (0) : __assert_fail ("(!N1IsConst || ConstValue1.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2865, __extension__ __PRETTY_FUNCTION__))
2864 ConstValue1.getBitWidth() == VT.getScalarSizeInBits()) &&(static_cast <bool> ((!N1IsConst || ConstValue1.getBitWidth
() == VT.getScalarSizeInBits()) && "Splat APInt should be element width"
) ? void (0) : __assert_fail ("(!N1IsConst || ConstValue1.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2865, __extension__ __PRETTY_FUNCTION__))
2865 "Splat APInt should be element width")(static_cast <bool> ((!N1IsConst || ConstValue1.getBitWidth
() == VT.getScalarSizeInBits()) && "Splat APInt should be element width"
) ? void (0) : __assert_fail ("(!N1IsConst || ConstValue1.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2865, __extension__ __PRETTY_FUNCTION__))
;
2866 } else {
2867 N0IsConst = isa<ConstantSDNode>(N0);
2868 if (N0IsConst) {
2869 ConstValue0 = cast<ConstantSDNode>(N0)->getAPIntValue();
2870 N0IsOpaqueConst = cast<ConstantSDNode>(N0)->isOpaque();
2871 }
2872 N1IsConst = isa<ConstantSDNode>(N1);
2873 if (N1IsConst) {
2874 ConstValue1 = cast<ConstantSDNode>(N1)->getAPIntValue();
2875 N1IsOpaqueConst = cast<ConstantSDNode>(N1)->isOpaque();
2876 }
2877 }
2878
2879 // fold (mul c1, c2) -> c1*c2
2880 if (N0IsConst && N1IsConst && !N0IsOpaqueConst && !N1IsOpaqueConst)
2881 return DAG.FoldConstantArithmetic(ISD::MUL, SDLoc(N), VT,
2882 N0.getNode(), N1.getNode());
2883
2884 // canonicalize constant to RHS (vector doesn't have to splat)
2885 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
2886 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
2887 return DAG.getNode(ISD::MUL, SDLoc(N), VT, N1, N0);
2888 // fold (mul x, 0) -> 0
2889 if (N1IsConst && ConstValue1.isNullValue())
2890 return N1;
2891 // fold (mul x, 1) -> x
2892 if (N1IsConst && ConstValue1.isOneValue())
2893 return N0;
2894
2895 if (SDValue NewSel = foldBinOpIntoSelect(N))
2896 return NewSel;
2897
2898 // fold (mul x, -1) -> 0-x
2899 if (N1IsConst && ConstValue1.isAllOnesValue()) {
2900 SDLoc DL(N);
2901 return DAG.getNode(ISD::SUB, DL, VT,
2902 DAG.getConstant(0, DL, VT), N0);
2903 }
2904 // fold (mul x, (1 << c)) -> x << c
2905 if (isConstantOrConstantVector(N1, /*NoOpaques*/ true) &&
2906 DAG.isKnownToBeAPowerOfTwo(N1) &&
2907 (!VT.isVector() || Level <= AfterLegalizeVectorOps)) {
2908 SDLoc DL(N);
2909 SDValue LogBase2 = BuildLogBase2(N1, DL);
2910 EVT ShiftVT = getShiftAmountTy(N0.getValueType());
2911 SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ShiftVT);
2912 return DAG.getNode(ISD::SHL, DL, VT, N0, Trunc);
2913 }
2914 // fold (mul x, -(1 << c)) -> -(x << c) or (-x) << c
2915 if (N1IsConst && !N1IsOpaqueConst && (-ConstValue1).isPowerOf2()) {
2916 unsigned Log2Val = (-ConstValue1).logBase2();
2917 SDLoc DL(N);
2918 // FIXME: If the input is something that is easily negated (e.g. a
2919 // single-use add), we should put the negate there.
2920 return DAG.getNode(ISD::SUB, DL, VT,
2921 DAG.getConstant(0, DL, VT),
2922 DAG.getNode(ISD::SHL, DL, VT, N0,
2923 DAG.getConstant(Log2Val, DL,
2924 getShiftAmountTy(N0.getValueType()))));
2925 }
2926
2927 // (mul (shl X, c1), c2) -> (mul X, c2 << c1)
2928 if (N0.getOpcode() == ISD::SHL &&
2929 isConstantOrConstantVector(N1, /* NoOpaques */ true) &&
2930 isConstantOrConstantVector(N0.getOperand(1), /* NoOpaques */ true)) {
2931 SDValue C3 = DAG.getNode(ISD::SHL, SDLoc(N), VT, N1, N0.getOperand(1));
2932 if (isConstantOrConstantVector(C3))
2933 return DAG.getNode(ISD::MUL, SDLoc(N), VT, N0.getOperand(0), C3);
2934 }
2935
2936 // Change (mul (shl X, C), Y) -> (shl (mul X, Y), C) when the shift has one
2937 // use.
2938 {
2939 SDValue Sh(nullptr, 0), Y(nullptr, 0);
2940
2941 // Check for both (mul (shl X, C), Y) and (mul Y, (shl X, C)).
2942 if (N0.getOpcode() == ISD::SHL &&
2943 isConstantOrConstantVector(N0.getOperand(1)) &&
2944 N0.getNode()->hasOneUse()) {
2945 Sh = N0; Y = N1;
2946 } else if (N1.getOpcode() == ISD::SHL &&
2947 isConstantOrConstantVector(N1.getOperand(1)) &&
2948 N1.getNode()->hasOneUse()) {
2949 Sh = N1; Y = N0;
2950 }
2951
2952 if (Sh.getNode()) {
2953 SDValue Mul = DAG.getNode(ISD::MUL, SDLoc(N), VT, Sh.getOperand(0), Y);
2954 return DAG.getNode(ISD::SHL, SDLoc(N), VT, Mul, Sh.getOperand(1));
2955 }
2956 }
2957
2958 // fold (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2)
2959 if (DAG.isConstantIntBuildVectorOrConstantInt(N1) &&
2960 N0.getOpcode() == ISD::ADD &&
2961 DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1)) &&
2962 isMulAddWithConstProfitable(N, N0, N1))
2963 return DAG.getNode(ISD::ADD, SDLoc(N), VT,
2964 DAG.getNode(ISD::MUL, SDLoc(N0), VT,
2965 N0.getOperand(0), N1),
2966 DAG.getNode(ISD::MUL, SDLoc(N1), VT,
2967 N0.getOperand(1), N1));
2968
2969 // reassociate mul
2970 if (SDValue RMUL = ReassociateOps(ISD::MUL, SDLoc(N), N0, N1))
2971 return RMUL;
2972
2973 return SDValue();
2974}
2975
2976/// Return true if divmod libcall is available.
2977static bool isDivRemLibcallAvailable(SDNode *Node, bool isSigned,
2978 const TargetLowering &TLI) {
2979 RTLIB::Libcall LC;
2980 EVT NodeType = Node->getValueType(0);
2981 if (!NodeType.isSimple())
2982 return false;
2983 switch (NodeType.getSimpleVT().SimpleTy) {
2984 default: return false; // No libcall for vector types.
2985 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2986 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2987 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2988 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2989 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2990 }
2991
2992 return TLI.getLibcallName(LC) != nullptr;
2993}
2994
2995/// Issue divrem if both quotient and remainder are needed.
2996SDValue DAGCombiner::useDivRem(SDNode *Node) {
2997 if (Node->use_empty())
2998 return SDValue(); // This is a dead node, leave it alone.
2999
3000 unsigned Opcode = Node->getOpcode();
3001 bool isSigned = (Opcode == ISD::SDIV) || (Opcode == ISD::SREM);
3002 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3003
3004 // DivMod lib calls can still work on non-legal types if using lib-calls.
3005 EVT VT = Node->getValueType(0);
3006 if (VT.isVector() || !VT.isInteger())
3007 return SDValue();
3008
3009 if (!TLI.isTypeLegal(VT) && !TLI.isOperationCustom(DivRemOpc, VT))
3010 return SDValue();
3011
3012 // If DIVREM is going to get expanded into a libcall,
3013 // but there is no libcall available, then don't combine.
3014 if (!TLI.isOperationLegalOrCustom(DivRemOpc, VT) &&
3015 !isDivRemLibcallAvailable(Node, isSigned, TLI))
3016 return SDValue();
3017
3018 // If div is legal, it's better to do the normal expansion
3019 unsigned OtherOpcode = 0;
3020 if ((Opcode == ISD::SDIV) || (Opcode == ISD::UDIV)) {
3021 OtherOpcode = isSigned ? ISD::SREM : ISD::UREM;
3022 if (TLI.isOperationLegalOrCustom(Opcode, VT))
3023 return SDValue();
3024 } else {
3025 OtherOpcode = isSigned ? ISD::SDIV : ISD::UDIV;
3026 if (TLI.isOperationLegalOrCustom(OtherOpcode, VT))
3027 return SDValue();
3028 }
3029
3030 SDValue Op0 = Node->getOperand(0);
3031 SDValue Op1 = Node->getOperand(1);
3032 SDValue combined;
3033 for (SDNode::use_iterator UI = Op0.getNode()->use_begin(),
3034 UE = Op0.getNode()->use_end(); UI != UE; ++UI) {
3035 SDNode *User = *UI;
3036 if (User == Node || User->getOpcode() == ISD::DELETED_NODE ||
3037 User->use_empty())
3038 continue;
3039 // Convert the other matching node(s), too;
3040 // otherwise, the DIVREM may get target-legalized into something
3041 // target-specific that we won't be able to recognize.
3042 unsigned UserOpc = User->getOpcode();
3043 if ((UserOpc == Opcode || UserOpc == OtherOpcode || UserOpc == DivRemOpc) &&
3044 User->getOperand(0) == Op0 &&
3045 User->getOperand(1) == Op1) {
3046 if (!combined) {
3047 if (UserOpc == OtherOpcode) {
3048 SDVTList VTs = DAG.getVTList(VT, VT);
3049 combined = DAG.getNode(DivRemOpc, SDLoc(Node), VTs, Op0, Op1);
3050 } else if (UserOpc == DivRemOpc) {
3051 combined = SDValue(User, 0);
3052 } else {
3053 assert(UserOpc == Opcode)(static_cast <bool> (UserOpc == Opcode) ? void (0) : __assert_fail
("UserOpc == Opcode", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3053, __extension__ __PRETTY_FUNCTION__))
;
3054 continue;
3055 }
3056 }
3057 if (UserOpc == ISD::SDIV || UserOpc == ISD::UDIV)
3058 CombineTo(User, combined);
3059 else if (UserOpc == ISD::SREM || UserOpc == ISD::UREM)
3060 CombineTo(User, combined.getValue(1));
3061 }
3062 }
3063 return combined;
3064}
3065
3066static SDValue simplifyDivRem(SDNode *N, SelectionDAG &DAG) {
3067 SDValue N0 = N->getOperand(0);
3068 SDValue N1 = N->getOperand(1);
3069 EVT VT = N->getValueType(0);
3070 SDLoc DL(N);
3071
3072 if (DAG.isUndef(N->getOpcode(), {N0, N1}))
3073 return DAG.getUNDEF(VT);
3074
3075 // undef / X -> 0
3076 // undef % X -> 0
3077 if (N0.isUndef())
3078 return DAG.getConstant(0, DL, VT);
3079
3080 return SDValue();
3081}
3082
3083SDValue DAGCombiner::visitSDIV(SDNode *N) {
3084 SDValue N0 = N->getOperand(0);
3085 SDValue N1 = N->getOperand(1);
3086 EVT VT = N->getValueType(0);
3087 EVT CCVT = getSetCCResultType(VT);
3088
3089 // fold vector ops
3090 if (VT.isVector())
3091 if (SDValue FoldedVOp = SimplifyVBinOp(N))
3092 return FoldedVOp;
3093
3094 SDLoc DL(N);
3095
3096 // fold (sdiv c1, c2) -> c1/c2
3097 ConstantSDNode *N0C = isConstOrConstSplat(N0);
3098 ConstantSDNode *N1C = isConstOrConstSplat(N1);
3099 if (N0C && N1C && !N0C->isOpaque() && !N1C->isOpaque())
3100 return DAG.FoldConstantArithmetic(ISD::SDIV, DL, VT, N0C, N1C);
3101 // fold (sdiv X, 1) -> X
3102 if (N1C && N1C->isOne())
3103 return N0;
3104 // fold (sdiv X, -1) -> 0-X
3105 if (N1C && N1C->isAllOnesValue())
3106 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), N0);
3107 // fold (sdiv X, MIN_SIGNED) -> select(X == MIN_SIGNED, 1, 0)
3108 if (N1C && N1C->getAPIntValue().isMinSignedValue())
3109 return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ),
3110 DAG.getConstant(1, DL, VT),
3111 DAG.getConstant(0, DL, VT));
3112
3113 if (SDValue V = simplifyDivRem(N, DAG))
3114 return V;
3115
3116 if (SDValue NewSel = foldBinOpIntoSelect(N))
3117 return NewSel;
3118
3119 // If we know the sign bits of both operands are zero, strength reduce to a
3120 // udiv instead. Handles (X&15) /s 4 -> X&15 >> 2
3121 if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
3122 return DAG.getNode(ISD::UDIV, DL, N1.getValueType(), N0, N1);
3123
3124 if (SDValue V = visitSDIVLike(N0, N1, N))
3125 return V;
3126
3127 // sdiv, srem -> sdivrem
3128 // If the divisor is constant, then return DIVREM only if isIntDivCheap() is
3129 // true. Otherwise, we break the simplification logic in visitREM().
3130 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
3131 if (!N1C || TLI.isIntDivCheap(N->getValueType(0), Attr))
3132 if (SDValue DivRem = useDivRem(N))
3133 return DivRem;
3134
3135 return SDValue();
3136}
3137
3138SDValue DAGCombiner::visitSDIVLike(SDValue N0, SDValue N1, SDNode *N) {
3139 SDLoc DL(N);
3140 EVT VT = N->getValueType(0);
3141 EVT CCVT = getSetCCResultType(VT);
3142 unsigned BitWidth = VT.getScalarSizeInBits();
3143
3144 ConstantSDNode *N1C = isConstOrConstSplat(N1);
3145
3146 // Helper for determining whether a value is a power-2 constant scalar or a
3147 // vector of such elements.
3148 auto IsPowerOfTwo = [](ConstantSDNode *C) {
3149 if (C->isNullValue() || C->isOpaque())
3150 return false;
3151 if (C->getAPIntValue().isPowerOf2())
3152 return true;
3153 if ((-C->getAPIntValue()).isPowerOf2())
3154 return true;
3155 return false;
3156 };
3157
3158 // fold (sdiv X, pow2) -> simple ops after legalize
3159 // FIXME: We check for the exact bit here because the generic lowering gives
3160 // better results in that case. The target-specific lowering should learn how
3161 // to handle exact sdivs efficiently.
3162 if (!N->getFlags().hasExact() &&
3163 ISD::matchUnaryPredicate(N1C ? SDValue(N1C, 0) : N1, IsPowerOfTwo)) {
3164 // Target-specific implementation of sdiv x, pow2.
3165 if (SDValue Res = BuildSDIVPow2(N))
3166 return Res;
3167
3168 // Create constants that are functions of the shift amount value.
3169 EVT ShiftAmtTy = getShiftAmountTy(N0.getValueType());
3170 SDValue Bits = DAG.getConstant(BitWidth, DL, ShiftAmtTy);
3171 SDValue C1 = DAG.getNode(ISD::CTTZ, DL, VT, N1);
3172 C1 = DAG.getZExtOrTrunc(C1, DL, ShiftAmtTy);
3173 SDValue Inexact = DAG.getNode(ISD::SUB, DL, ShiftAmtTy, Bits, C1);
3174 if (!isConstantOrConstantVector(Inexact))
3175 return SDValue();
3176
3177 // Splat the sign bit into the register
3178 SDValue Sign = DAG.getNode(ISD::SRA, DL, VT, N0,
3179 DAG.getConstant(BitWidth - 1, DL, ShiftAmtTy));
3180 AddToWorklist(Sign.getNode());
3181
3182 // Add (N0 < 0) ? abs2 - 1 : 0;
3183 SDValue Srl = DAG.getNode(ISD::SRL, DL, VT, Sign, Inexact);
3184 AddToWorklist(Srl.getNode());
3185 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Srl);
3186 AddToWorklist(Add.getNode());
3187 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Add, C1);
3188 AddToWorklist(Sra.getNode());
3189
3190 // Special case: (sdiv X, 1) -> X
3191 // Special Case: (sdiv X, -1) -> 0-X
3192 SDValue One = DAG.getConstant(1, DL, VT);
3193 SDValue AllOnes = DAG.getAllOnesConstant(DL, VT);
3194 SDValue IsOne = DAG.getSetCC(DL, CCVT, N1, One, ISD::SETEQ);
3195 SDValue IsAllOnes = DAG.getSetCC(DL, CCVT, N1, AllOnes, ISD::SETEQ);
3196 SDValue IsOneOrAllOnes = DAG.getNode(ISD::OR, DL, CCVT, IsOne, IsAllOnes);
3197 Sra = DAG.getSelect(DL, VT, IsOneOrAllOnes, N0, Sra);
3198
3199 // If dividing by a positive value, we're done. Otherwise, the result must
3200 // be negated.
3201 SDValue Zero = DAG.getConstant(0, DL, VT);
3202 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, Zero, Sra);
3203
3204 // FIXME: Use SELECT_CC once we improve SELECT_CC constant-folding.
3205 SDValue IsNeg = DAG.getSetCC(DL, CCVT, N1, Zero, ISD::SETLT);
3206 SDValue Res = DAG.getSelect(DL, VT, IsNeg, Sub, Sra);
3207 return Res;
3208 }
3209
3210 // If integer divide is expensive and we satisfy the requirements, emit an
3211 // alternate sequence. Targets may check function attributes for size/speed
3212 // trade-offs.
3213 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
3214 if (N1C && !TLI.isIntDivCheap(N->getValueType(0), Attr))
3215 if (SDValue Op = BuildSDIV(N))
3216 return Op;
3217
3218 return SDValue();
3219}
3220
3221SDValue DAGCombiner::visitUDIV(SDNode *N) {
3222 SDValue N0 = N->getOperand(0);
3223 SDValue N1 = N->getOperand(1);
3224 EVT VT = N->getValueType(0);
3225 EVT CCVT = getSetCCResultType(VT);
3226
3227 // fold vector ops
3228 if (VT.isVector())
3229 if (SDValue FoldedVOp = SimplifyVBinOp(N))
3230 return FoldedVOp;
3231
3232 SDLoc DL(N);
3233
3234 // fold (udiv c1, c2) -> c1/c2
3235 ConstantSDNode *N0C = isConstOrConstSplat(N0);
3236 ConstantSDNode *N1C = isConstOrConstSplat(N1);
3237 if (N0C && N1C)
3238 if (SDValue Folded = DAG.FoldConstantArithmetic(ISD::UDIV, DL, VT,
3239 N0C, N1C))
3240 return Folded;
3241 // fold (udiv X, 1) -> X
3242 if (N1C && N1C->isOne())
3243 return N0;
3244 // fold (udiv X, -1) -> select(X == -1, 1, 0)
3245 if (N1C && N1C->getAPIntValue().isAllOnesValue())
3246 return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ),
3247 DAG.getConstant(1, DL, VT),
3248 DAG.getConstant(0, DL, VT));
3249
3250 if (SDValue V = simplifyDivRem(N, DAG))
3251 return V;
3252
3253 if (SDValue NewSel = foldBinOpIntoSelect(N))
3254 return NewSel;
3255
3256 if (SDValue V = visitUDIVLike(N0, N1, N))
3257 return V;
3258
3259 // sdiv, srem -> sdivrem
3260 // If the divisor is constant, then return DIVREM only if isIntDivCheap() is
3261 // true. Otherwise, we break the simplification logic in visitREM().
3262 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
3263 if (!N1C || TLI.isIntDivCheap(N->getValueType(0), Attr))
3264 if (SDValue DivRem = useDivRem(N))
3265 return DivRem;
3266
3267 return SDValue();
3268}
3269
3270SDValue DAGCombiner::visitUDIVLike(SDValue N0, SDValue N1, SDNode *N) {
3271 SDLoc DL(N);
3272 EVT VT = N->getValueType(0);
3273
3274 ConstantSDNode *N1C = isConstOrConstSplat(N1);
3275
3276 // fold (udiv x, (1 << c)) -> x >>u c
3277 if (isConstantOrConstantVector(N1, /*NoOpaques*/ true) &&
3278 DAG.isKnownToBeAPowerOfTwo(N1)) {
3279 SDValue LogBase2 = BuildLogBase2(N1, DL);
3280 AddToWorklist(LogBase2.getNode());
3281
3282 EVT ShiftVT = getShiftAmountTy(N0.getValueType());
3283 SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ShiftVT);
3284 AddToWorklist(Trunc.getNode());
3285 return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc);
3286 }
3287
3288 // fold (udiv x, (shl c, y)) -> x >>u (log2(c)+y) iff c is power of 2
3289 if (N1.getOpcode() == ISD::SHL) {
3290 SDValue N10 = N1.getOperand(0);
3291 if (isConstantOrConstantVector(N10, /*NoOpaques*/ true) &&
3292 DAG.isKnownToBeAPowerOfTwo(N10)) {
3293 SDValue LogBase2 = BuildLogBase2(N10, DL);
3294 AddToWorklist(LogBase2.getNode());
3295
3296 EVT ADDVT = N1.getOperand(1).getValueType();
3297 SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ADDVT);
3298 AddToWorklist(Trunc.getNode());
3299 SDValue Add = DAG.getNode(ISD::ADD, DL, ADDVT, N1.getOperand(1), Trunc);
3300 AddToWorklist(Add.getNode());
3301 return DAG.getNode(ISD::SRL, DL, VT, N0, Add);
3302 }
3303 }
3304
3305 // fold (udiv x, c) -> alternate
3306 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
3307 if (N1C && !TLI.isIntDivCheap(N->getValueType(0), Attr))
3308 if (SDValue Op = BuildUDIV(N))
3309 return Op;
3310
3311 return SDValue();
3312}
3313
3314// handles ISD::SREM and ISD::UREM
3315SDValue DAGCombiner::visitREM(SDNode *N) {
3316 unsigned Opcode = N->getOpcode();
3317 SDValue N0 = N->getOperand(0);
3318 SDValue N1 = N->getOperand(1);
3319 EVT VT = N->getValueType(0);
3320 EVT CCVT = getSetCCResultType(VT);
3321
3322 bool isSigned = (Opcode == ISD::SREM);
3323 SDLoc DL(N);
3324
3325 // fold (rem c1, c2) -> c1%c2
3326 ConstantSDNode *N0C = isConstOrConstSplat(N0);
3327 ConstantSDNode *N1C = isConstOrConstSplat(N1);
3328 if (N0C && N1C)
3329 if (SDValue Folded = DAG.FoldConstantArithmetic(Opcode, DL, VT, N0C, N1C))
3330 return Folded;
3331 // fold (urem X, -1) -> select(X == -1, 0, x)
3332 if (!isSigned && N1C && N1C->getAPIntValue().isAllOnesValue())
3333 return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ),
3334 DAG.getConstant(0, DL, VT), N0);
3335
3336 if (SDValue V = simplifyDivRem(N, DAG))
3337 return V;
3338
3339 if (SDValue NewSel = foldBinOpIntoSelect(N))
3340 return NewSel;
3341
3342 if (isSigned) {
3343 // If we know the sign bits of both operands are zero, strength reduce to a
3344 // urem instead. Handles (X & 0x0FFFFFFF) %s 16 -> X&15
3345 if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
3346 return DAG.getNode(ISD::UREM, DL, VT, N0, N1);
3347 } else {
3348 SDValue NegOne = DAG.getAllOnesConstant(DL, VT);
3349 if (DAG.isKnownToBeAPowerOfTwo(N1)) {
3350 // fold (urem x, pow2) -> (and x, pow2-1)
3351 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne);
3352 AddToWorklist(Add.getNode());
3353 return DAG.getNode(ISD::AND, DL, VT, N0, Add);
3354 }
3355 if (N1.getOpcode() == ISD::SHL &&
3356 DAG.isKnownToBeAPowerOfTwo(N1.getOperand(0))) {
3357 // fold (urem x, (shl pow2, y)) -> (and x, (add (shl pow2, y), -1))
3358 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne);
3359 AddToWorklist(Add.getNode());
3360 return DAG.getNode(ISD::AND, DL, VT, N0, Add);
3361 }
3362 }
3363
3364 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
3365
3366 // If X/C can be simplified by the division-by-constant logic, lower
3367 // X%C to the equivalent of X-X/C*C.
3368 // Reuse the SDIVLike/UDIVLike combines - to avoid mangling nodes, the
3369 // speculative DIV must not cause a DIVREM conversion. We guard against this
3370 // by skipping the simplification if isIntDivCheap(). When div is not cheap,
3371 // combine will not return a DIVREM. Regardless, checking cheapness here
3372 // makes sense since the simplification results in fatter code.
3373 if (DAG.isKnownNeverZero(N1) && !TLI.isIntDivCheap(VT, Attr)) {
3374 SDValue OptimizedDiv =
3375 isSigned ? visitSDIVLike(N0, N1, N) : visitUDIVLike(N0, N1, N);
3376 if (OptimizedDiv.getNode() && OptimizedDiv.getOpcode() != ISD::UDIVREM &&
3377 OptimizedDiv.getOpcode() != ISD::SDIVREM) {
3378 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, OptimizedDiv, N1);
3379 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
3380 AddToWorklist(OptimizedDiv.getNode());
3381 AddToWorklist(Mul.getNode());
3382 return Sub;
3383 }
3384 }
3385
3386 // sdiv, srem -> sdivrem
3387 if (SDValue DivRem = useDivRem(N))
3388 return DivRem.getValue(1);
3389
3390 return SDValue();
3391}
3392
3393SDValue DAGCombiner::visitMULHS(SDNode *N) {
3394 SDValue N0 = N->getOperand(0);
3395 SDValue N1 = N->getOperand(1);
3396 EVT VT = N->getValueType(0);
3397 SDLoc DL(N);
3398
3399 if (VT.isVector()) {
3400 // fold (mulhs x, 0) -> 0
3401 if (ISD::isBuildVectorAllZeros(N1.getNode()))
3402 return N1;
3403 if (ISD::isBuildVectorAllZeros(N0.getNode()))
3404 return N0;
3405 }
3406
3407 // fold (mulhs x, 0) -> 0
3408 if (isNullConstant(N1))
3409 return N1;
3410 // fold (mulhs x, 1) -> (sra x, size(x)-1)
3411 if (isOneConstant(N1))
3412 return DAG.getNode(ISD::SRA, DL, N0.getValueType(), N0,
3413 DAG.getConstant(N0.getValueSizeInBits() - 1, DL,
3414 getShiftAmountTy(N0.getValueType())));
3415
3416 // fold (mulhs x, undef) -> 0
3417 if (N0.isUndef() || N1.isUndef())
3418 return DAG.getConstant(0, DL, VT);
3419
3420 // If the type twice as wide is legal, transform the mulhs to a wider multiply
3421 // plus a shift.
3422 if (VT.isSimple() && !VT.isVector()) {
3423 MVT Simple = VT.getSimpleVT();
3424 unsigned SimpleSize = Simple.getSizeInBits();
3425 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
3426 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
3427 N0 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N0);
3428 N1 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N1);
3429 N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
3430 N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
3431 DAG.getConstant(SimpleSize, DL,
3432 getShiftAmountTy(N1.getValueType())));
3433 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
3434 }
3435 }
3436
3437 return SDValue();
3438}
3439
3440SDValue DAGCombiner::visitMULHU(SDNode *N) {
3441 SDValue N0 = N->getOperand(0);
3442 SDValue N1 = N->getOperand(1);
3443 EVT VT = N->getValueType(0);
3444 SDLoc DL(N);
3445
3446 if (VT.isVector()) {
3447 // fold (mulhu x, 0) -> 0
3448 if (ISD::isBuildVectorAllZeros(N1.getNode()))
3449 return N1;
3450 if (ISD::isBuildVectorAllZeros(N0.getNode()))
3451 return N0;
3452 }
3453
3454 // fold (mulhu x, 0) -> 0
3455 if (isNullConstant(N1))
3456 return N1;
3457 // fold (mulhu x, 1) -> 0
3458 if (isOneConstant(N1))
3459 return DAG.getConstant(0, DL, N0.getValueType());
3460 // fold (mulhu x, undef) -> 0
3461 if (N0.isUndef() || N1.isUndef())
3462 return DAG.getConstant(0, DL, VT);
3463
3464 // If the type twice as wide is legal, transform the mulhu to a wider multiply
3465 // plus a shift.
3466 if (VT.isSimple() && !VT.isVector()) {
3467 MVT Simple = VT.getSimpleVT();
3468 unsigned SimpleSize = Simple.getSizeInBits();
3469 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
3470 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
3471 N0 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N0);
3472 N1 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N1);
3473 N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
3474 N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
3475 DAG.getConstant(SimpleSize, DL,
3476 getShiftAmountTy(N1.getValueType())));
3477 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
3478 }
3479 }
3480
3481 return SDValue();
3482}
3483
3484/// Perform optimizations common to nodes that compute two values. LoOp and HiOp
3485/// give the opcodes for the two computations that are being performed. Return
3486/// true if a simplification was made.
3487SDValue DAGCombiner::SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
3488 unsigned HiOp) {
3489 // If the high half is not needed, just compute the low half.
3490 bool HiExists = N->hasAnyUseOfValue(1);
3491 if (!HiExists &&
3492 (!LegalOperations ||
3493 TLI.isOperationLegalOrCustom(LoOp, N->getValueType(0)))) {
3494 SDValue Res = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0), N->ops());
3495 return CombineTo(N, Res, Res);
3496 }
3497
3498 // If the low half is not needed, just compute the high half.
3499 bool LoExists = N->hasAnyUseOfValue(0);
3500 if (!LoExists &&
3501 (!LegalOperations ||
3502 TLI.isOperationLegal(HiOp, N->getValueType(1)))) {
3503 SDValue Res = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1), N->ops());
3504 return CombineTo(N, Res, Res);
3505 }
3506
3507 // If both halves are used, return as it is.
3508 if (LoExists && HiExists)
3509 return SDValue();
3510
3511 // If the two computed results can be simplified separately, separate them.
3512 if (LoExists) {
3513 SDValue Lo = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0), N->ops());
3514 AddToWorklist(Lo.getNode());
3515 SDValue LoOpt = combine(Lo.getNode());
3516 if (LoOpt.getNode() && LoOpt.getNode() != Lo.getNode() &&
3517 (!LegalOperations ||
3518 TLI.isOperationLegal(LoOpt.getOpcode(), LoOpt.getValueType())))
3519 return CombineTo(N, LoOpt, LoOpt);
3520 }
3521
3522 if (HiExists) {
3523 SDValue Hi = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1), N->ops());
3524 AddToWorklist(Hi.getNode());
3525 SDValue HiOpt = combine(Hi.getNode());
3526 if (HiOpt.getNode() && HiOpt != Hi &&
3527 (!LegalOperations ||
3528 TLI.isOperationLegal(HiOpt.getOpcode(), HiOpt.getValueType())))
3529 return CombineTo(N, HiOpt, HiOpt);
3530 }
3531
3532 return SDValue();
3533}
3534
3535SDValue DAGCombiner::visitSMUL_LOHI(SDNode *N) {
3536 if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHS))
3537 return Res;
3538
3539 EVT VT = N->getValueType(0);
3540 SDLoc DL(N);
3541
3542 // If the type is twice as wide is legal, transform the mulhu to a wider
3543 // multiply plus a shift.
3544 if (VT.isSimple() && !VT.isVector()) {
3545 MVT Simple = VT.getSimpleVT();
3546 unsigned SimpleSize = Simple.getSizeInBits();
3547 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
3548 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
3549 SDValue Lo = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(0));
3550 SDValue Hi = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(1));
3551 Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
3552 // Compute the high part as N1.
3553 Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
3554 DAG.getConstant(SimpleSize, DL,
3555 getShiftAmountTy(Lo.getValueType())));
3556 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
3557 // Compute the low part as N0.
3558 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
3559 return CombineTo(N, Lo, Hi);
3560 }
3561 }
3562
3563 return SDValue();
3564}
3565
3566SDValue DAGCombiner::visitUMUL_LOHI(SDNode *N) {
3567 if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHU))
3568 return Res;
3569
3570 EVT VT = N->getValueType(0);
3571 SDLoc DL(N);
3572
3573 // If the type is twice as wide is legal, transform the mulhu to a wider
3574 // multiply plus a shift.
3575 if (VT.isSimple() && !VT.isVector()) {
3576 MVT Simple = VT.getSimpleVT();
3577 unsigned SimpleSize = Simple.getSizeInBits();
3578 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
3579 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
3580 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(0));
3581 SDValue Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(1));
3582 Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
3583 // Compute the high part as N1.
3584 Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
3585 DAG.getConstant(SimpleSize, DL,
3586 getShiftAmountTy(Lo.getValueType())));
3587 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
3588 // Compute the low part as N0.
3589 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
3590 return CombineTo(N, Lo, Hi);
3591 }
3592 }
3593
3594 return SDValue();
3595}
3596
3597SDValue DAGCombiner::visitSMULO(SDNode *N) {
3598 // (smulo x, 2) -> (saddo x, x)
3599 if (ConstantSDNode *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1)))
3600 if (C2->getAPIntValue() == 2)
3601 return DAG.getNode(ISD::SADDO, SDLoc(N), N->getVTList(),
3602 N->getOperand(0), N->getOperand(0));
3603
3604 return SDValue();
3605}
3606
3607SDValue DAGCombiner::visitUMULO(SDNode *N) {
3608 // (umulo x, 2) -> (uaddo x, x)
3609 if (ConstantSDNode *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1)))
3610 if (C2->getAPIntValue() == 2)
3611 return DAG.getNode(ISD::UADDO, SDLoc(N), N->getVTList(),
3612 N->getOperand(0), N->getOperand(0));
3613
3614 return SDValue();
3615}
3616
3617SDValue DAGCombiner::visitIMINMAX(SDNode *N) {
3618 SDValue N0 = N->getOperand(0);
3619 SDValue N1 = N->getOperand(1);
3620 EVT VT = N0.getValueType();
3621
3622 // fold vector ops
3623 if (VT.isVector())
3624 if (SDValue FoldedVOp = SimplifyVBinOp(N))
3625 return FoldedVOp;
3626
3627 // fold operation with constant operands.
3628 ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
3629 ConstantSDNode *N1C = getAsNonOpaqueConstant(N1);
3630 if (N0C && N1C)
3631 return DAG.FoldConstantArithmetic(N->getOpcode(), SDLoc(N), VT, N0C, N1C);
3632
3633 // canonicalize constant to RHS
3634 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
3635 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
3636 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N1, N0);
3637
3638 // Is sign bits are zero, flip between UMIN/UMAX and SMIN/SMAX.
3639 // Only do this if the current op isn't legal and the flipped is.
3640 unsigned Opcode = N->getOpcode();
3641 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3642 if (!TLI.isOperationLegal(Opcode, VT) &&
3643 (N0.isUndef() || DAG.SignBitIsZero(N0)) &&
3644 (N1.isUndef() || DAG.SignBitIsZero(N1))) {
3645 unsigned AltOpcode;
3646 switch (Opcode) {
3647 case ISD::SMIN: AltOpcode = ISD::UMIN; break;
3648 case ISD::SMAX: AltOpcode = ISD::UMAX; break;
3649 case ISD::UMIN: AltOpcode = ISD::SMIN; break;
3650 case ISD::UMAX: AltOpcode = ISD::SMAX; break;
3651 default: llvm_unreachable("Unknown MINMAX opcode")::llvm::llvm_unreachable_internal("Unknown MINMAX opcode", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3651)
;
3652 }
3653 if (TLI.isOperationLegal(AltOpcode, VT))
3654 return DAG.getNode(AltOpcode, SDLoc(N), VT, N0, N1);
3655 }
3656
3657 return SDValue();
3658}
3659
3660/// If this is a binary operator with two operands of the same opcode, try to
3661/// simplify it.
3662SDValue DAGCombiner::SimplifyBinOpWithSameOpcodeHands(SDNode *N) {
3663 SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
3664 EVT VT = N0.getValueType();
3665 assert(N0.getOpcode() == N1.getOpcode() && "Bad input!")(static_cast <bool> (N0.getOpcode() == N1.getOpcode() &&
"Bad input!") ? void (0) : __assert_fail ("N0.getOpcode() == N1.getOpcode() && \"Bad input!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3665, __extension__ __PRETTY_FUNCTION__))
;
3666
3667 // Bail early if none of these transforms apply.
3668 if (N0.getNumOperands() == 0) return SDValue();
3669
3670 // For each of OP in AND/OR/XOR:
3671 // fold (OP (zext x), (zext y)) -> (zext (OP x, y))
3672 // fold (OP (sext x), (sext y)) -> (sext (OP x, y))
3673 // fold (OP (aext x), (aext y)) -> (aext (OP x, y))
3674 // fold (OP (bswap x), (bswap y)) -> (bswap (OP x, y))
3675 // fold (OP (trunc x), (trunc y)) -> (trunc (OP x, y)) (if trunc isn't free)
3676 //
3677 // do not sink logical op inside of a vector extend, since it may combine
3678 // into a vsetcc.
3679 EVT Op0VT = N0.getOperand(0).getValueType();
3680 if ((N0.getOpcode() == ISD::ZERO_EXTEND ||
3681 N0.getOpcode() == ISD::SIGN_EXTEND ||
3682 N0.getOpcode() == ISD::BSWAP ||
3683 // Avoid infinite looping with PromoteIntBinOp.
3684 (N0.getOpcode() == ISD::ANY_EXTEND &&
3685 (!LegalTypes || TLI.isTypeDesirableForOp(N->getOpcode(), Op0VT))) ||
3686 (N0.getOpcode() == ISD::TRUNCATE &&
3687 (!TLI.isZExtFree(VT, Op0VT) ||
3688 !TLI.isTruncateFree(Op0VT, VT)) &&
3689 TLI.isTypeLegal(Op0VT))) &&
3690 !VT.isVector() &&
3691 Op0VT == N1.getOperand(0).getValueType() &&
3692 (!LegalOperations || TLI.isOperationLegal(N->getOpcode(), Op0VT))) {
3693 SDValue ORNode = DAG.getNode(N->getOpcode(), SDLoc(N0),
3694 N0.getOperand(0).getValueType(),
3695 N0.getOperand(0), N1.getOperand(0));
3696 AddToWorklist(ORNode.getNode());
3697 return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, ORNode);
3698 }
3699
3700 // For each of OP in SHL/SRL/SRA/AND...
3701 // fold (and (OP x, z), (OP y, z)) -> (OP (and x, y), z)
3702 // fold (or (OP x, z), (OP y, z)) -> (OP (or x, y), z)
3703 // fold (xor (OP x, z), (OP y, z)) -> (OP (xor x, y), z)
3704 if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL ||
3705 N0.getOpcode() == ISD::SRA || N0.getOpcode() == ISD::AND) &&
3706 N0.getOperand(1) == N1.getOperand(1)) {
3707 SDValue ORNode = DAG.getNode(N->getOpcode(), SDLoc(N0),
3708 N0.getOperand(0).getValueType(),
3709 N0.getOperand(0), N1.getOperand(0));
3710 AddToWorklist(ORNode.getNode());
3711 return DAG.getNode(N0.getOpcode(), SDLoc(N), VT,
3712 ORNode, N0.getOperand(1));
3713 }
3714
3715 // Simplify xor/and/or (bitcast(A), bitcast(B)) -> bitcast(op (A,B))
3716 // Only perform this optimization up until type legalization, before
3717 // LegalizeVectorOprs. LegalizeVectorOprs promotes vector operations by
3718 // adding bitcasts. For example (xor v4i32) is promoted to (v2i64), and
3719 // we don't want to undo this promotion.
3720 // We also handle SCALAR_TO_VECTOR because xor/or/and operations are cheaper
3721 // on scalars.
3722 if ((N0.getOpcode() == ISD::BITCAST ||
3723 N0.getOpcode() == ISD::SCALAR_TO_VECTOR) &&
3724 Level <= AfterLegalizeTypes) {
3725 SDValue In0 = N0.getOperand(0);
3726 SDValue In1 = N1.getOperand(0);
3727 EVT In0Ty = In0.getValueType();
3728 EVT In1Ty = In1.getValueType();
3729 SDLoc DL(N);
3730 // If both incoming values are integers, and the original types are the
3731 // same.
3732 if (In0Ty.isInteger() && In1Ty.isInteger() && In0Ty == In1Ty) {
3733 SDValue Op = DAG.getNode(N->getOpcode(), DL, In0Ty, In0, In1);
3734 SDValue BC = DAG.getNode(N0.getOpcode(), DL, VT, Op);
3735 AddToWorklist(Op.getNode());
3736 return BC;
3737 }
3738 }
3739
3740 // Xor/and/or are indifferent to the swizzle operation (shuffle of one value).
3741 // Simplify xor/and/or (shuff(A), shuff(B)) -> shuff(op (A,B))
3742 // If both shuffles use the same mask, and both shuffle within a single
3743 // vector, then it is worthwhile to move the swizzle after the operation.
3744 // The type-legalizer generates this pattern when loading illegal
3745 // vector types from memory. In many cases this allows additional shuffle
3746 // optimizations.
3747 // There are other cases where moving the shuffle after the xor/and/or
3748 // is profitable even if shuffles don't perform a swizzle.
3749 // If both shuffles use the same mask, and both shuffles have the same first
3750 // or second operand, then it might still be profitable to move the shuffle
3751 // after the xor/and/or operation.
3752 if (N0.getOpcode() == ISD::VECTOR_SHUFFLE && Level < AfterLegalizeDAG) {
3753 ShuffleVectorSDNode *SVN0 = cast<ShuffleVectorSDNode>(N0);
3754 ShuffleVectorSDNode *SVN1 = cast<ShuffleVectorSDNode>(N1);
3755
3756 assert(N0.getOperand(0).getValueType() == N1.getOperand(0).getValueType() &&(static_cast <bool> (N0.getOperand(0).getValueType() ==
N1.getOperand(0).getValueType() && "Inputs to shuffles are not the same type"
) ? void (0) : __assert_fail ("N0.getOperand(0).getValueType() == N1.getOperand(0).getValueType() && \"Inputs to shuffles are not the same type\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3757, __extension__ __PRETTY_FUNCTION__))
3757 "Inputs to shuffles are not the same type")(static_cast <bool> (N0.getOperand(0).getValueType() ==
N1.getOperand(0).getValueType() && "Inputs to shuffles are not the same type"
) ? void (0) : __assert_fail ("N0.getOperand(0).getValueType() == N1.getOperand(0).getValueType() && \"Inputs to shuffles are not the same type\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3757, __extension__ __PRETTY_FUNCTION__))
;
3758
3759 // Check that both shuffles use the same mask. The masks are known to be of
3760 // the same length because the result vector type is the same.
3761 // Check also that shuffles have only one use to avoid introducing extra
3762 // instructions.
3763 if (SVN0->hasOneUse() && SVN1->hasOneUse() &&
3764 SVN0->getMask().equals(SVN1->getMask())) {
3765 SDValue ShOp = N0->getOperand(1);
3766
3767 // Don't try to fold this node if it requires introducing a
3768 // build vector of all zeros that might be illegal at this stage.
3769 if (N->getOpcode() == ISD::XOR && !ShOp.isUndef()) {
3770 if (!LegalTypes)
3771 ShOp = DAG.getConstant(0, SDLoc(N), VT);
3772 else
3773 ShOp = SDValue();
3774 }
3775
3776 // (AND (shuf (A, C), shuf (B, C))) -> shuf (AND (A, B), C)
3777 // (OR (shuf (A, C), shuf (B, C))) -> shuf (OR (A, B), C)
3778 // (XOR (shuf (A, C), shuf (B, C))) -> shuf (XOR (A, B), V_0)
3779 if (N0.getOperand(1) == N1.getOperand(1) && ShOp.getNode()) {
3780 SDValue NewNode = DAG.getNode(N->getOpcode(), SDLoc(N), VT,
3781 N0->getOperand(0), N1->getOperand(0));
3782 AddToWorklist(NewNode.getNode());
3783 return DAG.getVectorShuffle(VT, SDLoc(N), NewNode, ShOp,
3784 SVN0->getMask());
3785 }
3786
3787 // Don't try to fold this node if it requires introducing a
3788 // build vector of all zeros that might be illegal at this stage.
3789 ShOp = N0->getOperand(0);
3790 if (N->getOpcode() == ISD::XOR && !ShOp.isUndef()) {
3791 if (!LegalTypes)
3792 ShOp = DAG.getConstant(0, SDLoc(N), VT);
3793 else
3794 ShOp = SDValue();
3795 }
3796
3797 // (AND (shuf (C, A), shuf (C, B))) -> shuf (C, AND (A, B))
3798 // (OR (shuf (C, A), shuf (C, B))) -> shuf (C, OR (A, B))
3799 // (XOR (shuf (C, A), shuf (C, B))) -> shuf (V_0, XOR (A, B))
3800 if (N0->getOperand(0) == N1->getOperand(0) && ShOp.getNode()) {
3801 SDValue NewNode = DAG.getNode(N->getOpcode(), SDLoc(N), VT,
3802 N0->getOperand(1), N1->getOperand(1));
3803 AddToWorklist(NewNode.getNode());
3804 return DAG.getVectorShuffle(VT, SDLoc(N), ShOp, NewNode,
3805 SVN0->getMask());
3806 }
3807 }
3808 }
3809
3810 return SDValue();
3811}
3812
3813/// Try to make (and/or setcc (LL, LR), setcc (RL, RR)) more efficient.
3814SDValue DAGCombiner::foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1,
3815 const SDLoc &DL) {
3816 SDValue LL, LR, RL, RR, N0CC, N1CC;
3817 if (!isSetCCEquivalent(N0, LL, LR, N0CC) ||
3818 !isSetCCEquivalent(N1, RL, RR, N1CC))
3819 return SDValue();
3820
3821 assert(N0.getValueType() == N1.getValueType() &&(static_cast <bool> (N0.getValueType() == N1.getValueType
() && "Unexpected operand types for bitwise logic op"
) ? void (0) : __assert_fail ("N0.getValueType() == N1.getValueType() && \"Unexpected operand types for bitwise logic op\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3822, __extension__ __PRETTY_FUNCTION__))
3822 "Unexpected operand types for bitwise logic op")(static_cast <bool> (N0.getValueType() == N1.getValueType
() && "Unexpected operand types for bitwise logic op"
) ? void (0) : __assert_fail ("N0.getValueType() == N1.getValueType() && \"Unexpected operand types for bitwise logic op\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3822, __extension__ __PRETTY_FUNCTION__))
;
3823 assert(LL.getValueType() == LR.getValueType() &&(static_cast <bool> (LL.getValueType() == LR.getValueType
() && RL.getValueType() == RR.getValueType() &&
"Unexpected operand types for setcc") ? void (0) : __assert_fail
("LL.getValueType() == LR.getValueType() && RL.getValueType() == RR.getValueType() && \"Unexpected operand types for setcc\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3825, __extension__ __PRETTY_FUNCTION__))
3824 RL.getValueType() == RR.getValueType() &&(static_cast <bool> (LL.getValueType() == LR.getValueType
() && RL.getValueType() == RR.getValueType() &&
"Unexpected operand types for setcc") ? void (0) : __assert_fail
("LL.getValueType() == LR.getValueType() && RL.getValueType() == RR.getValueType() && \"Unexpected operand types for setcc\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3825, __extension__ __PRETTY_FUNCTION__))
3825 "Unexpected operand types for setcc")(static_cast <bool> (LL.getValueType() == LR.getValueType
() && RL.getValueType() == RR.getValueType() &&
"Unexpected operand types for setcc") ? void (0) : __assert_fail
("LL.getValueType() == LR.getValueType() && RL.getValueType() == RR.getValueType() && \"Unexpected operand types for setcc\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3825, __extension__ __PRETTY_FUNCTION__))
;
3826
3827 // If we're here post-legalization or the logic op type is not i1, the logic
3828 // op type must match a setcc result type. Also, all folds require new
3829 // operations on the left and right operands, so those types must match.
3830 EVT VT = N0.getValueType();
3831 EVT OpVT = LL.getValueType();
3832 if (LegalOperations || VT.getScalarType() != MVT::i1)
3833 if (VT != getSetCCResultType(OpVT))
3834 return SDValue();
3835 if (OpVT != RL.getValueType())
3836 return SDValue();
3837
3838 ISD::CondCode CC0 = cast<CondCodeSDNode>(N0CC)->get();
3839 ISD::CondCode CC1 = cast<CondCodeSDNode>(N1CC)->get();
3840 bool IsInteger = OpVT.isInteger();
3841 if (LR == RR && CC0 == CC1 && IsInteger) {
3842 bool IsZero = isNullConstantOrNullSplatConstant(LR);
3843 bool IsNeg1 = isAllOnesConstantOrAllOnesSplatConstant(LR);
3844
3845 // All bits clear?
3846 bool AndEqZero = IsAnd && CC1 == ISD::SETEQ && IsZero;
3847 // All sign bits clear?
3848 bool AndGtNeg1 = IsAnd && CC1 == ISD::SETGT && IsNeg1;
3849 // Any bits set?
3850 bool OrNeZero = !IsAnd && CC1 == ISD::SETNE && IsZero;
3851 // Any sign bits set?
3852 bool OrLtZero = !IsAnd && CC1 == ISD::SETLT && IsZero;
3853
3854 // (and (seteq X, 0), (seteq Y, 0)) --> (seteq (or X, Y), 0)
3855 // (and (setgt X, -1), (setgt Y, -1)) --> (setgt (or X, Y), -1)
3856 // (or (setne X, 0), (setne Y, 0)) --> (setne (or X, Y), 0)
3857 // (or (setlt X, 0), (setlt Y, 0)) --> (setlt (or X, Y), 0)
3858 if (AndEqZero || AndGtNeg1 || OrNeZero || OrLtZero) {
3859 SDValue Or = DAG.getNode(ISD::OR, SDLoc(N0), OpVT, LL, RL);
3860 AddToWorklist(Or.getNode());
3861 return DAG.getSetCC(DL, VT, Or, LR, CC1);
3862 }
3863
3864 // All bits set?
3865 bool AndEqNeg1 = IsAnd && CC1 == ISD::SETEQ && IsNeg1;
3866 // All sign bits set?
3867 bool AndLtZero = IsAnd && CC1 == ISD::SETLT && IsZero;
3868 // Any bits clear?
3869 bool OrNeNeg1 = !IsAnd && CC1 == ISD::SETNE && IsNeg1;
3870 // Any sign bits clear?
3871 bool OrGtNeg1 = !IsAnd && CC1 == ISD::SETGT && IsNeg1;
3872
3873 // (and (seteq X, -1), (seteq Y, -1)) --> (seteq (and X, Y), -1)
3874 // (and (setlt X, 0), (setlt Y, 0)) --> (setlt (and X, Y), 0)
3875 // (or (setne X, -1), (setne Y, -1)) --> (setne (and X, Y), -1)
3876 // (or (setgt X, -1), (setgt Y -1)) --> (setgt (and X, Y), -1)
3877 if (AndEqNeg1 || AndLtZero || OrNeNeg1 || OrGtNeg1) {
3878 SDValue And = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, LL, RL);
3879 AddToWorklist(And.getNode());
3880 return DAG.getSetCC(DL, VT, And, LR, CC1);
3881 }
3882 }
3883
3884 // TODO: What is the 'or' equivalent of this fold?
3885 // (and (setne X, 0), (setne X, -1)) --> (setuge (add X, 1), 2)
3886 if (IsAnd && LL == RL && CC0 == CC1 && OpVT.getScalarSizeInBits() > 1 &&
3887 IsInteger && CC0 == ISD::SETNE &&
3888 ((isNullConstant(LR) && isAllOnesConstant(RR)) ||
3889 (isAllOnesConstant(LR) && isNullConstant(RR)))) {
3890 SDValue One = DAG.getConstant(1, DL, OpVT);
3891 SDValue Two = DAG.getConstant(2, DL, OpVT);
3892 SDValue Add = DAG.getNode(ISD::ADD, SDLoc(N0), OpVT, LL, One);
3893 AddToWorklist(Add.getNode());
3894 return DAG.getSetCC(DL, VT, Add, Two, ISD::SETUGE);
3895 }
3896
3897 // Try more general transforms if the predicates match and the only user of
3898 // the compares is the 'and' or 'or'.
3899 if (IsInteger && TLI.convertSetCCLogicToBitwiseLogic(OpVT) && CC0 == CC1 &&
3900 N0.hasOneUse() && N1.hasOneUse()) {
3901 // and (seteq A, B), (seteq C, D) --> seteq (or (xor A, B), (xor C, D)), 0
3902 // or (setne A, B), (setne C, D) --> setne (or (xor A, B), (xor C, D)), 0
3903 if ((IsAnd && CC1 == ISD::SETEQ) || (!IsAnd && CC1 == ISD::SETNE)) {
3904 SDValue XorL = DAG.getNode(ISD::XOR, SDLoc(N0), OpVT, LL, LR);
3905 SDValue XorR = DAG.getNode(ISD::XOR, SDLoc(N1), OpVT, RL, RR);
3906 SDValue Or = DAG.getNode(ISD::OR, DL, OpVT, XorL, XorR);
3907 SDValue Zero = DAG.getConstant(0, DL, OpVT);
3908 return DAG.getSetCC(DL, VT, Or, Zero, CC1);
3909 }
3910 }
3911
3912 // Canonicalize equivalent operands to LL == RL.
3913 if (LL == RR && LR == RL) {
3914 CC1 = ISD::getSetCCSwappedOperands(CC1);
3915 std::swap(RL, RR);
3916 }
3917
3918 // (and (setcc X, Y, CC0), (setcc X, Y, CC1)) --> (setcc X, Y, NewCC)
3919 // (or (setcc X, Y, CC0), (setcc X, Y, CC1)) --> (setcc X, Y, NewCC)
3920 if (LL == RL && LR == RR) {
3921 ISD::CondCode NewCC = IsAnd ? ISD::getSetCCAndOperation(CC0, CC1, IsInteger)
3922 : ISD::getSetCCOrOperation(CC0, CC1, IsInteger);
3923 if (NewCC != ISD::SETCC_INVALID &&
3924 (!LegalOperations ||
3925 (TLI.isCondCodeLegal(NewCC, LL.getSimpleValueType()) &&
3926 TLI.isOperationLegal(ISD::SETCC, OpVT))))
3927 return DAG.getSetCC(DL, VT, LL, LR, NewCC);
3928 }
3929
3930 return SDValue();
3931}
3932
3933/// This contains all DAGCombine rules which reduce two values combined by
3934/// an And operation to a single value. This makes them reusable in the context
3935/// of visitSELECT(). Rules involving constants are not included as
3936/// visitSELECT() already handles those cases.
3937SDValue DAGCombiner::visitANDLike(SDValue N0, SDValue N1, SDNode *N) {
3938 EVT VT = N1.getValueType();
3939 SDLoc DL(N);
3940
3941 // fold (and x, undef) -> 0
3942 if (N0.isUndef() || N1.isUndef())
3943 return DAG.getConstant(0, DL, VT);
3944
3945 if (SDValue V = foldLogicOfSetCCs(true, N0, N1, DL))
3946 return V;
3947
3948 if (N0.getOpcode() == ISD::ADD && N1.getOpcode() == ISD::SRL &&
3949 VT.getSizeInBits() <= 64) {
3950 if (ConstantSDNode *ADDI = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
3951 if (ConstantSDNode *SRLI = dyn_cast<ConstantSDNode>(N1.getOperand(1))) {
3952 // Look for (and (add x, c1), (lshr y, c2)). If C1 wasn't a legal
3953 // immediate for an add, but it is legal if its top c2 bits are set,
3954 // transform the ADD so the immediate doesn't need to be materialized
3955 // in a register.
3956 APInt ADDC = ADDI->getAPIntValue();
3957 APInt SRLC = SRLI->getAPIntValue();
3958 if (ADDC.getMinSignedBits() <= 64 &&
3959 SRLC.ult(VT.getSizeInBits()) &&
3960 !TLI.isLegalAddImmediate(ADDC.getSExtValue())) {
3961 APInt Mask = APInt::getHighBitsSet(VT.getSizeInBits(),
3962 SRLC.getZExtValue());
3963 if (DAG.MaskedValueIsZero(N0.getOperand(1), Mask)) {
3964 ADDC |= Mask;
3965 if (TLI.isLegalAddImmediate(ADDC.getSExtValue())) {
3966 SDLoc DL0(N0);
3967 SDValue NewAdd =
3968 DAG.getNode(ISD::ADD, DL0, VT,
3969 N0.getOperand(0), DAG.getConstant(ADDC, DL, VT));
3970 CombineTo(N0.getNode(), NewAdd);
3971 // Return N so it doesn't get rechecked!
3972 return SDValue(N, 0);
3973 }
3974 }
3975 }
3976 }
3977 }
3978 }
3979
3980 // Reduce bit extract of low half of an integer to the narrower type.
3981 // (and (srl i64:x, K), KMask) ->
3982 // (i64 zero_extend (and (srl (i32 (trunc i64:x)), K)), KMask)
3983 if (N0.getOpcode() == ISD::SRL && N0.hasOneUse()) {
3984 if (ConstantSDNode *CAnd = dyn_cast<ConstantSDNode>(N1)) {
3985 if (ConstantSDNode *CShift = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
3986 unsigned Size = VT.getSizeInBits();
3987 const APInt &AndMask = CAnd->getAPIntValue();
3988 unsigned ShiftBits = CShift->getZExtValue();
3989
3990 // Bail out, this node will probably disappear anyway.
3991 if (ShiftBits == 0)
3992 return SDValue();
3993
3994 unsigned MaskBits = AndMask.countTrailingOnes();
3995 EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), Size / 2);
3996
3997 if (AndMask.isMask() &&
3998 // Required bits must not span the two halves of the integer and
3999 // must fit in the half size type.
4000 (ShiftBits + MaskBits <= Size / 2) &&
4001 TLI.isNarrowingProfitable(VT, HalfVT) &&
4002 TLI.isTypeDesirableForOp(ISD::AND, HalfVT) &&
4003 TLI.isTypeDesirableForOp(ISD::SRL, HalfVT) &&
4004 TLI.isTruncateFree(VT, HalfVT) &&
4005 TLI.isZExtFree(HalfVT, VT)) {
4006 // The isNarrowingProfitable is to avoid regressions on PPC and
4007 // AArch64 which match a few 64-bit bit insert / bit extract patterns
4008 // on downstream users of this. Those patterns could probably be
4009 // extended to handle extensions mixed in.
4010
4011 SDValue SL(N0);
4012 assert(MaskBits <= Size)(static_cast <bool> (MaskBits <= Size) ? void (0) : __assert_fail
("MaskBits <= Size", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4012, __extension__ __PRETTY_FUNCTION__))
;
4013
4014 // Extracting the highest bit of the low half.
4015 EVT ShiftVT = TLI.getShiftAmountTy(HalfVT, DAG.getDataLayout());
4016 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, HalfVT,
4017 N0.getOperand(0));
4018
4019 SDValue NewMask = DAG.getConstant(AndMask.trunc(Size / 2), SL, HalfVT);
4020 SDValue ShiftK = DAG.getConstant(ShiftBits, SL, ShiftVT);
4021 SDValue Shift = DAG.getNode(ISD::SRL, SL, HalfVT, Trunc, ShiftK);
4022 SDValue And = DAG.getNode(ISD::AND, SL, HalfVT, Shift, NewMask);
4023 return DAG.getNode(ISD::ZERO_EXTEND, SL, VT, And);
4024 }
4025 }
4026 }
4027 }
4028
4029 return SDValue();
4030}
4031
4032bool DAGCombiner::isAndLoadExtLoad(ConstantSDNode *AndC, LoadSDNode *LoadN,
4033 EVT LoadResultTy, EVT &ExtVT) {
4034 if (!AndC->getAPIntValue().isMask())
4035 return false;
4036
4037 unsigned ActiveBits = AndC->getAPIntValue().countTrailingOnes();
4038
4039 ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
4040 EVT LoadedVT = LoadN->getMemoryVT();
4041
4042 if (ExtVT == LoadedVT &&
4043 (!LegalOperations ||
4044 TLI.isLoadExtLegal(ISD::ZEXTLOAD, LoadResultTy, ExtVT))) {
4045 // ZEXTLOAD will match without needing to change the size of the value being
4046 // loaded.
4047 return true;
4048 }
4049
4050 // Do not change the width of a volatile load.
4051 if (LoadN->isVolatile())
4052 return false;
4053
4054 // Do not generate loads of non-round integer types since these can
4055 // be expensive (and would be wrong if the type is not byte sized).
4056 if (!LoadedVT.bitsGT(ExtVT) || !ExtVT.isRound())
4057 return false;
4058
4059 if (LegalOperations &&
4060 !TLI.isLoadExtLegal(ISD::ZEXTLOAD, LoadResultTy, ExtVT))
4061 return false;
4062
4063 if (!TLI.shouldReduceLoadWidth(LoadN, ISD::ZEXTLOAD, ExtVT))
4064 return false;
4065
4066 return true;
4067}
4068
4069bool DAGCombiner::isLegalNarrowLdSt(LSBaseSDNode *LDST,
4070 ISD::LoadExtType ExtType, EVT &MemVT,
4071 unsigned ShAmt) {
4072 if (!LDST)
4073 return false;
4074 // Only allow byte offsets.
4075 if (ShAmt % 8)
4076 return false;
4077
4078 // Do not generate loads of non-round integer types since these can
4079 // be expensive (and would be wrong if the type is not byte sized).
4080 if (!MemVT.isRound())
4081 return false;
4082
4083 // Don't change the width of a volatile load.
4084 if (LDST->isVolatile())
4085 return false;
4086
4087 // Verify that we are actually reducing a load width here.
4088 if (LDST->getMemoryVT().getSizeInBits() < MemVT.getSizeInBits())
4089 return false;
4090
4091 // Ensure that this isn't going to produce an unsupported unaligned access.
4092 if (ShAmt &&
4093 !TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), MemVT,
4094 LDST->getAddressSpace(), ShAmt / 8))
4095 return false;
4096
4097 // It's not possible to generate a constant of extended or untyped type.
4098 EVT PtrType = LDST->getBasePtr().getValueType();
4099 if (PtrType == MVT::Untyped || PtrType.isExtended())
4100 return false;
4101
4102 if (isa<LoadSDNode>(LDST)) {
4103 LoadSDNode *Load = cast<LoadSDNode>(LDST);
4104 // Don't transform one with multiple uses, this would require adding a new
4105 // load.
4106 if (!SDValue(Load, 0).hasOneUse())
4107 return false;
4108
4109 if (LegalOperations &&
4110 !TLI.isLoadExtLegal(ExtType, Load->getValueType(0), MemVT))
4111 return false;
4112
4113 // For the transform to be legal, the load must produce only two values
4114 // (the value loaded and the chain). Don't transform a pre-increment
4115 // load, for example, which produces an extra value. Otherwise the
4116 // transformation is not equivalent, and the downstream logic to replace
4117 // uses gets things wrong.
4118 if (Load->getNumValues() > 2)
4119 return false;
4120
4121 // If the load that we're shrinking is an extload and we're not just
4122 // discarding the extension we can't simply shrink the load. Bail.
4123 // TODO: It would be possible to merge the extensions in some cases.
4124 if (Load->getExtensionType() != ISD::NON_EXTLOAD &&
4125 Load->getMemoryVT().getSizeInBits() < MemVT.getSizeInBits() + ShAmt)
4126 return false;
4127
4128 if (!TLI.shouldReduceLoadWidth(Load, ExtType, MemVT))
4129 return false;
4130 } else {
4131 assert(isa<StoreSDNode>(LDST) && "It is not a Load nor a Store SDNode")(static_cast <bool> (isa<StoreSDNode>(LDST) &&
"It is not a Load nor a Store SDNode") ? void (0) : __assert_fail
("isa<StoreSDNode>(LDST) && \"It is not a Load nor a Store SDNode\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4131, __extension__ __PRETTY_FUNCTION__))
;
4132 StoreSDNode *Store = cast<StoreSDNode>(LDST);
4133 // Can't write outside the original store
4134 if (Store->getMemoryVT().getSizeInBits() < MemVT.getSizeInBits() + ShAmt)
4135 return false;
4136
4137 if (LegalOperations &&
4138 !TLI.isTruncStoreLegal(Store->getValue().getValueType(), MemVT))
4139 return false;
4140 }
4141 return true;
4142}
4143
4144bool DAGCombiner::SearchForAndLoads(SDNode *N,
4145 SmallPtrSetImpl<LoadSDNode*> &Loads,
4146 SmallPtrSetImpl<SDNode*> &NodesWithConsts,
4147 ConstantSDNode *Mask,
4148 SDNode *&NodeToMask) {
4149 // Recursively search for the operands, looking for loads which can be
4150 // narrowed.
4151 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) {
4152 SDValue Op = N->getOperand(i);
4153
4154 if (Op.getValueType().isVector())
4155 return false;
4156
4157 // Some constants may need fixing up later if they are too large.
4158 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
4159 if ((N->getOpcode() == ISD::OR || N->getOpcode() == ISD::XOR) &&
4160 (Mask->getAPIntValue() & C->getAPIntValue()) != C->getAPIntValue())
4161 NodesWithConsts.insert(N);
4162 continue;
4163 }
4164
4165 if (!Op.hasOneUse())
4166 return false;
4167
4168 switch(Op.getOpcode()) {
4169 case ISD::LOAD: {
4170 auto *Load = cast<LoadSDNode>(Op);
4171 EVT ExtVT;
4172 if (isAndLoadExtLoad(Mask, Load, Load->getValueType(0), ExtVT) &&
4173 isLegalNarrowLdSt(Load, ISD::ZEXTLOAD, ExtVT)) {
4174
4175 // ZEXTLOAD is already small enough.
4176 if (Load->getExtensionType() == ISD::ZEXTLOAD &&
4177 ExtVT.bitsGE(Load->getMemoryVT()))
4178 continue;
4179
4180 // Use LE to convert equal sized loads to zext.
4181 if (ExtVT.bitsLE(Load->getMemoryVT()))
4182 Loads.insert(Load);
4183
4184 continue;
4185 }
4186 return false;
4187 }
4188 case ISD::ZERO_EXTEND:
4189 case ISD::AssertZext: {
4190 unsigned ActiveBits = Mask->getAPIntValue().countTrailingOnes();
4191 EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
4192 EVT VT = Op.getOpcode() == ISD::AssertZext ?
4193 cast<VTSDNode>(Op.getOperand(1))->getVT() :
4194 Op.getOperand(0).getValueType();
4195
4196 // We can accept extending nodes if the mask is wider or an equal
4197 // width to the original type.
4198 if (ExtVT.bitsGE(VT))
4199 continue;
4200 break;
4201 }
4202 case ISD::OR:
4203 case ISD::XOR:
4204 case ISD::AND:
4205 if (!SearchForAndLoads(Op.getNode(), Loads, NodesWithConsts, Mask,
4206 NodeToMask))
4207 return false;
4208 continue;
4209 }
4210
4211 // Allow one node which will masked along with any loads found.
4212 if (NodeToMask)
4213 return false;
4214
4215 // Also ensure that the node to be masked only produces one data result.
4216 NodeToMask = Op.getNode();
4217 if (NodeToMask->getNumValues() > 1) {
4218 bool HasValue = false;
4219 for (unsigned i = 0, e = NodeToMask->getNumValues(); i < e; ++i) {
4220 MVT VT = SDValue(NodeToMask, i).getSimpleValueType();
4221 if (VT != MVT::Glue && VT != MVT::Other) {
4222 if (HasValue) {
4223 NodeToMask = nullptr;
4224 return false;
4225 }
4226 HasValue = true;
4227 }
4228 }
4229 assert(HasValue && "Node to be masked has no data result?")(static_cast <bool> (HasValue && "Node to be masked has no data result?"
) ? void (0) : __assert_fail ("HasValue && \"Node to be masked has no data result?\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4229, __extension__ __PRETTY_FUNCTION__))
;
4230 }
4231 }
4232 return true;
4233}
4234
4235bool DAGCombiner::BackwardsPropagateMask(SDNode *N, SelectionDAG &DAG) {
4236 auto *Mask = dyn_cast<ConstantSDNode>(N->getOperand(1));
4237 if (!Mask)
4238 return false;
4239
4240 if (!Mask->getAPIntValue().isMask())
4241 return false;
4242
4243 // No need to do anything if the and directly uses a load.
4244 if (isa<LoadSDNode>(N->getOperand(0)))
4245 return false;
4246
4247 SmallPtrSet<LoadSDNode*, 8> Loads;
4248 SmallPtrSet<SDNode*, 2> NodesWithConsts;
4249 SDNode *FixupNode = nullptr;
4250 if (SearchForAndLoads(N, Loads, NodesWithConsts, Mask, FixupNode)) {
4251 if (Loads.size() == 0)
4252 return false;
4253
4254 LLVM_DEBUG(dbgs() << "Backwards propagate AND: "; N->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "Backwards propagate AND: "
; N->dump(); } } while (false)
;
4255 SDValue MaskOp = N->getOperand(1);
4256
4257 // If it exists, fixup the single node we allow in the tree that needs
4258 // masking.
4259 if (FixupNode) {
4260 LLVM_DEBUG(dbgs() << "First, need to fix up: "; FixupNode->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "First, need to fix up: "; FixupNode
->dump(); } } while (false)
;
4261 SDValue And = DAG.getNode(ISD::AND, SDLoc(FixupNode),
4262 FixupNode->getValueType(0),
4263 SDValue(FixupNode, 0), MaskOp);
4264 DAG.ReplaceAllUsesOfValueWith(SDValue(FixupNode, 0), And);
4265 if (And.getOpcode() == ISD ::AND)
4266 DAG.UpdateNodeOperands(And.getNode(), SDValue(FixupNode, 0), MaskOp);
4267 }
4268
4269 // Narrow any constants that need it.
4270 for (auto *LogicN : NodesWithConsts) {
4271 SDValue Op0 = LogicN->getOperand(0);
4272 SDValue Op1 = LogicN->getOperand(1);
4273
4274 if (isa<ConstantSDNode>(Op0))
4275 std::swap(Op0, Op1);
4276
4277 SDValue And = DAG.getNode(ISD::AND, SDLoc(Op1), Op1.getValueType(),
4278 Op1, MaskOp);
4279
4280 DAG.UpdateNodeOperands(LogicN, Op0, And);
4281 }
4282
4283 // Create narrow loads.
4284 for (auto *Load : Loads) {
4285 LLVM_DEBUG(dbgs() << "Propagate AND back to: "; Load->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "Propagate AND back to: "; Load
->dump(); } } while (false)
;
4286 SDValue And = DAG.getNode(ISD::AND, SDLoc(Load), Load->getValueType(0),
4287 SDValue(Load, 0), MaskOp);
4288 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), And);
4289 if (And.getOpcode() == ISD ::AND)
4290 And = SDValue(
4291 DAG.UpdateNodeOperands(And.getNode(), SDValue(Load, 0), MaskOp), 0);
4292 SDValue NewLoad = ReduceLoadWidth(And.getNode());
4293 assert(NewLoad &&(static_cast <bool> (NewLoad && "Shouldn't be masking the load if it can't be narrowed"
) ? void (0) : __assert_fail ("NewLoad && \"Shouldn't be masking the load if it can't be narrowed\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4294, __extension__ __PRETTY_FUNCTION__))
4294 "Shouldn't be masking the load if it can't be narrowed")(static_cast <bool> (NewLoad && "Shouldn't be masking the load if it can't be narrowed"
) ? void (0) : __assert_fail ("NewLoad && \"Shouldn't be masking the load if it can't be narrowed\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4294, __extension__ __PRETTY_FUNCTION__))
;
4295 CombineTo(Load, NewLoad, NewLoad.getValue(1));
4296 }
4297 DAG.ReplaceAllUsesWith(N, N->getOperand(0).getNode());
4298 return true;
4299 }
4300 return false;
4301}
4302
4303// Unfold
4304// x & (-1 'logical shift' y)
4305// To
4306// (x 'opposite logical shift' y) 'logical shift' y
4307// if it is better for performance.
4308SDValue DAGCombiner::unfoldExtremeBitClearingToShifts(SDNode *N) {
4309 assert(N->getOpcode() == ISD::AND)(static_cast <bool> (N->getOpcode() == ISD::AND) ? void
(0) : __assert_fail ("N->getOpcode() == ISD::AND", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4309, __extension__ __PRETTY_FUNCTION__))
;
4310
4311 SDValue N0 = N->getOperand(0);
4312 SDValue N1 = N->getOperand(1);
4313
4314 // Do we actually prefer shifts over mask?
4315 if (!TLI.preferShiftsToClearExtremeBits(N0))
4316 return SDValue();
4317
4318 // Try to match (-1 '[outer] logical shift' y)
4319 unsigned OuterShift;
4320 unsigned InnerShift; // The opposite direction to the OuterShift.
4321 SDValue Y; // Shift amount.
4322 auto matchMask = [&OuterShift, &InnerShift, &Y](SDValue M) -> bool {
4323 if (!M.hasOneUse())
4324 return false;
4325 OuterShift = M->getOpcode();
4326 if (OuterShift == ISD::SHL)
4327 InnerShift = ISD::SRL;
4328 else if (OuterShift == ISD::SRL)
4329 InnerShift = ISD::SHL;
4330 else
4331 return false;
4332 if (!isAllOnesConstant(M->getOperand(0)))
4333 return false;
4334 Y = M->getOperand(1);
4335 return true;
4336 };
4337
4338 SDValue X;
4339 if (matchMask(N1))
4340 X = N0;
4341 else if (matchMask(N0))
4342 X = N1;
4343 else
4344 return SDValue();
4345
4346 SDLoc DL(N);
4347 EVT VT = N->getValueType(0);
4348
4349 // tmp = x 'opposite logical shift' y
4350 SDValue T0 = DAG.getNode(InnerShift, DL, VT, X, Y);
4351 // ret = tmp 'logical shift' y
4352 SDValue T1 = DAG.getNode(OuterShift, DL, VT, T0, Y);
4353
4354 return T1;
4355}
4356
4357SDValue DAGCombiner::visitAND(SDNode *N) {
4358 SDValue N0 = N->getOperand(0);
4359 SDValue N1 = N->getOperand(1);
4360 EVT VT = N1.getValueType();
4361
4362 // x & x --> x
4363 if (N0 == N1)
4364 return N0;
4365
4366 // fold vector ops
4367 if (VT.isVector()) {
4368 if (SDValue FoldedVOp = SimplifyVBinOp(N))
4369 return FoldedVOp;
4370
4371 // fold (and x, 0) -> 0, vector edition
4372 if (ISD::isBuildVectorAllZeros(N0.getNode()))
4373 // do not return N0, because undef node may exist in N0
4374 return DAG.getConstant(APInt::getNullValue(N0.getScalarValueSizeInBits()),
4375 SDLoc(N), N0.getValueType());
4376 if (ISD::isBuildVectorAllZeros(N1.getNode()))
4377 // do not return N1, because undef node may exist in N1
4378 return DAG.getConstant(APInt::getNullValue(N1.getScalarValueSizeInBits()),
4379 SDLoc(N), N1.getValueType());
4380
4381 // fold (and x, -1) -> x, vector edition
4382 if (ISD::isBuildVectorAllOnes(N0.getNode()))
4383 return N1;
4384 if (ISD::isBuildVectorAllOnes(N1.getNode()))
4385 return N0;
4386 }
4387
4388 // fold (and c1, c2) -> c1&c2
4389 ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
4390 ConstantSDNode *N1C = isConstOrConstSplat(N1);
4391 if (N0C && N1C && !N1C->isOpaque())
4392 return DAG.FoldConstantArithmetic(ISD::AND, SDLoc(N), VT, N0C, N1C);
4393 // canonicalize constant to RHS
4394 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
4395 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
4396 return DAG.getNode(ISD::AND, SDLoc(N), VT, N1, N0);
4397 // fold (and x, -1) -> x
4398 if (isAllOnesConstant(N1))
4399 return N0;
4400 // if (and x, c) is known to be zero, return 0
4401 unsigned BitWidth = VT.getScalarSizeInBits();
4402 if (N1C && DAG.MaskedValueIsZero(SDValue(N, 0),
4403 APInt::getAllOnesValue(BitWidth)))
4404 return DAG.getConstant(0, SDLoc(N), VT);
4405
4406 if (SDValue NewSel = foldBinOpIntoSelect(N))
4407 return NewSel;
4408
4409 // reassociate and
4410 if (SDValue RAND = ReassociateOps(ISD::AND, SDLoc(N), N0, N1))
4411 return RAND;
4412
4413 // Try to convert a constant mask AND into a shuffle clear mask.
4414 if (VT.isVector())
4415 if (SDValue Shuffle = XformToShuffleWithZero(N))
4416 return Shuffle;
4417
4418 // fold (and (or x, C), D) -> D if (C & D) == D
4419 auto MatchSubset = [](ConstantSDNode *LHS, ConstantSDNode *RHS) {
4420 return RHS->getAPIntValue().isSubsetOf(LHS->getAPIntValue());
4421 };
4422 if (N0.getOpcode() == ISD::OR &&
4423 ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchSubset))
4424 return N1;
4425 // fold (and (any_ext V), c) -> (zero_ext V) if 'and' only clears top bits.
4426 if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) {
4427 SDValue N0Op0 = N0.getOperand(0);
4428 APInt Mask = ~N1C->getAPIntValue();
4429 Mask = Mask.trunc(N0Op0.getScalarValueSizeInBits());
4430 if (DAG.MaskedValueIsZero(N0Op0, Mask)) {
4431 SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N),
4432 N0.getValueType(), N0Op0);
4433
4434 // Replace uses of the AND with uses of the Zero extend node.
4435 CombineTo(N, Zext);
4436
4437 // We actually want to replace all uses of the any_extend with the
4438 // zero_extend, to avoid duplicating things. This will later cause this
4439 // AND to be folded.
4440 CombineTo(N0.getNode(), Zext);
4441 return SDValue(N, 0); // Return N so it doesn't get rechecked!
4442 }
4443 }
4444 // similarly fold (and (X (load ([non_ext|any_ext|zero_ext] V))), c) ->
4445 // (X (load ([non_ext|zero_ext] V))) if 'and' only clears top bits which must
4446 // already be zero by virtue of the width of the base type of the load.
4447 //
4448 // the 'X' node here can either be nothing or an extract_vector_elt to catch
4449 // more cases.
4450 if ((N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
4451 N0.getValueSizeInBits() == N0.getOperand(0).getScalarValueSizeInBits() &&
4452 N0.getOperand(0).getOpcode() == ISD::LOAD &&
4453 N0.getOperand(0).getResNo() == 0) ||
4454 (N0.getOpcode() == ISD::LOAD && N0.getResNo() == 0)) {
4455 LoadSDNode *Load = cast<LoadSDNode>( (N0.getOpcode() == ISD::LOAD) ?
4456 N0 : N0.getOperand(0) );
4457
4458 // Get the constant (if applicable) the zero'th operand is being ANDed with.
4459 // This can be a pure constant or a vector splat, in which case we treat the
4460 // vector as a scalar and use the splat value.
4461 APInt Constant = APInt::getNullValue(1);
4462 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) {
4463 Constant = C->getAPIntValue();
4464 } else if (BuildVectorSDNode *Vector = dyn_cast<BuildVectorSDNode>(N1)) {
4465 APInt SplatValue, SplatUndef;
4466 unsigned SplatBitSize;
4467 bool HasAnyUndefs;
4468 bool IsSplat = Vector->isConstantSplat(SplatValue, SplatUndef,
4469 SplatBitSize, HasAnyUndefs);
4470 if (IsSplat) {
4471 // Undef bits can contribute to a possible optimisation if set, so
4472 // set them.
4473 SplatValue |= SplatUndef;
4474
4475 // The splat value may be something like "0x00FFFFFF", which means 0 for
4476 // the first vector value and FF for the rest, repeating. We need a mask
4477 // that will apply equally to all members of the vector, so AND all the
4478 // lanes of the constant together.
4479 EVT VT = Vector->getValueType(0);
4480 unsigned BitWidth = VT.getScalarSizeInBits();
4481
4482 // If the splat value has been compressed to a bitlength lower
4483 // than the size of the vector lane, we need to re-expand it to
4484 // the lane size.
4485 if (BitWidth > SplatBitSize)
4486 for (SplatValue = SplatValue.zextOrTrunc(BitWidth);
4487 SplatBitSize < BitWidth;
4488 SplatBitSize = SplatBitSize * 2)
4489 SplatValue |= SplatValue.shl(SplatBitSize);
4490
4491 // Make sure that variable 'Constant' is only set if 'SplatBitSize' is a
4492 // multiple of 'BitWidth'. Otherwise, we could propagate a wrong value.
4493 if (SplatBitSize % BitWidth == 0) {
4494 Constant = APInt::getAllOnesValue(BitWidth);
4495 for (unsigned i = 0, n = SplatBitSize/BitWidth; i < n; ++i)
4496 Constant &= SplatValue.lshr(i*BitWidth).zextOrTrunc(BitWidth);
4497 }
4498 }
4499 }
4500
4501 // If we want to change an EXTLOAD to a ZEXTLOAD, ensure a ZEXTLOAD is
4502 // actually legal and isn't going to get expanded, else this is a false
4503 // optimisation.
4504 bool CanZextLoadProfitably = TLI.isLoadExtLegal(ISD::ZEXTLOAD,
4505 Load->getValueType(0),
4506 Load->getMemoryVT());
4507
4508 // Resize the constant to the same size as the original memory access before
4509 // extension. If it is still the AllOnesValue then this AND is completely
4510 // unneeded.
4511 Constant = Constant.zextOrTrunc(Load->getMemoryVT().getScalarSizeInBits());
4512
4513 bool B;
4514 switch (Load->getExtensionType()) {
4515 default: B = false; break;
4516 case ISD::EXTLOAD: B = CanZextLoadProfitably; break;
4517 case ISD::ZEXTLOAD:
4518 case ISD::NON_EXTLOAD: B = true; break;
4519 }
4520
4521 if (B && Constant.isAllOnesValue()) {
4522 // If the load type was an EXTLOAD, convert to ZEXTLOAD in order to
4523 // preserve semantics once we get rid of the AND.
4524 SDValue NewLoad(Load, 0);
4525
4526 // Fold the AND away. NewLoad may get replaced immediately.
4527 CombineTo(N, (N0.getNode() == Load) ? NewLoad : N0);
4528
4529 if (Load->getExtensionType() == ISD::EXTLOAD) {
4530 NewLoad = DAG.getLoad(Load->getAddressingMode(), ISD::ZEXTLOAD,
4531 Load->getValueType(0), SDLoc(Load),
4532 Load->getChain(), Load->getBasePtr(),
4533 Load->getOffset(), Load->getMemoryVT(),
4534 Load->getMemOperand());
4535 // Replace uses of the EXTLOAD with the new ZEXTLOAD.
4536 if (Load->getNumValues() == 3) {
4537 // PRE/POST_INC loads have 3 values.
4538 SDValue To[] = { NewLoad.getValue(0), NewLoad.getValue(1),
4539 NewLoad.getValue(2) };
4540 CombineTo(Load, To, 3, true);
4541 } else {
4542 CombineTo(Load, NewLoad.getValue(0), NewLoad.getValue(1));
4543 }
4544 }
4545
4546 return SDValue(N, 0); // Return N so it doesn't get rechecked!
4547 }
4548 }
4549
4550 // fold (and (load x), 255) -> (zextload x, i8)
4551 // fold (and (extload x, i16), 255) -> (zextload x, i8)
4552 // fold (and (any_ext (extload x, i16)), 255) -> (zextload x, i8)
4553 if (!VT.isVector() && N1C && (N0.getOpcode() == ISD::LOAD ||
4554 (N0.getOpcode() == ISD::ANY_EXTEND &&
4555 N0.getOperand(0).getOpcode() == ISD::LOAD))) {
4556 if (SDValue Res = ReduceLoadWidth(N)) {
4557 LoadSDNode *LN0 = N0->getOpcode() == ISD::ANY_EXTEND
4558 ? cast<LoadSDNode>(N0.getOperand(0)) : cast<LoadSDNode>(N0);
4559
4560 AddToWorklist(N);
4561 CombineTo(LN0, Res, Res.getValue(1));
4562 return SDValue(N, 0);
4563 }
4564 }
4565
4566 if (Level >= AfterLegalizeTypes) {
4567 // Attempt to propagate the AND back up to the leaves which, if they're
4568 // loads, can be combined to narrow loads and the AND node can be removed.
4569 // Perform after legalization so that extend nodes will already be
4570 // combined into the loads.
4571 if (BackwardsPropagateMask(N, DAG)) {
4572 return SDValue(N, 0);
4573 }
4574 }
4575
4576 if (SDValue Combined = visitANDLike(N0, N1, N))
4577 return Combined;
4578
4579 // Simplify: (and (op x...), (op y...)) -> (op (and x, y))
4580 if (N0.getOpcode() == N1.getOpcode())
4581 if (SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N))
4582 return Tmp;
4583
4584 // Masking the negated extension of a boolean is just the zero-extended
4585 // boolean:
4586 // and (sub 0, zext(bool X)), 1 --> zext(bool X)
4587 // and (sub 0, sext(bool X)), 1 --> zext(bool X)
4588 //
4589 // Note: the SimplifyDemandedBits fold below can make an information-losing
4590 // transform, and then we have no way to find this better fold.
4591 if (N1C && N1C->isOne() && N0.getOpcode() == ISD::SUB) {
4592 if (isNullConstantOrNullSplatConstant(N0.getOperand(0))) {
4593 SDValue SubRHS = N0.getOperand(1);
4594 if (SubRHS.getOpcode() == ISD::ZERO_EXTEND &&
4595 SubRHS.getOperand(0).getScalarValueSizeInBits() == 1)
4596 return SubRHS;
4597 if (SubRHS.getOpcode() == ISD::SIGN_EXTEND &&
4598 SubRHS.getOperand(0).getScalarValueSizeInBits() == 1)
4599 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, SubRHS.getOperand(0));
4600 }
4601 }
4602
4603 // fold (and (sign_extend_inreg x, i16 to i32), 1) -> (and x, 1)
4604 // fold (and (sra)) -> (and (srl)) when possible.
4605 if (SimplifyDemandedBits(SDValue(N, 0)))
4606 return SDValue(N, 0);
4607
4608 // fold (zext_inreg (extload x)) -> (zextload x)
4609 if (ISD::isEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode())) {
4610 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
4611 EVT MemVT = LN0->getMemoryVT();
4612 // If we zero all the possible extended bits, then we can turn this into
4613 // a zextload if we are running before legalize or the operation is legal.
4614 unsigned BitWidth = N1.getScalarValueSizeInBits();
4615 if (DAG.MaskedValueIsZero(N1, APInt::getHighBitsSet(BitWidth,
4616 BitWidth - MemVT.getScalarSizeInBits())) &&
4617 ((!LegalOperations && !LN0->isVolatile()) ||
4618 TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT))) {
4619 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT,
4620 LN0->getChain(), LN0->getBasePtr(),
4621 MemVT, LN0->getMemOperand());
4622 AddToWorklist(N);
4623 CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
4624 return SDValue(N, 0); // Return N so it doesn't get rechecked!
4625 }
4626 }
4627 // fold (zext_inreg (sextload x)) -> (zextload x) iff load has one use
4628 if (ISD::isSEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) &&
4629 N0.hasOneUse()) {
4630 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
4631 EVT MemVT = LN0->getMemoryVT();
4632 // If we zero all the possible extended bits, then we can turn this into
4633 // a zextload if we are running before legalize or the operation is legal.
4634 unsigned BitWidth = N1.getScalarValueSizeInBits();
4635 if (DAG.MaskedValueIsZero(N1, APInt::getHighBitsSet(BitWidth,
4636 BitWidth - MemVT.getScalarSizeInBits())) &&
4637 ((!LegalOperations && !LN0->isVolatile()) ||
4638 TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT))) {
4639 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT,
4640 LN0->getChain(), LN0->getBasePtr(),
4641 MemVT, LN0->getMemOperand());
4642 AddToWorklist(N);
4643 CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
4644 return SDValue(N, 0); // Return N so it doesn't get rechecked!
4645 }
4646 }
4647 // fold (and (or (srl N, 8), (shl N, 8)), 0xffff) -> (srl (bswap N), const)
4648 if (N1C && N1C->getAPIntValue() == 0xffff && N0.getOpcode() == ISD::OR) {
4649 if (SDValue BSwap = MatchBSwapHWordLow(N0.getNode(), N0.getOperand(0),
4650 N0.getOperand(1), false))
4651 return BSwap;
4652 }
4653
4654 if (SDValue Shifts = unfoldExtremeBitClearingToShifts(N))
4655 return Shifts;
4656
4657 return SDValue();
4658}
4659
4660/// Match (a >> 8) | (a << 8) as (bswap a) >> 16.
4661SDValue DAGCombiner::MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
4662 bool DemandHighBits) {
4663 if (!LegalOperations)
4664 return SDValue();
4665
4666 EVT VT = N->getValueType(0);
4667 if (VT != MVT::i64 && VT != MVT::i32 && VT != MVT::i16)
4668 return SDValue();
4669 if (!TLI.isOperationLegalOrCustom(ISD::BSWAP, VT))
4670 return SDValue();
4671
4672 // Recognize (and (shl a, 8), 0xff00), (and (srl a, 8), 0xff)
4673 bool LookPassAnd0 = false;
4674 bool LookPassAnd1 = false;
4675 if (N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::SRL)
4676 std::swap(N0, N1);
4677 if (N1.getOpcode() == ISD::AND && N1.getOperand(0).getOpcode() == ISD::SHL)
4678 std::swap(N0, N1);
4679 if (N0.getOpcode() == ISD::AND) {
4680 if (!N0.getNode()->hasOneUse())
4681 return SDValue();
4682 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
4683 // Also handle 0xffff since the LHS is guaranteed to have zeros there.
4684 // This is needed for X86.
4685 if (!N01C || (N01C->getZExtValue() != 0xFF00 &&
4686 N01C->getZExtValue() != 0xFFFF))
4687 return SDValue();
4688 N0 = N0.getOperand(0);
4689 LookPassAnd0 = true;
4690 }
4691
4692 if (N1.getOpcode() == ISD::AND) {
4693 if (!N1.getNode()->hasOneUse())
4694 return SDValue();
4695 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
4696 if (!N11C || N11C->getZExtValue() != 0xFF)
4697 return SDValue();
4698 N1 = N1.getOperand(0);
4699 LookPassAnd1 = true;
4700 }
4701
4702 if (N0.getOpcode() == ISD::SRL && N1.getOpcode() == ISD::SHL)
4703 std::swap(N0, N1);
4704 if (N0.getOpcode() != ISD::SHL || N1.getOpcode() != ISD::SRL)
4705 return SDValue();
4706 if (!N0.getNode()->hasOneUse() || !N1.getNode()->hasOneUse())
4707 return SDValue();
4708
4709 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
4710 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
4711 if (!N01C || !N11C)
4712 return SDValue();
4713 if (N01C->getZExtValue() != 8 || N11C->getZExtValue() != 8)
4714 return SDValue();
4715
4716 // Look for (shl (and a, 0xff), 8), (srl (and a, 0xff00), 8)
4717 SDValue N00 = N0->getOperand(0);
4718 if (!LookPassAnd0 && N00.getOpcode() == ISD::AND) {
4719 if (!N00.getNode()->hasOneUse())
4720 return SDValue();
4721 ConstantSDNode *N001C = dyn_cast<ConstantSDNode>(N00.getOperand(1));
4722 if (!N001C || N001C->getZExtValue() != 0xFF)
4723 return SDValue();
4724 N00 = N00.getOperand(0);
4725 LookPassAnd0 = true;
4726 }
4727
4728 SDValue N10 = N1->getOperand(0);
4729 if (!LookPassAnd1 && N10.getOpcode() == ISD::AND) {
4730 if (!N10.getNode()->hasOneUse())
4731 return SDValue();
4732 ConstantSDNode *N101C = dyn_cast<ConstantSDNode>(N10.getOperand(1));
4733 // Also allow 0xFFFF since the bits will be shifted out. This is needed
4734 // for X86.
4735 if (!N101C || (N101C->getZExtValue() != 0xFF00 &&
4736 N101C->getZExtValue() != 0xFFFF))
4737 return SDValue();
4738 N10 = N10.getOperand(0);
4739 LookPassAnd1 = true;
4740 }
4741
4742 if (N00 != N10)
4743 return SDValue();
4744
4745 // Make sure everything beyond the low halfword gets set to zero since the SRL
4746 // 16 will clear the top bits.
4747 unsigned OpSizeInBits = VT.getSizeInBits();
4748 if (DemandHighBits && OpSizeInBits > 16) {
4749 // If the left-shift isn't masked out then the only way this is a bswap is
4750 // if all bits beyond the low 8 are 0. In that case the entire pattern
4751 // reduces to a left shift anyway: leave it for other parts of the combiner.
4752 if (!LookPassAnd0)
4753 return SDValue();
4754
4755 // However, if the right shift isn't masked out then it might be because
4756 // it's not needed. See if we can spot that too.
4757 if (!LookPassAnd1 &&
4758 !DAG.MaskedValueIsZero(
4759 N10, APInt::getHighBitsSet(OpSizeInBits, OpSizeInBits - 16)))
4760 return SDValue();
4761 }
4762
4763 SDValue Res = DAG.getNode(ISD::BSWAP, SDLoc(N), VT, N00);
4764 if (OpSizeInBits > 16) {
4765 SDLoc DL(N);
4766 Res = DAG.getNode(ISD::SRL, DL, VT, Res,
4767 DAG.getConstant(OpSizeInBits - 16, DL,
4768 getShiftAmountTy(VT)));
4769 }
4770 return Res;
4771}
4772
4773/// Return true if the specified node is an element that makes up a 32-bit
4774/// packed halfword byteswap.
4775/// ((x & 0x000000ff) << 8) |
4776/// ((x & 0x0000ff00) >> 8) |
4777/// ((x & 0x00ff0000) << 8) |
4778/// ((x & 0xff000000) >> 8)
4779static bool isBSwapHWordElement(SDValue N, MutableArrayRef<SDNode *> Parts) {
4780 if (!N.getNode()->hasOneUse())
4781 return false;
4782
4783 unsigned Opc = N.getOpcode();
4784 if (Opc != ISD::AND && Opc != ISD::SHL && Opc != ISD::SRL)
4785 return false;
4786
4787 SDValue N0 = N.getOperand(0);
4788 unsigned Opc0 = N0.getOpcode();
4789 if (Opc0 != ISD::AND && Opc0 != ISD::SHL && Opc0 != ISD::SRL)
4790 return false;
4791
4792 ConstantSDNode *N1C = nullptr;
4793 // SHL or SRL: look upstream for AND mask operand
4794 if (Opc == ISD::AND)
4795 N1C = dyn_cast<ConstantSDNode>(N.getOperand(1));
4796 else if (Opc0 == ISD::AND)
4797 N1C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
4798 if (!N1C)
4799 return false;
4800
4801 unsigned MaskByteOffset;
4802 switch (N1C->getZExtValue()) {
4803 default:
4804 return false;
4805 case 0xFF: MaskByteOffset = 0; break;
4806 case 0xFF00: MaskByteOffset = 1; break;
4807 case 0xFFFF:
4808 // In case demanded bits didn't clear the bits that will be shifted out.
4809 // This is needed for X86.
4810 if (Opc == ISD::SRL || (Opc == ISD::AND && Opc0 == ISD::SHL)) {
4811 MaskByteOffset = 1;
4812 break;
4813 }
4814 return false;
4815 case 0xFF0000: MaskByteOffset = 2; break;
4816 case 0xFF000000: MaskByteOffset = 3; break;
4817 }
4818
4819 // Look for (x & 0xff) << 8 as well as ((x << 8) & 0xff00).
4820 if (Opc == ISD::AND) {
4821 if (MaskByteOffset == 0 || MaskByteOffset == 2) {
4822 // (x >> 8) & 0xff
4823 // (x >> 8) & 0xff0000
4824 if (Opc0 != ISD::SRL)
4825 return false;
4826 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
4827 if (!C || C->getZExtValue() != 8)
4828 return false;
4829 } else {
4830 // (x << 8) & 0xff00
4831 // (x << 8) & 0xff000000
4832 if (Opc0 != ISD::SHL)
4833 return false;
4834 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
4835 if (!C || C->getZExtValue() != 8)
4836 return false;
4837 }
4838 } else if (Opc == ISD::SHL) {
4839 // (x & 0xff) << 8
4840 // (x & 0xff0000) << 8
4841 if (MaskByteOffset != 0 && MaskByteOffset != 2)
4842 return false;
4843 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
4844 if (!C || C->getZExtValue() != 8)
4845 return false;
4846 } else { // Opc == ISD::SRL
4847 // (x & 0xff00) >> 8
4848 // (x & 0xff000000) >> 8
4849 if (MaskByteOffset != 1 && MaskByteOffset != 3)
4850 return false;
4851 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
4852 if (!C || C->getZExtValue() != 8)
4853 return false;
4854 }
4855
4856 if (Parts[MaskByteOffset])
4857 return false;
4858
4859 Parts[MaskByteOffset] = N0.getOperand(0).getNode();
4860 return true;
4861}
4862
4863/// Match a 32-bit packed halfword bswap. That is
4864/// ((x & 0x000000ff) << 8) |
4865/// ((x & 0x0000ff00) >> 8) |
4866/// ((x & 0x00ff0000) << 8) |
4867/// ((x & 0xff000000) >> 8)
4868/// => (rotl (bswap x), 16)
4869SDValue DAGCombiner::MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1) {
4870 if (!LegalOperations)
4871 return SDValue();
4872
4873 EVT VT = N->getValueType(0);
4874 if (VT != MVT::i32)
4875 return SDValue();
4876 if (!TLI.isOperationLegalOrCustom(ISD::BSWAP, VT))
4877 return SDValue();
4878
4879 // Look for either
4880 // (or (or (and), (and)), (or (and), (and)))
4881 // (or (or (or (and), (and)), (and)), (and))
4882 if (N0.getOpcode() != ISD::OR)
4883 return SDValue();
4884 SDValue N00 = N0.getOperand(0);
4885 SDValue N01 = N0.getOperand(1);
4886 SDNode *Parts[4] = {};
4887
4888 if (N1.getOpcode() == ISD::OR &&
4889 N00.getNumOperands() == 2 && N01.getNumOperands() == 2) {
4890 // (or (or (and), (and)), (or (and), (and)))
4891 if (!isBSwapHWordElement(N00, Parts))
4892 return SDValue();
4893
4894 if (!isBSwapHWordElement(N01, Parts))
4895 return SDValue();
4896 SDValue N10 = N1.getOperand(0);
4897 if (!isBSwapHWordElement(N10, Parts))
4898 return SDValue();
4899 SDValue N11 = N1.getOperand(1);
4900 if (!isBSwapHWordElement(N11, Parts))
4901 return SDValue();
4902 } else {
4903 // (or (or (or (and), (and)), (and)), (and))
4904 if (!isBSwapHWordElement(N1, Parts))
4905 return SDValue();
4906 if (!isBSwapHWordElement(N01, Parts))
4907 return SDValue();
4908 if (N00.getOpcode() != ISD::OR)
4909 return SDValue();
4910 SDValue N000 = N00.getOperand(0);
4911 if (!isBSwapHWordElement(N000, Parts))
4912 return SDValue();
4913 SDValue N001 = N00.getOperand(1);
4914 if (!isBSwapHWordElement(N001, Parts))
4915 return SDValue();
4916 }
4917
4918 // Make sure the parts are all coming from the same node.
4919 if (Parts[0] != Parts[1] || Parts[0] != Parts[2] || Parts[0] != Parts[3])
4920 return SDValue();
4921
4922 SDLoc DL(N);
4923 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT,
4924 SDValue(Parts[0], 0));
4925
4926 // Result of the bswap should be rotated by 16. If it's not legal, then
4927 // do (x << 16) | (x >> 16).
4928 SDValue ShAmt = DAG.getConstant(16, DL, getShiftAmountTy(VT));
4929 if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT))
4930 return DAG.getNode(ISD::ROTL, DL, VT, BSwap, ShAmt);
4931 if (TLI.isOperationLegalOrCustom(ISD::ROTR, VT))
4932 return DAG.getNode(ISD::ROTR, DL, VT, BSwap, ShAmt);
4933 return DAG.getNode(ISD::OR, DL, VT,
4934 DAG.getNode(ISD::SHL, DL, VT, BSwap, ShAmt),
4935 DAG.getNode(ISD::SRL, DL, VT, BSwap, ShAmt));
4936}
4937
4938/// This contains all DAGCombine rules which reduce two values combined by
4939/// an Or operation to a single value \see visitANDLike().
4940SDValue DAGCombiner::visitORLike(SDValue N0, SDValue N1, SDNode *N) {
4941 EVT VT = N1.getValueType();
4942 SDLoc DL(N);
4943
4944 // fold (or x, undef) -> -1
4945 if (!LegalOperations && (N0.isUndef() || N1.isUndef()))
4946 return DAG.getAllOnesConstant(DL, VT);
4947
4948 if (SDValue V = foldLogicOfSetCCs(false, N0, N1, DL))
4949 return V;
4950
4951 // (or (and X, C1), (and Y, C2)) -> (and (or X, Y), C3) if possible.
4952 if (N0.getOpcode() == ISD::AND && N1.getOpcode() == ISD::AND &&
4953 // Don't increase # computations.
4954 (N0.getNode()->hasOneUse() || N1.getNode()->hasOneUse())) {
4955 // We can only do this xform if we know that bits from X that are set in C2
4956 // but not in C1 are already zero. Likewise for Y.
4957 if (const ConstantSDNode *N0O1C =
4958 getAsNonOpaqueConstant(N0.getOperand(1))) {
4959 if (const ConstantSDNode *N1O1C =
4960 getAsNonOpaqueConstant(N1.getOperand(1))) {
4961 // We can only do this xform if we know that bits from X that are set in
4962 // C2 but not in C1 are already zero. Likewise for Y.
4963 const APInt &LHSMask = N0O1C->getAPIntValue();
4964 const APInt &RHSMask = N1O1C->getAPIntValue();
4965
4966 if (DAG.MaskedValueIsZero(N0.getOperand(0), RHSMask&~LHSMask) &&
4967 DAG.MaskedValueIsZero(N1.getOperand(0), LHSMask&~RHSMask)) {
4968 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT,
4969 N0.getOperand(0), N1.getOperand(0));
4970 return DAG.getNode(ISD::AND, DL, VT, X,
4971 DAG.getConstant(LHSMask | RHSMask, DL, VT));
4972 }
4973 }
4974 }
4975 }
4976
4977 // (or (and X, M), (and X, N)) -> (and X, (or M, N))
4978 if (N0.getOpcode() == ISD::AND &&
4979 N1.getOpcode() == ISD::AND &&
4980 N0.getOperand(0) == N1.getOperand(0) &&
4981 // Don't increase # computations.
4982 (N0.getNode()->hasOneUse() || N1.getNode()->hasOneUse())) {
4983 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT,
4984 N0.getOperand(1), N1.getOperand(1));
4985 return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), X);
4986 }
4987
4988 return SDValue();
4989}
4990
4991SDValue DAGCombiner::visitOR(SDNode *N) {
4992 SDValue N0 = N->getOperand(0);
4993 SDValue N1 = N->getOperand(1);
4994 EVT VT = N1.getValueType();
4995
4996 // x | x --> x
4997 if (N0 == N1)
4998 return N0;
4999
5000 // fold vector ops
5001 if (VT.isVector()) {
5002 if (SDValue FoldedVOp = SimplifyVBinOp(N))
5003 return FoldedVOp;
5004
5005 // fold (or x, 0) -> x, vector edition
5006 if (ISD::isBuildVectorAllZeros(N0.getNode()))
5007 return N1;
5008 if (ISD::isBuildVectorAllZeros(N1.getNode()))
5009 return N0;
5010
5011 // fold (or x, -1) -> -1, vector edition
5012 if (ISD::isBuildVectorAllOnes(N0.getNode()))
5013 // do not return N0, because undef node may exist in N0
5014 return DAG.getAllOnesConstant(SDLoc(N), N0.getValueType());
5015 if (ISD::isBuildVectorAllOnes(N1.getNode()))
5016 // do not return N1, because undef node may exist in N1
5017 return DAG.getAllOnesConstant(SDLoc(N), N1.getValueType());
5018
5019 // fold (or (shuf A, V_0, MA), (shuf B, V_0, MB)) -> (shuf A, B, Mask)
5020 // Do this only if the resulting shuffle is legal.
5021 if (isa<ShuffleVectorSDNode>(N0) &&
5022 isa<ShuffleVectorSDNode>(N1) &&
5023 // Avoid folding a node with illegal type.
5024 TLI.isTypeLegal(VT)) {
5025 bool ZeroN00 = ISD::isBuildVectorAllZeros(N0.getOperand(0).getNode());
5026 bool ZeroN01 = ISD::isBuildVectorAllZeros(N0.getOperand(1).getNode());
5027 bool ZeroN10 = ISD::isBuildVectorAllZeros(N1.getOperand(0).getNode());
5028 bool ZeroN11 = ISD::isBuildVectorAllZeros(N1.getOperand(1).getNode());
5029 // Ensure both shuffles have a zero input.
5030 if ((ZeroN00 != ZeroN01) && (ZeroN10 != ZeroN11)) {
5031 assert((!ZeroN00 || !ZeroN01) && "Both inputs zero!")(static_cast <bool> ((!ZeroN00 || !ZeroN01) && "Both inputs zero!"
) ? void (0) : __assert_fail ("(!ZeroN00 || !ZeroN01) && \"Both inputs zero!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5031, __extension__ __PRETTY_FUNCTION__))
;
5032 assert((!ZeroN10 || !ZeroN11) && "Both inputs zero!")(static_cast <bool> ((!ZeroN10 || !ZeroN11) && "Both inputs zero!"
) ? void (0) : __assert_fail ("(!ZeroN10 || !ZeroN11) && \"Both inputs zero!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5032, __extension__ __PRETTY_FUNCTION__))
;
5033 const ShuffleVectorSDNode *SV0 = cast<ShuffleVectorSDNode>(N0);
5034 const ShuffleVectorSDNode *SV1 = cast<ShuffleVectorSDNode>(N1);
5035 bool CanFold = true;
5036 int NumElts = VT.getVectorNumElements();
5037 SmallVector<int, 4> Mask(NumElts);
5038
5039 for (int i = 0; i != NumElts; ++i) {
5040 int M0 = SV0->getMaskElt(i);
5041 int M1 = SV1->getMaskElt(i);
5042
5043 // Determine if either index is pointing to a zero vector.
5044 bool M0Zero = M0 < 0 || (ZeroN00 == (M0 < NumElts));
5045 bool M1Zero = M1 < 0 || (ZeroN10 == (M1 < NumElts));
5046
5047 // If one element is zero and the otherside is undef, keep undef.
5048 // This also handles the case that both are undef.
5049 if ((M0Zero && M1 < 0) || (M1Zero && M0 < 0)) {
5050 Mask[i] = -1;
5051 continue;
5052 }
5053
5054 // Make sure only one of the elements is zero.
5055 if (M0Zero == M1Zero) {
5056 CanFold = false;
5057 break;
5058 }
5059
5060 assert((M0 >= 0 || M1 >= 0) && "Undef index!")(static_cast <bool> ((M0 >= 0 || M1 >= 0) &&
"Undef index!") ? void (0) : __assert_fail ("(M0 >= 0 || M1 >= 0) && \"Undef index!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5060, __extension__ __PRETTY_FUNCTION__))
;
5061
5062 // We have a zero and non-zero element. If the non-zero came from
5063 // SV0 make the index a LHS index. If it came from SV1, make it
5064 // a RHS index. We need to mod by NumElts because we don't care
5065 // which operand it came from in the original shuffles.
5066 Mask[i] = M1Zero ? M0 % NumElts : (M1 % NumElts) + NumElts;
5067 }
5068
5069 if (CanFold) {
5070 SDValue NewLHS = ZeroN00 ? N0.getOperand(1) : N0.getOperand(0);
5071 SDValue NewRHS = ZeroN10 ? N1.getOperand(1) : N1.getOperand(0);
5072
5073 bool LegalMask = TLI.isShuffleMaskLegal(Mask, VT);
5074 if (!LegalMask) {
5075 std::swap(NewLHS, NewRHS);
5076 ShuffleVectorSDNode::commuteMask(Mask);
5077 LegalMask = TLI.isShuffleMaskLegal(Mask, VT);
5078 }
5079
5080 if (LegalMask)
5081 return DAG.getVectorShuffle(VT, SDLoc(N), NewLHS, NewRHS, Mask);
5082 }
5083 }
5084 }
5085 }
5086
5087 // fold (or c1, c2) -> c1|c2
5088 ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
5089 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
5090 if (N0C && N1C && !N1C->isOpaque())
5091 return DAG.FoldConstantArithmetic(ISD::OR, SDLoc(N), VT, N0C, N1C);
5092 // canonicalize constant to RHS
5093 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
5094 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
5095 return DAG.getNode(ISD::OR, SDLoc(N), VT, N1, N0);
5096 // fold (or x, 0) -> x
5097 if (isNullConstant(N1))
5098 return N0;
5099 // fold (or x, -1) -> -1
5100 if (isAllOnesConstant(N1))
5101 return N1;
5102
5103 if (SDValue NewSel = foldBinOpIntoSelect(N))
5104 return NewSel;
5105
5106 // fold (or x, c) -> c iff (x & ~c) == 0
5107 if (N1C && DAG.MaskedValueIsZero(N0, ~N1C->getAPIntValue()))
5108 return N1;
5109
5110 if (SDValue Combined = visitORLike(N0, N1, N))
5111 return Combined;
5112
5113 // Recognize halfword bswaps as (bswap + rotl 16) or (bswap + shl 16)
5114 if (SDValue BSwap = MatchBSwapHWord(N, N0, N1))
5115 return BSwap;
5116 if (SDValue BSwap = MatchBSwapHWordLow(N, N0, N1))
5117 return BSwap;
5118
5119 // reassociate or
5120 if (SDValue ROR = ReassociateOps(ISD::OR, SDLoc(N), N0, N1))
5121 return ROR;
5122
5123 // Canonicalize (or (and X, c1), c2) -> (and (or X, c2), c1|c2)
5124 // iff (c1 & c2) != 0.
5125 auto MatchIntersect = [](ConstantSDNode *LHS, ConstantSDNode *RHS) {
5126 return LHS->getAPIntValue().intersects(RHS->getAPIntValue());
5127 };
5128 if (N0.getOpcode() == ISD::AND && N0.getNode()->hasOneUse() &&
5129 ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchIntersect)) {
5130 if (SDValue COR = DAG.FoldConstantArithmetic(
5131 ISD::OR, SDLoc(N1), VT, N1.getNode(), N0.getOperand(1).getNode())) {
5132 SDValue IOR = DAG.getNode(ISD::OR, SDLoc(N0), VT, N0.getOperand(0), N1);
5133 AddToWorklist(IOR.getNode());
5134 return DAG.getNode(ISD::AND, SDLoc(N), VT, COR, IOR);
5135 }
5136 }
5137
5138 // Simplify: (or (op x...), (op y...)) -> (op (or x, y))
5139 if (N0.getOpcode() == N1.getOpcode())
5140 if (SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N))
5141 return Tmp;
5142
5143 // See if this is some rotate idiom.
5144 if (SDNode *Rot = MatchRotate(N0, N1, SDLoc(N)))
5145 return SDValue(Rot, 0);
5146
5147 if (SDValue Load = MatchLoadCombine(N))
5148 return Load;
5149
5150 // Simplify the operands using demanded-bits information.
5151 if (SimplifyDemandedBits(SDValue(N, 0)))
5152 return SDValue(N, 0);
5153
5154 return SDValue();
5155}
5156
5157/// Match "(X shl/srl V1) & V2" where V2 may not be present.
5158bool DAGCombiner::MatchRotateHalf(SDValue Op, SDValue &Shift, SDValue &Mask) {
5159 if (Op.getOpcode() == ISD::AND) {
5160 if (DAG.isConstantIntBuildVectorOrConstantInt(Op.getOperand(1))) {
5161 Mask = Op.getOperand(1);
5162 Op = Op.getOperand(0);
5163 } else {
5164 return false;
5165 }
5166 }
5167
5168 if (Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SHL) {
5169 Shift = Op;
5170 return true;
5171 }
5172
5173 return false;
5174}
5175
5176// Return true if we can prove that, whenever Neg and Pos are both in the
5177// range [0, EltSize), Neg == (Pos == 0 ? 0 : EltSize - Pos). This means that
5178// for two opposing shifts shift1 and shift2 and a value X with OpBits bits:
5179//
5180// (or (shift1 X, Neg), (shift2 X, Pos))
5181//
5182// reduces to a rotate in direction shift2 by Pos or (equivalently) a rotate
5183// in direction shift1 by Neg. The range [0, EltSize) means that we only need
5184// to consider shift amounts with defined behavior.
5185static bool matchRotateSub(SDValue Pos, SDValue Neg, unsigned EltSize,
5186 SelectionDAG &DAG) {
5187 // If EltSize is a power of 2 then:
5188 //
5189 // (a) (Pos == 0 ? 0 : EltSize - Pos) == (EltSize - Pos) & (EltSize - 1)
5190 // (b) Neg == Neg & (EltSize - 1) whenever Neg is in [0, EltSize).
5191 //
5192 // So if EltSize is a power of 2 and Neg is (and Neg', EltSize-1), we check
5193 // for the stronger condition:
5194 //
5195 // Neg & (EltSize - 1) == (EltSize - Pos) & (EltSize - 1) [A]
5196 //
5197 // for all Neg and Pos. Since Neg & (EltSize - 1) == Neg' & (EltSize - 1)
5198 // we can just replace Neg with Neg' for the rest of the function.
5199 //
5200 // In other cases we check for the even stronger condition:
5201 //
5202 // Neg == EltSize - Pos [B]
5203 //
5204 // for all Neg and Pos. Note that the (or ...) then invokes undefined
5205 // behavior if Pos == 0 (and consequently Neg == EltSize).
5206 //
5207 // We could actually use [A] whenever EltSize is a power of 2, but the
5208 // only extra cases that it would match are those uninteresting ones
5209 // where Neg and Pos are never in range at the same time. E.g. for
5210 // EltSize == 32, using [A] would allow a Neg of the form (sub 64, Pos)
5211 // as well as (sub 32, Pos), but:
5212 //
5213 // (or (shift1 X, (sub 64, Pos)), (shift2 X, Pos))
5214 //
5215 // always invokes undefined behavior for 32-bit X.
5216 //
5217 // Below, Mask == EltSize - 1 when using [A] and is all-ones otherwise.
5218 unsigned MaskLoBits = 0;
5219 if (Neg.getOpcode() == ISD::AND && isPowerOf2_64(EltSize)) {
5220 if (ConstantSDNode *NegC = isConstOrConstSplat(Neg.getOperand(1))) {
5221 KnownBits Known;
5222 DAG.computeKnownBits(Neg.getOperand(0), Known);
5223 unsigned Bits = Log2_64(EltSize);
5224 if (NegC->getAPIntValue().getActiveBits() <= Bits &&
5225 ((NegC->getAPIntValue() | Known.Zero).countTrailingOnes() >= Bits)) {
5226 Neg = Neg.getOperand(0);
5227 MaskLoBits = Bits;
5228 }
5229 }
5230 }
5231
5232 // Check whether Neg has the form (sub NegC, NegOp1) for some NegC and NegOp1.
5233 if (Neg.getOpcode() != ISD::SUB)
5234 return false;
5235 ConstantSDNode *NegC = isConstOrConstSplat(Neg.getOperand(0));
5236 if (!NegC)
5237 return false;
5238 SDValue NegOp1 = Neg.getOperand(1);
5239
5240 // On the RHS of [A], if Pos is Pos' & (EltSize - 1), just replace Pos with
5241 // Pos'. The truncation is redundant for the purpose of the equality.
5242 if (MaskLoBits && Pos.getOpcode() == ISD::AND) {
5243 if (ConstantSDNode *PosC = isConstOrConstSplat(Pos.getOperand(1))) {
5244 KnownBits Known;
5245 DAG.computeKnownBits(Pos.getOperand(0), Known);
5246 if (PosC->getAPIntValue().getActiveBits() <= MaskLoBits &&
5247 ((PosC->getAPIntValue() | Known.Zero).countTrailingOnes() >=
5248 MaskLoBits))
5249 Pos = Pos.getOperand(0);
5250 }
5251 }
5252
5253 // The condition we need is now:
5254 //
5255 // (NegC - NegOp1) & Mask == (EltSize - Pos) & Mask
5256 //
5257 // If NegOp1 == Pos then we need:
5258 //
5259 // EltSize & Mask == NegC & Mask
5260 //
5261 // (because "x & Mask" is a truncation and distributes through subtraction).
5262 APInt Width;
5263 if (Pos == NegOp1)
5264 Width = NegC->getAPIntValue();
5265
5266 // Check for cases where Pos has the form (add NegOp1, PosC) for some PosC.
5267 // Then the condition we want to prove becomes:
5268 //
5269 // (NegC - NegOp1) & Mask == (EltSize - (NegOp1 + PosC)) & Mask
5270 //
5271 // which, again because "x & Mask" is a truncation, becomes:
5272 //
5273 // NegC & Mask == (EltSize - PosC) & Mask
5274 // EltSize & Mask == (NegC + PosC) & Mask
5275 else if (Pos.getOpcode() == ISD::ADD && Pos.getOperand(0) == NegOp1) {
5276 if (ConstantSDNode *PosC = isConstOrConstSplat(Pos.getOperand(1)))
5277 Width = PosC->getAPIntValue() + NegC->getAPIntValue();
5278 else
5279 return false;
5280 } else
5281 return false;
5282
5283 // Now we just need to check that EltSize & Mask == Width & Mask.
5284 if (MaskLoBits)
5285 // EltSize & Mask is 0 since Mask is EltSize - 1.
5286 return Width.getLoBits(MaskLoBits) == 0;
5287 return Width == EltSize;
5288}
5289
5290// A subroutine of MatchRotate used once we have found an OR of two opposite
5291// shifts of Shifted. If Neg == <operand size> - Pos then the OR reduces
5292// to both (PosOpcode Shifted, Pos) and (NegOpcode Shifted, Neg), with the
5293// former being preferred if supported. InnerPos and InnerNeg are Pos and
5294// Neg with outer conversions stripped away.
5295SDNode *DAGCombiner::MatchRotatePosNeg(SDValue Shifted, SDValue Pos,
5296 SDValue Neg, SDValue InnerPos,
5297 SDValue InnerNeg, unsigned PosOpcode,
5298 unsigned NegOpcode, const SDLoc &DL) {
5299 // fold (or (shl x, (*ext y)),
5300 // (srl x, (*ext (sub 32, y)))) ->
5301 // (rotl x, y) or (rotr x, (sub 32, y))
5302 //
5303 // fold (or (shl x, (*ext (sub 32, y))),
5304 // (srl x, (*ext y))) ->
5305 // (rotr x, y) or (rotl x, (sub 32, y))
5306 EVT VT = Shifted.getValueType();
5307 if (matchRotateSub(InnerPos, InnerNeg, VT.getScalarSizeInBits(), DAG)) {
5308 bool HasPos = TLI.isOperationLegalOrCustom(PosOpcode, VT);
5309 return DAG.getNode(HasPos ? PosOpcode : NegOpcode, DL, VT, Shifted,
5310 HasPos ? Pos : Neg).getNode();
5311 }
5312
5313 return nullptr;
5314}
5315
5316// MatchRotate - Handle an 'or' of two operands. If this is one of the many
5317// idioms for rotate, and if the target supports rotation instructions, generate
5318// a rot[lr].
5319SDNode *DAGCombiner::MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL) {
5320 // Must be a legal type. Expanded 'n promoted things won't work with rotates.
5321 EVT VT = LHS.getValueType();
5322 if (!TLI.isTypeLegal(VT)) return nullptr;
5323
5324 // The target must have at least one rotate flavor.
5325 bool HasROTL = hasOperation(ISD::ROTL, VT);
5326 bool HasROTR = hasOperation(ISD::ROTR, VT);
5327 if (!HasROTL && !HasROTR) return nullptr;
5328
5329 // Check for truncated rotate.
5330 if (LHS.getOpcode() == ISD::TRUNCATE && RHS.getOpcode() == ISD::TRUNCATE &&
5331 LHS.getOperand(0).getValueType() == RHS.getOperand(0).getValueType()) {
5332 assert(LHS.getValueType() == RHS.getValueType())(static_cast <bool> (LHS.getValueType() == RHS.getValueType
()) ? void (0) : __assert_fail ("LHS.getValueType() == RHS.getValueType()"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5332, __extension__ __PRETTY_FUNCTION__))
;
5333 if (SDNode *Rot = MatchRotate(LHS.getOperand(0), RHS.getOperand(0), DL)) {
5334 return DAG.getNode(ISD::TRUNCATE, SDLoc(LHS), LHS.getValueType(),
5335 SDValue(Rot, 0)).getNode();
5336 }
5337 }
5338
5339 // Match "(X shl/srl V1) & V2" where V2 may not be present.
5340 SDValue LHSShift; // The shift.
5341 SDValue LHSMask; // AND value if any.
5342 if (!MatchRotateHalf(LHS, LHSShift, LHSMask))
5343 return nullptr; // Not part of a rotate.
5344
5345 SDValue RHSShift; // The shift.
5346 SDValue RHSMask; // AND value if any.
5347 if (!MatchRotateHalf(RHS, RHSShift, RHSMask))
5348 return nullptr; // Not part of a rotate.
5349
5350 if (LHSShift.getOperand(0) != RHSShift.getOperand(0))
5351 return nullptr; // Not shifting the same value.
5352
5353 if (LHSShift.getOpcode() == RHSShift.getOpcode())
5354 return nullptr; // Shifts must disagree.
5355
5356 // Canonicalize shl to left side in a shl/srl pair.
5357 if (RHSShift.getOpcode() == ISD::SHL) {
5358 std::swap(LHS, RHS);
5359 std::swap(LHSShift, RHSShift);
5360 std::swap(LHSMask, RHSMask);
5361 }
5362
5363 unsigned EltSizeInBits = VT.getScalarSizeInBits();
5364 SDValue LHSShiftArg = LHSShift.getOperand(0);
5365 SDValue LHSShiftAmt = LHSShift.getOperand(1);
5366 SDValue RHSShiftArg = RHSShift.getOperand(0);
5367 SDValue RHSShiftAmt = RHSShift.getOperand(1);
5368
5369 // fold (or (shl x, C1), (srl x, C2)) -> (rotl x, C1)
5370 // fold (or (shl x, C1), (srl x, C2)) -> (rotr x, C2)
5371 auto MatchRotateSum = [EltSizeInBits](ConstantSDNode *LHS,
5372 ConstantSDNode *RHS) {
5373 return (LHS->getAPIntValue() + RHS->getAPIntValue()) == EltSizeInBits;
5374 };
5375 if (ISD::matchBinaryPredicate(LHSShiftAmt, RHSShiftAmt, MatchRotateSum)) {
5376 SDValue Rot = DAG.getNode(HasROTL ? ISD::ROTL : ISD::ROTR, DL, VT,
5377 LHSShiftArg, HasROTL ? LHSShiftAmt : RHSShiftAmt);
5378
5379 // If there is an AND of either shifted operand, apply it to the result.
5380 if (LHSMask.getNode() || RHSMask.getNode()) {
5381 SDValue AllOnes = DAG.getAllOnesConstant(DL, VT);
5382 SDValue Mask = AllOnes;
5383
5384 if (LHSMask.getNode()) {
5385 SDValue RHSBits = DAG.getNode(ISD::SRL, DL, VT, AllOnes, RHSShiftAmt);
5386 Mask = DAG.getNode(ISD::AND, DL, VT, Mask,
5387 DAG.getNode(ISD::OR, DL, VT, LHSMask, RHSBits));
5388 }
5389 if (RHSMask.getNode()) {
5390 SDValue LHSBits = DAG.getNode(ISD::SHL, DL, VT, AllOnes, LHSShiftAmt);
5391 Mask = DAG.getNode(ISD::AND, DL, VT, Mask,
5392 DAG.getNode(ISD::OR, DL, VT, RHSMask, LHSBits));
5393 }
5394
5395 Rot = DAG.getNode(ISD::AND, DL, VT, Rot, Mask);
5396 }
5397
5398 return Rot.getNode();
5399 }
5400
5401 // If there is a mask here, and we have a variable shift, we can't be sure
5402 // that we're masking out the right stuff.
5403 if (LHSMask.getNode() || RHSMask.getNode())
5404 return nullptr;
5405
5406 // If the shift amount is sign/zext/any-extended just peel it off.
5407 SDValue LExtOp0 = LHSShiftAmt;
5408 SDValue RExtOp0 = RHSShiftAmt;
5409 if ((LHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND ||
5410 LHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND ||
5411 LHSShiftAmt.getOpcode() == ISD::ANY_EXTEND ||
5412 LHSShiftAmt.getOpcode() == ISD::TRUNCATE) &&
5413 (RHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND ||
5414 RHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND ||
5415 RHSShiftAmt.getOpcode() == ISD::ANY_EXTEND ||
5416 RHSShiftAmt.getOpcode() == ISD::TRUNCATE)) {
5417 LExtOp0 = LHSShiftAmt.getOperand(0);
5418 RExtOp0 = RHSShiftAmt.getOperand(0);
5419 }
5420
5421 SDNode *TryL = MatchRotatePosNeg(LHSShiftArg, LHSShiftAmt, RHSShiftAmt,
5422 LExtOp0, RExtOp0, ISD::ROTL, ISD::ROTR, DL);
5423 if (TryL)
5424 return TryL;
5425
5426 SDNode *TryR = MatchRotatePosNeg(RHSShiftArg, RHSShiftAmt, LHSShiftAmt,
5427 RExtOp0, LExtOp0, ISD::ROTR, ISD::ROTL, DL);
5428 if (TryR)
5429 return TryR;
5430
5431 return nullptr;
5432}
5433
5434namespace {
5435
5436/// Represents known origin of an individual byte in load combine pattern. The
5437/// value of the byte is either constant zero or comes from memory.
5438struct ByteProvider {
5439 // For constant zero providers Load is set to nullptr. For memory providers
5440 // Load represents the node which loads the byte from memory.
5441 // ByteOffset is the offset of the byte in the value produced by the load.
5442 LoadSDNode *Load = nullptr;
5443 unsigned ByteOffset = 0;
5444
5445 ByteProvider() = default;
5446
5447 static ByteProvider getMemory(LoadSDNode *Load, unsigned ByteOffset) {
5448 return ByteProvider(Load, ByteOffset);
5449 }
5450
5451 static ByteProvider getConstantZero() { return ByteProvider(nullptr, 0); }
5452
5453 bool isConstantZero() const { return !Load; }
5454 bool isMemory() const { return Load; }
5455
5456 bool operator==(const ByteProvider &Other) const {
5457 return Other.Load == Load && Other.ByteOffset == ByteOffset;
5458 }
5459
5460private:
5461 ByteProvider(LoadSDNode *Load, unsigned ByteOffset)
5462 : Load(Load), ByteOffset(ByteOffset) {}
5463};
5464
5465} // end anonymous namespace
5466
5467/// Recursively traverses the expression calculating the origin of the requested
5468/// byte of the given value. Returns None if the provider can't be calculated.
5469///
5470/// For all the values except the root of the expression verifies that the value
5471/// has exactly one use and if it's not true return None. This way if the origin
5472/// of the byte is returned it's guaranteed that the values which contribute to
5473/// the byte are not used outside of this expression.
5474///
5475/// Because the parts of the expression are not allowed to have more than one
5476/// use this function iterates over trees, not DAGs. So it never visits the same
5477/// node more than once.
5478static const Optional<ByteProvider>
5479calculateByteProvider(SDValue Op, unsigned Index, unsigned Depth,
5480 bool Root = false) {
5481 // Typical i64 by i8 pattern requires recursion up to 8 calls depth
5482 if (Depth == 10)
5483 return None;
5484
5485 if (!Root && !Op.hasOneUse())
5486 return None;
5487
5488 assert(Op.getValueType().isScalarInteger() && "can't handle other types")(static_cast <bool> (Op.getValueType().isScalarInteger(
) && "can't handle other types") ? void (0) : __assert_fail
("Op.getValueType().isScalarInteger() && \"can't handle other types\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5488, __extension__ __PRETTY_FUNCTION__))
;
5489 unsigned BitWidth = Op.getValueSizeInBits();
5490 if (BitWidth % 8 != 0)
5491 return None;
5492 unsigned ByteWidth = BitWidth / 8;
5493 assert(Index < ByteWidth && "invalid index requested")(static_cast <bool> (Index < ByteWidth && "invalid index requested"
) ? void (0) : __assert_fail ("Index < ByteWidth && \"invalid index requested\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5493, __extension__ __PRETTY_FUNCTION__))
;
5494 (void) ByteWidth;
5495
5496 switch (Op.getOpcode()) {
5497 case ISD::OR: {
5498 auto LHS = calculateByteProvider(Op->getOperand(0), Index, Depth + 1);
5499 if (!LHS)
5500 return None;
5501 auto RHS = calculateByteProvider(Op->getOperand(1), Index, Depth + 1);
5502 if (!RHS)
5503 return None;
5504
5505 if (LHS->isConstantZero())
5506 return RHS;
5507 if (RHS->isConstantZero())
5508 return LHS;
5509 return None;
5510 }
5511 case ISD::SHL: {
5512 auto ShiftOp = dyn_cast<ConstantSDNode>(Op->getOperand(1));
5513 if (!ShiftOp)
5514 return None;
5515
5516 uint64_t BitShift = ShiftOp->getZExtValue();
5517 if (BitShift % 8 != 0)
5518 return None;
5519 uint64_t ByteShift = BitShift / 8;
5520
5521 return Index < ByteShift
5522 ? ByteProvider::getConstantZero()
5523 : calculateByteProvider(Op->getOperand(0), Index - ByteShift,
5524 Depth + 1);
5525 }
5526 case ISD::ANY_EXTEND:
5527 case ISD::SIGN_EXTEND:
5528 case ISD::ZERO_EXTEND: {
5529 SDValue NarrowOp = Op->getOperand(0);
5530 unsigned NarrowBitWidth = NarrowOp.getScalarValueSizeInBits();
5531 if (NarrowBitWidth % 8 != 0)
5532 return None;
5533 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
5534
5535 if (Index >= NarrowByteWidth)
5536 return Op.getOpcode() == ISD::ZERO_EXTEND
5537 ? Optional<ByteProvider>(ByteProvider::getConstantZero())
5538 : None;
5539 return calculateByteProvider(NarrowOp, Index, Depth + 1);
5540 }
5541 case ISD::BSWAP:
5542 return calculateByteProvider(Op->getOperand(0), ByteWidth - Index - 1,
5543 Depth + 1);
5544 case ISD::LOAD: {
5545 auto L = cast<LoadSDNode>(Op.getNode());
5546 if (L->isVolatile() || L->isIndexed())
5547 return None;
5548
5549 unsigned NarrowBitWidth = L->getMemoryVT().getSizeInBits();
5550 if (NarrowBitWidth % 8 != 0)
5551 return None;
5552 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
5553
5554 if (Index >= NarrowByteWidth)
5555 return L->getExtensionType() == ISD::ZEXTLOAD
5556 ? Optional<ByteProvider>(ByteProvider::getConstantZero())
5557 : None;
5558 return ByteProvider::getMemory(L, Index);
5559 }
5560 }
5561
5562 return None;
5563}
5564
5565/// Match a pattern where a wide type scalar value is loaded by several narrow
5566/// loads and combined by shifts and ors. Fold it into a single load or a load
5567/// and a BSWAP if the targets supports it.
5568///
5569/// Assuming little endian target:
5570/// i8 *a = ...
5571/// i32 val = a[0] | (a[1] << 8) | (a[2] << 16) | (a[3] << 24)
5572/// =>
5573/// i32 val = *((i32)a)
5574///
5575/// i8 *a = ...
5576/// i32 val = (a[0] << 24) | (a[1] << 16) | (a[2] << 8) | a[3]
5577/// =>
5578/// i32 val = BSWAP(*((i32)a))
5579///
5580/// TODO: This rule matches complex patterns with OR node roots and doesn't
5581/// interact well with the worklist mechanism. When a part of the pattern is
5582/// updated (e.g. one of the loads) its direct users are put into the worklist,
5583/// but the root node of the pattern which triggers the load combine is not
5584/// necessarily a direct user of the changed node. For example, once the address
5585/// of t28 load is reassociated load combine won't be triggered:
5586/// t25: i32 = add t4, Constant:i32<2>
5587/// t26: i64 = sign_extend t25
5588/// t27: i64 = add t2, t26
5589/// t28: i8,ch = load<LD1[%tmp9]> t0, t27, undef:i64
5590/// t29: i32 = zero_extend t28
5591/// t32: i32 = shl t29, Constant:i8<8>
5592/// t33: i32 = or t23, t32
5593/// As a possible fix visitLoad can check if the load can be a part of a load
5594/// combine pattern and add corresponding OR roots to the worklist.
5595SDValue DAGCombiner::MatchLoadCombine(SDNode *N) {
5596 assert(N->getOpcode() == ISD::OR &&(static_cast <bool> (N->getOpcode() == ISD::OR &&
"Can only match load combining against OR nodes") ? void (0)
: __assert_fail ("N->getOpcode() == ISD::OR && \"Can only match load combining against OR nodes\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5597, __extension__ __PRETTY_FUNCTION__))
5597 "Can only match load combining against OR nodes")(static_cast <bool> (N->getOpcode() == ISD::OR &&
"Can only match load combining against OR nodes") ? void (0)
: __assert_fail ("N->getOpcode() == ISD::OR && \"Can only match load combining against OR nodes\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5597, __extension__ __PRETTY_FUNCTION__))
;
5598
5599 // Handles simple types only
5600 EVT VT = N->getValueType(0);
5601 if (VT != MVT::i16 && VT != MVT::i32 && VT != MVT::i64)
5602 return SDValue();
5603 unsigned ByteWidth = VT.getSizeInBits() / 8;
5604
5605 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5606 // Before legalize we can introduce too wide illegal loads which will be later
5607 // split into legal sized loads. This enables us to combine i64 load by i8
5608 // patterns to a couple of i32 loads on 32 bit targets.
5609 if (LegalOperations && !TLI.isOperationLegal(ISD::LOAD, VT))
5610 return SDValue();
5611
5612 std::function<unsigned(unsigned, unsigned)> LittleEndianByteAt = [](
5613 unsigned BW, unsigned i) { return i; };
5614 std::function<unsigned(unsigned, unsigned)> BigEndianByteAt = [](
5615 unsigned BW, unsigned i) { return BW - i - 1; };
5616
5617 bool IsBigEndianTarget = DAG.getDataLayout().isBigEndian();
5618 auto MemoryByteOffset = [&] (ByteProvider P) {
5619 assert(P.isMemory() && "Must be a memory byte provider")(static_cast <bool> (P.isMemory() && "Must be a memory byte provider"
) ? void (0) : __assert_fail ("P.isMemory() && \"Must be a memory byte provider\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5619, __extension__ __PRETTY_FUNCTION__))
;
5620 unsigned LoadBitWidth = P.Load->getMemoryVT().getSizeInBits();
5621 assert(LoadBitWidth % 8 == 0 &&(static_cast <bool> (LoadBitWidth % 8 == 0 && "can only analyze providers for individual bytes not bit"
) ? void (0) : __assert_fail ("LoadBitWidth % 8 == 0 && \"can only analyze providers for individual bytes not bit\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5622, __extension__ __PRETTY_FUNCTION__))
5622 "can only analyze providers for individual bytes not bit")(static_cast <bool> (LoadBitWidth % 8 == 0 && "can only analyze providers for individual bytes not bit"
) ? void (0) : __assert_fail ("LoadBitWidth % 8 == 0 && \"can only analyze providers for individual bytes not bit\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5622, __extension__ __PRETTY_FUNCTION__))
;
5623 unsigned LoadByteWidth = LoadBitWidth / 8;
5624 return IsBigEndianTarget
5625 ? BigEndianByteAt(LoadByteWidth, P.ByteOffset)
5626 : LittleEndianByteAt(LoadByteWidth, P.ByteOffset);
5627 };
5628
5629 Optional<BaseIndexOffset> Base;
5630 SDValue Chain;
5631
5632 SmallPtrSet<LoadSDNode *, 8> Loads;
5633 Optional<ByteProvider> FirstByteProvider;
5634 int64_t FirstOffset = INT64_MAX(9223372036854775807L);
5635
5636 // Check if all the bytes of the OR we are looking at are loaded from the same
5637 // base address. Collect bytes offsets from Base address in ByteOffsets.
5638 SmallVector<int64_t, 4> ByteOffsets(ByteWidth);
5639 for (unsigned i = 0; i < ByteWidth; i++) {
5640 auto P = calculateByteProvider(SDValue(N, 0), i, 0, /*Root=*/true);
5641 if (!P || !P->isMemory()) // All the bytes must be loaded from memory
5642 return SDValue();
5643
5644 LoadSDNode *L = P->Load;
5645 assert(L->hasNUsesOfValue(1, 0) && !L->isVolatile() && !L->isIndexed() &&(static_cast <bool> (L->hasNUsesOfValue(1, 0) &&
!L->isVolatile() && !L->isIndexed() &&
"Must be enforced by calculateByteProvider") ? void (0) : __assert_fail
("L->hasNUsesOfValue(1, 0) && !L->isVolatile() && !L->isIndexed() && \"Must be enforced by calculateByteProvider\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5646, __extension__ __PRETTY_FUNCTION__))
5646 "Must be enforced by calculateByteProvider")(static_cast <bool> (L->hasNUsesOfValue(1, 0) &&
!L->isVolatile() && !L->isIndexed() &&
"Must be enforced by calculateByteProvider") ? void (0) : __assert_fail
("L->hasNUsesOfValue(1, 0) && !L->isVolatile() && !L->isIndexed() && \"Must be enforced by calculateByteProvider\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5646, __extension__ __PRETTY_FUNCTION__))
;
5647 assert(L->getOffset().isUndef() && "Unindexed load must have undef offset")(static_cast <bool> (L->getOffset().isUndef() &&
"Unindexed load must have undef offset") ? void (0) : __assert_fail
("L->getOffset().isUndef() && \"Unindexed load must have undef offset\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5647, __extension__ __PRETTY_FUNCTION__))
;
5648
5649 // All loads must share the same chain
5650 SDValue LChain = L->getChain();
5651 if (!Chain)
5652 Chain = LChain;
5653 else if (Chain != LChain)
5654 return SDValue();
5655
5656 // Loads must share the same base address
5657 BaseIndexOffset Ptr = BaseIndexOffset::match(L, DAG);
5658 int64_t ByteOffsetFromBase = 0;
5659 if (!Base)
5660 Base = Ptr;
5661 else if (!Base->equalBaseIndex(Ptr, DAG, ByteOffsetFromBase))
5662 return SDValue();
5663
5664 // Calculate the offset of the current byte from the base address
5665 ByteOffsetFromBase += MemoryByteOffset(*P);
5666 ByteOffsets[i] = ByteOffsetFromBase;
5667
5668 // Remember the first byte load
5669 if (ByteOffsetFromBase < FirstOffset) {
5670 FirstByteProvider = P;
5671 FirstOffset = ByteOffsetFromBase;
5672 }
5673
5674 Loads.insert(L);
5675 }
5676 assert(!Loads.empty() && "All the bytes of the value must be loaded from "(static_cast <bool> (!Loads.empty() && "All the bytes of the value must be loaded from "
"memory, so there must be at least one load which produces the value"
) ? void (0) : __assert_fail ("!Loads.empty() && \"All the bytes of the value must be loaded from \" \"memory, so there must be at least one load which produces the value\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5677, __extension__ __PRETTY_FUNCTION__))
5677 "memory, so there must be at least one load which produces the value")(static_cast <bool> (!Loads.empty() && "All the bytes of the value must be loaded from "
"memory, so there must be at least one load which produces the value"
) ? void (0) : __assert_fail ("!Loads.empty() && \"All the bytes of the value must be loaded from \" \"memory, so there must be at least one load which produces the value\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5677, __extension__ __PRETTY_FUNCTION__))
;
5678 assert(Base && "Base address of the accessed memory location must be set")(static_cast <bool> (Base && "Base address of the accessed memory location must be set"
) ? void (0) : __assert_fail ("Base && \"Base address of the accessed memory location must be set\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5678, __extension__ __PRETTY_FUNCTION__))
;
5679 assert(FirstOffset != INT64_MAX && "First byte offset must be set")(static_cast <bool> (FirstOffset != (9223372036854775807L
) && "First byte offset must be set") ? void (0) : __assert_fail
("FirstOffset != INT64_MAX && \"First byte offset must be set\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5679, __extension__ __PRETTY_FUNCTION__))
;
5680
5681 // Check if the bytes of the OR we are looking at match with either big or
5682 // little endian value load
5683 bool BigEndian = true, LittleEndian = true;
5684 for (unsigned i = 0; i < ByteWidth; i++) {
5685 int64_t CurrentByteOffset = ByteOffsets[i] - FirstOffset;
5686 LittleEndian &= CurrentByteOffset == LittleEndianByteAt(ByteWidth, i);
5687 BigEndian &= CurrentByteOffset == BigEndianByteAt(ByteWidth, i);
5688 if (!BigEndian && !LittleEndian)
5689 return SDValue();
5690 }
5691 assert((BigEndian != LittleEndian) && "should be either or")(static_cast <bool> ((BigEndian != LittleEndian) &&
"should be either or") ? void (0) : __assert_fail ("(BigEndian != LittleEndian) && \"should be either or\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5691, __extension__ __PRETTY_FUNCTION__))
;
5692 assert(FirstByteProvider && "must be set")(static_cast <bool> (FirstByteProvider && "must be set"
) ? void (0) : __assert_fail ("FirstByteProvider && \"must be set\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5692, __extension__ __PRETTY_FUNCTION__))
;
5693
5694 // Ensure that the first byte is loaded from zero offset of the first load.
5695 // So the combined value can be loaded from the first load address.
5696 if (MemoryByteOffset(*FirstByteProvider) != 0)
5697 return SDValue();
5698 LoadSDNode *FirstLoad = FirstByteProvider->Load;
5699
5700 // The node we are looking at matches with the pattern, check if we can
5701 // replace it with a single load and bswap if needed.
5702
5703 // If the load needs byte swap check if the target supports it
5704 bool NeedsBswap = IsBigEndianTarget != BigEndian;
5705
5706 // Before legalize we can introduce illegal bswaps which will be later
5707 // converted to an explicit bswap sequence. This way we end up with a single
5708 // load and byte shuffling instead of several loads and byte shuffling.
5709 if (NeedsBswap && LegalOperations && !TLI.isOperationLegal(ISD::BSWAP, VT))
5710 return SDValue();
5711
5712 // Check that a load of the wide type is both allowed and fast on the target
5713 bool Fast = false;
5714 bool Allowed = TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(),
5715 VT, FirstLoad->getAddressSpace(),
5716 FirstLoad->getAlignment(), &Fast);
5717 if (!Allowed || !Fast)
5718 return SDValue();
5719
5720 SDValue NewLoad =
5721 DAG.getLoad(VT, SDLoc(N), Chain, FirstLoad->getBasePtr(),
5722 FirstLoad->getPointerInfo(), FirstLoad->getAlignment());
5723
5724 // Transfer chain users from old loads to the new load.
5725 for (LoadSDNode *L : Loads)
5726 DAG.ReplaceAllUsesOfValueWith(SDValue(L, 1), SDValue(NewLoad.getNode(), 1));
5727
5728 return NeedsBswap ? DAG.getNode(ISD::BSWAP, SDLoc(N), VT, NewLoad) : NewLoad;
5729}
5730
5731// If the target has andn, bsl, or a similar bit-select instruction,
5732// we want to unfold masked merge, with canonical pattern of:
5733// | A | |B|
5734// ((x ^ y) & m) ^ y
5735// | D |
5736// Into:
5737// (x & m) | (y & ~m)
5738// If y is a constant, and the 'andn' does not work with immediates,
5739// we unfold into a different pattern:
5740// ~(~x & m) & (m | y)
5741// NOTE: we don't unfold the pattern if 'xor' is actually a 'not', because at
5742// the very least that breaks andnpd / andnps patterns, and because those
5743// patterns are simplified in IR and shouldn't be created in the DAG
5744SDValue DAGCombiner::unfoldMaskedMerge(SDNode *N) {
5745 assert(N->getOpcode() == ISD::XOR)(static_cast <bool> (N->getOpcode() == ISD::XOR) ? void
(0) : __assert_fail ("N->getOpcode() == ISD::XOR", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5745, __extension__ __PRETTY_FUNCTION__))
;
5746
5747 // Don't touch 'not' (i.e. where y = -1).
5748 if (isAllOnesConstantOrAllOnesSplatConstant(N->getOperand(1)))
5749 return SDValue();
5750
5751 EVT VT = N->getValueType(0);
5752
5753 // There are 3 commutable operators in the pattern,
5754 // so we have to deal with 8 possible variants of the basic pattern.
5755 SDValue X, Y, M;
5756 auto matchAndXor = [&X, &Y, &M](SDValue And, unsigned XorIdx, SDValue Other) {
5757 if (And.getOpcode() != ISD::AND || !And.hasOneUse())
5758 return false;
5759 SDValue Xor = And.getOperand(XorIdx);
5760 if (Xor.getOpcode() != ISD::XOR || !Xor.hasOneUse())
5761 return false;
5762 SDValue Xor0 = Xor.getOperand(0);
5763 SDValue Xor1 = Xor.getOperand(1);
5764 // Don't touch 'not' (i.e. where y = -1).
5765 if (isAllOnesConstantOrAllOnesSplatConstant(Xor1))
5766 return false;
5767 if (Other == Xor0)
5768 std::swap(Xor0, Xor1);
5769 if (Other != Xor1)
5770 return false;
5771 X = Xor0;
5772 Y = Xor1;
5773 M = And.getOperand(XorIdx ? 0 : 1);
5774 return true;
5775 };
5776
5777 SDValue N0 = N->getOperand(0);
5778 SDValue N1 = N->getOperand(1);
5779 if (!matchAndXor(N0, 0, N1) && !matchAndXor(N0, 1, N1) &&
5780 !matchAndXor(N1, 0, N0) && !matchAndXor(N1, 1, N0))
5781 return SDValue();
5782
5783 // Don't do anything if the mask is constant. This should not be reachable.
5784 // InstCombine should have already unfolded this pattern, and DAGCombiner
5785 // probably shouldn't produce it, too.
5786 if (isa<ConstantSDNode>(M.getNode()))
5787 return SDValue();
5788
5789 // We can transform if the target has AndNot
5790 if (!TLI.hasAndNot(M))
5791 return SDValue();
5792
5793 SDLoc DL(N);
5794
5795 // If Y is a constant, check that 'andn' works with immediates.
5796 if (!TLI.hasAndNot(Y)) {
5797 assert(TLI.hasAndNot(X) && "Only mask is a variable? Unreachable.")(static_cast <bool> (TLI.hasAndNot(X) && "Only mask is a variable? Unreachable."
) ? void (0) : __assert_fail ("TLI.hasAndNot(X) && \"Only mask is a variable? Unreachable.\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5797, __extension__ __PRETTY_FUNCTION__))
;
5798 // If not, we need to do a bit more work to make sure andn is still used.
5799 SDValue NotX = DAG.getNOT(DL, X, VT);
5800 SDValue LHS = DAG.getNode(ISD::AND, DL, VT, NotX, M);
5801 SDValue NotLHS = DAG.getNOT(DL, LHS, VT);
5802 SDValue RHS = DAG.getNode(ISD::OR, DL, VT, M, Y);
5803 return DAG.getNode(ISD::AND, DL, VT, NotLHS, RHS);
5804 }
5805
5806 SDValue LHS = DAG.getNode(ISD::AND, DL, VT, X, M);
5807 SDValue NotM = DAG.getNOT(DL, M, VT);
5808 SDValue RHS = DAG.getNode(ISD::AND, DL, VT, Y, NotM);
5809
5810 return DAG.getNode(ISD::OR, DL, VT, LHS, RHS);
5811}
5812
5813SDValue DAGCombiner::visitXOR(SDNode *N) {
5814 SDValue N0 = N->getOperand(0);
5815 SDValue N1 = N->getOperand(1);
5816 EVT VT = N0.getValueType();
5817
5818 // fold vector ops
5819 if (VT.isVector()) {
5820 if (SDValue FoldedVOp = SimplifyVBinOp(N))
5821 return FoldedVOp;
5822
5823 // fold (xor x, 0) -> x, vector edition
5824 if (ISD::isBuildVectorAllZeros(N0.getNode()))
5825 return N1;
5826 if (ISD::isBuildVectorAllZeros(N1.getNode()))
5827 return N0;
5828 }
5829
5830 // fold (xor undef, undef) -> 0. This is a common idiom (misuse).
5831 if (N0.isUndef() && N1.isUndef())
5832 return DAG.getConstant(0, SDLoc(N), VT);
5833 // fold (xor x, undef) -> undef
5834 if (N0.isUndef())
5835 return N0;
5836 if (N1.isUndef())
5837 return N1;
5838 // fold (xor c1, c2) -> c1^c2
5839 ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
5840 ConstantSDNode *N1C = getAsNonOpaqueConstant(N1);
5841 if (N0C && N1C)
5842 return DAG.FoldConstantArithmetic(ISD::XOR, SDLoc(N), VT, N0C, N1C);
5843 // canonicalize constant to RHS
5844 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
5845 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
5846 return DAG.getNode(ISD::XOR, SDLoc(N), VT, N1, N0);
5847 // fold (xor x, 0) -> x
5848 if (isNullConstant(N1))
5849 return N0;
5850
5851 if (SDValue NewSel = foldBinOpIntoSelect(N))
5852 return NewSel;
5853
5854 // reassociate xor
5855 if (SDValue RXOR = ReassociateOps(ISD::XOR, SDLoc(N), N0, N1))
5856 return RXOR;
5857
5858 // fold !(x cc y) -> (x !cc y)
5859 SDValue LHS, RHS, CC;
5860 if (TLI.isConstTrueVal(N1.getNode()) && isSetCCEquivalent(N0, LHS, RHS, CC)) {
5861 bool isInt = LHS.getValueType().isInteger();
5862 ISD::CondCode NotCC = ISD::getSetCCInverse(cast<CondCodeSDNode>(CC)->get(),
5863 isInt);
5864
5865 if (!LegalOperations ||
5866 TLI.isCondCodeLegal(NotCC, LHS.getSimpleValueType())) {
5867 switch (N0.getOpcode()) {
5868 default:
5869 llvm_unreachable("Unhandled SetCC Equivalent!")::llvm::llvm_unreachable_internal("Unhandled SetCC Equivalent!"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5869)
;
5870 case ISD::SETCC:
5871 return DAG.getSetCC(SDLoc(N0), VT, LHS, RHS, NotCC);
5872 case ISD::SELECT_CC:
5873 return DAG.getSelectCC(SDLoc(N0), LHS, RHS, N0.getOperand(2),
5874 N0.getOperand(3), NotCC);
5875 }
5876 }
5877 }
5878
5879 // fold (not (zext (setcc x, y))) -> (zext (not (setcc x, y)))
5880 if (isOneConstant(N1) && N0.getOpcode() == ISD::ZERO_EXTEND &&
5881 N0.getNode()->hasOneUse() &&
5882 isSetCCEquivalent(N0.getOperand(0), LHS, RHS, CC)){
5883 SDValue V = N0.getOperand(0);
5884 SDLoc DL(N0);
5885 V = DAG.getNode(ISD::XOR, DL, V.getValueType(), V,
5886 DAG.getConstant(1, DL, V.getValueType()));
5887 AddToWorklist(V.getNode());
5888 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, V);
5889 }
5890
5891 // fold (not (or x, y)) -> (and (not x), (not y)) iff x or y are setcc
5892 if (isOneConstant(N1) && VT == MVT::i1 && N0.hasOneUse() &&
5893 (N0.getOpcode() == ISD::OR || N0.getOpcode() == ISD::AND)) {
5894 SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1);
5895 if (isOneUseSetCC(RHS) || isOneUseSetCC(LHS)) {
5896 unsigned NewOpcode = N0.getOpcode() == ISD::AND ? ISD::OR : ISD::AND;
5897 LHS = DAG.getNode(ISD::XOR, SDLoc(LHS), VT, LHS, N1); // LHS = ~LHS
5898 RHS = DAG.getNode(ISD::XOR, SDLoc(RHS), VT, RHS, N1); // RHS = ~RHS
5899 AddToWorklist(LHS.getNode()); AddToWorklist(RHS.getNode());
5900 return DAG.getNode(NewOpcode, SDLoc(N), VT, LHS, RHS);
5901 }
5902 }
5903 // fold (not (or x, y)) -> (and (not x), (not y)) iff x or y are constants
5904 if (isAllOnesConstant(N1) && N0.hasOneUse() &&
5905 (N0.getOpcode() == ISD::OR || N0.getOpcode() == ISD::AND)) {
5906 SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1);
5907 if (isa<ConstantSDNode>(RHS) || isa<ConstantSDNode>(LHS)) {
5908 unsigned NewOpcode = N0.getOpcode() == ISD::AND ? ISD::OR : ISD::AND;
5909 LHS = DAG.getNode(ISD::XOR, SDLoc(LHS), VT, LHS, N1); // LHS = ~LHS
5910 RHS = DAG.getNode(ISD::XOR, SDLoc(RHS), VT, RHS, N1); // RHS = ~RHS
5911 AddToWorklist(LHS.getNode()); AddToWorklist(RHS.getNode());
5912 return DAG.getNode(NewOpcode, SDLoc(N), VT, LHS, RHS);
5913 }
5914 }
5915 // fold (xor (and x, y), y) -> (and (not x), y)
5916 if (N0.getOpcode() == ISD::AND && N0.getNode()->hasOneUse() &&
5917 N0->getOperand(1) == N1) {
5918 SDValue X = N0->getOperand(0);
5919 SDValue NotX = DAG.getNOT(SDLoc(X), X, VT);
5920 AddToWorklist(NotX.getNode());
5921 return DAG.getNode(ISD::AND, SDLoc(N), VT, NotX, N1);
5922 }
5923
5924 // fold Y = sra (X, size(X)-1); xor (add (X, Y), Y) -> (abs X)
5925 if (TLI.isOperationLegalOrCustom(ISD::ABS, VT)) {
5926 SDValue A = N0.getOpcode() == ISD::ADD ? N0 : N1;
5927 SDValue S = N0.getOpcode() == ISD::SRA ? N0 : N1;
5928 if (A.getOpcode() == ISD::ADD && S.getOpcode() == ISD::SRA) {
5929 SDValue A0 = A.getOperand(0), A1 = A.getOperand(1);
5930 SDValue S0 = S.getOperand(0);
5931 if ((A0 == S && A1 == S0) || (A1 == S && A0 == S0)) {
5932 unsigned OpSizeInBits = VT.getScalarSizeInBits();
5933 if (ConstantSDNode *C = isConstOrConstSplat(S.getOperand(1)))
5934 if (C->getAPIntValue() == (OpSizeInBits - 1))
5935 return DAG.getNode(ISD::ABS, SDLoc(N), VT, S0);
5936 }
5937 }
5938 }
5939
5940 // fold (xor x, x) -> 0
5941 if (N0 == N1)
5942 return tryFoldToZero(SDLoc(N), TLI, VT, DAG, LegalOperations, LegalTypes);
5943
5944 // fold (xor (shl 1, x), -1) -> (rotl ~1, x)
5945 // Here is a concrete example of this equivalence: