Bug Summary

File:lib/CodeGen/SelectionDAG/DAGCombiner.cpp
Warning:line 391, column 32
The result of the right shift is undefined due to shifting by '64', which is greater or equal to the width of type 'llvm::APInt::WordType'

Annotated Source Code

[?] Use j/k keys for keyboard navigation

/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

1//===- DAGCombiner.cpp - Implement a DAG node combiner --------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This pass combines dag nodes to form fewer, simpler DAG nodes. It can be run
11// both before and after the DAG is legalized.
12//
13// This pass is not a substitute for the LLVM IR instcombine pass. This pass is
14// primarily intended to handle simplification opportunities that are implicit
15// in the LLVM IR and exposed by the various codegen lowering phases.
16//
17//===----------------------------------------------------------------------===//
18
19#include "llvm/ADT/APFloat.h"
20#include "llvm/ADT/APInt.h"
21#include "llvm/ADT/ArrayRef.h"
22#include "llvm/ADT/DenseMap.h"
23#include "llvm/ADT/None.h"
24#include "llvm/ADT/Optional.h"
25#include "llvm/ADT/STLExtras.h"
26#include "llvm/ADT/SetVector.h"
27#include "llvm/ADT/SmallBitVector.h"
28#include "llvm/ADT/SmallPtrSet.h"
29#include "llvm/ADT/SmallSet.h"
30#include "llvm/ADT/SmallVector.h"
31#include "llvm/ADT/Statistic.h"
32#include "llvm/Analysis/AliasAnalysis.h"
33#include "llvm/Analysis/MemoryLocation.h"
34#include "llvm/CodeGen/DAGCombine.h"
35#include "llvm/CodeGen/ISDOpcodes.h"
36#include "llvm/CodeGen/MachineFrameInfo.h"
37#include "llvm/CodeGen/MachineFunction.h"
38#include "llvm/CodeGen/MachineMemOperand.h"
39#include "llvm/CodeGen/MachineValueType.h"
40#include "llvm/CodeGen/RuntimeLibcalls.h"
41#include "llvm/CodeGen/SelectionDAG.h"
42#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
43#include "llvm/CodeGen/SelectionDAGNodes.h"
44#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
45#include "llvm/CodeGen/TargetLowering.h"
46#include "llvm/CodeGen/TargetRegisterInfo.h"
47#include "llvm/CodeGen/TargetSubtargetInfo.h"
48#include "llvm/CodeGen/ValueTypes.h"
49#include "llvm/IR/Attributes.h"
50#include "llvm/IR/Constant.h"
51#include "llvm/IR/DataLayout.h"
52#include "llvm/IR/DerivedTypes.h"
53#include "llvm/IR/Function.h"
54#include "llvm/IR/LLVMContext.h"
55#include "llvm/IR/Metadata.h"
56#include "llvm/Support/Casting.h"
57#include "llvm/Support/CodeGen.h"
58#include "llvm/Support/CommandLine.h"
59#include "llvm/Support/Compiler.h"
60#include "llvm/Support/Debug.h"
61#include "llvm/Support/ErrorHandling.h"
62#include "llvm/Support/KnownBits.h"
63#include "llvm/Support/MathExtras.h"
64#include "llvm/Support/raw_ostream.h"
65#include "llvm/Target/TargetMachine.h"
66#include "llvm/Target/TargetOptions.h"
67#include <algorithm>
68#include <cassert>
69#include <cstdint>
70#include <functional>
71#include <iterator>
72#include <string>
73#include <tuple>
74#include <utility>
75#include <vector>
76
77using namespace llvm;
78
79#define DEBUG_TYPE"dagcombine" "dagcombine"
80
81STATISTIC(NodesCombined , "Number of dag nodes combined")static llvm::Statistic NodesCombined = {"dagcombine", "NodesCombined"
, "Number of dag nodes combined", {0}, false}
;
82STATISTIC(PreIndexedNodes , "Number of pre-indexed nodes created")static llvm::Statistic PreIndexedNodes = {"dagcombine", "PreIndexedNodes"
, "Number of pre-indexed nodes created", {0}, false}
;
83STATISTIC(PostIndexedNodes, "Number of post-indexed nodes created")static llvm::Statistic PostIndexedNodes = {"dagcombine", "PostIndexedNodes"
, "Number of post-indexed nodes created", {0}, false}
;
84STATISTIC(OpsNarrowed , "Number of load/op/store narrowed")static llvm::Statistic OpsNarrowed = {"dagcombine", "OpsNarrowed"
, "Number of load/op/store narrowed", {0}, false}
;
85STATISTIC(LdStFP2Int , "Number of fp load/store pairs transformed to int")static llvm::Statistic LdStFP2Int = {"dagcombine", "LdStFP2Int"
, "Number of fp load/store pairs transformed to int", {0}, false
}
;
86STATISTIC(SlicedLoads, "Number of load sliced")static llvm::Statistic SlicedLoads = {"dagcombine", "SlicedLoads"
, "Number of load sliced", {0}, false}
;
87
88static cl::opt<bool>
89CombinerGlobalAA("combiner-global-alias-analysis", cl::Hidden,
90 cl::desc("Enable DAG combiner's use of IR alias analysis"));
91
92static cl::opt<bool>
93UseTBAA("combiner-use-tbaa", cl::Hidden, cl::init(true),
94 cl::desc("Enable DAG combiner's use of TBAA"));
95
96#ifndef NDEBUG
97static cl::opt<std::string>
98CombinerAAOnlyFunc("combiner-aa-only-func", cl::Hidden,
99 cl::desc("Only use DAG-combiner alias analysis in this"
100 " function"));
101#endif
102
103/// Hidden option to stress test load slicing, i.e., when this option
104/// is enabled, load slicing bypasses most of its profitability guards.
105static cl::opt<bool>
106StressLoadSlicing("combiner-stress-load-slicing", cl::Hidden,
107 cl::desc("Bypass the profitability model of load slicing"),
108 cl::init(false));
109
110static cl::opt<bool>
111 MaySplitLoadIndex("combiner-split-load-index", cl::Hidden, cl::init(true),
112 cl::desc("DAG combiner may split indexing from loads"));
113
114namespace {
115
116 class DAGCombiner {
117 SelectionDAG &DAG;
118 const TargetLowering &TLI;
119 CombineLevel Level;
120 CodeGenOpt::Level OptLevel;
121 bool LegalOperations = false;
122 bool LegalTypes = false;
123 bool ForCodeSize;
124
125 /// \brief Worklist of all of the nodes that need to be simplified.
126 ///
127 /// This must behave as a stack -- new nodes to process are pushed onto the
128 /// back and when processing we pop off of the back.
129 ///
130 /// The worklist will not contain duplicates but may contain null entries
131 /// due to nodes being deleted from the underlying DAG.
132 SmallVector<SDNode *, 64> Worklist;
133
134 /// \brief Mapping from an SDNode to its position on the worklist.
135 ///
136 /// This is used to find and remove nodes from the worklist (by nulling
137 /// them) when they are deleted from the underlying DAG. It relies on
138 /// stable indices of nodes within the worklist.
139 DenseMap<SDNode *, unsigned> WorklistMap;
140
141 /// \brief Set of nodes which have been combined (at least once).
142 ///
143 /// This is used to allow us to reliably add any operands of a DAG node
144 /// which have not yet been combined to the worklist.
145 SmallPtrSet<SDNode *, 32> CombinedNodes;
146
147 // AA - Used for DAG load/store alias analysis.
148 AliasAnalysis *AA;
149
150 /// When an instruction is simplified, add all users of the instruction to
151 /// the work lists because they might get more simplified now.
152 void AddUsersToWorklist(SDNode *N) {
153 for (SDNode *Node : N->uses())
154 AddToWorklist(Node);
155 }
156
157 /// Call the node-specific routine that folds each particular type of node.
158 SDValue visit(SDNode *N);
159
160 public:
161 DAGCombiner(SelectionDAG &D, AliasAnalysis *AA, CodeGenOpt::Level OL)
162 : DAG(D), TLI(D.getTargetLoweringInfo()), Level(BeforeLegalizeTypes),
163 OptLevel(OL), AA(AA) {
164 ForCodeSize = DAG.getMachineFunction().getFunction().optForSize();
165
166 MaximumLegalStoreInBits = 0;
167 for (MVT VT : MVT::all_valuetypes())
168 if (EVT(VT).isSimple() && VT != MVT::Other &&
169 TLI.isTypeLegal(EVT(VT)) &&
170 VT.getSizeInBits() >= MaximumLegalStoreInBits)
171 MaximumLegalStoreInBits = VT.getSizeInBits();
172 }
173
174 /// Add to the worklist making sure its instance is at the back (next to be
175 /// processed.)
176 void AddToWorklist(SDNode *N) {
177 assert(N->getOpcode() != ISD::DELETED_NODE &&(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& "Deleted Node added to Worklist") ? void (0) : __assert_fail
("N->getOpcode() != ISD::DELETED_NODE && \"Deleted Node added to Worklist\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 178, __extension__ __PRETTY_FUNCTION__))
178 "Deleted Node added to Worklist")(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& "Deleted Node added to Worklist") ? void (0) : __assert_fail
("N->getOpcode() != ISD::DELETED_NODE && \"Deleted Node added to Worklist\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 178, __extension__ __PRETTY_FUNCTION__))
;
179
180 // Skip handle nodes as they can't usefully be combined and confuse the
181 // zero-use deletion strategy.
182 if (N->getOpcode() == ISD::HANDLENODE)
183 return;
184
185 if (WorklistMap.insert(std::make_pair(N, Worklist.size())).second)
186 Worklist.push_back(N);
187 }
188
189 /// Remove all instances of N from the worklist.
190 void removeFromWorklist(SDNode *N) {
191 CombinedNodes.erase(N);
192
193 auto It = WorklistMap.find(N);
194 if (It == WorklistMap.end())
195 return; // Not in the worklist.
196
197 // Null out the entry rather than erasing it to avoid a linear operation.
198 Worklist[It->second] = nullptr;
199 WorklistMap.erase(It);
200 }
201
202 void deleteAndRecombine(SDNode *N);
203 bool recursivelyDeleteUnusedNodes(SDNode *N);
204
205 /// Replaces all uses of the results of one DAG node with new values.
206 SDValue CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
207 bool AddTo = true);
208
209 /// Replaces all uses of the results of one DAG node with new values.
210 SDValue CombineTo(SDNode *N, SDValue Res, bool AddTo = true) {
211 return CombineTo(N, &Res, 1, AddTo);
212 }
213
214 /// Replaces all uses of the results of one DAG node with new values.
215 SDValue CombineTo(SDNode *N, SDValue Res0, SDValue Res1,
216 bool AddTo = true) {
217 SDValue To[] = { Res0, Res1 };
218 return CombineTo(N, To, 2, AddTo);
219 }
220
221 void CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO);
222
223 private:
224 unsigned MaximumLegalStoreInBits;
225
226 /// Check the specified integer node value to see if it can be simplified or
227 /// if things it uses can be simplified by bit propagation.
228 /// If so, return true.
229 bool SimplifyDemandedBits(SDValue Op) {
230 unsigned BitWidth = Op.getScalarValueSizeInBits();
231 APInt Demanded = APInt::getAllOnesValue(BitWidth);
232 return SimplifyDemandedBits(Op, Demanded);
233 }
234
235 bool SimplifyDemandedBits(SDValue Op, const APInt &Demanded);
236
237 bool CombineToPreIndexedLoadStore(SDNode *N);
238 bool CombineToPostIndexedLoadStore(SDNode *N);
239 SDValue SplitIndexingFromLoad(LoadSDNode *LD);
240 bool SliceUpLoad(SDNode *N);
241
242 /// \brief Replace an ISD::EXTRACT_VECTOR_ELT of a load with a narrowed
243 /// load.
244 ///
245 /// \param EVE ISD::EXTRACT_VECTOR_ELT to be replaced.
246 /// \param InVecVT type of the input vector to EVE with bitcasts resolved.
247 /// \param EltNo index of the vector element to load.
248 /// \param OriginalLoad load that EVE came from to be replaced.
249 /// \returns EVE on success SDValue() on failure.
250 SDValue ReplaceExtractVectorEltOfLoadWithNarrowedLoad(
251 SDNode *EVE, EVT InVecVT, SDValue EltNo, LoadSDNode *OriginalLoad);
252 void ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad);
253 SDValue PromoteOperand(SDValue Op, EVT PVT, bool &Replace);
254 SDValue SExtPromoteOperand(SDValue Op, EVT PVT);
255 SDValue ZExtPromoteOperand(SDValue Op, EVT PVT);
256 SDValue PromoteIntBinOp(SDValue Op);
257 SDValue PromoteIntShiftOp(SDValue Op);
258 SDValue PromoteExtend(SDValue Op);
259 bool PromoteLoad(SDValue Op);
260
261 void ExtendSetCCUses(const SmallVectorImpl<SDNode *> &SetCCs, SDValue Trunc,
262 SDValue ExtLoad, const SDLoc &DL,
263 ISD::NodeType ExtType);
264
265 /// Call the node-specific routine that knows how to fold each
266 /// particular type of node. If that doesn't do anything, try the
267 /// target-specific DAG combines.
268 SDValue combine(SDNode *N);
269
270 // Visitation implementation - Implement dag node combining for different
271 // node types. The semantics are as follows:
272 // Return Value:
273 // SDValue.getNode() == 0 - No change was made
274 // SDValue.getNode() == N - N was replaced, is dead and has been handled.
275 // otherwise - N should be replaced by the returned Operand.
276 //
277 SDValue visitTokenFactor(SDNode *N);
278 SDValue visitMERGE_VALUES(SDNode *N);
279 SDValue visitADD(SDNode *N);
280 SDValue visitADDLike(SDValue N0, SDValue N1, SDNode *LocReference);
281 SDValue visitSUB(SDNode *N);
282 SDValue visitADDC(SDNode *N);
283 SDValue visitUADDO(SDNode *N);
284 SDValue visitUADDOLike(SDValue N0, SDValue N1, SDNode *N);
285 SDValue visitSUBC(SDNode *N);
286 SDValue visitUSUBO(SDNode *N);
287 SDValue visitADDE(SDNode *N);
288 SDValue visitADDCARRY(SDNode *N);
289 SDValue visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn, SDNode *N);
290 SDValue visitSUBE(SDNode *N);
291 SDValue visitSUBCARRY(SDNode *N);
292 SDValue visitMUL(SDNode *N);
293 SDValue useDivRem(SDNode *N);
294 SDValue visitSDIV(SDNode *N);
295 SDValue visitUDIV(SDNode *N);
296 SDValue visitREM(SDNode *N);
297 SDValue visitMULHU(SDNode *N);
298 SDValue visitMULHS(SDNode *N);
299 SDValue visitSMUL_LOHI(SDNode *N);
300 SDValue visitUMUL_LOHI(SDNode *N);
301 SDValue visitSMULO(SDNode *N);
302 SDValue visitUMULO(SDNode *N);
303 SDValue visitIMINMAX(SDNode *N);
304 SDValue visitAND(SDNode *N);
305 SDValue visitANDLike(SDValue N0, SDValue N1, SDNode *LocReference);
306 SDValue visitOR(SDNode *N);
307 SDValue visitORLike(SDValue N0, SDValue N1, SDNode *LocReference);
308 SDValue visitXOR(SDNode *N);
309 SDValue SimplifyVBinOp(SDNode *N);
310 SDValue visitSHL(SDNode *N);
311 SDValue visitSRA(SDNode *N);
312 SDValue visitSRL(SDNode *N);
313 SDValue visitRotate(SDNode *N);
314 SDValue visitABS(SDNode *N);
315 SDValue visitBSWAP(SDNode *N);
316 SDValue visitBITREVERSE(SDNode *N);
317 SDValue visitCTLZ(SDNode *N);
318 SDValue visitCTLZ_ZERO_UNDEF(SDNode *N);
319 SDValue visitCTTZ(SDNode *N);
320 SDValue visitCTTZ_ZERO_UNDEF(SDNode *N);
321 SDValue visitCTPOP(SDNode *N);
322 SDValue visitSELECT(SDNode *N);
323 SDValue visitVSELECT(SDNode *N);
324 SDValue visitSELECT_CC(SDNode *N);
325 SDValue visitSETCC(SDNode *N);
326 SDValue visitSETCCE(SDNode *N);
327 SDValue visitSETCCCARRY(SDNode *N);
328 SDValue visitSIGN_EXTEND(SDNode *N);
329 SDValue visitZERO_EXTEND(SDNode *N);
330 SDValue visitANY_EXTEND(SDNode *N);
331 SDValue visitAssertExt(SDNode *N);
332 SDValue visitSIGN_EXTEND_INREG(SDNode *N);
333 SDValue visitSIGN_EXTEND_VECTOR_INREG(SDNode *N);
334 SDValue visitZERO_EXTEND_VECTOR_INREG(SDNode *N);
335 SDValue visitTRUNCATE(SDNode *N);
336 SDValue visitBITCAST(SDNode *N);
337 SDValue visitBUILD_PAIR(SDNode *N);
338 SDValue visitFADD(SDNode *N);
339 SDValue visitFSUB(SDNode *N);
340 SDValue visitFMUL(SDNode *N);
341 SDValue visitFMA(SDNode *N);
342 SDValue visitFDIV(SDNode *N);
343 SDValue visitFREM(SDNode *N);
344 SDValue visitFSQRT(SDNode *N);
345 SDValue visitFCOPYSIGN(SDNode *N);
346 SDValue visitSINT_TO_FP(SDNode *N);
347 SDValue visitUINT_TO_FP(SDNode *N);
348 SDValue visitFP_TO_SINT(SDNode *N);
349 SDValue visitFP_TO_UINT(SDNode *N);
350 SDValue visitFP_ROUND(SDNode *N);
351 SDValue visitFP_ROUND_INREG(SDNode *N);
352 SDValue visitFP_EXTEND(SDNode *N);
353 SDValue visitFNEG(SDNode *N);
354 SDValue visitFABS(SDNode *N);
355 SDValue visitFCEIL(SDNode *N);
356 SDValue visitFTRUNC(SDNode *N);
357 SDValue visitFFLOOR(SDNode *N);
358 SDValue visitFMINNUM(SDNode *N);
359 SDValue visitFMAXNUM(SDNode *N);
360 SDValue visitBRCOND(SDNode *N);
361 SDValue visitBR_CC(SDNode *N);
362 SDValue visitLOAD(SDNode *N);
363
364 SDValue replaceStoreChain(StoreSDNode *ST, SDValue BetterChain);
365 SDValue replaceStoreOfFPConstant(StoreSDNode *ST);
366
367 SDValue visitSTORE(SDNode *N);
368 SDValue visitINSERT_VECTOR_ELT(SDNode *N);
369 SDValue visitEXTRACT_VECTOR_ELT(SDNode *N);
370 SDValue visitBUILD_VECTOR(SDNode *N);
371 SDValue visitCONCAT_VECTORS(SDNode *N);
372 SDValue visitEXTRACT_SUBVECTOR(SDNode *N);
373 SDValue visitVECTOR_SHUFFLE(SDNode *N);
374 SDValue visitSCALAR_TO_VECTOR(SDNode *N);
375 SDValue visitINSERT_SUBVECTOR(SDNode *N);
376 SDValue visitMLOAD(SDNode *N);
377 SDValue visitMSTORE(SDNode *N);
378 SDValue visitMGATHER(SDNode *N);
379 SDValue visitMSCATTER(SDNode *N);
380 SDValue visitFP_TO_FP16(SDNode *N);
381 SDValue visitFP16_TO_FP(SDNode *N);
382
383 SDValue visitFADDForFMACombine(SDNode *N);
384 SDValue visitFSUBForFMACombine(SDNode *N);
385 SDValue visitFMULForFMADistributiveCombine(SDNode *N);
386
387 SDValue XformToShuffleWithZero(SDNode *N);
388 SDValue ReassociateOps(unsigned Opc, const SDLoc &DL, SDValue LHS,
389 SDValue RHS);
390
391 SDValue visitShiftByConstant(SDNode *N, ConstantSDNode *Amt);
392
393 SDValue foldSelectOfConstants(SDNode *N);
394 SDValue foldVSelectOfConstants(SDNode *N);
395 SDValue foldBinOpIntoSelect(SDNode *BO);
396 bool SimplifySelectOps(SDNode *SELECT, SDValue LHS, SDValue RHS);
397 SDValue SimplifyBinOpWithSameOpcodeHands(SDNode *N);
398 SDValue SimplifySelect(const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2);
399 SDValue SimplifySelectCC(const SDLoc &DL, SDValue N0, SDValue N1,
400 SDValue N2, SDValue N3, ISD::CondCode CC,
401 bool NotExtCompare = false);
402 SDValue foldSelectCCToShiftAnd(const SDLoc &DL, SDValue N0, SDValue N1,
403 SDValue N2, SDValue N3, ISD::CondCode CC);
404 SDValue foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1,
405 const SDLoc &DL);
406 SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
407 const SDLoc &DL, bool foldBooleans = true);
408
409 bool isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
410 SDValue &CC) const;
411 bool isOneUseSetCC(SDValue N) const;
412
413 SDValue SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
414 unsigned HiOp);
415 SDValue CombineConsecutiveLoads(SDNode *N, EVT VT);
416 SDValue CombineExtLoad(SDNode *N);
417 SDValue combineRepeatedFPDivisors(SDNode *N);
418 SDValue combineInsertEltToShuffle(SDNode *N, unsigned InsIndex);
419 SDValue ConstantFoldBITCASTofBUILD_VECTOR(SDNode *, EVT);
420 SDValue BuildSDIV(SDNode *N);
421 SDValue BuildSDIVPow2(SDNode *N);
422 SDValue BuildUDIV(SDNode *N);
423 SDValue BuildLogBase2(SDValue Op, const SDLoc &DL);
424 SDValue BuildReciprocalEstimate(SDValue Op, SDNodeFlags Flags);
425 SDValue buildRsqrtEstimate(SDValue Op, SDNodeFlags Flags);
426 SDValue buildSqrtEstimate(SDValue Op, SDNodeFlags Flags);
427 SDValue buildSqrtEstimateImpl(SDValue Op, SDNodeFlags Flags, bool Recip);
428 SDValue buildSqrtNROneConst(SDValue Op, SDValue Est, unsigned Iterations,
429 SDNodeFlags Flags, bool Reciprocal);
430 SDValue buildSqrtNRTwoConst(SDValue Op, SDValue Est, unsigned Iterations,
431 SDNodeFlags Flags, bool Reciprocal);
432 SDValue MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
433 bool DemandHighBits = true);
434 SDValue MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1);
435 SDNode *MatchRotatePosNeg(SDValue Shifted, SDValue Pos, SDValue Neg,
436 SDValue InnerPos, SDValue InnerNeg,
437 unsigned PosOpcode, unsigned NegOpcode,
438 const SDLoc &DL);
439 SDNode *MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL);
440 SDValue MatchLoadCombine(SDNode *N);
441 SDValue ReduceLoadWidth(SDNode *N);
442 SDValue ReduceLoadOpStoreWidth(SDNode *N);
443 SDValue splitMergedValStore(StoreSDNode *ST);
444 SDValue TransformFPLoadStorePair(SDNode *N);
445 SDValue reduceBuildVecExtToExtBuildVec(SDNode *N);
446 SDValue reduceBuildVecConvertToConvertBuildVec(SDNode *N);
447 SDValue reduceBuildVecToShuffle(SDNode *N);
448 SDValue createBuildVecShuffle(const SDLoc &DL, SDNode *N,
449 ArrayRef<int> VectorMask, SDValue VecIn1,
450 SDValue VecIn2, unsigned LeftIdx);
451 SDValue matchVSelectOpSizesWithSetCC(SDNode *N);
452
453 /// Walk up chain skipping non-aliasing memory nodes,
454 /// looking for aliasing nodes and adding them to the Aliases vector.
455 void GatherAllAliases(SDNode *N, SDValue OriginalChain,
456 SmallVectorImpl<SDValue> &Aliases);
457
458 /// Return true if there is any possibility that the two addresses overlap.
459 bool isAlias(LSBaseSDNode *Op0, LSBaseSDNode *Op1) const;
460
461 /// Walk up chain skipping non-aliasing memory nodes, looking for a better
462 /// chain (aliasing node.)
463 SDValue FindBetterChain(SDNode *N, SDValue Chain);
464
465 /// Try to replace a store and any possibly adjacent stores on
466 /// consecutive chains with better chains. Return true only if St is
467 /// replaced.
468 ///
469 /// Notice that other chains may still be replaced even if the function
470 /// returns false.
471 bool findBetterNeighborChains(StoreSDNode *St);
472
473 /// Match "(X shl/srl V1) & V2" where V2 may not be present.
474 bool MatchRotateHalf(SDValue Op, SDValue &Shift, SDValue &Mask);
475
476 /// Holds a pointer to an LSBaseSDNode as well as information on where it
477 /// is located in a sequence of memory operations connected by a chain.
478 struct MemOpLink {
479 // Ptr to the mem node.
480 LSBaseSDNode *MemNode;
481
482 // Offset from the base ptr.
483 int64_t OffsetFromBase;
484
485 MemOpLink(LSBaseSDNode *N, int64_t Offset)
486 : MemNode(N), OffsetFromBase(Offset) {}
487 };
488
489 /// This is a helper function for visitMUL to check the profitability
490 /// of folding (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2).
491 /// MulNode is the original multiply, AddNode is (add x, c1),
492 /// and ConstNode is c2.
493 bool isMulAddWithConstProfitable(SDNode *MulNode,
494 SDValue &AddNode,
495 SDValue &ConstNode);
496
497 /// This is a helper function for visitAND and visitZERO_EXTEND. Returns
498 /// true if the (and (load x) c) pattern matches an extload. ExtVT returns
499 /// the type of the loaded value to be extended.
500 bool isAndLoadExtLoad(ConstantSDNode *AndC, LoadSDNode *LoadN,
501 EVT LoadResultTy, EVT &ExtVT);
502
503 /// Helper function to calculate whether the given Load can have its
504 /// width reduced to ExtVT.
505 bool isLegalNarrowLoad(LoadSDNode *LoadN, ISD::LoadExtType ExtType,
506 EVT &ExtVT, unsigned ShAmt = 0);
507
508 /// Used by BackwardsPropagateMask to find suitable loads.
509 bool SearchForAndLoads(SDNode *N, SmallPtrSetImpl<LoadSDNode*> &Loads,
510 SmallPtrSetImpl<SDNode*> &NodeWithConsts,
511 ConstantSDNode *Mask, SDNode *&UncombinedNode);
512 /// Attempt to propagate a given AND node back to load leaves so that they
513 /// can be combined into narrow loads.
514 bool BackwardsPropagateMask(SDNode *N, SelectionDAG &DAG);
515
516 /// Helper function for MergeConsecutiveStores which merges the
517 /// component store chains.
518 SDValue getMergeStoreChains(SmallVectorImpl<MemOpLink> &StoreNodes,
519 unsigned NumStores);
520
521 /// This is a helper function for MergeConsecutiveStores. When the
522 /// source elements of the consecutive stores are all constants or
523 /// all extracted vector elements, try to merge them into one
524 /// larger store introducing bitcasts if necessary. \return True
525 /// if a merged store was created.
526 bool MergeStoresOfConstantsOrVecElts(SmallVectorImpl<MemOpLink> &StoreNodes,
527 EVT MemVT, unsigned NumStores,
528 bool IsConstantSrc, bool UseVector,
529 bool UseTrunc);
530
531 /// This is a helper function for MergeConsecutiveStores. Stores
532 /// that potentially may be merged with St are placed in
533 /// StoreNodes.
534 void getStoreMergeCandidates(StoreSDNode *St,
535 SmallVectorImpl<MemOpLink> &StoreNodes);
536
537 /// Helper function for MergeConsecutiveStores. Checks if
538 /// candidate stores have indirect dependency through their
539 /// operands. \return True if safe to merge.
540 bool checkMergeStoreCandidatesForDependencies(
541 SmallVectorImpl<MemOpLink> &StoreNodes, unsigned NumStores);
542
543 /// Merge consecutive store operations into a wide store.
544 /// This optimization uses wide integers or vectors when possible.
545 /// \return number of stores that were merged into a merged store (the
546 /// affected nodes are stored as a prefix in \p StoreNodes).
547 bool MergeConsecutiveStores(StoreSDNode *N);
548
549 /// \brief Try to transform a truncation where C is a constant:
550 /// (trunc (and X, C)) -> (and (trunc X), (trunc C))
551 ///
552 /// \p N needs to be a truncation and its first operand an AND. Other
553 /// requirements are checked by the function (e.g. that trunc is
554 /// single-use) and if missed an empty SDValue is returned.
555 SDValue distributeTruncateThroughAnd(SDNode *N);
556
557 public:
558 /// Runs the dag combiner on all nodes in the work list
559 void Run(CombineLevel AtLevel);
560
561 SelectionDAG &getDAG() const { return DAG; }
562
563 /// Returns a type large enough to hold any valid shift amount - before type
564 /// legalization these can be huge.
565 EVT getShiftAmountTy(EVT LHSTy) {
566 assert(LHSTy.isInteger() && "Shift amount is not an integer type!")(static_cast <bool> (LHSTy.isInteger() && "Shift amount is not an integer type!"
) ? void (0) : __assert_fail ("LHSTy.isInteger() && \"Shift amount is not an integer type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 566, __extension__ __PRETTY_FUNCTION__))
;
567 if (LHSTy.isVector())
568 return LHSTy;
569 auto &DL = DAG.getDataLayout();
570 return LegalTypes ? TLI.getScalarShiftAmountTy(DL, LHSTy)
571 : TLI.getPointerTy(DL);
572 }
573
574 /// This method returns true if we are running before type legalization or
575 /// if the specified VT is legal.
576 bool isTypeLegal(const EVT &VT) {
577 if (!LegalTypes) return true;
578 return TLI.isTypeLegal(VT);
579 }
580
581 /// Convenience wrapper around TargetLowering::getSetCCResultType
582 EVT getSetCCResultType(EVT VT) const {
583 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
584 }
585 };
586
587/// This class is a DAGUpdateListener that removes any deleted
588/// nodes from the worklist.
589class WorklistRemover : public SelectionDAG::DAGUpdateListener {
590 DAGCombiner &DC;
591
592public:
593 explicit WorklistRemover(DAGCombiner &dc)
594 : SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
595
596 void NodeDeleted(SDNode *N, SDNode *E) override {
597 DC.removeFromWorklist(N);
598 }
599};
600
601} // end anonymous namespace
602
603//===----------------------------------------------------------------------===//
604// TargetLowering::DAGCombinerInfo implementation
605//===----------------------------------------------------------------------===//
606
607void TargetLowering::DAGCombinerInfo::AddToWorklist(SDNode *N) {
608 ((DAGCombiner*)DC)->AddToWorklist(N);
609}
610
611SDValue TargetLowering::DAGCombinerInfo::
612CombineTo(SDNode *N, ArrayRef<SDValue> To, bool AddTo) {
613 return ((DAGCombiner*)DC)->CombineTo(N, &To[0], To.size(), AddTo);
614}
615
616SDValue TargetLowering::DAGCombinerInfo::
617CombineTo(SDNode *N, SDValue Res, bool AddTo) {
618 return ((DAGCombiner*)DC)->CombineTo(N, Res, AddTo);
619}
620
621SDValue TargetLowering::DAGCombinerInfo::
622CombineTo(SDNode *N, SDValue Res0, SDValue Res1, bool AddTo) {
623 return ((DAGCombiner*)DC)->CombineTo(N, Res0, Res1, AddTo);
624}
625
626void TargetLowering::DAGCombinerInfo::
627CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
628 return ((DAGCombiner*)DC)->CommitTargetLoweringOpt(TLO);
629}
630
631//===----------------------------------------------------------------------===//
632// Helper Functions
633//===----------------------------------------------------------------------===//
634
635void DAGCombiner::deleteAndRecombine(SDNode *N) {
636 removeFromWorklist(N);
637
638 // If the operands of this node are only used by the node, they will now be
639 // dead. Make sure to re-visit them and recursively delete dead nodes.
640 for (const SDValue &Op : N->ops())
641 // For an operand generating multiple values, one of the values may
642 // become dead allowing further simplification (e.g. split index
643 // arithmetic from an indexed load).
644 if (Op->hasOneUse() || Op->getNumValues() > 1)
645 AddToWorklist(Op.getNode());
646
647 DAG.DeleteNode(N);
648}
649
650/// Return 1 if we can compute the negated form of the specified expression for
651/// the same cost as the expression itself, or 2 if we can compute the negated
652/// form more cheaply than the expression itself.
653static char isNegatibleForFree(SDValue Op, bool LegalOperations,
654 const TargetLowering &TLI,
655 const TargetOptions *Options,
656 unsigned Depth = 0) {
657 // fneg is removable even if it has multiple uses.
658 if (Op.getOpcode() == ISD::FNEG) return 2;
659
660 // Don't allow anything with multiple uses.
661 if (!Op.hasOneUse()) return 0;
662
663 // Don't recurse exponentially.
664 if (Depth > 6) return 0;
665
666 switch (Op.getOpcode()) {
667 default: return false;
668 case ISD::ConstantFP: {
669 if (!LegalOperations)
670 return 1;
671
672 // Don't invert constant FP values after legalization unless the target says
673 // the negated constant is legal.
674 EVT VT = Op.getValueType();
675 return TLI.isOperationLegal(ISD::ConstantFP, VT) ||
676 TLI.isFPImmLegal(neg(cast<ConstantFPSDNode>(Op)->getValueAPF()), VT);
677 }
678 case ISD::FADD:
679 // FIXME: determine better conditions for this xform.
680 if (!Options->UnsafeFPMath) return 0;
681
682 // After operation legalization, it might not be legal to create new FSUBs.
683 if (LegalOperations &&
684 !TLI.isOperationLegalOrCustom(ISD::FSUB, Op.getValueType()))
685 return 0;
686
687 // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
688 if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI,
689 Options, Depth + 1))
690 return V;
691 // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
692 return isNegatibleForFree(Op.getOperand(1), LegalOperations, TLI, Options,
693 Depth + 1);
694 case ISD::FSUB:
695 // We can't turn -(A-B) into B-A when we honor signed zeros.
696 if (!Options->NoSignedZerosFPMath &&
697 !Op.getNode()->getFlags().hasNoSignedZeros())
698 return 0;
699
700 // fold (fneg (fsub A, B)) -> (fsub B, A)
701 return 1;
702
703 case ISD::FMUL:
704 case ISD::FDIV:
705 if (Options->HonorSignDependentRoundingFPMath()) return 0;
706
707 // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y) or (fmul X, (fneg Y))
708 if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI,
709 Options, Depth + 1))
710 return V;
711
712 return isNegatibleForFree(Op.getOperand(1), LegalOperations, TLI, Options,
713 Depth + 1);
714
715 case ISD::FP_EXTEND:
716 case ISD::FP_ROUND:
717 case ISD::FSIN:
718 return isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI, Options,
719 Depth + 1);
720 }
721}
722
723/// If isNegatibleForFree returns true, return the newly negated expression.
724static SDValue GetNegatedExpression(SDValue Op, SelectionDAG &DAG,
725 bool LegalOperations, unsigned Depth = 0) {
726 const TargetOptions &Options = DAG.getTarget().Options;
727 // fneg is removable even if it has multiple uses.
728 if (Op.getOpcode() == ISD::FNEG) return Op.getOperand(0);
729
730 // Don't allow anything with multiple uses.
731 assert(Op.hasOneUse() && "Unknown reuse!")(static_cast <bool> (Op.hasOneUse() && "Unknown reuse!"
) ? void (0) : __assert_fail ("Op.hasOneUse() && \"Unknown reuse!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 731, __extension__ __PRETTY_FUNCTION__))
;
732
733 assert(Depth <= 6 && "GetNegatedExpression doesn't match isNegatibleForFree")(static_cast <bool> (Depth <= 6 && "GetNegatedExpression doesn't match isNegatibleForFree"
) ? void (0) : __assert_fail ("Depth <= 6 && \"GetNegatedExpression doesn't match isNegatibleForFree\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 733, __extension__ __PRETTY_FUNCTION__))
;
734
735 const SDNodeFlags Flags = Op.getNode()->getFlags();
736
737 switch (Op.getOpcode()) {
738 default: llvm_unreachable("Unknown code")::llvm::llvm_unreachable_internal("Unknown code", "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 738)
;
739 case ISD::ConstantFP: {
740 APFloat V = cast<ConstantFPSDNode>(Op)->getValueAPF();
741 V.changeSign();
742 return DAG.getConstantFP(V, SDLoc(Op), Op.getValueType());
743 }
744 case ISD::FADD:
745 // FIXME: determine better conditions for this xform.
746 assert(Options.UnsafeFPMath)(static_cast <bool> (Options.UnsafeFPMath) ? void (0) :
__assert_fail ("Options.UnsafeFPMath", "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 746, __extension__ __PRETTY_FUNCTION__))
;
747
748 // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
749 if (isNegatibleForFree(Op.getOperand(0), LegalOperations,
750 DAG.getTargetLoweringInfo(), &Options, Depth+1))
751 return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
752 GetNegatedExpression(Op.getOperand(0), DAG,
753 LegalOperations, Depth+1),
754 Op.getOperand(1), Flags);
755 // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
756 return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
757 GetNegatedExpression(Op.getOperand(1), DAG,
758 LegalOperations, Depth+1),
759 Op.getOperand(0), Flags);
760 case ISD::FSUB:
761 // fold (fneg (fsub 0, B)) -> B
762 if (ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(Op.getOperand(0)))
763 if (N0CFP->isZero())
764 return Op.getOperand(1);
765
766 // fold (fneg (fsub A, B)) -> (fsub B, A)
767 return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
768 Op.getOperand(1), Op.getOperand(0), Flags);
769
770 case ISD::FMUL:
771 case ISD::FDIV:
772 assert(!Options.HonorSignDependentRoundingFPMath())(static_cast <bool> (!Options.HonorSignDependentRoundingFPMath
()) ? void (0) : __assert_fail ("!Options.HonorSignDependentRoundingFPMath()"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 772, __extension__ __PRETTY_FUNCTION__))
;
773
774 // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y)
775 if (isNegatibleForFree(Op.getOperand(0), LegalOperations,
776 DAG.getTargetLoweringInfo(), &Options, Depth+1))
777 return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
778 GetNegatedExpression(Op.getOperand(0), DAG,
779 LegalOperations, Depth+1),
780 Op.getOperand(1), Flags);
781
782 // fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y))
783 return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
784 Op.getOperand(0),
785 GetNegatedExpression(Op.getOperand(1), DAG,
786 LegalOperations, Depth+1), Flags);
787
788 case ISD::FP_EXTEND:
789 case ISD::FSIN:
790 return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
791 GetNegatedExpression(Op.getOperand(0), DAG,
792 LegalOperations, Depth+1));
793 case ISD::FP_ROUND:
794 return DAG.getNode(ISD::FP_ROUND, SDLoc(Op), Op.getValueType(),
795 GetNegatedExpression(Op.getOperand(0), DAG,
796 LegalOperations, Depth+1),
797 Op.getOperand(1));
798 }
799}
800
801// APInts must be the same size for most operations, this helper
802// function zero extends the shorter of the pair so that they match.
803// We provide an Offset so that we can create bitwidths that won't overflow.
804static void zeroExtendToMatch(APInt &LHS, APInt &RHS, unsigned Offset = 0) {
805 unsigned Bits = Offset + std::max(LHS.getBitWidth(), RHS.getBitWidth());
806 LHS = LHS.zextOrSelf(Bits);
807 RHS = RHS.zextOrSelf(Bits);
808}
809
810// Return true if this node is a setcc, or is a select_cc
811// that selects between the target values used for true and false, making it
812// equivalent to a setcc. Also, set the incoming LHS, RHS, and CC references to
813// the appropriate nodes based on the type of node we are checking. This
814// simplifies life a bit for the callers.
815bool DAGCombiner::isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
816 SDValue &CC) const {
817 if (N.getOpcode() == ISD::SETCC) {
818 LHS = N.getOperand(0);
819 RHS = N.getOperand(1);
820 CC = N.getOperand(2);
821 return true;
822 }
823
824 if (N.getOpcode() != ISD::SELECT_CC ||
825 !TLI.isConstTrueVal(N.getOperand(2).getNode()) ||
826 !TLI.isConstFalseVal(N.getOperand(3).getNode()))
827 return false;
828
829 if (TLI.getBooleanContents(N.getValueType()) ==
830 TargetLowering::UndefinedBooleanContent)
831 return false;
832
833 LHS = N.getOperand(0);
834 RHS = N.getOperand(1);
835 CC = N.getOperand(4);
836 return true;
837}
838
839/// Return true if this is a SetCC-equivalent operation with only one use.
840/// If this is true, it allows the users to invert the operation for free when
841/// it is profitable to do so.
842bool DAGCombiner::isOneUseSetCC(SDValue N) const {
843 SDValue N0, N1, N2;
844 if (isSetCCEquivalent(N, N0, N1, N2) && N.getNode()->hasOneUse())
845 return true;
846 return false;
847}
848
849// \brief Returns the SDNode if it is a constant float BuildVector
850// or constant float.
851static SDNode *isConstantFPBuildVectorOrConstantFP(SDValue N) {
852 if (isa<ConstantFPSDNode>(N))
853 return N.getNode();
854 if (ISD::isBuildVectorOfConstantFPSDNodes(N.getNode()))
855 return N.getNode();
856 return nullptr;
857}
858
859// Determines if it is a constant integer or a build vector of constant
860// integers (and undefs).
861// Do not permit build vector implicit truncation.
862static bool isConstantOrConstantVector(SDValue N, bool NoOpaques = false) {
863 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N))
864 return !(Const->isOpaque() && NoOpaques);
865 if (N.getOpcode() != ISD::BUILD_VECTOR)
866 return false;
867 unsigned BitWidth = N.getScalarValueSizeInBits();
868 for (const SDValue &Op : N->op_values()) {
869 if (Op.isUndef())
870 continue;
871 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Op);
872 if (!Const || Const->getAPIntValue().getBitWidth() != BitWidth ||
873 (Const->isOpaque() && NoOpaques))
874 return false;
875 }
876 return true;
877}
878
879// Determines if it is a constant null integer or a splatted vector of a
880// constant null integer (with no undefs).
881// Build vector implicit truncation is not an issue for null values.
882static bool isNullConstantOrNullSplatConstant(SDValue N) {
883 if (ConstantSDNode *Splat = isConstOrConstSplat(N))
884 return Splat->isNullValue();
885 return false;
886}
887
888// Determines if it is a constant integer of one or a splatted vector of a
889// constant integer of one (with no undefs).
890// Do not permit build vector implicit truncation.
891static bool isOneConstantOrOneSplatConstant(SDValue N) {
892 unsigned BitWidth = N.getScalarValueSizeInBits();
893 if (ConstantSDNode *Splat = isConstOrConstSplat(N))
894 return Splat->isOne() && Splat->getAPIntValue().getBitWidth() == BitWidth;
895 return false;
896}
897
898// Determines if it is a constant integer of all ones or a splatted vector of a
899// constant integer of all ones (with no undefs).
900// Do not permit build vector implicit truncation.
901static bool isAllOnesConstantOrAllOnesSplatConstant(SDValue N) {
902 unsigned BitWidth = N.getScalarValueSizeInBits();
903 if (ConstantSDNode *Splat = isConstOrConstSplat(N))
904 return Splat->isAllOnesValue() &&
905 Splat->getAPIntValue().getBitWidth() == BitWidth;
906 return false;
907}
908
909// Determines if a BUILD_VECTOR is composed of all-constants possibly mixed with
910// undef's.
911static bool isAnyConstantBuildVector(const SDNode *N) {
912 return ISD::isBuildVectorOfConstantSDNodes(N) ||
913 ISD::isBuildVectorOfConstantFPSDNodes(N);
914}
915
916// Attempt to match a unary predicate against a scalar/splat constant or
917// every element of a constant BUILD_VECTOR.
918static bool matchUnaryPredicate(SDValue Op,
919 std::function<bool(ConstantSDNode *)> Match) {
920 if (auto *Cst = dyn_cast<ConstantSDNode>(Op))
921 return Match(Cst);
922
923 if (ISD::BUILD_VECTOR != Op.getOpcode())
924 return false;
925
926 EVT SVT = Op.getValueType().getScalarType();
927 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
928 auto *Cst = dyn_cast<ConstantSDNode>(Op.getOperand(i));
929 if (!Cst || Cst->getValueType(0) != SVT || !Match(Cst))
930 return false;
931 }
932 return true;
933}
934
935// Attempt to match a binary predicate against a pair of scalar/splat constants
936// or every element of a pair of constant BUILD_VECTORs.
937static bool matchBinaryPredicate(
938 SDValue LHS, SDValue RHS,
939 std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match) {
940 if (LHS.getValueType() != RHS.getValueType())
941 return false;
942
943 if (auto *LHSCst = dyn_cast<ConstantSDNode>(LHS))
944 if (auto *RHSCst = dyn_cast<ConstantSDNode>(RHS))
945 return Match(LHSCst, RHSCst);
946
947 if (ISD::BUILD_VECTOR != LHS.getOpcode() ||
948 ISD::BUILD_VECTOR != RHS.getOpcode())
949 return false;
950
951 EVT SVT = LHS.getValueType().getScalarType();
952 for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) {
953 auto *LHSCst = dyn_cast<ConstantSDNode>(LHS.getOperand(i));
954 auto *RHSCst = dyn_cast<ConstantSDNode>(RHS.getOperand(i));
955 if (!LHSCst || !RHSCst)
956 return false;
957 if (LHSCst->getValueType(0) != SVT ||
958 LHSCst->getValueType(0) != RHSCst->getValueType(0))
959 return false;
960 if (!Match(LHSCst, RHSCst))
961 return false;
962 }
963 return true;
964}
965
966SDValue DAGCombiner::ReassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0,
967 SDValue N1) {
968 EVT VT = N0.getValueType();
969 if (N0.getOpcode() == Opc) {
970 if (SDNode *L = DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1))) {
971 if (SDNode *R = DAG.isConstantIntBuildVectorOrConstantInt(N1)) {
972 // reassoc. (op (op x, c1), c2) -> (op x, (op c1, c2))
973 if (SDValue OpNode = DAG.FoldConstantArithmetic(Opc, DL, VT, L, R))
974 return DAG.getNode(Opc, DL, VT, N0.getOperand(0), OpNode);
975 return SDValue();
976 }
977 if (N0.hasOneUse()) {
978 // reassoc. (op (op x, c1), y) -> (op (op x, y), c1) iff x+c1 has one
979 // use
980 SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N0.getOperand(0), N1);
981 if (!OpNode.getNode())
982 return SDValue();
983 AddToWorklist(OpNode.getNode());
984 return DAG.getNode(Opc, DL, VT, OpNode, N0.getOperand(1));
985 }
986 }
987 }
988
989 if (N1.getOpcode() == Opc) {
990 if (SDNode *R = DAG.isConstantIntBuildVectorOrConstantInt(N1.getOperand(1))) {
991 if (SDNode *L = DAG.isConstantIntBuildVectorOrConstantInt(N0)) {
992 // reassoc. (op c2, (op x, c1)) -> (op x, (op c1, c2))
993 if (SDValue OpNode = DAG.FoldConstantArithmetic(Opc, DL, VT, R, L))
994 return DAG.getNode(Opc, DL, VT, N1.getOperand(0), OpNode);
995 return SDValue();
996 }
997 if (N1.hasOneUse()) {
998 // reassoc. (op x, (op y, c1)) -> (op (op x, y), c1) iff x+c1 has one
999 // use
1000 SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N0, N1.getOperand(0));
1001 if (!OpNode.getNode())
1002 return SDValue();
1003 AddToWorklist(OpNode.getNode());
1004 return DAG.getNode(Opc, DL, VT, OpNode, N1.getOperand(1));
1005 }
1006 }
1007 }
1008
1009 return SDValue();
1010}
1011
1012SDValue DAGCombiner::CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
1013 bool AddTo) {
1014 assert(N->getNumValues() == NumTo && "Broken CombineTo call!")(static_cast <bool> (N->getNumValues() == NumTo &&
"Broken CombineTo call!") ? void (0) : __assert_fail ("N->getNumValues() == NumTo && \"Broken CombineTo call!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1014, __extension__ __PRETTY_FUNCTION__))
;
1015 ++NodesCombined;
1016 DEBUG(dbgs() << "\nReplacing.1 ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.1 "; N->dump
(&DAG); dbgs() << "\nWith: "; To[0].getNode()->dump
(&DAG); dbgs() << " and " << NumTo-1 <<
" other values\n"; } } while (false)
1017 N->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.1 "; N->dump
(&DAG); dbgs() << "\nWith: "; To[0].getNode()->dump
(&DAG); dbgs() << " and " << NumTo-1 <<
" other values\n"; } } while (false)
1018 dbgs() << "\nWith: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.1 "; N->dump
(&DAG); dbgs() << "\nWith: "; To[0].getNode()->dump
(&DAG); dbgs() << " and " << NumTo-1 <<
" other values\n"; } } while (false)
1019 To[0].getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.1 "; N->dump
(&DAG); dbgs() << "\nWith: "; To[0].getNode()->dump
(&DAG); dbgs() << " and " << NumTo-1 <<
" other values\n"; } } while (false)
1020 dbgs() << " and " << NumTo-1 << " other values\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.1 "; N->dump
(&DAG); dbgs() << "\nWith: "; To[0].getNode()->dump
(&DAG); dbgs() << " and " << NumTo-1 <<
" other values\n"; } } while (false)
;
1021 for (unsigned i = 0, e = NumTo; i != e; ++i)
1022 assert((!To[i].getNode() ||(static_cast <bool> ((!To[i].getNode() || N->getValueType
(i) == To[i].getValueType()) && "Cannot combine value to value of different type!"
) ? void (0) : __assert_fail ("(!To[i].getNode() || N->getValueType(i) == To[i].getValueType()) && \"Cannot combine value to value of different type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1024, __extension__ __PRETTY_FUNCTION__))
1023 N->getValueType(i) == To[i].getValueType()) &&(static_cast <bool> ((!To[i].getNode() || N->getValueType
(i) == To[i].getValueType()) && "Cannot combine value to value of different type!"
) ? void (0) : __assert_fail ("(!To[i].getNode() || N->getValueType(i) == To[i].getValueType()) && \"Cannot combine value to value of different type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1024, __extension__ __PRETTY_FUNCTION__))
1024 "Cannot combine value to value of different type!")(static_cast <bool> ((!To[i].getNode() || N->getValueType
(i) == To[i].getValueType()) && "Cannot combine value to value of different type!"
) ? void (0) : __assert_fail ("(!To[i].getNode() || N->getValueType(i) == To[i].getValueType()) && \"Cannot combine value to value of different type!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1024, __extension__ __PRETTY_FUNCTION__))
;
1025
1026 WorklistRemover DeadNodes(*this);
1027 DAG.ReplaceAllUsesWith(N, To);
1028 if (AddTo) {
1029 // Push the new nodes and any users onto the worklist
1030 for (unsigned i = 0, e = NumTo; i != e; ++i) {
1031 if (To[i].getNode()) {
1032 AddToWorklist(To[i].getNode());
1033 AddUsersToWorklist(To[i].getNode());
1034 }
1035 }
1036 }
1037
1038 // Finally, if the node is now dead, remove it from the graph. The node
1039 // may not be dead if the replacement process recursively simplified to
1040 // something else needing this node.
1041 if (N->use_empty())
1042 deleteAndRecombine(N);
1043 return SDValue(N, 0);
1044}
1045
1046void DAGCombiner::
1047CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
1048 // Replace all uses. If any nodes become isomorphic to other nodes and
1049 // are deleted, make sure to remove them from our worklist.
1050 WorklistRemover DeadNodes(*this);
1051 DAG.ReplaceAllUsesOfValueWith(TLO.Old, TLO.New);
1052
1053 // Push the new node and any (possibly new) users onto the worklist.
1054 AddToWorklist(TLO.New.getNode());
1055 AddUsersToWorklist(TLO.New.getNode());
1056
1057 // Finally, if the node is now dead, remove it from the graph. The node
1058 // may not be dead if the replacement process recursively simplified to
1059 // something else needing this node.
1060 if (TLO.Old.getNode()->use_empty())
1061 deleteAndRecombine(TLO.Old.getNode());
1062}
1063
1064/// Check the specified integer node value to see if it can be simplified or if
1065/// things it uses can be simplified by bit propagation. If so, return true.
1066bool DAGCombiner::SimplifyDemandedBits(SDValue Op, const APInt &Demanded) {
1067 TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations);
1068 KnownBits Known;
1069 if (!TLI.SimplifyDemandedBits(Op, Demanded, Known, TLO))
1070 return false;
1071
1072 // Revisit the node.
1073 AddToWorklist(Op.getNode());
1074
1075 // Replace the old value with the new one.
1076 ++NodesCombined;
1077 DEBUG(dbgs() << "\nReplacing.2 ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.getNode
()->dump(&DAG); dbgs() << "\nWith: "; TLO.New.getNode
()->dump(&DAG); dbgs() << '\n'; } } while (false
)
1078 TLO.Old.getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.getNode
()->dump(&DAG); dbgs() << "\nWith: "; TLO.New.getNode
()->dump(&DAG); dbgs() << '\n'; } } while (false
)
1079 dbgs() << "\nWith: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.getNode
()->dump(&DAG); dbgs() << "\nWith: "; TLO.New.getNode
()->dump(&DAG); dbgs() << '\n'; } } while (false
)
1080 TLO.New.getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.getNode
()->dump(&DAG); dbgs() << "\nWith: "; TLO.New.getNode
()->dump(&DAG); dbgs() << '\n'; } } while (false
)
1081 dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.2 "; TLO.Old.getNode
()->dump(&DAG); dbgs() << "\nWith: "; TLO.New.getNode
()->dump(&DAG); dbgs() << '\n'; } } while (false
)
;
1082
1083 CommitTargetLoweringOpt(TLO);
1084 return true;
1085}
1086
1087void DAGCombiner::ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad) {
1088 SDLoc DL(Load);
1089 EVT VT = Load->getValueType(0);
1090 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, VT, SDValue(ExtLoad, 0));
1091
1092 DEBUG(dbgs() << "\nReplacing.9 ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.9 "; Load->
dump(&DAG); dbgs() << "\nWith: "; Trunc.getNode()->
dump(&DAG); dbgs() << '\n'; } } while (false)
1093 Load->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.9 "; Load->
dump(&DAG); dbgs() << "\nWith: "; Trunc.getNode()->
dump(&DAG); dbgs() << '\n'; } } while (false)
1094 dbgs() << "\nWith: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.9 "; Load->
dump(&DAG); dbgs() << "\nWith: "; Trunc.getNode()->
dump(&DAG); dbgs() << '\n'; } } while (false)
1095 Trunc.getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.9 "; Load->
dump(&DAG); dbgs() << "\nWith: "; Trunc.getNode()->
dump(&DAG); dbgs() << '\n'; } } while (false)
1096 dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nReplacing.9 "; Load->
dump(&DAG); dbgs() << "\nWith: "; Trunc.getNode()->
dump(&DAG); dbgs() << '\n'; } } while (false)
;
1097 WorklistRemover DeadNodes(*this);
1098 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), Trunc);
1099 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), SDValue(ExtLoad, 1));
1100 deleteAndRecombine(Load);
1101 AddToWorklist(Trunc.getNode());
1102}
1103
1104SDValue DAGCombiner::PromoteOperand(SDValue Op, EVT PVT, bool &Replace) {
1105 Replace = false;
1106 SDLoc DL(Op);
1107 if (ISD::isUNINDEXEDLoad(Op.getNode())) {
1108 LoadSDNode *LD = cast<LoadSDNode>(Op);
1109 EVT MemVT = LD->getMemoryVT();
1110 ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD)
1111 ? (TLI.isLoadExtLegal(ISD::ZEXTLOAD, PVT, MemVT) ? ISD::ZEXTLOAD
1112 : ISD::EXTLOAD)
1113 : LD->getExtensionType();
1114 Replace = true;
1115 return DAG.getExtLoad(ExtType, DL, PVT,
1116 LD->getChain(), LD->getBasePtr(),
1117 MemVT, LD->getMemOperand());
1118 }
1119
1120 unsigned Opc = Op.getOpcode();
1121 switch (Opc) {
1122 default: break;
1123 case ISD::AssertSext:
1124 if (SDValue Op0 = SExtPromoteOperand(Op.getOperand(0), PVT))
1125 return DAG.getNode(ISD::AssertSext, DL, PVT, Op0, Op.getOperand(1));
1126 break;
1127 case ISD::AssertZext:
1128 if (SDValue Op0 = ZExtPromoteOperand(Op.getOperand(0), PVT))
1129 return DAG.getNode(ISD::AssertZext, DL, PVT, Op0, Op.getOperand(1));
1130 break;
1131 case ISD::Constant: {
1132 unsigned ExtOpc =
1133 Op.getValueType().isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
1134 return DAG.getNode(ExtOpc, DL, PVT, Op);
1135 }
1136 }
1137
1138 if (!TLI.isOperationLegal(ISD::ANY_EXTEND, PVT))
1139 return SDValue();
1140 return DAG.getNode(ISD::ANY_EXTEND, DL, PVT, Op);
1141}
1142
1143SDValue DAGCombiner::SExtPromoteOperand(SDValue Op, EVT PVT) {
1144 if (!TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, PVT))
1145 return SDValue();
1146 EVT OldVT = Op.getValueType();
1147 SDLoc DL(Op);
1148 bool Replace = false;
1149 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
1150 if (!NewOp.getNode())
1151 return SDValue();
1152 AddToWorklist(NewOp.getNode());
1153
1154 if (Replace)
1155 ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
1156 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, NewOp.getValueType(), NewOp,
1157 DAG.getValueType(OldVT));
1158}
1159
1160SDValue DAGCombiner::ZExtPromoteOperand(SDValue Op, EVT PVT) {
1161 EVT OldVT = Op.getValueType();
1162 SDLoc DL(Op);
1163 bool Replace = false;
1164 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
1165 if (!NewOp.getNode())
1166 return SDValue();
1167 AddToWorklist(NewOp.getNode());
1168
1169 if (Replace)
1170 ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
1171 return DAG.getZeroExtendInReg(NewOp, DL, OldVT);
1172}
1173
1174/// Promote the specified integer binary operation if the target indicates it is
1175/// beneficial. e.g. On x86, it's usually better to promote i16 operations to
1176/// i32 since i16 instructions are longer.
1177SDValue DAGCombiner::PromoteIntBinOp(SDValue Op) {
1178 if (!LegalOperations)
1179 return SDValue();
1180
1181 EVT VT = Op.getValueType();
1182 if (VT.isVector() || !VT.isInteger())
1183 return SDValue();
1184
1185 // If operation type is 'undesirable', e.g. i16 on x86, consider
1186 // promoting it.
1187 unsigned Opc = Op.getOpcode();
1188 if (TLI.isTypeDesirableForOp(Opc, VT))
1189 return SDValue();
1190
1191 EVT PVT = VT;
1192 // Consult target whether it is a good idea to promote this operation and
1193 // what's the right type to promote it to.
1194 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1195 assert(PVT != VT && "Don't know what type to promote to!")(static_cast <bool> (PVT != VT && "Don't know what type to promote to!"
) ? void (0) : __assert_fail ("PVT != VT && \"Don't know what type to promote to!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1195, __extension__ __PRETTY_FUNCTION__))
;
1196
1197 DEBUG(dbgs() << "\nPromoting "; Op.getNode()->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; Op.getNode(
)->dump(&DAG); } } while (false)
;
1198
1199 bool Replace0 = false;
1200 SDValue N0 = Op.getOperand(0);
1201 SDValue NN0 = PromoteOperand(N0, PVT, Replace0);
1202
1203 bool Replace1 = false;
1204 SDValue N1 = Op.getOperand(1);
1205 SDValue NN1 = PromoteOperand(N1, PVT, Replace1);
1206 SDLoc DL(Op);
1207
1208 SDValue RV =
1209 DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, NN0, NN1));
1210
1211 // We are always replacing N0/N1's use in N and only need
1212 // additional replacements if there are additional uses.
1213 Replace0 &= !N0->hasOneUse();
1214 Replace1 &= (N0 != N1) && !N1->hasOneUse();
1215
1216 // Combine Op here so it is preserved past replacements.
1217 CombineTo(Op.getNode(), RV);
1218
1219 // If operands have a use ordering, make sure we deal with
1220 // predecessor first.
1221 if (Replace0 && Replace1 && N0.getNode()->isPredecessorOf(N1.getNode())) {
1222 std::swap(N0, N1);
1223 std::swap(NN0, NN1);
1224 }
1225
1226 if (Replace0) {
1227 AddToWorklist(NN0.getNode());
1228 ReplaceLoadWithPromotedLoad(N0.getNode(), NN0.getNode());
1229 }
1230 if (Replace1) {
1231 AddToWorklist(NN1.getNode());
1232 ReplaceLoadWithPromotedLoad(N1.getNode(), NN1.getNode());
1233 }
1234 return Op;
1235 }
1236 return SDValue();
1237}
1238
1239/// Promote the specified integer shift operation if the target indicates it is
1240/// beneficial. e.g. On x86, it's usually better to promote i16 operations to
1241/// i32 since i16 instructions are longer.
1242SDValue DAGCombiner::PromoteIntShiftOp(SDValue Op) {
1243 if (!LegalOperations)
1244 return SDValue();
1245
1246 EVT VT = Op.getValueType();
1247 if (VT.isVector() || !VT.isInteger())
1248 return SDValue();
1249
1250 // If operation type is 'undesirable', e.g. i16 on x86, consider
1251 // promoting it.
1252 unsigned Opc = Op.getOpcode();
1253 if (TLI.isTypeDesirableForOp(Opc, VT))
1254 return SDValue();
1255
1256 EVT PVT = VT;
1257 // Consult target whether it is a good idea to promote this operation and
1258 // what's the right type to promote it to.
1259 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1260 assert(PVT != VT && "Don't know what type to promote to!")(static_cast <bool> (PVT != VT && "Don't know what type to promote to!"
) ? void (0) : __assert_fail ("PVT != VT && \"Don't know what type to promote to!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1260, __extension__ __PRETTY_FUNCTION__))
;
1261
1262 DEBUG(dbgs() << "\nPromoting "; Op.getNode()->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; Op.getNode(
)->dump(&DAG); } } while (false)
;
1263
1264 bool Replace = false;
1265 SDValue N0 = Op.getOperand(0);
1266 SDValue N1 = Op.getOperand(1);
1267 if (Opc == ISD::SRA)
1268 N0 = SExtPromoteOperand(N0, PVT);
1269 else if (Opc == ISD::SRL)
1270 N0 = ZExtPromoteOperand(N0, PVT);
1271 else
1272 N0 = PromoteOperand(N0, PVT, Replace);
1273
1274 if (!N0.getNode())
1275 return SDValue();
1276
1277 SDLoc DL(Op);
1278 SDValue RV =
1279 DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, N0, N1));
1280
1281 AddToWorklist(N0.getNode());
1282 if (Replace)
1283 ReplaceLoadWithPromotedLoad(Op.getOperand(0).getNode(), N0.getNode());
1284
1285 // Deal with Op being deleted.
1286 if (Op && Op.getOpcode() != ISD::DELETED_NODE)
1287 return RV;
1288 }
1289 return SDValue();
1290}
1291
1292SDValue DAGCombiner::PromoteExtend(SDValue Op) {
1293 if (!LegalOperations)
1294 return SDValue();
1295
1296 EVT VT = Op.getValueType();
1297 if (VT.isVector() || !VT.isInteger())
1298 return SDValue();
1299
1300 // If operation type is 'undesirable', e.g. i16 on x86, consider
1301 // promoting it.
1302 unsigned Opc = Op.getOpcode();
1303 if (TLI.isTypeDesirableForOp(Opc, VT))
1304 return SDValue();
1305
1306 EVT PVT = VT;
1307 // Consult target whether it is a good idea to promote this operation and
1308 // what's the right type to promote it to.
1309 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1310 assert(PVT != VT && "Don't know what type to promote to!")(static_cast <bool> (PVT != VT && "Don't know what type to promote to!"
) ? void (0) : __assert_fail ("PVT != VT && \"Don't know what type to promote to!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1310, __extension__ __PRETTY_FUNCTION__))
;
1311 // fold (aext (aext x)) -> (aext x)
1312 // fold (aext (zext x)) -> (zext x)
1313 // fold (aext (sext x)) -> (sext x)
1314 DEBUG(dbgs() << "\nPromoting ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; Op.getNode(
)->dump(&DAG); } } while (false)
1315 Op.getNode()->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; Op.getNode(
)->dump(&DAG); } } while (false)
;
1316 return DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, Op.getOperand(0));
1317 }
1318 return SDValue();
1319}
1320
1321bool DAGCombiner::PromoteLoad(SDValue Op) {
1322 if (!LegalOperations)
1323 return false;
1324
1325 if (!ISD::isUNINDEXEDLoad(Op.getNode()))
1326 return false;
1327
1328 EVT VT = Op.getValueType();
1329 if (VT.isVector() || !VT.isInteger())
1330 return false;
1331
1332 // If operation type is 'undesirable', e.g. i16 on x86, consider
1333 // promoting it.
1334 unsigned Opc = Op.getOpcode();
1335 if (TLI.isTypeDesirableForOp(Opc, VT))
1336 return false;
1337
1338 EVT PVT = VT;
1339 // Consult target whether it is a good idea to promote this operation and
1340 // what's the right type to promote it to.
1341 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1342 assert(PVT != VT && "Don't know what type to promote to!")(static_cast <bool> (PVT != VT && "Don't know what type to promote to!"
) ? void (0) : __assert_fail ("PVT != VT && \"Don't know what type to promote to!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1342, __extension__ __PRETTY_FUNCTION__))
;
1343
1344 SDLoc DL(Op);
1345 SDNode *N = Op.getNode();
1346 LoadSDNode *LD = cast<LoadSDNode>(N);
1347 EVT MemVT = LD->getMemoryVT();
1348 ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD)
1349 ? (TLI.isLoadExtLegal(ISD::ZEXTLOAD, PVT, MemVT) ? ISD::ZEXTLOAD
1350 : ISD::EXTLOAD)
1351 : LD->getExtensionType();
1352 SDValue NewLD = DAG.getExtLoad(ExtType, DL, PVT,
1353 LD->getChain(), LD->getBasePtr(),
1354 MemVT, LD->getMemOperand());
1355 SDValue Result = DAG.getNode(ISD::TRUNCATE, DL, VT, NewLD);
1356
1357 DEBUG(dbgs() << "\nPromoting ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; N->dump(
&DAG); dbgs() << "\nTo: "; Result.getNode()->dump
(&DAG); dbgs() << '\n'; } } while (false)
1358 N->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; N->dump(
&DAG); dbgs() << "\nTo: "; Result.getNode()->dump
(&DAG); dbgs() << '\n'; } } while (false)
1359 dbgs() << "\nTo: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; N->dump(
&DAG); dbgs() << "\nTo: "; Result.getNode()->dump
(&DAG); dbgs() << '\n'; } } while (false)
1360 Result.getNode()->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; N->dump(
&DAG); dbgs() << "\nTo: "; Result.getNode()->dump
(&DAG); dbgs() << '\n'; } } while (false)
1361 dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nPromoting "; N->dump(
&DAG); dbgs() << "\nTo: "; Result.getNode()->dump
(&DAG); dbgs() << '\n'; } } while (false)
;
1362 WorklistRemover DeadNodes(*this);
1363 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result);
1364 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), NewLD.getValue(1));
1365 deleteAndRecombine(N);
1366 AddToWorklist(Result.getNode());
1367 return true;
1368 }
1369 return false;
1370}
1371
1372/// \brief Recursively delete a node which has no uses and any operands for
1373/// which it is the only use.
1374///
1375/// Note that this both deletes the nodes and removes them from the worklist.
1376/// It also adds any nodes who have had a user deleted to the worklist as they
1377/// may now have only one use and subject to other combines.
1378bool DAGCombiner::recursivelyDeleteUnusedNodes(SDNode *N) {
1379 if (!N->use_empty())
1380 return false;
1381
1382 SmallSetVector<SDNode *, 16> Nodes;
1383 Nodes.insert(N);
1384 do {
1385 N = Nodes.pop_back_val();
1386 if (!N)
1387 continue;
1388
1389 if (N->use_empty()) {
1390 for (const SDValue &ChildN : N->op_values())
1391 Nodes.insert(ChildN.getNode());
1392
1393 removeFromWorklist(N);
1394 DAG.DeleteNode(N);
1395 } else {
1396 AddToWorklist(N);
1397 }
1398 } while (!Nodes.empty());
1399 return true;
1400}
1401
1402//===----------------------------------------------------------------------===//
1403// Main DAG Combiner implementation
1404//===----------------------------------------------------------------------===//
1405
1406void DAGCombiner::Run(CombineLevel AtLevel) {
1407 // set the instance variables, so that the various visit routines may use it.
1408 Level = AtLevel;
1409 LegalOperations = Level >= AfterLegalizeVectorOps;
1410 LegalTypes = Level >= AfterLegalizeTypes;
1411
1412 // Add all the dag nodes to the worklist.
1413 for (SDNode &Node : DAG.allnodes())
1414 AddToWorklist(&Node);
1415
1416 // Create a dummy node (which is not added to allnodes), that adds a reference
1417 // to the root node, preventing it from being deleted, and tracking any
1418 // changes of the root.
1419 HandleSDNode Dummy(DAG.getRoot());
1420
1421 // While the worklist isn't empty, find a node and try to combine it.
1422 while (!WorklistMap.empty()) {
1423 SDNode *N;
1424 // The Worklist holds the SDNodes in order, but it may contain null entries.
1425 do {
1426 N = Worklist.pop_back_val();
1427 } while (!N);
1428
1429 bool GoodWorklistEntry = WorklistMap.erase(N);
1430 (void)GoodWorklistEntry;
1431 assert(GoodWorklistEntry &&(static_cast <bool> (GoodWorklistEntry && "Found a worklist entry without a corresponding map entry!"
) ? void (0) : __assert_fail ("GoodWorklistEntry && \"Found a worklist entry without a corresponding map entry!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1432, __extension__ __PRETTY_FUNCTION__))
1432 "Found a worklist entry without a corresponding map entry!")(static_cast <bool> (GoodWorklistEntry && "Found a worklist entry without a corresponding map entry!"
) ? void (0) : __assert_fail ("GoodWorklistEntry && \"Found a worklist entry without a corresponding map entry!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1432, __extension__ __PRETTY_FUNCTION__))
;
1433
1434 // If N has no uses, it is dead. Make sure to revisit all N's operands once
1435 // N is deleted from the DAG, since they too may now be dead or may have a
1436 // reduced number of uses, allowing other xforms.
1437 if (recursivelyDeleteUnusedNodes(N))
1438 continue;
1439
1440 WorklistRemover DeadNodes(*this);
1441
1442 // If this combine is running after legalizing the DAG, re-legalize any
1443 // nodes pulled off the worklist.
1444 if (Level == AfterLegalizeDAG) {
1445 SmallSetVector<SDNode *, 16> UpdatedNodes;
1446 bool NIsValid = DAG.LegalizeOp(N, UpdatedNodes);
1447
1448 for (SDNode *LN : UpdatedNodes) {
1449 AddToWorklist(LN);
1450 AddUsersToWorklist(LN);
1451 }
1452 if (!NIsValid)
1453 continue;
1454 }
1455
1456 DEBUG(dbgs() << "\nCombining: "; N->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << "\nCombining: "; N->dump
(&DAG); } } while (false)
;
1457
1458 // Add any operands of the new node which have not yet been combined to the
1459 // worklist as well. Because the worklist uniques things already, this
1460 // won't repeatedly process the same operand.
1461 CombinedNodes.insert(N);
1462 for (const SDValue &ChildN : N->op_values())
1463 if (!CombinedNodes.count(ChildN.getNode()))
1464 AddToWorklist(ChildN.getNode());
1465
1466 SDValue RV = combine(N);
1467
1468 if (!RV.getNode())
1469 continue;
1470
1471 ++NodesCombined;
1472
1473 // If we get back the same node we passed in, rather than a new node or
1474 // zero, we know that the node must have defined multiple values and
1475 // CombineTo was used. Since CombineTo takes care of the worklist
1476 // mechanics for us, we have no work to do in this case.
1477 if (RV.getNode() == N)
1478 continue;
1479
1480 assert(N->getOpcode() != ISD::DELETED_NODE &&(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& RV.getOpcode() != ISD::DELETED_NODE && "Node was deleted but visit returned new node!"
) ? void (0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && RV.getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned new node!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1482, __extension__ __PRETTY_FUNCTION__))
1481 RV.getOpcode() != ISD::DELETED_NODE &&(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& RV.getOpcode() != ISD::DELETED_NODE && "Node was deleted but visit returned new node!"
) ? void (0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && RV.getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned new node!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1482, __extension__ __PRETTY_FUNCTION__))
1482 "Node was deleted but visit returned new node!")(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& RV.getOpcode() != ISD::DELETED_NODE && "Node was deleted but visit returned new node!"
) ? void (0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && RV.getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned new node!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1482, __extension__ __PRETTY_FUNCTION__))
;
1483
1484 DEBUG(dbgs() << " ... into: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << " ... into: "; RV.getNode()
->dump(&DAG); } } while (false)
1485 RV.getNode()->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("dagcombine")) { dbgs() << " ... into: "; RV.getNode()
->dump(&DAG); } } while (false)
;
1486
1487 if (N->getNumValues() == RV.getNode()->getNumValues())
1488 DAG.ReplaceAllUsesWith(N, RV.getNode());
1489 else {
1490 assert(N->getValueType(0) == RV.getValueType() &&(static_cast <bool> (N->getValueType(0) == RV.getValueType
() && N->getNumValues() == 1 && "Type mismatch"
) ? void (0) : __assert_fail ("N->getValueType(0) == RV.getValueType() && N->getNumValues() == 1 && \"Type mismatch\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1491, __extension__ __PRETTY_FUNCTION__))
1491 N->getNumValues() == 1 && "Type mismatch")(static_cast <bool> (N->getValueType(0) == RV.getValueType
() && N->getNumValues() == 1 && "Type mismatch"
) ? void (0) : __assert_fail ("N->getValueType(0) == RV.getValueType() && N->getNumValues() == 1 && \"Type mismatch\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1491, __extension__ __PRETTY_FUNCTION__))
;
1492 DAG.ReplaceAllUsesWith(N, &RV);
1493 }
1494
1495 // Push the new node and any users onto the worklist
1496 AddToWorklist(RV.getNode());
1497 AddUsersToWorklist(RV.getNode());
1498
1499 // Finally, if the node is now dead, remove it from the graph. The node
1500 // may not be dead if the replacement process recursively simplified to
1501 // something else needing this node. This will also take care of adding any
1502 // operands which have lost a user to the worklist.
1503 recursivelyDeleteUnusedNodes(N);
1504 }
1505
1506 // If the root changed (e.g. it was a dead load, update the root).
1507 DAG.setRoot(Dummy.getValue());
1508 DAG.RemoveDeadNodes();
1509}
1510
1511SDValue DAGCombiner::visit(SDNode *N) {
1512 switch (N->getOpcode()) {
1513 default: break;
1514 case ISD::TokenFactor: return visitTokenFactor(N);
1515 case ISD::MERGE_VALUES: return visitMERGE_VALUES(N);
1516 case ISD::ADD: return visitADD(N);
1517 case ISD::SUB: return visitSUB(N);
1518 case ISD::ADDC: return visitADDC(N);
1519 case ISD::UADDO: return visitUADDO(N);
1520 case ISD::SUBC: return visitSUBC(N);
1521 case ISD::USUBO: return visitUSUBO(N);
1522 case ISD::ADDE: return visitADDE(N);
1523 case ISD::ADDCARRY: return visitADDCARRY(N);
1524 case ISD::SUBE: return visitSUBE(N);
1525 case ISD::SUBCARRY: return visitSUBCARRY(N);
1526 case ISD::MUL: return visitMUL(N);
1527 case ISD::SDIV: return visitSDIV(N);
1528 case ISD::UDIV: return visitUDIV(N);
1529 case ISD::SREM:
1530 case ISD::UREM: return visitREM(N);
1531 case ISD::MULHU: return visitMULHU(N);
1532 case ISD::MULHS: return visitMULHS(N);
1533 case ISD::SMUL_LOHI: return visitSMUL_LOHI(N);
1534 case ISD::UMUL_LOHI: return visitUMUL_LOHI(N);
1535 case ISD::SMULO: return visitSMULO(N);
1536 case ISD::UMULO: return visitUMULO(N);
1537 case ISD::SMIN:
1538 case ISD::SMAX:
1539 case ISD::UMIN:
1540 case ISD::UMAX: return visitIMINMAX(N);
1541 case ISD::AND: return visitAND(N);
1542 case ISD::OR: return visitOR(N);
1543 case ISD::XOR: return visitXOR(N);
1544 case ISD::SHL: return visitSHL(N);
1545 case ISD::SRA: return visitSRA(N);
1546 case ISD::SRL: return visitSRL(N);
1547 case ISD::ROTR:
1548 case ISD::ROTL: return visitRotate(N);
1549 case ISD::ABS: return visitABS(N);
1550 case ISD::BSWAP: return visitBSWAP(N);
1551 case ISD::BITREVERSE: return visitBITREVERSE(N);
1552 case ISD::CTLZ: return visitCTLZ(N);
1553 case ISD::CTLZ_ZERO_UNDEF: return visitCTLZ_ZERO_UNDEF(N);
1554 case ISD::CTTZ: return visitCTTZ(N);
1555 case ISD::CTTZ_ZERO_UNDEF: return visitCTTZ_ZERO_UNDEF(N);
1556 case ISD::CTPOP: return visitCTPOP(N);
1557 case ISD::SELECT: return visitSELECT(N);
1558 case ISD::VSELECT: return visitVSELECT(N);
1559 case ISD::SELECT_CC: return visitSELECT_CC(N);
1560 case ISD::SETCC: return visitSETCC(N);
1561 case ISD::SETCCE: return visitSETCCE(N);
1562 case ISD::SETCCCARRY: return visitSETCCCARRY(N);
1563 case ISD::SIGN_EXTEND: return visitSIGN_EXTEND(N);
1564 case ISD::ZERO_EXTEND: return visitZERO_EXTEND(N);
1565 case ISD::ANY_EXTEND: return visitANY_EXTEND(N);
1566 case ISD::AssertSext:
1567 case ISD::AssertZext: return visitAssertExt(N);
1568 case ISD::SIGN_EXTEND_INREG: return visitSIGN_EXTEND_INREG(N);
1569 case ISD::SIGN_EXTEND_VECTOR_INREG: return visitSIGN_EXTEND_VECTOR_INREG(N);
1570 case ISD::ZERO_EXTEND_VECTOR_INREG: return visitZERO_EXTEND_VECTOR_INREG(N);
1571 case ISD::TRUNCATE: return visitTRUNCATE(N);
1572 case ISD::BITCAST: return visitBITCAST(N);
1573 case ISD::BUILD_PAIR: return visitBUILD_PAIR(N);
1574 case ISD::FADD: return visitFADD(N);
1575 case ISD::FSUB: return visitFSUB(N);
1576 case ISD::FMUL: return visitFMUL(N);
1577 case ISD::FMA: return visitFMA(N);
1578 case ISD::FDIV: return visitFDIV(N);
1579 case ISD::FREM: return visitFREM(N);
1580 case ISD::FSQRT: return visitFSQRT(N);
1581 case ISD::FCOPYSIGN: return visitFCOPYSIGN(N);
1582 case ISD::SINT_TO_FP: return visitSINT_TO_FP(N);
1583 case ISD::UINT_TO_FP: return visitUINT_TO_FP(N);
1584 case ISD::FP_TO_SINT: return visitFP_TO_SINT(N);
1585 case ISD::FP_TO_UINT: return visitFP_TO_UINT(N);
1586 case ISD::FP_ROUND: return visitFP_ROUND(N);
1587 case ISD::FP_ROUND_INREG: return visitFP_ROUND_INREG(N);
1588 case ISD::FP_EXTEND: return visitFP_EXTEND(N);
1589 case ISD::FNEG: return visitFNEG(N);
1590 case ISD::FABS: return visitFABS(N);
1591 case ISD::FFLOOR: return visitFFLOOR(N);
1592 case ISD::FMINNUM: return visitFMINNUM(N);
1593 case ISD::FMAXNUM: return visitFMAXNUM(N);
1594 case ISD::FCEIL: return visitFCEIL(N);
1595 case ISD::FTRUNC: return visitFTRUNC(N);
1596 case ISD::BRCOND: return visitBRCOND(N);
1597 case ISD::BR_CC: return visitBR_CC(N);
1598 case ISD::LOAD: return visitLOAD(N);
1599 case ISD::STORE: return visitSTORE(N);
1600 case ISD::INSERT_VECTOR_ELT: return visitINSERT_VECTOR_ELT(N);
1601 case ISD::EXTRACT_VECTOR_ELT: return visitEXTRACT_VECTOR_ELT(N);
1602 case ISD::BUILD_VECTOR: return visitBUILD_VECTOR(N);
1603 case ISD::CONCAT_VECTORS: return visitCONCAT_VECTORS(N);
1604 case ISD::EXTRACT_SUBVECTOR: return visitEXTRACT_SUBVECTOR(N);
1605 case ISD::VECTOR_SHUFFLE: return visitVECTOR_SHUFFLE(N);
1606 case ISD::SCALAR_TO_VECTOR: return visitSCALAR_TO_VECTOR(N);
1607 case ISD::INSERT_SUBVECTOR: return visitINSERT_SUBVECTOR(N);
1608 case ISD::MGATHER: return visitMGATHER(N);
1609 case ISD::MLOAD: return visitMLOAD(N);
1610 case ISD::MSCATTER: return visitMSCATTER(N);
1611 case ISD::MSTORE: return visitMSTORE(N);
1612 case ISD::FP_TO_FP16: return visitFP_TO_FP16(N);
1613 case ISD::FP16_TO_FP: return visitFP16_TO_FP(N);
1614 }
1615 return SDValue();
1616}
1617
1618SDValue DAGCombiner::combine(SDNode *N) {
1619 SDValue RV = visit(N);
1620
1621 // If nothing happened, try a target-specific DAG combine.
1622 if (!RV.getNode()) {
1623 assert(N->getOpcode() != ISD::DELETED_NODE &&(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& "Node was deleted but visit returned NULL!") ? void
(0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned NULL!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1624, __extension__ __PRETTY_FUNCTION__))
1624 "Node was deleted but visit returned NULL!")(static_cast <bool> (N->getOpcode() != ISD::DELETED_NODE
&& "Node was deleted but visit returned NULL!") ? void
(0) : __assert_fail ("N->getOpcode() != ISD::DELETED_NODE && \"Node was deleted but visit returned NULL!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1624, __extension__ __PRETTY_FUNCTION__))
;
1625
1626 if (N->getOpcode() >= ISD::BUILTIN_OP_END ||
1627 TLI.hasTargetDAGCombine((ISD::NodeType)N->getOpcode())) {
1628
1629 // Expose the DAG combiner to the target combiner impls.
1630 TargetLowering::DAGCombinerInfo
1631 DagCombineInfo(DAG, Level, false, this);
1632
1633 RV = TLI.PerformDAGCombine(N, DagCombineInfo);
1634 }
1635 }
1636
1637 // If nothing happened still, try promoting the operation.
1638 if (!RV.getNode()) {
1639 switch (N->getOpcode()) {
1640 default: break;
1641 case ISD::ADD:
1642 case ISD::SUB:
1643 case ISD::MUL:
1644 case ISD::AND:
1645 case ISD::OR:
1646 case ISD::XOR:
1647 RV = PromoteIntBinOp(SDValue(N, 0));
1648 break;
1649 case ISD::SHL:
1650 case ISD::SRA:
1651 case ISD::SRL:
1652 RV = PromoteIntShiftOp(SDValue(N, 0));
1653 break;
1654 case ISD::SIGN_EXTEND:
1655 case ISD::ZERO_EXTEND:
1656 case ISD::ANY_EXTEND:
1657 RV = PromoteExtend(SDValue(N, 0));
1658 break;
1659 case ISD::LOAD:
1660 if (PromoteLoad(SDValue(N, 0)))
1661 RV = SDValue(N, 0);
1662 break;
1663 }
1664 }
1665
1666 // If N is a commutative binary node, try eliminate it if the commuted
1667 // version is already present in the DAG.
1668 if (!RV.getNode() && TLI.isCommutativeBinOp(N->getOpcode()) &&
1669 N->getNumValues() == 1) {
1670 SDValue N0 = N->getOperand(0);
1671 SDValue N1 = N->getOperand(1);
1672
1673 // Constant operands are canonicalized to RHS.
1674 if (N0 != N1 && (isa<ConstantSDNode>(N0) || !isa<ConstantSDNode>(N1))) {
1675 SDValue Ops[] = {N1, N0};
1676 SDNode *CSENode = DAG.getNodeIfExists(N->getOpcode(), N->getVTList(), Ops,
1677 N->getFlags());
1678 if (CSENode)
1679 return SDValue(CSENode, 0);
1680 }
1681 }
1682
1683 return RV;
1684}
1685
1686/// Given a node, return its input chain if it has one, otherwise return a null
1687/// sd operand.
1688static SDValue getInputChainForNode(SDNode *N) {
1689 if (unsigned NumOps = N->getNumOperands()) {
1690 if (N->getOperand(0).getValueType() == MVT::Other)
1691 return N->getOperand(0);
1692 if (N->getOperand(NumOps-1).getValueType() == MVT::Other)
1693 return N->getOperand(NumOps-1);
1694 for (unsigned i = 1; i < NumOps-1; ++i)
1695 if (N->getOperand(i).getValueType() == MVT::Other)
1696 return N->getOperand(i);
1697 }
1698 return SDValue();
1699}
1700
1701SDValue DAGCombiner::visitTokenFactor(SDNode *N) {
1702 // If N has two operands, where one has an input chain equal to the other,
1703 // the 'other' chain is redundant.
1704 if (N->getNumOperands() == 2) {
1705 if (getInputChainForNode(N->getOperand(0).getNode()) == N->getOperand(1))
1706 return N->getOperand(0);
1707 if (getInputChainForNode(N->getOperand(1).getNode()) == N->getOperand(0))
1708 return N->getOperand(1);
1709 }
1710
1711 SmallVector<SDNode *, 8> TFs; // List of token factors to visit.
1712 SmallVector<SDValue, 8> Ops; // Ops for replacing token factor.
1713 SmallPtrSet<SDNode*, 16> SeenOps;
1714 bool Changed = false; // If we should replace this token factor.
1715
1716 // Start out with this token factor.
1717 TFs.push_back(N);
1718
1719 // Iterate through token factors. The TFs grows when new token factors are
1720 // encountered.
1721 for (unsigned i = 0; i < TFs.size(); ++i) {
1722 SDNode *TF = TFs[i];
1723
1724 // Check each of the operands.
1725 for (const SDValue &Op : TF->op_values()) {
1726 switch (Op.getOpcode()) {
1727 case ISD::EntryToken:
1728 // Entry tokens don't need to be added to the list. They are
1729 // redundant.
1730 Changed = true;
1731 break;
1732
1733 case ISD::TokenFactor:
1734 if (Op.hasOneUse() && !is_contained(TFs, Op.getNode())) {
1735 // Queue up for processing.
1736 TFs.push_back(Op.getNode());
1737 // Clean up in case the token factor is removed.
1738 AddToWorklist(Op.getNode());
1739 Changed = true;
1740 break;
1741 }
1742 LLVM_FALLTHROUGH[[clang::fallthrough]];
1743
1744 default:
1745 // Only add if it isn't already in the list.
1746 if (SeenOps.insert(Op.getNode()).second)
1747 Ops.push_back(Op);
1748 else
1749 Changed = true;
1750 break;
1751 }
1752 }
1753 }
1754
1755 // Remove Nodes that are chained to another node in the list. Do so
1756 // by walking up chains breath-first stopping when we've seen
1757 // another operand. In general we must climb to the EntryNode, but we can exit
1758 // early if we find all remaining work is associated with just one operand as
1759 // no further pruning is possible.
1760
1761 // List of nodes to search through and original Ops from which they originate.
1762 SmallVector<std::pair<SDNode *, unsigned>, 8> Worklist;
1763 SmallVector<unsigned, 8> OpWorkCount; // Count of work for each Op.
1764 SmallPtrSet<SDNode *, 16> SeenChains;
1765 bool DidPruneOps = false;
1766
1767 unsigned NumLeftToConsider = 0;
1768 for (const SDValue &Op : Ops) {
1769 Worklist.push_back(std::make_pair(Op.getNode(), NumLeftToConsider++));
1770 OpWorkCount.push_back(1);
1771 }
1772
1773 auto AddToWorklist = [&](unsigned CurIdx, SDNode *Op, unsigned OpNumber) {
1774 // If this is an Op, we can remove the op from the list. Remark any
1775 // search associated with it as from the current OpNumber.
1776 if (SeenOps.count(Op) != 0) {
1777 Changed = true;
1778 DidPruneOps = true;
1779 unsigned OrigOpNumber = 0;
1780 while (OrigOpNumber < Ops.size() && Ops[OrigOpNumber].getNode() != Op)
1781 OrigOpNumber++;
1782 assert((OrigOpNumber != Ops.size()) &&(static_cast <bool> ((OrigOpNumber != Ops.size()) &&
"expected to find TokenFactor Operand") ? void (0) : __assert_fail
("(OrigOpNumber != Ops.size()) && \"expected to find TokenFactor Operand\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1783, __extension__ __PRETTY_FUNCTION__))
1783 "expected to find TokenFactor Operand")(static_cast <bool> ((OrigOpNumber != Ops.size()) &&
"expected to find TokenFactor Operand") ? void (0) : __assert_fail
("(OrigOpNumber != Ops.size()) && \"expected to find TokenFactor Operand\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1783, __extension__ __PRETTY_FUNCTION__))
;
1784 // Re-mark worklist from OrigOpNumber to OpNumber
1785 for (unsigned i = CurIdx + 1; i < Worklist.size(); ++i) {
1786 if (Worklist[i].second == OrigOpNumber) {
1787 Worklist[i].second = OpNumber;
1788 }
1789 }
1790 OpWorkCount[OpNumber] += OpWorkCount[OrigOpNumber];
1791 OpWorkCount[OrigOpNumber] = 0;
1792 NumLeftToConsider--;
1793 }
1794 // Add if it's a new chain
1795 if (SeenChains.insert(Op).second) {
1796 OpWorkCount[OpNumber]++;
1797 Worklist.push_back(std::make_pair(Op, OpNumber));
1798 }
1799 };
1800
1801 for (unsigned i = 0; i < Worklist.size() && i < 1024; ++i) {
1802 // We need at least be consider at least 2 Ops to prune.
1803 if (NumLeftToConsider <= 1)
1804 break;
1805 auto CurNode = Worklist[i].first;
1806 auto CurOpNumber = Worklist[i].second;
1807 assert((OpWorkCount[CurOpNumber] > 0) &&(static_cast <bool> ((OpWorkCount[CurOpNumber] > 0) &&
"Node should not appear in worklist") ? void (0) : __assert_fail
("(OpWorkCount[CurOpNumber] > 0) && \"Node should not appear in worklist\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1808, __extension__ __PRETTY_FUNCTION__))
1808 "Node should not appear in worklist")(static_cast <bool> ((OpWorkCount[CurOpNumber] > 0) &&
"Node should not appear in worklist") ? void (0) : __assert_fail
("(OpWorkCount[CurOpNumber] > 0) && \"Node should not appear in worklist\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1808, __extension__ __PRETTY_FUNCTION__))
;
1809 switch (CurNode->getOpcode()) {
1810 case ISD::EntryToken:
1811 // Hitting EntryToken is the only way for the search to terminate without
1812 // hitting
1813 // another operand's search. Prevent us from marking this operand
1814 // considered.
1815 NumLeftToConsider++;
1816 break;
1817 case ISD::TokenFactor:
1818 for (const SDValue &Op : CurNode->op_values())
1819 AddToWorklist(i, Op.getNode(), CurOpNumber);
1820 break;
1821 case ISD::CopyFromReg:
1822 case ISD::CopyToReg:
1823 AddToWorklist(i, CurNode->getOperand(0).getNode(), CurOpNumber);
1824 break;
1825 default:
1826 if (auto *MemNode = dyn_cast<MemSDNode>(CurNode))
1827 AddToWorklist(i, MemNode->getChain().getNode(), CurOpNumber);
1828 break;
1829 }
1830 OpWorkCount[CurOpNumber]--;
1831 if (OpWorkCount[CurOpNumber] == 0)
1832 NumLeftToConsider--;
1833 }
1834
1835 // If we've changed things around then replace token factor.
1836 if (Changed) {
1837 SDValue Result;
1838 if (Ops.empty()) {
1839 // The entry token is the only possible outcome.
1840 Result = DAG.getEntryNode();
1841 } else {
1842 if (DidPruneOps) {
1843 SmallVector<SDValue, 8> PrunedOps;
1844 //
1845 for (const SDValue &Op : Ops) {
1846 if (SeenChains.count(Op.getNode()) == 0)
1847 PrunedOps.push_back(Op);
1848 }
1849 Result = DAG.getNode(ISD::TokenFactor, SDLoc(N), MVT::Other, PrunedOps);
1850 } else {
1851 Result = DAG.getNode(ISD::TokenFactor, SDLoc(N), MVT::Other, Ops);
1852 }
1853 }
1854 return Result;
1855 }
1856 return SDValue();
1857}
1858
1859/// MERGE_VALUES can always be eliminated.
1860SDValue DAGCombiner::visitMERGE_VALUES(SDNode *N) {
1861 WorklistRemover DeadNodes(*this);
1862 // Replacing results may cause a different MERGE_VALUES to suddenly
1863 // be CSE'd with N, and carry its uses with it. Iterate until no
1864 // uses remain, to ensure that the node can be safely deleted.
1865 // First add the users of this node to the work list so that they
1866 // can be tried again once they have new operands.
1867 AddUsersToWorklist(N);
1868 do {
1869 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
1870 DAG.ReplaceAllUsesOfValueWith(SDValue(N, i), N->getOperand(i));
1871 } while (!N->use_empty());
1872 deleteAndRecombine(N);
1873 return SDValue(N, 0); // Return N so it doesn't get rechecked!
1874}
1875
1876/// If \p N is a ConstantSDNode with isOpaque() == false return it casted to a
1877/// ConstantSDNode pointer else nullptr.
1878static ConstantSDNode *getAsNonOpaqueConstant(SDValue N) {
1879 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N);
1880 return Const != nullptr && !Const->isOpaque() ? Const : nullptr;
1881}
1882
1883SDValue DAGCombiner::foldBinOpIntoSelect(SDNode *BO) {
1884 auto BinOpcode = BO->getOpcode();
1885 assert((BinOpcode == ISD::ADD || BinOpcode == ISD::SUB ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1894, __extension__ __PRETTY_FUNCTION__))
1886 BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1894, __extension__ __PRETTY_FUNCTION__))
1887 BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1894, __extension__ __PRETTY_FUNCTION__))
1888 BinOpcode == ISD::UREM || BinOpcode == ISD::AND ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1894, __extension__ __PRETTY_FUNCTION__))
1889 BinOpcode == ISD::OR || BinOpcode == ISD::XOR ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1894, __extension__ __PRETTY_FUNCTION__))
1890 BinOpcode == ISD::SHL || BinOpcode == ISD::SRL ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1894, __extension__ __PRETTY_FUNCTION__))
1891 BinOpcode == ISD::SRA || BinOpcode == ISD::FADD ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1894, __extension__ __PRETTY_FUNCTION__))
1892 BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL ||(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1894, __extension__ __PRETTY_FUNCTION__))
1893 BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) &&(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1894, __extension__ __PRETTY_FUNCTION__))
1894 "Unexpected binary operator")(static_cast <bool> ((BinOpcode == ISD::ADD || BinOpcode
== ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV
|| BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode
== ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR
|| BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode
== ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD
|| BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode
== ISD::FDIV || BinOpcode == ISD::FREM) && "Unexpected binary operator"
) ? void (0) : __assert_fail ("(BinOpcode == ISD::ADD || BinOpcode == ISD::SUB || BinOpcode == ISD::MUL || BinOpcode == ISD::SDIV || BinOpcode == ISD::UDIV || BinOpcode == ISD::SREM || BinOpcode == ISD::UREM || BinOpcode == ISD::AND || BinOpcode == ISD::OR || BinOpcode == ISD::XOR || BinOpcode == ISD::SHL || BinOpcode == ISD::SRL || BinOpcode == ISD::SRA || BinOpcode == ISD::FADD || BinOpcode == ISD::FSUB || BinOpcode == ISD::FMUL || BinOpcode == ISD::FDIV || BinOpcode == ISD::FREM) && \"Unexpected binary operator\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 1894, __extension__ __PRETTY_FUNCTION__))
;
1895
1896 // Bail out if any constants are opaque because we can't constant fold those.
1897 SDValue C1 = BO->getOperand(1);
1898 if (!isConstantOrConstantVector(C1, true) &&
1899 !isConstantFPBuildVectorOrConstantFP(C1))
1900 return SDValue();
1901
1902 // Don't do this unless the old select is going away. We want to eliminate the
1903 // binary operator, not replace a binop with a select.
1904 // TODO: Handle ISD::SELECT_CC.
1905 SDValue Sel = BO->getOperand(0);
1906 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse())
1907 return SDValue();
1908
1909 SDValue CT = Sel.getOperand(1);
1910 if (!isConstantOrConstantVector(CT, true) &&
1911 !isConstantFPBuildVectorOrConstantFP(CT))
1912 return SDValue();
1913
1914 SDValue CF = Sel.getOperand(2);
1915 if (!isConstantOrConstantVector(CF, true) &&
1916 !isConstantFPBuildVectorOrConstantFP(CF))
1917 return SDValue();
1918
1919 // We have a select-of-constants followed by a binary operator with a
1920 // constant. Eliminate the binop by pulling the constant math into the select.
1921 // Example: add (select Cond, CT, CF), C1 --> select Cond, CT + C1, CF + C1
1922 EVT VT = Sel.getValueType();
1923 SDLoc DL(Sel);
1924 SDValue NewCT = DAG.getNode(BinOpcode, DL, VT, CT, C1);
1925 if (!NewCT.isUndef() &&
1926 !isConstantOrConstantVector(NewCT, true) &&
1927 !isConstantFPBuildVectorOrConstantFP(NewCT))
1928 return SDValue();
1929
1930 SDValue NewCF = DAG.getNode(BinOpcode, DL, VT, CF, C1);
1931 if (!NewCF.isUndef() &&
1932 !isConstantOrConstantVector(NewCF, true) &&
1933 !isConstantFPBuildVectorOrConstantFP(NewCF))
1934 return SDValue();
1935
1936 return DAG.getSelect(DL, VT, Sel.getOperand(0), NewCT, NewCF);
1937}
1938
1939SDValue DAGCombiner::visitADD(SDNode *N) {
1940 SDValue N0 = N->getOperand(0);
1941 SDValue N1 = N->getOperand(1);
1942 EVT VT = N0.getValueType();
1943 SDLoc DL(N);
1944
1945 // fold vector ops
1946 if (VT.isVector()) {
1947 if (SDValue FoldedVOp = SimplifyVBinOp(N))
1948 return FoldedVOp;
1949
1950 // fold (add x, 0) -> x, vector edition
1951 if (ISD::isBuildVectorAllZeros(N1.getNode()))
1952 return N0;
1953 if (ISD::isBuildVectorAllZeros(N0.getNode()))
1954 return N1;
1955 }
1956
1957 // fold (add x, undef) -> undef
1958 if (N0.isUndef())
1959 return N0;
1960
1961 if (N1.isUndef())
1962 return N1;
1963
1964 if (DAG.isConstantIntBuildVectorOrConstantInt(N0)) {
1965 // canonicalize constant to RHS
1966 if (!DAG.isConstantIntBuildVectorOrConstantInt(N1))
1967 return DAG.getNode(ISD::ADD, DL, VT, N1, N0);
1968 // fold (add c1, c2) -> c1+c2
1969 return DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, N0.getNode(),
1970 N1.getNode());
1971 }
1972
1973 // fold (add x, 0) -> x
1974 if (isNullConstant(N1))
1975 return N0;
1976
1977 if (isConstantOrConstantVector(N1, /* NoOpaque */ true)) {
1978 // fold ((c1-A)+c2) -> (c1+c2)-A
1979 if (N0.getOpcode() == ISD::SUB &&
1980 isConstantOrConstantVector(N0.getOperand(0), /* NoOpaque */ true)) {
1981 // FIXME: Adding 2 constants should be handled by FoldConstantArithmetic.
1982 return DAG.getNode(ISD::SUB, DL, VT,
1983 DAG.getNode(ISD::ADD, DL, VT, N1, N0.getOperand(0)),
1984 N0.getOperand(1));
1985 }
1986
1987 // add (sext i1 X), 1 -> zext (not i1 X)
1988 // We don't transform this pattern:
1989 // add (zext i1 X), -1 -> sext (not i1 X)
1990 // because most (?) targets generate better code for the zext form.
1991 if (N0.getOpcode() == ISD::SIGN_EXTEND && N0.hasOneUse() &&
1992 isOneConstantOrOneSplatConstant(N1)) {
1993 SDValue X = N0.getOperand(0);
1994 if ((!LegalOperations ||
1995 (TLI.isOperationLegal(ISD::XOR, X.getValueType()) &&
1996 TLI.isOperationLegal(ISD::ZERO_EXTEND, VT))) &&
1997 X.getScalarValueSizeInBits() == 1) {
1998 SDValue Not = DAG.getNOT(DL, X, X.getValueType());
1999 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Not);
2000 }
2001 }
2002
2003 // Undo the add -> or combine to merge constant offsets from a frame index.
2004 if (N0.getOpcode() == ISD::OR &&
2005 isa<FrameIndexSDNode>(N0.getOperand(0)) &&
2006 isa<ConstantSDNode>(N0.getOperand(1)) &&
2007 DAG.haveNoCommonBitsSet(N0.getOperand(0), N0.getOperand(1))) {
2008 SDValue Add0 = DAG.getNode(ISD::ADD, DL, VT, N1, N0.getOperand(1));
2009 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), Add0);
2010 }
2011 }
2012
2013 if (SDValue NewSel = foldBinOpIntoSelect(N))
2014 return NewSel;
2015
2016 // reassociate add
2017 if (SDValue RADD = ReassociateOps(ISD::ADD, DL, N0, N1))
2018 return RADD;
2019
2020 // fold ((0-A) + B) -> B-A
2021 if (N0.getOpcode() == ISD::SUB &&
2022 isNullConstantOrNullSplatConstant(N0.getOperand(0)))
2023 return DAG.getNode(ISD::SUB, DL, VT, N1, N0.getOperand(1));
2024
2025 // fold (A + (0-B)) -> A-B
2026 if (N1.getOpcode() == ISD::SUB &&
2027 isNullConstantOrNullSplatConstant(N1.getOperand(0)))
2028 return DAG.getNode(ISD::SUB, DL, VT, N0, N1.getOperand(1));
2029
2030 // fold (A+(B-A)) -> B
2031 if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(1))
2032 return N1.getOperand(0);
2033
2034 // fold ((B-A)+A) -> B
2035 if (N0.getOpcode() == ISD::SUB && N1 == N0.getOperand(1))
2036 return N0.getOperand(0);
2037
2038 // fold (A+(B-(A+C))) to (B-C)
2039 if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD &&
2040 N0 == N1.getOperand(1).getOperand(0))
2041 return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0),
2042 N1.getOperand(1).getOperand(1));
2043
2044 // fold (A+(B-(C+A))) to (B-C)
2045 if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD &&
2046 N0 == N1.getOperand(1).getOperand(1))
2047 return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0),
2048 N1.getOperand(1).getOperand(0));
2049
2050 // fold (A+((B-A)+or-C)) to (B+or-C)
2051 if ((N1.getOpcode() == ISD::SUB || N1.getOpcode() == ISD::ADD) &&
2052 N1.getOperand(0).getOpcode() == ISD::SUB &&
2053 N0 == N1.getOperand(0).getOperand(1))
2054 return DAG.getNode(N1.getOpcode(), DL, VT, N1.getOperand(0).getOperand(0),
2055 N1.getOperand(1));
2056
2057 // fold (A-B)+(C-D) to (A+C)-(B+D) when A or C is constant
2058 if (N0.getOpcode() == ISD::SUB && N1.getOpcode() == ISD::SUB) {
2059 SDValue N00 = N0.getOperand(0);
2060 SDValue N01 = N0.getOperand(1);
2061 SDValue N10 = N1.getOperand(0);
2062 SDValue N11 = N1.getOperand(1);
2063
2064 if (isConstantOrConstantVector(N00) || isConstantOrConstantVector(N10))
2065 return DAG.getNode(ISD::SUB, DL, VT,
2066 DAG.getNode(ISD::ADD, SDLoc(N0), VT, N00, N10),
2067 DAG.getNode(ISD::ADD, SDLoc(N1), VT, N01, N11));
2068 }
2069
2070 if (SimplifyDemandedBits(SDValue(N, 0)))
2071 return SDValue(N, 0);
2072
2073 // fold (a+b) -> (a|b) iff a and b share no bits.
2074 if ((!LegalOperations || TLI.isOperationLegal(ISD::OR, VT)) &&
2075 DAG.haveNoCommonBitsSet(N0, N1))
2076 return DAG.getNode(ISD::OR, DL, VT, N0, N1);
2077
2078 if (SDValue Combined = visitADDLike(N0, N1, N))
2079 return Combined;
2080
2081 if (SDValue Combined = visitADDLike(N1, N0, N))
2082 return Combined;
2083
2084 return SDValue();
2085}
2086
2087static SDValue getAsCarry(const TargetLowering &TLI, SDValue V) {
2088 bool Masked = false;
2089
2090 // First, peel away TRUNCATE/ZERO_EXTEND/AND nodes due to legalization.
2091 while (true) {
2092 if (V.getOpcode() == ISD::TRUNCATE || V.getOpcode() == ISD::ZERO_EXTEND) {
2093 V = V.getOperand(0);
2094 continue;
2095 }
2096
2097 if (V.getOpcode() == ISD::AND && isOneConstant(V.getOperand(1))) {
2098 Masked = true;
2099 V = V.getOperand(0);
2100 continue;
2101 }
2102
2103 break;
2104 }
2105
2106 // If this is not a carry, return.
2107 if (V.getResNo() != 1)
2108 return SDValue();
2109
2110 if (V.getOpcode() != ISD::ADDCARRY && V.getOpcode() != ISD::SUBCARRY &&
2111 V.getOpcode() != ISD::UADDO && V.getOpcode() != ISD::USUBO)
2112 return SDValue();
2113
2114 // If the result is masked, then no matter what kind of bool it is we can
2115 // return. If it isn't, then we need to make sure the bool type is either 0 or
2116 // 1 and not other values.
2117 if (Masked ||
2118 TLI.getBooleanContents(V.getValueType()) ==
2119 TargetLoweringBase::ZeroOrOneBooleanContent)
2120 return V;
2121
2122 return SDValue();
2123}
2124
2125SDValue DAGCombiner::visitADDLike(SDValue N0, SDValue N1, SDNode *LocReference) {
2126 EVT VT = N0.getValueType();
2127 SDLoc DL(LocReference);
2128
2129 // fold (add x, shl(0 - y, n)) -> sub(x, shl(y, n))
2130 if (N1.getOpcode() == ISD::SHL && N1.getOperand(0).getOpcode() == ISD::SUB &&
2131 isNullConstantOrNullSplatConstant(N1.getOperand(0).getOperand(0)))
2132 return DAG.getNode(ISD::SUB, DL, VT, N0,
2133 DAG.getNode(ISD::SHL, DL, VT,
2134 N1.getOperand(0).getOperand(1),
2135 N1.getOperand(1)));
2136
2137 if (N1.getOpcode() == ISD::AND) {
2138 SDValue AndOp0 = N1.getOperand(0);
2139 unsigned NumSignBits = DAG.ComputeNumSignBits(AndOp0);
2140 unsigned DestBits = VT.getScalarSizeInBits();
2141
2142 // (add z, (and (sbbl x, x), 1)) -> (sub z, (sbbl x, x))
2143 // and similar xforms where the inner op is either ~0 or 0.
2144 if (NumSignBits == DestBits &&
2145 isOneConstantOrOneSplatConstant(N1->getOperand(1)))
2146 return DAG.getNode(ISD::SUB, DL, VT, N0, AndOp0);
2147 }
2148
2149 // add (sext i1), X -> sub X, (zext i1)
2150 if (N0.getOpcode() == ISD::SIGN_EXTEND &&
2151 N0.getOperand(0).getValueType() == MVT::i1 &&
2152 !TLI.isOperationLegal(ISD::SIGN_EXTEND, MVT::i1)) {
2153 SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0));
2154 return DAG.getNode(ISD::SUB, DL, VT, N1, ZExt);
2155 }
2156
2157 // add X, (sextinreg Y i1) -> sub X, (and Y 1)
2158 if (N1.getOpcode() == ISD::SIGN_EXTEND_INREG) {
2159 VTSDNode *TN = cast<VTSDNode>(N1.getOperand(1));
2160 if (TN->getVT() == MVT::i1) {
2161 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0),
2162 DAG.getConstant(1, DL, VT));
2163 return DAG.getNode(ISD::SUB, DL, VT, N0, ZExt);
2164 }
2165 }
2166
2167 // (add X, (addcarry Y, 0, Carry)) -> (addcarry X, Y, Carry)
2168 if (N1.getOpcode() == ISD::ADDCARRY && isNullConstant(N1.getOperand(1)) &&
2169 N1.getResNo() == 0)
2170 return DAG.getNode(ISD::ADDCARRY, DL, N1->getVTList(),
2171 N0, N1.getOperand(0), N1.getOperand(2));
2172
2173 // (add X, Carry) -> (addcarry X, 0, Carry)
2174 if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT))
2175 if (SDValue Carry = getAsCarry(TLI, N1))
2176 return DAG.getNode(ISD::ADDCARRY, DL,
2177 DAG.getVTList(VT, Carry.getValueType()), N0,
2178 DAG.getConstant(0, DL, VT), Carry);
2179
2180 return SDValue();
2181}
2182
2183SDValue DAGCombiner::visitADDC(SDNode *N) {
2184 SDValue N0 = N->getOperand(0);
2185 SDValue N1 = N->getOperand(1);
2186 EVT VT = N0.getValueType();
2187 SDLoc DL(N);
2188
2189 // If the flag result is dead, turn this into an ADD.
2190 if (!N->hasAnyUseOfValue(1))
2191 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2192 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2193
2194 // canonicalize constant to RHS.
2195 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
2196 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2197 if (N0C && !N1C)
2198 return DAG.getNode(ISD::ADDC, DL, N->getVTList(), N1, N0);
2199
2200 // fold (addc x, 0) -> x + no carry out
2201 if (isNullConstant(N1))
2202 return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE,
2203 DL, MVT::Glue));
2204
2205 // If it cannot overflow, transform into an add.
2206 if (DAG.computeOverflowKind(N0, N1) == SelectionDAG::OFK_Never)
2207 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2208 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2209
2210 return SDValue();
2211}
2212
2213SDValue DAGCombiner::visitUADDO(SDNode *N) {
2214 SDValue N0 = N->getOperand(0);
2215 SDValue N1 = N->getOperand(1);
2216 EVT VT = N0.getValueType();
2217 if (VT.isVector())
2218 return SDValue();
2219
2220 EVT CarryVT = N->getValueType(1);
2221 SDLoc DL(N);
2222
2223 // If the flag result is dead, turn this into an ADD.
2224 if (!N->hasAnyUseOfValue(1))
2225 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2226 DAG.getUNDEF(CarryVT));
2227
2228 // canonicalize constant to RHS.
2229 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
2230 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2231 if (N0C && !N1C)
2232 return DAG.getNode(ISD::UADDO, DL, N->getVTList(), N1, N0);
2233
2234 // fold (uaddo x, 0) -> x + no carry out
2235 if (isNullConstant(N1))
2236 return CombineTo(N, N0, DAG.getConstant(0, DL, CarryVT));
2237
2238 // If it cannot overflow, transform into an add.
2239 if (DAG.computeOverflowKind(N0, N1) == SelectionDAG::OFK_Never)
2240 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
2241 DAG.getConstant(0, DL, CarryVT));
2242
2243 if (SDValue Combined = visitUADDOLike(N0, N1, N))
2244 return Combined;
2245
2246 if (SDValue Combined = visitUADDOLike(N1, N0, N))
2247 return Combined;
2248
2249 return SDValue();
2250}
2251
2252SDValue DAGCombiner::visitUADDOLike(SDValue N0, SDValue N1, SDNode *N) {
2253 auto VT = N0.getValueType();
2254
2255 // (uaddo X, (addcarry Y, 0, Carry)) -> (addcarry X, Y, Carry)
2256 // If Y + 1 cannot overflow.
2257 if (N1.getOpcode() == ISD::ADDCARRY && isNullConstant(N1.getOperand(1))) {
2258 SDValue Y = N1.getOperand(0);
2259 SDValue One = DAG.getConstant(1, SDLoc(N), Y.getValueType());
2260 if (DAG.computeOverflowKind(Y, One) == SelectionDAG::OFK_Never)
2261 return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0, Y,
2262 N1.getOperand(2));
2263 }
2264
2265 // (uaddo X, Carry) -> (addcarry X, 0, Carry)
2266 if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT))
2267 if (SDValue Carry = getAsCarry(TLI, N1))
2268 return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0,
2269 DAG.getConstant(0, SDLoc(N), VT), Carry);
2270
2271 return SDValue();
2272}
2273
2274SDValue DAGCombiner::visitADDE(SDNode *N) {
2275 SDValue N0 = N->getOperand(0);
2276 SDValue N1 = N->getOperand(1);
2277 SDValue CarryIn = N->getOperand(2);
2278
2279 // canonicalize constant to RHS
2280 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
2281 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2282 if (N0C && !N1C)
2283 return DAG.getNode(ISD::ADDE, SDLoc(N), N->getVTList(),
2284 N1, N0, CarryIn);
2285
2286 // fold (adde x, y, false) -> (addc x, y)
2287 if (CarryIn.getOpcode() == ISD::CARRY_FALSE)
2288 return DAG.getNode(ISD::ADDC, SDLoc(N), N->getVTList(), N0, N1);
2289
2290 return SDValue();
2291}
2292
2293SDValue DAGCombiner::visitADDCARRY(SDNode *N) {
2294 SDValue N0 = N->getOperand(0);
2295 SDValue N1 = N->getOperand(1);
2296 SDValue CarryIn = N->getOperand(2);
2297 SDLoc DL(N);
2298
2299 // canonicalize constant to RHS
2300 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
2301 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2302 if (N0C && !N1C)
2303 return DAG.getNode(ISD::ADDCARRY, DL, N->getVTList(), N1, N0, CarryIn);
2304
2305 // fold (addcarry x, y, false) -> (uaddo x, y)
2306 if (isNullConstant(CarryIn))
2307 return DAG.getNode(ISD::UADDO, DL, N->getVTList(), N0, N1);
2308
2309 // fold (addcarry 0, 0, X) -> (and (ext/trunc X), 1) and no carry.
2310 if (isNullConstant(N0) && isNullConstant(N1)) {
2311 EVT VT = N0.getValueType();
2312 EVT CarryVT = CarryIn.getValueType();
2313 SDValue CarryExt = DAG.getBoolExtOrTrunc(CarryIn, DL, VT, CarryVT);
2314 AddToWorklist(CarryExt.getNode());
2315 return CombineTo(N, DAG.getNode(ISD::AND, DL, VT, CarryExt,
2316 DAG.getConstant(1, DL, VT)),
2317 DAG.getConstant(0, DL, CarryVT));
2318 }
2319
2320 if (SDValue Combined = visitADDCARRYLike(N0, N1, CarryIn, N))
2321 return Combined;
2322
2323 if (SDValue Combined = visitADDCARRYLike(N1, N0, CarryIn, N))
2324 return Combined;
2325
2326 return SDValue();
2327}
2328
2329SDValue DAGCombiner::visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn,
2330 SDNode *N) {
2331 // Iff the flag result is dead:
2332 // (addcarry (add|uaddo X, Y), 0, Carry) -> (addcarry X, Y, Carry)
2333 if ((N0.getOpcode() == ISD::ADD ||
2334 (N0.getOpcode() == ISD::UADDO && N0.getResNo() == 0)) &&
2335 isNullConstant(N1) && !N->hasAnyUseOfValue(1))
2336 return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(),
2337 N0.getOperand(0), N0.getOperand(1), CarryIn);
2338
2339 /**
2340 * When one of the addcarry argument is itself a carry, we may be facing
2341 * a diamond carry propagation. In which case we try to transform the DAG
2342 * to ensure linear carry propagation if that is possible.
2343 *
2344 * We are trying to get:
2345 * (addcarry X, 0, (addcarry A, B, Z):Carry)
2346 */
2347 if (auto Y = getAsCarry(TLI, N1)) {
2348 /**
2349 * (uaddo A, B)
2350 * / \
2351 * Carry Sum
2352 * | \
2353 * | (addcarry *, 0, Z)
2354 * | /
2355 * \ Carry
2356 * | /
2357 * (addcarry X, *, *)
2358 */
2359 if (Y.getOpcode() == ISD::UADDO &&
2360 CarryIn.getResNo() == 1 &&
2361 CarryIn.getOpcode() == ISD::ADDCARRY &&
2362 isNullConstant(CarryIn.getOperand(1)) &&
2363 CarryIn.getOperand(0) == Y.getValue(0)) {
2364 auto NewY = DAG.getNode(ISD::ADDCARRY, SDLoc(N), Y->getVTList(),
2365 Y.getOperand(0), Y.getOperand(1),
2366 CarryIn.getOperand(2));
2367 AddToWorklist(NewY.getNode());
2368 return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0,
2369 DAG.getConstant(0, SDLoc(N), N0.getValueType()),
2370 NewY.getValue(1));
2371 }
2372 }
2373
2374 return SDValue();
2375}
2376
2377// Since it may not be valid to emit a fold to zero for vector initializers
2378// check if we can before folding.
2379static SDValue tryFoldToZero(const SDLoc &DL, const TargetLowering &TLI, EVT VT,
2380 SelectionDAG &DAG, bool LegalOperations,
2381 bool LegalTypes) {
2382 if (!VT.isVector())
2383 return DAG.getConstant(0, DL, VT);
2384 if (!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT))
2385 return DAG.getConstant(0, DL, VT);
2386 return SDValue();
2387}
2388
2389SDValue DAGCombiner::visitSUB(SDNode *N) {
2390 SDValue N0 = N->getOperand(0);
2391 SDValue N1 = N->getOperand(1);
2392 EVT VT = N0.getValueType();
2393 SDLoc DL(N);
2394
2395 // fold vector ops
2396 if (VT.isVector()) {
2397 if (SDValue FoldedVOp = SimplifyVBinOp(N))
2398 return FoldedVOp;
2399
2400 // fold (sub x, 0) -> x, vector edition
2401 if (ISD::isBuildVectorAllZeros(N1.getNode()))
2402 return N0;
2403 }
2404
2405 // fold (sub x, x) -> 0
2406 // FIXME: Refactor this and xor and other similar operations together.
2407 if (N0 == N1)
2408 return tryFoldToZero(DL, TLI, VT, DAG, LegalOperations, LegalTypes);
2409 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
2410 DAG.isConstantIntBuildVectorOrConstantInt(N1)) {
2411 // fold (sub c1, c2) -> c1-c2
2412 return DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, N0.getNode(),
2413 N1.getNode());
2414 }
2415
2416 if (SDValue NewSel = foldBinOpIntoSelect(N))
2417 return NewSel;
2418
2419 ConstantSDNode *N1C = getAsNonOpaqueConstant(N1);
2420
2421 // fold (sub x, c) -> (add x, -c)
2422 if (N1C) {
2423 return DAG.getNode(ISD::ADD, DL, VT, N0,
2424 DAG.getConstant(-N1C->getAPIntValue(), DL, VT));
2425 }
2426
2427 if (isNullConstantOrNullSplatConstant(N0)) {
2428 unsigned BitWidth = VT.getScalarSizeInBits();
2429 // Right-shifting everything out but the sign bit followed by negation is
2430 // the same as flipping arithmetic/logical shift type without the negation:
2431 // -(X >>u 31) -> (X >>s 31)
2432 // -(X >>s 31) -> (X >>u 31)
2433 if (N1->getOpcode() == ISD::SRA || N1->getOpcode() == ISD::SRL) {
2434 ConstantSDNode *ShiftAmt = isConstOrConstSplat(N1.getOperand(1));
2435 if (ShiftAmt && ShiftAmt->getZExtValue() == BitWidth - 1) {
2436 auto NewSh = N1->getOpcode() == ISD::SRA ? ISD::SRL : ISD::SRA;
2437 if (!LegalOperations || TLI.isOperationLegal(NewSh, VT))
2438 return DAG.getNode(NewSh, DL, VT, N1.getOperand(0), N1.getOperand(1));
2439 }
2440 }
2441
2442 // 0 - X --> 0 if the sub is NUW.
2443 if (N->getFlags().hasNoUnsignedWrap())
2444 return N0;
2445
2446 if (DAG.MaskedValueIsZero(N1, ~APInt::getSignMask(BitWidth))) {
2447 // N1 is either 0 or the minimum signed value. If the sub is NSW, then
2448 // N1 must be 0 because negating the minimum signed value is undefined.
2449 if (N->getFlags().hasNoSignedWrap())
2450 return N0;
2451
2452 // 0 - X --> X if X is 0 or the minimum signed value.
2453 return N1;
2454 }
2455 }
2456
2457 // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1)
2458 if (isAllOnesConstantOrAllOnesSplatConstant(N0))
2459 return DAG.getNode(ISD::XOR, DL, VT, N1, N0);
2460
2461 // fold A-(A-B) -> B
2462 if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(0))
2463 return N1.getOperand(1);
2464
2465 // fold (A+B)-A -> B
2466 if (N0.getOpcode() == ISD::ADD && N0.getOperand(0) == N1)
2467 return N0.getOperand(1);
2468
2469 // fold (A+B)-B -> A
2470 if (N0.getOpcode() == ISD::ADD && N0.getOperand(1) == N1)
2471 return N0.getOperand(0);
2472
2473 // fold C2-(A+C1) -> (C2-C1)-A
2474 if (N1.getOpcode() == ISD::ADD) {
2475 SDValue N11 = N1.getOperand(1);
2476 if (isConstantOrConstantVector(N0, /* NoOpaques */ true) &&
2477 isConstantOrConstantVector(N11, /* NoOpaques */ true)) {
2478 SDValue NewC = DAG.getNode(ISD::SUB, DL, VT, N0, N11);
2479 return DAG.getNode(ISD::SUB, DL, VT, NewC, N1.getOperand(0));
2480 }
2481 }
2482
2483 // fold ((A+(B+or-C))-B) -> A+or-C
2484 if (N0.getOpcode() == ISD::ADD &&
2485 (N0.getOperand(1).getOpcode() == ISD::SUB ||
2486 N0.getOperand(1).getOpcode() == ISD::ADD) &&
2487 N0.getOperand(1).getOperand(0) == N1)
2488 return DAG.getNode(N0.getOperand(1).getOpcode(), DL, VT, N0.getOperand(0),
2489 N0.getOperand(1).getOperand(1));
2490
2491 // fold ((A+(C+B))-B) -> A+C
2492 if (N0.getOpcode() == ISD::ADD && N0.getOperand(1).getOpcode() == ISD::ADD &&
2493 N0.getOperand(1).getOperand(1) == N1)
2494 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0),
2495 N0.getOperand(1).getOperand(0));
2496
2497 // fold ((A-(B-C))-C) -> A-B
2498 if (N0.getOpcode() == ISD::SUB && N0.getOperand(1).getOpcode() == ISD::SUB &&
2499 N0.getOperand(1).getOperand(1) == N1)
2500 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0),
2501 N0.getOperand(1).getOperand(0));
2502
2503 // If either operand of a sub is undef, the result is undef
2504 if (N0.isUndef())
2505 return N0;
2506 if (N1.isUndef())
2507 return N1;
2508
2509 // If the relocation model supports it, consider symbol offsets.
2510 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N0))
2511 if (!LegalOperations && TLI.isOffsetFoldingLegal(GA)) {
2512 // fold (sub Sym, c) -> Sym-c
2513 if (N1C && GA->getOpcode() == ISD::GlobalAddress)
2514 return DAG.getGlobalAddress(GA->getGlobal(), SDLoc(N1C), VT,
2515 GA->getOffset() -
2516 (uint64_t)N1C->getSExtValue());
2517 // fold (sub Sym+c1, Sym+c2) -> c1-c2
2518 if (GlobalAddressSDNode *GB = dyn_cast<GlobalAddressSDNode>(N1))
2519 if (GA->getGlobal() == GB->getGlobal())
2520 return DAG.getConstant((uint64_t)GA->getOffset() - GB->getOffset(),
2521 DL, VT);
2522 }
2523
2524 // sub X, (sextinreg Y i1) -> add X, (and Y 1)
2525 if (N1.getOpcode() == ISD::SIGN_EXTEND_INREG) {
2526 VTSDNode *TN = cast<VTSDNode>(N1.getOperand(1));
2527 if (TN->getVT() == MVT::i1) {
2528 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0),
2529 DAG.getConstant(1, DL, VT));
2530 return DAG.getNode(ISD::ADD, DL, VT, N0, ZExt);
2531 }
2532 }
2533
2534 return SDValue();
2535}
2536
2537SDValue DAGCombiner::visitSUBC(SDNode *N) {
2538 SDValue N0 = N->getOperand(0);
2539 SDValue N1 = N->getOperand(1);
2540 EVT VT = N0.getValueType();
2541 SDLoc DL(N);
2542
2543 // If the flag result is dead, turn this into an SUB.
2544 if (!N->hasAnyUseOfValue(1))
2545 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1),
2546 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2547
2548 // fold (subc x, x) -> 0 + no borrow
2549 if (N0 == N1)
2550 return CombineTo(N, DAG.getConstant(0, DL, VT),
2551 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2552
2553 // fold (subc x, 0) -> x + no borrow
2554 if (isNullConstant(N1))
2555 return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2556
2557 // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1) + no borrow
2558 if (isAllOnesConstant(N0))
2559 return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0),
2560 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2561
2562 return SDValue();
2563}
2564
2565SDValue DAGCombiner::visitUSUBO(SDNode *N) {
2566 SDValue N0 = N->getOperand(0);
2567 SDValue N1 = N->getOperand(1);
2568 EVT VT = N0.getValueType();
2569 if (VT.isVector())
2570 return SDValue();
2571
2572 EVT CarryVT = N->getValueType(1);
2573 SDLoc DL(N);
2574
2575 // If the flag result is dead, turn this into an SUB.
2576 if (!N->hasAnyUseOfValue(1))
2577 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1),
2578 DAG.getUNDEF(CarryVT));
2579
2580 // fold (usubo x, x) -> 0 + no borrow
2581 if (N0 == N1)
2582 return CombineTo(N, DAG.getConstant(0, DL, VT),
2583 DAG.getConstant(0, DL, CarryVT));
2584
2585 // fold (usubo x, 0) -> x + no borrow
2586 if (isNullConstant(N1))
2587 return CombineTo(N, N0, DAG.getConstant(0, DL, CarryVT));
2588
2589 // Canonicalize (usubo -1, x) -> ~x, i.e. (xor x, -1) + no borrow
2590 if (isAllOnesConstant(N0))
2591 return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0),
2592 DAG.getConstant(0, DL, CarryVT));
2593
2594 return SDValue();
2595}
2596
2597SDValue DAGCombiner::visitSUBE(SDNode *N) {
2598 SDValue N0 = N->getOperand(0);
2599 SDValue N1 = N->getOperand(1);
2600 SDValue CarryIn = N->getOperand(2);
2601
2602 // fold (sube x, y, false) -> (subc x, y)
2603 if (CarryIn.getOpcode() == ISD::CARRY_FALSE)
2604 return DAG.getNode(ISD::SUBC, SDLoc(N), N->getVTList(), N0, N1);
2605
2606 return SDValue();
2607}
2608
2609SDValue DAGCombiner::visitSUBCARRY(SDNode *N) {
2610 SDValue N0 = N->getOperand(0);
2611 SDValue N1 = N->getOperand(1);
2612 SDValue CarryIn = N->getOperand(2);
2613
2614 // fold (subcarry x, y, false) -> (usubo x, y)
2615 if (isNullConstant(CarryIn))
2616 return DAG.getNode(ISD::USUBO, SDLoc(N), N->getVTList(), N0, N1);
2617
2618 return SDValue();
2619}
2620
2621SDValue DAGCombiner::visitMUL(SDNode *N) {
2622 SDValue N0 = N->getOperand(0);
2623 SDValue N1 = N->getOperand(1);
2624 EVT VT = N0.getValueType();
2625
2626 // fold (mul x, undef) -> 0
2627 if (N0.isUndef() || N1.isUndef())
2628 return DAG.getConstant(0, SDLoc(N), VT);
2629
2630 bool N0IsConst = false;
2631 bool N1IsConst = false;
2632 bool N1IsOpaqueConst = false;
2633 bool N0IsOpaqueConst = false;
2634 APInt ConstValue0, ConstValue1;
2635 // fold vector ops
2636 if (VT.isVector()) {
2637 if (SDValue FoldedVOp = SimplifyVBinOp(N))
2638 return FoldedVOp;
2639
2640 N0IsConst = ISD::isConstantSplatVector(N0.getNode(), ConstValue0);
2641 N1IsConst = ISD::isConstantSplatVector(N1.getNode(), ConstValue1);
2642 assert((!N0IsConst ||(static_cast <bool> ((!N0IsConst || ConstValue0.getBitWidth
() == VT.getScalarSizeInBits()) && "Splat APInt should be element width"
) ? void (0) : __assert_fail ("(!N0IsConst || ConstValue0.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2644, __extension__ __PRETTY_FUNCTION__))
2643 ConstValue0.getBitWidth() == VT.getScalarSizeInBits()) &&(static_cast <bool> ((!N0IsConst || ConstValue0.getBitWidth
() == VT.getScalarSizeInBits()) && "Splat APInt should be element width"
) ? void (0) : __assert_fail ("(!N0IsConst || ConstValue0.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2644, __extension__ __PRETTY_FUNCTION__))
2644 "Splat APInt should be element width")(static_cast <bool> ((!N0IsConst || ConstValue0.getBitWidth
() == VT.getScalarSizeInBits()) && "Splat APInt should be element width"
) ? void (0) : __assert_fail ("(!N0IsConst || ConstValue0.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2644, __extension__ __PRETTY_FUNCTION__))
;
2645 assert((!N1IsConst ||(static_cast <bool> ((!N1IsConst || ConstValue1.getBitWidth
() == VT.getScalarSizeInBits()) && "Splat APInt should be element width"
) ? void (0) : __assert_fail ("(!N1IsConst || ConstValue1.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2647, __extension__ __PRETTY_FUNCTION__))
2646 ConstValue1.getBitWidth() == VT.getScalarSizeInBits()) &&(static_cast <bool> ((!N1IsConst || ConstValue1.getBitWidth
() == VT.getScalarSizeInBits()) && "Splat APInt should be element width"
) ? void (0) : __assert_fail ("(!N1IsConst || ConstValue1.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2647, __extension__ __PRETTY_FUNCTION__))
2647 "Splat APInt should be element width")(static_cast <bool> ((!N1IsConst || ConstValue1.getBitWidth
() == VT.getScalarSizeInBits()) && "Splat APInt should be element width"
) ? void (0) : __assert_fail ("(!N1IsConst || ConstValue1.getBitWidth() == VT.getScalarSizeInBits()) && \"Splat APInt should be element width\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2647, __extension__ __PRETTY_FUNCTION__))
;
2648 } else {
2649 N0IsConst = isa<ConstantSDNode>(N0);
2650 if (N0IsConst) {
2651 ConstValue0 = cast<ConstantSDNode>(N0)->getAPIntValue();
2652 N0IsOpaqueConst = cast<ConstantSDNode>(N0)->isOpaque();
2653 }
2654 N1IsConst = isa<ConstantSDNode>(N1);
2655 if (N1IsConst) {
2656 ConstValue1 = cast<ConstantSDNode>(N1)->getAPIntValue();
2657 N1IsOpaqueConst = cast<ConstantSDNode>(N1)->isOpaque();
2658 }
2659 }
2660
2661 // fold (mul c1, c2) -> c1*c2
2662 if (N0IsConst && N1IsConst && !N0IsOpaqueConst && !N1IsOpaqueConst)
2663 return DAG.FoldConstantArithmetic(ISD::MUL, SDLoc(N), VT,
2664 N0.getNode(), N1.getNode());
2665
2666 // canonicalize constant to RHS (vector doesn't have to splat)
2667 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
2668 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
2669 return DAG.getNode(ISD::MUL, SDLoc(N), VT, N1, N0);
2670 // fold (mul x, 0) -> 0
2671 if (N1IsConst && ConstValue1.isNullValue())
2672 return N1;
2673 // fold (mul x, 1) -> x
2674 if (N1IsConst && ConstValue1.isOneValue())
2675 return N0;
2676
2677 if (SDValue NewSel = foldBinOpIntoSelect(N))
2678 return NewSel;
2679
2680 // fold (mul x, -1) -> 0-x
2681 if (N1IsConst && ConstValue1.isAllOnesValue()) {
2682 SDLoc DL(N);
2683 return DAG.getNode(ISD::SUB, DL, VT,
2684 DAG.getConstant(0, DL, VT), N0);
2685 }
2686 // fold (mul x, (1 << c)) -> x << c
2687 if (isConstantOrConstantVector(N1, /*NoOpaques*/ true) &&
2688 DAG.isKnownToBeAPowerOfTwo(N1) &&
2689 (!VT.isVector() || Level <= AfterLegalizeVectorOps)) {
2690 SDLoc DL(N);
2691 SDValue LogBase2 = BuildLogBase2(N1, DL);
2692 AddToWorklist(LogBase2.getNode());
2693
2694 EVT ShiftVT = getShiftAmountTy(N0.getValueType());
2695 SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ShiftVT);
2696 AddToWorklist(Trunc.getNode());
2697 return DAG.getNode(ISD::SHL, DL, VT, N0, Trunc);
2698 }
2699 // fold (mul x, -(1 << c)) -> -(x << c) or (-x) << c
2700 if (N1IsConst && !N1IsOpaqueConst && (-ConstValue1).isPowerOf2()) {
2701 unsigned Log2Val = (-ConstValue1).logBase2();
2702 SDLoc DL(N);
2703 // FIXME: If the input is something that is easily negated (e.g. a
2704 // single-use add), we should put the negate there.
2705 return DAG.getNode(ISD::SUB, DL, VT,
2706 DAG.getConstant(0, DL, VT),
2707 DAG.getNode(ISD::SHL, DL, VT, N0,
2708 DAG.getConstant(Log2Val, DL,
2709 getShiftAmountTy(N0.getValueType()))));
2710 }
2711
2712 // (mul (shl X, c1), c2) -> (mul X, c2 << c1)
2713 if (N0.getOpcode() == ISD::SHL &&
2714 isConstantOrConstantVector(N1, /* NoOpaques */ true) &&
2715 isConstantOrConstantVector(N0.getOperand(1), /* NoOpaques */ true)) {
2716 SDValue C3 = DAG.getNode(ISD::SHL, SDLoc(N), VT, N1, N0.getOperand(1));
2717 if (isConstantOrConstantVector(C3))
2718 return DAG.getNode(ISD::MUL, SDLoc(N), VT, N0.getOperand(0), C3);
2719 }
2720
2721 // Change (mul (shl X, C), Y) -> (shl (mul X, Y), C) when the shift has one
2722 // use.
2723 {
2724 SDValue Sh(nullptr, 0), Y(nullptr, 0);
2725
2726 // Check for both (mul (shl X, C), Y) and (mul Y, (shl X, C)).
2727 if (N0.getOpcode() == ISD::SHL &&
2728 isConstantOrConstantVector(N0.getOperand(1)) &&
2729 N0.getNode()->hasOneUse()) {
2730 Sh = N0; Y = N1;
2731 } else if (N1.getOpcode() == ISD::SHL &&
2732 isConstantOrConstantVector(N1.getOperand(1)) &&
2733 N1.getNode()->hasOneUse()) {
2734 Sh = N1; Y = N0;
2735 }
2736
2737 if (Sh.getNode()) {
2738 SDValue Mul = DAG.getNode(ISD::MUL, SDLoc(N), VT, Sh.getOperand(0), Y);
2739 return DAG.getNode(ISD::SHL, SDLoc(N), VT, Mul, Sh.getOperand(1));
2740 }
2741 }
2742
2743 // fold (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2)
2744 if (DAG.isConstantIntBuildVectorOrConstantInt(N1) &&
2745 N0.getOpcode() == ISD::ADD &&
2746 DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1)) &&
2747 isMulAddWithConstProfitable(N, N0, N1))
2748 return DAG.getNode(ISD::ADD, SDLoc(N), VT,
2749 DAG.getNode(ISD::MUL, SDLoc(N0), VT,
2750 N0.getOperand(0), N1),
2751 DAG.getNode(ISD::MUL, SDLoc(N1), VT,
2752 N0.getOperand(1), N1));
2753
2754 // reassociate mul
2755 if (SDValue RMUL = ReassociateOps(ISD::MUL, SDLoc(N), N0, N1))
2756 return RMUL;
2757
2758 return SDValue();
2759}
2760
2761/// Return true if divmod libcall is available.
2762static bool isDivRemLibcallAvailable(SDNode *Node, bool isSigned,
2763 const TargetLowering &TLI) {
2764 RTLIB::Libcall LC;
2765 EVT NodeType = Node->getValueType(0);
2766 if (!NodeType.isSimple())
2767 return false;
2768 switch (NodeType.getSimpleVT().SimpleTy) {
2769 default: return false; // No libcall for vector types.
2770 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2771 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2772 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2773 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2774 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2775 }
2776
2777 return TLI.getLibcallName(LC) != nullptr;
2778}
2779
2780/// Issue divrem if both quotient and remainder are needed.
2781SDValue DAGCombiner::useDivRem(SDNode *Node) {
2782 if (Node->use_empty())
2783 return SDValue(); // This is a dead node, leave it alone.
2784
2785 unsigned Opcode = Node->getOpcode();
2786 bool isSigned = (Opcode == ISD::SDIV) || (Opcode == ISD::SREM);
2787 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
2788
2789 // DivMod lib calls can still work on non-legal types if using lib-calls.
2790 EVT VT = Node->getValueType(0);
2791 if (VT.isVector() || !VT.isInteger())
2792 return SDValue();
2793
2794 if (!TLI.isTypeLegal(VT) && !TLI.isOperationCustom(DivRemOpc, VT))
2795 return SDValue();
2796
2797 // If DIVREM is going to get expanded into a libcall,
2798 // but there is no libcall available, then don't combine.
2799 if (!TLI.isOperationLegalOrCustom(DivRemOpc, VT) &&
2800 !isDivRemLibcallAvailable(Node, isSigned, TLI))
2801 return SDValue();
2802
2803 // If div is legal, it's better to do the normal expansion
2804 unsigned OtherOpcode = 0;
2805 if ((Opcode == ISD::SDIV) || (Opcode == ISD::UDIV)) {
2806 OtherOpcode = isSigned ? ISD::SREM : ISD::UREM;
2807 if (TLI.isOperationLegalOrCustom(Opcode, VT))
2808 return SDValue();
2809 } else {
2810 OtherOpcode = isSigned ? ISD::SDIV : ISD::UDIV;
2811 if (TLI.isOperationLegalOrCustom(OtherOpcode, VT))
2812 return SDValue();
2813 }
2814
2815 SDValue Op0 = Node->getOperand(0);
2816 SDValue Op1 = Node->getOperand(1);
2817 SDValue combined;
2818 for (SDNode::use_iterator UI = Op0.getNode()->use_begin(),
2819 UE = Op0.getNode()->use_end(); UI != UE;) {
2820 SDNode *User = *UI++;
2821 if (User == Node || User->use_empty())
2822 continue;
2823 // Convert the other matching node(s), too;
2824 // otherwise, the DIVREM may get target-legalized into something
2825 // target-specific that we won't be able to recognize.
2826 unsigned UserOpc = User->getOpcode();
2827 if ((UserOpc == Opcode || UserOpc == OtherOpcode || UserOpc == DivRemOpc) &&
2828 User->getOperand(0) == Op0 &&
2829 User->getOperand(1) == Op1) {
2830 if (!combined) {
2831 if (UserOpc == OtherOpcode) {
2832 SDVTList VTs = DAG.getVTList(VT, VT);
2833 combined = DAG.getNode(DivRemOpc, SDLoc(Node), VTs, Op0, Op1);
2834 } else if (UserOpc == DivRemOpc) {
2835 combined = SDValue(User, 0);
2836 } else {
2837 assert(UserOpc == Opcode)(static_cast <bool> (UserOpc == Opcode) ? void (0) : __assert_fail
("UserOpc == Opcode", "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 2837, __extension__ __PRETTY_FUNCTION__))
;
2838 continue;
2839 }
2840 }
2841 if (UserOpc == ISD::SDIV || UserOpc == ISD::UDIV)
2842 CombineTo(User, combined);
2843 else if (UserOpc == ISD::SREM || UserOpc == ISD::UREM)
2844 CombineTo(User, combined.getValue(1));
2845 }
2846 }
2847 return combined;
2848}
2849
2850static SDValue simplifyDivRem(SDNode *N, SelectionDAG &DAG) {
2851 SDValue N0 = N->getOperand(0);
2852 SDValue N1 = N->getOperand(1);
2853 EVT VT = N->getValueType(0);
2854 SDLoc DL(N);
2855
2856 if (DAG.isUndef(N->getOpcode(), {N0, N1}))
2857 return DAG.getUNDEF(VT);
2858
2859 // undef / X -> 0
2860 // undef % X -> 0
2861 if (N0.isUndef())
2862 return DAG.getConstant(0, DL, VT);
2863
2864 return SDValue();
2865}
2866
2867SDValue DAGCombiner::visitSDIV(SDNode *N) {
2868 SDValue N0 = N->getOperand(0);
2869 SDValue N1 = N->getOperand(1);
2870 EVT VT = N->getValueType(0);
2871
2872 // fold vector ops
2873 if (VT.isVector())
2874 if (SDValue FoldedVOp = SimplifyVBinOp(N))
2875 return FoldedVOp;
2876
2877 SDLoc DL(N);
2878
2879 // fold (sdiv c1, c2) -> c1/c2
2880 ConstantSDNode *N0C = isConstOrConstSplat(N0);
2881 ConstantSDNode *N1C = isConstOrConstSplat(N1);
2882 if (N0C && N1C && !N0C->isOpaque() && !N1C->isOpaque())
2883 return DAG.FoldConstantArithmetic(ISD::SDIV, DL, VT, N0C, N1C);
2884 // fold (sdiv X, 1) -> X
2885 if (N1C && N1C->isOne())
2886 return N0;
2887 // fold (sdiv X, -1) -> 0-X
2888 if (N1C && N1C->isAllOnesValue())
2889 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), N0);
2890
2891 if (SDValue V = simplifyDivRem(N, DAG))
2892 return V;
2893
2894 if (SDValue NewSel = foldBinOpIntoSelect(N))
2895 return NewSel;
2896
2897 // If we know the sign bits of both operands are zero, strength reduce to a
2898 // udiv instead. Handles (X&15) /s 4 -> X&15 >> 2
2899 if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
2900 return DAG.getNode(ISD::UDIV, DL, N1.getValueType(), N0, N1);
2901
2902 // fold (sdiv X, pow2) -> simple ops after legalize
2903 // FIXME: We check for the exact bit here because the generic lowering gives
2904 // better results in that case. The target-specific lowering should learn how
2905 // to handle exact sdivs efficiently.
2906 if (N1C && !N1C->isNullValue() && !N1C->isOpaque() &&
2907 !N->getFlags().hasExact() && (N1C->getAPIntValue().isPowerOf2() ||
2908 (-N1C->getAPIntValue()).isPowerOf2())) {
2909 // Target-specific implementation of sdiv x, pow2.
2910 if (SDValue Res = BuildSDIVPow2(N))
2911 return Res;
2912
2913 unsigned lg2 = N1C->getAPIntValue().countTrailingZeros();
2914
2915 // Splat the sign bit into the register
2916 SDValue SGN =
2917 DAG.getNode(ISD::SRA, DL, VT, N0,
2918 DAG.getConstant(VT.getScalarSizeInBits() - 1, DL,
2919 getShiftAmountTy(N0.getValueType())));
2920 AddToWorklist(SGN.getNode());
2921
2922 // Add (N0 < 0) ? abs2 - 1 : 0;
2923 SDValue SRL =
2924 DAG.getNode(ISD::SRL, DL, VT, SGN,
2925 DAG.getConstant(VT.getScalarSizeInBits() - lg2, DL,
2926 getShiftAmountTy(SGN.getValueType())));
2927 SDValue ADD = DAG.getNode(ISD::ADD, DL, VT, N0, SRL);
2928 AddToWorklist(SRL.getNode());
2929 AddToWorklist(ADD.getNode()); // Divide by pow2
2930 SDValue SRA = DAG.getNode(ISD::SRA, DL, VT, ADD,
2931 DAG.getConstant(lg2, DL,
2932 getShiftAmountTy(ADD.getValueType())));
2933
2934 // If we're dividing by a positive value, we're done. Otherwise, we must
2935 // negate the result.
2936 if (N1C->getAPIntValue().isNonNegative())
2937 return SRA;
2938
2939 AddToWorklist(SRA.getNode());
2940 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), SRA);
2941 }
2942
2943 // If integer divide is expensive and we satisfy the requirements, emit an
2944 // alternate sequence. Targets may check function attributes for size/speed
2945 // trade-offs.
2946 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
2947 if (N1C && !TLI.isIntDivCheap(N->getValueType(0), Attr))
2948 if (SDValue Op = BuildSDIV(N))
2949 return Op;
2950
2951 // sdiv, srem -> sdivrem
2952 // If the divisor is constant, then return DIVREM only if isIntDivCheap() is
2953 // true. Otherwise, we break the simplification logic in visitREM().
2954 if (!N1C || TLI.isIntDivCheap(N->getValueType(0), Attr))
2955 if (SDValue DivRem = useDivRem(N))
2956 return DivRem;
2957
2958 return SDValue();
2959}
2960
2961SDValue DAGCombiner::visitUDIV(SDNode *N) {
2962 SDValue N0 = N->getOperand(0);
2963 SDValue N1 = N->getOperand(1);
2964 EVT VT = N->getValueType(0);
2965
2966 // fold vector ops
2967 if (VT.isVector())
2968 if (SDValue FoldedVOp = SimplifyVBinOp(N))
2969 return FoldedVOp;
2970
2971 SDLoc DL(N);
2972
2973 // fold (udiv c1, c2) -> c1/c2
2974 ConstantSDNode *N0C = isConstOrConstSplat(N0);
2975 ConstantSDNode *N1C = isConstOrConstSplat(N1);
2976 if (N0C && N1C)
2977 if (SDValue Folded = DAG.FoldConstantArithmetic(ISD::UDIV, DL, VT,
2978 N0C, N1C))
2979 return Folded;
2980
2981 if (SDValue V = simplifyDivRem(N, DAG))
2982 return V;
2983
2984 if (SDValue NewSel = foldBinOpIntoSelect(N))
2985 return NewSel;
2986
2987 // fold (udiv x, (1 << c)) -> x >>u c
2988 if (isConstantOrConstantVector(N1, /*NoOpaques*/ true) &&
2989 DAG.isKnownToBeAPowerOfTwo(N1)) {
2990 SDValue LogBase2 = BuildLogBase2(N1, DL);
2991 AddToWorklist(LogBase2.getNode());
2992
2993 EVT ShiftVT = getShiftAmountTy(N0.getValueType());
2994 SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ShiftVT);
2995 AddToWorklist(Trunc.getNode());
2996 return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc);
2997 }
2998
2999 // fold (udiv x, (shl c, y)) -> x >>u (log2(c)+y) iff c is power of 2
3000 if (N1.getOpcode() == ISD::SHL) {
3001 SDValue N10 = N1.getOperand(0);
3002 if (isConstantOrConstantVector(N10, /*NoOpaques*/ true) &&
3003 DAG.isKnownToBeAPowerOfTwo(N10)) {
3004 SDValue LogBase2 = BuildLogBase2(N10, DL);
3005 AddToWorklist(LogBase2.getNode());
3006
3007 EVT ADDVT = N1.getOperand(1).getValueType();
3008 SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ADDVT);
3009 AddToWorklist(Trunc.getNode());
3010 SDValue Add = DAG.getNode(ISD::ADD, DL, ADDVT, N1.getOperand(1), Trunc);
3011 AddToWorklist(Add.getNode());
3012 return DAG.getNode(ISD::SRL, DL, VT, N0, Add);
3013 }
3014 }
3015
3016 // fold (udiv x, c) -> alternate
3017 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
3018 if (N1C && !TLI.isIntDivCheap(N->getValueType(0), Attr))
3019 if (SDValue Op = BuildUDIV(N))
3020 return Op;
3021
3022 // sdiv, srem -> sdivrem
3023 // If the divisor is constant, then return DIVREM only if isIntDivCheap() is
3024 // true. Otherwise, we break the simplification logic in visitREM().
3025 if (!N1C || TLI.isIntDivCheap(N->getValueType(0), Attr))
3026 if (SDValue DivRem = useDivRem(N))
3027 return DivRem;
3028
3029 return SDValue();
3030}
3031
3032// handles ISD::SREM and ISD::UREM
3033SDValue DAGCombiner::visitREM(SDNode *N) {
3034 unsigned Opcode = N->getOpcode();
3035 SDValue N0 = N->getOperand(0);
3036 SDValue N1 = N->getOperand(1);
3037 EVT VT = N->getValueType(0);
3038 bool isSigned = (Opcode == ISD::SREM);
3039 SDLoc DL(N);
3040
3041 // fold (rem c1, c2) -> c1%c2
3042 ConstantSDNode *N0C = isConstOrConstSplat(N0);
3043 ConstantSDNode *N1C = isConstOrConstSplat(N1);
3044 if (N0C && N1C)
3045 if (SDValue Folded = DAG.FoldConstantArithmetic(Opcode, DL, VT, N0C, N1C))
3046 return Folded;
3047
3048 if (SDValue V = simplifyDivRem(N, DAG))
3049 return V;
3050
3051 if (SDValue NewSel = foldBinOpIntoSelect(N))
3052 return NewSel;
3053
3054 if (isSigned) {
3055 // If we know the sign bits of both operands are zero, strength reduce to a
3056 // urem instead. Handles (X & 0x0FFFFFFF) %s 16 -> X&15
3057 if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
3058 return DAG.getNode(ISD::UREM, DL, VT, N0, N1);
3059 } else {
3060 SDValue NegOne = DAG.getAllOnesConstant(DL, VT);
3061 if (DAG.isKnownToBeAPowerOfTwo(N1)) {
3062 // fold (urem x, pow2) -> (and x, pow2-1)
3063 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne);
3064 AddToWorklist(Add.getNode());
3065 return DAG.getNode(ISD::AND, DL, VT, N0, Add);
3066 }
3067 if (N1.getOpcode() == ISD::SHL &&
3068 DAG.isKnownToBeAPowerOfTwo(N1.getOperand(0))) {
3069 // fold (urem x, (shl pow2, y)) -> (and x, (add (shl pow2, y), -1))
3070 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne);
3071 AddToWorklist(Add.getNode());
3072 return DAG.getNode(ISD::AND, DL, VT, N0, Add);
3073 }
3074 }
3075
3076 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
3077
3078 // If X/C can be simplified by the division-by-constant logic, lower
3079 // X%C to the equivalent of X-X/C*C.
3080 // To avoid mangling nodes, this simplification requires that the combine()
3081 // call for the speculative DIV must not cause a DIVREM conversion. We guard
3082 // against this by skipping the simplification if isIntDivCheap(). When
3083 // div is not cheap, combine will not return a DIVREM. Regardless,
3084 // checking cheapness here makes sense since the simplification results in
3085 // fatter code.
3086 if (N1C && !N1C->isNullValue() && !TLI.isIntDivCheap(VT, Attr)) {
3087 unsigned DivOpcode = isSigned ? ISD::SDIV : ISD::UDIV;
3088 SDValue Div = DAG.getNode(DivOpcode, DL, VT, N0, N1);
3089 AddToWorklist(Div.getNode());
3090 SDValue OptimizedDiv = combine(Div.getNode());
3091 if (OptimizedDiv.getNode() && OptimizedDiv.getNode() != Div.getNode()) {
3092 assert((OptimizedDiv.getOpcode() != ISD::UDIVREM) &&(static_cast <bool> ((OptimizedDiv.getOpcode() != ISD::
UDIVREM) && (OptimizedDiv.getOpcode() != ISD::SDIVREM
)) ? void (0) : __assert_fail ("(OptimizedDiv.getOpcode() != ISD::UDIVREM) && (OptimizedDiv.getOpcode() != ISD::SDIVREM)"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3093, __extension__ __PRETTY_FUNCTION__))
3093 (OptimizedDiv.getOpcode() != ISD::SDIVREM))(static_cast <bool> ((OptimizedDiv.getOpcode() != ISD::
UDIVREM) && (OptimizedDiv.getOpcode() != ISD::SDIVREM
)) ? void (0) : __assert_fail ("(OptimizedDiv.getOpcode() != ISD::UDIVREM) && (OptimizedDiv.getOpcode() != ISD::SDIVREM)"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3093, __extension__ __PRETTY_FUNCTION__))
;
3094 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, OptimizedDiv, N1);
3095 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
3096 AddToWorklist(Mul.getNode());
3097 return Sub;
3098 }
3099 }
3100
3101 // sdiv, srem -> sdivrem
3102 if (SDValue DivRem = useDivRem(N))
3103 return DivRem.getValue(1);
3104
3105 return SDValue();
3106}
3107
3108SDValue DAGCombiner::visitMULHS(SDNode *N) {
3109 SDValue N0 = N->getOperand(0);
3110 SDValue N1 = N->getOperand(1);
3111 EVT VT = N->getValueType(0);
3112 SDLoc DL(N);
3113
3114 if (VT.isVector()) {
3115 // fold (mulhs x, 0) -> 0
3116 if (ISD::isBuildVectorAllZeros(N1.getNode()))
3117 return N1;
3118 if (ISD::isBuildVectorAllZeros(N0.getNode()))
3119 return N0;
3120 }
3121
3122 // fold (mulhs x, 0) -> 0
3123 if (isNullConstant(N1))
3124 return N1;
3125 // fold (mulhs x, 1) -> (sra x, size(x)-1)
3126 if (isOneConstant(N1))
3127 return DAG.getNode(ISD::SRA, DL, N0.getValueType(), N0,
3128 DAG.getConstant(N0.getValueSizeInBits() - 1, DL,
3129 getShiftAmountTy(N0.getValueType())));
3130
3131 // fold (mulhs x, undef) -> 0
3132 if (N0.isUndef() || N1.isUndef())
3133 return DAG.getConstant(0, DL, VT);
3134
3135 // If the type twice as wide is legal, transform the mulhs to a wider multiply
3136 // plus a shift.
3137 if (VT.isSimple() && !VT.isVector()) {
3138 MVT Simple = VT.getSimpleVT();
3139 unsigned SimpleSize = Simple.getSizeInBits();
3140 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
3141 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
3142 N0 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N0);
3143 N1 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N1);
3144 N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
3145 N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
3146 DAG.getConstant(SimpleSize, DL,
3147 getShiftAmountTy(N1.getValueType())));
3148 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
3149 }
3150 }
3151
3152 return SDValue();
3153}
3154
3155SDValue DAGCombiner::visitMULHU(SDNode *N) {
3156 SDValue N0 = N->getOperand(0);
3157 SDValue N1 = N->getOperand(1);
3158 EVT VT = N->getValueType(0);
3159 SDLoc DL(N);
3160
3161 if (VT.isVector()) {
3162 // fold (mulhu x, 0) -> 0
3163 if (ISD::isBuildVectorAllZeros(N1.getNode()))
3164 return N1;
3165 if (ISD::isBuildVectorAllZeros(N0.getNode()))
3166 return N0;
3167 }
3168
3169 // fold (mulhu x, 0) -> 0
3170 if (isNullConstant(N1))
3171 return N1;
3172 // fold (mulhu x, 1) -> 0
3173 if (isOneConstant(N1))
3174 return DAG.getConstant(0, DL, N0.getValueType());
3175 // fold (mulhu x, undef) -> 0
3176 if (N0.isUndef() || N1.isUndef())
3177 return DAG.getConstant(0, DL, VT);
3178
3179 // If the type twice as wide is legal, transform the mulhu to a wider multiply
3180 // plus a shift.
3181 if (VT.isSimple() && !VT.isVector()) {
3182 MVT Simple = VT.getSimpleVT();
3183 unsigned SimpleSize = Simple.getSizeInBits();
3184 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
3185 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
3186 N0 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N0);
3187 N1 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N1);
3188 N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
3189 N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
3190 DAG.getConstant(SimpleSize, DL,
3191 getShiftAmountTy(N1.getValueType())));
3192 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
3193 }
3194 }
3195
3196 return SDValue();
3197}
3198
3199/// Perform optimizations common to nodes that compute two values. LoOp and HiOp
3200/// give the opcodes for the two computations that are being performed. Return
3201/// true if a simplification was made.
3202SDValue DAGCombiner::SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
3203 unsigned HiOp) {
3204 // If the high half is not needed, just compute the low half.
3205 bool HiExists = N->hasAnyUseOfValue(1);
3206 if (!HiExists &&
3207 (!LegalOperations ||
3208 TLI.isOperationLegalOrCustom(LoOp, N->getValueType(0)))) {
3209 SDValue Res = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0), N->ops());
3210 return CombineTo(N, Res, Res);
3211 }
3212
3213 // If the low half is not needed, just compute the high half.
3214 bool LoExists = N->hasAnyUseOfValue(0);
3215 if (!LoExists &&
3216 (!LegalOperations ||
3217 TLI.isOperationLegal(HiOp, N->getValueType(1)))) {
3218 SDValue Res = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1), N->ops());
3219 return CombineTo(N, Res, Res);
3220 }
3221
3222 // If both halves are used, return as it is.
3223 if (LoExists && HiExists)
3224 return SDValue();
3225
3226 // If the two computed results can be simplified separately, separate them.
3227 if (LoExists) {
3228 SDValue Lo = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0), N->ops());
3229 AddToWorklist(Lo.getNode());
3230 SDValue LoOpt = combine(Lo.getNode());
3231 if (LoOpt.getNode() && LoOpt.getNode() != Lo.getNode() &&
3232 (!LegalOperations ||
3233 TLI.isOperationLegal(LoOpt.getOpcode(), LoOpt.getValueType())))
3234 return CombineTo(N, LoOpt, LoOpt);
3235 }
3236
3237 if (HiExists) {
3238 SDValue Hi = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1), N->ops());
3239 AddToWorklist(Hi.getNode());
3240 SDValue HiOpt = combine(Hi.getNode());
3241 if (HiOpt.getNode() && HiOpt != Hi &&
3242 (!LegalOperations ||
3243 TLI.isOperationLegal(HiOpt.getOpcode(), HiOpt.getValueType())))
3244 return CombineTo(N, HiOpt, HiOpt);
3245 }
3246
3247 return SDValue();
3248}
3249
3250SDValue DAGCombiner::visitSMUL_LOHI(SDNode *N) {
3251 if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHS))
3252 return Res;
3253
3254 EVT VT = N->getValueType(0);
3255 SDLoc DL(N);
3256
3257 // If the type is twice as wide is legal, transform the mulhu to a wider
3258 // multiply plus a shift.
3259 if (VT.isSimple() && !VT.isVector()) {
3260 MVT Simple = VT.getSimpleVT();
3261 unsigned SimpleSize = Simple.getSizeInBits();
3262 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
3263 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
3264 SDValue Lo = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(0));
3265 SDValue Hi = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(1));
3266 Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
3267 // Compute the high part as N1.
3268 Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
3269 DAG.getConstant(SimpleSize, DL,
3270 getShiftAmountTy(Lo.getValueType())));
3271 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
3272 // Compute the low part as N0.
3273 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
3274 return CombineTo(N, Lo, Hi);
3275 }
3276 }
3277
3278 return SDValue();
3279}
3280
3281SDValue DAGCombiner::visitUMUL_LOHI(SDNode *N) {
3282 if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHU))
3283 return Res;
3284
3285 EVT VT = N->getValueType(0);
3286 SDLoc DL(N);
3287
3288 // If the type is twice as wide is legal, transform the mulhu to a wider
3289 // multiply plus a shift.
3290 if (VT.isSimple() && !VT.isVector()) {
3291 MVT Simple = VT.getSimpleVT();
3292 unsigned SimpleSize = Simple.getSizeInBits();
3293 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
3294 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
3295 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(0));
3296 SDValue Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(1));
3297 Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
3298 // Compute the high part as N1.
3299 Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
3300 DAG.getConstant(SimpleSize, DL,
3301 getShiftAmountTy(Lo.getValueType())));
3302 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
3303 // Compute the low part as N0.
3304 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
3305 return CombineTo(N, Lo, Hi);
3306 }
3307 }
3308
3309 return SDValue();
3310}
3311
3312SDValue DAGCombiner::visitSMULO(SDNode *N) {
3313 // (smulo x, 2) -> (saddo x, x)
3314 if (ConstantSDNode *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1)))
3315 if (C2->getAPIntValue() == 2)
3316 return DAG.getNode(ISD::SADDO, SDLoc(N), N->getVTList(),
3317 N->getOperand(0), N->getOperand(0));
3318
3319 return SDValue();
3320}
3321
3322SDValue DAGCombiner::visitUMULO(SDNode *N) {
3323 // (umulo x, 2) -> (uaddo x, x)
3324 if (ConstantSDNode *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1)))
3325 if (C2->getAPIntValue() == 2)
3326 return DAG.getNode(ISD::UADDO, SDLoc(N), N->getVTList(),
3327 N->getOperand(0), N->getOperand(0));
3328
3329 return SDValue();
3330}
3331
3332SDValue DAGCombiner::visitIMINMAX(SDNode *N) {
3333 SDValue N0 = N->getOperand(0);
3334 SDValue N1 = N->getOperand(1);
3335 EVT VT = N0.getValueType();
3336
3337 // fold vector ops
3338 if (VT.isVector())
3339 if (SDValue FoldedVOp = SimplifyVBinOp(N))
3340 return FoldedVOp;
3341
3342 // fold operation with constant operands.
3343 ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
3344 ConstantSDNode *N1C = getAsNonOpaqueConstant(N1);
3345 if (N0C && N1C)
3346 return DAG.FoldConstantArithmetic(N->getOpcode(), SDLoc(N), VT, N0C, N1C);
3347
3348 // canonicalize constant to RHS
3349 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
3350 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
3351 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N1, N0);
3352
3353 return SDValue();
3354}
3355
3356/// If this is a binary operator with two operands of the same opcode, try to
3357/// simplify it.
3358SDValue DAGCombiner::SimplifyBinOpWithSameOpcodeHands(SDNode *N) {
3359 SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
3360 EVT VT = N0.getValueType();
3361 assert(N0.getOpcode() == N1.getOpcode() && "Bad input!")(static_cast <bool> (N0.getOpcode() == N1.getOpcode() &&
"Bad input!") ? void (0) : __assert_fail ("N0.getOpcode() == N1.getOpcode() && \"Bad input!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3361, __extension__ __PRETTY_FUNCTION__))
;
3362
3363 // Bail early if none of these transforms apply.
3364 if (N0.getNumOperands() == 0) return SDValue();
3365
3366 // For each of OP in AND/OR/XOR:
3367 // fold (OP (zext x), (zext y)) -> (zext (OP x, y))
3368 // fold (OP (sext x), (sext y)) -> (sext (OP x, y))
3369 // fold (OP (aext x), (aext y)) -> (aext (OP x, y))
3370 // fold (OP (bswap x), (bswap y)) -> (bswap (OP x, y))
3371 // fold (OP (trunc x), (trunc y)) -> (trunc (OP x, y)) (if trunc isn't free)
3372 //
3373 // do not sink logical op inside of a vector extend, since it may combine
3374 // into a vsetcc.
3375 EVT Op0VT = N0.getOperand(0).getValueType();
3376 if ((N0.getOpcode() == ISD::ZERO_EXTEND ||
3377 N0.getOpcode() == ISD::SIGN_EXTEND ||
3378 N0.getOpcode() == ISD::BSWAP ||
3379 // Avoid infinite looping with PromoteIntBinOp.
3380 (N0.getOpcode() == ISD::ANY_EXTEND &&
3381 (!LegalTypes || TLI.isTypeDesirableForOp(N->getOpcode(), Op0VT))) ||
3382 (N0.getOpcode() == ISD::TRUNCATE &&
3383 (!TLI.isZExtFree(VT, Op0VT) ||
3384 !TLI.isTruncateFree(Op0VT, VT)) &&
3385 TLI.isTypeLegal(Op0VT))) &&
3386 !VT.isVector() &&
3387 Op0VT == N1.getOperand(0).getValueType() &&
3388 (!LegalOperations || TLI.isOperationLegal(N->getOpcode(), Op0VT))) {
3389 SDValue ORNode = DAG.getNode(N->getOpcode(), SDLoc(N0),
3390 N0.getOperand(0).getValueType(),
3391 N0.getOperand(0), N1.getOperand(0));
3392 AddToWorklist(ORNode.getNode());
3393 return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, ORNode);
3394 }
3395
3396 // For each of OP in SHL/SRL/SRA/AND...
3397 // fold (and (OP x, z), (OP y, z)) -> (OP (and x, y), z)
3398 // fold (or (OP x, z), (OP y, z)) -> (OP (or x, y), z)
3399 // fold (xor (OP x, z), (OP y, z)) -> (OP (xor x, y), z)
3400 if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL ||
3401 N0.getOpcode() == ISD::SRA || N0.getOpcode() == ISD::AND) &&
3402 N0.getOperand(1) == N1.getOperand(1)) {
3403 SDValue ORNode = DAG.getNode(N->getOpcode(), SDLoc(N0),
3404 N0.getOperand(0).getValueType(),
3405 N0.getOperand(0), N1.getOperand(0));
3406 AddToWorklist(ORNode.getNode());
3407 return DAG.getNode(N0.getOpcode(), SDLoc(N), VT,
3408 ORNode, N0.getOperand(1));
3409 }
3410
3411 // Simplify xor/and/or (bitcast(A), bitcast(B)) -> bitcast(op (A,B))
3412 // Only perform this optimization up until type legalization, before
3413 // LegalizeVectorOprs. LegalizeVectorOprs promotes vector operations by
3414 // adding bitcasts. For example (xor v4i32) is promoted to (v2i64), and
3415 // we don't want to undo this promotion.
3416 // We also handle SCALAR_TO_VECTOR because xor/or/and operations are cheaper
3417 // on scalars.
3418 if ((N0.getOpcode() == ISD::BITCAST ||
3419 N0.getOpcode() == ISD::SCALAR_TO_VECTOR) &&
3420 Level <= AfterLegalizeTypes) {
3421 SDValue In0 = N0.getOperand(0);
3422 SDValue In1 = N1.getOperand(0);
3423 EVT In0Ty = In0.getValueType();
3424 EVT In1Ty = In1.getValueType();
3425 SDLoc DL(N);
3426 // If both incoming values are integers, and the original types are the
3427 // same.
3428 if (In0Ty.isInteger() && In1Ty.isInteger() && In0Ty == In1Ty) {
3429 SDValue Op = DAG.getNode(N->getOpcode(), DL, In0Ty, In0, In1);
3430 SDValue BC = DAG.getNode(N0.getOpcode(), DL, VT, Op);
3431 AddToWorklist(Op.getNode());
3432 return BC;
3433 }
3434 }
3435
3436 // Xor/and/or are indifferent to the swizzle operation (shuffle of one value).
3437 // Simplify xor/and/or (shuff(A), shuff(B)) -> shuff(op (A,B))
3438 // If both shuffles use the same mask, and both shuffle within a single
3439 // vector, then it is worthwhile to move the swizzle after the operation.
3440 // The type-legalizer generates this pattern when loading illegal
3441 // vector types from memory. In many cases this allows additional shuffle
3442 // optimizations.
3443 // There are other cases where moving the shuffle after the xor/and/or
3444 // is profitable even if shuffles don't perform a swizzle.
3445 // If both shuffles use the same mask, and both shuffles have the same first
3446 // or second operand, then it might still be profitable to move the shuffle
3447 // after the xor/and/or operation.
3448 if (N0.getOpcode() == ISD::VECTOR_SHUFFLE && Level < AfterLegalizeDAG) {
3449 ShuffleVectorSDNode *SVN0 = cast<ShuffleVectorSDNode>(N0);
3450 ShuffleVectorSDNode *SVN1 = cast<ShuffleVectorSDNode>(N1);
3451
3452 assert(N0.getOperand(0).getValueType() == N1.getOperand(0).getValueType() &&(static_cast <bool> (N0.getOperand(0).getValueType() ==
N1.getOperand(0).getValueType() && "Inputs to shuffles are not the same type"
) ? void (0) : __assert_fail ("N0.getOperand(0).getValueType() == N1.getOperand(0).getValueType() && \"Inputs to shuffles are not the same type\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3453, __extension__ __PRETTY_FUNCTION__))
3453 "Inputs to shuffles are not the same type")(static_cast <bool> (N0.getOperand(0).getValueType() ==
N1.getOperand(0).getValueType() && "Inputs to shuffles are not the same type"
) ? void (0) : __assert_fail ("N0.getOperand(0).getValueType() == N1.getOperand(0).getValueType() && \"Inputs to shuffles are not the same type\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3453, __extension__ __PRETTY_FUNCTION__))
;
3454
3455 // Check that both shuffles use the same mask. The masks are known to be of
3456 // the same length because the result vector type is the same.
3457 // Check also that shuffles have only one use to avoid introducing extra
3458 // instructions.
3459 if (SVN0->hasOneUse() && SVN1->hasOneUse() &&
3460 SVN0->getMask().equals(SVN1->getMask())) {
3461 SDValue ShOp = N0->getOperand(1);
3462
3463 // Don't try to fold this node if it requires introducing a
3464 // build vector of all zeros that might be illegal at this stage.
3465 if (N->getOpcode() == ISD::XOR && !ShOp.isUndef()) {
3466 if (!LegalTypes)
3467 ShOp = DAG.getConstant(0, SDLoc(N), VT);
3468 else
3469 ShOp = SDValue();
3470 }
3471
3472 // (AND (shuf (A, C), shuf (B, C)) -> shuf (AND (A, B), C)
3473 // (OR (shuf (A, C), shuf (B, C)) -> shuf (OR (A, B), C)
3474 // (XOR (shuf (A, C), shuf (B, C)) -> shuf (XOR (A, B), V_0)
3475 if (N0.getOperand(1) == N1.getOperand(1) && ShOp.getNode()) {
3476 SDValue NewNode = DAG.getNode(N->getOpcode(), SDLoc(N), VT,
3477 N0->getOperand(0), N1->getOperand(0));
3478 AddToWorklist(NewNode.getNode());
3479 return DAG.getVectorShuffle(VT, SDLoc(N), NewNode, ShOp,
3480 SVN0->getMask());
3481 }
3482
3483 // Don't try to fold this node if it requires introducing a
3484 // build vector of all zeros that might be illegal at this stage.
3485 ShOp = N0->getOperand(0);
3486 if (N->getOpcode() == ISD::XOR && !ShOp.isUndef()) {
3487 if (!LegalTypes)
3488 ShOp = DAG.getConstant(0, SDLoc(N), VT);
3489 else
3490 ShOp = SDValue();
3491 }
3492
3493 // (AND (shuf (C, A), shuf (C, B)) -> shuf (C, AND (A, B))
3494 // (OR (shuf (C, A), shuf (C, B)) -> shuf (C, OR (A, B))
3495 // (XOR (shuf (C, A), shuf (C, B)) -> shuf (V_0, XOR (A, B))
3496 if (N0->getOperand(0) == N1->getOperand(0) && ShOp.getNode()) {
3497 SDValue NewNode = DAG.getNode(N->getOpcode(), SDLoc(N), VT,
3498 N0->getOperand(1), N1->getOperand(1));
3499 AddToWorklist(NewNode.getNode());
3500 return DAG.getVectorShuffle(VT, SDLoc(N), ShOp, NewNode,
3501 SVN0->getMask());
3502 }
3503 }
3504 }
3505
3506 return SDValue();
3507}
3508
3509/// Try to make (and/or setcc (LL, LR), setcc (RL, RR)) more efficient.
3510SDValue DAGCombiner::foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1,
3511 const SDLoc &DL) {
3512 SDValue LL, LR, RL, RR, N0CC, N1CC;
3513 if (!isSetCCEquivalent(N0, LL, LR, N0CC) ||
3514 !isSetCCEquivalent(N1, RL, RR, N1CC))
3515 return SDValue();
3516
3517 assert(N0.getValueType() == N1.getValueType() &&(static_cast <bool> (N0.getValueType() == N1.getValueType
() && "Unexpected operand types for bitwise logic op"
) ? void (0) : __assert_fail ("N0.getValueType() == N1.getValueType() && \"Unexpected operand types for bitwise logic op\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3518, __extension__ __PRETTY_FUNCTION__))
3518 "Unexpected operand types for bitwise logic op")(static_cast <bool> (N0.getValueType() == N1.getValueType
() && "Unexpected operand types for bitwise logic op"
) ? void (0) : __assert_fail ("N0.getValueType() == N1.getValueType() && \"Unexpected operand types for bitwise logic op\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3518, __extension__ __PRETTY_FUNCTION__))
;
3519 assert(LL.getValueType() == LR.getValueType() &&(static_cast <bool> (LL.getValueType() == LR.getValueType
() && RL.getValueType() == RR.getValueType() &&
"Unexpected operand types for setcc") ? void (0) : __assert_fail
("LL.getValueType() == LR.getValueType() && RL.getValueType() == RR.getValueType() && \"Unexpected operand types for setcc\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3521, __extension__ __PRETTY_FUNCTION__))
3520 RL.getValueType() == RR.getValueType() &&(static_cast <bool> (LL.getValueType() == LR.getValueType
() && RL.getValueType() == RR.getValueType() &&
"Unexpected operand types for setcc") ? void (0) : __assert_fail
("LL.getValueType() == LR.getValueType() && RL.getValueType() == RR.getValueType() && \"Unexpected operand types for setcc\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3521, __extension__ __PRETTY_FUNCTION__))
3521 "Unexpected operand types for setcc")(static_cast <bool> (LL.getValueType() == LR.getValueType
() && RL.getValueType() == RR.getValueType() &&
"Unexpected operand types for setcc") ? void (0) : __assert_fail
("LL.getValueType() == LR.getValueType() && RL.getValueType() == RR.getValueType() && \"Unexpected operand types for setcc\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3521, __extension__ __PRETTY_FUNCTION__))
;
3522
3523 // If we're here post-legalization or the logic op type is not i1, the logic
3524 // op type must match a setcc result type. Also, all folds require new
3525 // operations on the left and right operands, so those types must match.
3526 EVT VT = N0.getValueType();
3527 EVT OpVT = LL.getValueType();
3528 if (LegalOperations || VT != MVT::i1)
3529 if (VT != getSetCCResultType(OpVT))
3530 return SDValue();
3531 if (OpVT != RL.getValueType())
3532 return SDValue();
3533
3534 ISD::CondCode CC0 = cast<CondCodeSDNode>(N0CC)->get();
3535 ISD::CondCode CC1 = cast<CondCodeSDNode>(N1CC)->get();
3536 bool IsInteger = OpVT.isInteger();
3537 if (LR == RR && CC0 == CC1 && IsInteger) {
3538 bool IsZero = isNullConstantOrNullSplatConstant(LR);
3539 bool IsNeg1 = isAllOnesConstantOrAllOnesSplatConstant(LR);
3540
3541 // All bits clear?
3542 bool AndEqZero = IsAnd && CC1 == ISD::SETEQ && IsZero;
3543 // All sign bits clear?
3544 bool AndGtNeg1 = IsAnd && CC1 == ISD::SETGT && IsNeg1;
3545 // Any bits set?
3546 bool OrNeZero = !IsAnd && CC1 == ISD::SETNE && IsZero;
3547 // Any sign bits set?
3548 bool OrLtZero = !IsAnd && CC1 == ISD::SETLT && IsZero;
3549
3550 // (and (seteq X, 0), (seteq Y, 0)) --> (seteq (or X, Y), 0)
3551 // (and (setgt X, -1), (setgt Y, -1)) --> (setgt (or X, Y), -1)
3552 // (or (setne X, 0), (setne Y, 0)) --> (setne (or X, Y), 0)
3553 // (or (setlt X, 0), (setlt Y, 0)) --> (setlt (or X, Y), 0)
3554 if (AndEqZero || AndGtNeg1 || OrNeZero || OrLtZero) {
3555 SDValue Or = DAG.getNode(ISD::OR, SDLoc(N0), OpVT, LL, RL);
3556 AddToWorklist(Or.getNode());
3557 return DAG.getSetCC(DL, VT, Or, LR, CC1);
3558 }
3559
3560 // All bits set?
3561 bool AndEqNeg1 = IsAnd && CC1 == ISD::SETEQ && IsNeg1;
3562 // All sign bits set?
3563 bool AndLtZero = IsAnd && CC1 == ISD::SETLT && IsZero;
3564 // Any bits clear?
3565 bool OrNeNeg1 = !IsAnd && CC1 == ISD::SETNE && IsNeg1;
3566 // Any sign bits clear?
3567 bool OrGtNeg1 = !IsAnd && CC1 == ISD::SETGT && IsNeg1;
3568
3569 // (and (seteq X, -1), (seteq Y, -1)) --> (seteq (and X, Y), -1)
3570 // (and (setlt X, 0), (setlt Y, 0)) --> (setlt (and X, Y), 0)
3571 // (or (setne X, -1), (setne Y, -1)) --> (setne (and X, Y), -1)
3572 // (or (setgt X, -1), (setgt Y -1)) --> (setgt (and X, Y), -1)
3573 if (AndEqNeg1 || AndLtZero || OrNeNeg1 || OrGtNeg1) {
3574 SDValue And = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, LL, RL);
3575 AddToWorklist(And.getNode());
3576 return DAG.getSetCC(DL, VT, And, LR, CC1);
3577 }
3578 }
3579
3580 // TODO: What is the 'or' equivalent of this fold?
3581 // (and (setne X, 0), (setne X, -1)) --> (setuge (add X, 1), 2)
3582 if (IsAnd && LL == RL && CC0 == CC1 && OpVT.getScalarSizeInBits() > 1 &&
3583 IsInteger && CC0 == ISD::SETNE &&
3584 ((isNullConstant(LR) && isAllOnesConstant(RR)) ||
3585 (isAllOnesConstant(LR) && isNullConstant(RR)))) {
3586 SDValue One = DAG.getConstant(1, DL, OpVT);
3587 SDValue Two = DAG.getConstant(2, DL, OpVT);
3588 SDValue Add = DAG.getNode(ISD::ADD, SDLoc(N0), OpVT, LL, One);
3589 AddToWorklist(Add.getNode());
3590 return DAG.getSetCC(DL, VT, Add, Two, ISD::SETUGE);
3591 }
3592
3593 // Try more general transforms if the predicates match and the only user of
3594 // the compares is the 'and' or 'or'.
3595 if (IsInteger && TLI.convertSetCCLogicToBitwiseLogic(OpVT) && CC0 == CC1 &&
3596 N0.hasOneUse() && N1.hasOneUse()) {
3597 // and (seteq A, B), (seteq C, D) --> seteq (or (xor A, B), (xor C, D)), 0
3598 // or (setne A, B), (setne C, D) --> setne (or (xor A, B), (xor C, D)), 0
3599 if ((IsAnd && CC1 == ISD::SETEQ) || (!IsAnd && CC1 == ISD::SETNE)) {
3600 SDValue XorL = DAG.getNode(ISD::XOR, SDLoc(N0), OpVT, LL, LR);
3601 SDValue XorR = DAG.getNode(ISD::XOR, SDLoc(N1), OpVT, RL, RR);
3602 SDValue Or = DAG.getNode(ISD::OR, DL, OpVT, XorL, XorR);
3603 SDValue Zero = DAG.getConstant(0, DL, OpVT);
3604 return DAG.getSetCC(DL, VT, Or, Zero, CC1);
3605 }
3606 }
3607
3608 // Canonicalize equivalent operands to LL == RL.
3609 if (LL == RR && LR == RL) {
3610 CC1 = ISD::getSetCCSwappedOperands(CC1);
3611 std::swap(RL, RR);
3612 }
3613
3614 // (and (setcc X, Y, CC0), (setcc X, Y, CC1)) --> (setcc X, Y, NewCC)
3615 // (or (setcc X, Y, CC0), (setcc X, Y, CC1)) --> (setcc X, Y, NewCC)
3616 if (LL == RL && LR == RR) {
3617 ISD::CondCode NewCC = IsAnd ? ISD::getSetCCAndOperation(CC0, CC1, IsInteger)
3618 : ISD::getSetCCOrOperation(CC0, CC1, IsInteger);
3619 if (NewCC != ISD::SETCC_INVALID &&
3620 (!LegalOperations ||
3621 (TLI.isCondCodeLegal(NewCC, LL.getSimpleValueType()) &&
3622 TLI.isOperationLegal(ISD::SETCC, OpVT))))
3623 return DAG.getSetCC(DL, VT, LL, LR, NewCC);
3624 }
3625
3626 return SDValue();
3627}
3628
3629/// This contains all DAGCombine rules which reduce two values combined by
3630/// an And operation to a single value. This makes them reusable in the context
3631/// of visitSELECT(). Rules involving constants are not included as
3632/// visitSELECT() already handles those cases.
3633SDValue DAGCombiner::visitANDLike(SDValue N0, SDValue N1, SDNode *N) {
3634 EVT VT = N1.getValueType();
3635 SDLoc DL(N);
3636
3637 // fold (and x, undef) -> 0
3638 if (N0.isUndef() || N1.isUndef())
3639 return DAG.getConstant(0, DL, VT);
3640
3641 if (SDValue V = foldLogicOfSetCCs(true, N0, N1, DL))
3642 return V;
3643
3644 if (N0.getOpcode() == ISD::ADD && N1.getOpcode() == ISD::SRL &&
3645 VT.getSizeInBits() <= 64) {
3646 if (ConstantSDNode *ADDI = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
3647 if (ConstantSDNode *SRLI = dyn_cast<ConstantSDNode>(N1.getOperand(1))) {
3648 // Look for (and (add x, c1), (lshr y, c2)). If C1 wasn't a legal
3649 // immediate for an add, but it is legal if its top c2 bits are set,
3650 // transform the ADD so the immediate doesn't need to be materialized
3651 // in a register.
3652 APInt ADDC = ADDI->getAPIntValue();
3653 APInt SRLC = SRLI->getAPIntValue();
3654 if (ADDC.getMinSignedBits() <= 64 &&
3655 SRLC.ult(VT.getSizeInBits()) &&
3656 !TLI.isLegalAddImmediate(ADDC.getSExtValue())) {
3657 APInt Mask = APInt::getHighBitsSet(VT.getSizeInBits(),
3658 SRLC.getZExtValue());
3659 if (DAG.MaskedValueIsZero(N0.getOperand(1), Mask)) {
3660 ADDC |= Mask;
3661 if (TLI.isLegalAddImmediate(ADDC.getSExtValue())) {
3662 SDLoc DL0(N0);
3663 SDValue NewAdd =
3664 DAG.getNode(ISD::ADD, DL0, VT,
3665 N0.getOperand(0), DAG.getConstant(ADDC, DL, VT));
3666 CombineTo(N0.getNode(), NewAdd);
3667 // Return N so it doesn't get rechecked!
3668 return SDValue(N, 0);
3669 }
3670 }
3671 }
3672 }
3673 }
3674 }
3675
3676 // Reduce bit extract of low half of an integer to the narrower type.
3677 // (and (srl i64:x, K), KMask) ->
3678 // (i64 zero_extend (and (srl (i32 (trunc i64:x)), K)), KMask)
3679 if (N0.getOpcode() == ISD::SRL && N0.hasOneUse()) {
3680 if (ConstantSDNode *CAnd = dyn_cast<ConstantSDNode>(N1)) {
3681 if (ConstantSDNode *CShift = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
3682 unsigned Size = VT.getSizeInBits();
3683 const APInt &AndMask = CAnd->getAPIntValue();
3684 unsigned ShiftBits = CShift->getZExtValue();
3685
3686 // Bail out, this node will probably disappear anyway.
3687 if (ShiftBits == 0)
3688 return SDValue();
3689
3690 unsigned MaskBits = AndMask.countTrailingOnes();
3691 EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), Size / 2);
3692
3693 if (AndMask.isMask() &&
3694 // Required bits must not span the two halves of the integer and
3695 // must fit in the half size type.
3696 (ShiftBits + MaskBits <= Size / 2) &&
3697 TLI.isNarrowingProfitable(VT, HalfVT) &&
3698 TLI.isTypeDesirableForOp(ISD::AND, HalfVT) &&
3699 TLI.isTypeDesirableForOp(ISD::SRL, HalfVT) &&
3700 TLI.isTruncateFree(VT, HalfVT) &&
3701 TLI.isZExtFree(HalfVT, VT)) {
3702 // The isNarrowingProfitable is to avoid regressions on PPC and
3703 // AArch64 which match a few 64-bit bit insert / bit extract patterns
3704 // on downstream users of this. Those patterns could probably be
3705 // extended to handle extensions mixed in.
3706
3707 SDValue SL(N0);
3708 assert(MaskBits <= Size)(static_cast <bool> (MaskBits <= Size) ? void (0) : __assert_fail
("MaskBits <= Size", "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3708, __extension__ __PRETTY_FUNCTION__))
;
3709
3710 // Extracting the highest bit of the low half.
3711 EVT ShiftVT = TLI.getShiftAmountTy(HalfVT, DAG.getDataLayout());
3712 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, HalfVT,
3713 N0.getOperand(0));
3714
3715 SDValue NewMask = DAG.getConstant(AndMask.trunc(Size / 2), SL, HalfVT);
3716 SDValue ShiftK = DAG.getConstant(ShiftBits, SL, ShiftVT);
3717 SDValue Shift = DAG.getNode(ISD::SRL, SL, HalfVT, Trunc, ShiftK);
3718 SDValue And = DAG.getNode(ISD::AND, SL, HalfVT, Shift, NewMask);
3719 return DAG.getNode(ISD::ZERO_EXTEND, SL, VT, And);
3720 }
3721 }
3722 }
3723 }
3724
3725 return SDValue();
3726}
3727
3728bool DAGCombiner::isAndLoadExtLoad(ConstantSDNode *AndC, LoadSDNode *LoadN,
3729 EVT LoadResultTy, EVT &ExtVT) {
3730 if (!AndC->getAPIntValue().isMask())
3731 return false;
3732
3733 unsigned ActiveBits = AndC->getAPIntValue().countTrailingOnes();
3734
3735 ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
3736 EVT LoadedVT = LoadN->getMemoryVT();
3737
3738 if (ExtVT == LoadedVT &&
3739 (!LegalOperations ||
3740 TLI.isLoadExtLegal(ISD::ZEXTLOAD, LoadResultTy, ExtVT))) {
3741 // ZEXTLOAD will match without needing to change the size of the value being
3742 // loaded.
3743 return true;
3744 }
3745
3746 // Do not change the width of a volatile load.
3747 if (LoadN->isVolatile())
3748 return false;
3749
3750 // Do not generate loads of non-round integer types since these can
3751 // be expensive (and would be wrong if the type is not byte sized).
3752 if (!LoadedVT.bitsGT(ExtVT) || !ExtVT.isRound())
3753 return false;
3754
3755 if (LegalOperations &&
3756 !TLI.isLoadExtLegal(ISD::ZEXTLOAD, LoadResultTy, ExtVT))
3757 return false;
3758
3759 if (!TLI.shouldReduceLoadWidth(LoadN, ISD::ZEXTLOAD, ExtVT))
3760 return false;
3761
3762 return true;
3763}
3764
3765bool DAGCombiner::isLegalNarrowLoad(LoadSDNode *LoadN, ISD::LoadExtType ExtType,
3766 EVT &ExtVT, unsigned ShAmt) {
3767 // Don't transform one with multiple uses, this would require adding a new
3768 // load.
3769 if (!SDValue(LoadN, 0).hasOneUse())
3770 return false;
3771
3772 if (LegalOperations &&
3773 !TLI.isLoadExtLegal(ExtType, LoadN->getValueType(0), ExtVT))
3774 return false;
3775
3776 // Do not generate loads of non-round integer types since these can
3777 // be expensive (and would be wrong if the type is not byte sized).
3778 if (!ExtVT.isRound())
3779 return false;
3780
3781 // Don't change the width of a volatile load.
3782 if (LoadN->isVolatile())
3783 return false;
3784
3785 // Verify that we are actually reducing a load width here.
3786 if (LoadN->getMemoryVT().getSizeInBits() < ExtVT.getSizeInBits())
3787 return false;
3788
3789 // For the transform to be legal, the load must produce only two values
3790 // (the value loaded and the chain). Don't transform a pre-increment
3791 // load, for example, which produces an extra value. Otherwise the
3792 // transformation is not equivalent, and the downstream logic to replace
3793 // uses gets things wrong.
3794 if (LoadN->getNumValues() > 2)
3795 return false;
3796
3797 // If the load that we're shrinking is an extload and we're not just
3798 // discarding the extension we can't simply shrink the load. Bail.
3799 // TODO: It would be possible to merge the extensions in some cases.
3800 if (LoadN->getExtensionType() != ISD::NON_EXTLOAD &&
3801 LoadN->getMemoryVT().getSizeInBits() < ExtVT.getSizeInBits() + ShAmt)
3802 return false;
3803
3804 if (!TLI.shouldReduceLoadWidth(LoadN, ExtType, ExtVT))
3805 return false;
3806
3807 // It's not possible to generate a constant of extended or untyped type.
3808 EVT PtrType = LoadN->getOperand(1).getValueType();
3809 if (PtrType == MVT::Untyped || PtrType.isExtended())
3810 return false;
3811
3812 return true;
3813}
3814
3815bool DAGCombiner::SearchForAndLoads(SDNode *N,
3816 SmallPtrSetImpl<LoadSDNode*> &Loads,
3817 SmallPtrSetImpl<SDNode*> &NodesWithConsts,
3818 ConstantSDNode *Mask,
3819 SDNode *&NodeToMask) {
3820 // Recursively search for the operands, looking for loads which can be
3821 // narrowed.
3822 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) {
3823 SDValue Op = N->getOperand(i);
3824
3825 if (Op.getValueType().isVector())
3826 return false;
3827
3828 // Some constants may need fixing up later if they are too large.
3829 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
3830 if ((N->getOpcode() == ISD::OR || N->getOpcode() == ISD::XOR) &&
3831 (Mask->getAPIntValue() & C->getAPIntValue()) != C->getAPIntValue())
3832 NodesWithConsts.insert(N);
3833 continue;
3834 }
3835
3836 if (!Op.hasOneUse())
3837 return false;
3838
3839 switch(Op.getOpcode()) {
3840 case ISD::LOAD: {
3841 auto *Load = cast<LoadSDNode>(Op);
3842 EVT ExtVT;
3843 if (isAndLoadExtLoad(Mask, Load, Load->getValueType(0), ExtVT) &&
3844 isLegalNarrowLoad(Load, ISD::ZEXTLOAD, ExtVT)) {
3845 // Only add this load if we can make it more narrow.
3846 if (ExtVT.bitsLT(Load->getMemoryVT()))
3847 Loads.insert(Load);
3848 continue;
3849 }
3850 return false;
3851 }
3852 case ISD::ZERO_EXTEND:
3853 case ISD::ANY_EXTEND:
3854 case ISD::AssertZext: {
3855 unsigned ActiveBits = Mask->getAPIntValue().countTrailingOnes();
3856 EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
3857 EVT VT = Op.getOpcode() == ISD::AssertZext ?
3858 cast<VTSDNode>(Op.getOperand(1))->getVT() :
3859 Op.getOperand(0).getValueType();
3860
3861 // We can accept extending nodes if the mask is wider or an equal
3862 // width to the original type.
3863 if (ExtVT.bitsGE(VT))
3864 continue;
3865 break;
3866 }
3867 case ISD::OR:
3868 case ISD::XOR:
3869 case ISD::AND:
3870 if (!SearchForAndLoads(Op.getNode(), Loads, NodesWithConsts, Mask,
3871 NodeToMask))
3872 return false;
3873 continue;
3874 }
3875
3876 // Allow one node which will masked along with any loads found.
3877 if (NodeToMask)
3878 return false;
3879 NodeToMask = Op.getNode();
3880 }
3881 return true;
3882}
3883
3884bool DAGCombiner::BackwardsPropagateMask(SDNode *N, SelectionDAG &DAG) {
3885 auto *Mask = dyn_cast<ConstantSDNode>(N->getOperand(1));
3886 if (!Mask)
3887 return false;
3888
3889 if (!Mask->getAPIntValue().isMask())
3890 return false;
3891
3892 // No need to do anything if the and directly uses a load.
3893 if (isa<LoadSDNode>(N->getOperand(0)))
3894 return false;
3895
3896 SmallPtrSet<LoadSDNode*, 8> Loads;
3897 SmallPtrSet<SDNode*, 2> NodesWithConsts;
3898 SDNode *FixupNode = nullptr;
3899 if (SearchForAndLoads(N, Loads, NodesWithConsts, Mask, FixupNode)) {
3900 if (Loads.size() == 0)
3901 return false;
3902
3903 SDValue MaskOp = N->getOperand(1);
3904
3905 // If it exists, fixup the single node we allow in the tree that needs
3906 // masking.
3907 if (FixupNode) {
3908 SDValue And = DAG.getNode(ISD::AND, SDLoc(FixupNode),
3909 FixupNode->getValueType(0),
3910 SDValue(FixupNode, 0), MaskOp);
3911 DAG.ReplaceAllUsesOfValueWith(SDValue(FixupNode, 0), And);
3912 DAG.UpdateNodeOperands(And.getNode(), SDValue(FixupNode, 0),
3913 MaskOp);
3914 }
3915
3916 // Narrow any constants that need it.
3917 for (auto *LogicN : NodesWithConsts) {
3918 auto *C = cast<ConstantSDNode>(LogicN->getOperand(1));
3919 SDValue And = DAG.getNode(ISD::AND, SDLoc(C), C->getValueType(0),
3920 SDValue(C, 0), MaskOp);
3921 DAG.UpdateNodeOperands(LogicN, LogicN->getOperand(0), And);
3922 }
3923
3924 // Create narrow loads.
3925 for (auto *Load : Loads) {
3926 SDValue And = DAG.getNode(ISD::AND, SDLoc(Load), Load->getValueType(0),
3927 SDValue(Load, 0), MaskOp);
3928 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), And);
3929 DAG.UpdateNodeOperands(And.getNode(), SDValue(Load, 0), MaskOp);
3930 SDValue NewLoad = ReduceLoadWidth(And.getNode());
3931 assert(NewLoad &&(static_cast <bool> (NewLoad && "Shouldn't be masking the load if it can't be narrowed"
) ? void (0) : __assert_fail ("NewLoad && \"Shouldn't be masking the load if it can't be narrowed\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3932, __extension__ __PRETTY_FUNCTION__))
3932 "Shouldn't be masking the load if it can't be narrowed")(static_cast <bool> (NewLoad && "Shouldn't be masking the load if it can't be narrowed"
) ? void (0) : __assert_fail ("NewLoad && \"Shouldn't be masking the load if it can't be narrowed\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 3932, __extension__ __PRETTY_FUNCTION__))
;
3933 CombineTo(Load, NewLoad, NewLoad.getValue(1));
3934 }
3935 DAG.ReplaceAllUsesWith(N, N->getOperand(0).getNode());
3936 return true;
3937 }
3938 return false;
3939}
3940
3941SDValue DAGCombiner::visitAND(SDNode *N) {
3942 SDValue N0 = N->getOperand(0);
3943 SDValue N1 = N->getOperand(1);
3944 EVT VT = N1.getValueType();
3945
3946 // x & x --> x
3947 if (N0 == N1)
3948 return N0;
3949
3950 // fold vector ops
3951 if (VT.isVector()) {
3952 if (SDValue FoldedVOp = SimplifyVBinOp(N))
3953 return FoldedVOp;
3954
3955 // fold (and x, 0) -> 0, vector edition
3956 if (ISD::isBuildVectorAllZeros(N0.getNode()))
3957 // do not return N0, because undef node may exist in N0
3958 return DAG.getConstant(APInt::getNullValue(N0.getScalarValueSizeInBits()),
3959 SDLoc(N), N0.getValueType());
3960 if (ISD::isBuildVectorAllZeros(N1.getNode()))
3961 // do not return N1, because undef node may exist in N1
3962 return DAG.getConstant(APInt::getNullValue(N1.getScalarValueSizeInBits()),
3963 SDLoc(N), N1.getValueType());
3964
3965 // fold (and x, -1) -> x, vector edition
3966 if (ISD::isBuildVectorAllOnes(N0.getNode()))
3967 return N1;
3968 if (ISD::isBuildVectorAllOnes(N1.getNode()))
3969 return N0;
3970 }
3971
3972 // fold (and c1, c2) -> c1&c2
3973 ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
3974 ConstantSDNode *N1C = isConstOrConstSplat(N1);
3975 if (N0C && N1C && !N1C->isOpaque())
3976 return DAG.FoldConstantArithmetic(ISD::AND, SDLoc(N), VT, N0C, N1C);
3977 // canonicalize constant to RHS
3978 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
3979 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
3980 return DAG.getNode(ISD::AND, SDLoc(N), VT, N1, N0);
3981 // fold (and x, -1) -> x
3982 if (isAllOnesConstant(N1))
3983 return N0;
3984 // if (and x, c) is known to be zero, return 0
3985 unsigned BitWidth = VT.getScalarSizeInBits();
3986 if (N1C && DAG.MaskedValueIsZero(SDValue(N, 0),
3987 APInt::getAllOnesValue(BitWidth)))
3988 return DAG.getConstant(0, SDLoc(N), VT);
3989
3990 if (SDValue NewSel = foldBinOpIntoSelect(N))
3991 return NewSel;
3992
3993 // reassociate and
3994 if (SDValue RAND = ReassociateOps(ISD::AND, SDLoc(N), N0, N1))
3995 return RAND;
3996
3997 // Try to convert a constant mask AND into a shuffle clear mask.
3998 if (VT.isVector())
3999 if (SDValue Shuffle = XformToShuffleWithZero(N))
4000 return Shuffle;
4001
4002 // fold (and (or x, C), D) -> D if (C & D) == D
4003 auto MatchSubset = [](ConstantSDNode *LHS, ConstantSDNode *RHS) {
4004 return RHS->getAPIntValue().isSubsetOf(LHS->getAPIntValue());
4005 };
4006 if (N0.getOpcode() == ISD::OR &&
4007 matchBinaryPredicate(N0.getOperand(1), N1, MatchSubset))
4008 return N1;
4009 // fold (and (any_ext V), c) -> (zero_ext V) if 'and' only clears top bits.
4010 if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) {
4011 SDValue N0Op0 = N0.getOperand(0);
4012 APInt Mask = ~N1C->getAPIntValue();
4013 Mask = Mask.trunc(N0Op0.getScalarValueSizeInBits());
4014 if (DAG.MaskedValueIsZero(N0Op0, Mask)) {
4015 SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N),
4016 N0.getValueType(), N0Op0);
4017
4018 // Replace uses of the AND with uses of the Zero extend node.
4019 CombineTo(N, Zext);
4020
4021 // We actually want to replace all uses of the any_extend with the
4022 // zero_extend, to avoid duplicating things. This will later cause this
4023 // AND to be folded.
4024 CombineTo(N0.getNode(), Zext);
4025 return SDValue(N, 0); // Return N so it doesn't get rechecked!
4026 }
4027 }
4028 // similarly fold (and (X (load ([non_ext|any_ext|zero_ext] V))), c) ->
4029 // (X (load ([non_ext|zero_ext] V))) if 'and' only clears top bits which must
4030 // already be zero by virtue of the width of the base type of the load.
4031 //
4032 // the 'X' node here can either be nothing or an extract_vector_elt to catch
4033 // more cases.
4034 if ((N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
4035 N0.getValueSizeInBits() == N0.getOperand(0).getScalarValueSizeInBits() &&
4036 N0.getOperand(0).getOpcode() == ISD::LOAD &&
4037 N0.getOperand(0).getResNo() == 0) ||
4038 (N0.getOpcode() == ISD::LOAD && N0.getResNo() == 0)) {
4039 LoadSDNode *Load = cast<LoadSDNode>( (N0.getOpcode() == ISD::LOAD) ?
4040 N0 : N0.getOperand(0) );
4041
4042 // Get the constant (if applicable) the zero'th operand is being ANDed with.
4043 // This can be a pure constant or a vector splat, in which case we treat the
4044 // vector as a scalar and use the splat value.
4045 APInt Constant = APInt::getNullValue(1);
4046 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) {
4047 Constant = C->getAPIntValue();
4048 } else if (BuildVectorSDNode *Vector = dyn_cast<BuildVectorSDNode>(N1)) {
4049 APInt SplatValue, SplatUndef;
4050 unsigned SplatBitSize;
4051 bool HasAnyUndefs;
4052 bool IsSplat = Vector->isConstantSplat(SplatValue, SplatUndef,
4053 SplatBitSize, HasAnyUndefs);
4054 if (IsSplat) {
4055 // Undef bits can contribute to a possible optimisation if set, so
4056 // set them.
4057 SplatValue |= SplatUndef;
4058
4059 // The splat value may be something like "0x00FFFFFF", which means 0 for
4060 // the first vector value and FF for the rest, repeating. We need a mask
4061 // that will apply equally to all members of the vector, so AND all the
4062 // lanes of the constant together.
4063 EVT VT = Vector->getValueType(0);
4064 unsigned BitWidth = VT.getScalarSizeInBits();
4065
4066 // If the splat value has been compressed to a bitlength lower
4067 // than the size of the vector lane, we need to re-expand it to
4068 // the lane size.
4069 if (BitWidth > SplatBitSize)
4070 for (SplatValue = SplatValue.zextOrTrunc(BitWidth);
4071 SplatBitSize < BitWidth;
4072 SplatBitSize = SplatBitSize * 2)
4073 SplatValue |= SplatValue.shl(SplatBitSize);
4074
4075 // Make sure that variable 'Constant' is only set if 'SplatBitSize' is a
4076 // multiple of 'BitWidth'. Otherwise, we could propagate a wrong value.
4077 if (SplatBitSize % BitWidth == 0) {
4078 Constant = APInt::getAllOnesValue(BitWidth);
4079 for (unsigned i = 0, n = SplatBitSize/BitWidth; i < n; ++i)
4080 Constant &= SplatValue.lshr(i*BitWidth).zextOrTrunc(BitWidth);
4081 }
4082 }
4083 }
4084
4085 // If we want to change an EXTLOAD to a ZEXTLOAD, ensure a ZEXTLOAD is
4086 // actually legal and isn't going to get expanded, else this is a false
4087 // optimisation.
4088 bool CanZextLoadProfitably = TLI.isLoadExtLegal(ISD::ZEXTLOAD,
4089 Load->getValueType(0),
4090 Load->getMemoryVT());
4091
4092 // Resize the constant to the same size as the original memory access before
4093 // extension. If it is still the AllOnesValue then this AND is completely
4094 // unneeded.
4095 Constant = Constant.zextOrTrunc(Load->getMemoryVT().getScalarSizeInBits());
4096
4097 bool B;
4098 switch (Load->getExtensionType()) {
4099 default: B = false; break;
4100 case ISD::EXTLOAD: B = CanZextLoadProfitably; break;
4101 case ISD::ZEXTLOAD:
4102 case ISD::NON_EXTLOAD: B = true; break;
4103 }
4104
4105 if (B && Constant.isAllOnesValue()) {
4106 // If the load type was an EXTLOAD, convert to ZEXTLOAD in order to
4107 // preserve semantics once we get rid of the AND.
4108 SDValue NewLoad(Load, 0);
4109
4110 // Fold the AND away. NewLoad may get replaced immediately.
4111 CombineTo(N, (N0.getNode() == Load) ? NewLoad : N0);
4112
4113 if (Load->getExtensionType() == ISD::EXTLOAD) {
4114 NewLoad = DAG.getLoad(Load->getAddressingMode(), ISD::ZEXTLOAD,
4115 Load->getValueType(0), SDLoc(Load),
4116 Load->getChain(), Load->getBasePtr(),
4117 Load->getOffset(), Load->getMemoryVT(),
4118 Load->getMemOperand());
4119 // Replace uses of the EXTLOAD with the new ZEXTLOAD.
4120 if (Load->getNumValues() == 3) {
4121 // PRE/POST_INC loads have 3 values.
4122 SDValue To[] = { NewLoad.getValue(0), NewLoad.getValue(1),
4123 NewLoad.getValue(2) };
4124 CombineTo(Load, To, 3, true);
4125 } else {
4126 CombineTo(Load, NewLoad.getValue(0), NewLoad.getValue(1));
4127 }
4128 }
4129
4130 return SDValue(N, 0); // Return N so it doesn't get rechecked!
4131 }
4132 }
4133
4134 // fold (and (load x), 255) -> (zextload x, i8)
4135 // fold (and (extload x, i16), 255) -> (zextload x, i8)
4136 // fold (and (any_ext (extload x, i16)), 255) -> (zextload x, i8)
4137 if (!VT.isVector() && N1C && (N0.getOpcode() == ISD::LOAD ||
4138 (N0.getOpcode() == ISD::ANY_EXTEND &&
4139 N0.getOperand(0).getOpcode() == ISD::LOAD))) {
4140 if (SDValue Res = ReduceLoadWidth(N)) {
4141 LoadSDNode *LN0 = N0->getOpcode() == ISD::ANY_EXTEND
4142 ? cast<LoadSDNode>(N0.getOperand(0)) : cast<LoadSDNode>(N0);
4143
4144 AddToWorklist(N);
4145 CombineTo(LN0, Res, Res.getValue(1));
4146 return SDValue(N, 0);
4147 }
4148 }
4149
4150 if (Level >= AfterLegalizeTypes) {
4151 // Attempt to propagate the AND back up to the leaves which, if they're
4152 // loads, can be combined to narrow loads and the AND node can be removed.
4153 // Perform after legalization so that extend nodes will already be
4154 // combined into the loads.
4155 if (BackwardsPropagateMask(N, DAG)) {
4156 return SDValue(N, 0);
4157 }
4158 }
4159
4160 if (SDValue Combined = visitANDLike(N0, N1, N))
4161 return Combined;
4162
4163 // Simplify: (and (op x...), (op y...)) -> (op (and x, y))
4164 if (N0.getOpcode() == N1.getOpcode())
4165 if (SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N))
4166 return Tmp;
4167
4168 // Masking the negated extension of a boolean is just the zero-extended
4169 // boolean:
4170 // and (sub 0, zext(bool X)), 1 --> zext(bool X)
4171 // and (sub 0, sext(bool X)), 1 --> zext(bool X)
4172 //
4173 // Note: the SimplifyDemandedBits fold below can make an information-losing
4174 // transform, and then we have no way to find this better fold.
4175 if (N1C && N1C->isOne() && N0.getOpcode() == ISD::SUB) {
4176 if (isNullConstantOrNullSplatConstant(N0.getOperand(0))) {
4177 SDValue SubRHS = N0.getOperand(1);
4178 if (SubRHS.getOpcode() == ISD::ZERO_EXTEND &&
4179 SubRHS.getOperand(0).getScalarValueSizeInBits() == 1)
4180 return SubRHS;
4181 if (SubRHS.getOpcode() == ISD::SIGN_EXTEND &&
4182 SubRHS.getOperand(0).getScalarValueSizeInBits() == 1)
4183 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, SubRHS.getOperand(0));
4184 }
4185 }
4186
4187 // fold (and (sign_extend_inreg x, i16 to i32), 1) -> (and x, 1)
4188 // fold (and (sra)) -> (and (srl)) when possible.
4189 if (SimplifyDemandedBits(SDValue(N, 0)))
4190 return SDValue(N, 0);
4191
4192 // fold (zext_inreg (extload x)) -> (zextload x)
4193 if (ISD::isEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode())) {
4194 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
4195 EVT MemVT = LN0->getMemoryVT();
4196 // If we zero all the possible extended bits, then we can turn this into
4197 // a zextload if we are running before legalize or the operation is legal.
4198 unsigned BitWidth = N1.getScalarValueSizeInBits();
4199 if (DAG.MaskedValueIsZero(N1, APInt::getHighBitsSet(BitWidth,
4200 BitWidth - MemVT.getScalarSizeInBits())) &&
4201 ((!LegalOperations && !LN0->isVolatile()) ||
4202 TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT))) {
4203 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT,
4204 LN0->getChain(), LN0->getBasePtr(),
4205 MemVT, LN0->getMemOperand());
4206 AddToWorklist(N);
4207 CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
4208 return SDValue(N, 0); // Return N so it doesn't get rechecked!
4209 }
4210 }
4211 // fold (zext_inreg (sextload x)) -> (zextload x) iff load has one use
4212 if (ISD::isSEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) &&
4213 N0.hasOneUse()) {
4214 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
4215 EVT MemVT = LN0->getMemoryVT();
4216 // If we zero all the possible extended bits, then we can turn this into
4217 // a zextload if we are running before legalize or the operation is legal.
4218 unsigned BitWidth = N1.getScalarValueSizeInBits();
4219 if (DAG.MaskedValueIsZero(N1, APInt::getHighBitsSet(BitWidth,
4220 BitWidth - MemVT.getScalarSizeInBits())) &&
4221 ((!LegalOperations && !LN0->isVolatile()) ||
4222 TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT))) {
4223 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT,
4224 LN0->getChain(), LN0->getBasePtr(),
4225 MemVT, LN0->getMemOperand());
4226 AddToWorklist(N);
4227 CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
4228 return SDValue(N, 0); // Return N so it doesn't get rechecked!
4229 }
4230 }
4231 // fold (and (or (srl N, 8), (shl N, 8)), 0xffff) -> (srl (bswap N), const)
4232 if (N1C && N1C->getAPIntValue() == 0xffff && N0.getOpcode() == ISD::OR) {
4233 if (SDValue BSwap = MatchBSwapHWordLow(N0.getNode(), N0.getOperand(0),
4234 N0.getOperand(1), false))
4235 return BSwap;
4236 }
4237
4238 return SDValue();
4239}
4240
4241/// Match (a >> 8) | (a << 8) as (bswap a) >> 16.
4242SDValue DAGCombiner::MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
4243 bool DemandHighBits) {
4244 if (!LegalOperations)
4245 return SDValue();
4246
4247 EVT VT = N->getValueType(0);
4248 if (VT != MVT::i64 && VT != MVT::i32 && VT != MVT::i16)
4249 return SDValue();
4250 if (!TLI.isOperationLegalOrCustom(ISD::BSWAP, VT))
4251 return SDValue();
4252
4253 // Recognize (and (shl a, 8), 0xff00), (and (srl a, 8), 0xff)
4254 bool LookPassAnd0 = false;
4255 bool LookPassAnd1 = false;
4256 if (N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::SRL)
4257 std::swap(N0, N1);
4258 if (N1.getOpcode() == ISD::AND && N1.getOperand(0).getOpcode() == ISD::SHL)
4259 std::swap(N0, N1);
4260 if (N0.getOpcode() == ISD::AND) {
4261 if (!N0.getNode()->hasOneUse())
4262 return SDValue();
4263 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
4264 if (!N01C || N01C->getZExtValue() != 0xFF00)
4265 return SDValue();
4266 N0 = N0.getOperand(0);
4267 LookPassAnd0 = true;
4268 }
4269
4270 if (N1.getOpcode() == ISD::AND) {
4271 if (!N1.getNode()->hasOneUse())
4272 return SDValue();
4273 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
4274 if (!N11C || N11C->getZExtValue() != 0xFF)
4275 return SDValue();
4276 N1 = N1.getOperand(0);
4277 LookPassAnd1 = true;
4278 }
4279
4280 if (N0.getOpcode() == ISD::SRL && N1.getOpcode() == ISD::SHL)
4281 std::swap(N0, N1);
4282 if (N0.getOpcode() != ISD::SHL || N1.getOpcode() != ISD::SRL)
4283 return SDValue();
4284 if (!N0.getNode()->hasOneUse() || !N1.getNode()->hasOneUse())
4285 return SDValue();
4286
4287 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
4288 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
4289 if (!N01C || !N11C)
4290 return SDValue();
4291 if (N01C->getZExtValue() != 8 || N11C->getZExtValue() != 8)
4292 return SDValue();
4293
4294 // Look for (shl (and a, 0xff), 8), (srl (and a, 0xff00), 8)
4295 SDValue N00 = N0->getOperand(0);
4296 if (!LookPassAnd0 && N00.getOpcode() == ISD::AND) {
4297 if (!N00.getNode()->hasOneUse())
4298 return SDValue();
4299 ConstantSDNode *N001C = dyn_cast<ConstantSDNode>(N00.getOperand(1));
4300 if (!N001C || N001C->getZExtValue() != 0xFF)
4301 return SDValue();
4302 N00 = N00.getOperand(0);
4303 LookPassAnd0 = true;
4304 }
4305
4306 SDValue N10 = N1->getOperand(0);
4307 if (!LookPassAnd1 && N10.getOpcode() == ISD::AND) {
4308 if (!N10.getNode()->hasOneUse())
4309 return SDValue();
4310 ConstantSDNode *N101C = dyn_cast<ConstantSDNode>(N10.getOperand(1));
4311 if (!N101C || N101C->getZExtValue() != 0xFF00)
4312 return SDValue();
4313 N10 = N10.getOperand(0);
4314 LookPassAnd1 = true;
4315 }
4316
4317 if (N00 != N10)
4318 return SDValue();
4319
4320 // Make sure everything beyond the low halfword gets set to zero since the SRL
4321 // 16 will clear the top bits.
4322 unsigned OpSizeInBits = VT.getSizeInBits();
4323 if (DemandHighBits && OpSizeInBits > 16) {
4324 // If the left-shift isn't masked out then the only way this is a bswap is
4325 // if all bits beyond the low 8 are 0. In that case the entire pattern
4326 // reduces to a left shift anyway: leave it for other parts of the combiner.
4327 if (!LookPassAnd0)
4328 return SDValue();
4329
4330 // However, if the right shift isn't masked out then it might be because
4331 // it's not needed. See if we can spot that too.
4332 if (!LookPassAnd1 &&
4333 !DAG.MaskedValueIsZero(
4334 N10, APInt::getHighBitsSet(OpSizeInBits, OpSizeInBits - 16)))
4335 return SDValue();
4336 }
4337
4338 SDValue Res = DAG.getNode(ISD::BSWAP, SDLoc(N), VT, N00);
4339 if (OpSizeInBits > 16) {
4340 SDLoc DL(N);
4341 Res = DAG.getNode(ISD::SRL, DL, VT, Res,
4342 DAG.getConstant(OpSizeInBits - 16, DL,
4343 getShiftAmountTy(VT)));
4344 }
4345 return Res;
4346}
4347
4348/// Return true if the specified node is an element that makes up a 32-bit
4349/// packed halfword byteswap.
4350/// ((x & 0x000000ff) << 8) |
4351/// ((x & 0x0000ff00) >> 8) |
4352/// ((x & 0x00ff0000) << 8) |
4353/// ((x & 0xff000000) >> 8)
4354static bool isBSwapHWordElement(SDValue N, MutableArrayRef<SDNode *> Parts) {
4355 if (!N.getNode()->hasOneUse())
4356 return false;
4357
4358 unsigned Opc = N.getOpcode();
4359 if (Opc != ISD::AND && Opc != ISD::SHL && Opc != ISD::SRL)
4360 return false;
4361
4362 SDValue N0 = N.getOperand(0);
4363 unsigned Opc0 = N0.getOpcode();
4364 if (Opc0 != ISD::AND && Opc0 != ISD::SHL && Opc0 != ISD::SRL)
4365 return false;
4366
4367 ConstantSDNode *N1C = nullptr;
4368 // SHL or SRL: look upstream for AND mask operand
4369 if (Opc == ISD::AND)
4370 N1C = dyn_cast<ConstantSDNode>(N.getOperand(1));
4371 else if (Opc0 == ISD::AND)
4372 N1C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
4373 if (!N1C)
4374 return false;
4375
4376 unsigned MaskByteOffset;
4377 switch (N1C->getZExtValue()) {
4378 default:
4379 return false;
4380 case 0xFF: MaskByteOffset = 0; break;
4381 case 0xFF00: MaskByteOffset = 1; break;
4382 case 0xFF0000: MaskByteOffset = 2; break;
4383 case 0xFF000000: MaskByteOffset = 3; break;
4384 }
4385
4386 // Look for (x & 0xff) << 8 as well as ((x << 8) & 0xff00).
4387 if (Opc == ISD::AND) {
4388 if (MaskByteOffset == 0 || MaskByteOffset == 2) {
4389 // (x >> 8) & 0xff
4390 // (x >> 8) & 0xff0000
4391 if (Opc0 != ISD::SRL)
4392 return false;
4393 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
4394 if (!C || C->getZExtValue() != 8)
4395 return false;
4396 } else {
4397 // (x << 8) & 0xff00
4398 // (x << 8) & 0xff000000
4399 if (Opc0 != ISD::SHL)
4400 return false;
4401 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
4402 if (!C || C->getZExtValue() != 8)
4403 return false;
4404 }
4405 } else if (Opc == ISD::SHL) {
4406 // (x & 0xff) << 8
4407 // (x & 0xff0000) << 8
4408 if (MaskByteOffset != 0 && MaskByteOffset != 2)
4409 return false;
4410 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
4411 if (!C || C->getZExtValue() != 8)
4412 return false;
4413 } else { // Opc == ISD::SRL
4414 // (x & 0xff00) >> 8
4415 // (x & 0xff000000) >> 8
4416 if (MaskByteOffset != 1 && MaskByteOffset != 3)
4417 return false;
4418 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
4419 if (!C || C->getZExtValue() != 8)
4420 return false;
4421 }
4422
4423 if (Parts[MaskByteOffset])
4424 return false;
4425
4426 Parts[MaskByteOffset] = N0.getOperand(0).getNode();
4427 return true;
4428}
4429
4430/// Match a 32-bit packed halfword bswap. That is
4431/// ((x & 0x000000ff) << 8) |
4432/// ((x & 0x0000ff00) >> 8) |
4433/// ((x & 0x00ff0000) << 8) |
4434/// ((x & 0xff000000) >> 8)
4435/// => (rotl (bswap x), 16)
4436SDValue DAGCombiner::MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1) {
4437 if (!LegalOperations)
4438 return SDValue();
4439
4440 EVT VT = N->getValueType(0);
4441 if (VT != MVT::i32)
4442 return SDValue();
4443 if (!TLI.isOperationLegalOrCustom(ISD::BSWAP, VT))
4444 return SDValue();
4445
4446 // Look for either
4447 // (or (or (and), (and)), (or (and), (and)))
4448 // (or (or (or (and), (and)), (and)), (and))
4449 if (N0.getOpcode() != ISD::OR)
4450 return SDValue();
4451 SDValue N00 = N0.getOperand(0);
4452 SDValue N01 = N0.getOperand(1);
4453 SDNode *Parts[4] = {};
4454
4455 if (N1.getOpcode() == ISD::OR &&
4456 N00.getNumOperands() == 2 && N01.getNumOperands() == 2) {
4457 // (or (or (and), (and)), (or (and), (and)))
4458 if (!isBSwapHWordElement(N00, Parts))
4459 return SDValue();
4460
4461 if (!isBSwapHWordElement(N01, Parts))
4462 return SDValue();
4463 SDValue N10 = N1.getOperand(0);
4464 if (!isBSwapHWordElement(N10, Parts))
4465 return SDValue();
4466 SDValue N11 = N1.getOperand(1);
4467 if (!isBSwapHWordElement(N11, Parts))
4468 return SDValue();
4469 } else {
4470 // (or (or (or (and), (and)), (and)), (and))
4471 if (!isBSwapHWordElement(N1, Parts))
4472 return SDValue();
4473 if (!isBSwapHWordElement(N01, Parts))
4474 return SDValue();
4475 if (N00.getOpcode() != ISD::OR)
4476 return SDValue();
4477 SDValue N000 = N00.getOperand(0);
4478 if (!isBSwapHWordElement(N000, Parts))
4479 return SDValue();
4480 SDValue N001 = N00.getOperand(1);
4481 if (!isBSwapHWordElement(N001, Parts))
4482 return SDValue();
4483 }
4484
4485 // Make sure the parts are all coming from the same node.
4486 if (Parts[0] != Parts[1] || Parts[0] != Parts[2] || Parts[0] != Parts[3])
4487 return SDValue();
4488
4489 SDLoc DL(N);
4490 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT,
4491 SDValue(Parts[0], 0));
4492
4493 // Result of the bswap should be rotated by 16. If it's not legal, then
4494 // do (x << 16) | (x >> 16).
4495 SDValue ShAmt = DAG.getConstant(16, DL, getShiftAmountTy(VT));
4496 if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT))
4497 return DAG.getNode(ISD::ROTL, DL, VT, BSwap, ShAmt);
4498 if (TLI.isOperationLegalOrCustom(ISD::ROTR, VT))
4499 return DAG.getNode(ISD::ROTR, DL, VT, BSwap, ShAmt);
4500 return DAG.getNode(ISD::OR, DL, VT,
4501 DAG.getNode(ISD::SHL, DL, VT, BSwap, ShAmt),
4502 DAG.getNode(ISD::SRL, DL, VT, BSwap, ShAmt));
4503}
4504
4505/// This contains all DAGCombine rules which reduce two values combined by
4506/// an Or operation to a single value \see visitANDLike().
4507SDValue DAGCombiner::visitORLike(SDValue N0, SDValue N1, SDNode *N) {
4508 EVT VT = N1.getValueType();
4509 SDLoc DL(N);
4510
4511 // fold (or x, undef) -> -1
4512 if (!LegalOperations && (N0.isUndef() || N1.isUndef()))
4513 return DAG.getAllOnesConstant(DL, VT);
4514
4515 if (SDValue V = foldLogicOfSetCCs(false, N0, N1, DL))
4516 return V;
4517
4518 // (or (and X, C1), (and Y, C2)) -> (and (or X, Y), C3) if possible.
4519 if (N0.getOpcode() == ISD::AND && N1.getOpcode() == ISD::AND &&
4520 // Don't increase # computations.
4521 (N0.getNode()->hasOneUse() || N1.getNode()->hasOneUse())) {
4522 // We can only do this xform if we know that bits from X that are set in C2
4523 // but not in C1 are already zero. Likewise for Y.
4524 if (const ConstantSDNode *N0O1C =
4525 getAsNonOpaqueConstant(N0.getOperand(1))) {
4526 if (const ConstantSDNode *N1O1C =
4527 getAsNonOpaqueConstant(N1.getOperand(1))) {
4528 // We can only do this xform if we know that bits from X that are set in
4529 // C2 but not in C1 are already zero. Likewise for Y.
4530 const APInt &LHSMask = N0O1C->getAPIntValue();
4531 const APInt &RHSMask = N1O1C->getAPIntValue();
4532
4533 if (DAG.MaskedValueIsZero(N0.getOperand(0), RHSMask&~LHSMask) &&
4534 DAG.MaskedValueIsZero(N1.getOperand(0), LHSMask&~RHSMask)) {
4535 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT,
4536 N0.getOperand(0), N1.getOperand(0));
4537 return DAG.getNode(ISD::AND, DL, VT, X,
4538 DAG.getConstant(LHSMask | RHSMask, DL, VT));
4539 }
4540 }
4541 }
4542 }
4543
4544 // (or (and X, M), (and X, N)) -> (and X, (or M, N))
4545 if (N0.getOpcode() == ISD::AND &&
4546 N1.getOpcode() == ISD::AND &&
4547 N0.getOperand(0) == N1.getOperand(0) &&
4548 // Don't increase # computations.
4549 (N0.getNode()->hasOneUse() || N1.getNode()->hasOneUse())) {
4550 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT,
4551 N0.getOperand(1), N1.getOperand(1));
4552 return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), X);
4553 }
4554
4555 return SDValue();
4556}
4557
4558SDValue DAGCombiner::visitOR(SDNode *N) {
4559 SDValue N0 = N->getOperand(0);
4560 SDValue N1 = N->getOperand(1);
4561 EVT VT = N1.getValueType();
4562
4563 // x | x --> x
4564 if (N0 == N1)
4565 return N0;
4566
4567 // fold vector ops
4568 if (VT.isVector()) {
4569 if (SDValue FoldedVOp = SimplifyVBinOp(N))
4570 return FoldedVOp;
4571
4572 // fold (or x, 0) -> x, vector edition
4573 if (ISD::isBuildVectorAllZeros(N0.getNode()))
4574 return N1;
4575 if (ISD::isBuildVectorAllZeros(N1.getNode()))
4576 return N0;
4577
4578 // fold (or x, -1) -> -1, vector edition
4579 if (ISD::isBuildVectorAllOnes(N0.getNode()))
4580 // do not return N0, because undef node may exist in N0
4581 return DAG.getAllOnesConstant(SDLoc(N), N0.getValueType());
4582 if (ISD::isBuildVectorAllOnes(N1.getNode()))
4583 // do not return N1, because undef node may exist in N1
4584 return DAG.getAllOnesConstant(SDLoc(N), N1.getValueType());
4585
4586 // fold (or (shuf A, V_0, MA), (shuf B, V_0, MB)) -> (shuf A, B, Mask)
4587 // Do this only if the resulting shuffle is legal.
4588 if (isa<ShuffleVectorSDNode>(N0) &&
4589 isa<ShuffleVectorSDNode>(N1) &&
4590 // Avoid folding a node with illegal type.
4591 TLI.isTypeLegal(VT)) {
4592 bool ZeroN00 = ISD::isBuildVectorAllZeros(N0.getOperand(0).getNode());
4593 bool ZeroN01 = ISD::isBuildVectorAllZeros(N0.getOperand(1).getNode());
4594 bool ZeroN10 = ISD::isBuildVectorAllZeros(N1.getOperand(0).getNode());
4595 bool ZeroN11 = ISD::isBuildVectorAllZeros(N1.getOperand(1).getNode());
4596 // Ensure both shuffles have a zero input.
4597 if ((ZeroN00 != ZeroN01) && (ZeroN10 != ZeroN11)) {
4598 assert((!ZeroN00 || !ZeroN01) && "Both inputs zero!")(static_cast <bool> ((!ZeroN00 || !ZeroN01) && "Both inputs zero!"
) ? void (0) : __assert_fail ("(!ZeroN00 || !ZeroN01) && \"Both inputs zero!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4598, __extension__ __PRETTY_FUNCTION__))
;
4599 assert((!ZeroN10 || !ZeroN11) && "Both inputs zero!")(static_cast <bool> ((!ZeroN10 || !ZeroN11) && "Both inputs zero!"
) ? void (0) : __assert_fail ("(!ZeroN10 || !ZeroN11) && \"Both inputs zero!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4599, __extension__ __PRETTY_FUNCTION__))
;
4600 const ShuffleVectorSDNode *SV0 = cast<ShuffleVectorSDNode>(N0);
4601 const ShuffleVectorSDNode *SV1 = cast<ShuffleVectorSDNode>(N1);
4602 bool CanFold = true;
4603 int NumElts = VT.getVectorNumElements();
4604 SmallVector<int, 4> Mask(NumElts);
4605
4606 for (int i = 0; i != NumElts; ++i) {
4607 int M0 = SV0->getMaskElt(i);
4608 int M1 = SV1->getMaskElt(i);
4609
4610 // Determine if either index is pointing to a zero vector.
4611 bool M0Zero = M0 < 0 || (ZeroN00 == (M0 < NumElts));
4612 bool M1Zero = M1 < 0 || (ZeroN10 == (M1 < NumElts));
4613
4614 // If one element is zero and the otherside is undef, keep undef.
4615 // This also handles the case that both are undef.
4616 if ((M0Zero && M1 < 0) || (M1Zero && M0 < 0)) {
4617 Mask[i] = -1;
4618 continue;
4619 }
4620
4621 // Make sure only one of the elements is zero.
4622 if (M0Zero == M1Zero) {
4623 CanFold = false;
4624 break;
4625 }
4626
4627 assert((M0 >= 0 || M1 >= 0) && "Undef index!")(static_cast <bool> ((M0 >= 0 || M1 >= 0) &&
"Undef index!") ? void (0) : __assert_fail ("(M0 >= 0 || M1 >= 0) && \"Undef index!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4627, __extension__ __PRETTY_FUNCTION__))
;
4628
4629 // We have a zero and non-zero element. If the non-zero came from
4630 // SV0 make the index a LHS index. If it came from SV1, make it
4631 // a RHS index. We need to mod by NumElts because we don't care
4632 // which operand it came from in the original shuffles.
4633 Mask[i] = M1Zero ? M0 % NumElts : (M1 % NumElts) + NumElts;
4634 }
4635
4636 if (CanFold) {
4637 SDValue NewLHS = ZeroN00 ? N0.getOperand(1) : N0.getOperand(0);
4638 SDValue NewRHS = ZeroN10 ? N1.getOperand(1) : N1.getOperand(0);
4639
4640 bool LegalMask = TLI.isShuffleMaskLegal(Mask, VT);
4641 if (!LegalMask) {
4642 std::swap(NewLHS, NewRHS);
4643 ShuffleVectorSDNode::commuteMask(Mask);
4644 LegalMask = TLI.isShuffleMaskLegal(Mask, VT);
4645 }
4646
4647 if (LegalMask)
4648 return DAG.getVectorShuffle(VT, SDLoc(N), NewLHS, NewRHS, Mask);
4649 }
4650 }
4651 }
4652 }
4653
4654 // fold (or c1, c2) -> c1|c2
4655 ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
4656 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
4657 if (N0C && N1C && !N1C->isOpaque())
4658 return DAG.FoldConstantArithmetic(ISD::OR, SDLoc(N), VT, N0C, N1C);
4659 // canonicalize constant to RHS
4660 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
4661 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
4662 return DAG.getNode(ISD::OR, SDLoc(N), VT, N1, N0);
4663 // fold (or x, 0) -> x
4664 if (isNullConstant(N1))
4665 return N0;
4666 // fold (or x, -1) -> -1
4667 if (isAllOnesConstant(N1))
4668 return N1;
4669
4670 if (SDValue NewSel = foldBinOpIntoSelect(N))
4671 return NewSel;
4672
4673 // fold (or x, c) -> c iff (x & ~c) == 0
4674 if (N1C && DAG.MaskedValueIsZero(N0, ~N1C->getAPIntValue()))
4675 return N1;
4676
4677 if (SDValue Combined = visitORLike(N0, N1, N))
4678 return Combined;
4679
4680 // Recognize halfword bswaps as (bswap + rotl 16) or (bswap + shl 16)
4681 if (SDValue BSwap = MatchBSwapHWord(N, N0, N1))
4682 return BSwap;
4683 if (SDValue BSwap = MatchBSwapHWordLow(N, N0, N1))
4684 return BSwap;
4685
4686 // reassociate or
4687 if (SDValue ROR = ReassociateOps(ISD::OR, SDLoc(N), N0, N1))
4688 return ROR;
4689
4690 // Canonicalize (or (and X, c1), c2) -> (and (or X, c2), c1|c2)
4691 // iff (c1 & c2) != 0.
4692 auto MatchIntersect = [](ConstantSDNode *LHS, ConstantSDNode *RHS) {
4693 return LHS->getAPIntValue().intersects(RHS->getAPIntValue());
4694 };
4695 if (N0.getOpcode() == ISD::AND && N0.getNode()->hasOneUse() &&
4696 matchBinaryPredicate(N0.getOperand(1), N1, MatchIntersect)) {
4697 if (SDValue COR = DAG.FoldConstantArithmetic(
4698 ISD::OR, SDLoc(N1), VT, N1.getNode(), N0.getOperand(1).getNode())) {
4699 SDValue IOR = DAG.getNode(ISD::OR, SDLoc(N0), VT, N0.getOperand(0), N1);
4700 AddToWorklist(IOR.getNode());
4701 return DAG.getNode(ISD::AND, SDLoc(N), VT, COR, IOR);
4702 }
4703 }
4704
4705 // Simplify: (or (op x...), (op y...)) -> (op (or x, y))
4706 if (N0.getOpcode() == N1.getOpcode())
4707 if (SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N))
4708 return Tmp;
4709
4710 // See if this is some rotate idiom.
4711 if (SDNode *Rot = MatchRotate(N0, N1, SDLoc(N)))
4712 return SDValue(Rot, 0);
4713
4714 if (SDValue Load = MatchLoadCombine(N))
4715 return Load;
4716
4717 // Simplify the operands using demanded-bits information.
4718 if (SimplifyDemandedBits(SDValue(N, 0)))
4719 return SDValue(N, 0);
4720
4721 return SDValue();
4722}
4723
4724/// Match "(X shl/srl V1) & V2" where V2 may not be present.
4725bool DAGCombiner::MatchRotateHalf(SDValue Op, SDValue &Shift, SDValue &Mask) {
4726 if (Op.getOpcode() == ISD::AND) {
4727 if (DAG.isConstantIntBuildVectorOrConstantInt(Op.getOperand(1))) {
4728 Mask = Op.getOperand(1);
4729 Op = Op.getOperand(0);
4730 } else {
4731 return false;
4732 }
4733 }
4734
4735 if (Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SHL) {
4736 Shift = Op;
4737 return true;
4738 }
4739
4740 return false;
4741}
4742
4743// Return true if we can prove that, whenever Neg and Pos are both in the
4744// range [0, EltSize), Neg == (Pos == 0 ? 0 : EltSize - Pos). This means that
4745// for two opposing shifts shift1 and shift2 and a value X with OpBits bits:
4746//
4747// (or (shift1 X, Neg), (shift2 X, Pos))
4748//
4749// reduces to a rotate in direction shift2 by Pos or (equivalently) a rotate
4750// in direction shift1 by Neg. The range [0, EltSize) means that we only need
4751// to consider shift amounts with defined behavior.
4752static bool matchRotateSub(SDValue Pos, SDValue Neg, unsigned EltSize) {
4753 // If EltSize is a power of 2 then:
4754 //
4755 // (a) (Pos == 0 ? 0 : EltSize - Pos) == (EltSize - Pos) & (EltSize - 1)
4756 // (b) Neg == Neg & (EltSize - 1) whenever Neg is in [0, EltSize).
4757 //
4758 // So if EltSize is a power of 2 and Neg is (and Neg', EltSize-1), we check
4759 // for the stronger condition:
4760 //
4761 // Neg & (EltSize - 1) == (EltSize - Pos) & (EltSize - 1) [A]
4762 //
4763 // for all Neg and Pos. Since Neg & (EltSize - 1) == Neg' & (EltSize - 1)
4764 // we can just replace Neg with Neg' for the rest of the function.
4765 //
4766 // In other cases we check for the even stronger condition:
4767 //
4768 // Neg == EltSize - Pos [B]
4769 //
4770 // for all Neg and Pos. Note that the (or ...) then invokes undefined
4771 // behavior if Pos == 0 (and consequently Neg == EltSize).
4772 //
4773 // We could actually use [A] whenever EltSize is a power of 2, but the
4774 // only extra cases that it would match are those uninteresting ones
4775 // where Neg and Pos are never in range at the same time. E.g. for
4776 // EltSize == 32, using [A] would allow a Neg of the form (sub 64, Pos)
4777 // as well as (sub 32, Pos), but:
4778 //
4779 // (or (shift1 X, (sub 64, Pos)), (shift2 X, Pos))
4780 //
4781 // always invokes undefined behavior for 32-bit X.
4782 //
4783 // Below, Mask == EltSize - 1 when using [A] and is all-ones otherwise.
4784 unsigned MaskLoBits = 0;
4785 if (Neg.getOpcode() == ISD::AND && isPowerOf2_64(EltSize)) {
4786 if (ConstantSDNode *NegC = isConstOrConstSplat(Neg.getOperand(1))) {
4787 if (NegC->getAPIntValue() == EltSize - 1) {
4788 Neg = Neg.getOperand(0);
4789 MaskLoBits = Log2_64(EltSize);
4790 }
4791 }
4792 }
4793
4794 // Check whether Neg has the form (sub NegC, NegOp1) for some NegC and NegOp1.
4795 if (Neg.getOpcode() != ISD::SUB)
4796 return false;
4797 ConstantSDNode *NegC = isConstOrConstSplat(Neg.getOperand(0));
4798 if (!NegC)
4799 return false;
4800 SDValue NegOp1 = Neg.getOperand(1);
4801
4802 // On the RHS of [A], if Pos is Pos' & (EltSize - 1), just replace Pos with
4803 // Pos'. The truncation is redundant for the purpose of the equality.
4804 if (MaskLoBits && Pos.getOpcode() == ISD::AND)
4805 if (ConstantSDNode *PosC = isConstOrConstSplat(Pos.getOperand(1)))
4806 if (PosC->getAPIntValue() == EltSize - 1)
4807 Pos = Pos.getOperand(0);
4808
4809 // The condition we need is now:
4810 //
4811 // (NegC - NegOp1) & Mask == (EltSize - Pos) & Mask
4812 //
4813 // If NegOp1 == Pos then we need:
4814 //
4815 // EltSize & Mask == NegC & Mask
4816 //
4817 // (because "x & Mask" is a truncation and distributes through subtraction).
4818 APInt Width;
4819 if (Pos == NegOp1)
4820 Width = NegC->getAPIntValue();
4821
4822 // Check for cases where Pos has the form (add NegOp1, PosC) for some PosC.
4823 // Then the condition we want to prove becomes:
4824 //
4825 // (NegC - NegOp1) & Mask == (EltSize - (NegOp1 + PosC)) & Mask
4826 //
4827 // which, again because "x & Mask" is a truncation, becomes:
4828 //
4829 // NegC & Mask == (EltSize - PosC) & Mask
4830 // EltSize & Mask == (NegC + PosC) & Mask
4831 else if (Pos.getOpcode() == ISD::ADD && Pos.getOperand(0) == NegOp1) {
4832 if (ConstantSDNode *PosC = isConstOrConstSplat(Pos.getOperand(1)))
4833 Width = PosC->getAPIntValue() + NegC->getAPIntValue();
4834 else
4835 return false;
4836 } else
4837 return false;
4838
4839 // Now we just need to check that EltSize & Mask == Width & Mask.
4840 if (MaskLoBits)
4841 // EltSize & Mask is 0 since Mask is EltSize - 1.
4842 return Width.getLoBits(MaskLoBits) == 0;
4843 return Width == EltSize;
4844}
4845
4846// A subroutine of MatchRotate used once we have found an OR of two opposite
4847// shifts of Shifted. If Neg == <operand size> - Pos then the OR reduces
4848// to both (PosOpcode Shifted, Pos) and (NegOpcode Shifted, Neg), with the
4849// former being preferred if supported. InnerPos and InnerNeg are Pos and
4850// Neg with outer conversions stripped away.
4851SDNode *DAGCombiner::MatchRotatePosNeg(SDValue Shifted, SDValue Pos,
4852 SDValue Neg, SDValue InnerPos,
4853 SDValue InnerNeg, unsigned PosOpcode,
4854 unsigned NegOpcode, const SDLoc &DL) {
4855 // fold (or (shl x, (*ext y)),
4856 // (srl x, (*ext (sub 32, y)))) ->
4857 // (rotl x, y) or (rotr x, (sub 32, y))
4858 //
4859 // fold (or (shl x, (*ext (sub 32, y))),
4860 // (srl x, (*ext y))) ->
4861 // (rotr x, y) or (rotl x, (sub 32, y))
4862 EVT VT = Shifted.getValueType();
4863 if (matchRotateSub(InnerPos, InnerNeg, VT.getScalarSizeInBits())) {
4864 bool HasPos = TLI.isOperationLegalOrCustom(PosOpcode, VT);
4865 return DAG.getNode(HasPos ? PosOpcode : NegOpcode, DL, VT, Shifted,
4866 HasPos ? Pos : Neg).getNode();
4867 }
4868
4869 return nullptr;
4870}
4871
4872// MatchRotate - Handle an 'or' of two operands. If this is one of the many
4873// idioms for rotate, and if the target supports rotation instructions, generate
4874// a rot[lr].
4875SDNode *DAGCombiner::MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL) {
4876 // Must be a legal type. Expanded 'n promoted things won't work with rotates.
4877 EVT VT = LHS.getValueType();
4878 if (!TLI.isTypeLegal(VT)) return nullptr;
4879
4880 // The target must have at least one rotate flavor.
4881 bool HasROTL = TLI.isOperationLegalOrCustom(ISD::ROTL, VT);
4882 bool HasROTR = TLI.isOperationLegalOrCustom(ISD::ROTR, VT);
4883 if (!HasROTL && !HasROTR) return nullptr;
4884
4885 // Check for truncated rotate.
4886 if (LHS.getOpcode() == ISD::TRUNCATE && RHS.getOpcode() == ISD::TRUNCATE &&
4887 LHS.getOperand(0).getValueType() == RHS.getOperand(0).getValueType()) {
4888 assert(LHS.getValueType() == RHS.getValueType())(static_cast <bool> (LHS.getValueType() == RHS.getValueType
()) ? void (0) : __assert_fail ("LHS.getValueType() == RHS.getValueType()"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 4888, __extension__ __PRETTY_FUNCTION__))
;
4889 if (SDNode *Rot = MatchRotate(LHS.getOperand(0), RHS.getOperand(0), DL)) {
4890 return DAG.getNode(ISD::TRUNCATE, SDLoc(LHS), LHS.getValueType(),
4891 SDValue(Rot, 0)).getNode();
4892 }
4893 }
4894
4895 // Match "(X shl/srl V1) & V2" where V2 may not be present.
4896 SDValue LHSShift; // The shift.
4897 SDValue LHSMask; // AND value if any.
4898 if (!MatchRotateHalf(LHS, LHSShift, LHSMask))
4899 return nullptr; // Not part of a rotate.
4900
4901 SDValue RHSShift; // The shift.
4902 SDValue RHSMask; // AND value if any.
4903 if (!MatchRotateHalf(RHS, RHSShift, RHSMask))
4904 return nullptr; // Not part of a rotate.
4905
4906 if (LHSShift.getOperand(0) != RHSShift.getOperand(0))
4907 return nullptr; // Not shifting the same value.
4908
4909 if (LHSShift.getOpcode() == RHSShift.getOpcode())
4910 return nullptr; // Shifts must disagree.
4911
4912 // Canonicalize shl to left side in a shl/srl pair.
4913 if (RHSShift.getOpcode() == ISD::SHL) {
4914 std::swap(LHS, RHS);
4915 std::swap(LHSShift, RHSShift);
4916 std::swap(LHSMask, RHSMask);
4917 }
4918
4919 unsigned EltSizeInBits = VT.getScalarSizeInBits();
4920 SDValue LHSShiftArg = LHSShift.getOperand(0);
4921 SDValue LHSShiftAmt = LHSShift.getOperand(1);
4922 SDValue RHSShiftArg = RHSShift.getOperand(0);
4923 SDValue RHSShiftAmt = RHSShift.getOperand(1);
4924
4925 // fold (or (shl x, C1), (srl x, C2)) -> (rotl x, C1)
4926 // fold (or (shl x, C1), (srl x, C2)) -> (rotr x, C2)
4927 auto MatchRotateSum = [EltSizeInBits](ConstantSDNode *LHS,
4928 ConstantSDNode *RHS) {
4929 return (LHS->getAPIntValue() + RHS->getAPIntValue()) == EltSizeInBits;
4930 };
4931 if (matchBinaryPredicate(LHSShiftAmt, RHSShiftAmt, MatchRotateSum)) {
4932 SDValue Rot = DAG.getNode(HasROTL ? ISD::ROTL : ISD::ROTR, DL, VT,
4933 LHSShiftArg, HasROTL ? LHSShiftAmt : RHSShiftAmt);
4934
4935 // If there is an AND of either shifted operand, apply it to the result.
4936 if (LHSMask.getNode() || RHSMask.getNode()) {
4937 SDValue AllOnes = DAG.getAllOnesConstant(DL, VT);
4938 SDValue Mask = AllOnes;
4939
4940 if (LHSMask.getNode()) {
4941 SDValue RHSBits = DAG.getNode(ISD::SRL, DL, VT, AllOnes, RHSShiftAmt);
4942 Mask = DAG.getNode(ISD::AND, DL, VT, Mask,
4943 DAG.getNode(ISD::OR, DL, VT, LHSMask, RHSBits));
4944 }
4945 if (RHSMask.getNode()) {
4946 SDValue LHSBits = DAG.getNode(ISD::SHL, DL, VT, AllOnes, LHSShiftAmt);
4947 Mask = DAG.getNode(ISD::AND, DL, VT, Mask,
4948 DAG.getNode(ISD::OR, DL, VT, RHSMask, LHSBits));
4949 }
4950
4951 Rot = DAG.getNode(ISD::AND, DL, VT, Rot, Mask);
4952 }
4953
4954 return Rot.getNode();
4955 }
4956
4957 // If there is a mask here, and we have a variable shift, we can't be sure
4958 // that we're masking out the right stuff.
4959 if (LHSMask.getNode() || RHSMask.getNode())
4960 return nullptr;
4961
4962 // If the shift amount is sign/zext/any-extended just peel it off.
4963 SDValue LExtOp0 = LHSShiftAmt;
4964 SDValue RExtOp0 = RHSShiftAmt;
4965 if ((LHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND ||
4966 LHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND ||
4967 LHSShiftAmt.getOpcode() == ISD::ANY_EXTEND ||
4968 LHSShiftAmt.getOpcode() == ISD::TRUNCATE) &&
4969 (RHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND ||
4970 RHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND ||
4971 RHSShiftAmt.getOpcode() == ISD::ANY_EXTEND ||
4972 RHSShiftAmt.getOpcode() == ISD::TRUNCATE)) {
4973 LExtOp0 = LHSShiftAmt.getOperand(0);
4974 RExtOp0 = RHSShiftAmt.getOperand(0);
4975 }
4976
4977 SDNode *TryL = MatchRotatePosNeg(LHSShiftArg, LHSShiftAmt, RHSShiftAmt,
4978 LExtOp0, RExtOp0, ISD::ROTL, ISD::ROTR, DL);
4979 if (TryL)
4980 return TryL;
4981
4982 SDNode *TryR = MatchRotatePosNeg(RHSShiftArg, RHSShiftAmt, LHSShiftAmt,
4983 RExtOp0, LExtOp0, ISD::ROTR, ISD::ROTL, DL);
4984 if (TryR)
4985 return TryR;
4986
4987 return nullptr;
4988}
4989
4990namespace {
4991
4992/// Represents known origin of an individual byte in load combine pattern. The
4993/// value of the byte is either constant zero or comes from memory.
4994struct ByteProvider {
4995 // For constant zero providers Load is set to nullptr. For memory providers
4996 // Load represents the node which loads the byte from memory.
4997 // ByteOffset is the offset of the byte in the value produced by the load.
4998 LoadSDNode *Load = nullptr;
4999 unsigned ByteOffset = 0;
5000
5001 ByteProvider() = default;
5002
5003 static ByteProvider getMemory(LoadSDNode *Load, unsigned ByteOffset) {
5004 return ByteProvider(Load, ByteOffset);
5005 }
5006
5007 static ByteProvider getConstantZero() { return ByteProvider(nullptr, 0); }
5008
5009 bool isConstantZero() const { return !Load; }
5010 bool isMemory() const { return Load; }
5011
5012 bool operator==(const ByteProvider &Other) const {
5013 return Other.Load == Load && Other.ByteOffset == ByteOffset;
5014 }
5015
5016private:
5017 ByteProvider(LoadSDNode *Load, unsigned ByteOffset)
5018 : Load(Load), ByteOffset(ByteOffset) {}
5019};
5020
5021} // end anonymous namespace
5022
5023/// Recursively traverses the expression calculating the origin of the requested
5024/// byte of the given value. Returns None if the provider can't be calculated.
5025///
5026/// For all the values except the root of the expression verifies that the value
5027/// has exactly one use and if it's not true return None. This way if the origin
5028/// of the byte is returned it's guaranteed that the values which contribute to
5029/// the byte are not used outside of this expression.
5030///
5031/// Because the parts of the expression are not allowed to have more than one
5032/// use this function iterates over trees, not DAGs. So it never visits the same
5033/// node more than once.
5034static const Optional<ByteProvider>
5035calculateByteProvider(SDValue Op, unsigned Index, unsigned Depth,
5036 bool Root = false) {
5037 // Typical i64 by i8 pattern requires recursion up to 8 calls depth
5038 if (Depth == 10)
5039 return None;
5040
5041 if (!Root && !Op.hasOneUse())
5042 return None;
5043
5044 assert(Op.getValueType().isScalarInteger() && "can't handle other types")(static_cast <bool> (Op.getValueType().isScalarInteger(
) && "can't handle other types") ? void (0) : __assert_fail
("Op.getValueType().isScalarInteger() && \"can't handle other types\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5044, __extension__ __PRETTY_FUNCTION__))
;
5045 unsigned BitWidth = Op.getValueSizeInBits();
5046 if (BitWidth % 8 != 0)
5047 return None;
5048 unsigned ByteWidth = BitWidth / 8;
5049 assert(Index < ByteWidth && "invalid index requested")(static_cast <bool> (Index < ByteWidth && "invalid index requested"
) ? void (0) : __assert_fail ("Index < ByteWidth && \"invalid index requested\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5049, __extension__ __PRETTY_FUNCTION__))
;
5050 (void) ByteWidth;
5051
5052 switch (Op.getOpcode()) {
5053 case ISD::OR: {
5054 auto LHS = calculateByteProvider(Op->getOperand(0), Index, Depth + 1);
5055 if (!LHS)
5056 return None;
5057 auto RHS = calculateByteProvider(Op->getOperand(1), Index, Depth + 1);
5058 if (!RHS)
5059 return None;
5060
5061 if (LHS->isConstantZero())
5062 return RHS;
5063 if (RHS->isConstantZero())
5064 return LHS;
5065 return None;
5066 }
5067 case ISD::SHL: {
5068 auto ShiftOp = dyn_cast<ConstantSDNode>(Op->getOperand(1));
5069 if (!ShiftOp)
5070 return None;
5071
5072 uint64_t BitShift = ShiftOp->getZExtValue();
5073 if (BitShift % 8 != 0)
5074 return None;
5075 uint64_t ByteShift = BitShift / 8;
5076
5077 return Index < ByteShift
5078 ? ByteProvider::getConstantZero()
5079 : calculateByteProvider(Op->getOperand(0), Index - ByteShift,
5080 Depth + 1);
5081 }
5082 case ISD::ANY_EXTEND:
5083 case ISD::SIGN_EXTEND:
5084 case ISD::ZERO_EXTEND: {
5085 SDValue NarrowOp = Op->getOperand(0);
5086 unsigned NarrowBitWidth = NarrowOp.getScalarValueSizeInBits();
5087 if (NarrowBitWidth % 8 != 0)
5088 return None;
5089 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
5090
5091 if (Index >= NarrowByteWidth)
5092 return Op.getOpcode() == ISD::ZERO_EXTEND
5093 ? Optional<ByteProvider>(ByteProvider::getConstantZero())
5094 : None;
5095 return calculateByteProvider(NarrowOp, Index, Depth + 1);
5096 }
5097 case ISD::BSWAP:
5098 return calculateByteProvider(Op->getOperand(0), ByteWidth - Index - 1,
5099 Depth + 1);
5100 case ISD::LOAD: {
5101 auto L = cast<LoadSDNode>(Op.getNode());
5102 if (L->isVolatile() || L->isIndexed())
5103 return None;
5104
5105 unsigned NarrowBitWidth = L->getMemoryVT().getSizeInBits();
5106 if (NarrowBitWidth % 8 != 0)
5107 return None;
5108 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
5109
5110 if (Index >= NarrowByteWidth)
5111 return L->getExtensionType() == ISD::ZEXTLOAD
5112 ? Optional<ByteProvider>(ByteProvider::getConstantZero())
5113 : None;
5114 return ByteProvider::getMemory(L, Index);
5115 }
5116 }
5117
5118 return None;
5119}
5120
5121/// Match a pattern where a wide type scalar value is loaded by several narrow
5122/// loads and combined by shifts and ors. Fold it into a single load or a load
5123/// and a BSWAP if the targets supports it.
5124///
5125/// Assuming little endian target:
5126/// i8 *a = ...
5127/// i32 val = a[0] | (a[1] << 8) | (a[2] << 16) | (a[3] << 24)
5128/// =>
5129/// i32 val = *((i32)a)
5130///
5131/// i8 *a = ...
5132/// i32 val = (a[0] << 24) | (a[1] << 16) | (a[2] << 8) | a[3]
5133/// =>
5134/// i32 val = BSWAP(*((i32)a))
5135///
5136/// TODO: This rule matches complex patterns with OR node roots and doesn't
5137/// interact well with the worklist mechanism. When a part of the pattern is
5138/// updated (e.g. one of the loads) its direct users are put into the worklist,
5139/// but the root node of the pattern which triggers the load combine is not
5140/// necessarily a direct user of the changed node. For example, once the address
5141/// of t28 load is reassociated load combine won't be triggered:
5142/// t25: i32 = add t4, Constant:i32<2>
5143/// t26: i64 = sign_extend t25
5144/// t27: i64 = add t2, t26
5145/// t28: i8,ch = load<LD1[%tmp9]> t0, t27, undef:i64
5146/// t29: i32 = zero_extend t28
5147/// t32: i32 = shl t29, Constant:i8<8>
5148/// t33: i32 = or t23, t32
5149/// As a possible fix visitLoad can check if the load can be a part of a load
5150/// combine pattern and add corresponding OR roots to the worklist.
5151SDValue DAGCombiner::MatchLoadCombine(SDNode *N) {
5152 assert(N->getOpcode() == ISD::OR &&(static_cast <bool> (N->getOpcode() == ISD::OR &&
"Can only match load combining against OR nodes") ? void (0)
: __assert_fail ("N->getOpcode() == ISD::OR && \"Can only match load combining against OR nodes\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5153, __extension__ __PRETTY_FUNCTION__))
5153 "Can only match load combining against OR nodes")(static_cast <bool> (N->getOpcode() == ISD::OR &&
"Can only match load combining against OR nodes") ? void (0)
: __assert_fail ("N->getOpcode() == ISD::OR && \"Can only match load combining against OR nodes\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5153, __extension__ __PRETTY_FUNCTION__))
;
5154
5155 // Handles simple types only
5156 EVT VT = N->getValueType(0);
5157 if (VT != MVT::i16 && VT != MVT::i32 && VT != MVT::i64)
5158 return SDValue();
5159 unsigned ByteWidth = VT.getSizeInBits() / 8;
5160
5161 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5162 // Before legalize we can introduce too wide illegal loads which will be later
5163 // split into legal sized loads. This enables us to combine i64 load by i8
5164 // patterns to a couple of i32 loads on 32 bit targets.
5165 if (LegalOperations && !TLI.isOperationLegal(ISD::LOAD, VT))
5166 return SDValue();
5167
5168 std::function<unsigned(unsigned, unsigned)> LittleEndianByteAt = [](
5169 unsigned BW, unsigned i) { return i; };
5170 std::function<unsigned(unsigned, unsigned)> BigEndianByteAt = [](
5171 unsigned BW, unsigned i) { return BW - i - 1; };
5172
5173 bool IsBigEndianTarget = DAG.getDataLayout().isBigEndian();
5174 auto MemoryByteOffset = [&] (ByteProvider P) {
5175 assert(P.isMemory() && "Must be a memory byte provider")(static_cast <bool> (P.isMemory() && "Must be a memory byte provider"
) ? void (0) : __assert_fail ("P.isMemory() && \"Must be a memory byte provider\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5175, __extension__ __PRETTY_FUNCTION__))
;
5176 unsigned LoadBitWidth = P.Load->getMemoryVT().getSizeInBits();
5177 assert(LoadBitWidth % 8 == 0 &&(static_cast <bool> (LoadBitWidth % 8 == 0 && "can only analyze providers for individual bytes not bit"
) ? void (0) : __assert_fail ("LoadBitWidth % 8 == 0 && \"can only analyze providers for individual bytes not bit\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5178, __extension__ __PRETTY_FUNCTION__))
5178 "can only analyze providers for individual bytes not bit")(static_cast <bool> (LoadBitWidth % 8 == 0 && "can only analyze providers for individual bytes not bit"
) ? void (0) : __assert_fail ("LoadBitWidth % 8 == 0 && \"can only analyze providers for individual bytes not bit\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5178, __extension__ __PRETTY_FUNCTION__))
;
5179 unsigned LoadByteWidth = LoadBitWidth / 8;
5180 return IsBigEndianTarget
5181 ? BigEndianByteAt(LoadByteWidth, P.ByteOffset)
5182 : LittleEndianByteAt(LoadByteWidth, P.ByteOffset);
5183 };
5184
5185 Optional<BaseIndexOffset> Base;
5186 SDValue Chain;
5187
5188 SmallSet<LoadSDNode *, 8> Loads;
5189 Optional<ByteProvider> FirstByteProvider;
5190 int64_t FirstOffset = INT64_MAX(9223372036854775807L);
5191
5192 // Check if all the bytes of the OR we are looking at are loaded from the same
5193 // base address. Collect bytes offsets from Base address in ByteOffsets.
5194 SmallVector<int64_t, 4> ByteOffsets(ByteWidth);
5195 for (unsigned i = 0; i < ByteWidth; i++) {
5196 auto P = calculateByteProvider(SDValue(N, 0), i, 0, /*Root=*/true);
5197 if (!P || !P->isMemory()) // All the bytes must be loaded from memory
5198 return SDValue();
5199
5200 LoadSDNode *L = P->Load;
5201 assert(L->hasNUsesOfValue(1, 0) && !L->isVolatile() && !L->isIndexed() &&(static_cast <bool> (L->hasNUsesOfValue(1, 0) &&
!L->isVolatile() && !L->isIndexed() &&
"Must be enforced by calculateByteProvider") ? void (0) : __assert_fail
("L->hasNUsesOfValue(1, 0) && !L->isVolatile() && !L->isIndexed() && \"Must be enforced by calculateByteProvider\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5202, __extension__ __PRETTY_FUNCTION__))
5202 "Must be enforced by calculateByteProvider")(static_cast <bool> (L->hasNUsesOfValue(1, 0) &&
!L->isVolatile() && !L->isIndexed() &&
"Must be enforced by calculateByteProvider") ? void (0) : __assert_fail
("L->hasNUsesOfValue(1, 0) && !L->isVolatile() && !L->isIndexed() && \"Must be enforced by calculateByteProvider\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5202, __extension__ __PRETTY_FUNCTION__))
;
5203 assert(L->getOffset().isUndef() && "Unindexed load must have undef offset")(static_cast <bool> (L->getOffset().isUndef() &&
"Unindexed load must have undef offset") ? void (0) : __assert_fail
("L->getOffset().isUndef() && \"Unindexed load must have undef offset\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5203, __extension__ __PRETTY_FUNCTION__))
;
5204
5205 // All loads must share the same chain
5206 SDValue LChain = L->getChain();
5207 if (!Chain)
5208 Chain = LChain;
5209 else if (Chain != LChain)
5210 return SDValue();
5211
5212 // Loads must share the same base address
5213 BaseIndexOffset Ptr = BaseIndexOffset::match(L->getBasePtr(), DAG);
5214 int64_t ByteOffsetFromBase = 0;
5215 if (!Base)
5216 Base = Ptr;
5217 else if (!Base->equalBaseIndex(Ptr, DAG, ByteOffsetFromBase))
5218 return SDValue();
5219
5220 // Calculate the offset of the current byte from the base address
5221 ByteOffsetFromBase += MemoryByteOffset(*P);
5222 ByteOffsets[i] = ByteOffsetFromBase;
5223
5224 // Remember the first byte load
5225 if (ByteOffsetFromBase < FirstOffset) {
5226 FirstByteProvider = P;
5227 FirstOffset = ByteOffsetFromBase;
5228 }
5229
5230 Loads.insert(L);
5231 }
5232 assert(!Loads.empty() && "All the bytes of the value must be loaded from "(static_cast <bool> (!Loads.empty() && "All the bytes of the value must be loaded from "
"memory, so there must be at least one load which produces the value"
) ? void (0) : __assert_fail ("!Loads.empty() && \"All the bytes of the value must be loaded from \" \"memory, so there must be at least one load which produces the value\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5233, __extension__ __PRETTY_FUNCTION__))
5233 "memory, so there must be at least one load which produces the value")(static_cast <bool> (!Loads.empty() && "All the bytes of the value must be loaded from "
"memory, so there must be at least one load which produces the value"
) ? void (0) : __assert_fail ("!Loads.empty() && \"All the bytes of the value must be loaded from \" \"memory, so there must be at least one load which produces the value\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5233, __extension__ __PRETTY_FUNCTION__))
;
5234 assert(Base && "Base address of the accessed memory location must be set")(static_cast <bool> (Base && "Base address of the accessed memory location must be set"
) ? void (0) : __assert_fail ("Base && \"Base address of the accessed memory location must be set\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5234, __extension__ __PRETTY_FUNCTION__))
;
5235 assert(FirstOffset != INT64_MAX && "First byte offset must be set")(static_cast <bool> (FirstOffset != (9223372036854775807L
) && "First byte offset must be set") ? void (0) : __assert_fail
("FirstOffset != INT64_MAX && \"First byte offset must be set\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5235, __extension__ __PRETTY_FUNCTION__))
;
5236
5237 // Check if the bytes of the OR we are looking at match with either big or
5238 // little endian value load
5239 bool BigEndian = true, LittleEndian = true;
5240 for (unsigned i = 0; i < ByteWidth; i++) {
5241 int64_t CurrentByteOffset = ByteOffsets[i] - FirstOffset;
5242 LittleEndian &= CurrentByteOffset == LittleEndianByteAt(ByteWidth, i);
5243 BigEndian &= CurrentByteOffset == BigEndianByteAt(ByteWidth, i);
5244 if (!BigEndian && !LittleEndian)
5245 return SDValue();
5246 }
5247 assert((BigEndian != LittleEndian) && "should be either or")(static_cast <bool> ((BigEndian != LittleEndian) &&
"should be either or") ? void (0) : __assert_fail ("(BigEndian != LittleEndian) && \"should be either or\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5247, __extension__ __PRETTY_FUNCTION__))
;
5248 assert(FirstByteProvider && "must be set")(static_cast <bool> (FirstByteProvider && "must be set"
) ? void (0) : __assert_fail ("FirstByteProvider && \"must be set\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5248, __extension__ __PRETTY_FUNCTION__))
;
5249
5250 // Ensure that the first byte is loaded from zero offset of the first load.
5251 // So the combined value can be loaded from the first load address.
5252 if (MemoryByteOffset(*FirstByteProvider) != 0)
5253 return SDValue();
5254 LoadSDNode *FirstLoad = FirstByteProvider->Load;
5255
5256 // The node we are looking at matches with the pattern, check if we can
5257 // replace it with a single load and bswap if needed.
5258
5259 // If the load needs byte swap check if the target supports it
5260 bool NeedsBswap = IsBigEndianTarget != BigEndian;
5261
5262 // Before legalize we can introduce illegal bswaps which will be later
5263 // converted to an explicit bswap sequence. This way we end up with a single
5264 // load and byte shuffling instead of several loads and byte shuffling.
5265 if (NeedsBswap && LegalOperations && !TLI.isOperationLegal(ISD::BSWAP, VT))
5266 return SDValue();
5267
5268 // Check that a load of the wide type is both allowed and fast on the target
5269 bool Fast = false;
5270 bool Allowed = TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(),
5271 VT, FirstLoad->getAddressSpace(),
5272 FirstLoad->getAlignment(), &Fast);
5273 if (!Allowed || !Fast)
5274 return SDValue();
5275
5276 SDValue NewLoad =
5277 DAG.getLoad(VT, SDLoc(N), Chain, FirstLoad->getBasePtr(),
5278 FirstLoad->getPointerInfo(), FirstLoad->getAlignment());
5279
5280 // Transfer chain users from old loads to the new load.
5281 for (LoadSDNode *L : Loads)
5282 DAG.ReplaceAllUsesOfValueWith(SDValue(L, 1), SDValue(NewLoad.getNode(), 1));
5283
5284 return NeedsBswap ? DAG.getNode(ISD::BSWAP, SDLoc(N), VT, NewLoad) : NewLoad;
5285}
5286
5287SDValue DAGCombiner::visitXOR(SDNode *N) {
5288 SDValue N0 = N->getOperand(0);
5289 SDValue N1 = N->getOperand(1);
5290 EVT VT = N0.getValueType();
5291
5292 // fold vector ops
5293 if (VT.isVector()) {
5294 if (SDValue FoldedVOp = SimplifyVBinOp(N))
5295 return FoldedVOp;
5296
5297 // fold (xor x, 0) -> x, vector edition
5298 if (ISD::isBuildVectorAllZeros(N0.getNode()))
5299 return N1;
5300 if (ISD::isBuildVectorAllZeros(N1.getNode()))
5301 return N0;
5302 }
5303
5304 // fold (xor undef, undef) -> 0. This is a common idiom (misuse).
5305 if (N0.isUndef() && N1.isUndef())
5306 return DAG.getConstant(0, SDLoc(N), VT);
5307 // fold (xor x, undef) -> undef
5308 if (N0.isUndef())
5309 return N0;
5310 if (N1.isUndef())
5311 return N1;
5312 // fold (xor c1, c2) -> c1^c2
5313 ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
5314 ConstantSDNode *N1C = getAsNonOpaqueConstant(N1);
5315 if (N0C && N1C)
5316 return DAG.FoldConstantArithmetic(ISD::XOR, SDLoc(N), VT, N0C, N1C);
5317 // canonicalize constant to RHS
5318 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
5319 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
5320 return DAG.getNode(ISD::XOR, SDLoc(N), VT, N1, N0);
5321 // fold (xor x, 0) -> x
5322 if (isNullConstant(N1))
5323 return N0;
5324
5325 if (SDValue NewSel = foldBinOpIntoSelect(N))
5326 return NewSel;
5327
5328 // reassociate xor
5329 if (SDValue RXOR = ReassociateOps(ISD::XOR, SDLoc(N), N0, N1))
5330 return RXOR;
5331
5332 // fold !(x cc y) -> (x !cc y)
5333 SDValue LHS, RHS, CC;
5334 if (TLI.isConstTrueVal(N1.getNode()) && isSetCCEquivalent(N0, LHS, RHS, CC)) {
5335 bool isInt = LHS.getValueType().isInteger();
5336 ISD::CondCode NotCC = ISD::getSetCCInverse(cast<CondCodeSDNode>(CC)->get(),
5337 isInt);
5338
5339 if (!LegalOperations ||
5340 TLI.isCondCodeLegal(NotCC, LHS.getSimpleValueType())) {
5341 switch (N0.getOpcode()) {
5342 default:
5343 llvm_unreachable("Unhandled SetCC Equivalent!")::llvm::llvm_unreachable_internal("Unhandled SetCC Equivalent!"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5343)
;
5344 case ISD::SETCC:
5345 return DAG.getSetCC(SDLoc(N0), VT, LHS, RHS, NotCC);
5346 case ISD::SELECT_CC:
5347 return DAG.getSelectCC(SDLoc(N0), LHS, RHS, N0.getOperand(2),
5348 N0.getOperand(3), NotCC);
5349 }
5350 }
5351 }
5352
5353 // fold (not (zext (setcc x, y))) -> (zext (not (setcc x, y)))
5354 if (isOneConstant(N1) && N0.getOpcode() == ISD::ZERO_EXTEND &&
5355 N0.getNode()->hasOneUse() &&
5356 isSetCCEquivalent(N0.getOperand(0), LHS, RHS, CC)){
5357 SDValue V = N0.getOperand(0);
5358 SDLoc DL(N0);
5359 V = DAG.getNode(ISD::XOR, DL, V.getValueType(), V,
5360 DAG.getConstant(1, DL, V.getValueType()));
5361 AddToWorklist(V.getNode());
5362 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, V);
5363 }
5364
5365 // fold (not (or x, y)) -> (and (not x), (not y)) iff x or y are setcc
5366 if (isOneConstant(N1) && VT == MVT::i1 &&
5367 (N0.getOpcode() == ISD::OR || N0.getOpcode() == ISD::AND)) {
5368 SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1);
5369 if (isOneUseSetCC(RHS) || isOneUseSetCC(LHS)) {
5370 unsigned NewOpcode = N0.getOpcode() == ISD::AND ? ISD::OR : ISD::AND;
5371 LHS = DAG.getNode(ISD::XOR, SDLoc(LHS), VT, LHS, N1); // LHS = ~LHS
5372 RHS = DAG.getNode(ISD::XOR, SDLoc(RHS), VT, RHS, N1); // RHS = ~RHS
5373 AddToWorklist(LHS.getNode()); AddToWorklist(RHS.getNode());
5374 return DAG.getNode(NewOpcode, SDLoc(N), VT, LHS, RHS);
5375 }
5376 }
5377 // fold (not (or x, y)) -> (and (not x), (not y)) iff x or y are constants
5378 if (isAllOnesConstant(N1) &&
5379 (N0.getOpcode() == ISD::OR || N0.getOpcode() == ISD::AND)) {
5380 SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1);
5381 if (isa<ConstantSDNode>(RHS) || isa<ConstantSDNode>(LHS)) {
5382 unsigned NewOpcode = N0.getOpcode() == ISD::AND ? ISD::OR : ISD::AND;
5383 LHS = DAG.getNode(ISD::XOR, SDLoc(LHS), VT, LHS, N1); // LHS = ~LHS
5384 RHS = DAG.getNode(ISD::XOR, SDLoc(RHS), VT, RHS, N1); // RHS = ~RHS
5385 AddToWorklist(LHS.getNode()); AddToWorklist(RHS.getNode());
5386 return DAG.getNode(NewOpcode, SDLoc(N), VT, LHS, RHS);
5387 }
5388 }
5389 // fold (xor (and x, y), y) -> (and (not x), y)
5390 if (N0.getOpcode() == ISD::AND && N0.getNode()->hasOneUse() &&
5391 N0->getOperand(1) == N1) {
5392 SDValue X = N0->getOperand(0);
5393 SDValue NotX = DAG.getNOT(SDLoc(X), X, VT);
5394 AddToWorklist(NotX.getNode());
5395 return DAG.getNode(ISD::AND, SDLoc(N), VT, NotX, N1);
5396 }
5397
5398 // fold Y = sra (X, size(X)-1); xor (add (X, Y), Y) -> (abs X)
5399 unsigned OpSizeInBits = VT.getScalarSizeInBits();
5400 if (N0.getOpcode() == ISD::ADD && N0.getOperand(1) == N1 &&
5401 N1.getOpcode() == ISD::SRA && N1.getOperand(0) == N0.getOperand(0) &&
5402 TLI.isOperationLegalOrCustom(ISD::ABS, VT)) {
5403 if (ConstantSDNode *C = isConstOrConstSplat(N1.getOperand(1)))
5404 if (C->getAPIntValue() == (OpSizeInBits - 1))
5405 return DAG.getNode(ISD::ABS, SDLoc(N), VT, N0.getOperand(0));
5406 }
5407
5408 // fold (xor x, x) -> 0
5409 if (N0 == N1)
5410 return tryFoldToZero(SDLoc(N), TLI, VT, DAG, LegalOperations, LegalTypes);
5411
5412 // fold (xor (shl 1, x), -1) -> (rotl ~1, x)
5413 // Here is a concrete example of this equivalence:
5414 // i16 x == 14
5415 // i16 shl == 1 << 14 == 16384 == 0b0100000000000000
5416 // i16 xor == ~(1 << 14) == 49151 == 0b1011111111111111
5417 //
5418 // =>
5419 //
5420 // i16 ~1 == 0b1111111111111110
5421 // i16 rol(~1, 14) == 0b1011111111111111
5422 //
5423 // Some additional tips to help conceptualize this transform:
5424 // - Try to see the operation as placing a single zero in a value of all ones.
5425 // - There exists no value for x which would allow the result to contain zero.
5426 // - Values of x larger than the bitwidth are undefined and do not require a
5427 // consistent result.
5428 // - Pushing the zero left requires shifting one bits in from the right.
5429 // A rotate left of ~1 is a nice way of achieving the desired result.
5430 if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT) && N0.getOpcode() == ISD::SHL
5431 && isAllOnesConstant(N1) && isOneConstant(N0.getOperand(0))) {
5432 SDLoc DL(N);
5433 return DAG.getNode(ISD::ROTL, DL, VT, DAG.getConstant(~1, DL, VT),
5434 N0.getOperand(1));
5435 }
5436
5437 // Simplify: xor (op x...), (op y...) -> (op (xor x, y))
5438 if (N0.getOpcode() == N1.getOpcode())
5439 if (SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N))
5440 return Tmp;
5441
5442 // Simplify the expression using non-local knowledge.
5443 if (SimplifyDemandedBits(SDValue(N, 0)))
5444 return SDValue(N, 0);
5445
5446 return SDValue();
5447}
5448
5449/// Handle transforms common to the three shifts, when the shift amount is a
5450/// constant.
5451SDValue DAGCombiner::visitShiftByConstant(SDNode *N, ConstantSDNode *Amt) {
5452 SDNode *LHS = N->getOperand(0).getNode();
5453 if (!LHS->hasOneUse()) return SDValue();
5454
5455 // We want to pull some binops through shifts, so that we have (and (shift))
5456 // instead of (shift (and)), likewise for add, or, xor, etc. This sort of
5457 // thing happens with address calculations, so it's important to canonicalize
5458 // it.
5459 bool HighBitSet = false; // Can we transform this if the high bit is set?
5460
5461 switch (LHS->getOpcode()) {
5462 default: return SDValue();
5463 case ISD::OR:
5464 case ISD::XOR:
5465 HighBitSet = false; // We can only transform sra if the high bit is clear.
5466 break;
5467 case ISD::AND:
5468 HighBitSet = true; // We can only transform sra if the high bit is set.
5469 break;
5470 case ISD::ADD:
5471 if (N->getOpcode() != ISD::SHL)
5472 return SDValue(); // only shl(add) not sr[al](add).
5473 HighBitSet = false; // We can only transform sra if the high bit is clear.
5474 break;
5475 }
5476
5477 // We require the RHS of the binop to be a constant and not opaque as well.
5478 ConstantSDNode *BinOpCst = getAsNonOpaqueConstant(LHS->getOperand(1));
5479 if (!BinOpCst) return SDValue();
5480
5481 // FIXME: disable this unless the input to the binop is a shift by a constant
5482 // or is copy/select.Enable this in other cases when figure out it's exactly profitable.
5483 SDNode *BinOpLHSVal = LHS->getOperand(0).getNode();
5484 bool isShift = BinOpLHSVal->getOpcode() == ISD::SHL ||
5485 BinOpLHSVal->getOpcode() == ISD::SRA ||
5486 BinOpLHSVal->getOpcode() == ISD::SRL;
5487 bool isCopyOrSelect = BinOpLHSVal->getOpcode() == ISD::CopyFromReg ||
5488 BinOpLHSVal->getOpcode() == ISD::SELECT;
5489
5490 if ((!isShift || !isa<ConstantSDNode>(BinOpLHSVal->getOperand(1))) &&
5491 !isCopyOrSelect)
5492 return SDValue();
5493
5494 if (isCopyOrSelect && N->hasOneUse())
5495 return SDValue();
5496
5497 EVT VT = N->getValueType(0);
5498
5499 // If this is a signed shift right, and the high bit is modified by the
5500 // logical operation, do not perform the transformation. The highBitSet
5501 // boolean indicates the value of the high bit of the constant which would
5502 // cause it to be modified for this operation.
5503 if (N->getOpcode() == ISD::SRA) {
5504 bool BinOpRHSSignSet = BinOpCst->getAPIntValue().isNegative();
5505 if (BinOpRHSSignSet != HighBitSet)
5506 return SDValue();
5507 }
5508
5509 if (!TLI.isDesirableToCommuteWithShift(LHS))
5510 return SDValue();
5511
5512 // Fold the constants, shifting the binop RHS by the shift amount.
5513 SDValue NewRHS = DAG.getNode(N->getOpcode(), SDLoc(LHS->getOperand(1)),
5514 N->getValueType(0),
5515 LHS->getOperand(1), N->getOperand(1));
5516 assert(isa<ConstantSDNode>(NewRHS) && "Folding was not successful!")(static_cast <bool> (isa<ConstantSDNode>(NewRHS) &&
"Folding was not successful!") ? void (0) : __assert_fail ("isa<ConstantSDNode>(NewRHS) && \"Folding was not successful!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5516, __extension__ __PRETTY_FUNCTION__))
;
5517
5518 // Create the new shift.
5519 SDValue NewShift = DAG.getNode(N->getOpcode(),
5520 SDLoc(LHS->getOperand(0)),
5521 VT, LHS->getOperand(0), N->getOperand(1));
5522
5523 // Create the new binop.
5524 return DAG.getNode(LHS->getOpcode(), SDLoc(N), VT, NewShift, NewRHS);
5525}
5526
5527SDValue DAGCombiner::distributeTruncateThroughAnd(SDNode *N) {
5528 assert(N->getOpcode() == ISD::TRUNCATE)(static_cast <bool> (N->getOpcode() == ISD::TRUNCATE
) ? void (0) : __assert_fail ("N->getOpcode() == ISD::TRUNCATE"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5528, __extension__ __PRETTY_FUNCTION__))
;
5529 assert(N->getOperand(0).getOpcode() == ISD::AND)(static_cast <bool> (N->getOperand(0).getOpcode() ==
ISD::AND) ? void (0) : __assert_fail ("N->getOperand(0).getOpcode() == ISD::AND"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 5529, __extension__ __PRETTY_FUNCTION__))
;
5530
5531 // (truncate:TruncVT (and N00, N01C)) -> (and (truncate:TruncVT N00), TruncC)
5532 if (N->hasOneUse() && N->getOperand(0).hasOneUse()) {
5533 SDValue N01 = N->getOperand(0).getOperand(1);
5534 if (isConstantOrConstantVector(N01, /* NoOpaques */ true)) {
5535 SDLoc DL(N);
5536 EVT TruncVT = N->getValueType(0);
5537 SDValue N00 = N->getOperand(0).getOperand(0);
5538 SDValue Trunc00 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N00);
5539 SDValue Trunc01 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N01);
5540 AddToWorklist(Trunc00.getNode());
5541 AddToWorklist(Trunc01.getNode());
5542 return DAG.getNode(ISD::AND, DL, TruncVT, Trunc00, Trunc01);
5543 }
5544 }
5545
5546 return SDValue();
5547}
5548
5549SDValue DAGCombiner::visitRotate(SDNode *N) {
5550 SDLoc dl(N);
5551 SDValue N0 = N->getOperand(0);
5552 SDValue N1 = N->getOperand(1);
5553 EVT VT = N->getValueType(0);
5554 unsigned Bitsize = VT.getScalarSizeInBits();
5555
5556 // fold (rot x, 0) -> x
5557 if (isNullConstantOrNullSplatConstant(N1))
5558 return N0;
5559
5560 // fold (rot x, c) -> (rot x, c % BitSize)
5561 if (ConstantSDNode *Cst = isConstOrConstSplat(N1)) {
5562 if (Cst->getAPIntValue().uge(Bitsize)) {
5563 uint64_t RotAmt = Cst->getAPIntValue().urem(Bitsize);
5564 return DAG.getNode(N->getOpcode(), dl, VT, N0,
5565 DAG.getConstant(RotAmt, dl, N1.getValueType()));
5566 }
5567 }
5568
5569 // fold (rot* x, (trunc (and y, c))) -> (rot* x, (and (trunc y), (trunc c))).
5570 if (N1.getOpcode() == ISD::TRUNCATE &&
5571 N1.getOperand(0).getOpcode() == ISD::AND) {
5572 if (SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode()))
5573 return DAG.getNode(N->getOpcode(), dl, VT, N0, NewOp1);
5574 }
5575
5576 unsigned NextOp = N0.getOpcode();
5577 // fold (rot* (rot* x, c2), c1) -> (rot* x, c1 +- c2 % bitsize)
5578 if (NextOp == ISD::ROTL || NextOp == ISD::ROTR) {
5579 SDNode *C1 = DAG.isConstantIntBuildVectorOrConstantInt(N1);
5580 SDNode *C2 = DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1));
5581 if (C1 && C2 && C1->getValueType(0) == C2->getValueType(0)) {
5582 EVT ShiftVT = C1->getValueType(0);
5583 bool SameSide = (N->getOpcode() == NextOp);
5584 unsigned CombineOp = SameSide ? ISD::ADD : ISD::SUB;
5585 if (SDValue CombinedShift =
5586 DAG.FoldConstantArithmetic(CombineOp, dl, ShiftVT, C1, C2)) {
5587 SDValue BitsizeC = DAG.getConstant(Bitsize, dl, ShiftVT);
5588 SDValue CombinedShiftNorm = DAG.FoldConstantArithmetic(
5589 ISD::SREM, dl, ShiftVT, CombinedShift.getNode(),
5590 BitsizeC.getNode());
5591 return DAG.getNode(N->getOpcode(), dl, VT, N0->getOperand(0),
5592 CombinedShiftNorm);
5593 }
5594 }
5595 }
5596 return SDValue();
5597}
5598
5599SDValue DAGCombiner::visitSHL(SDNode *N) {
5600 SDValue N0 = N->getOperand(0);
5601 SDValue N1 = N->getOperand(1);
5602 EVT VT = N0.getValueType();
5603 unsigned OpSizeInBits = VT.getScalarSizeInBits();
5604
5605 // fold vector ops
5606 if (VT.isVector()) {
5607 if (SDValue FoldedVOp = SimplifyVBinOp(N))
5608 return FoldedVOp;
5609
5610 BuildVectorSDNode *N1CV = dyn_cast<BuildVectorSDNode>(N1);
5611 // If setcc produces all-one true value then:
5612 // (shl (and (setcc) N01CV) N1CV) -> (and (setcc) N01CV<<N1CV)
5613 if (N1CV && N1CV->isConstant()) {
5614 if (N0.getOpcode() == ISD::AND) {
5615 SDValue N00 = N0->getOperand(0);
5616 SDValue N01 = N0->getOperand(1);
5617 BuildVectorSDNode *N01CV = dyn_cast<BuildVectorSDNode>(N01);
5618
5619 if (N01CV && N01CV->isConstant() && N00.getOpcode() == ISD::SETCC &&
5620 TLI.getBooleanContents(N00.getOperand(0).getValueType()) ==
5621 TargetLowering::ZeroOrNegativeOneBooleanContent) {
5622 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT,
5623 N01CV, N1CV))
5624 return DAG.getNode(ISD::AND, SDLoc(N), VT, N00, C);
5625 }
5626 }
5627 }
5628 }
5629
5630 ConstantSDNode *N1C = isConstOrConstSplat(N1);
5631
5632 // fold (shl c1, c2) -> c1<<c2
5633 ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
5634 if (N0C && N1C && !N1C->isOpaque())
5635 return DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT, N0C, N1C);
5636 // fold (shl 0, x) -> 0
5637 if (isNullConstantOrNullSplatConstant(N0))
5638 return N0;
5639 // fold (shl x, c >= size(x)) -> undef
5640 // NOTE: ALL vector elements must be too big to avoid partial UNDEFs.
5641 auto MatchShiftTooBig = [OpSizeInBits](ConstantSDNode *Val) {
5642 return Val->getAPIntValue().uge(OpSizeInBits);
5643 };
5644 if (matchUnaryPredicate(N1, MatchShiftTooBig))
5645 return DAG.getUNDEF(VT);
5646 // fold (shl x, 0) -> x
5647 if (N1C && N1C->isNullValue())
5648 return N0;
5649 // fold (shl undef, x) -> 0
5650 if (N0.isUndef())
5651 return DAG.getConstant(0, SDLoc(N), VT);
5652
5653 if (SDValue NewSel = foldBinOpIntoSelect(N))
5654 return NewSel;
5655
5656 // if (shl x, c) is known to be zero, return 0
5657 if (DAG.MaskedValueIsZero(SDValue(N, 0),
5658 APInt::getAllOnesValue(OpSizeInBits)))
5659 return DAG.getConstant(0, SDLoc(N), VT);
5660 // fold (shl x, (trunc (and y, c))) -> (shl x, (and (trunc y), (trunc c))).
5661 if (N1.getOpcode() == ISD::TRUNCATE &&
5662 N1.getOperand(0).getOpcode() == ISD::AND) {
5663 if (SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode()))
5664 return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, NewOp1);
5665 }
5666
5667 if (N1C && SimplifyDemandedBits(SDValue(N, 0)))
5668 return SDValue(N, 0);
5669
5670 // fold (shl (shl x, c1), c2) -> 0 or (shl x, (add c1, c2))
5671 if (N0.getOpcode() == ISD::SHL) {
5672 auto MatchOutOfRange = [OpSizeInBits](ConstantSDNode *LHS,
5673 ConstantSDNode *RHS) {
5674 APInt c1 = LHS->getAPIntValue();
5675 APInt c2 = RHS->getAPIntValue();
5676 zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
5677 return (c1 + c2).uge(OpSizeInBits);
5678 };
5679 if (matchBinaryPredicate(N1, N0.getOperand(1), MatchOutOfRange))
5680 return DAG.getConstant(0, SDLoc(N), VT);
5681
5682 auto MatchInRange = [OpSizeInBits](ConstantSDNode *LHS,
5683 ConstantSDNode *RHS) {
5684 APInt c1 = LHS->getAPIntValue();
5685 APInt c2 = RHS->getAPIntValue();
5686 zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
5687 return (c1 + c2).ult(OpSizeInBits);
5688 };
5689 if (matchBinaryPredicate(N1, N0.getOperand(1), MatchInRange)) {
5690 SDLoc DL(N);
5691 EVT ShiftVT = N1.getValueType();
5692 SDValue Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, N1, N0.getOperand(1));
5693 return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Sum);
5694 }
5695 }
5696
5697 // fold (shl (ext (shl x, c1)), c2) -> (ext (shl x, (add c1, c2)))
5698 // For this to be valid, the second form must not preserve any of the bits
5699 // that are shifted out by the inner shift in the first form. This means
5700 // the outer shift size must be >= the number of bits added by the ext.
5701 // As a corollary, we don't care what kind of ext it is.
5702 if (N1C && (N0.getOpcode() == ISD::ZERO_EXTEND ||
5703 N0.getOpcode() == ISD::ANY_EXTEND ||
5704 N0.getOpcode() == ISD::SIGN_EXTEND) &&
5705 N0.getOperand(0).getOpcode() == ISD::SHL) {
5706 SDValue N0Op0 = N0.getOperand(0);
5707 if (ConstantSDNode *N0Op0C1 = isConstOrConstSplat(N0Op0.getOperand(1))) {
5708 APInt c1 = N0Op0C1->getAPIntValue();
5709 APInt c2 = N1C->getAPIntValue();
5710 zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
5711
5712 EVT InnerShiftVT = N0Op0.getValueType();
5713 uint64_t InnerShiftSize = InnerShiftVT.getScalarSizeInBits();
5714 if (c2.uge(OpSizeInBits - InnerShiftSize)) {
5715 SDLoc DL(N0);
5716 APInt Sum = c1 + c2;
5717 if (Sum.uge(OpSizeInBits))
5718 return DAG.getConstant(0, DL, VT);
5719
5720 return DAG.getNode(
5721 ISD::SHL, DL, VT,
5722 DAG.getNode(N0.getOpcode(), DL, VT, N0Op0->getOperand(0)),
5723 DAG.getConstant(Sum.getZExtValue(), DL, N1.getValueType()));
5724 }
5725 }
5726 }
5727
5728 // fold (shl (zext (srl x, C)), C) -> (zext (shl (srl x, C), C))
5729 // Only fold this if the inner zext has no other uses to avoid increasing
5730 // the total number of instructions.
5731 if (N1C && N0.getOpcode() == ISD::ZERO_EXTEND && N0.hasOneUse() &&
5732 N0.getOperand(0).getOpcode() == ISD::SRL) {
5733 SDValue N0Op0 = N0.getOperand(0);
5734 if (ConstantSDNode *N0Op0C1 = isConstOrConstSplat(N0Op0.getOperand(1))) {
5735 if (N0Op0C1->getAPIntValue().ult(VT.getScalarSizeInBits())) {
5736 uint64_t c1 = N0Op0C1->getZExtValue();
5737 uint64_t c2 = N1C->getZExtValue();
5738 if (c1 == c2) {
5739 SDValue NewOp0 = N0.getOperand(0);
5740 EVT CountVT = NewOp0.getOperand(1).getValueType();
5741 SDLoc DL(N);
5742 SDValue NewSHL = DAG.getNode(ISD::SHL, DL, NewOp0.getValueType(),
5743 NewOp0,
5744 DAG.getConstant(c2, DL, CountVT));
5745 AddToWorklist(NewSHL.getNode());
5746 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N0), VT, NewSHL);
5747 }
5748 }
5749 }
5750 }
5751
5752 // fold (shl (sr[la] exact X, C1), C2) -> (shl X, (C2-C1)) if C1 <= C2
5753 // fold (shl (sr[la] exact X, C1), C2) -> (sr[la] X, (C2-C1)) if C1 > C2
5754 if (N1C && (N0.getOpcode() == ISD::SRL || N0.getOpcode() == ISD::SRA) &&
5755 N0->getFlags().hasExact()) {
5756 if (ConstantSDNode *N0C1 = isConstOrConstSplat(N0.getOperand(1))) {
5757 uint64_t C1 = N0C1->getZExtValue();
5758 uint64_t C2 = N1C->getZExtValue();
5759 SDLoc DL(N);
5760 if (C1 <= C2)
5761 return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0),
5762 DAG.getConstant(C2 - C1, DL, N1.getValueType()));
5763 return DAG.getNode(N0.getOpcode(), DL, VT, N0.getOperand(0),
5764 DAG.getConstant(C1 - C2, DL, N1.getValueType()));
5765 }
5766 }
5767
5768 // fold (shl (srl x, c1), c2) -> (and (shl x, (sub c2, c1), MASK) or
5769 // (and (srl x, (sub c1, c2), MASK)
5770 // Only fold this if the inner shift has no other uses -- if it does, folding
5771 // this will increase the total number of instructions.
5772 if (N1C && N0.getOpcode() == ISD::SRL && N0.hasOneUse()) {
5773 if (ConstantSDNode *N0C1 = isConstOrConstSplat(N0.getOperand(1))) {
5774 uint64_t c1 = N0C1->getZExtValue();
5775 if (c1 < OpSizeInBits) {
5776 uint64_t c2 = N1C->getZExtValue();
5777 APInt Mask = APInt::getHighBitsSet(OpSizeInBits, OpSizeInBits - c1);
5778 SDValue Shift;
5779 if (c2 > c1) {
5780 Mask <<= c2 - c1;
5781 SDLoc DL(N);
5782 Shift = DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0),
5783 DAG.getConstant(c2 - c1, DL, N1.getValueType()));
5784 } else {
5785 Mask.lshrInPlace(c1 - c2);
5786 SDLoc DL(N);
5787 Shift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0),
5788 DAG.getConstant(c1 - c2, DL, N1.getValueType()));
5789 }
5790 SDLoc DL(N0);
5791 return DAG.getNode(ISD::AND, DL, VT, Shift,
5792 DAG.getConstant(Mask, DL, VT));
5793 }
5794 }
5795 }
5796
5797 // fold (shl (sra x, c1), c1) -> (and x, (shl -1, c1))
5798 if (N0.getOpcode() == ISD::SRA && N1 == N0.getOperand(1) &&
5799 isConstantOrConstantVector(N1, /* No Opaques */ true)) {
5800 SDLoc DL(N);
5801 SDValue AllBits = DAG.getAllOnesConstant(DL, VT);
5802 SDValue HiBitsMask = DAG.getNode(ISD::SHL, DL, VT, AllBits, N1);
5803 return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), HiBitsMask);
5804 }
5805
5806 // fold (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2)
5807 // fold (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2)
5808 // Variant of version done on multiply, except mul by a power of 2 is turned
5809 // into a shift.
5810 if ((N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::OR) &&
5811 N0.getNode()->hasOneUse() &&
5812 isConstantOrConstantVector(N1, /* No Opaques */ true) &&
5813 isConstantOrConstantVector(N0.getOperand(1), /* No Opaques */ true)) {
5814 SDValue Shl0 = DAG.getNode(ISD::SHL, SDLoc(N0), VT, N0.getOperand(0), N1);
5815 SDValue Shl1 = DAG.getNode(ISD::SHL, SDLoc(N1), VT, N0.getOperand(1), N1);
5816 AddToWorklist(Shl0.getNode());
5817 AddToWorklist(Shl1.getNode());
5818 return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, Shl0, Shl1);
5819 }
5820
5821 // fold (shl (mul x, c1), c2) -> (mul x, c1 << c2)
5822 if (N0.getOpcode() == ISD::MUL && N0.getNode()->hasOneUse() &&
5823 isConstantOrConstantVector(N1, /* No Opaques */ true) &&
5824 isConstantOrConstantVector(N0.getOperand(1), /* No Opaques */ true)) {
5825 SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(N1), VT, N0.getOperand(1), N1);
5826 if (isConstantOrConstantVector(Shl))
5827 return DAG.getNode(ISD::MUL, SDLoc(N), VT, N0.getOperand(0), Shl);
5828 }
5829
5830 if (N1C && !N1C->isOpaque())
5831 if (SDValue NewSHL = visitShiftByConstant(N, N1C))
5832 return NewSHL;
5833
5834 return SDValue();
5835}
5836
5837SDValue DAGCombiner::visitSRA(SDNode *N) {
5838 SDValue N0 = N->getOperand(0);
5839 SDValue N1 = N->getOperand(1);
5840 EVT VT = N0.getValueType();
5841 unsigned OpSizeInBits = VT.getScalarSizeInBits();
5842
5843 // Arithmetic shifting an all-sign-bit value is a no-op.
5844 // fold (sra 0, x) -> 0
5845 // fold (sra -1, x) -> -1
5846 if (DAG.ComputeNumSignBits(N0) == OpSizeInBits)
5847 return N0;
5848
5849 // fold vector ops
5850 if (VT.isVector())
5851 if (SDValue FoldedVOp = SimplifyVBinOp(N))
5852 return FoldedVOp;
5853
5854 ConstantSDNode *N1C = isConstOrConstSplat(N1);
5855
5856 // fold (sra c1, c2) -> (sra c1, c2)
5857 ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
5858 if (N0C && N1C && !N1C->isOpaque())
5859 return DAG.FoldConstantArithmetic(ISD::SRA, SDLoc(N), VT, N0C, N1C);
5860 // fold (sra x, c >= size(x)) -> undef
5861 // NOTE: ALL vector elements must be too big to avoid partial UNDEFs.
5862 auto MatchShiftTooBig = [OpSizeInBits](ConstantSDNode *Val) {
5863 return Val->getAPIntValue().uge(OpSizeInBits);
5864 };
5865 if (matchUnaryPredicate(N1, MatchShiftTooBig))
5866 return DAG.getUNDEF(VT);
5867 // fold (sra x, 0) -> x
5868 if (N1C && N1C->isNullValue())
5869 return N0;
5870
5871 if (SDValue NewSel = foldBinOpIntoSelect(N))
5872 return NewSel;
5873
5874 // fold (sra (shl x, c1), c1) -> sext_inreg for some c1 and target supports
5875 // sext_inreg.
5876 if (N1C && N0.getOpcode() == ISD::SHL && N1 == N0.getOperand(1)) {
5877 unsigned LowBits = OpSizeInBits - (unsigned)N1C->getZExtValue();
5878 EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), LowBits);
5879 if (VT.isVector())
5880 ExtVT = EVT::getVectorVT(*DAG.getContext(),
5881 ExtVT, VT.getVectorNumElements());
5882 if ((!LegalOperations ||
5883 TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, ExtVT)))
5884 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT,
5885 N0.getOperand(0), DAG.getValueType(ExtVT));
5886 }
5887
5888 // fold (sra (sra x, c1), c2) -> (sra x, (add c1, c2))
5889 if (N0.getOpcode() == ISD::SRA) {
5890 SDLoc DL(N);
5891 EVT ShiftVT = N1.getValueType();
5892
5893 auto MatchOutOfRange = [OpSizeInBits](ConstantSDNode *LHS,
5894 ConstantSDNode *RHS) {
5895 APInt c1 = LHS->getAPIntValue();
5896 APInt c2 = RHS->getAPIntValue();
5897 zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
5898 return (c1 + c2).uge(OpSizeInBits);
5899 };
5900 if (matchBinaryPredicate(N1, N0.getOperand(1), MatchOutOfRange))
5901 return DAG.getNode(ISD::SRA, DL, VT, N0.getOperand(0),
5902 DAG.getConstant(OpSizeInBits - 1, DL, ShiftVT));
5903
5904 auto MatchInRange = [OpSizeInBits](ConstantSDNode *LHS,
5905 ConstantSDNode *RHS) {
5906 APInt c1 = LHS->getAPIntValue();
5907 APInt c2 = RHS->getAPIntValue();
5908 zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
5909 return (c1 + c2).ult(OpSizeInBits);
5910 };
5911 if (matchBinaryPredicate(N1, N0.getOperand(1), MatchInRange)) {
5912 SDValue Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, N1, N0.getOperand(1));
5913 return DAG.getNode(ISD::SRA, DL, VT, N0.getOperand(0), Sum);
5914 }
5915 }
5916
5917 // fold (sra (shl X, m), (sub result_size, n))
5918 // -> (sign_extend (trunc (shl X, (sub (sub result_size, n), m)))) for
5919 // result_size - n != m.
5920 // If truncate is free for the target sext(shl) is likely to result in better
5921 // code.
5922 if (N0.getOpcode() == ISD::SHL && N1C) {
5923 // Get the two constanst of the shifts, CN0 = m, CN = n.
5924 const ConstantSDNode *N01C = isConstOrConstSplat(N0.getOperand(1));
5925 if (N01C) {
5926 LLVMContext &Ctx = *DAG.getContext();
5927 // Determine what the truncate's result bitsize and type would be.
5928 EVT TruncVT = EVT::getIntegerVT(Ctx, OpSizeInBits - N1C->getZExtValue());
5929
5930 if (VT.isVector())
5931 TruncVT = EVT::getVectorVT(Ctx, TruncVT, VT.getVectorNumElements());
5932
5933 // Determine the residual right-shift amount.
5934 int ShiftAmt = N1C->getZExtValue() - N01C->getZExtValue();
5935
5936 // If the shift is not a no-op (in which case this should be just a sign
5937 // extend already), the truncated to type is legal, sign_extend is legal
5938 // on that type, and the truncate to that type is both legal and free,
5939 // perform the transform.
5940 if ((ShiftAmt > 0) &&
5941 TLI.isOperationLegalOrCustom(ISD::SIGN_EXTEND, TruncVT) &&
5942 TLI.isOperationLegalOrCustom(ISD::TRUNCATE, VT) &&
5943 TLI.isTruncateFree(VT, TruncVT)) {
5944 SDLoc DL(N);
5945 SDValue Amt = DAG.getConstant(ShiftAmt, DL,
5946 getShiftAmountTy(N0.getOperand(0).getValueType()));
5947 SDValue Shift = DAG.getNode(ISD::SRL, DL, VT,
5948 N0.getOperand(0), Amt);
5949 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, TruncVT,
5950 Shift);
5951 return DAG.getNode(ISD::SIGN_EXTEND, DL,
5952 N->getValueType(0), Trunc);
5953 }
5954 }
5955 }
5956
5957 // fold (sra x, (trunc (and y, c))) -> (sra x, (and (trunc y), (trunc c))).
5958 if (N1.getOpcode() == ISD::TRUNCATE &&
5959 N1.getOperand(0).getOpcode() == ISD::AND) {
5960 if (SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode()))
5961 return DAG.getNode(ISD::SRA, SDLoc(N), VT, N0, NewOp1);
5962 }
5963
5964 // fold (sra (trunc (srl x, c1)), c2) -> (trunc (sra x, c1 + c2))
5965 // if c1 is equal to the number of bits the trunc removes
5966 if (N0.getOpcode() == ISD::TRUNCATE &&
5967 (N0.getOperand(0).getOpcode() == ISD::SRL ||
5968 N0.getOperand(0).getOpcode() == ISD::SRA) &&
5969 N0.getOperand(0).hasOneUse() &&
5970 N0.getOperand(0).getOperand(1).hasOneUse() &&
5971 N1C) {
5972 SDValue N0Op0 = N0.getOperand(0);
5973 if (ConstantSDNode *LargeShift = isConstOrConstSplat(N0Op0.getOperand(1))) {
5974 unsigned LargeShiftVal = LargeShift->getZExtValue();
5975 EVT LargeVT = N0Op0.getValueType();
5976
5977 if (LargeVT.getScalarSizeInBits() - OpSizeInBits == LargeShiftVal) {
5978 SDLoc DL(N);
5979 SDValue Amt =
5980 DAG.getConstant(LargeShiftVal + N1C->getZExtValue(), DL,
5981 getShiftAmountTy(N0Op0.getOperand(0).getValueType()));
5982 SDValue SRA = DAG.getNode(ISD::SRA, DL, LargeVT,
5983 N0Op0.getOperand(0), Amt);
5984 return DAG.getNode(ISD::TRUNCATE, DL, VT, SRA);
5985 }
5986 }
5987 }
5988
5989 // Simplify, based on bits shifted out of the LHS.
5990 if (N1C && SimplifyDemandedBits(SDValue(N, 0)))
5991 return SDValue(N, 0);
5992
5993 // If the sign bit is known to be zero, switch this to a SRL.
5994 if (DAG.SignBitIsZero(N0))
5995 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, N1);
5996
5997 if (N1C && !N1C->isOpaque())
5998 if (SDValue NewSRA = visitShiftByConstant(N, N1C))
5999 return NewSRA;
6000
6001 return SDValue();
6002}
6003
6004SDValue DAGCombiner::visitSRL(SDNode *N) {
6005 SDValue N0 = N->getOperand(0);
6006 SDValue N1 = N->getOperand(1);
6007 EVT VT = N0.getValueType();
6008 unsigned OpSizeInBits = VT.getScalarSizeInBits();
6009
6010 // fold vector ops
6011 if (VT.isVector())
6012 if (SDValue FoldedVOp = SimplifyVBinOp(N))
6013 return FoldedVOp;
6014
6015 ConstantSDNode *N1C = isConstOrConstSplat(N1);
6016
6017 // fold (srl c1, c2) -> c1 >>u c2
6018 ConstantSDNode *N0C = getAsNonOpaqueConstant(N0);
6019 if (N0C && N1C && !N1C->isOpaque())
6020 return DAG.FoldConstantArithmetic(ISD::SRL, SDLoc(N), VT, N0C, N1C);
6021 // fold (srl 0, x) -> 0
6022 if (isNullConstantOrNullSplatConstant(N0))
6023 return N0;
6024 // fold (srl x, c >= size(x)) -> undef
6025 // NOTE: ALL vector elements must be too big to avoid partial UNDEFs.
6026 auto MatchShiftTooBig = [OpSizeInBits](ConstantSDNode *Val) {
6027 return Val->getAPIntValue().uge(OpSizeInBits);
6028 };
6029 if (matchUnaryPredicate(N1, MatchShiftTooBig))
6030 return DAG.getUNDEF(VT);
6031 // fold (srl x, 0) -> x
6032 if (N1C && N1C->isNullValue())
6033 return N0;
6034
6035 if (SDValue NewSel = foldBinOpIntoSelect(N))
6036 return NewSel;
6037
6038 // if (srl x, c) is known to be zero, return 0
6039 if (N1C && DAG.MaskedValueIsZero(SDValue(N, 0),
6040 APInt::getAllOnesValue(OpSizeInBits)))
6041 return DAG.getConstant(0, SDLoc(N), VT);
6042
6043 // fold (srl (srl x, c1), c2) -> 0 or (srl x, (add c1, c2))
6044 if (N0.getOpcode() == ISD::SRL) {
6045 auto MatchOutOfRange = [OpSizeInBits](ConstantSDNode *LHS,
6046 ConstantSDNode *RHS) {
6047 APInt c1 = LHS->getAPIntValue();
6048 APInt c2 = RHS->getAPIntValue();
6049 zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
6050 return (c1 + c2).uge(OpSizeInBits);
6051 };
6052 if (matchBinaryPredicate(N1, N0.getOperand(1), MatchOutOfRange))
6053 return DAG.getConstant(0, SDLoc(N), VT);
6054
6055 auto MatchInRange = [OpSizeInBits](ConstantSDNode *LHS,
6056 ConstantSDNode *RHS) {
6057 APInt c1 = LHS->getAPIntValue();
6058 APInt c2 = RHS->getAPIntValue();
6059 zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
6060 return (c1 + c2).ult(OpSizeInBits);
6061 };
6062 if (matchBinaryPredicate(N1, N0.getOperand(1), MatchInRange)) {
6063 SDLoc DL(N);
6064 EVT ShiftVT = N1.getValueType();
6065 SDValue Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, N1, N0.getOperand(1));
6066 return DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), Sum);
6067 }
6068 }
6069
6070 // fold (srl (trunc (srl x, c1)), c2) -> 0 or (trunc (srl x, (add c1, c2)))
6071 if (N1C && N0.getOpcode() == ISD::TRUNCATE &&
6072 N0.getOperand(0).getOpcode() == ISD::SRL) {
6073 if (auto N001C = isConstOrConstSplat(N0.getOperand(0).getOperand(1))) {
6074 uint64_t c1 = N001C->getZExtValue();
6075 uint64_t c2 = N1C->getZExtValue();
6076 EVT InnerShiftVT = N0.getOperand(0).getValueType();
6077 EVT ShiftCountVT = N0.getOperand(0).getOperand(1).getValueType();
6078 uint64_t InnerShiftSize = InnerShiftVT.getScalarSizeInBits();
6079 // This is only valid if the OpSizeInBits + c1 = size of inner shift.
6080 if (c1 + OpSizeInBits == InnerShiftSize) {
6081 SDLoc DL(N0);
6082 if (c1 + c2 >= InnerShiftSize)
6083 return DAG.getConstant(0, DL, VT);
6084 return DAG.getNode(ISD::TRUNCATE, DL, VT,
6085 DAG.getNode(ISD::SRL, DL, InnerShiftVT,
6086 N0.getOperand(0).getOperand(0),
6087 DAG.getConstant(c1 + c2, DL,
6088 ShiftCountVT)));
6089 }
6090 }
6091 }
6092
6093 // fold (srl (shl x, c), c) -> (and x, cst2)
6094 if (N0.getOpcode() == ISD::SHL && N0.getOperand(1) == N1 &&
6095 isConstantOrConstantVector(N1, /* NoOpaques */ true)) {
6096 SDLoc DL(N);
6097 SDValue Mask =
6098 DAG.getNode(ISD::SRL, DL, VT, DAG.getAllOnesConstant(DL, VT), N1);
6099 AddToWorklist(Mask.getNode());
6100 return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), Mask);
6101 }
6102
6103 // fold (srl (anyextend x), c) -> (and (anyextend (srl x, c)), mask)
6104 if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) {
6105 // Shifting in all undef bits?
6106 EVT SmallVT = N0.getOperand(0).getValueType();
6107 unsigned BitSize = SmallVT.getScalarSizeInBits();
6108 if (N1C->getZExtValue() >= BitSize)
6109 return DAG.getUNDEF(VT);
6110
6111 if (!LegalTypes || TLI.isTypeDesirableForOp(ISD::SRL, SmallVT)) {
6112 uint64_t ShiftAmt = N1C->getZExtValue();
6113 SDLoc DL0(N0);
6114 SDValue SmallShift = DAG.getNode(ISD::SRL, DL0, SmallVT,
6115 N0.getOperand(0),
6116 DAG.getConstant(ShiftAmt, DL0,
6117 getShiftAmountTy(SmallVT)));
6118 AddToWorklist(SmallShift.getNode());
6119 APInt Mask = APInt::getLowBitsSet(OpSizeInBits, OpSizeInBits - ShiftAmt);
6120 SDLoc DL(N);
6121 return DAG.getNode(ISD::AND, DL, VT,
6122 DAG.getNode(ISD::ANY_EXTEND, DL, VT, SmallShift),
6123 DAG.getConstant(Mask, DL, VT));
6124 }
6125 }
6126
6127 // fold (srl (sra X, Y), 31) -> (srl X, 31). This srl only looks at the sign
6128 // bit, which is unmodified by sra.
6129 if (N1C && N1C->getZExtValue() + 1 == OpSizeInBits) {
6130 if (N0.getOpcode() == ISD::SRA)
6131 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0.getOperand(0), N1);
6132 }
6133
6134 // fold (srl (ctlz x), "5") -> x iff x has one bit set (the low bit).
6135 if (N1C && N0.getOpcode() == ISD::CTLZ &&
6136 N1C->getAPIntValue() == Log2_32(OpSizeInBits)) {
6137 KnownBits Known;
6138 DAG.computeKnownBits(N0.getOperand(0), Known);
6139
6140 // If any of the input bits are KnownOne, then the input couldn't be all
6141 // zeros, thus the result of the srl will always be zero.
6142 if (Known.One.getBoolValue()) return DAG.getConstant(0, SDLoc(N0), VT);
6143
6144 // If all of the bits input the to ctlz node are known to be zero, then
6145 // the result of the ctlz is "32" and the result of the shift is one.
6146 APInt UnknownBits = ~Known.Zero;
6147 if (UnknownBits == 0) return DAG.getConstant(1, SDLoc(N0), VT);
6148
6149 // Otherwise, check to see if there is exactly one bit input to the ctlz.
6150 if (UnknownBits.isPowerOf2()) {
6151 // Okay, we know that only that the single bit specified by UnknownBits
6152 // could be set on input to the CTLZ node. If this bit is set, the SRL
6153 // will return 0, if it is clear, it returns 1. Change the CTLZ/SRL pair
6154 // to an SRL/XOR pair, which is likely to simplify more.
6155 unsigned ShAmt = UnknownBits.countTrailingZeros();
6156 SDValue Op = N0.getOperand(0);
6157
6158 if (ShAmt) {
6159 SDLoc DL(N0);
6160 Op = DAG.getNode(ISD::SRL, DL, VT, Op,
6161 DAG.getConstant(ShAmt, DL,
6162 getShiftAmountTy(Op.getValueType())));
6163 AddToWorklist(Op.getNode());
6164 }
6165
6166 SDLoc DL(N);
6167 return DAG.getNode(ISD::XOR, DL, VT,
6168 Op, DAG.getConstant(1, DL, VT));
6169 }
6170 }
6171
6172 // fold (srl x, (trunc (and y, c))) -> (srl x, (and (trunc y), (trunc c))).
6173 if (N1.getOpcode() == ISD::TRUNCATE &&
6174 N1.getOperand(0).getOpcode() == ISD::AND) {
6175 if (SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode()))
6176 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, NewOp1);
6177 }
6178
6179 // fold operands of srl based on knowledge that the low bits are not
6180 // demanded.
6181 if (N1C && SimplifyDemandedBits(SDValue(N, 0)))
6182 return SDValue(N, 0);
6183
6184 if (N1C && !N1C->isOpaque())
6185 if (SDValue NewSRL = visitShiftByConstant(N, N1C))
6186 return NewSRL;
6187
6188 // Attempt to convert a srl of a load into a narrower zero-extending load.
6189 if (SDValue NarrowLoad = ReduceLoadWidth(N))
6190 return NarrowLoad;
6191
6192 // Here is a common situation. We want to optimize:
6193 //
6194 // %a = ...
6195 // %b = and i32 %a, 2
6196 // %c = srl i32 %b, 1
6197 // brcond i32 %c ...
6198 //
6199 // into
6200 //
6201 // %a = ...
6202 // %b = and %a, 2
6203 // %c = setcc eq %b, 0
6204 // brcond %c ...
6205 //
6206 // However when after the source operand of SRL is optimized into AND, the SRL
6207 // itself may not be optimized further. Look for it and add the BRCOND into
6208 // the worklist.
6209 if (N->hasOneUse()) {
6210 SDNode *Use = *N->use_begin();
6211 if (Use->getOpcode() == ISD::BRCOND)
6212 AddToWorklist(Use);
6213 else if (Use->getOpcode() == ISD::TRUNCATE && Use->hasOneUse()) {
6214 // Also look pass the truncate.
6215 Use = *Use->use_begin();
6216 if (Use->getOpcode() == ISD::BRCOND)
6217 AddToWorklist(Use);
6218 }
6219 }
6220
6221 return SDValue();
6222}
6223
6224SDValue DAGCombiner::visitABS(SDNode *N) {
6225 SDValue N0 = N->getOperand(0);
6226 EVT VT = N->getValueType(0);
6227
6228 // fold (abs c1) -> c2
6229 if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
6230 return DAG.getNode(ISD::ABS, SDLoc(N), VT, N0);
6231 // fold (abs (abs x)) -> (abs x)
6232 if (N0.getOpcode() == ISD::ABS)
6233 return N0;
6234 // fold (abs x) -> x iff not-negative
6235 if (DAG.SignBitIsZero(N0))
6236 return N0;
6237 return SDValue();
6238}
6239
6240SDValue DAGCombiner::visitBSWAP(SDNode *N) {
6241 SDValue N0 = N->getOperand(0);
6242 EVT VT = N->getValueType(0);
6243
6244 // fold (bswap c1) -> c2
6245 if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
6246 return DAG.getNode(ISD::BSWAP, SDLoc(N), VT, N0);
6247 // fold (bswap (bswap x)) -> x
6248 if (N0.getOpcode() == ISD::BSWAP)
6249 return N0->getOperand(0);
6250 return SDValue();
6251}
6252
6253SDValue DAGCombiner::visitBITREVERSE(SDNode *N) {
6254 SDValue N0 = N->getOperand(0);
6255 EVT VT = N->getValueType(0);
6256
6257 // fold (bitreverse c1) -> c2
6258 if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
6259 return DAG.getNode(ISD::BITREVERSE, SDLoc(N), VT, N0);
6260 // fold (bitreverse (bitreverse x)) -> x
6261 if (N0.getOpcode() == ISD::BITREVERSE)
6262 return N0.getOperand(0);
6263 return SDValue();
6264}
6265
6266SDValue DAGCombiner::visitCTLZ(SDNode *N) {
6267 SDValue N0 = N->getOperand(0);
6268 EVT VT = N->getValueType(0);
6269
6270 // fold (ctlz c1) -> c2
6271 if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
6272 return DAG.getNode(ISD::CTLZ, SDLoc(N), VT, N0);
6273 return SDValue();
6274}
6275
6276SDValue DAGCombiner::visitCTLZ_ZERO_UNDEF(SDNode *N) {
6277 SDValue N0 = N->getOperand(0);
6278 EVT VT = N->getValueType(0);
6279
6280 // fold (ctlz_zero_undef c1) -> c2
6281 if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
6282 return DAG.getNode(ISD::CTLZ_ZERO_UNDEF, SDLoc(N), VT, N0);
6283 return SDValue();
6284}
6285
6286SDValue DAGCombiner::visitCTTZ(SDNode *N) {
6287 SDValue N0 = N->getOperand(0);
6288 EVT VT = N->getValueType(0);
6289
6290 // fold (cttz c1) -> c2
6291 if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
6292 return DAG.getNode(ISD::CTTZ, SDLoc(N), VT, N0);
6293 return SDValue();
6294}
6295
6296SDValue DAGCombiner::visitCTTZ_ZERO_UNDEF(SDNode *N) {
6297 SDValue N0 = N->getOperand(0);
6298 EVT VT = N->getValueType(0);
6299
6300 // fold (cttz_zero_undef c1) -> c2
6301 if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
6302 return DAG.getNode(ISD::CTTZ_ZERO_UNDEF, SDLoc(N), VT, N0);
6303 return SDValue();
6304}
6305
6306SDValue DAGCombiner::visitCTPOP(SDNode *N) {
6307 SDValue N0 = N->getOperand(0);
6308 EVT VT = N->getValueType(0);
6309
6310 // fold (ctpop c1) -> c2
6311 if (DAG.isConstantIntBuildVectorOrConstantInt(N0))
6312 return DAG.getNode(ISD::CTPOP, SDLoc(N), VT, N0);
6313 return SDValue();
6314}
6315
6316/// \brief Generate Min/Max node
6317static SDValue combineMinNumMaxNum(const SDLoc &DL, EVT VT, SDValue LHS,
6318 SDValue RHS, SDValue True, SDValue False,
6319 ISD::CondCode CC, const TargetLowering &TLI,
6320 SelectionDAG &DAG) {
6321 if (!(LHS == True && RHS == False) && !(LHS == False && RHS == True))
6322 return SDValue();
6323
6324 switch (CC) {
6325 case ISD::SETOLT:
6326 case ISD::SETOLE:
6327 case ISD::SETLT:
6328 case ISD::SETLE:
6329 case ISD::SETULT:
6330 case ISD::SETULE: {
6331 unsigned Opcode = (LHS == True) ? ISD::FMINNUM : ISD::FMAXNUM;
6332 if (TLI.isOperationLegal(Opcode, VT))
6333 return DAG.getNode(Opcode, DL, VT, LHS, RHS);
6334 return SDValue();
6335 }
6336 case ISD::SETOGT:
6337 case ISD::SETOGE:
6338 case ISD::SETGT:
6339 case ISD::SETGE:
6340 case ISD::SETUGT:
6341 case ISD::SETUGE: {
6342 unsigned Opcode = (LHS == True) ? ISD::FMAXNUM : ISD::FMINNUM;
6343 if (TLI.isOperationLegal(Opcode, VT))
6344 return DAG.getNode(Opcode, DL, VT, LHS, RHS);
6345 return SDValue();
6346 }
6347 default:
6348 return SDValue();
6349 }
6350}
6351
6352SDValue DAGCombiner::foldSelectOfConstants(SDNode *N) {
6353 SDValue Cond = N->getOperand(0);
6354 SDValue N1 = N->getOperand(1);
6355 SDValue N2 = N->getOperand(2);
6356 EVT VT = N->getValueType(0);
6357 EVT CondVT = Cond.getValueType();
6358 SDLoc DL(N);
6359
6360 if (!VT.isInteger())
6361 return SDValue();
6362
6363 auto *C1 = dyn_cast<ConstantSDNode>(N1);
6364 auto *C2 = dyn_cast<ConstantSDNode>(N2);
6365 if (!C1 || !C2)
6366 return SDValue();
6367
6368 // Only do this before legalization to avoid conflicting with target-specific
6369 // transforms in the other direction (create a select from a zext/sext). There
6370 // is also a target-independent combine here in DAGCombiner in the other
6371 // direction for (select Cond, -1, 0) when the condition is not i1.
6372 if (CondVT == MVT::i1 && !LegalOperations) {
6373 if (C1->isNullValue() && C2->isOne()) {
6374 // select Cond, 0, 1 --> zext (!Cond)
6375 SDValue NotCond = DAG.getNOT(DL, Cond, MVT::i1);
6376 if (VT != MVT::i1)
6377 NotCond = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, NotCond);
6378 return NotCond;
6379 }
6380 if (C1->isNullValue() && C2->isAllOnesValue()) {
6381 // select Cond, 0, -1 --> sext (!Cond)
6382 SDValue NotCond = DAG.getNOT(DL, Cond, MVT::i1);
6383 if (VT != MVT::i1)
6384 NotCond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, NotCond);
6385 return NotCond;
6386 }
6387 if (C1->isOne() && C2->isNullValue()) {
6388 // select Cond, 1, 0 --> zext (Cond)
6389 if (VT != MVT::i1)
6390 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond);
6391 return Cond;
6392 }
6393 if (C1->isAllOnesValue() && C2->isNullValue()) {
6394 // select Cond, -1, 0 --> sext (Cond)
6395 if (VT != MVT::i1)
6396 Cond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Cond);
6397 return Cond;
6398 }
6399
6400 // For any constants that differ by 1, we can transform the select into an
6401 // extend and add. Use a target hook because some targets may prefer to
6402 // transform in the other direction.
6403 if (TLI.convertSelectOfConstantsToMath(VT)) {
6404 if (C1->getAPIntValue() - 1 == C2->getAPIntValue()) {
6405 // select Cond, C1, C1-1 --> add (zext Cond), C1-1
6406 if (VT != MVT::i1)
6407 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond);
6408 return DAG.getNode(ISD::ADD, DL, VT, Cond, N2);
6409 }
6410 if (C1->getAPIntValue() + 1 == C2->getAPIntValue()) {
6411 // select Cond, C1, C1+1 --> add (sext Cond), C1+1
6412 if (VT != MVT::i1)
6413 Cond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Cond);
6414 return DAG.getNode(ISD::ADD, DL, VT, Cond, N2);
6415 }
6416 }
6417
6418 return SDValue();
6419 }
6420
6421 // fold (select Cond, 0, 1) -> (xor Cond, 1)
6422 // We can't do this reliably if integer based booleans have different contents
6423 // to floating point based booleans. This is because we can't tell whether we
6424 // have an integer-based boolean or a floating-point-based boolean unless we
6425 // can find the SETCC that produced it and inspect its operands. This is
6426 // fairly easy if C is the SETCC node, but it can potentially be
6427 // undiscoverable (or not reasonably discoverable). For example, it could be
6428 // in another basic block or it could require searching a complicated
6429 // expression.
6430 if (CondVT.isInteger() &&
6431 TLI.getBooleanContents(false, true) ==
6432 TargetLowering::ZeroOrOneBooleanContent &&
6433 TLI.getBooleanContents(false, false) ==
6434 TargetLowering::ZeroOrOneBooleanContent &&
6435 C1->isNullValue() && C2->isOne()) {
6436 SDValue NotCond =
6437 DAG.getNode(ISD::XOR, DL, CondVT, Cond, DAG.getConstant(1, DL, CondVT));
6438 if (VT.bitsEq(CondVT))
6439 return NotCond;
6440 return DAG.getZExtOrTrunc(NotCond, DL, VT);
6441 }
6442
6443 return SDValue();
6444}
6445
6446SDValue DAGCombiner::visitSELECT(SDNode *N) {
6447 SDValue N0 = N->getOperand(0);
6448 SDValue N1 = N->getOperand(1);
6449 SDValue N2 = N->getOperand(2);
6450 EVT VT = N->getValueType(0);
6451 EVT VT0 = N0.getValueType();
6452 SDLoc DL(N);
6453
6454 // fold (select C, X, X) -> X
6455 if (N1 == N2)
6456 return N1;
6457
6458 if (const ConstantSDNode *N0C = dyn_cast<const ConstantSDNode>(N0)) {
6459 // fold (select true, X, Y) -> X
6460 // fold (select false, X, Y) -> Y
6461 return !N0C->isNullValue() ? N1 : N2;
6462 }
6463
6464 // fold (select X, X, Y) -> (or X, Y)
6465 // fold (select X, 1, Y) -> (or C, Y)
6466 if (VT == VT0 && VT == MVT::i1 && (N0 == N1 || isOneConstant(N1)))
6467 return DAG.getNode(ISD::OR, DL, VT, N0, N2);
6468
6469 if (SDValue V = foldSelectOfConstants(N))
6470 return V;
6471
6472 // fold (select C, 0, X) -> (and (not C), X)
6473 if (VT == VT0 && VT == MVT::i1 && isNullConstant(N1)) {
6474 SDValue NOTNode = DAG.getNOT(SDLoc(N0), N0, VT);
6475 AddToWorklist(NOTNode.getNode());
6476 return DAG.getNode(ISD::AND, DL, VT, NOTNode, N2);
6477 }
6478 // fold (select C, X, 1) -> (or (not C), X)
6479 if (VT == VT0 && VT == MVT::i1 && isOneConstant(N2)) {
6480 SDValue NOTNode = DAG.getNOT(SDLoc(N0), N0, VT);
6481 AddToWorklist(NOTNode.getNode());
6482 return DAG.getNode(ISD::OR, DL, VT, NOTNode, N1);
6483 }
6484 // fold (select X, Y, X) -> (and X, Y)
6485 // fold (select X, Y, 0) -> (and X, Y)
6486 if (VT == VT0 && VT == MVT::i1 && (N0 == N2 || isNullConstant(N2)))
6487 return DAG.getNode(ISD::AND, DL, VT, N0, N1);
6488
6489 // If we can fold this based on the true/false value, do so.
6490 if (SimplifySelectOps(N, N1, N2))
6491 return SDValue(N, 0); // Don't revisit N.
6492
6493 if (VT0 == MVT::i1) {
6494 // The code in this block deals with the following 2 equivalences:
6495 // select(C0|C1, x, y) <=> select(C0, x, select(C1, x, y))
6496 // select(C0&C1, x, y) <=> select(C0, select(C1, x, y), y)
6497 // The target can specify its preferred form with the
6498 // shouldNormalizeToSelectSequence() callback. However we always transform
6499 // to the right anyway if we find the inner select exists in the DAG anyway
6500 // and we always transform to the left side if we know that we can further
6501 // optimize the combination of the conditions.
6502 bool normalizeToSequence =
6503 TLI.shouldNormalizeToSelectSequence(*DAG.getContext(), VT);
6504 // select (and Cond0, Cond1), X, Y
6505 // -> select Cond0, (select Cond1, X, Y), Y
6506 if (N0->getOpcode() == ISD::AND && N0->hasOneUse()) {
6507 SDValue Cond0 = N0->getOperand(0);
6508 SDValue Cond1 = N0->getOperand(1);
6509 SDValue InnerSelect =
6510 DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond1, N1, N2);
6511 if (normalizeToSequence || !InnerSelect.use_empty())
6512 return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond0,
6513 InnerSelect, N2);
6514 }
6515 // select (or Cond0, Cond1), X, Y -> select Cond0, X, (select Cond1, X, Y)
6516 if (N0->getOpcode() == ISD::OR && N0->hasOneUse()) {
6517 SDValue Cond0 = N0->getOperand(0);
6518 SDValue Cond1 = N0->getOperand(1);
6519 SDValue InnerSelect =
6520 DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond1, N1, N2);
6521 if (normalizeToSequence || !InnerSelect.use_empty())
6522 return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond0, N1,
6523 InnerSelect);
6524 }
6525
6526 // select Cond0, (select Cond1, X, Y), Y -> select (and Cond0, Cond1), X, Y
6527 if (N1->getOpcode() == ISD::SELECT && N1->hasOneUse()) {
6528 SDValue N1_0 = N1->getOperand(0);
6529 SDValue N1_1 = N1->getOperand(1);
6530 SDValue N1_2 = N1->getOperand(2);
6531 if (N1_2 == N2 && N0.getValueType() == N1_0.getValueType()) {
6532 // Create the actual and node if we can generate good code for it.
6533 if (!normalizeToSequence) {
6534 SDValue And = DAG.getNode(ISD::AND, DL, N0.getValueType(), N0, N1_0);
6535 return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), And, N1_1, N2);
6536 }
6537 // Otherwise see if we can optimize the "and" to a better pattern.
6538 if (SDValue Combined = visitANDLike(N0, N1_0, N))
6539 return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Combined, N1_1,
6540 N2);
6541 }
6542 }
6543 // select Cond0, X, (select Cond1, X, Y) -> select (or Cond0, Cond1), X, Y
6544 if (N2->getOpcode() == ISD::SELECT && N2->hasOneUse()) {
6545 SDValue N2_0 = N2->getOperand(0);
6546 SDValue N2_1 = N2->getOperand(1);
6547 SDValue N2_2 = N2->getOperand(2);
6548 if (N2_1 == N1 && N0.getValueType() == N2_0.getValueType()) {
6549 // Create the actual or node if we can generate good code for it.
6550 if (!normalizeToSequence) {
6551 SDValue Or = DAG.getNode(ISD::OR, DL, N0.getValueType(), N0, N2_0);
6552 return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Or, N1, N2_2);
6553 }
6554 // Otherwise see if we can optimize to a better pattern.
6555 if (SDValue Combined = visitORLike(N0, N2_0, N))
6556 return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Combined, N1,
6557 N2_2);
6558 }
6559 }
6560 }
6561
6562 // select (xor Cond, 1), X, Y -> select Cond, Y, X
6563 if (VT0 == MVT::i1) {
6564 if (N0->getOpcode() == ISD::XOR) {
6565 if (auto *C = dyn_cast<ConstantSDNode>(N0->getOperand(1))) {
6566 SDValue Cond0 = N0->getOperand(0);
6567 if (C->isOne())
6568 return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond0, N2, N1);
6569 }
6570 }
6571 }
6572
6573 // fold selects based on a setcc into other things, such as min/max/abs
6574 if (N0.getOpcode() == ISD::SETCC) {
6575 // select x, y (fcmp lt x, y) -> fminnum x, y
6576 // select x, y (fcmp gt x, y) -> fmaxnum x, y
6577 //
6578 // This is OK if we don't care about what happens if either operand is a
6579 // NaN.
6580 //
6581
6582 // FIXME: Instead of testing for UnsafeFPMath, this should be checking for
6583 // no signed zeros as well as no nans.
6584 const TargetOptions &Options = DAG.getTarget().Options;
6585 if (Options.UnsafeFPMath && VT.isFloatingPoint() && N0.hasOneUse() &&
6586 DAG.isKnownNeverNaN(N1) && DAG.isKnownNeverNaN(N2)) {
6587 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
6588
6589 if (SDValue FMinMax = combineMinNumMaxNum(
6590 DL, VT, N0.getOperand(0), N0.getOperand(1), N1, N2, CC, TLI, DAG))
6591 return FMinMax;
6592 }
6593
6594 if ((!LegalOperations &&
6595 TLI.isOperationLegalOrCustom(ISD::SELECT_CC, VT)) ||
6596 TLI.isOperationLegal(ISD::SELECT_CC, VT))
6597 return DAG.getNode(ISD::SELECT_CC, DL, VT, N0.getOperand(0),
6598 N0.getOperand(1), N1, N2, N0.getOperand(2));
6599 return SimplifySelect(DL, N0, N1, N2);
6600 }
6601
6602 return SDValue();
6603}
6604
6605static
6606std::pair<SDValue, SDValue> SplitVSETCC(const SDNode *N, SelectionDAG &DAG) {
6607 SDLoc DL(N);
6608 EVT LoVT, HiVT;
6609 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
6610
6611 // Split the inputs.
6612 SDValue Lo, Hi, LL, LH, RL, RH;
6613 std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0);
6614 std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1);
6615
6616 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2));
6617 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2));
6618
6619 return std::make_pair(Lo, Hi);
6620}
6621
6622// This function assumes all the vselect's arguments are CONCAT_VECTOR
6623// nodes and that the condition is a BV of ConstantSDNodes (or undefs).
6624static SDValue ConvertSelectToConcatVector(SDNode *N, SelectionDAG &DAG) {
6625 SDLoc DL(N);
6626 SDValue Cond = N->getOperand(0);
6627 SDValue LHS = N->getOperand(1);
6628 SDValue RHS = N->getOperand(2);
6629 EVT VT = N->getValueType(0);
6630 int NumElems = VT.getVectorNumElements();
6631 assert(LHS.getOpcode() == ISD::CONCAT_VECTORS &&(static_cast <bool> (LHS.getOpcode() == ISD::CONCAT_VECTORS
&& RHS.getOpcode() == ISD::CONCAT_VECTORS &&
Cond.getOpcode() == ISD::BUILD_VECTOR) ? void (0) : __assert_fail
("LHS.getOpcode() == ISD::CONCAT_VECTORS && RHS.getOpcode() == ISD::CONCAT_VECTORS && Cond.getOpcode() == ISD::BUILD_VECTOR"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 6633, __extension__ __PRETTY_FUNCTION__))
6632 RHS.getOpcode() == ISD::CONCAT_VECTORS &&(static_cast <bool> (LHS.getOpcode() == ISD::CONCAT_VECTORS
&& RHS.getOpcode() == ISD::CONCAT_VECTORS &&
Cond.getOpcode() == ISD::BUILD_VECTOR) ? void (0) : __assert_fail
("LHS.getOpcode() == ISD::CONCAT_VECTORS && RHS.getOpcode() == ISD::CONCAT_VECTORS && Cond.getOpcode() == ISD::BUILD_VECTOR"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 6633, __extension__ __PRETTY_FUNCTION__))
6633 Cond.getOpcode() == ISD::BUILD_VECTOR)(static_cast <bool> (LHS.getOpcode() == ISD::CONCAT_VECTORS
&& RHS.getOpcode() == ISD::CONCAT_VECTORS &&
Cond.getOpcode() == ISD::BUILD_VECTOR) ? void (0) : __assert_fail
("LHS.getOpcode() == ISD::CONCAT_VECTORS && RHS.getOpcode() == ISD::CONCAT_VECTORS && Cond.getOpcode() == ISD::BUILD_VECTOR"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 6633, __extension__ __PRETTY_FUNCTION__))
;
6634
6635 // CONCAT_VECTOR can take an arbitrary number of arguments. We only care about
6636 // binary ones here.
6637 if (LHS->getNumOperands() != 2 || RHS->getNumOperands() != 2)
6638 return SDValue();
6639
6640 // We're sure we have an even number of elements due to the
6641 // concat_vectors we have as arguments to vselect.
6642 // Skip BV elements until we find one that's not an UNDEF
6643 // After we find an UNDEF element, keep looping until we get to half the
6644 // length of the BV and see if all the non-undef nodes are the same.
6645 ConstantSDNode *BottomHalf = nullptr;
6646 for (int i = 0; i < NumElems / 2; ++i) {
6647 if (Cond->getOperand(i)->isUndef())
6648 continue;
6649
6650 if (BottomHalf == nullptr)
6651 BottomHalf = cast<ConstantSDNode>(Cond.getOperand(i));
6652 else if (Cond->getOperand(i).getNode() != BottomHalf)
6653 return SDValue();
6654 }
6655
6656 // Do the same for the second half of the BuildVector
6657 ConstantSDNode *TopHalf = nullptr;
6658 for (int i = NumElems / 2; i < NumElems; ++i) {
6659 if (Cond->getOperand(i)->isUndef())
6660 continue;
6661
6662 if (TopHalf == nullptr)
6663 TopHalf = cast<ConstantSDNode>(Cond.getOperand(i));
6664 else if (Cond->getOperand(i).getNode() != TopHalf)
6665 return SDValue();
6666 }
6667
6668 assert(TopHalf && BottomHalf &&(static_cast <bool> (TopHalf && BottomHalf &&
"One half of the selector was all UNDEFs and the other was all the "
"same value. This should have been addressed before this function."
) ? void (0) : __assert_fail ("TopHalf && BottomHalf && \"One half of the selector was all UNDEFs and the other was all the \" \"same value. This should have been addressed before this function.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 6670, __extension__ __PRETTY_FUNCTION__))
6669 "One half of the selector was all UNDEFs and the other was all the "(static_cast <bool> (TopHalf && BottomHalf &&
"One half of the selector was all UNDEFs and the other was all the "
"same value. This should have been addressed before this function."
) ? void (0) : __assert_fail ("TopHalf && BottomHalf && \"One half of the selector was all UNDEFs and the other was all the \" \"same value. This should have been addressed before this function.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 6670, __extension__ __PRETTY_FUNCTION__))
6670 "same value. This should have been addressed before this function.")(static_cast <bool> (TopHalf && BottomHalf &&
"One half of the selector was all UNDEFs and the other was all the "
"same value. This should have been addressed before this function."
) ? void (0) : __assert_fail ("TopHalf && BottomHalf && \"One half of the selector was all UNDEFs and the other was all the \" \"same value. This should have been addressed before this function.\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 6670, __extension__ __PRETTY_FUNCTION__))
;
6671 return DAG.getNode(
6672 ISD::CONCAT_VECTORS, DL, VT,
6673 BottomHalf->isNullValue() ? RHS->getOperand(0) : LHS->getOperand(0),
6674 TopHalf->isNullValue() ? RHS->getOperand(1) : LHS->getOperand(1));
6675}
6676
6677SDValue DAGCombiner::visitMSCATTER(SDNode *N) {
6678 if (Level >= AfterLegalizeTypes)
6679 return SDValue();
6680
6681 MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(N);
6682 SDValue Mask = MSC->getMask();
6683 SDValue Data = MSC->getValue();
6684 SDLoc DL(N);
6685
6686 // If the MSCATTER data type requires splitting and the mask is provided by a
6687 // SETCC, then split both nodes and its operands before legalization. This
6688 // prevents the type legalizer from unrolling SETCC into scalar comparisons
6689 // and enables future optimizations (e.g. min/max pattern matching on X86).
6690 if (Mask.getOpcode() != ISD::SETCC)
6691 return SDValue();
6692
6693 // Check if any splitting is required.
6694 if (TLI.getTypeAction(*DAG.getContext(), Data.getValueType()) !=
6695 TargetLowering::TypeSplitVector)
6696 return SDValue();
6697 SDValue MaskLo, MaskHi, Lo, Hi;
6698 std::tie(MaskLo, MaskHi) = SplitVSETCC(Mask.getNode(), DAG);
6699
6700 EVT LoVT, HiVT;
6701 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MSC->getValueType(0));
6702
6703 SDValue Chain = MSC->getChain();
6704
6705 EVT MemoryVT = MSC->getMemoryVT();
6706 unsigned Alignment = MSC->getOriginalAlignment();
6707
6708 EVT LoMemVT, HiMemVT;
6709 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
6710
6711 SDValue DataLo, DataHi;
6712 std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
6713
6714 SDValue BasePtr = MSC->getBasePtr();
6715 SDValue IndexLo, IndexHi;
6716 std::tie(IndexLo, IndexHi) = DAG.SplitVector(MSC->getIndex(), DL);
6717
6718 MachineMemOperand *MMO = DAG.getMachineFunction().
6719 getMachineMemOperand(MSC->getPointerInfo(),
6720 MachineMemOperand::MOStore, LoMemVT.getStoreSize(),
6721 Alignment, MSC->getAAInfo(), MSC->getRanges());
6722
6723 SDValue OpsLo[] = { Chain, DataLo, MaskLo, BasePtr, IndexLo };
6724 Lo = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataLo.getValueType(),
6725 DL, OpsLo, MMO);
6726
6727 SDValue OpsHi[] = {Chain, DataHi, MaskHi, BasePtr, IndexHi};
6728 Hi = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataHi.getValueType(),
6729 DL, OpsHi, MMO);
6730
6731 AddToWorklist(Lo.getNode());
6732 AddToWorklist(Hi.getNode());
6733
6734 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
6735}
6736
6737SDValue DAGCombiner::visitMSTORE(SDNode *N) {
6738 if (Level >= AfterLegalizeTypes)
6739 return SDValue();
6740
6741 MaskedStoreSDNode *MST = dyn_cast<MaskedStoreSDNode>(N);
6742 SDValue Mask = MST->getMask();
6743 SDValue Data = MST->getValue();
6744 EVT VT = Data.getValueType();
6745 SDLoc DL(N);
6746
6747 // If the MSTORE data type requires splitting and the mask is provided by a
6748 // SETCC, then split both nodes and its operands before legalization. This
6749 // prevents the type legalizer from unrolling SETCC into scalar comparisons
6750 // and enables future optimizations (e.g. min/max pattern matching on X86).
6751 if (Mask.getOpcode() == ISD::SETCC) {
6752 // Check if any splitting is required.
6753 if (TLI.getTypeAction(*DAG.getContext(), VT) !=
6754 TargetLowering::TypeSplitVector)
6755 return SDValue();
6756
6757 SDValue MaskLo, MaskHi, Lo, Hi;
6758 std::tie(MaskLo, MaskHi) = SplitVSETCC(Mask.getNode(), DAG);
6759
6760 SDValue Chain = MST->getChain();
6761 SDValue Ptr = MST->getBasePtr();
6762
6763 EVT MemoryVT = MST->getMemoryVT();
6764 unsigned Alignment = MST->getOriginalAlignment();
6765
6766 // if Alignment is equal to the vector size,
6767 // take the half of it for the second part
6768 unsigned SecondHalfAlignment =
6769 (Alignment == VT.getSizeInBits() / 8) ? Alignment / 2 : Alignment;
6770
6771 EVT LoMemVT, HiMemVT;
6772 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
6773
6774 SDValue DataLo, DataHi;
6775 std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
6776
6777 MachineMemOperand *MMO = DAG.getMachineFunction().
6778 getMachineMemOperand(MST->getPointerInfo(),
6779 MachineMemOperand::MOStore, LoMemVT.getStoreSize(),
6780 Alignment, MST->getAAInfo(), MST->getRanges());
6781
6782 Lo = DAG.getMaskedStore(Chain, DL, DataLo, Ptr, MaskLo, LoMemVT, MMO,
6783 MST->isTruncatingStore(),
6784 MST->isCompressingStore());
6785
6786 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
6787 MST->isCompressingStore());
6788
6789 MMO = DAG.getMachineFunction().
6790 getMachineMemOperand(MST->getPointerInfo(),
6791 MachineMemOperand::MOStore, HiMemVT.getStoreSize(),
6792 SecondHalfAlignment, MST->getAAInfo(),
6793 MST->getRanges());
6794
6795 Hi = DAG.getMaskedStore(Chain, DL, DataHi, Ptr, MaskHi, HiMemVT, MMO,
6796 MST->isTruncatingStore(),
6797 MST->isCompressingStore());
6798
6799 AddToWorklist(Lo.getNode());
6800 AddToWorklist(Hi.getNode());
6801
6802 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
6803 }
6804 return SDValue();
6805}
6806
6807SDValue DAGCombiner::visitMGATHER(SDNode *N) {
6808 if (Level >= AfterLegalizeTypes)
6809 return SDValue();
6810
6811 MaskedGatherSDNode *MGT = cast<MaskedGatherSDNode>(N);
6812 SDValue Mask = MGT->getMask();
6813 SDLoc DL(N);
6814
6815 // If the MGATHER result requires splitting and the mask is provided by a
6816 // SETCC, then split both nodes and its operands before legalization. This
6817 // prevents the type legalizer from unrolling SETCC into scalar comparisons
6818 // and enables future optimizations (e.g. min/max pattern matching on X86).
6819
6820 if (Mask.getOpcode() != ISD::SETCC)
6821 return SDValue();
6822
6823 EVT VT = N->getValueType(0);
6824
6825 // Check if any splitting is required.
6826 if (TLI.getTypeAction(*DAG.getContext(), VT) !=
6827 TargetLowering::TypeSplitVector)
6828 return SDValue();
6829
6830 SDValue MaskLo, MaskHi, Lo, Hi;
6831 std::tie(MaskLo, MaskHi) = SplitVSETCC(Mask.getNode(), DAG);
6832
6833 SDValue Src0 = MGT->getValue();
6834 SDValue Src0Lo, Src0Hi;
6835 std::tie(Src0Lo, Src0Hi) = DAG.SplitVector(Src0, DL);
6836
6837 EVT LoVT, HiVT;
6838 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
6839
6840 SDValue Chain = MGT->getChain();
6841 EVT MemoryVT = MGT->getMemoryVT();
6842 unsigned Alignment = MGT->getOriginalAlignment();
6843
6844 EVT LoMemVT, HiMemVT;
6845 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
6846
6847 SDValue BasePtr = MGT->getBasePtr();
6848 SDValue Index = MGT->getIndex();
6849 SDValue IndexLo, IndexHi;
6850 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, DL);
6851
6852 MachineMemOperand *MMO = DAG.getMachineFunction().
6853 getMachineMemOperand(MGT->getPointerInfo(),
6854 MachineMemOperand::MOLoad, LoMemVT.getStoreSize(),
6855 Alignment, MGT->getAAInfo(), MGT->getRanges());
6856
6857 SDValue OpsLo[] = { Chain, Src0Lo, MaskLo, BasePtr, IndexLo };
6858 Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, DL, OpsLo,
6859 MMO);
6860
6861 SDValue OpsHi[] = {Chain, Src0Hi, MaskHi, BasePtr, IndexHi};
6862 Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, DL, OpsHi,
6863 MMO);
6864
6865 AddToWorklist(Lo.getNode());
6866 AddToWorklist(Hi.getNode());
6867
6868 // Build a factor node to remember that this load is independent of the
6869 // other one.
6870 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo.getValue(1),
6871 Hi.getValue(1));
6872
6873 // Legalized the chain result - switch anything that used the old chain to
6874 // use the new one.
6875 DAG.ReplaceAllUsesOfValueWith(SDValue(MGT, 1), Chain);
6876
6877 SDValue GatherRes = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
6878
6879 SDValue RetOps[] = { GatherRes, Chain };
6880 return DAG.getMergeValues(RetOps, DL);
6881}
6882
6883SDValue DAGCombiner::visitMLOAD(SDNode *N) {
6884 if (Level >= AfterLegalizeTypes)
6885 return SDValue();
6886
6887 MaskedLoadSDNode *MLD = dyn_cast<MaskedLoadSDNode>(N);
6888 SDValue Mask = MLD->getMask();
6889 SDLoc DL(N);
6890
6891 // If the MLOAD result requires splitting and the mask is provided by a
6892 // SETCC, then split both nodes and its operands before legalization. This
6893 // prevents the type legalizer from unrolling SETCC into scalar comparisons
6894 // and enables future optimizations (e.g. min/max pattern matching on X86).
6895 if (Mask.getOpcode() == ISD::SETCC) {
6896 EVT VT = N->getValueType(0);
6897
6898 // Check if any splitting is required.
6899 if (TLI.getTypeAction(*DAG.getContext(), VT) !=
6900 TargetLowering::TypeSplitVector)
6901 return SDValue();
6902
6903 SDValue MaskLo, MaskHi, Lo, Hi;
6904 std::tie(MaskLo, MaskHi) = SplitVSETCC(Mask.getNode(), DAG);
6905
6906 SDValue Src0 = MLD->getSrc0();
6907 SDValue Src0Lo, Src0Hi;
6908 std::tie(Src0Lo, Src0Hi) = DAG.SplitVector(Src0, DL);
6909
6910 EVT LoVT, HiVT;
6911 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0));
6912
6913 SDValue Chain = MLD->getChain();
6914 SDValue Ptr = MLD->getBasePtr();
6915 EVT MemoryVT = MLD->getMemoryVT();
6916 unsigned Alignment = MLD->getOriginalAlignment();
6917
6918 // if Alignment is equal to the vector size,
6919 // take the half of it for the second part
6920 unsigned SecondHalfAlignment =
6921 (Alignment == MLD->getValueType(0).getSizeInBits()/8) ?
6922 Alignment/2 : Alignment;
6923
6924 EVT LoMemVT, HiMemVT;
6925 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
6926
6927 MachineMemOperand *MMO = DAG.getMachineFunction().
6928 getMachineMemOperand(MLD->getPointerInfo(),
6929 MachineMemOperand::MOLoad, LoMemVT.getStoreSize(),
6930 Alignment, MLD->getAAInfo(), MLD->getRanges());
6931
6932 Lo = DAG.getMaskedLoad(LoVT, DL, Chain, Ptr, MaskLo, Src0Lo, LoMemVT, MMO,
6933 ISD::NON_EXTLOAD, MLD->isExpandingLoad());
6934
6935 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
6936 MLD->isExpandingLoad());
6937
6938 MMO = DAG.getMachineFunction().
6939 getMachineMemOperand(MLD->getPointerInfo(),
6940 MachineMemOperand::MOLoad, HiMemVT.getStoreSize(),
6941 SecondHalfAlignment, MLD->getAAInfo(), MLD->getRanges());
6942
6943 Hi = DAG.getMaskedLoad(HiVT, DL, Chain, Ptr, MaskHi, Src0Hi, HiMemVT, MMO,
6944 ISD::NON_EXTLOAD, MLD->isExpandingLoad());
6945
6946 AddToWorklist(Lo.getNode());
6947 AddToWorklist(Hi.getNode());
6948
6949 // Build a factor node to remember that this load is independent of the
6950 // other one.
6951 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo.getValue(1),
6952 Hi.getValue(1));
6953
6954 // Legalized the chain result - switch anything that used the old chain to
6955 // use the new one.
6956 DAG.ReplaceAllUsesOfValueWith(SDValue(MLD, 1), Chain);
6957
6958 SDValue LoadRes = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
6959
6960 SDValue RetOps[] = { LoadRes, Chain };
6961 return DAG.getMergeValues(RetOps, DL);
6962 }
6963 return SDValue();
6964}
6965
6966/// A vector select of 2 constant vectors can be simplified to math/logic to
6967/// avoid a variable select instruction and possibly avoid constant loads.
6968SDValue DAGCombiner::foldVSelectOfConstants(SDNode *N) {
6969 SDValue Cond = N->getOperand(0);
6970 SDValue N1 = N->getOperand(1);
6971 SDValue N2 = N->getOperand(2);
6972 EVT VT = N->getValueType(0);
6973 if (!Cond.hasOneUse() || Cond.getScalarValueSizeInBits() != 1 ||
6974 !TLI.convertSelectOfConstantsToMath(VT) ||
6975 !ISD::isBuildVectorOfConstantSDNodes(N1.getNode()) ||
6976 !ISD::isBuildVectorOfConstantSDNodes(N2.getNode()))
6977 return SDValue();
6978
6979 // Check if we can use the condition value to increment/decrement a single
6980 // constant value. This simplifies a select to an add and removes a constant
6981 // load/materialization from the general case.
6982 bool AllAddOne = true;
6983 bool AllSubOne = true;
6984 unsigned Elts = VT.getVectorNumElements();
6985 for (unsigned i = 0; i != Elts; ++i) {
6986 SDValue N1Elt = N1.getOperand(i);
6987 SDValue N2Elt = N2.getOperand(i);
6988 if (N1Elt.isUndef() || N2Elt.isUndef())
6989 continue;
6990
6991 const APInt &C1 = cast<ConstantSDNode>(N1Elt)->getAPIntValue();
6992 const APInt &C2 = cast<ConstantSDNode>(N2Elt)->getAPIntValue();
6993 if (C1 != C2 + 1)
6994 AllAddOne = false;
6995 if (C1 != C2 - 1)
6996 AllSubOne = false;
6997 }
6998
6999 // Further simplifications for the extra-special cases where the constants are
7000 // all 0 or all -1 should be implemented as folds of these patterns.
7001 SDLoc DL(N);
7002 if (AllAddOne || AllSubOne) {
7003 // vselect <N x i1> Cond, C+1, C --> add (zext Cond), C
7004 // vselect <N x i1> Cond, C-1, C --> add (sext Cond), C
7005 auto ExtendOpcode = AllAddOne ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND;
7006 SDValue ExtendedCond = DAG.getNode(ExtendOpcode, DL, VT, Cond);
7007 return DAG.getNode(ISD::ADD, DL, VT, ExtendedCond, N2);
7008 }
7009
7010 // The general case for select-of-constants:
7011 // vselect <N x i1> Cond, C1, C2 --> xor (and (sext Cond), (C1^C2)), C2
7012 // ...but that only makes sense if a vselect is slower than 2 logic ops, so
7013 // leave that to a machine-specific pass.
7014 return SDValue();
7015}
7016
7017SDValue DAGCombiner::visitVSELECT(SDNode *N) {
7018 SDValue N0 = N->getOperand(0);
7019 SDValue N1 = N->getOperand(1);
7020 SDValue N2 = N->getOperand(2);
7021 SDLoc DL(N);
7022
7023 // fold (vselect C, X, X) -> X
7024 if (N1 == N2)
7025 return N1;
7026
7027 // Canonicalize integer abs.
7028 // vselect (setg[te] X, 0), X, -X ->
7029 // vselect (setgt X, -1), X, -X ->
7030 // vselect (setl[te] X, 0), -X, X ->
7031 // Y = sra (X, size(X)-1); xor (add (X, Y), Y)
7032 if (N0.getOpcode() == ISD::SETCC) {
7033 SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1);
7034 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
7035 bool isAbs = false;
7036 bool RHSIsAllZeros = ISD::isBuildVectorAllZeros(RHS.getNode());
7037
7038 if (((RHSIsAllZeros && (CC == ISD::SETGT || CC == ISD::SETGE)) ||
7039 (ISD::isBuildVectorAllOnes(RHS.getNode()) && CC == ISD::SETGT)) &&
7040 N1 == LHS && N2.getOpcode() == ISD::SUB && N1 == N2.getOperand(1))
7041 isAbs = ISD::isBuildVectorAllZeros(N2.getOperand(0).getNode());
7042 else if ((RHSIsAllZeros && (CC == ISD::SETLT || CC == ISD::SETLE)) &&
7043 N2 == LHS && N1.getOpcode() == ISD::SUB && N2 == N1.getOperand(1))
7044 isAbs = ISD::isBuildVectorAllZeros(N1.getOperand(0).getNode());
7045
7046 if (isAbs) {
7047 EVT VT = LHS.getValueType();
7048 if (TLI.isOperationLegalOrCustom(ISD::ABS, VT))
7049 return DAG.getNode(ISD::ABS, DL, VT, LHS);
7050
7051 SDValue Shift = DAG.getNode(
7052 ISD::SRA, DL, VT, LHS,
7053 DAG.getConstant(VT.getScalarSizeInBits() - 1, DL, VT));
7054 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, LHS, Shift);
7055 AddToWorklist(Shift.getNode());
7056 AddToWorklist(Add.getNode());
7057 return DAG.getNode(ISD::XOR, DL, VT, Add, Shift);
7058 }
7059 }
7060
7061 if (SimplifySelectOps(N, N1, N2))
7062 return SDValue(N, 0); // Don't revisit N.
7063
7064 // Fold (vselect (build_vector all_ones), N1, N2) -> N1
7065 if (ISD::isBuildVectorAllOnes(N0.getNode()))
7066 return N1;
7067 // Fold (vselect (build_vector all_zeros), N1, N2) -> N2
7068 if (ISD::isBuildVectorAllZeros(N0.getNode()))
7069 return N2;
7070
7071 // The ConvertSelectToConcatVector function is assuming both the above
7072 // checks for (vselect (build_vector all{ones,zeros) ...) have been made
7073 // and addressed.
7074 if (N1.getOpcode() == ISD::CONCAT_VECTORS &&
7075 N2.getOpcode() == ISD::CONCAT_VECTORS &&
7076 ISD::isBuildVectorOfConstantSDNodes(N0.getNode())) {
7077 if (SDValue CV = ConvertSelectToConcatVector(N, DAG))
7078 return CV;
7079 }
7080
7081 if (SDValue V = foldVSelectOfConstants(N))
7082 return V;
7083
7084 return SDValue();
7085}
7086
7087SDValue DAGCombiner::visitSELECT_CC(SDNode *N) {
7088 SDValue N0 = N->getOperand(0);
7089 SDValue N1 = N->getOperand(1);
7090 SDValue N2 = N->getOperand(2);
7091 SDValue N3 = N->getOperand(3);
7092 SDValue N4 = N->getOperand(4);
7093 ISD::CondCode CC = cast<CondCodeSDNode>(N4)->get();
7094
7095 // fold select_cc lhs, rhs, x, x, cc -> x
7096 if (N2 == N3)
7097 return N2;
7098
7099 // Determine if the condition we're dealing with is constant
7100 if (SDValue SCC = SimplifySetCC(getSetCCResultType(N0.getValueType()), N0, N1,
7101 CC, SDLoc(N), false)) {
7102 AddToWorklist(SCC.getNode());
7103
7104 if (ConstantSDNode *SCCC = dyn_cast<ConstantSDNode>(SCC.getNode())) {
7105 if (!SCCC->isNullValue())
7106 return N2; // cond always true -> true val
7107 else
7108 return N3; // cond always false -> false val
7109 } else if (SCC->isUndef()) {
7110 // When the condition is UNDEF, just return the first operand. This is
7111 // coherent the DAG creation, no setcc node is created in this case
7112 return N2;
7113 } else if (SCC.getOpcode() == ISD::SETCC) {
7114 // Fold to a simpler select_cc
7115 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N2.getValueType(),
7116 SCC.getOperand(0), SCC.getOperand(1), N2, N3,
7117 SCC.getOperand(2));
7118 }
7119 }
7120
7121 // If we can fold this based on the true/false value, do so.
7122 if (SimplifySelectOps(N, N2, N3))
7123 return SDValue(N, 0); // Don't revisit N.
7124
7125 // fold select_cc into other things, such as min/max/abs
7126 return SimplifySelectCC(SDLoc(N), N0, N1, N2, N3, CC);
7127}
7128
7129SDValue DAGCombiner::visitSETCC(SDNode *N) {
7130 return SimplifySetCC(N->getValueType(0), N->getOperand(0), N->getOperand(1),
7131 cast<CondCodeSDNode>(N->getOperand(2))->get(),
7132 SDLoc(N));
7133}
7134
7135SDValue DAGCombiner::visitSETCCE(SDNode *N) {
7136 SDValue LHS = N->getOperand(0);
7137 SDValue RHS = N->getOperand(1);
7138 SDValue Carry = N->getOperand(2);
7139 SDValue Cond = N->getOperand(3);
7140
7141 // If Carry is false, fold to a regular SETCC.
7142 if (Carry.getOpcode() == ISD::CARRY_FALSE)
7143 return DAG.getNode(ISD::SETCC, SDLoc(N), N->getVTList(), LHS, RHS, Cond);
7144
7145 return SDValue();
7146}
7147
7148SDValue DAGCombiner::visitSETCCCARRY(SDNode *N) {
7149 SDValue LHS = N->getOperand(0);
7150 SDValue RHS = N->getOperand(1);
7151 SDValue Carry = N->getOperand(2);
7152 SDValue Cond = N->getOperand(3);
7153
7154 // If Carry is false, fold to a regular SETCC.
7155 if (isNullConstant(Carry))
7156 return DAG.getNode(ISD::SETCC, SDLoc(N), N->getVTList(), LHS, RHS, Cond);
7157
7158 return SDValue();
7159}
7160
7161/// Try to fold a sext/zext/aext dag node into a ConstantSDNode or
7162/// a build_vector of constants.
7163/// This function is called by the DAGCombiner when visiting sext/zext/aext
7164/// dag nodes (see for example method DAGCombiner::visitSIGN_EXTEND).
7165/// Vector extends are not folded if operations are legal; this is to
7166/// avoid introducing illegal build_vector dag nodes.
7167static SDNode *tryToFoldExtendOfConstant(SDNode *N, const TargetLowering &TLI,
7168 SelectionDAG &DAG, bool LegalTypes,
7169 bool LegalOperations) {
7170 unsigned Opcode = N->getOpcode();
7171 SDValue N0 = N->getOperand(0);
7172 EVT VT = N->getValueType(0);
7173
7174 assert((Opcode == ISD::SIGN_EXTEND || Opcode == ISD::ZERO_EXTEND ||(static_cast <bool> ((Opcode == ISD::SIGN_EXTEND || Opcode
== ISD::ZERO_EXTEND || Opcode == ISD::ANY_EXTEND || Opcode ==
ISD::SIGN_EXTEND_VECTOR_INREG || Opcode == ISD::ZERO_EXTEND_VECTOR_INREG
) && "Expected EXTEND dag node in input!") ? void (0)
: __assert_fail ("(Opcode == ISD::SIGN_EXTEND || Opcode == ISD::ZERO_EXTEND || Opcode == ISD::ANY_EXTEND || Opcode == ISD::SIGN_EXTEND_VECTOR_INREG || Opcode == ISD::ZERO_EXTEND_VECTOR_INREG) && \"Expected EXTEND dag node in input!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 7177, __extension__ __PRETTY_FUNCTION__))
7175 Opcode == ISD::ANY_EXTEND || Opcode == ISD::SIGN_EXTEND_VECTOR_INREG ||(static_cast <bool> ((Opcode == ISD::SIGN_EXTEND || Opcode
== ISD::ZERO_EXTEND || Opcode == ISD::ANY_EXTEND || Opcode ==
ISD::SIGN_EXTEND_VECTOR_INREG || Opcode == ISD::ZERO_EXTEND_VECTOR_INREG
) && "Expected EXTEND dag node in input!") ? void (0)
: __assert_fail ("(Opcode == ISD::SIGN_EXTEND || Opcode == ISD::ZERO_EXTEND || Opcode == ISD::ANY_EXTEND || Opcode == ISD::SIGN_EXTEND_VECTOR_INREG || Opcode == ISD::ZERO_EXTEND_VECTOR_INREG) && \"Expected EXTEND dag node in input!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 7177, __extension__ __PRETTY_FUNCTION__))
7176 Opcode == ISD::ZERO_EXTEND_VECTOR_INREG)(static_cast <bool> ((Opcode == ISD::SIGN_EXTEND || Opcode
== ISD::ZERO_EXTEND || Opcode == ISD::ANY_EXTEND || Opcode ==
ISD::SIGN_EXTEND_VECTOR_INREG || Opcode == ISD::ZERO_EXTEND_VECTOR_INREG
) && "Expected EXTEND dag node in input!") ? void (0)
: __assert_fail ("(Opcode == ISD::SIGN_EXTEND || Opcode == ISD::ZERO_EXTEND || Opcode == ISD::ANY_EXTEND || Opcode == ISD::SIGN_EXTEND_VECTOR_INREG || Opcode == ISD::ZERO_EXTEND_VECTOR_INREG) && \"Expected EXTEND dag node in input!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 7177, __extension__ __PRETTY_FUNCTION__))
7177 && "Expected EXTEND dag node in input!")(static_cast <bool> ((Opcode == ISD::SIGN_EXTEND || Opcode
== ISD::ZERO_EXTEND || Opcode == ISD::ANY_EXTEND || Opcode ==
ISD::SIGN_EXTEND_VECTOR_INREG || Opcode == ISD::ZERO_EXTEND_VECTOR_INREG
) && "Expected EXTEND dag node in input!") ? void (0)
: __assert_fail ("(Opcode == ISD::SIGN_EXTEND || Opcode == ISD::ZERO_EXTEND || Opcode == ISD::ANY_EXTEND || Opcode == ISD::SIGN_EXTEND_VECTOR_INREG || Opcode == ISD::ZERO_EXTEND_VECTOR_INREG) && \"Expected EXTEND dag node in input!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 7177, __extension__ __PRETTY_FUNCTION__))
;
7178
7179 // fold (sext c1) -> c1
7180 // fold (zext c1) -> c1
7181 // fold (aext c1) -> c1
7182 if (isa<ConstantSDNode>(N0))
7183 return DAG.getNode(Opcode, SDLoc(N), VT, N0).getNode();
7184
7185 // fold (sext (build_vector AllConstants) -> (build_vector AllConstants)
7186 // fold (zext (build_vector AllConstants) -> (build_vector AllConstants)
7187 // fold (aext (build_vector AllConstants) -> (build_vector AllConstants)
7188 EVT SVT = VT.getScalarType();
7189 if (!(VT.isVector() &&
7190 (!LegalTypes || (!LegalOperations && TLI.isTypeLegal(SVT))) &&
7191 ISD::isBuildVectorOfConstantSDNodes(N0.getNode())))
7192 return nullptr;
7193
7194 // We can fold this node into a build_vector.
7195 unsigned VTBits = SVT.getSizeInBits();
7196 unsigned EVTBits = N0->getValueType(0).getScalarSizeInBits();
7197 SmallVector<SDValue, 8> Elts;
7198 unsigned NumElts = VT.getVectorNumElements();
7199 SDLoc DL(N);
7200
7201 for (unsigned i=0; i != NumElts; ++i) {
7202 SDValue Op = N0->getOperand(i);
7203 if (Op->isUndef()) {
7204 Elts.push_back(DAG.getUNDEF(SVT));
7205 continue;
7206 }
7207
7208 SDLoc DL(Op);
7209 // Get the constant value and if needed trunc it to the size of the type.
7210 // Nodes like build_vector might have constants wider than the scalar type.
7211 APInt C = cast<ConstantSDNode>(Op)->getAPIntValue().zextOrTrunc(EVTBits);
7212 if (Opcode == ISD::SIGN_EXTEND || Opcode == ISD::SIGN_EXTEND_VECTOR_INREG)
7213 Elts.push_back(DAG.getConstant(C.sext(VTBits), DL, SVT));
7214 else
7215 Elts.push_back(DAG.getConstant(C.zext(VTBits), DL, SVT));
7216 }
7217
7218 return DAG.getBuildVector(VT, DL, Elts).getNode();
7219}
7220
7221// ExtendUsesToFormExtLoad - Trying to extend uses of a load to enable this:
7222// "fold ({s|z|a}ext (load x)) -> ({s|z|a}ext (truncate ({s|z|a}extload x)))"
7223// transformation. Returns true if extension are possible and the above
7224// mentioned transformation is profitable.
7225static bool ExtendUsesToFormExtLoad(SDNode *N, SDValue N0,
7226 unsigned ExtOpc,
7227 SmallVectorImpl<SDNode *> &ExtendNodes,
7228 const TargetLowering &TLI) {
7229 bool HasCopyToRegUses = false;
7230 bool isTruncFree = TLI.isTruncateFree(N->getValueType(0), N0.getValueType());
7231 for (SDNode::use_iterator UI = N0.getNode()->use_begin(),
7232 UE = N0.getNode()->use_end();
7233 UI != UE; ++UI) {
7234 SDNode *User = *UI;
7235 if (User == N)
7236 continue;
7237 if (UI.getUse().getResNo() != N0.getResNo())
7238 continue;
7239 // FIXME: Only extend SETCC N, N and SETCC N, c for now.
7240 if (ExtOpc != ISD::ANY_EXTEND && User->getOpcode() == ISD::SETCC) {
7241 ISD::CondCode CC = cast<CondCodeSDNode>(User->getOperand(2))->get();
7242 if (ExtOpc == ISD::ZERO_EXTEND && ISD::isSignedIntSetCC(CC))
7243 // Sign bits will be lost after a zext.
7244 return false;
7245 bool Add = false;
7246 for (unsigned i = 0; i != 2; ++i) {
7247 SDValue UseOp = User->getOperand(i);
7248 if (UseOp == N0)
7249 continue;
7250 if (!isa<ConstantSDNode>(UseOp))
7251 return false;
7252 Add = true;
7253 }
7254 if (Add)
7255 ExtendNodes.push_back(User);
7256 continue;
7257 }
7258 // If truncates aren't free and there are users we can't
7259 // extend, it isn't worthwhile.
7260 if (!isTruncFree)
7261 return false;
7262 // Remember if this value is live-out.
7263 if (User->getOpcode() == ISD::CopyToReg)
7264 HasCopyToRegUses = true;
7265 }
7266
7267 if (HasCopyToRegUses) {
7268 bool BothLiveOut = false;
7269 for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
7270 UI != UE; ++UI) {
7271 SDUse &Use = UI.getUse();
7272 if (Use.getResNo() == 0 && Use.getUser()->getOpcode() == ISD::CopyToReg) {
7273 BothLiveOut = true;
7274 break;
7275 }
7276 }
7277 if (BothLiveOut)
7278 // Both unextended and extended values are live out. There had better be
7279 // a good reason for the transformation.
7280 return ExtendNodes.size();
7281 }
7282 return true;
7283}
7284
7285void DAGCombiner::ExtendSetCCUses(const SmallVectorImpl<SDNode *> &SetCCs,
7286 SDValue Trunc, SDValue ExtLoad,
7287 const SDLoc &DL, ISD::NodeType ExtType) {
7288 // Extend SetCC uses if necessary.
7289 for (unsigned i = 0, e = SetCCs.size(); i != e; ++i) {
7290 SDNode *SetCC = SetCCs[i];
7291 SmallVector<SDValue, 4> Ops;
7292
7293 for (unsigned j = 0; j != 2; ++j) {
7294 SDValue SOp = SetCC->getOperand(j);
7295 if (SOp == Trunc)
7296 Ops.push_back(ExtLoad);
7297 else
7298 Ops.push_back(DAG.getNode(ExtType, DL, ExtLoad->getValueType(0), SOp));
7299 }
7300
7301 Ops.push_back(SetCC->getOperand(2));
7302 CombineTo(SetCC, DAG.getNode(ISD::SETCC, DL, SetCC->getValueType(0), Ops));
7303 }
7304}
7305
7306// FIXME: Bring more similar combines here, common to sext/zext (maybe aext?).
7307SDValue DAGCombiner::CombineExtLoad(SDNode *N) {
7308 SDValue N0 = N->getOperand(0);
7309 EVT DstVT = N->getValueType(0);
7310 EVT SrcVT = N0.getValueType();
7311
7312 assert((N->getOpcode() == ISD::SIGN_EXTEND ||(static_cast <bool> ((N->getOpcode() == ISD::SIGN_EXTEND
|| N->getOpcode() == ISD::ZERO_EXTEND) && "Unexpected node type (not an extend)!"
) ? void (0) : __assert_fail ("(N->getOpcode() == ISD::SIGN_EXTEND || N->getOpcode() == ISD::ZERO_EXTEND) && \"Unexpected node type (not an extend)!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 7314, __extension__ __PRETTY_FUNCTION__))
7313 N->getOpcode() == ISD::ZERO_EXTEND) &&(static_cast <bool> ((N->getOpcode() == ISD::SIGN_EXTEND
|| N->getOpcode() == ISD::ZERO_EXTEND) && "Unexpected node type (not an extend)!"
) ? void (0) : __assert_fail ("(N->getOpcode() == ISD::SIGN_EXTEND || N->getOpcode() == ISD::ZERO_EXTEND) && \"Unexpected node type (not an extend)!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 7314, __extension__ __PRETTY_FUNCTION__))
7314 "Unexpected node type (not an extend)!")(static_cast <bool> ((N->getOpcode() == ISD::SIGN_EXTEND
|| N->getOpcode() == ISD::ZERO_EXTEND) && "Unexpected node type (not an extend)!"
) ? void (0) : __assert_fail ("(N->getOpcode() == ISD::SIGN_EXTEND || N->getOpcode() == ISD::ZERO_EXTEND) && \"Unexpected node type (not an extend)!\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 7314, __extension__ __PRETTY_FUNCTION__))
;
7315
7316 // fold (sext (load x)) to multiple smaller sextloads; same for zext.
7317 // For example, on a target with legal v4i32, but illegal v8i32, turn:
7318 // (v8i32 (sext (v8i16 (load x))))
7319 // into:
7320 // (v8i32 (concat_vectors (v4i32 (sextload x)),
7321 // (v4i32 (sextload (x + 16)))))
7322 // Where uses of the original load, i.e.:
7323 // (v8i16 (load x))
7324 // are replaced with:
7325 // (v8i16 (truncate
7326 // (v8i32 (concat_vectors (v4i32 (sextload x)),
7327 // (v4i32 (sextload (x + 16)))))))
7328 //
7329 // This combine is only applicable to illegal, but splittable, vectors.
7330 // All legal types, and illegal non-vector types, are handled elsewhere.
7331 // This combine is controlled by TargetLowering::isVectorLoadExtDesirable.
7332 //
7333 if (N0->getOpcode() != ISD::LOAD)
7334 return SDValue();
7335
7336 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
7337
7338 if (!ISD::isNON_EXTLoad(LN0) || !ISD::isUNINDEXEDLoad(LN0) ||
7339 !N0.hasOneUse() || LN0->isVolatile() || !DstVT.isVector() ||
7340 !DstVT.isPow2VectorType() || !TLI.isVectorLoadExtDesirable(SDValue(N, 0)))
7341 return SDValue();
7342
7343 SmallVector<SDNode *, 4> SetCCs;
7344 if (!ExtendUsesToFormExtLoad(N, N0, N->getOpcode(), SetCCs, TLI))
7345 return SDValue();
7346
7347 ISD::LoadExtType ExtType =
7348 N->getOpcode() == ISD::SIGN_EXTEND ? ISD::SEXTLOAD : ISD::ZEXTLOAD;
7349
7350 // Try to split the vector types to get down to legal types.
7351 EVT SplitSrcVT = SrcVT;
7352 EVT SplitDstVT = DstVT;
7353 while (!TLI.isLoadExtLegalOrCustom(ExtType, SplitDstVT, SplitSrcVT) &&
7354 SplitSrcVT.getVectorNumElements() > 1) {
7355 SplitDstVT = DAG.GetSplitDestVTs(SplitDstVT).first;
7356 SplitSrcVT = DAG.GetSplitDestVTs(SplitSrcVT).first;
7357 }
7358
7359 if (!TLI.isLoadExtLegalOrCustom(ExtType, SplitDstVT, SplitSrcVT))
7360 return SDValue();
7361
7362 SDLoc DL(N);
7363 const unsigned NumSplits =
7364 DstVT.getVectorNumElements() / SplitDstVT.getVectorNumElements();
7365 const unsigned Stride = SplitSrcVT.getStoreSize();
7366 SmallVector<SDValue, 4> Loads;
7367 SmallVector<SDValue, 4> Chains;
7368
7369 SDValue BasePtr = LN0->getBasePtr();
7370 for (unsigned Idx = 0; Idx < NumSplits; Idx++) {
7371 const unsigned Offset = Idx * Stride;
7372 const unsigned Align = MinAlign(LN0->getAlignment(), Offset);
7373
7374 SDValue SplitLoad = DAG.getExtLoad(
7375 ExtType, DL, SplitDstVT, LN0->getChain(), BasePtr,
7376 LN0->getPointerInfo().getWithOffset(Offset), SplitSrcVT, Align,
7377 LN0->getMemOperand()->getFlags(), LN0->getAAInfo());
7378
7379 BasePtr = DAG.getNode(ISD::ADD, DL, BasePtr.getValueType(), BasePtr,
7380 DAG.getConstant(Stride, DL, BasePtr.getValueType()));
7381
7382 Loads.push_back(SplitLoad.getValue(0));
7383 Chains.push_back(SplitLoad.getValue(1));
7384 }
7385
7386 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
7387 SDValue NewValue = DAG.getNode(ISD::CONCAT_VECTORS, DL, DstVT, Loads);
7388
7389 // Simplify TF.
7390 AddToWorklist(NewChain.getNode());
7391
7392 CombineTo(N, NewValue);
7393
7394 // Replace uses of the original load (before extension)
7395 // with a truncate of the concatenated sextloaded vectors.
7396 SDValue Trunc =
7397 DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), NewValue);
7398 CombineTo(N0.getNode(), Trunc, NewChain);
7399 ExtendSetCCUses(SetCCs, Trunc, NewValue, DL,
7400 (ISD::NodeType)N->getOpcode());
7401 return SDValue(N, 0); // Return N so it doesn't get rechecked!
7402}
7403
7404/// If we're narrowing or widening the result of a vector select and the final
7405/// size is the same size as a setcc (compare) feeding the select, then try to
7406/// apply the cast operation to the select's operands because matching vector
7407/// sizes for a select condition and other operands should be more efficient.
7408SDValue DAGCombiner::matchVSelectOpSizesWithSetCC(SDNode *Cast) {
7409 unsigned CastOpcode = Cast->getOpcode();
7410 assert((CastOpcode == ISD::SIGN_EXTEND || CastOpcode == ISD::ZERO_EXTEND ||(static_cast <bool> ((CastOpcode == ISD::SIGN_EXTEND ||
CastOpcode == ISD::ZERO_EXTEND || CastOpcode == ISD::TRUNCATE
|| CastOpcode == ISD::FP_EXTEND || CastOpcode == ISD::FP_ROUND
) && "Unexpected opcode for vector select narrowing/widening"
) ? void (0) : __assert_fail ("(CastOpcode == ISD::SIGN_EXTEND || CastOpcode == ISD::ZERO_EXTEND || CastOpcode == ISD::TRUNCATE || CastOpcode == ISD::FP_EXTEND || CastOpcode == ISD::FP_ROUND) && \"Unexpected opcode for vector select narrowing/widening\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 7413, __extension__ __PRETTY_FUNCTION__))
7411 CastOpcode == ISD::TRUNCATE || CastOpcode == ISD::FP_EXTEND ||(static_cast <bool> ((CastOpcode == ISD::SIGN_EXTEND ||
CastOpcode == ISD::ZERO_EXTEND || CastOpcode == ISD::TRUNCATE
|| CastOpcode == ISD::FP_EXTEND || CastOpcode == ISD::FP_ROUND
) && "Unexpected opcode for vector select narrowing/widening"
) ? void (0) : __assert_fail ("(CastOpcode == ISD::SIGN_EXTEND || CastOpcode == ISD::ZERO_EXTEND || CastOpcode == ISD::TRUNCATE || CastOpcode == ISD::FP_EXTEND || CastOpcode == ISD::FP_ROUND) && \"Unexpected opcode for vector select narrowing/widening\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 7413, __extension__ __PRETTY_FUNCTION__))
7412 CastOpcode == ISD::FP_ROUND) &&(static_cast <bool> ((CastOpcode == ISD::SIGN_EXTEND ||
CastOpcode == ISD::ZERO_EXTEND || CastOpcode == ISD::TRUNCATE
|| CastOpcode == ISD::FP_EXTEND || CastOpcode == ISD::FP_ROUND
) && "Unexpected opcode for vector select narrowing/widening"
) ? void (0) : __assert_fail ("(CastOpcode == ISD::SIGN_EXTEND || CastOpcode == ISD::ZERO_EXTEND || CastOpcode == ISD::TRUNCATE || CastOpcode == ISD::FP_EXTEND || CastOpcode == ISD::FP_ROUND) && \"Unexpected opcode for vector select narrowing/widening\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 7413, __extension__ __PRETTY_FUNCTION__))
7413 "Unexpected opcode for vector select narrowing/widening")(static_cast <bool> ((CastOpcode == ISD::SIGN_EXTEND ||
CastOpcode == ISD::ZERO_EXTEND || CastOpcode == ISD::TRUNCATE
|| CastOpcode == ISD::FP_EXTEND || CastOpcode == ISD::FP_ROUND
) && "Unexpected opcode for vector select narrowing/widening"
) ? void (0) : __assert_fail ("(CastOpcode == ISD::SIGN_EXTEND || CastOpcode == ISD::ZERO_EXTEND || CastOpcode == ISD::TRUNCATE || CastOpcode == ISD::FP_EXTEND || CastOpcode == ISD::FP_ROUND) && \"Unexpected opcode for vector select narrowing/widening\""
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 7413, __extension__ __PRETTY_FUNCTION__))
;
7414
7415 // We only do this transform before legal ops because the pattern may be
7416 // obfuscated by target-specific operations after legalization. Do not create
7417 // an illegal select op, however, because that may be difficult to lower.
7418 EVT VT = Cast->getValueType(0);
7419 if (LegalOperations || !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT))
7420 return SDValue();
7421
7422 SDValue VSel = Cast->getOperand(0);
7423 if (VSel.getOpcode() != ISD::VSELECT || !VSel.hasOneUse() ||
7424 VSel.getOperand(0).getOpcode() != ISD::SETCC)
7425 return SDValue();
7426
7427 // Does the setcc have the same vector size as the casted select?
7428 SDValue SetCC = VSel.getOperand(0);
7429 EVT SetCCVT = getSetCCResultType(SetCC.getOperand(0).getValueType());
7430 if (SetCCVT.getSizeInBits() != VT.getSizeInBits())
7431 return SDValue();
7432
7433 // cast (vsel (setcc X), A, B) --> vsel (setcc X), (cast A), (cast B)
7434 SDValue A = VSel.getOperand(1);
7435 SDValue B = VSel.getOperand(2);
7436 SDValue CastA, CastB;
7437 SDLoc DL(Cast);
7438 if (CastOpcode == ISD::FP_ROUND) {
7439 // FP_ROUND (fptrunc) has an extra flag operand to pass along.
7440 CastA = DAG.getNode(CastOpcode, DL, VT, A, Cast->getOperand(1));
7441 CastB = DAG.getNode(CastOpcode, DL, VT, B, Cast->getOperand(1));
7442 } else {
7443 CastA = DAG.getNode(CastOpcode, DL, VT, A);
7444 CastB = DAG.getNode(CastOpcode, DL, VT, B);
7445 }
7446 return DAG.getNode(ISD::VSELECT, DL, VT, SetCC, CastA, CastB);
7447}
7448
7449SDValue DAGCombiner::visitSIGN_EXTEND(SDNode *N) {
7450 SDValue N0 = N->getOperand(0);
7451 EVT VT = N->getValueType(0);
7452 SDLoc DL(N);
7453
7454 if (SDNode *Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes,
7455 LegalOperations))
7456 return SDValue(Res, 0);
7457
7458 // fold (sext (sext x)) -> (sext x)
7459 // fold (sext (aext x)) -> (sext x)
7460 if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND)
7461 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, N0.getOperand(0));
7462
7463 if (N0.getOpcode() == ISD::TRUNCATE) {
7464 // fold (sext (truncate (load x))) -> (sext (smaller load x))
7465 // fold (sext (truncate (srl (load x), c))) -> (sext (smaller load (x+c/n)))
7466 if (SDValue NarrowLoad = ReduceLoadWidth(N0.getNode())) {
7467 SDNode *oye = N0.getOperand(0).getNode();
7468 if (NarrowLoad.getNode() != N0.getNode()) {
7469 CombineTo(N0.getNode(), NarrowLoad);
7470 // CombineTo deleted the truncate, if needed, but not what's under it.
7471 AddToWorklist(oye);
7472 }
7473 return SDValue(N, 0); // Return N so it doesn't get rechecked!
7474 }
7475
7476 // See if the value being truncated is already sign extended. If so, just
7477 // eliminate the trunc/sext pair.
7478 SDValue Op = N0.getOperand(0);
7479 unsigned OpBits = Op.getScalarValueSizeInBits();
7480 unsigned MidBits = N0.getScalarValueSizeInBits();
7481 unsigned DestBits = VT.getScalarSizeInBits();
7482 unsigned NumSignBits = DAG.ComputeNumSignBits(Op);
7483
7484 if (OpBits == DestBits) {
7485 // Op is i32, Mid is i8, and Dest is i32. If Op has more than 24 sign
7486 // bits, it is already ready.
7487 if (NumSignBits > DestBits-MidBits)
7488 return Op;
7489 } else if (OpBits < DestBits) {
7490 // Op is i32, Mid is i8, and Dest is i64. If Op has more than 24 sign
7491 // bits, just sext from i32.
7492 if (NumSignBits > OpBits-MidBits)
7493 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op);
7494 } else {
7495 // Op is i64, Mid is i8, and Dest is i32. If Op has more than 56 sign
7496 // bits, just truncate to i32.
7497 if (NumSignBits > OpBits-MidBits)
7498 return DAG.getNode(ISD::TRUNCATE, DL, VT, Op);
7499 }
7500
7501 // fold (sext (truncate x)) -> (sextinreg x).
7502 if (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG,
7503 N0.getValueType())) {
7504 if (OpBits < DestBits)
7505 Op = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N0), VT, Op);
7506 else if (OpBits > DestBits)
7507 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N0), VT, Op);
7508 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Op,
7509 DAG.getValueType(N0.getValueType()));
7510 }
7511 }
7512
7513 // fold (sext (load x)) -> (sext (truncate (sextload x)))
7514 // Only generate vector extloads when 1) they're legal, and 2) they are
7515 // deemed desirable by the target.
7516 if (ISD::isNON_EXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) &&
7517 ((!LegalOperations && !VT.isVector() &&
7518 !cast<LoadSDNode>(N0)->isVolatile()) ||
7519 TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, N0.getValueType()))) {
7520 bool DoXform = true;
7521 SmallVector<SDNode*, 4> SetCCs;
7522 if (!N0.hasOneUse())
7523 DoXform = ExtendUsesToFormExtLoad(N, N0, ISD::SIGN_EXTEND, SetCCs, TLI);
7524 if (VT.isVector())
7525 DoXform &= TLI.isVectorLoadExtDesirable(SDValue(N, 0));
7526 if (DoXform) {
7527 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
7528 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, DL, VT, LN0->getChain(),
7529 LN0->getBasePtr(), N0.getValueType(),
7530 LN0->getMemOperand());
7531 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0),
7532 N0.getValueType(), ExtLoad);
7533 ExtendSetCCUses(SetCCs, Trunc, ExtLoad, DL, ISD::SIGN_EXTEND);
7534 // If the load value is used only by N, replace it via CombineTo N.
7535 bool NoReplaceTrunc = SDValue(LN0, 0).hasOneUse();
7536 CombineTo(N, ExtLoad);
7537 if (NoReplaceTrunc)
7538 DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1));
7539 else
7540 CombineTo(LN0, Trunc, ExtLoad.getValue(1));
7541 return SDValue(N, 0);
7542 }
7543 }
7544
7545 // fold (sext (load x)) to multiple smaller sextloads.
7546 // Only on illegal but splittable vectors.
7547 if (SDValue ExtLoad = CombineExtLoad(N))
7548 return ExtLoad;
7549
7550 // fold (sext (sextload x)) -> (sext (truncate (sextload x)))
7551 // fold (sext ( extload x)) -> (sext (truncate (sextload x)))
7552 if ((ISD::isSEXTLoad(N0.getNode()) || ISD::isEXTLoad(N0.getNode())) &&
7553 ISD::isUNINDEXEDLoad(N0.getNode()) && N0.hasOneUse()) {
7554 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
7555 EVT MemVT = LN0->getMemoryVT();
7556 if ((!LegalOperations && !LN0->isVolatile()) ||
7557 TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, MemVT)) {
7558 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, DL, VT, LN0->getChain(),
7559 LN0->getBasePtr(), MemVT,
7560 LN0->getMemOperand());
7561 CombineTo(N, ExtLoad);
7562 CombineTo(N0.getNode(),
7563 DAG.getNode(ISD::TRUNCATE, SDLoc(N0),
7564 N0.getValueType(), ExtLoad),
7565 ExtLoad.getValue(1));
7566 return SDValue(N, 0); // Return N so it doesn't get rechecked!
7567 }
7568 }
7569
7570 // fold (sext (and/or/xor (load x), cst)) ->
7571 // (and/or/xor (sextload x), (sext cst))
7572 if ((N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR ||
7573 N0.getOpcode() == ISD::XOR) &&
7574 isa<LoadSDNode>(N0.getOperand(0)) &&
7575 N0.getOperand(1).getOpcode() == ISD::Constant &&
7576 TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, N0.getValueType()) &&
7577 (!LegalOperations && TLI.isOperationLegal(N0.getOpcode(), VT))) {
7578 LoadSDNode *LN0 = cast<LoadSDNode>(N0.getOperand(0));
7579 if (LN0->getExtensionType() != ISD::ZEXTLOAD && LN0->isUnindexed()) {
7580 bool DoXform = true;
7581 SmallVector<SDNode*, 4> SetCCs;
7582 if (!N0.hasOneUse())
7583 DoXform = ExtendUsesToFormExtLoad(N, N0.getOperand(0), ISD::SIGN_EXTEND,
7584 SetCCs, TLI);
7585 if (DoXform) {
7586 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(LN0), VT,
7587 LN0->getChain(), LN0->getBasePtr(),
7588 LN0->getMemoryVT(),
7589 LN0->getMemOperand());
7590 APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
7591 Mask = Mask.sext(VT.getSizeInBits());
7592 SDValue And = DAG.getNode(N0.getOpcode(), DL, VT,
7593 ExtLoad, DAG.getConstant(Mask, DL, VT));
7594 SDValue Trunc = DAG.getNode(ISD::TRUNCATE,
7595 SDLoc(N0.getOperand(0)),
7596 N0.getOperand(0).getValueType(), ExtLoad);
7597 ExtendSetCCUses(SetCCs, Trunc, ExtLoad, DL, ISD::SIGN_EXTEND);
7598 bool NoReplaceTruncAnd = !N0.hasOneUse();
7599 bool NoReplaceTrunc = SDValue(LN0, 0).hasOneUse();
7600 CombineTo(N, And);
7601 // If N0 has multiple uses, change other uses as well.
7602 if (NoReplaceTruncAnd) {
7603 SDValue TruncAnd =
7604 DAG.getNode(ISD::TRUNCATE, DL, N0.getValueType(), And);
7605 CombineTo(N0.getNode(), TruncAnd);
7606 }
7607 if (NoReplaceTrunc)
7608 DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1));
7609 else
7610 CombineTo(LN0, Trunc, ExtLoad.getValue(1));
7611 return SDValue(N,0); // Return N so it doesn't get rechecked!
7612 }
7613 }
7614 }
7615
7616 if (N0.getOpcode() == ISD::SETCC) {
7617 SDValue N00 = N0.getOperand(0);
7618 SDValue N01 = N0.getOperand(1);
7619 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
7620 EVT N00VT = N0.getOperand(0).getValueType();
7621
7622 // sext(setcc) -> sext_in_reg(vsetcc) for vectors.
7623 // Only do this before legalize for now.
7624 if (VT.isVector() && !LegalOperations &&
7625 TLI.getBooleanContents(N00VT) ==
7626 TargetLowering::ZeroOrNegativeOneBooleanContent) {
7627 // On some architectures (such as SSE/NEON/etc) the SETCC result type is
7628 // of the same size as the compared operands. Only optimize sext(setcc())
7629 // if this is the case.
7630 EVT SVT = getSetCCResultType(N00VT);
7631
7632 // We know that the # elements of the results is the same as the
7633 // # elements of the compare (and the # elements of the compare result
7634 // for that matter). Check to see that they are the same size. If so,
7635 // we know that the element size of the sext'd result matches the
7636 // element size of the compare operands.
7637 if (VT.getSizeInBits() == SVT.getSizeInBits())
7638 return DAG.getSetCC(DL, VT, N00, N01, CC);
7639
7640 // If the desired elements are smaller or larger than the source
7641 // elements, we can use a matching integer vector type and then
7642 // truncate/sign extend.
7643 EVT MatchingVecType = N00VT.changeVectorElementTypeToInteger();
7644 if (SVT == MatchingVecType) {
7645 SDValue VsetCC = DAG.getSetCC(DL, MatchingVecType, N00, N01, CC);
7646 return DAG.getSExtOrTrunc(VsetCC, DL, VT);
7647 }
7648 }
7649
7650 // sext(setcc x, y, cc) -> (select (setcc x, y, cc), T, 0)
7651 // Here, T can be 1 or -1, depending on the type of the setcc and
7652 // getBooleanContents().
7653 unsigned SetCCWidth = N0.getScalarValueSizeInBits();
7654
7655 // To determine the "true" side of the select, we need to know the high bit
7656 // of the value returned by the setcc if it evaluates to true.
7657 // If the type of the setcc is i1, then the true case of the select is just
7658 // sext(i1 1), that is, -1.
7659 // If the type of the setcc is larger (say, i8) then the value of the high
7660 // bit depends on getBooleanContents(), so ask TLI for a real "true" value
7661 // of the appropriate width.
7662 SDValue ExtTrueVal = (SetCCWidth == 1) ? DAG.getAllOnesConstant(DL, VT)
7663 : TLI.getConstTrueVal(DAG, VT, DL);
7664 SDValue Zero = DAG.getConstant(0, DL, VT);
7665 if (SDValue SCC =
7666 SimplifySelectCC(DL, N00, N01, ExtTrueVal, Zero, CC, true))
7667 return SCC;
7668
7669 if (!VT.isVector() && !TLI.convertSelectOfConstantsToMath(VT)) {
7670 EVT SetCCVT = getSetCCResultType(N00VT);
7671 // Don't do this transform for i1 because there's a select transform
7672 // that would reverse it.
7673 // TODO: We should not do this transform at all without a target hook
7674 // because a sext is likely cheaper than a select?
7675 if (SetCCVT.getScalarSizeInBits() != 1 &&
7676 (!LegalOperations || TLI.isOperationLegal(ISD::SETCC, N00VT))) {
7677 SDValue SetCC = DAG.getSetCC(DL, SetCCVT, N00, N01, CC);
7678 return DAG.getSelect(DL, VT, SetCC, ExtTrueVal, Zero);
7679 }
7680 }
7681 }
7682
7683 // fold (sext x) -> (zext x) if the sign bit is known zero.
7684 if ((!LegalOperations || TLI.isOperationLegal(ISD::ZERO_EXTEND, VT)) &&
7685 DAG.SignBitIsZero(N0))
7686 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0);
7687
7688 if (SDValue NewVSel = matchVSelectOpSizesWithSetCC(N))
7689 return NewVSel;
7690
7691 return SDValue();
7692}
7693
7694// isTruncateOf - If N is a truncate of some other value, return true, record
7695// the value being truncated in Op and which of Op's bits are zero/one in Known.
7696// This function computes KnownBits to avoid a duplicated call to
7697// computeKnownBits in the caller.
7698static bool isTruncateOf(SelectionDAG &DAG, SDValue N, SDValue &Op,
7699 KnownBits &Known) {
7700 if (N->getOpcode() == ISD::TRUNCATE) {
7701 Op = N->getOperand(0);
7702 DAG.computeKnownBits(Op, Known);
7703 return true;
7704 }
7705
7706 if (N->getOpcode() != ISD::SETCC || N->getValueType(0) != MVT::i1 ||
7707 cast<CondCodeSDNode>(N->getOperand(2))->get() != ISD::SETNE)
7708 return false;
7709
7710 SDValue Op0 = N->getOperand(0);
7711 SDValue Op1 = N->getOperand(1);
7712 assert(Op0.getValueType() == Op1.getValueType())(static_cast <bool> (Op0.getValueType() == Op1.getValueType
()) ? void (0) : __assert_fail ("Op0.getValueType() == Op1.getValueType()"
, "/build/llvm-toolchain-snapshot-6.0~svn321639/lib/CodeGen/SelectionDAG/DAGCombiner.cpp"
, 7712, __extension__ __PRETTY_FUNCTION__))
;
7713
7714 if (isNullConstant(Op0))
7715 Op = Op1;
7716 else if (isNullConstant(Op1))
7717 Op = Op0;
7718 else
7719 return false;
7720
7721 DAG.computeKnownBits(Op, Known);
7722
7723 if (!(Known.Zero | 1).isAllOnesValue())
7724 return false;
7725
7726 return true;
7727}
7728
7729SDValue DAGCombiner::visitZERO_EXTEND(SDNode *N) {
7730 SDValue N0 = N->getOperand(0);
7731 EVT VT = N->getValueType(0);
7732
7733 if (SDNode *Res = tryToFoldExtendOfConstant(N, TLI, DAG, LegalTypes,
7734 LegalOperations))
7735 return SDValue(Res, 0);
7736
7737 // fold (zext (zext x)) -> (zext x)
7738 // fold (zext (aext x)) -> (zext x)
7739 if (N0.getOpcode() == ISD::ZERO_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND)
7740 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT,
7741 N0.getOperand(0));
7742
7743 // fold (zext (truncate x)) -> (zext x) or
7744 // (zext (truncate x)) -> (truncate x)
7745 // This is valid when the truncated bits of x are already zero.
7746 // FIXME: We should extend this to work for vectors too.
7747 SDValue Op;
7748 KnownBits Known;
7749 if (!VT.isVector() && isTruncateOf(DAG, N0, Op, Known)) {
7750 APInt TruncatedBits =
7751 (Op.getValueSizeInBits() == N0.getValueSizeInBits()) ?
7752 APInt(Op.getValueSizeInBits(), 0) :
7753 APInt::getBitsSet(Op.getValueSizeInBits(),
7754 N0.getValueSizeInBits(),
7755 std::min(Op.getValueSizeInBits(),
7756 VT.getSizeInBits()));
7757 if (TruncatedBits.isSubsetOf(Known.Zero))
7758 return DAG.getZExtOrTrunc(Op, SDLoc(N), VT);
7759 }
7760
7761 // fold (zext (truncate x)) -> (and x, mask)
7762 if (N0.getOpcode() == ISD::TRUNCATE) {
7763 // fold (zext (truncate (load x))) -> (zext (smaller load x))
7764 // fold (zext (truncate (srl (load x), c))) -> (zext (smaller load (x+c/n)))
7765 if (SDValue NarrowLoad = ReduceLoadWidth(N0.getNode())) {
7766 SDNode *oye = N0.getOperand(0).getNode();
7767 if (NarrowLoad.getNode() != N0.getNode()) {
7768 CombineTo(N0.getNode(), NarrowLoad);
7769 // CombineTo deleted the truncate, if needed, but not what's under it.
7770 AddToWorklist(oye);
7771 }
7772 return SDValue(N, 0); // Return N so it doesn't get rechecked!
7773 }
7774
7775 EVT SrcVT = N0.getOperand(0).getValueType();
7776 EVT MinVT = N0.getValueType();
7777
7778 // Try to mask before the extension to avoid having to generate a larger mask,
7779 // possibly over several sub-vectors.
7780 if (SrcVT.bitsLT(VT)) {
7781 if (!LegalOperations || (TLI.isOperationLegal(ISD::AND, SrcVT) &&
7782 TLI.isOperationLegal(ISD::ZERO_EXTEND, VT))) {
7783 SDValue Op = N0.getOperand(0);
7784 Op = DAG.getZeroExtendInReg(Op, SDLoc(N), MinVT.getScalarType());
7785 AddToWorklist(Op.getNode());
7786 return DAG.getZExtOrTrunc(Op, SDLoc(N), VT);
7787 }
7788 }
7789
7790 if (!LegalOperations || TLI.isOperationLegal(ISD::AND, VT)) {
7791 SDValue Op = DAG.getAnyExtOrTrunc(N0.getOperand(0), SDLoc(N), VT);
7792 AddToWorklist(Op.getNode());
7793 SDValue And = DAG.getZeroExtendInReg(Op, SDLoc(N), MinVT.getScalarType());
7794 // We may safely transfer the debug info describing the truncate node over
7795 // to the equivalent and operation.
7796 DAG.transferDbgValues(N0, And);
7797 return And;
7798 }
7799 }
7800
7801 // Fold (zext (and (trunc x), cst)) -> (and x, cst),
7802 // if either of the casts is not free.
7803 if (N0.getOpcode() == ISD::AND &&
7804 N0.getOperand(0).getOpcode() == ISD::TRUNCATE &&
7805 N0.getOperand(1).getOpcode() == ISD::Constant &&
7806 (!TLI.isTruncateFree(N0.getOperand(0).getOperand(0).getValueType(),
7807 N0.getValueType()) ||
7808 !TLI.isZExtFree(N0.getValueType(), VT))) {
7809 SDValue X = N0.getOperand(0).getOperand(0);
7810 X = DAG.getAnyExtOrTrunc(X, SDLoc(X), VT);
7811 APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
7812 Mask = Mask.zext(VT.getSizeInBits());
7813 SDLoc DL(N);
7814 return DAG.getNode(ISD::AND, DL, VT,
7815 X, DAG.getConstant(Mask, DL, VT));
7816 }
7817
7818 // fold (zext (load x)) -> (zext (truncate (zextload x)))
7819 // Only generate vector extloads when 1) they're legal, and 2) they are
7820 // deemed desirable by the target.
7821 if (ISD::isNON_EXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) &&
7822 ((!LegalOperations && !VT.isVector() &&
7823 !cast<LoadSDNode>(N0)->isVolatile()) ||
7824 TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, N0.getValueType()))) {
7825 bool DoXform = true;
7826 SmallVector<SDNode*, 4> SetCCs;
7827 if (!N0.hasOneUse())
7828 DoXform = ExtendUsesToFormExtLoad(N, N0, ISD::ZERO_EXTEND, SetCCs, TLI);
7829 if (VT.isVector())
7830 DoXform &= TLI.isVectorLoadExtDesirable(SDValue(N, 0));
7831 if (DoXform) {
7832 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
7833 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N), VT,
7834 LN0->getChain(),
7835 LN0->getBasePtr(), N0.getValueType(),
7836 LN0->getMemOperand());
7837
7838 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0),
7839 N0.getValueType(), ExtLoad);