LLVM 24.0.0git
DAGCombiner.cpp
Go to the documentation of this file.
1//===- DAGCombiner.cpp - Implement a DAG node combiner --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass combines dag nodes to form fewer, simpler DAG nodes. It can be run
10// both before and after the DAG is legalized.
11//
12// This pass is not a substitute for the LLVM IR instcombine pass. This pass is
13// primarily intended to handle simplification opportunities that are implicit
14// in the LLVM IR and exposed by the various codegen lowering phases.
15//
16//===----------------------------------------------------------------------===//
17
18#include "SelectionDAGOptions.h"
19#include "llvm/ADT/APFloat.h"
20#include "llvm/ADT/APInt.h"
21#include "llvm/ADT/APSInt.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/ADT/DenseMap.h"
25#include "llvm/ADT/STLExtras.h"
26#include "llvm/ADT/SetVector.h"
29#include "llvm/ADT/SmallSet.h"
31#include "llvm/ADT/Statistic.h"
53#include "llvm/IR/Attributes.h"
54#include "llvm/IR/Constant.h"
55#include "llvm/IR/DataLayout.h"
58#include "llvm/IR/Function.h"
59#include "llvm/IR/Metadata.h"
64#include "llvm/Support/Debug.h"
72#include <algorithm>
73#include <cassert>
74#include <cstdint>
75#include <functional>
76#include <initializer_list>
77#include <iterator>
78#include <optional>
79#include <string>
80#include <tuple>
81#include <utility>
82#include <variant>
83
84#include "SDNodeDbgValue.h"
85
86using namespace llvm;
87using namespace llvm::SDPatternMatch;
88
89#define DEBUG_TYPE "dagcombine"
90
91STATISTIC(NodesCombined , "Number of dag nodes combined");
92STATISTIC(PreIndexedNodes , "Number of pre-indexed nodes created");
93STATISTIC(PostIndexedNodes, "Number of post-indexed nodes created");
94STATISTIC(OpsNarrowed , "Number of load/op/store narrowed");
95STATISTIC(LdStFP2Int , "Number of fp load/store pairs transformed to int");
96STATISTIC(SlicedLoads, "Number of load sliced");
97STATISTIC(NumFPLogicOpsConv, "Number of logic ops converted to fp ops");
98
99DEBUG_COUNTER(DAGCombineCounter, "dagcombine",
100 "Controls whether a DAG combine is performed for a node");
101
102namespace {
103
104template <typename RangeT = std::initializer_list<const MemSDNode *>>
105static const MDNode *getCommonMemCacheHint(const RangeT &Nodes) {
106 auto Begin = Nodes.begin();
107 if (Begin == Nodes.end())
108 return nullptr;
109
110 const MDNode *MemCacheHint = (*Begin)->getMemCacheHint();
111 if (all_of(Nodes, [MemCacheHint](const MemSDNode *Node) {
112 return Node->getMemCacheHint() == MemCacheHint;
113 }))
114 return MemCacheHint;
115 return nullptr;
116}
117
118 class DAGCombiner {
119 SelectionDAG &DAG;
120 const TargetLowering &TLI;
121 const SelectionDAGTargetInfo *STI;
123 CodeGenOptLevel OptLevel;
124 bool LegalDAG = false;
125 bool LegalOperations = false;
126 bool LegalTypes = false;
127 bool ForCodeSize;
128 bool DisableGenericCombines;
129
130 /// Worklist of all of the nodes that need to be simplified.
131 ///
132 /// This must behave as a stack -- new nodes to process are pushed onto the
133 /// back and when processing we pop off of the back.
134 ///
135 /// The worklist will not contain duplicates but may contain null entries
136 /// due to nodes being deleted from the underlying DAG. For fast lookup and
137 /// deduplication, the index of the node in this vector is stored in the
138 /// node in SDNode::CombinerWorklistIndex.
140
141 /// This records all nodes attempted to be added to the worklist since we
142 /// considered a new worklist entry. As we keep do not add duplicate nodes
143 /// in the worklist, this is different from the tail of the worklist.
144 SmallSetVector<SDNode *, 32> PruningList;
145
146 /// Map from candidate StoreNode to the pair of RootNode and count.
147 /// The count is used to track how many times we have seen the StoreNode
148 /// with the same RootNode bail out in dependence check. If we have seen
149 /// the bail out for the same pair many times over a limit, we won't
150 /// consider the StoreNode with the same RootNode as store merging
151 /// candidate again.
152 DenseMap<SDNode *, std::pair<SDNode *, unsigned>> StoreRootCountMap;
153
154 // BatchAA - Used for DAG load/store alias analysis.
155 BatchAAResults *BatchAA;
156
157 /// This caches all chains that have already been processed in
158 /// DAGCombiner::getStoreMergeCandidates() and found to have no mergeable
159 /// stores candidates.
160 SmallPtrSet<SDNode *, 4> ChainsWithoutMergeableStores;
161
162 /// When an instruction is simplified, add all users of the instruction to
163 /// the work lists because they might get more simplified now.
164 void AddUsersToWorklist(SDNode *N) {
165 for (SDNode *Node : N->users())
166 AddToWorklist(Node);
167 }
168
169 /// Convenient shorthand to add a node and all of its user to the worklist.
170 void AddToWorklistWithUsers(SDNode *N) {
171 AddUsersToWorklist(N);
172 AddToWorklist(N);
173 }
174
175 // Prune potentially dangling nodes. This is called after
176 // any visit to a node, but should also be called during a visit after any
177 // failed combine which may have created a DAG node.
178 void clearAddedDanglingWorklistEntries() {
179 // Check any nodes added to the worklist to see if they are prunable.
180 while (!PruningList.empty()) {
181 auto *N = PruningList.pop_back_val();
182 if (N->use_empty())
183 recursivelyDeleteUnusedNodes(N);
184 }
185 }
186
187 SDNode *getNextWorklistEntry() {
188 // Before we do any work, remove nodes that are not in use.
189 clearAddedDanglingWorklistEntries();
190 SDNode *N = nullptr;
191 // The Worklist holds the SDNodes in order, but it may contain null
192 // entries.
193 while (!N && !Worklist.empty()) {
194 N = Worklist.pop_back_val();
195 }
196
197 if (N) {
198 assert(N->getCombinerWorklistIndex() >= 0 &&
199 "Found a worklist entry without a corresponding map entry!");
200 // Set to -2 to indicate that we combined the node.
201 N->setCombinerWorklistIndex(-2);
202 }
203 return N;
204 }
205
206 /// Call the node-specific routine that folds each particular type of node.
207 SDValue visit(SDNode *N);
208
209 public:
210 DAGCombiner(SelectionDAG &D, BatchAAResults *BatchAA, CodeGenOptLevel OL)
211 : DAG(D), TLI(D.getTargetLoweringInfo()),
212 STI(D.getSubtarget().getSelectionDAGInfo()), OptLevel(OL),
213 BatchAA(BatchAA) {
214 ForCodeSize = DAG.shouldOptForSize();
215 DisableGenericCombines = DAG.getSelectionDAGOptions().combiner_disabled ||
216 (STI && STI->disableGenericCombines(OptLevel));
217 }
218
219 void ConsiderForPruning(SDNode *N) {
220 // Mark this for potential pruning.
221 PruningList.insert(N);
222 }
223
224 /// Add to the worklist making sure its instance is at the back (next to be
225 /// processed.)
226 void AddToWorklist(SDNode *N, bool IsCandidateForPruning = true,
227 bool SkipIfCombinedBefore = false) {
228 assert(N->getOpcode() != ISD::DELETED_NODE &&
229 "Deleted Node added to Worklist");
230
231 // Skip handle nodes as they can't usefully be combined and confuse the
232 // zero-use deletion strategy.
233 if (N->getOpcode() == ISD::HANDLENODE)
234 return;
235
236 if (SkipIfCombinedBefore && N->getCombinerWorklistIndex() == -2)
237 return;
238
239 if (IsCandidateForPruning)
240 ConsiderForPruning(N);
241
242 if (N->getCombinerWorklistIndex() < 0) {
243 N->setCombinerWorklistIndex(Worklist.size());
244 Worklist.push_back(N);
245 }
246 }
247
248 /// Remove all instances of N from the worklist.
249 void removeFromWorklist(SDNode *N) {
250 PruningList.remove(N);
251 StoreRootCountMap.erase(N);
252
253 int WorklistIndex = N->getCombinerWorklistIndex();
254 // If not in the worklist, the index might be -1 or -2 (was combined
255 // before). As the node gets deleted anyway, there's no need to update
256 // the index.
257 if (WorklistIndex < 0)
258 return; // Not in the worklist.
259
260 // Null out the entry rather than erasing it to avoid a linear operation.
261 Worklist[WorklistIndex] = nullptr;
262 N->setCombinerWorklistIndex(-1);
263 }
264
265 void deleteAndRecombine(SDNode *N);
266 bool recursivelyDeleteUnusedNodes(SDNode *N);
267
268 /// Replaces all uses of the results of one DAG node with new values.
269 SDValue CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
270 bool AddTo = true);
271
272 /// Replaces all uses of the results of one DAG node with new values.
273 SDValue CombineTo(SDNode *N, SDValue Res, bool AddTo = true) {
274 return CombineTo(N, &Res, 1, AddTo);
275 }
276
277 /// Replaces all uses of the results of one DAG node with new values.
278 SDValue CombineTo(SDNode *N, SDValue Res0, SDValue Res1,
279 bool AddTo = true) {
280 SDValue To[] = { Res0, Res1 };
281 return CombineTo(N, To, 2, AddTo);
282 }
283
284 SDValue CombineTo(SDNode *N, SmallVectorImpl<SDValue> *To,
285 bool AddTo = true) {
286 return CombineTo(N, To->data(), To->size(), AddTo);
287 }
288
289 void CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO);
290
291 private:
292 /// Check the specified integer node value to see if it can be simplified or
293 /// if things it uses can be simplified by bit propagation.
294 /// If so, return true.
295 bool SimplifyDemandedBits(SDValue Op) {
296 unsigned BitWidth = Op.getScalarValueSizeInBits();
297 APInt DemandedBits = APInt::getAllOnes(BitWidth);
298 return SimplifyDemandedBits(Op, DemandedBits);
299 }
300
301 bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits) {
302 EVT VT = Op.getValueType();
303 APInt DemandedElts = VT.isFixedLengthVector()
305 : APInt(1, 1);
306 return SimplifyDemandedBits(Op, DemandedBits, DemandedElts, false);
307 }
308
309 /// Check the specified vector node value to see if it can be simplified or
310 /// if things it uses can be simplified as it only uses some of the
311 /// elements. If so, return true.
312 bool SimplifyDemandedVectorElts(SDValue Op) {
313 // TODO: For now just pretend it cannot be simplified.
314 if (Op.getValueType().isScalableVector())
315 return false;
316
317 unsigned NumElts = Op.getValueType().getVectorNumElements();
318 APInt DemandedElts = APInt::getAllOnes(NumElts);
319 return SimplifyDemandedVectorElts(Op, DemandedElts);
320 }
321
322 bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
323 const APInt &DemandedElts,
324 bool AssumeSingleUse = false);
325 bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedElts,
326 bool AssumeSingleUse = false);
327
328 bool CombineToPreIndexedLoadStore(SDNode *N);
329 bool CombineToPostIndexedLoadStore(SDNode *N);
330 SDValue SplitIndexingFromLoad(LoadSDNode *LD);
331 bool SliceUpLoad(SDNode *N);
332
333 // Looks up the chain to find a unique (unaliased) store feeding the passed
334 // load. If no such store is found, returns a nullptr.
335 // Note: This will look past a CALLSEQ_START if the load is chained to it so
336 // so that it can find stack stores for byval params.
337 StoreSDNode *getUniqueStoreFeeding(LoadSDNode *LD, int64_t &Offset);
338 // Scalars have size 0 to distinguish from singleton vectors.
339 SDValue ForwardStoreValueToDirectLoad(LoadSDNode *LD);
340 bool getTruncatedStoreValue(StoreSDNode *ST, SDValue &Val);
341 bool extendLoadedValueToExtension(LoadSDNode *LD, SDValue &Val);
342
343 void ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad);
344 SDValue PromoteOperand(SDValue Op, EVT PVT, bool &Replace);
345 SDValue SExtPromoteOperand(SDValue Op, EVT PVT);
346 SDValue ZExtPromoteOperand(SDValue Op, EVT PVT);
347 SDValue PromoteIntBinOp(SDValue Op);
348 SDValue PromoteIntShiftOp(SDValue Op);
349 SDValue PromoteExtend(SDValue Op);
350 bool PromoteLoad(SDValue Op);
351
352 SDValue foldShiftToAvg(SDNode *N, const SDLoc &DL);
353 // Fold `a bitwiseop (~b +/- c)` -> `a bitwiseop ~(b -/+ c)`
354 SDValue foldBitwiseOpWithNeg(SDNode *N, const SDLoc &DL, EVT VT);
355
356 SDValue combineMinNumMaxNum(const SDLoc &DL, EVT VT, SDValue LHS,
359
360 /// Call the node-specific routine that knows how to fold each
361 /// particular type of node. If that doesn't do anything, try the
362 /// target-specific DAG combines.
363 SDValue combine(SDNode *N);
364
365 // Visitation implementation - Implement dag node combining for different
366 // node types. The semantics are as follows:
367 // Return Value:
368 // SDValue.getNode() == 0 - No change was made
369 // SDValue.getNode() == N - N was replaced, is dead and has been handled.
370 // otherwise - N should be replaced by the returned Operand.
371 //
372 SDValue visitTokenFactor(SDNode *N);
373 SDValue visitGET_ROUNDING(SDNode *N);
374 SDValue visitMERGE_VALUES(SDNode *N);
375 SDValue visitADD(SDNode *N);
376 SDValue visitADDLike(SDNode *N);
377 SDValue visitADDLikeCommutative(SDValue N0, SDValue N1, const SDLoc &DL);
378 SDValue visitPTRADD(SDNode *N);
379 SDValue visitSUB(SDNode *N);
380 SDValue visitADDSAT(SDNode *N);
381 SDValue visitSUBSAT(SDNode *N);
382 SDValue visitADDC(SDNode *N);
383 SDValue visitADDO(SDNode *N);
384 SDValue visitUADDOLike(SDValue N0, SDValue N1, SDNode *N);
385 SDValue visitSUBC(SDNode *N);
386 SDValue visitSUBO(SDNode *N);
387 SDValue visitADDE(SDNode *N);
388 SDValue visitUADDO_CARRY(SDNode *N);
389 SDValue visitSADDO_CARRY(SDNode *N);
390 SDValue visitUADDO_CARRYLike(SDValue N0, SDValue N1, SDValue CarryIn,
391 SDNode *N);
392 SDValue visitSADDO_CARRYLike(SDValue N0, SDValue N1, SDValue CarryIn,
393 SDNode *N);
394 SDValue visitSUBE(SDNode *N);
395 SDValue visitUSUBO_CARRY(SDNode *N);
396 SDValue visitSSUBO_CARRY(SDNode *N);
397 SDValue visitMUL(SDNode *N);
398 SDValue visitMULFIX(SDNode *N);
399 SDValue useDivRem(SDNode *N);
400 SDValue visitSDIV(SDNode *N);
401 SDValue visitSDIVLike(SDValue N0, SDValue N1, SDNode *N);
402 SDValue visitUDIV(SDNode *N);
403 SDValue visitUDIVLike(SDValue N0, SDValue N1, SDNode *N);
404 SDValue visitREM(SDNode *N);
405 SDValue visitMULHU(SDNode *N);
406 SDValue visitMULHS(SDNode *N);
407 SDValue visitAVG(SDNode *N);
408 SDValue visitAVGSUB(SDNode *N);
409 SDValue visitABD(SDNode *N);
410 SDValue visitSMUL_LOHI(SDNode *N);
411 SDValue visitUMUL_LOHI(SDNode *N);
412 SDValue visitMULO(SDNode *N);
413 SDValue visitIMINMAX(SDNode *N);
414 SDValue visitAND(SDNode *N);
415 SDValue visitANDLike(SDValue N0, SDValue N1, SDNode *N);
416 SDValue visitOR(SDNode *N);
417 SDValue visitORLike(SDValue N0, SDValue N1, const SDLoc &DL);
418 SDValue visitXOR(SDNode *N);
419 SDValue SimplifyVCastOp(SDNode *N, const SDLoc &DL);
420 SDValue SimplifyVBinOp(SDNode *N, const SDLoc &DL);
421 SDValue visitSHL(SDNode *N);
422 SDValue visitSRA(SDNode *N);
423 SDValue visitSRL(SDNode *N);
424 SDValue visitFunnelShift(SDNode *N);
425 SDValue visitSHLSAT(SDNode *N);
426 SDValue visitRotate(SDNode *N);
427 SDValue visitABS(SDNode *N);
428 SDValue visitABS_MIN_POISON(SDNode *N);
429 SDValue visitCLMUL(SDNode *N);
430 SDValue visitPEXT(SDNode *N);
431 SDValue visitPDEP(SDNode *N);
432 SDValue visitBSWAP(SDNode *N);
433 SDValue visitBITREVERSE(SDNode *N);
434 SDValue visitCTLZ(SDNode *N);
435 SDValue visitCTLZ_ZERO_POISON(SDNode *N);
436 SDValue visitCTTZ(SDNode *N);
437 SDValue visitCTTZ_ZERO_POISON(SDNode *N);
438 SDValue visitCTPOP(SDNode *N);
439 SDValue visitPARITY(SDNode *N);
440 SDValue visitSELECT(SDNode *N);
441 SDValue visitVSELECT(SDNode *N);
442 SDValue visitSELECT_CC(SDNode *N);
443 SDValue visitSETCC(SDNode *N);
444 SDValue visitSETCCCARRY(SDNode *N);
445 SDValue visitSIGN_EXTEND(SDNode *N);
446 SDValue visitZERO_EXTEND(SDNode *N);
447 SDValue visitANY_EXTEND(SDNode *N);
448 SDValue visitAssertExt(SDNode *N);
449 SDValue visitAssertAlign(SDNode *N);
450 SDValue visitIS_FPCLASS(SDNode *N);
451 SDValue visitSIGN_EXTEND_INREG(SDNode *N);
452 SDValue visitEXTEND_VECTOR_INREG(SDNode *N);
453 SDValue visitTRUNCATE(SDNode *N);
454 SDValue visitTRUNCATE_USAT_U(SDNode *N);
455 SDValue visitBITCAST(SDNode *N);
456 SDValue visitFREEZE(SDNode *N);
457 SDValue visitBUILD_PAIR(SDNode *N);
458 SDValue visitFADD(SDNode *N);
459 SDValue visitSTRICT_FADD(SDNode *N);
460 SDValue visitFSUB(SDNode *N);
461 SDValue visitFMUL(SDNode *N);
462 SDValue visitFMA(SDNode *N);
463 SDValue visitFMAD(SDNode *N);
464 SDValue visitFMULADD(SDNode *N);
465 SDValue visitFDIV(SDNode *N);
466 SDValue visitFREM(SDNode *N);
467 SDValue visitFSQRT(SDNode *N);
468 SDValue visitFCOPYSIGN(SDNode *N);
469 SDValue visitFPOW(SDNode *N);
470 SDValue visitFCANONICALIZE(SDNode *N);
471 SDValue visitSINT_TO_FP(SDNode *N);
472 SDValue visitUINT_TO_FP(SDNode *N);
473 SDValue visitFP_TO_SINT(SDNode *N);
474 SDValue visitFP_TO_UINT(SDNode *N);
475 SDValue visitFP_TO_INT_SAT(SDNode *N);
476 SDValue visitXROUND(SDNode *N);
477 SDValue visitFP_ROUND(SDNode *N);
478 SDValue visitFP_EXTEND(SDNode *N);
479 SDValue visitFNEG(SDNode *N);
480 SDValue visitFABS(SDNode *N);
481 SDValue visitFCEIL(SDNode *N);
482 SDValue visitFTRUNC(SDNode *N);
483 SDValue visitFFREXP(SDNode *N);
484 SDValue visitFFLOOR(SDNode *N);
485 SDValue visitFMinMax(SDNode *N);
486 SDValue visitBRCOND(SDNode *N);
487 SDValue visitBR_CC(SDNode *N);
488 SDValue visitLOAD(SDNode *N);
489
490 SDValue replaceStoreChain(StoreSDNode *ST, SDValue BetterChain);
491 SDValue replaceStoreOfFPConstant(StoreSDNode *ST);
492 SDValue replaceStoreOfInsertLoad(StoreSDNode *ST);
493
494 bool refineExtractVectorEltIntoMultipleNarrowExtractVectorElts(SDNode *N);
495 SDValue combineStoreConcatTruncVector(StoreSDNode *N);
496 SDValue visitSTORE(SDNode *N);
497 SDValue visitATOMIC_STORE(SDNode *N);
498 SDValue visitLIFETIME_END(SDNode *N);
499 SDValue visitINSERT_VECTOR_ELT(SDNode *N);
500 SDValue visitEXTRACT_VECTOR_ELT(SDNode *N);
501 SDValue visitBUILD_VECTOR(SDNode *N);
502 SDValue visitCONCAT_VECTORS(SDNode *N);
503 SDValue visitVECTOR_INTERLEAVE(SDNode *N);
504 SDValue visitVECTOR_DEINTERLEAVE(SDNode *N);
505 SDValue visitEXTRACT_SUBVECTOR(SDNode *N);
506 SDValue visitVECTOR_SHUFFLE(SDNode *N);
507 SDValue visitSCALAR_TO_VECTOR(SDNode *N);
508 SDValue visitINSERT_SUBVECTOR(SDNode *N);
509 SDValue visitVECTOR_COMPRESS(SDNode *N);
510 SDValue visitMLOAD(SDNode *N);
511 SDValue visitMSTORE(SDNode *N);
512 SDValue visitMGATHER(SDNode *N);
513 SDValue visitMSCATTER(SDNode *N);
514 SDValue visitMHISTOGRAM(SDNode *N);
515 SDValue visitPARTIAL_REDUCE_MLA(SDNode *N);
516 SDValue visitLOOP_DEPENDENCE_MASK(SDNode *N);
517 SDValue visitVPGATHER(SDNode *N);
518 SDValue visitVPSCATTER(SDNode *N);
519 SDValue visitVP_STRIDED_LOAD(SDNode *N);
520 SDValue visitVP_STRIDED_STORE(SDNode *N);
521 SDValue visitFP_TO_FP16(SDNode *N);
522 SDValue visitFP16_TO_FP(SDNode *N);
523 SDValue visitFP_TO_BF16(SDNode *N);
524 SDValue visitBF16_TO_FP(SDNode *N);
525 SDValue visitVECREDUCE(SDNode *N);
526 SDValue visitVPOp(SDNode *N);
527 SDValue visitGET_FPENV_MEM(SDNode *N);
528 SDValue visitSET_FPENV_MEM(SDNode *N);
529
530 SDValue visitFADDForFMACombine(SDNode *N);
531 SDValue visitFSUBForFMACombine(SDNode *N);
532 SDValue visitFMULForFMADistributiveCombine(SDNode *N);
533
534 SDValue XformToShuffleWithZero(SDNode *N);
535 bool reassociationCanBreakAddressingModePattern(unsigned Opc,
536 const SDLoc &DL,
537 SDNode *N,
538 SDValue N0,
539 SDValue N1);
540 SDValue reassociateOpsCommutative(unsigned Opc, const SDLoc &DL, SDValue N0,
541 SDValue N1, SDNodeFlags Flags);
542 SDValue reassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0,
543 SDValue N1, SDNodeFlags Flags);
544 SDValue reassociateReduction(unsigned RedOpc, unsigned Opc, const SDLoc &DL,
545 EVT VT, SDValue N0, SDValue N1,
546 SDNodeFlags Flags = SDNodeFlags());
547
548 SDValue visitShiftByConstant(SDNode *N);
549
550 SDValue foldSelectOfConstants(SDNode *N);
551 SDValue foldVSelectOfConstants(SDNode *N);
552 SDValue foldBinOpIntoSelect(SDNode *BO);
553 bool SimplifySelectOps(SDNode *SELECT, SDValue LHS, SDValue RHS);
554 SDValue hoistLogicOpWithSameOpcodeHands(SDNode *N);
555 SDValue SimplifySelect(const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2);
556 SDValue SimplifySelectCC(const SDLoc &DL, SDValue N0, SDValue N1,
558 bool NotExtCompare = false);
559 SDValue convertSelectOfFPConstantsToLoadOffset(
560 const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2, SDValue N3,
562 SDValue foldSignChangeInBitcast(SDNode *N);
563 SDValue foldSelectCCToShiftAnd(const SDLoc &DL, SDValue N0, SDValue N1,
565 SDValue foldSelectOfBinops(SDNode *N);
566 SDValue foldSextSetcc(SDNode *N);
567 SDValue foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1,
568 const SDLoc &DL);
569 SDValue foldSubToUSubSat(EVT DstVT, SDNode *N, const SDLoc &DL);
570 SDValue foldABSToABD(SDNode *N, const SDLoc &DL);
571 SDValue foldSelectToABD(SDValue LHS, SDValue RHS, SDValue True,
572 SDValue False, ISD::CondCode CC, const SDLoc &DL);
573 SDValue foldSelectToUMin(SDValue LHS, SDValue RHS, SDValue True,
574 SDValue False, ISD::CondCode CC, const SDLoc &DL);
575 SDValue unfoldMaskedMerge(SDNode *N);
576 SDValue unfoldExtremeBitClearingToShifts(SDNode *N);
577 SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
578 const SDLoc &DL, bool foldBooleans);
579 SDValue rebuildSetCC(SDValue N);
580
581 bool isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
582 SDValue &CC, bool MatchStrict = false) const;
583 bool isOneUseSetCC(SDValue N) const;
584
585 SDValue foldAddToAvg(SDNode *N, const SDLoc &DL);
586 SDValue foldSubToAvg(SDNode *N, const SDLoc &DL);
587
588 SDValue foldCTLZToCTLS(SDValue Src, const SDLoc &DL);
589
590 SDValue SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
591 unsigned HiOp);
592 SDValue CombineConsecutiveLoads(SDNode *N, EVT VT);
593 SDValue foldBitcastedFPLogic(SDNode *N, SelectionDAG &DAG,
594 const TargetLowering &TLI);
595 SDValue foldPartialReduceMLAMulOp(SDNode *N);
596 SDValue foldPartialReduceAdd(SDNode *N);
597
598 SDValue CombineExtLoad(SDNode *N);
599 SDValue CombineZExtLogicopShiftLoad(SDNode *N);
600 SDValue combineRepeatedFPDivisors(SDNode *N);
601 SDValue combineFMulOrFDivWithIntPow2(SDNode *N);
602 SDValue replaceShuffleOfInsert(ShuffleVectorSDNode *Shuf);
603 SDValue mergeInsertEltWithShuffle(SDNode *N, unsigned InsIndex);
604 SDValue combineInsertEltToShuffle(SDNode *N, unsigned InsIndex);
605 SDValue combineInsertEltToLoad(SDNode *N, unsigned InsIndex);
606 SDValue foldExtractSubvectorFromConcatVectors(EVT VT, SDValue V,
607 uint64_t ExtIdx,
608 const SDLoc &DL);
609 SDValue BuildSDIV(SDNode *N);
610 SDValue BuildSDIVPow2(SDNode *N);
611 SDValue BuildUDIV(SDNode *N);
612 SDValue BuildSREMPow2(SDNode *N);
613 SDValue buildOptimizedSREM(SDValue N0, SDValue N1, SDNode *N);
614 SDValue BuildLogBase2(SDValue V, const SDLoc &DL,
615 bool KnownNeverZero = false,
616 bool InexpensiveOnly = false,
617 std::optional<EVT> OutVT = std::nullopt);
618 SDValue BuildDivEstimate(SDValue N, SDValue Op, SDNodeFlags Flags);
619 SDValue buildRsqrtEstimate(SDValue Op, SDNodeFlags Flags);
620 SDValue buildSqrtEstimate(SDValue Op, SDNodeFlags Flags);
621 SDValue buildSqrtEstimateImpl(SDValue Op, bool Recip, SDNodeFlags Flags);
622 SDValue buildSqrtNROneConst(SDValue Arg, SDValue Est, unsigned Iterations,
623 bool Reciprocal);
624 SDValue buildSqrtNRTwoConst(SDValue Arg, SDValue Est, unsigned Iterations,
625 bool Reciprocal);
626 SDValue MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
627 bool DemandHighBits = true);
628 SDValue MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1);
629 SDValue MatchRotatePosNeg(SDValue Shifted, SDValue Pos, SDValue Neg,
630 SDValue InnerPos, SDValue InnerNeg, bool FromAdd,
631 bool HasPos, unsigned PosOpcode,
632 unsigned NegOpcode, const SDLoc &DL);
633 SDValue MatchFunnelPosNeg(SDValue N0, SDValue N1, SDValue Pos, SDValue Neg,
634 SDValue InnerPos, SDValue InnerNeg, bool FromAdd,
635 bool HasPos, unsigned PosOpcode,
636 unsigned NegOpcode, const SDLoc &DL);
637 SDValue MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL,
638 bool FromAdd);
639 SDValue MatchLoadCombine(SDNode *N);
640 SDValue mergeTruncStores(StoreSDNode *N);
641 SDValue reduceLoadWidth(SDNode *N);
642 SDValue ReduceLoadOpStoreWidth(SDNode *N);
643 SDValue splitMergedValStore(StoreSDNode *ST);
644 SDValue TransformFPLoadStorePair(SDNode *N);
645 SDValue convertBuildVecExtToExt(SDNode *N);
646 SDValue convertBuildVecZextToBuildVecWithZeros(SDNode *N);
647 SDValue reduceBuildVecExtToExtBuildVec(SDNode *N);
648 SDValue reduceBuildVecTruncToBitCast(SDNode *N);
649 SDValue reduceBuildVecToShuffle(SDNode *N);
650 SDValue createBuildVecShuffle(const SDLoc &DL, SDNode *N,
651 ArrayRef<int> VectorMask, SDValue VecIn1,
652 SDValue VecIn2, unsigned LeftIdx,
653 bool DidSplitVec);
654 SDValue matchVSelectOpSizesWithSetCC(SDNode *Cast);
655
656 /// Walk up chain skipping non-aliasing memory nodes,
657 /// looking for aliasing nodes and adding them to the Aliases vector.
658 void GatherAllAliases(SDNode *N, SDValue OriginalChain,
659 SmallVectorImpl<SDValue> &Aliases);
660
661 /// Return true if there is any possibility that the two addresses overlap.
662 bool mayAlias(SDNode *Op0, SDNode *Op1) const;
663
664 /// Walk up chain skipping non-aliasing memory nodes, looking for a better
665 /// chain (aliasing node.)
666 SDValue FindBetterChain(SDNode *N, SDValue Chain);
667
668 /// Try to replace a store and any possibly adjacent stores on
669 /// consecutive chains with better chains. Return true only if St is
670 /// replaced.
671 ///
672 /// Notice that other chains may still be replaced even if the function
673 /// returns false.
674 bool findBetterNeighborChains(StoreSDNode *St);
675
676 // Helper for findBetterNeighborChains. Walk up store chain add additional
677 // chained stores that do not overlap and can be parallelized.
678 bool parallelizeChainedStores(StoreSDNode *St);
679
680 /// Holds a pointer to an LSBaseSDNode as well as information on where it
681 /// is located in a sequence of memory operations connected by a chain.
682 struct MemOpLink {
683 // Ptr to the mem node.
684 LSBaseSDNode *MemNode;
685
686 // Offset from the base ptr.
687 int64_t OffsetFromBase;
688
689 MemOpLink(LSBaseSDNode *N, int64_t Offset)
690 : MemNode(N), OffsetFromBase(Offset) {}
691 };
692
693 // Classify the origin of a stored value.
694 enum class StoreSource { Unknown, Constant, Extract, Load };
695 StoreSource getStoreSource(SDValue StoreVal) {
696 switch (StoreVal.getOpcode()) {
697 case ISD::Constant:
698 case ISD::ConstantFP:
699 return StoreSource::Constant;
703 return StoreSource::Constant;
704 return StoreSource::Unknown;
707 return StoreSource::Extract;
708 case ISD::LOAD:
709 return StoreSource::Load;
710 default:
711 return StoreSource::Unknown;
712 }
713 }
714
715 /// This is a helper function for visitMUL to check the profitability
716 /// of folding (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2).
717 /// MulNode is the original multiply, AddNode is (add x, c1),
718 /// and ConstNode is c2.
719 bool isMulAddWithConstProfitable(SDNode *MulNode, SDValue AddNode,
720 SDValue ConstNode);
721
722 /// This is a helper function for visitAND and visitZERO_EXTEND. Returns
723 /// true if the (and (load x) c) pattern matches an extload. ExtVT returns
724 /// the type of the loaded value to be extended.
725 bool isAndLoadExtLoad(ConstantSDNode *AndC, LoadSDNode *LoadN,
726 EVT LoadResultTy, EVT &ExtVT);
727
728 /// Helper function to calculate whether the given Load/Store can have its
729 /// width reduced to ExtVT.
730 bool isLegalNarrowLdSt(LSBaseSDNode *LDSTN, ISD::LoadExtType ExtType,
731 EVT &MemVT, unsigned ShAmt = 0);
732
733 /// Used by BackwardsPropagateMask to find suitable loads.
734 bool SearchForAndLoads(SDNode *N, SmallVectorImpl<LoadSDNode*> &Loads,
735 SmallPtrSetImpl<SDNode*> &NodesWithConsts,
736 ConstantSDNode *Mask, SDNode *&NodeToMask);
737 /// Attempt to propagate a given AND node back to load leaves so that they
738 /// can be combined into narrow loads.
739 bool BackwardsPropagateMask(SDNode *N);
740
741 /// Helper function for mergeConsecutiveStores which merges the component
742 /// store chains.
743 SDValue getMergeStoreChains(SmallVectorImpl<MemOpLink> &StoreNodes,
744 unsigned NumStores);
745
746 /// Helper function for mergeConsecutiveStores which checks if all the
747 /// load/store nodes have the same underlying object. We can still reuse the
748 /// first load/store's pointer info if all the load/stores are from the same
749 /// object.
750 bool hasSameUnderlyingObj(ArrayRef<MemOpLink> MemOps);
751
752 /// This is a helper function for mergeConsecutiveStores. When the source
753 /// elements of the consecutive stores are all constants or all extracted
754 /// vector elements, try to merge them into one larger store introducing
755 /// bitcasts if necessary. \return True if a merged store was created.
756 bool mergeStoresOfConstantsOrVecElts(SmallVectorImpl<MemOpLink> &StoreNodes,
757 EVT MemVT, unsigned NumStores,
758 bool IsConstantSrc, bool UseVector,
759 bool UseTrunc);
760
761 /// This is a helper function for mergeConsecutiveStores. Stores that
762 /// potentially may be merged with St are placed in StoreNodes. On success,
763 /// returns a chain predecessor to all store candidates.
764 SDNode *getStoreMergeCandidates(StoreSDNode *St,
765 SmallVectorImpl<MemOpLink> &StoreNodes);
766
767 /// Helper function for mergeConsecutiveStores. Checks if candidate stores
768 /// have indirect dependency through their operands. RootNode is the
769 /// predecessor to all stores calculated by getStoreMergeCandidates and is
770 /// used to prune the dependency check. \return True if safe to merge.
771 bool checkMergeStoreCandidatesForDependencies(
772 SmallVectorImpl<MemOpLink> &StoreNodes, unsigned NumStores,
773 SDNode *RootNode);
774
775 /// Helper function for tryStoreMergeOfLoads. Checks if the load/store
776 /// chain has a call in it. \return True if a call is found.
777 bool hasCallInLdStChain(StoreSDNode *St, LoadSDNode *Ld);
778
779 /// This is a helper function for mergeConsecutiveStores. Given a list of
780 /// store candidates, find the first N that are consecutive in memory.
781 /// Returns 0 if there are not at least 2 consecutive stores to try merging.
782 unsigned getConsecutiveStores(SmallVectorImpl<MemOpLink> &StoreNodes,
783 int64_t ElementSizeBytes) const;
784
785 /// This is a helper function for mergeConsecutiveStores. It is used for
786 /// store chains that are composed entirely of constant values.
787 bool tryStoreMergeOfConstants(SmallVectorImpl<MemOpLink> &StoreNodes,
788 unsigned NumConsecutiveStores,
789 EVT MemVT, SDNode *Root, bool AllowVectors);
790
791 /// This is a helper function for mergeConsecutiveStores. It is used for
792 /// store chains that are composed entirely of extracted vector elements.
793 /// When extracting multiple vector elements, try to store them in one
794 /// vector store rather than a sequence of scalar stores.
795 bool tryStoreMergeOfExtracts(SmallVectorImpl<MemOpLink> &StoreNodes,
796 unsigned NumConsecutiveStores, EVT MemVT,
797 SDNode *Root);
798
799 /// This is a helper function for mergeConsecutiveStores. It is used for
800 /// store chains that are composed entirely of loaded values.
801 bool tryStoreMergeOfLoads(SmallVectorImpl<MemOpLink> &StoreNodes,
802 unsigned NumConsecutiveStores, EVT MemVT,
803 SDNode *Root, bool AllowVectors,
804 bool IsNonTemporalStore, bool IsNonTemporalLoad);
805
806 /// Merge consecutive store operations into a wide store.
807 /// This optimization uses wide integers or vectors when possible.
808 /// \return true if stores were merged.
809 bool mergeConsecutiveStores(StoreSDNode *St);
810
811 /// Try to transform a truncation where C is a constant:
812 /// (trunc (and X, C)) -> (and (trunc X), (trunc C))
813 ///
814 /// \p N needs to be a truncation and its first operand an AND. Other
815 /// requirements are checked by the function (e.g. that trunc is
816 /// single-use) and if missed an empty SDValue is returned.
817 SDValue distributeTruncateThroughAnd(SDNode *N);
818
819 /// Helper function to determine whether the target supports operation
820 /// given by \p Opcode for type \p VT, that is, whether the operation
821 /// is legal or custom before legalizing operations, and whether is
822 /// legal (but not custom) after legalization.
823 bool hasOperation(unsigned Opcode, EVT VT) {
824 return TLI.isOperationLegalOrCustom(Opcode, VT, LegalOperations);
825 }
826
827 bool hasUMin(EVT VT) const {
828 auto LK = TLI.getTypeConversion(*DAG.getContext(), VT);
829 return (LK.first == TargetLoweringBase::TypeLegal ||
831 TLI.isOperationLegalOrCustom(ISD::UMIN, LK.second);
832 }
833
834 public:
835 /// Runs the dag combiner on all nodes in the work list
836 void Run(CombineLevel AtLevel);
837
838 SelectionDAG &getDAG() const { return DAG; }
839
840 /// Convenience wrapper around TargetLowering::getShiftAmountTy.
841 EVT getShiftAmountTy(EVT LHSTy) {
842 return TLI.getShiftAmountTy(LHSTy, DAG.getDataLayout());
843 }
844
845 /// This method returns true if we are running before type legalization or
846 /// if the specified VT is legal.
847 bool isTypeLegal(const EVT &VT) {
848 if (!LegalTypes) return true;
849 return TLI.isTypeLegal(VT);
850 }
851
852 /// Convenience wrapper around TargetLowering::getSetCCResultType
853 EVT getSetCCResultType(EVT VT) const {
854 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
855 }
856
857 void ExtendSetCCUses(const SmallVectorImpl<SDNode *> &SetCCs,
858 SDValue OrigLoad, SDValue ExtLoad,
859 ISD::NodeType ExtType);
860 };
861
862/// This class is a DAGUpdateListener that removes any deleted
863/// nodes from the worklist.
864class WorklistRemover : public SelectionDAG::DAGUpdateListener {
865 DAGCombiner &DC;
866
867public:
868 explicit WorklistRemover(DAGCombiner &dc)
869 : SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
870
871 void NodeDeleted(SDNode *N, SDNode *E) override {
872 DC.removeFromWorklist(N);
873 }
874};
875
876class WorklistInserter : public SelectionDAG::DAGUpdateListener {
877 DAGCombiner &DC;
878
879public:
880 explicit WorklistInserter(DAGCombiner &dc)
881 : SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
882
883 // FIXME: Ideally we could add N to the worklist, but this causes exponential
884 // compile time costs in large DAGs, e.g. Halide.
885 void NodeInserted(SDNode *N) override { DC.ConsiderForPruning(N); }
886};
887
888} // end anonymous namespace
889
890//===----------------------------------------------------------------------===//
891// TargetLowering::DAGCombinerInfo implementation
892//===----------------------------------------------------------------------===//
893
895 ((DAGCombiner*)DC)->AddToWorklist(N);
896}
897
899CombineTo(SDNode *N, ArrayRef<SDValue> To, bool AddTo) {
900 return ((DAGCombiner*)DC)->CombineTo(N, &To[0], To.size(), AddTo);
901}
902
904CombineTo(SDNode *N, SDValue Res, bool AddTo) {
905 return ((DAGCombiner*)DC)->CombineTo(N, Res, AddTo);
906}
907
909CombineTo(SDNode *N, SDValue Res0, SDValue Res1, bool AddTo) {
910 return ((DAGCombiner*)DC)->CombineTo(N, Res0, Res1, AddTo);
911}
912
915 return ((DAGCombiner*)DC)->recursivelyDeleteUnusedNodes(N);
916}
917
920 return ((DAGCombiner*)DC)->CommitTargetLoweringOpt(TLO);
921}
922
923//===----------------------------------------------------------------------===//
924// Helper Functions
925//===----------------------------------------------------------------------===//
926
927void DAGCombiner::deleteAndRecombine(SDNode *N) {
928 removeFromWorklist(N);
929
930 // If the operands of this node are only used by the node, they will now be
931 // dead. Make sure to re-visit them and recursively delete dead nodes.
932 for (const SDValue &Op : N->ops())
933 // For an operand generating multiple values, one of the values may
934 // become dead allowing further simplification (e.g. split index
935 // arithmetic from an indexed load).
936 if (Op->hasOneUse() || Op->getNumValues() > 1)
937 AddToWorklist(Op.getNode());
938
939 DAG.DeleteNode(N);
940}
941
942// APInts must be the same size for most operations, this helper
943// function zero extends the shorter of the pair so that they match.
944// We provide an Offset so that we can create bitwidths that won't overflow.
945static void zeroExtendToMatch(APInt &LHS, APInt &RHS, unsigned Offset = 0) {
946 unsigned Bits = Offset + std::max(LHS.getBitWidth(), RHS.getBitWidth());
947 LHS = LHS.zext(Bits);
948 RHS = RHS.zext(Bits);
949}
950
951// Return true if this node is a setcc, or is a select_cc
952// that selects between the target values used for true and false, making it
953// equivalent to a setcc. Also, set the incoming LHS, RHS, and CC references to
954// the appropriate nodes based on the type of node we are checking. This
955// simplifies life a bit for the callers.
956bool DAGCombiner::isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
957 SDValue &CC, bool MatchStrict) const {
958 if (N.getOpcode() == ISD::SETCC) {
959 LHS = N.getOperand(0);
960 RHS = N.getOperand(1);
961 CC = N.getOperand(2);
962 return true;
963 }
964
965 if (MatchStrict &&
966 (N.getOpcode() == ISD::STRICT_FSETCC ||
967 N.getOpcode() == ISD::STRICT_FSETCCS)) {
968 LHS = N.getOperand(1);
969 RHS = N.getOperand(2);
970 CC = N.getOperand(3);
971 return true;
972 }
973
974 if (N.getOpcode() != ISD::SELECT_CC || !TLI.isConstTrueVal(N.getOperand(2)) ||
975 !TLI.isConstFalseVal(N.getOperand(3)))
976 return false;
977
978 if (TLI.getBooleanContents(N.getValueType()) ==
980 return false;
981
982 LHS = N.getOperand(0);
983 RHS = N.getOperand(1);
984 CC = N.getOperand(4);
985 return true;
986}
987
988/// Return true if this is a SetCC-equivalent operation with only one use.
989/// If this is true, it allows the users to invert the operation for free when
990/// it is profitable to do so.
991bool DAGCombiner::isOneUseSetCC(SDValue N) const {
992 SDValue N0, N1, N2;
993 if (isSetCCEquivalent(N, N0, N1, N2) && N->hasOneUse())
994 return true;
995 return false;
996}
997
999 if (!ScalarTy.isSimple())
1000 return false;
1001
1002 uint64_t MaskForTy = 0ULL;
1003 switch (ScalarTy.getSimpleVT().SimpleTy) {
1004 case MVT::i8:
1005 MaskForTy = 0xFFULL;
1006 break;
1007 case MVT::i16:
1008 MaskForTy = 0xFFFFULL;
1009 break;
1010 case MVT::i32:
1011 MaskForTy = 0xFFFFFFFFULL;
1012 break;
1013 default:
1014 return false;
1015 break;
1016 }
1017
1018 APInt Val;
1019 if (ISD::isConstantSplatVector(N, Val))
1020 return Val.getLimitedValue() == MaskForTy;
1021
1022 return false;
1023}
1024
1025// Determines if it is a constant integer or a splat/build vector of constant
1026// integers (and undefs).
1027// Do not permit build vector implicit truncation unless AllowTruncation is set.
1028static bool isConstantOrConstantVector(SDValue N, bool NoOpaques = false,
1029 bool AllowTruncation = false) {
1031 return !(Const->isOpaque() && NoOpaques);
1032 if (N.getOpcode() != ISD::BUILD_VECTOR && N.getOpcode() != ISD::SPLAT_VECTOR)
1033 return false;
1034 unsigned BitWidth = N.getScalarValueSizeInBits();
1035 for (const SDValue &Op : N->op_values()) {
1036 if (Op.isUndef())
1037 continue;
1039 if (!Const || (Const->isOpaque() && NoOpaques))
1040 return false;
1041 // When AllowTruncation is true, allow constants that have been promoted
1042 // during type legalization as long as the value fits in the target type.
1043 if ((AllowTruncation &&
1044 Const->getAPIntValue().getActiveBits() > BitWidth) ||
1045 (!AllowTruncation && Const->getAPIntValue().getBitWidth() != BitWidth))
1046 return false;
1047 }
1048 return true;
1049}
1050
1051// Determines if a BUILD_VECTOR is composed of all-constants possibly mixed with
1052// undef's.
1053static bool isAnyConstantBuildVector(SDValue V, bool NoOpaques = false) {
1054 if (V.getOpcode() != ISD::BUILD_VECTOR)
1055 return false;
1056 return isConstantOrConstantVector(V, NoOpaques) ||
1058}
1059
1060// Determine if this an indexed load with an opaque target constant index.
1061static bool canSplitIdx(LoadSDNode *LD, const SelectionDAGOptions &Opts) {
1062 return Opts.combiner_split_load_index &&
1063 (LD->getOperand(2).getOpcode() != ISD::TargetConstant ||
1064 !cast<ConstantSDNode>(LD->getOperand(2))->isOpaque());
1065}
1066
1067bool DAGCombiner::reassociationCanBreakAddressingModePattern(unsigned Opc,
1068 const SDLoc &DL,
1069 SDNode *N,
1070 SDValue N0,
1071 SDValue N1) {
1072 // Currently this only tries to ensure we don't undo the GEP splits done by
1073 // CodeGenPrepare when shouldConsiderGEPOffsetSplit is true. To ensure this,
1074 // we check if the following transformation would be problematic:
1075 // (load/store (add, (add, x, offset1), offset2)) ->
1076 // (load/store (add, x, offset1+offset2)).
1077
1078 // (load/store (add, (add, x, y), offset2)) ->
1079 // (load/store (add, (add, x, offset2), y)).
1080
1081 if (!N0.isAnyAdd())
1082 return false;
1083
1084 // Check for vscale addressing modes.
1085 // (load/store (add/sub (add x, y), vscale))
1086 // (load/store (add/sub (add x, y), (lsl vscale, C)))
1087 // (load/store (add/sub (add x, y), (mul vscale, C)))
1088 if ((N1.getOpcode() == ISD::VSCALE ||
1089 ((N1.getOpcode() == ISD::SHL || N1.getOpcode() == ISD::MUL) &&
1090 N1.getOperand(0).getOpcode() == ISD::VSCALE &&
1092 N1.getValueType().getFixedSizeInBits() <= 64) {
1093 int64_t ScalableOffset = N1.getOpcode() == ISD::VSCALE
1094 ? N1.getConstantOperandVal(0)
1095 : (N1.getOperand(0).getConstantOperandVal(0) *
1096 (N1.getOpcode() == ISD::SHL
1097 ? (1LL << N1.getConstantOperandVal(1))
1098 : N1.getConstantOperandVal(1)));
1099 if (Opc == ISD::SUB)
1100 ScalableOffset = -ScalableOffset;
1101 if (all_of(N->users(), [&](SDNode *Node) {
1102 if (auto *LoadStore = dyn_cast<MemSDNode>(Node);
1103 LoadStore && LoadStore->hasUniqueMemOperand() &&
1104 LoadStore->getBasePtr().getNode() == N) {
1105 TargetLoweringBase::AddrMode AM;
1106 AM.HasBaseReg = true;
1107 AM.ScalableOffset = ScalableOffset;
1108 EVT VT = LoadStore->getMemoryVT();
1109 unsigned AS = LoadStore->getAddressSpace();
1110 Type *AccessTy = VT.getTypeForEVT(*DAG.getContext());
1111 return TLI.isLegalAddressingMode(DAG.getDataLayout(), AM, AccessTy,
1112 AS);
1113 }
1114 return false;
1115 }))
1116 return true;
1117 }
1118
1119 if (Opc != ISD::ADD && Opc != ISD::PTRADD)
1120 return false;
1121
1122 auto *C2 = dyn_cast<ConstantSDNode>(N1);
1123 if (!C2)
1124 return false;
1125
1126 const APInt &C2APIntVal = C2->getAPIntValue();
1127 if (C2APIntVal.getSignificantBits() > 64)
1128 return false;
1129
1130 if (auto *C1 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
1131 if (N0.hasOneUse())
1132 return false;
1133
1134 const APInt &C1APIntVal = C1->getAPIntValue();
1135 const APInt CombinedValueIntVal = C1APIntVal + C2APIntVal;
1136 if (CombinedValueIntVal.getSignificantBits() > 64)
1137 return false;
1138 const int64_t CombinedValue = CombinedValueIntVal.getSExtValue();
1139
1140 for (SDNode *Node : N->users()) {
1141 if (auto *LoadStore = dyn_cast<MemSDNode>(Node)) {
1142 if (!LoadStore->hasUniqueMemOperand())
1143 continue;
1144 // Is x[offset2] already not a legal addressing mode? If so then
1145 // reassociating the constants breaks nothing (we test offset2 because
1146 // that's the one we hope to fold into the load or store).
1147 TargetLoweringBase::AddrMode AM;
1148 AM.HasBaseReg = true;
1149 AM.BaseOffs = C2APIntVal.getSExtValue();
1150 EVT VT = LoadStore->getMemoryVT();
1151 unsigned AS = LoadStore->getAddressSpace();
1152 Type *AccessTy = VT.getTypeForEVT(*DAG.getContext());
1153 if (!TLI.isLegalAddressingMode(DAG.getDataLayout(), AM, AccessTy, AS))
1154 continue;
1155
1156 // Would x[offset1+offset2] still be a legal addressing mode?
1157 AM.BaseOffs = CombinedValue;
1158 if (!TLI.isLegalAddressingMode(DAG.getDataLayout(), AM, AccessTy, AS))
1159 return true;
1160 }
1161 }
1162 } else {
1163 if (auto *GA = dyn_cast<GlobalAddressSDNode>(N0.getOperand(1)))
1164 if (GA->getOpcode() == ISD::GlobalAddress && TLI.isOffsetFoldingLegal(GA))
1165 return false;
1166
1167 for (SDNode *Node : N->users()) {
1168 auto *LoadStore = dyn_cast<MemSDNode>(Node);
1169 if (!LoadStore || !LoadStore->hasUniqueMemOperand())
1170 return false;
1171
1172 // Is x[offset2] a legal addressing mode? If so then
1173 // reassociating the constants breaks address pattern
1174 TargetLoweringBase::AddrMode AM;
1175 AM.HasBaseReg = true;
1176 AM.BaseOffs = C2APIntVal.getSExtValue();
1177 EVT VT = LoadStore->getMemoryVT();
1178 unsigned AS = LoadStore->getAddressSpace();
1179 Type *AccessTy = VT.getTypeForEVT(*DAG.getContext());
1180 if (!TLI.isLegalAddressingMode(DAG.getDataLayout(), AM, AccessTy, AS))
1181 return false;
1182 }
1183 return true;
1184 }
1185
1186 return false;
1187}
1188
1189/// Helper for DAGCombiner::reassociateOps. Try to reassociate (Opc N0, N1) if
1190/// \p N0 is the same kind of operation as \p Opc.
1191SDValue DAGCombiner::reassociateOpsCommutative(unsigned Opc, const SDLoc &DL,
1192 SDValue N0, SDValue N1,
1193 SDNodeFlags Flags) {
1194 EVT VT = N0.getValueType();
1195
1196 if (N0.getOpcode() != Opc)
1197 return SDValue();
1198
1199 SDValue N00 = N0.getOperand(0);
1200 SDValue N01 = N0.getOperand(1);
1201
1203 SDNodeFlags NewFlags;
1204 if (N0.getOpcode() == ISD::ADD && N0->getFlags().hasNoUnsignedWrap() &&
1205 Flags.hasNoUnsignedWrap())
1206 NewFlags |= SDNodeFlags::NoUnsignedWrap;
1207
1209 // Reassociate: (op (op x, c1), c2) -> (op x, (op c1, c2))
1210 if (SDValue OpNode = DAG.FoldConstantArithmetic(Opc, DL, VT, {N01, N1})) {
1211 NewFlags.setDisjoint(Flags.hasDisjoint() &&
1212 N0->getFlags().hasDisjoint());
1213 return DAG.getNode(Opc, DL, VT, N00, OpNode, NewFlags);
1214 }
1215 return SDValue();
1216 }
1217 if (TLI.isReassocProfitable(DAG, N0, N1)) {
1218 // Reassociate: (op (op x, c1), y) -> (op (op x, y), c1)
1219 // iff (op x, c1) has one use
1220 SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N00, N1, NewFlags);
1221 return DAG.getNode(Opc, DL, VT, OpNode, N01, NewFlags);
1222 }
1223 }
1224
1225 // Check for repeated operand logic simplifications.
1226 if (Opc == ISD::AND || Opc == ISD::OR) {
1227 // (N00 & N01) & N00 --> N00 & N01
1228 // (N00 & N01) & N01 --> N00 & N01
1229 // (N00 | N01) | N00 --> N00 | N01
1230 // (N00 | N01) | N01 --> N00 | N01
1231 if (N1 == N00 || N1 == N01)
1232 return N0;
1233 }
1234 if (Opc == ISD::XOR) {
1235 // (N00 ^ N01) ^ N00 --> N01
1236 if (N1 == N00)
1237 return N01;
1238 // (N00 ^ N01) ^ N01 --> N00
1239 if (N1 == N01)
1240 return N00;
1241 }
1242
1243 if (TLI.isReassocProfitable(DAG, N0, N1)) {
1244 if (N1 != N01) {
1245 // Reassociate if (op N00, N1) already exist
1246 if (SDNode *NE = DAG.getNodeIfExists(Opc, DAG.getVTList(VT), {N00, N1})) {
1247 // if Op (Op N00, N1), N01 already exist
1248 // we need to stop reassciate to avoid dead loop
1249 if (!DAG.doesNodeExist(Opc, DAG.getVTList(VT), {SDValue(NE, 0), N01}))
1250 return DAG.getNode(Opc, DL, VT, SDValue(NE, 0), N01);
1251 }
1252 }
1253
1254 if (N1 != N00) {
1255 // Reassociate if (op N01, N1) already exist
1256 if (SDNode *NE = DAG.getNodeIfExists(Opc, DAG.getVTList(VT), {N01, N1})) {
1257 // if Op (Op N01, N1), N00 already exist
1258 // we need to stop reassciate to avoid dead loop
1259 if (!DAG.doesNodeExist(Opc, DAG.getVTList(VT), {SDValue(NE, 0), N00}))
1260 return DAG.getNode(Opc, DL, VT, SDValue(NE, 0), N00);
1261 }
1262 }
1263
1264 // Reassociate the operands from (OR/AND (OR/AND(N00, N001)), N1) to (OR/AND
1265 // (OR/AND(N00, N1)), N01) when N00 and N1 are comparisons with the same
1266 // predicate or to (OR/AND (OR/AND(N1, N01)), N00) when N01 and N1 are
1267 // comparisons with the same predicate. This enables optimizations as the
1268 // following one:
1269 // CMP(A,C)||CMP(B,C) => CMP(MIN/MAX(A,B), C)
1270 // CMP(A,C)&&CMP(B,C) => CMP(MIN/MAX(A,B), C)
1271 if (Opc == ISD::AND || Opc == ISD::OR) {
1272 if (N1->getOpcode() == ISD::SETCC && N00->getOpcode() == ISD::SETCC &&
1273 N01->getOpcode() == ISD::SETCC) {
1275 ISD::CondCode CC00 = N00.getOperandCondCode(2);
1276 ISD::CondCode CC01 = N01.getOperandCondCode(2);
1277 if (CC1 == CC00 && CC1 != CC01) {
1278 SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N00, N1, Flags);
1279 return DAG.getNode(Opc, DL, VT, OpNode, N01, Flags);
1280 }
1281 if (CC1 == CC01 && CC1 != CC00) {
1282 SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N01, N1, Flags);
1283 return DAG.getNode(Opc, DL, VT, OpNode, N00, Flags);
1284 }
1285 }
1286 }
1287 }
1288
1289 return SDValue();
1290}
1291
1292/// Try to reassociate commutative (Opc N0, N1) if either \p N0 or \p N1 is the
1293/// same kind of operation as \p Opc.
1294SDValue DAGCombiner::reassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0,
1295 SDValue N1, SDNodeFlags Flags) {
1296 assert(TLI.isCommutativeBinOp(Opc) && "Operation not commutative.");
1297
1298 // Floating-point reassociation is not allowed without loose FP math.
1299 if (N0.getValueType().isFloatingPoint() ||
1301 if (!Flags.hasAllowReassociation() || !Flags.hasNoSignedZeros())
1302 return SDValue();
1303
1304 if (SDValue Combined = reassociateOpsCommutative(Opc, DL, N0, N1, Flags))
1305 return Combined;
1306 if (SDValue Combined = reassociateOpsCommutative(Opc, DL, N1, N0, Flags))
1307 return Combined;
1308 return SDValue();
1309}
1310
1311// Try to fold Opc(vecreduce(x), vecreduce(y)) -> vecreduce(Opc(x, y))
1312// Note that we only expect Flags to be passed from FP operations. For integer
1313// operations they need to be dropped.
1314SDValue DAGCombiner::reassociateReduction(unsigned RedOpc, unsigned Opc,
1315 const SDLoc &DL, EVT VT, SDValue N0,
1316 SDValue N1, SDNodeFlags Flags) {
1317 if (N0.getOpcode() == RedOpc && N1.getOpcode() == RedOpc &&
1318 N0.getOperand(0).getValueType() == N1.getOperand(0).getValueType() &&
1319 N0->hasOneUse() && N1->hasOneUse() &&
1322 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
1323 return DAG.getNode(RedOpc, DL, VT,
1324 DAG.getNode(Opc, DL, N0.getOperand(0).getValueType(),
1325 N0.getOperand(0), N1.getOperand(0)));
1326 }
1327
1328 // Reassociate op(op(vecreduce(a), b), op(vecreduce(c), d)) into
1329 // op(vecreduce(op(a, c)), op(b, d)), to combine the reductions into a
1330 // single node.
1331 SDValue A, B, C, D, RedA, RedB;
1332 if (sd_match(N0,
1334 Opc, m_Value(RedA, m_OneUse(m_UnaryOp(RedOpc, m_Value(A)))),
1335 m_Value(B, m_Unless(m_UnaryOp(RedOpc, m_Value())))))) &&
1336 sd_match(N1,
1338 Opc, m_Value(RedB, m_OneUse(m_UnaryOp(RedOpc, m_Value(C)))),
1339 m_Value(D, m_Unless(m_UnaryOp(RedOpc, m_Value())))))) &&
1340 A.getValueType() == C.getValueType() &&
1341 hasOperation(Opc, A.getValueType()) &&
1342 TLI.shouldReassociateReduction(RedOpc, VT)) {
1343 if ((Opc == ISD::FADD || Opc == ISD::FMUL) &&
1344 (!N0->getFlags().hasAllowReassociation() ||
1346 !RedA->getFlags().hasAllowReassociation() ||
1347 !RedB->getFlags().hasAllowReassociation()))
1348 return SDValue();
1349 SelectionDAG::FlagInserter FlagsInserter(
1350 DAG, Flags & N0->getFlags() & N1->getFlags() & RedA->getFlags() &
1351 RedB->getFlags());
1352 SDValue Op = DAG.getNode(Opc, DL, A.getValueType(), A, C);
1353 SDValue Red = DAG.getNode(RedOpc, DL, VT, Op);
1354 SDValue Op2 = DAG.getNode(Opc, DL, VT, B, D);
1355 return DAG.getNode(Opc, DL, VT, Red, Op2);
1356 }
1357
1358 // Reassociate a reduction chain so two reductions become adjacent and the
1359 // folds above can merge them:
1360 // op(vecreduce(X), op(vecreduce(Y), Z))
1361 // -> op(vecreduce(op(X, Y)), Z)
1362 // Applied to fixpoint by the combiner worklist, this collapses an
1363 // arbitrarily long chain of reductions (such as the left-leaning chain SLP
1364 // emits) into a single reduction.
1365 auto FoldReductionChain = [&](SDValue Red0, SDValue Chain) -> SDValue {
1366 SDValue X, Y, Z, RedY;
1367 if (!sd_match(Red0, m_OneUse(m_UnaryOp(RedOpc, m_Value(X)))) ||
1368 !sd_match(
1369 Chain,
1371 Opc, m_Value(RedY, m_OneUse(m_UnaryOp(RedOpc, m_Value(Y)))),
1372 m_Value(Z, m_Unless(m_UnaryOp(RedOpc, m_Value())))))) ||
1373 X.getValueType() != Y.getValueType() ||
1374 !hasOperation(Opc, X.getValueType()) ||
1375 !TLI.shouldReassociateReduction(RedOpc, VT))
1376 return SDValue();
1377 if ((Opc == ISD::FADD || Opc == ISD::FMUL) &&
1378 (!Chain->getFlags().hasAllowReassociation() ||
1379 !Red0->getFlags().hasAllowReassociation() ||
1380 !RedY->getFlags().hasAllowReassociation()))
1381 return SDValue();
1382 SelectionDAG::FlagInserter FlagsInserter(
1383 DAG, Flags & Chain->getFlags() & Red0->getFlags() & RedY->getFlags());
1384 SDValue Op = DAG.getNode(Opc, DL, X.getValueType(), X, Y);
1385 SDValue Red = DAG.getNode(RedOpc, DL, VT, Op);
1386 return DAG.getNode(Opc, DL, VT, Red, Z);
1387 };
1388 if (SDValue V = FoldReductionChain(N0, N1))
1389 return V;
1390 if (SDValue V = FoldReductionChain(N1, N0))
1391 return V;
1392
1393 return SDValue();
1394}
1395
1396SDValue DAGCombiner::CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
1397 bool AddTo) {
1398 assert(N->getNumValues() == NumTo && "Broken CombineTo call!");
1399 ++NodesCombined;
1400 LLVM_DEBUG(dbgs() << "\nReplacing.1 "; N->dump(&DAG); dbgs() << "\nWith: ";
1401 To[0].dump(&DAG);
1402 dbgs() << " and " << NumTo - 1 << " other values\n");
1403 for (unsigned i = 0, e = NumTo; i != e; ++i)
1404 assert((!To[i].getNode() ||
1405 N->getValueType(i) == To[i].getValueType()) &&
1406 "Cannot combine value to value of different type!");
1407
1408 WorklistRemover DeadNodes(*this);
1409 DAG.ReplaceAllUsesWith(N, To);
1410 if (AddTo) {
1411 // Push the new nodes and any users onto the worklist
1412 for (unsigned i = 0, e = NumTo; i != e; ++i) {
1413 if (To[i].getNode())
1414 AddToWorklistWithUsers(To[i].getNode());
1415 }
1416 }
1417
1418 // Finally, if the node is now dead, remove it from the graph. The node
1419 // may not be dead if the replacement process recursively simplified to
1420 // something else needing this node.
1421 if (N->use_empty())
1422 deleteAndRecombine(N);
1423 return SDValue(N, 0);
1424}
1425
1426void DAGCombiner::
1427CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
1428 // Replace the old value with the new one.
1429 ++NodesCombined;
1430 LLVM_DEBUG(dbgs() << "\nReplacing.2 "; TLO.Old.dump(&DAG);
1431 dbgs() << "\nWith: "; TLO.New.dump(&DAG); dbgs() << '\n');
1432
1433 // Replace all uses.
1434 DAG.ReplaceAllUsesOfValueWith(TLO.Old, TLO.New);
1435
1436 // Push the new node and any (possibly new) users onto the worklist.
1437 AddToWorklistWithUsers(TLO.New.getNode());
1438
1439 // Finally, if the node is now dead, remove it from the graph.
1440 recursivelyDeleteUnusedNodes(TLO.Old.getNode());
1441}
1442
1443/// Check the specified integer node value to see if it can be simplified or if
1444/// things it uses can be simplified by bit propagation. If so, return true.
1445bool DAGCombiner::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
1446 const APInt &DemandedElts,
1447 bool AssumeSingleUse) {
1448 TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations);
1449 KnownBits Known;
1450 if (!TLI.SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO, 0,
1451 AssumeSingleUse))
1452 return false;
1453
1454 // Revisit the node.
1455 AddToWorklist(Op.getNode());
1456
1457 CommitTargetLoweringOpt(TLO);
1458 return true;
1459}
1460
1461/// Check the specified vector node value to see if it can be simplified or
1462/// if things it uses can be simplified as it only uses some of the elements.
1463/// If so, return true.
1464bool DAGCombiner::SimplifyDemandedVectorElts(SDValue Op,
1465 const APInt &DemandedElts,
1466 bool AssumeSingleUse) {
1467 TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations);
1468 APInt KnownUndef, KnownZero;
1469 if (!TLI.SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero,
1470 TLO, 0, AssumeSingleUse))
1471 return false;
1472
1473 // Revisit the node.
1474 AddToWorklist(Op.getNode());
1475
1476 CommitTargetLoweringOpt(TLO);
1477 return true;
1478}
1479
1480void DAGCombiner::ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad) {
1481 SDLoc DL(Load);
1482 EVT VT = Load->getValueType(0);
1483 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, VT, SDValue(ExtLoad, 0));
1484
1485 LLVM_DEBUG(dbgs() << "\nReplacing.9 "; Load->dump(&DAG); dbgs() << "\nWith: ";
1486 Trunc.dump(&DAG); dbgs() << '\n');
1487
1488 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), Trunc);
1489 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), SDValue(ExtLoad, 1));
1490
1491 AddToWorklist(Trunc.getNode());
1492 recursivelyDeleteUnusedNodes(Load);
1493}
1494
1495SDValue DAGCombiner::PromoteOperand(SDValue Op, EVT PVT, bool &Replace) {
1496 Replace = false;
1497 SDLoc DL(Op);
1498 if (ISD::isUNINDEXEDLoad(Op.getNode())) {
1499 LoadSDNode *LD = cast<LoadSDNode>(Op);
1500 EVT MemVT = LD->getMemoryVT();
1502 : LD->getExtensionType();
1503 Replace = true;
1504 return DAG.getExtLoad(ExtType, DL, PVT,
1505 LD->getChain(), LD->getBasePtr(),
1506 MemVT, LD->getMemOperand());
1507 }
1508
1509 unsigned Opc = Op.getOpcode();
1510 switch (Opc) {
1511 default: break;
1512 case ISD::AssertSext:
1513 if (SDValue Op0 = SExtPromoteOperand(Op.getOperand(0), PVT))
1514 return DAG.getNode(ISD::AssertSext, DL, PVT, Op0, Op.getOperand(1));
1515 break;
1516 case ISD::AssertZext:
1517 if (SDValue Op0 = ZExtPromoteOperand(Op.getOperand(0), PVT))
1518 return DAG.getNode(ISD::AssertZext, DL, PVT, Op0, Op.getOperand(1));
1519 break;
1520 case ISD::Constant: {
1521 unsigned ExtOpc =
1522 Op.getValueType().isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
1523 return DAG.getNode(ExtOpc, DL, PVT, Op);
1524 }
1525 }
1526
1528 return SDValue();
1529 return DAG.getNode(ISD::ANY_EXTEND, DL, PVT, Op);
1530}
1531
1532SDValue DAGCombiner::SExtPromoteOperand(SDValue Op, EVT PVT) {
1534 return SDValue();
1535 EVT OldVT = Op.getValueType();
1536 SDLoc DL(Op);
1537 bool Replace = false;
1538 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
1539 if (!NewOp.getNode())
1540 return SDValue();
1541 AddToWorklist(NewOp.getNode());
1542
1543 if (Replace)
1544 ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
1545 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, NewOp.getValueType(), NewOp,
1546 DAG.getValueType(OldVT));
1547}
1548
1549SDValue DAGCombiner::ZExtPromoteOperand(SDValue Op, EVT PVT) {
1550 EVT OldVT = Op.getValueType();
1551 SDLoc DL(Op);
1552 bool Replace = false;
1553 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
1554 if (!NewOp.getNode())
1555 return SDValue();
1556 AddToWorklist(NewOp.getNode());
1557
1558 if (Replace)
1559 ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
1560 return DAG.getZeroExtendInReg(NewOp, DL, OldVT);
1561}
1562
1563/// Promote the specified integer binary operation if the target indicates it is
1564/// beneficial. e.g. On x86, it's usually better to promote i16 operations to
1565/// i32 since i16 instructions are longer.
1566SDValue DAGCombiner::PromoteIntBinOp(SDValue Op) {
1567 if (!LegalOperations)
1568 return SDValue();
1569
1570 EVT VT = Op.getValueType();
1571 if (VT.isVector() || !VT.isInteger())
1572 return SDValue();
1573
1574 // If operation type is 'undesirable', e.g. i16 on x86, consider
1575 // promoting it.
1576 unsigned Opc = Op.getOpcode();
1577 if (TLI.isTypeDesirableForOp(Opc, VT))
1578 return SDValue();
1579
1580 EVT PVT = VT;
1581 // Consult target whether it is a good idea to promote this operation and
1582 // what's the right type to promote it to.
1583 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1584 assert(PVT != VT && "Don't know what type to promote to!");
1585
1586 LLVM_DEBUG(dbgs() << "\nPromoting "; Op.dump(&DAG));
1587
1588 bool Replace0 = false;
1589 SDValue N0 = Op.getOperand(0);
1590 SDValue NN0 = PromoteOperand(N0, PVT, Replace0);
1591
1592 bool Replace1 = false;
1593 SDValue N1 = Op.getOperand(1);
1594 SDValue NN1 = PromoteOperand(N1, PVT, Replace1);
1595 SDLoc DL(Op);
1596
1597 SDValue RV =
1598 DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, NN0, NN1));
1599
1600 // We are always replacing N0/N1's use in N and only need additional
1601 // replacements if there are additional uses.
1602 // Note: We are checking uses of the *nodes* (SDNode) rather than values
1603 // (SDValue) here because the node may reference multiple values
1604 // (for example, the chain value of a load node).
1605 Replace0 &= !N0->hasOneUse();
1606 Replace1 &= (N0 != N1) && !N1->hasOneUse();
1607
1608 // Combine Op here so it is preserved past replacements.
1609 CombineTo(Op.getNode(), RV);
1610
1611 // If operands have a use ordering, make sure we deal with
1612 // predecessor first.
1613 if (Replace0 && Replace1 && N0->isPredecessorOf(N1.getNode())) {
1614 std::swap(N0, N1);
1615 std::swap(NN0, NN1);
1616 }
1617
1618 if (Replace0) {
1619 AddToWorklist(NN0.getNode());
1620 ReplaceLoadWithPromotedLoad(N0.getNode(), NN0.getNode());
1621 }
1622 if (Replace1) {
1623 AddToWorklist(NN1.getNode());
1624 ReplaceLoadWithPromotedLoad(N1.getNode(), NN1.getNode());
1625 }
1626 return Op;
1627 }
1628 return SDValue();
1629}
1630
1631/// Promote the specified integer shift operation if the target indicates it is
1632/// beneficial. e.g. On x86, it's usually better to promote i16 operations to
1633/// i32 since i16 instructions are longer.
1634SDValue DAGCombiner::PromoteIntShiftOp(SDValue Op) {
1635 if (!LegalOperations)
1636 return SDValue();
1637
1638 EVT VT = Op.getValueType();
1639 if (VT.isVector() || !VT.isInteger())
1640 return SDValue();
1641
1642 // If operation type is 'undesirable', e.g. i16 on x86, consider
1643 // promoting it.
1644 unsigned Opc = Op.getOpcode();
1645 if (TLI.isTypeDesirableForOp(Opc, VT))
1646 return SDValue();
1647
1648 EVT PVT = VT;
1649 // Consult target whether it is a good idea to promote this operation and
1650 // what's the right type to promote it to.
1651 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1652 assert(PVT != VT && "Don't know what type to promote to!");
1653
1654 LLVM_DEBUG(dbgs() << "\nPromoting "; Op.dump(&DAG));
1655
1656 SDNodeFlags TruncFlags;
1657 bool Replace = false;
1658 SDValue N0 = Op.getOperand(0);
1659 if (Opc == ISD::SRA) {
1660 N0 = SExtPromoteOperand(N0, PVT);
1661 } else if (Opc == ISD::SRL) {
1662 N0 = ZExtPromoteOperand(N0, PVT);
1663 } else {
1664 if (Op->getFlags().hasNoUnsignedWrap()) {
1665 N0 = ZExtPromoteOperand(N0, PVT);
1666 TruncFlags = SDNodeFlags::NoUnsignedWrap;
1667 } else if (Op->getFlags().hasNoSignedWrap()) {
1668 N0 = SExtPromoteOperand(N0, PVT);
1669 TruncFlags = SDNodeFlags::NoSignedWrap;
1670 } else {
1671 N0 = PromoteOperand(N0, PVT, Replace);
1672 }
1673 }
1674
1675 if (!N0.getNode())
1676 return SDValue();
1677
1678 SDLoc DL(Op);
1679 SDValue N1 = Op.getOperand(1);
1680 SDValue RV = DAG.getNode(ISD::TRUNCATE, DL, VT,
1681 DAG.getNode(Opc, DL, PVT, N0, N1), TruncFlags);
1682
1683 if (Replace)
1684 ReplaceLoadWithPromotedLoad(Op.getOperand(0).getNode(), N0.getNode());
1685
1686 // Deal with Op being deleted.
1687 if (Op && Op.getOpcode() != ISD::DELETED_NODE)
1688 return RV;
1689 }
1690 return SDValue();
1691}
1692
1693SDValue DAGCombiner::PromoteExtend(SDValue Op) {
1694 if (!LegalOperations)
1695 return SDValue();
1696
1697 EVT VT = Op.getValueType();
1698 if (VT.isVector() || !VT.isInteger())
1699 return SDValue();
1700
1701 // If operation type is 'undesirable', e.g. i16 on x86, consider
1702 // promoting it.
1703 unsigned Opc = Op.getOpcode();
1704 if (TLI.isTypeDesirableForOp(Opc, VT))
1705 return SDValue();
1706
1707 EVT PVT = VT;
1708 // Consult target whether it is a good idea to promote this operation and
1709 // what's the right type to promote it to.
1710 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1711 assert(PVT != VT && "Don't know what type to promote to!");
1712 // fold (aext (aext x)) -> (aext x)
1713 // fold (aext (zext x)) -> (zext x)
1714 // fold (aext (sext x)) -> (sext x)
1715 LLVM_DEBUG(dbgs() << "\nPromoting "; Op.dump(&DAG));
1716 return DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, Op.getOperand(0));
1717 }
1718 return SDValue();
1719}
1720
1721bool DAGCombiner::PromoteLoad(SDValue Op) {
1722 if (!LegalOperations)
1723 return false;
1724
1725 if (!ISD::isUNINDEXEDLoad(Op.getNode()))
1726 return false;
1727
1728 EVT VT = Op.getValueType();
1729 if (VT.isVector() || !VT.isInteger())
1730 return false;
1731
1732 // If operation type is 'undesirable', e.g. i16 on x86, consider
1733 // promoting it.
1734 unsigned Opc = Op.getOpcode();
1735 if (TLI.isTypeDesirableForOp(Opc, VT))
1736 return false;
1737
1738 EVT PVT = VT;
1739 // Consult target whether it is a good idea to promote this operation and
1740 // what's the right type to promote it to.
1741 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1742 assert(PVT != VT && "Don't know what type to promote to!");
1743
1744 SDLoc DL(Op);
1745 SDNode *N = Op.getNode();
1746 LoadSDNode *LD = cast<LoadSDNode>(N);
1747 EVT MemVT = LD->getMemoryVT();
1749 : LD->getExtensionType();
1750 SDValue NewLD = DAG.getExtLoad(ExtType, DL, PVT,
1751 LD->getChain(), LD->getBasePtr(),
1752 MemVT, LD->getMemOperand());
1753 SDValue Result = DAG.getNode(ISD::TRUNCATE, DL, VT, NewLD);
1754
1755 LLVM_DEBUG(dbgs() << "\nPromoting "; N->dump(&DAG); dbgs() << "\nTo: ";
1756 Result.dump(&DAG); dbgs() << '\n');
1757
1758 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result);
1759 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), NewLD.getValue(1));
1760
1761 AddToWorklist(Result.getNode());
1762 recursivelyDeleteUnusedNodes(N);
1763 return true;
1764 }
1765
1766 return false;
1767}
1768
1769/// Recursively delete a node which has no uses and any operands for
1770/// which it is the only use.
1771///
1772/// Note that this both deletes the nodes and removes them from the worklist.
1773/// It also adds any nodes who have had a user deleted to the worklist as they
1774/// may now have only one use and subject to other combines.
1775bool DAGCombiner::recursivelyDeleteUnusedNodes(SDNode *N) {
1776 if (!N->use_empty())
1777 return false;
1778
1779 SmallSetVector<SDNode *, 16> Nodes;
1780 Nodes.insert(N);
1781 do {
1782 N = Nodes.pop_back_val();
1783 if (!N)
1784 continue;
1785
1786 if (N->use_empty()) {
1787 for (const SDValue &ChildN : N->op_values())
1788 Nodes.insert(ChildN.getNode());
1789
1790 removeFromWorklist(N);
1791 DAG.DeleteNode(N);
1792 } else {
1793 AddToWorklist(N);
1794 }
1795 } while (!Nodes.empty());
1796 return true;
1797}
1798
1799//===----------------------------------------------------------------------===//
1800// Main DAG Combiner implementation
1801//===----------------------------------------------------------------------===//
1802
1803void DAGCombiner::Run(CombineLevel AtLevel) {
1804 // set the instance variables, so that the various visit routines may use it.
1805 Level = AtLevel;
1806 LegalDAG = Level >= AfterLegalizeDAG;
1807 LegalOperations = Level >= AfterLegalizeVectorOps;
1808 LegalTypes = Level >= AfterLegalizeTypes;
1809
1810 const SelectionDAGOptions &Opts = DAG.getSelectionDAGOptions();
1811 bool UseTopologicalSorting =
1812 valueOr(Opts.combiner_topological_sorting, TLI.useTopologicalSorting());
1813
1814 WorklistInserter AddNodes(*this);
1815
1816 if (UseTopologicalSorting)
1818
1819 // Add all the dag nodes to the worklist.
1820 //
1821 // Note: All nodes are not added to PruningList here, this is because the only
1822 // nodes which can be deleted are those which have no uses and all other nodes
1823 // which would otherwise be added to the worklist by the first call to
1824 // getNextWorklistEntry are already present in it.
1825 if (UseTopologicalSorting) {
1826 for (SDNode &Node : reverse(DAG.allnodes()))
1827 AddToWorklist(&Node, /* IsCandidateForPruning */ Node.use_empty());
1828 } else {
1829 for (SDNode &Node : DAG.allnodes())
1830 AddToWorklist(&Node, /* IsCandidateForPruning */ Node.use_empty());
1831 }
1832
1833 // Create a dummy node (which is not added to allnodes), that adds a reference
1834 // to the root node, preventing it from being deleted, and tracking any
1835 // changes of the root.
1836 HandleSDNode Dummy(DAG.getRoot());
1837
1838 // While we have a valid worklist entry node, try to combine it.
1839 while (SDNode *N = getNextWorklistEntry()) {
1840 // If N has no uses, it is dead. Make sure to revisit all N's operands once
1841 // N is deleted from the DAG, since they too may now be dead or may have a
1842 // reduced number of uses, allowing other xforms.
1843 if (recursivelyDeleteUnusedNodes(N))
1844 continue;
1845
1846 WorklistRemover DeadNodes(*this);
1847
1848 // If this combine is running after legalizing the DAG, re-legalize any
1849 // nodes pulled off the worklist.
1850 if (LegalDAG) {
1851 SmallSetVector<SDNode *, 16> UpdatedNodes;
1852 bool NIsValid = DAG.LegalizeOp(N, UpdatedNodes);
1853
1854 for (SDNode *LN : UpdatedNodes)
1855 AddToWorklistWithUsers(LN);
1856
1857 if (!NIsValid)
1858 continue;
1859 }
1860
1861 LLVM_DEBUG(dbgs() << "\nCombining: "; N->dump(&DAG));
1862
1863 // Add any operands of the new node which have not yet been combined to the
1864 // worklist as well. getNextWorklistEntry flags nodes that have been
1865 // combined before. Because the worklist uniques things already, this won't
1866 // repeatedly process the same operand.
1867 for (const SDValue &ChildN : N->op_values())
1868 AddToWorklist(ChildN.getNode(), /*IsCandidateForPruning=*/true,
1869 /*SkipIfCombinedBefore=*/true);
1870
1871 SDValue RV = combine(N);
1872
1873 if (!RV.getNode())
1874 continue;
1875
1876 ++NodesCombined;
1877
1878 // Invalidate cached info.
1879 ChainsWithoutMergeableStores.clear();
1880
1881 // If we get back the same node we passed in, rather than a new node or
1882 // zero, we know that the node must have defined multiple values and
1883 // CombineTo was used. Since CombineTo takes care of the worklist
1884 // mechanics for us, we have no work to do in this case.
1885 if (RV.getNode() == N)
1886 continue;
1887
1888 assert(N->getOpcode() != ISD::DELETED_NODE &&
1889 RV.getOpcode() != ISD::DELETED_NODE &&
1890 "Node was deleted but visit returned new node!");
1891
1892 LLVM_DEBUG(dbgs() << " ... into: "; RV.dump(&DAG));
1893
1894 if (N->getNumValues() == RV->getNumValues())
1895 DAG.ReplaceAllUsesWith(N, RV.getNode());
1896 else {
1897 assert(N->getValueType(0) == RV.getValueType() &&
1898 N->getNumValues() == 1 && "Type mismatch");
1899 DAG.ReplaceAllUsesWith(N, &RV);
1900 }
1901
1902 // Push the new node and any users onto the worklist. Omit this if the
1903 // new node is the EntryToken (e.g. if a store managed to get optimized
1904 // out), because re-visiting the EntryToken and its users will not uncover
1905 // any additional opportunities, but there may be a large number of such
1906 // users, potentially causing compile time explosion.
1907 if (RV.getOpcode() != ISD::EntryToken)
1908 AddToWorklistWithUsers(RV.getNode());
1909
1910 // Finally, if the node is now dead, remove it from the graph. The node
1911 // may not be dead if the replacement process recursively simplified to
1912 // something else needing this node. This will also take care of adding any
1913 // operands which have lost a user to the worklist.
1914 recursivelyDeleteUnusedNodes(N);
1915 }
1916
1917 // If the root changed (e.g. it was a dead load, update the root).
1918 DAG.setRoot(Dummy.getValue());
1919 DAG.RemoveDeadNodes();
1920}
1921
1922SDValue DAGCombiner::visit(SDNode *N) {
1923 // clang-format off
1924 switch (N->getOpcode()) {
1925 default: break;
1926 case ISD::TokenFactor: return visitTokenFactor(N);
1927 case ISD::GET_ROUNDING: return visitGET_ROUNDING(N);
1928 case ISD::MERGE_VALUES: return visitMERGE_VALUES(N);
1929 case ISD::ADD: return visitADD(N);
1930 case ISD::PTRADD: return visitPTRADD(N);
1931 case ISD::SUB: return visitSUB(N);
1932 case ISD::SADDSAT:
1933 case ISD::UADDSAT: return visitADDSAT(N);
1934 case ISD::SSUBSAT:
1935 case ISD::USUBSAT: return visitSUBSAT(N);
1936 case ISD::ADDC: return visitADDC(N);
1937 case ISD::SADDO:
1938 case ISD::UADDO: return visitADDO(N);
1939 case ISD::SUBC: return visitSUBC(N);
1940 case ISD::SSUBO:
1941 case ISD::USUBO: return visitSUBO(N);
1942 case ISD::ADDE: return visitADDE(N);
1943 case ISD::UADDO_CARRY: return visitUADDO_CARRY(N);
1944 case ISD::SADDO_CARRY: return visitSADDO_CARRY(N);
1945 case ISD::SUBE: return visitSUBE(N);
1946 case ISD::USUBO_CARRY: return visitUSUBO_CARRY(N);
1947 case ISD::SSUBO_CARRY: return visitSSUBO_CARRY(N);
1948 case ISD::SMULFIX:
1949 case ISD::SMULFIXSAT:
1950 case ISD::UMULFIX:
1951 case ISD::UMULFIXSAT: return visitMULFIX(N);
1952 case ISD::MUL: return visitMUL(N);
1953 case ISD::SDIV: return visitSDIV(N);
1954 case ISD::UDIV: return visitUDIV(N);
1955 case ISD::SREM:
1956 case ISD::UREM: return visitREM(N);
1957 case ISD::MULHU: return visitMULHU(N);
1958 case ISD::MULHS: return visitMULHS(N);
1959 case ISD::AVGFLOORS:
1960 case ISD::AVGFLOORU:
1961 case ISD::AVGCEILS:
1962 case ISD::AVGCEILU: return visitAVG(N);
1963 case ISD::AVGSUBFLOORS:
1964 case ISD::AVGSUBFLOORU: return visitAVGSUB(N);
1965 case ISD::ABDS:
1966 case ISD::ABDU: return visitABD(N);
1967 case ISD::SMUL_LOHI: return visitSMUL_LOHI(N);
1968 case ISD::UMUL_LOHI: return visitUMUL_LOHI(N);
1969 case ISD::SMULO:
1970 case ISD::UMULO: return visitMULO(N);
1971 case ISD::SMIN:
1972 case ISD::SMAX:
1973 case ISD::UMIN:
1974 case ISD::UMAX: return visitIMINMAX(N);
1975 case ISD::AND: return visitAND(N);
1976 case ISD::OR: return visitOR(N);
1977 case ISD::XOR: return visitXOR(N);
1978 case ISD::SHL: return visitSHL(N);
1979 case ISD::SRA: return visitSRA(N);
1980 case ISD::SRL: return visitSRL(N);
1981 case ISD::ROTR:
1982 case ISD::ROTL: return visitRotate(N);
1983 case ISD::FSHL:
1984 case ISD::FSHR: return visitFunnelShift(N);
1985 case ISD::SSHLSAT:
1986 case ISD::USHLSAT: return visitSHLSAT(N);
1987 case ISD::ABS: return visitABS(N);
1988 case ISD::ABS_MIN_POISON: return visitABS_MIN_POISON(N);
1989 case ISD::CLMUL:
1990 case ISD::CLMULR:
1991 case ISD::CLMULH: return visitCLMUL(N);
1992 case ISD::PEXT: return visitPEXT(N);
1993 case ISD::PDEP: return visitPDEP(N);
1994 case ISD::BSWAP: return visitBSWAP(N);
1995 case ISD::BITREVERSE: return visitBITREVERSE(N);
1996 case ISD::CTLZ: return visitCTLZ(N);
1997 case ISD::CTLZ_ZERO_POISON: return visitCTLZ_ZERO_POISON(N);
1998 case ISD::CTTZ: return visitCTTZ(N);
1999 case ISD::CTTZ_ZERO_POISON: return visitCTTZ_ZERO_POISON(N);
2000 case ISD::CTPOP: return visitCTPOP(N);
2001 case ISD::PARITY: return visitPARITY(N);
2002 case ISD::SELECT: return visitSELECT(N);
2003 case ISD::VSELECT: return visitVSELECT(N);
2004 case ISD::SELECT_CC: return visitSELECT_CC(N);
2005 case ISD::SETCC: return visitSETCC(N);
2006 case ISD::SETCCCARRY: return visitSETCCCARRY(N);
2007 case ISD::SIGN_EXTEND: return visitSIGN_EXTEND(N);
2008 case ISD::ZERO_EXTEND: return visitZERO_EXTEND(N);
2009 case ISD::ANY_EXTEND: return visitANY_EXTEND(N);
2010 case ISD::AssertSext:
2011 case ISD::AssertZext: return visitAssertExt(N);
2012 case ISD::AssertAlign: return visitAssertAlign(N);
2013 case ISD::IS_FPCLASS: return visitIS_FPCLASS(N);
2014 case ISD::SIGN_EXTEND_INREG: return visitSIGN_EXTEND_INREG(N);
2017 case ISD::ANY_EXTEND_VECTOR_INREG: return visitEXTEND_VECTOR_INREG(N);
2018 case ISD::TRUNCATE: return visitTRUNCATE(N);
2019 case ISD::TRUNCATE_USAT_U: return visitTRUNCATE_USAT_U(N);
2020 case ISD::BITCAST: return visitBITCAST(N);
2021 case ISD::BUILD_PAIR: return visitBUILD_PAIR(N);
2022 case ISD::FADD: return visitFADD(N);
2023 case ISD::STRICT_FADD: return visitSTRICT_FADD(N);
2024 case ISD::FSUB: return visitFSUB(N);
2025 case ISD::FMUL: return visitFMUL(N);
2026 case ISD::FMA: return visitFMA(N);
2027 case ISD::FMAD: return visitFMAD(N);
2028 case ISD::FMULADD: return visitFMULADD(N);
2029 case ISD::FDIV: return visitFDIV(N);
2030 case ISD::FREM: return visitFREM(N);
2031 case ISD::FSQRT: return visitFSQRT(N);
2032 case ISD::FCOPYSIGN: return visitFCOPYSIGN(N);
2033 case ISD::FPOW: return visitFPOW(N);
2034 case ISD::SINT_TO_FP: return visitSINT_TO_FP(N);
2035 case ISD::UINT_TO_FP: return visitUINT_TO_FP(N);
2036 case ISD::FP_TO_SINT: return visitFP_TO_SINT(N);
2037 case ISD::FP_TO_UINT: return visitFP_TO_UINT(N);
2039 case ISD::FP_TO_UINT_SAT: return visitFP_TO_INT_SAT(N);
2040 case ISD::LROUND:
2041 case ISD::LLROUND:
2042 case ISD::LRINT:
2043 case ISD::LLRINT: return visitXROUND(N);
2044 case ISD::FP_ROUND: return visitFP_ROUND(N);
2045 case ISD::FP_EXTEND: return visitFP_EXTEND(N);
2046 case ISD::FNEG: return visitFNEG(N);
2047 case ISD::FABS: return visitFABS(N);
2048 case ISD::FFLOOR: return visitFFLOOR(N);
2049 case ISD::FMINNUM:
2050 case ISD::FMAXNUM:
2051 case ISD::FMINIMUM:
2052 case ISD::FMAXIMUM:
2053 case ISD::FMINIMUMNUM:
2054 case ISD::FMAXIMUMNUM: return visitFMinMax(N);
2055 case ISD::FCEIL: return visitFCEIL(N);
2056 case ISD::FTRUNC: return visitFTRUNC(N);
2057 case ISD::FFREXP: return visitFFREXP(N);
2058 case ISD::BRCOND: return visitBRCOND(N);
2059 case ISD::BR_CC: return visitBR_CC(N);
2060 case ISD::LOAD: return visitLOAD(N);
2061 case ISD::STORE: return visitSTORE(N);
2062 case ISD::ATOMIC_STORE: return visitATOMIC_STORE(N);
2063 case ISD::INSERT_VECTOR_ELT: return visitINSERT_VECTOR_ELT(N);
2064 case ISD::EXTRACT_VECTOR_ELT: return visitEXTRACT_VECTOR_ELT(N);
2065 case ISD::BUILD_VECTOR: return visitBUILD_VECTOR(N);
2066 case ISD::CONCAT_VECTORS: return visitCONCAT_VECTORS(N);
2067 case ISD::VECTOR_INTERLEAVE: return visitVECTOR_INTERLEAVE(N);
2068 case ISD::VECTOR_DEINTERLEAVE: return visitVECTOR_DEINTERLEAVE(N);
2069 case ISD::EXTRACT_SUBVECTOR: return visitEXTRACT_SUBVECTOR(N);
2070 case ISD::VECTOR_SHUFFLE: return visitVECTOR_SHUFFLE(N);
2071 case ISD::SCALAR_TO_VECTOR: return visitSCALAR_TO_VECTOR(N);
2072 case ISD::INSERT_SUBVECTOR: return visitINSERT_SUBVECTOR(N);
2073 case ISD::MGATHER: return visitMGATHER(N);
2074 case ISD::MLOAD: return visitMLOAD(N);
2075 case ISD::MSCATTER: return visitMSCATTER(N);
2076 case ISD::MSTORE: return visitMSTORE(N);
2077 case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM: return visitMHISTOGRAM(N);
2082 return visitPARTIAL_REDUCE_MLA(N);
2085 return visitLOOP_DEPENDENCE_MASK(N);
2086 case ISD::VECTOR_COMPRESS: return visitVECTOR_COMPRESS(N);
2087 case ISD::LIFETIME_END: return visitLIFETIME_END(N);
2088 case ISD::FP_TO_FP16: return visitFP_TO_FP16(N);
2089 case ISD::FP16_TO_FP: return visitFP16_TO_FP(N);
2090 case ISD::FP_TO_BF16: return visitFP_TO_BF16(N);
2091 case ISD::BF16_TO_FP: return visitBF16_TO_FP(N);
2092 case ISD::FREEZE: return visitFREEZE(N);
2093 case ISD::GET_FPENV_MEM: return visitGET_FPENV_MEM(N);
2094 case ISD::SET_FPENV_MEM: return visitSET_FPENV_MEM(N);
2095 case ISD::FCANONICALIZE: return visitFCANONICALIZE(N);
2098 case ISD::VECREDUCE_ADD:
2099 case ISD::VECREDUCE_MUL:
2100 case ISD::VECREDUCE_AND:
2101 case ISD::VECREDUCE_OR:
2102 case ISD::VECREDUCE_XOR:
2112 case ISD::VECREDUCE_FMINIMUMNUM: return visitVECREDUCE(N);
2113#define BEGIN_REGISTER_VP_SDNODE(SDOPC, ...) case ISD::SDOPC:
2114#include "llvm/IR/VPIntrinsics.def"
2115 return visitVPOp(N);
2116 }
2117 // clang-format on
2118 return SDValue();
2119}
2120
2121SDValue DAGCombiner::combine(SDNode *N) {
2122 if (!DebugCounter::shouldExecute(DAGCombineCounter))
2123 return SDValue();
2124
2125 SDValue RV;
2126 if (!DisableGenericCombines)
2127 RV = visit(N);
2128
2129 // If nothing happened, try a target-specific DAG combine.
2130 if (!RV.getNode()) {
2131 assert(N->getOpcode() != ISD::DELETED_NODE &&
2132 "Node was deleted but visit returned NULL!");
2133
2134 if (N->getOpcode() >= ISD::BUILTIN_OP_END ||
2135 TLI.hasTargetDAGCombine((ISD::NodeType)N->getOpcode())) {
2136
2137 // Expose the DAG combiner to the target combiner impls.
2138 TargetLowering::DAGCombinerInfo
2139 DagCombineInfo(DAG, Level, false, this);
2140
2141 RV = TLI.PerformDAGCombine(N, DagCombineInfo);
2142 }
2143 }
2144
2145 // If nothing happened still, try promoting the operation.
2146 if (!RV.getNode()) {
2147 switch (N->getOpcode()) {
2148 default: break;
2149 case ISD::ADD:
2150 case ISD::SUB:
2151 case ISD::MUL:
2152 case ISD::AND:
2153 case ISD::OR:
2154 case ISD::XOR:
2155 RV = PromoteIntBinOp(SDValue(N, 0));
2156 break;
2157 case ISD::SHL:
2158 case ISD::SRA:
2159 case ISD::SRL:
2160 RV = PromoteIntShiftOp(SDValue(N, 0));
2161 break;
2162 case ISD::SIGN_EXTEND:
2163 case ISD::ZERO_EXTEND:
2164 case ISD::ANY_EXTEND:
2165 RV = PromoteExtend(SDValue(N, 0));
2166 break;
2167 case ISD::LOAD:
2168 if (PromoteLoad(SDValue(N, 0)))
2169 RV = SDValue(N, 0);
2170 break;
2171 }
2172 }
2173
2174 // If N is a commutative binary node, try to eliminate it if the commuted
2175 // version is already present in the DAG.
2176 if (!RV.getNode() && TLI.isCommutativeBinOp(N->getOpcode())) {
2177 SDValue N0 = N->getOperand(0);
2178 SDValue N1 = N->getOperand(1);
2179
2180 // Constant operands are canonicalized to RHS.
2181 if (N0 != N1 && (isa<ConstantSDNode>(N0) || !isa<ConstantSDNode>(N1))) {
2182 SDValue Ops[] = {N1, N0};
2183 SDNode *CSENode = DAG.getNodeIfExists(N->getOpcode(), N->getVTList(), Ops,
2184 N->getFlags());
2185 if (CSENode)
2186 return SDValue(CSENode, 0);
2187 }
2188 }
2189
2190 return RV;
2191}
2192
2193/// Given a node, return its input chain if it has one, otherwise return a null
2194/// sd operand.
2196 if (unsigned NumOps = N->getNumOperands()) {
2197 if (N->getOperand(0).getValueType() == MVT::Other)
2198 return N->getOperand(0);
2199 if (N->getOperand(NumOps-1).getValueType() == MVT::Other)
2200 return N->getOperand(NumOps-1);
2201 for (unsigned i = 1; i < NumOps-1; ++i)
2202 if (N->getOperand(i).getValueType() == MVT::Other)
2203 return N->getOperand(i);
2204 }
2205 return SDValue();
2206}
2207
2208SDValue DAGCombiner::visitFCANONICALIZE(SDNode *N) {
2209 SDValue Operand = N->getOperand(0);
2210 EVT VT = Operand.getValueType();
2211 SDLoc dl(N);
2212
2213 // Canonicalize undef to quiet NaN.
2214 if (Operand.isUndef()) {
2215 APFloat CanonicalQNaN = APFloat::getQNaN(VT.getFltSemantics());
2216 return DAG.getConstantFP(CanonicalQNaN, dl, VT);
2217 }
2218 return SDValue();
2219}
2220
2221SDValue DAGCombiner::visitGET_ROUNDING(SDNode *N) {
2222 SDValue Chain = N->getOperand(0);
2223 if (Chain.getOpcode() != ISD::SET_ROUNDING)
2224 return SDValue();
2225
2226 auto *Mode = dyn_cast<ConstantSDNode>(Chain.getOperand(1));
2227 if (!Mode || Mode->getZExtValue() > 3)
2228 return SDValue();
2229
2230 // The standard rounding modes 0 through 3 round-trip through SET_ROUNDING.
2231 // Keep the setter, but avoid reading back the state it just established.
2232 return CombineTo(N, Chain.getOperand(1), Chain);
2233}
2234
2235SDValue DAGCombiner::visitTokenFactor(SDNode *N) {
2236 // If N has two operands, where one has an input chain equal to the other,
2237 // the 'other' chain is redundant.
2238 if (N->getNumOperands() == 2) {
2239 if (getInputChainForNode(N->getOperand(0).getNode()) == N->getOperand(1))
2240 return N->getOperand(0);
2241 if (getInputChainForNode(N->getOperand(1).getNode()) == N->getOperand(0))
2242 return N->getOperand(1);
2243 }
2244
2245 // Don't simplify token factors if optnone.
2246 if (OptLevel == CodeGenOptLevel::None)
2247 return SDValue();
2248
2249 const SelectionDAGOptions &Opts = DAG.getSelectionDAGOptions();
2250
2251 // Don't simplify the token factor if the node itself has too many operands.
2252 if (N->getNumOperands() > Opts.combiner_tokenfactor_inline_limit)
2253 return SDValue();
2254
2255 // If the sole user is a token factor, we should make sure we have a
2256 // chance to merge them together. This prevents TF chains from inhibiting
2257 // optimizations.
2258 if (N->hasOneUse() && N->user_begin()->getOpcode() == ISD::TokenFactor)
2259 AddToWorklist(*(N->user_begin()));
2260
2261 SmallVector<SDNode *, 8> TFs; // List of token factors to visit.
2262 SmallVector<SDValue, 8> Ops; // Ops for replacing token factor.
2263 SmallPtrSet<SDNode*, 16> SeenOps;
2264 bool Changed = false; // If we should replace this token factor.
2265
2266 // Start out with this token factor.
2267 TFs.push_back(N);
2268
2269 // Iterate through token factors. The TFs grows when new token factors are
2270 // encountered.
2271 for (unsigned i = 0; i < TFs.size(); ++i) {
2272 // Limit number of nodes to inline, to avoid quadratic compile times.
2273 // We have to add the outstanding Token Factors to Ops, otherwise we might
2274 // drop Ops from the resulting Token Factors.
2275 if (Ops.size() > Opts.combiner_tokenfactor_inline_limit) {
2276 for (unsigned j = i; j < TFs.size(); j++)
2277 Ops.emplace_back(TFs[j], 0);
2278 // Drop unprocessed Token Factors from TFs, so we do not add them to the
2279 // combiner worklist later.
2280 TFs.resize(i);
2281 break;
2282 }
2283
2284 SDNode *TF = TFs[i];
2285 // Check each of the operands.
2286 for (const SDValue &Op : TF->op_values()) {
2287 switch (Op.getOpcode()) {
2288 case ISD::EntryToken:
2289 // Entry tokens don't need to be added to the list. They are
2290 // redundant.
2291 Changed = true;
2292 break;
2293
2294 case ISD::TokenFactor:
2295 if (Op.hasOneUse() && !is_contained(TFs, Op.getNode())) {
2296 // Queue up for processing.
2297 TFs.push_back(Op.getNode());
2298 Changed = true;
2299 break;
2300 }
2301 [[fallthrough]];
2302
2303 default:
2304 // Only add if it isn't already in the list.
2305 if (SeenOps.insert(Op.getNode()).second)
2306 Ops.push_back(Op);
2307 else
2308 Changed = true;
2309 break;
2310 }
2311 }
2312 }
2313
2314 // Re-visit inlined Token Factors, to clean them up in case they have been
2315 // removed. Skip the first Token Factor, as this is the current node.
2316 for (unsigned i = 1, e = TFs.size(); i < e; i++)
2317 AddToWorklist(TFs[i]);
2318
2319 // Remove Nodes that are chained to another node in the list. Do so
2320 // by walking up chains breath-first stopping when we've seen
2321 // another operand. In general we must climb to the EntryNode, but we can exit
2322 // early if we find all remaining work is associated with just one operand as
2323 // no further pruning is possible.
2324
2325 // List of nodes to search through and original Ops from which they originate.
2327 SmallVector<unsigned, 8> OpWorkCount; // Count of work for each Op.
2328 SmallPtrSet<SDNode *, 16> SeenChains;
2329 bool DidPruneOps = false;
2330
2331 unsigned NumLeftToConsider = 0;
2332 for (const SDValue &Op : Ops) {
2333 Worklist.push_back(std::make_pair(Op.getNode(), NumLeftToConsider++));
2334 OpWorkCount.push_back(1);
2335 }
2336
2337 auto AddToWorklist = [&](unsigned CurIdx, SDNode *Op, unsigned OpNumber) {
2338 // If this is an Op, we can remove the op from the list. Remark any
2339 // search associated with it as from the current OpNumber.
2340 if (SeenOps.contains(Op)) {
2341 Changed = true;
2342 DidPruneOps = true;
2343 unsigned OrigOpNumber = 0;
2344 while (OrigOpNumber < Ops.size() && Ops[OrigOpNumber].getNode() != Op)
2345 OrigOpNumber++;
2346 assert((OrigOpNumber != Ops.size()) &&
2347 "expected to find TokenFactor Operand");
2348 // Re-mark worklist from OrigOpNumber to OpNumber
2349 for (unsigned i = CurIdx + 1; i < Worklist.size(); ++i) {
2350 if (Worklist[i].second == OrigOpNumber) {
2351 Worklist[i].second = OpNumber;
2352 }
2353 }
2354 OpWorkCount[OpNumber] += OpWorkCount[OrigOpNumber];
2355 OpWorkCount[OrigOpNumber] = 0;
2356 NumLeftToConsider--;
2357 }
2358 // Add if it's a new chain
2359 if (SeenChains.insert(Op).second) {
2360 OpWorkCount[OpNumber]++;
2361 Worklist.push_back(std::make_pair(Op, OpNumber));
2362 }
2363 };
2364
2365 for (unsigned i = 0; i < Worklist.size() && i < 1024; ++i) {
2366 // We need at least be consider at least 2 Ops to prune.
2367 if (NumLeftToConsider <= 1)
2368 break;
2369 auto CurNode = Worklist[i].first;
2370 auto CurOpNumber = Worklist[i].second;
2371 assert((OpWorkCount[CurOpNumber] > 0) &&
2372 "Node should not appear in worklist");
2373 switch (CurNode->getOpcode()) {
2374 case ISD::EntryToken:
2375 // Hitting EntryToken is the only way for the search to terminate without
2376 // hitting
2377 // another operand's search. Prevent us from marking this operand
2378 // considered.
2379 NumLeftToConsider++;
2380 break;
2381 case ISD::TokenFactor:
2382 for (const SDValue &Op : CurNode->op_values())
2383 AddToWorklist(i, Op.getNode(), CurOpNumber);
2384 break;
2386 case ISD::LIFETIME_END:
2387 case ISD::CopyFromReg:
2388 case ISD::CopyToReg:
2389 AddToWorklist(i, CurNode->getOperand(0).getNode(), CurOpNumber);
2390 break;
2391 default:
2392 if (auto *MemNode = dyn_cast<MemSDNode>(CurNode))
2393 AddToWorklist(i, MemNode->getChain().getNode(), CurOpNumber);
2394 break;
2395 }
2396 OpWorkCount[CurOpNumber]--;
2397 if (OpWorkCount[CurOpNumber] == 0)
2398 NumLeftToConsider--;
2399 }
2400
2401 // If we've changed things around then replace token factor.
2402 if (Changed) {
2404 if (Ops.empty()) {
2405 // The entry token is the only possible outcome.
2406 Result = DAG.getEntryNode();
2407 } else {
2408 if (DidPruneOps) {
2409 SmallVector<SDValue, 8> PrunedOps;
2410 //
2411 for (const SDValue &Op : Ops) {
2412 if (SeenChains.count(Op.getNode()) == 0)
2413 PrunedOps.push_back(Op);
2414 }
2415 Result = DAG.getTokenFactor(SDLoc(N), PrunedOps);
2416 } else {
2417 Result = DAG.getTokenFactor(SDLoc(N), Ops);
2418 }
2419 }
2420 return Result;
2421 }
2422 return SDValue();
2423}
2424
2425/// MERGE_VALUES can always be eliminated.
2426SDValue DAGCombiner::visitMERGE_VALUES(SDNode *N) {
2427 WorklistRemover DeadNodes(*this);
2428 // Replacing results may cause a different MERGE_VALUES to suddenly
2429 // be CSE'd with N, and carry its uses with it. Iterate until no
2430 // uses remain, to ensure that the node can be safely deleted.
2431 // First add the users of this node to the work list so that they
2432 // can be tried again once they have new operands.
2433 AddUsersToWorklist(N);
2434 do {
2435 // Do as a single replacement to avoid rewalking use lists.
2437 DAG.ReplaceAllUsesWith(N, Ops.data());
2438 } while (!N->use_empty());
2439 deleteAndRecombine(N);
2440 return SDValue(N, 0); // Return N so it doesn't get rechecked!
2441}
2442
2443/// If \p N is a ConstantSDNode with isOpaque() == false return it casted to a
2444/// ConstantSDNode pointer else nullptr.
2447 return Const != nullptr && !Const->isOpaque() ? Const : nullptr;
2448}
2449
2450// isTruncateOf - If N is a truncate of some other value, return true, record
2451// the value being truncated in Op and which of Op's bits are zero/one in Known.
2452// This function computes KnownBits to avoid a duplicated call to
2453// computeKnownBits in the caller.
2455 KnownBits &Known) {
2456 if (N->getOpcode() == ISD::TRUNCATE) {
2457 Op = N->getOperand(0);
2458 Known = DAG.computeKnownBits(Op);
2459 if (N->getFlags().hasNoUnsignedWrap())
2460 Known.Zero.setBitsFrom(N.getScalarValueSizeInBits());
2461 return true;
2462 }
2463
2464 if (N.getValueType().getScalarType() != MVT::i1 ||
2466 return false;
2467
2468 Known = DAG.computeKnownBits(Op);
2469 return (Known.Zero | 1).isAllOnes();
2470}
2471
2472/// Return true if 'Use' is a load or a store that uses N as its base pointer
2473/// and that N may be folded in the load / store addressing mode.
2475 const TargetLowering &TLI) {
2476 EVT VT;
2477 unsigned AS;
2478
2479 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Use)) {
2480 if (LD->isIndexed() || LD->getBasePtr().getNode() != N)
2481 return false;
2482 VT = LD->getMemoryVT();
2483 AS = LD->getAddressSpace();
2484 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Use)) {
2485 if (ST->isIndexed() || ST->getBasePtr().getNode() != N)
2486 return false;
2487 VT = ST->getMemoryVT();
2488 AS = ST->getAddressSpace();
2490 if (LD->isIndexed() || LD->getBasePtr().getNode() != N)
2491 return false;
2492 VT = LD->getMemoryVT();
2493 AS = LD->getAddressSpace();
2495 if (ST->isIndexed() || ST->getBasePtr().getNode() != N)
2496 return false;
2497 VT = ST->getMemoryVT();
2498 AS = ST->getAddressSpace();
2499 } else {
2500 return false;
2501 }
2502
2504 if (N->isAnyAdd()) {
2505 AM.HasBaseReg = true;
2507 if (Offset)
2508 // [reg +/- imm]
2509 AM.BaseOffs = Offset->getSExtValue();
2510 else
2511 // [reg +/- reg]
2512 AM.Scale = 1;
2513 } else if (N->getOpcode() == ISD::SUB) {
2514 AM.HasBaseReg = true;
2516 if (Offset)
2517 // [reg +/- imm]
2518 AM.BaseOffs = -Offset->getSExtValue();
2519 else
2520 // [reg +/- reg]
2521 AM.Scale = 1;
2522 } else {
2523 return false;
2524 }
2525
2526 return TLI.isLegalAddressingMode(DAG.getDataLayout(), AM,
2527 VT.getTypeForEVT(*DAG.getContext()), AS);
2528}
2529
2530/// This inverts a canonicalization in IR that replaces a variable select arm
2531/// with an identity constant. Codegen improves if we re-use the variable
2532/// operand rather than load a constant. This can also be converted into a
2533/// masked vector operation if the target supports it.
2535 bool ShouldCommuteOperands) {
2536 SDValue N0 = N->getOperand(0);
2537 SDValue N1 = N->getOperand(1);
2538
2539 // Match a select as operand 1. The identity constant that we are looking for
2540 // is only valid as operand 1 of a non-commutative binop.
2541 if (ShouldCommuteOperands)
2542 std::swap(N0, N1);
2543
2544 SDValue Cond, TVal, FVal;
2546 m_Value(FVal)))))
2547 return SDValue();
2548
2549 // We can't hoist all instructions because of immediate UB (not speculatable).
2550 // For example div/rem by zero.
2552 return SDValue();
2553
2554 unsigned SelOpcode = N1.getOpcode();
2555 unsigned Opcode = N->getOpcode();
2556 EVT VT = N->getValueType(0);
2558
2559 // This transform increases uses of N0, so freeze it to be safe.
2560 // binop N0, (vselect Cond, IDC, FVal) --> vselect Cond, N0, (binop N0, FVal)
2561 unsigned OpNo = ShouldCommuteOperands ? 0 : 1;
2562 if (DAG.isIdentityElement(Opcode, N->getFlags(), TVal, OpNo) &&
2563 TLI.shouldFoldSelectWithIdentityConstant(Opcode, VT, SelOpcode, N0,
2564 FVal)) {
2565 SDValue F0 = DAG.getFreeze(N0);
2566 SDValue NewBO = DAG.getNode(Opcode, SDLoc(N), VT, F0, FVal, N->getFlags());
2567 return DAG.getSelect(SDLoc(N), VT, Cond, F0, NewBO);
2568 }
2569 // binop N0, (vselect Cond, TVal, IDC) --> vselect Cond, (binop N0, TVal), N0
2570 if (DAG.isIdentityElement(Opcode, N->getFlags(), FVal, OpNo) &&
2571 TLI.shouldFoldSelectWithIdentityConstant(Opcode, VT, SelOpcode, N0,
2572 TVal)) {
2573 SDValue F0 = DAG.getFreeze(N0);
2574 SDValue NewBO = DAG.getNode(Opcode, SDLoc(N), VT, F0, TVal, N->getFlags());
2575 return DAG.getSelect(SDLoc(N), VT, Cond, NewBO, F0);
2576 }
2577
2578 return SDValue();
2579}
2580
2581SDValue DAGCombiner::foldBinOpIntoSelect(SDNode *BO) {
2583 assert(TLI.isBinOp(BO->getOpcode()) && BO->getNumValues() == 1 &&
2584 "Unexpected binary operator");
2585
2586 if (SDValue Sel = foldSelectWithIdentityConstant(BO, DAG, false))
2587 return Sel;
2588
2589 if (TLI.isCommutativeBinOp(BO->getOpcode()))
2590 if (SDValue Sel = foldSelectWithIdentityConstant(BO, DAG, true))
2591 return Sel;
2592
2593 // Don't do this unless the old select is going away. We want to eliminate the
2594 // binary operator, not replace a binop with a select.
2595 // TODO: Handle ISD::SELECT_CC.
2596 unsigned SelOpNo = 0;
2597 SDValue Sel = BO->getOperand(0);
2598 auto BinOpcode = BO->getOpcode();
2599 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse()) {
2600 SelOpNo = 1;
2601 Sel = BO->getOperand(1);
2602
2603 // Peek through trunc to shift amount type.
2604 if ((BinOpcode == ISD::SHL || BinOpcode == ISD::SRA ||
2605 BinOpcode == ISD::SRL) && Sel.hasOneUse()) {
2606 // This is valid when the truncated bits of x are already zero.
2607 SDValue Op;
2608 KnownBits Known;
2609 if (isTruncateOf(DAG, Sel, Op, Known) &&
2610 Known.countMaxActiveBits() < Sel.getScalarValueSizeInBits())
2611 Sel = Op;
2612 }
2613 }
2614
2615 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse())
2616 return SDValue();
2617
2618 SDValue CT = Sel.getOperand(1);
2619 if (!isConstantOrConstantVector(CT, true) &&
2621 return SDValue();
2622
2623 SDValue CF = Sel.getOperand(2);
2624 if (!isConstantOrConstantVector(CF, true) &&
2626 return SDValue();
2627
2628 // Bail out if any constants are opaque because we can't constant fold those.
2629 // The exception is "and" and "or" with either 0 or -1 in which case we can
2630 // propagate non constant operands into select. I.e.:
2631 // and (select Cond, 0, -1), X --> select Cond, 0, X
2632 // or X, (select Cond, -1, 0) --> select Cond, -1, X
2633 bool CanFoldNonConst =
2634 (BinOpcode == ISD::AND || BinOpcode == ISD::OR) &&
2637
2638 SDValue CBO = BO->getOperand(SelOpNo ^ 1);
2639 if (!CanFoldNonConst &&
2640 !isConstantOrConstantVector(CBO, true) &&
2642 return SDValue();
2643
2644 SDLoc DL(Sel);
2645 SDValue NewCT, NewCF;
2646 EVT VT = BO->getValueType(0);
2647
2648 if (CanFoldNonConst) {
2649 // If CBO is an opaque constant, we can't rely on getNode to constant fold.
2650 if ((BinOpcode == ISD::AND && isNullOrNullSplat(CT)) ||
2651 (BinOpcode == ISD::OR && isAllOnesOrAllOnesSplat(CT)))
2652 NewCT = CT;
2653 else
2654 NewCT = CBO;
2655
2656 if ((BinOpcode == ISD::AND && isNullOrNullSplat(CF)) ||
2657 (BinOpcode == ISD::OR && isAllOnesOrAllOnesSplat(CF)))
2658 NewCF = CF;
2659 else
2660 NewCF = CBO;
2661 } else {
2662 // We have a select-of-constants followed by a binary operator with a
2663 // constant. Eliminate the binop by pulling the constant math into the
2664 // select. Example: add (select Cond, CT, CF), CBO --> select Cond, CT +
2665 // CBO, CF + CBO
2666 NewCT = SelOpNo ? DAG.FoldConstantArithmetic(BinOpcode, DL, VT, {CBO, CT})
2667 : DAG.FoldConstantArithmetic(BinOpcode, DL, VT, {CT, CBO});
2668 if (!NewCT)
2669 return SDValue();
2670
2671 NewCF = SelOpNo ? DAG.FoldConstantArithmetic(BinOpcode, DL, VT, {CBO, CF})
2672 : DAG.FoldConstantArithmetic(BinOpcode, DL, VT, {CF, CBO});
2673 if (!NewCF)
2674 return SDValue();
2675 }
2676
2677 return DAG.getSelect(DL, VT, Sel.getOperand(0), NewCT, NewCF, BO->getFlags());
2678}
2679
2681 SelectionDAG &DAG) {
2682 assert((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) &&
2683 "Expecting add or sub");
2684
2685 // Match a constant operand and a zext operand for the math instruction:
2686 // add Z, C
2687 // sub C, Z
2688 bool IsAdd = N->getOpcode() == ISD::ADD;
2689 SDValue C = IsAdd ? N->getOperand(1) : N->getOperand(0);
2690 SDValue Z = IsAdd ? N->getOperand(0) : N->getOperand(1);
2691 auto *CN = dyn_cast<ConstantSDNode>(C);
2692 if (!CN || Z.getOpcode() != ISD::ZERO_EXTEND)
2693 return SDValue();
2694
2695 // Match the zext operand as a setcc of a boolean.
2696 if (Z.getOperand(0).getValueType() != MVT::i1)
2697 return SDValue();
2698
2699 // Match the compare as: setcc (X & 1), 0, eq.
2700 if (!sd_match(
2701 Z.getOperand(0),
2703 return SDValue();
2704
2705 // We are adding/subtracting a constant and an inverted low bit. Turn that
2706 // into a subtract/add of the low bit with incremented/decremented constant:
2707 // add (zext i1 (seteq (X & 1), 0)), C --> sub C+1, (zext (X & 1))
2708 // sub C, (zext i1 (seteq (X & 1), 0)) --> add C-1, (zext (X & 1))
2709 EVT VT = C.getValueType();
2710 SDValue LowBit = DAG.getZExtOrTrunc(Z.getOperand(0).getOperand(0), DL, VT);
2711 SDValue C1 = IsAdd ? DAG.getConstant(CN->getAPIntValue() + 1, DL, VT)
2712 : DAG.getConstant(CN->getAPIntValue() - 1, DL, VT);
2713 return DAG.getNode(IsAdd ? ISD::SUB : ISD::ADD, DL, VT, C1, LowBit);
2714}
2715
2716// Attempt to form avgceil(A, B) from (A | B) - ((A ^ B) >> 1)
2717SDValue DAGCombiner::foldSubToAvg(SDNode *N, const SDLoc &DL) {
2718 SDValue N0 = N->getOperand(0);
2719 EVT VT = N0.getValueType();
2720 SDValue A, B;
2721
2722 if ((!LegalOperations || hasOperation(ISD::AVGCEILU, VT)) &&
2724 m_Srl(m_Xor(m_Deferred(A), m_Deferred(B)), m_One())))) {
2725 return DAG.getNode(ISD::AVGCEILU, DL, VT, A, B);
2726 }
2727 if ((!LegalOperations || hasOperation(ISD::AVGCEILS, VT)) &&
2729 m_Sra(m_Xor(m_Deferred(A), m_Deferred(B)), m_One())))) {
2730 return DAG.getNode(ISD::AVGCEILS, DL, VT, A, B);
2731 }
2732 return SDValue();
2733}
2734
2735/// Try to fold a pointer arithmetic node.
2736/// This needs to be done separately from normal addition, because pointer
2737/// addition is not commutative.
2738SDValue DAGCombiner::visitPTRADD(SDNode *N) {
2739 SDValue N0 = N->getOperand(0);
2740 SDValue N1 = N->getOperand(1);
2741 EVT PtrVT = N0.getValueType();
2742 EVT IntVT = N1.getValueType();
2743 SDLoc DL(N);
2744
2745 // This is already ensured by an assert in SelectionDAG::getNode(). Several
2746 // combines here depend on this assumption.
2747 assert(PtrVT == IntVT &&
2748 "PTRADD with different operand types is not supported");
2749
2750 // fold (ptradd x, 0) -> x
2751 if (isNullConstant(N1))
2752 return N0;
2753
2754 // fold (ptradd 0, x) -> x
2755 if (PtrVT == IntVT && isNullConstant(N0))
2756 return N1;
2757
2758 if (N0.getOpcode() == ISD::PTRADD &&
2759 !reassociationCanBreakAddressingModePattern(ISD::PTRADD, DL, N, N0, N1)) {
2760 SDValue X = N0.getOperand(0);
2761 SDValue Y = N0.getOperand(1);
2762 SDValue Z = N1;
2763 bool N0OneUse = N0.hasOneUse();
2764 bool YIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(Y);
2765 bool ZIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(Z);
2766
2767 // (ptradd (ptradd x, y), z) -> (ptradd x, (add y, z)) if:
2768 // * y is a constant and (ptradd x, y) has one use; or
2769 // * y and z are both constants.
2770 if ((YIsConstant && N0OneUse) || (YIsConstant && ZIsConstant)) {
2771 // If both additions in the original were NUW, the new ones are as well.
2772 SDNodeFlags Flags =
2773 (N->getFlags() & N0->getFlags()) & SDNodeFlags::NoUnsignedWrap;
2774 SDValue Add = DAG.getNode(ISD::ADD, DL, IntVT, {Y, Z}, Flags);
2775 AddToWorklist(Add.getNode());
2776 // We can't set InBounds even if both original ptradds were InBounds and
2777 // NUW: SDAG usually represents pointers as integers, therefore, the
2778 // matched pattern behaves as if it had implicit casts:
2779 // (ptradd inbounds (inttoptr (ptrtoint (ptradd inbounds x, y))), z)
2780 // The outer inbounds ptradd might therefore rely on a provenance that x
2781 // does not have.
2782 return DAG.getMemBasePlusOffset(X, Add, DL, Flags);
2783 }
2784 }
2785
2786 // The following combines can turn in-bounds pointer arithmetic out of bounds.
2787 // That is problematic for settings like AArch64's CPA, which checks that
2788 // intermediate results of pointer arithmetic remain in bounds. The target
2789 // therefore needs to opt-in to enable them.
2791 DAG.getMachineFunction().getFunction(), PtrVT))
2792 return SDValue();
2793
2794 if (N0.getOpcode() == ISD::PTRADD && isa<ConstantSDNode>(N1)) {
2795 // Fold (ptradd (ptradd GA, v), c) -> (ptradd (ptradd GA, c) v) with
2796 // global address GA and constant c, such that c can be folded into GA.
2797 // TODO: Support constant vector splats.
2798 SDValue GAValue = N0.getOperand(0);
2799 if (const GlobalAddressSDNode *GA =
2802 if (!LegalOperations && TLI.isOffsetFoldingLegal(GA)) {
2803 // If both additions in the original were NUW, reassociation preserves
2804 // that.
2805 SDNodeFlags Flags =
2806 (N->getFlags() & N0->getFlags()) & SDNodeFlags::NoUnsignedWrap;
2807 // We can't set InBounds even if both original ptradds were InBounds and
2808 // NUW: SDAG usually represents pointers as integers, therefore, the
2809 // matched pattern behaves as if it had implicit casts:
2810 // (ptradd inbounds (inttoptr (ptrtoint (ptradd inbounds GA, v))), c)
2811 // The outer inbounds ptradd might therefore rely on a provenance that
2812 // GA does not have.
2813 SDValue Inner = DAG.getMemBasePlusOffset(GAValue, N1, DL, Flags);
2814 AddToWorklist(Inner.getNode());
2815 return DAG.getMemBasePlusOffset(Inner, N0.getOperand(1), DL, Flags);
2816 }
2817 }
2818 }
2819
2820 if (N1.getOpcode() == ISD::ADD && N1.hasOneUse()) {
2821 // (ptradd x, (add y, z)) -> (ptradd (ptradd x, y), z) if z is a constant,
2822 // y is not, and (add y, z) is used only once.
2823 // (ptradd x, (add y, z)) -> (ptradd (ptradd x, z), y) if y is a constant,
2824 // z is not, and (add y, z) is used only once.
2825 // The goal is to move constant offsets to the outermost ptradd, to create
2826 // more opportunities to fold offsets into memory instructions.
2827 // Together with the another combine above, this also implements
2828 // (ptradd (ptradd x, y), z) -> (ptradd (ptradd x, z), y)).
2829 SDValue X = N0;
2830 SDValue Y = N1.getOperand(0);
2831 SDValue Z = N1.getOperand(1);
2832 bool YIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(Y);
2833 bool ZIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(Z);
2834
2835 // If both additions in the original were NUW, reassociation preserves that.
2836 SDNodeFlags CommonFlags = N->getFlags() & N1->getFlags();
2837 SDNodeFlags ReassocFlags = CommonFlags & SDNodeFlags::NoUnsignedWrap;
2838 if (CommonFlags.hasNoUnsignedWrap()) {
2839 // If both operations are NUW and the PTRADD is inbounds, the offests are
2840 // both non-negative, so the reassociated PTRADDs are also inbounds.
2841 ReassocFlags |= N->getFlags() & SDNodeFlags::InBounds;
2842 }
2843
2844 if (ZIsConstant != YIsConstant) {
2845 if (YIsConstant)
2846 std::swap(Y, Z);
2847 SDValue Inner = DAG.getMemBasePlusOffset(X, Y, DL, ReassocFlags);
2848 AddToWorklist(Inner.getNode());
2849 return DAG.getMemBasePlusOffset(Inner, Z, DL, ReassocFlags);
2850 }
2851 }
2852
2853 // Transform (ptradd a, b) -> (or disjoint a, b) if it is equivalent and if
2854 // that transformation can't block an offset folding at any use of the ptradd.
2855 // This should be done late, after legalization, so that it doesn't block
2856 // other ptradd combines that could enable more offset folding.
2857 if (LegalOperations && DAG.haveNoCommonBitsSet(N0, N1)) {
2858 bool TransformCannotBreakAddrMode = none_of(N->users(), [&](SDNode *User) {
2859 return canFoldInAddressingMode(N, User, DAG, TLI);
2860 });
2861
2862 if (TransformCannotBreakAddrMode)
2863 return DAG.getNode(ISD::OR, DL, PtrVT, N0, N1, SDNodeFlags::Disjoint);
2864 }
2865
2866 return SDValue();
2867}
2868
2869/// Try to fold a 'not' shifted sign-bit with add/sub with constant operand into
2870/// a shift and add with a different constant.
2872 SelectionDAG &DAG) {
2873 assert((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) &&
2874 "Expecting add or sub");
2875
2876 // We need a constant operand for the add/sub, and the other operand is a
2877 // logical shift right: add (srl), C or sub C, (srl).
2878 bool IsAdd = N->getOpcode() == ISD::ADD;
2879 SDValue ConstantOp = IsAdd ? N->getOperand(1) : N->getOperand(0);
2880 SDValue ShiftOp = IsAdd ? N->getOperand(0) : N->getOperand(1);
2881 if (!DAG.isConstantIntBuildVectorOrConstantInt(ConstantOp) ||
2882 ShiftOp.getOpcode() != ISD::SRL)
2883 return SDValue();
2884
2885 // The shift must be of a 'not' value.
2886 SDValue Not = ShiftOp.getOperand(0);
2887 if (!Not.hasOneUse() || !isBitwiseNot(Not))
2888 return SDValue();
2889
2890 // The shift must be moving the sign bit to the least-significant-bit.
2891 EVT VT = ShiftOp.getValueType();
2892 SDValue ShAmt = ShiftOp.getOperand(1);
2893 ConstantSDNode *ShAmtC = isConstOrConstSplat(ShAmt);
2894 if (!ShAmtC || ShAmtC->getAPIntValue() != (VT.getScalarSizeInBits() - 1))
2895 return SDValue();
2896
2897 // Eliminate the 'not' by adjusting the shift and add/sub constant:
2898 // add (srl (not X), 31), C --> add (sra X, 31), (C + 1)
2899 // sub C, (srl (not X), 31) --> add (srl X, 31), (C - 1)
2900 if (SDValue NewC = DAG.FoldConstantArithmetic(
2901 IsAdd ? ISD::ADD : ISD::SUB, DL, VT,
2902 {ConstantOp, DAG.getConstant(1, DL, VT)})) {
2903 SDValue NewShift = DAG.getNode(IsAdd ? ISD::SRA : ISD::SRL, DL, VT,
2904 Not.getOperand(0), ShAmt);
2905 return DAG.getNode(ISD::ADD, DL, VT, NewShift, NewC);
2906 }
2907
2908 return SDValue();
2909}
2910
2911static bool
2913 return (isBitwiseNot(Op0) && Op0.getOperand(0) == Op1) ||
2914 (isBitwiseNot(Op1) && Op1.getOperand(0) == Op0);
2915}
2916
2917/// Try to fold a node that behaves like an ADD (note that N isn't necessarily
2918/// an ISD::ADD here, it could for example be an ISD::OR if we know that there
2919/// are no common bits set in the operands).
2920SDValue DAGCombiner::visitADDLike(SDNode *N) {
2921 SDValue N0 = N->getOperand(0);
2922 SDValue N1 = N->getOperand(1);
2923 EVT VT = N0.getValueType();
2924 SDLoc DL(N);
2925
2926 // fold (add x, undef) -> undef
2927 if (N0.isUndef())
2928 return N0;
2929 if (N1.isUndef())
2930 return N1;
2931
2932 // fold (add c1, c2) -> c1+c2
2933 // only forward flags if operation was originally an add
2935 ISD::ADD, DL, VT, {N0, N1},
2936 N->getOpcode() == ISD::ADD ? N->getFlags() : SDNodeFlags()))
2937 return C;
2938
2939 // canonicalize constant to RHS
2942 return DAG.getNode(ISD::ADD, DL, VT, N1, N0);
2943
2944 if (areBitwiseNotOfEachother(N0, N1))
2945 return DAG.getConstant(APInt::getAllOnes(VT.getScalarSizeInBits()), DL, VT);
2946
2947 // fold vector ops
2948 if (VT.isVector()) {
2949 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
2950 return FoldedVOp;
2951
2952 // fold (add x, 0) -> x, vector edition
2954 return N0;
2955 }
2956
2957 // fold (add x, 0) -> x
2958 if (isNullConstant(N1))
2959 return N0;
2960
2961 if (N0.getOpcode() == ISD::SUB) {
2962 SDValue N00 = N0.getOperand(0);
2963 SDValue N01 = N0.getOperand(1);
2964
2965 // fold ((A-c1)+c2) -> (A+(c2-c1))
2966 if (SDValue Sub = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N1, N01}))
2967 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), Sub);
2968
2969 // fold ((c1-A)+c2) -> (c1+c2)-A
2970 if (SDValue Add = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N1, N00}))
2971 return DAG.getNode(ISD::SUB, DL, VT, Add, N0.getOperand(1));
2972 }
2973
2974 // add (sext i1 X), 1 -> zext (not i1 X)
2975 // We don't transform this pattern:
2976 // add (zext i1 X), -1 -> sext (not i1 X)
2977 // because most (?) targets generate better code for the zext form.
2978 if (N0.getOpcode() == ISD::SIGN_EXTEND && N0.hasOneUse() &&
2979 isOneOrOneSplat(N1)) {
2980 SDValue X = N0.getOperand(0);
2981 if ((!LegalOperations ||
2982 (TLI.isOperationLegal(ISD::XOR, X.getValueType()) &&
2984 X.getScalarValueSizeInBits() == 1) {
2985 SDValue Not = DAG.getNOT(DL, X, X.getValueType());
2986 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Not);
2987 }
2988 }
2989
2990 // Fold (add (or x, c0), c1) -> (add x, (c0 + c1))
2991 // iff (or x, c0) is equivalent to (add x, c0).
2992 // Fold (add (xor x, c0), c1) -> (add x, (c0 + c1))
2993 // iff (xor x, c0) is equivalent to (add x, c0).
2994 if (DAG.isADDLike(N0)) {
2995 SDValue N01 = N0.getOperand(1);
2996 if (SDValue Add = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N1, N01}))
2997 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), Add);
2998 }
2999
3000 if (SDValue NewSel = foldBinOpIntoSelect(N))
3001 return NewSel;
3002
3003 // reassociate add
3004 if (!reassociationCanBreakAddressingModePattern(ISD::ADD, DL, N, N0, N1)) {
3005 if (SDValue RADD = reassociateOps(ISD::ADD, DL, N0, N1, N->getFlags()))
3006 return RADD;
3007
3008 // (X + Y) + X --> Y + (X + X)
3009 SDValue X, Y, InnerAdd;
3010 if (sd_match(
3011 N, m_Add(m_OneUse(m_Value(InnerAdd, m_Add(m_Value(X), m_Value(Y)))),
3012 m_Deferred(X)))) {
3013 if (X != Y) {
3014 // Redistribute shared NUW flag.
3015 // TODO: If NSW+NUW occurs on both adds, that can be redistributed too.
3016 SDNodeFlags NewFlags =
3017 N->getFlags() & InnerAdd->getFlags() & SDNodeFlags::NoUnsignedWrap;
3018 SDValue X2 = DAG.getNode(ISD::ADD, DL, VT, X, X, NewFlags);
3019 return DAG.getNode(ISD::ADD, DL, VT, Y, X2, NewFlags);
3020 }
3021 }
3022
3023 // Reassociate (add (or x, c), y) -> (add add(x, y), c)) if (or x, c) is
3024 // equivalent to (add x, c).
3025 // Reassociate (add (xor x, c), y) -> (add add(x, y), c)) if (xor x, c) is
3026 // equivalent to (add x, c).
3027 // Do this optimization only when adding c does not introduce instructions
3028 // for adding carries.
3029 auto ReassociateAddOr = [&](SDValue N0, SDValue N1) {
3030 if (DAG.isADDLike(N0) && N0.hasOneUse() &&
3031 isConstantOrConstantVector(N0.getOperand(1), /* NoOpaque */ true)) {
3032 // If N0's type does not split or is a sign mask, it does not introduce
3033 // add carry.
3034 auto TyActn = TLI.getTypeAction(*DAG.getContext(), N0.getValueType());
3035 bool NoAddCarry = TyActn == TargetLoweringBase::TypeLegal ||
3038 if (NoAddCarry)
3039 return DAG.getNode(
3040 ISD::ADD, DL, VT,
3041 DAG.getNode(ISD::ADD, DL, VT, N1, N0.getOperand(0)),
3042 N0.getOperand(1));
3043 }
3044 return SDValue();
3045 };
3046 if (SDValue Add = ReassociateAddOr(N0, N1))
3047 return Add;
3048 if (SDValue Add = ReassociateAddOr(N1, N0))
3049 return Add;
3050
3051 // Fold add(vecreduce(x), vecreduce(y)) -> vecreduce(add(x, y))
3052 if (SDValue SD =
3053 reassociateReduction(ISD::VECREDUCE_ADD, ISD::ADD, DL, VT, N0, N1))
3054 return SD;
3055 }
3056
3057 SDValue A, B, C, D;
3058
3059 // fold ((0-A) + B) -> B-A
3060 if (sd_match(N0, m_Neg(m_Value(A))))
3061 return DAG.getNode(ISD::SUB, DL, VT, N1, A);
3062
3063 // fold (A + (0-B)) -> A-B
3064 if (sd_match(N1, m_Neg(m_Value(B))))
3065 return DAG.getNode(ISD::SUB, DL, VT, N0, B);
3066
3067 // fold (A+(B-A)) -> B
3068 if (sd_match(N1, m_Sub(m_Value(B), m_Specific(N0))))
3069 return B;
3070
3071 // fold ((B-A)+A) -> B
3072 if (sd_match(N0, m_Sub(m_Value(B), m_Specific(N1))))
3073 return B;
3074
3075 // fold ((A-B)+(C-A)) -> (C-B)
3076 if (sd_match(N0, m_Sub(m_Value(A), m_Value(B))) &&
3078 return DAG.getNode(ISD::SUB, DL, VT, C, B);
3079
3080 // fold ((A-B)+(B-C)) -> (A-C)
3081 if (sd_match(N0, m_Sub(m_Value(A), m_Value(B))) &&
3083 return DAG.getNode(ISD::SUB, DL, VT, A, C);
3084
3085 // fold (A+(B-(A+C))) to (B-C)
3086 // fold (A+(B-(C+A))) to (B-C)
3087 if (sd_match(N1, m_Sub(m_Value(B), m_Add(m_Specific(N0), m_Value(C)))))
3088 return DAG.getNode(ISD::SUB, DL, VT, B, C);
3089
3090 // fold (A+((B-A)+or-C)) to (B+or-C)
3091 if (sd_match(N1,
3093 m_Sub(m_Sub(m_Value(B), m_Specific(N0)), m_Value(C)))))
3094 return DAG.getNode(N1.getOpcode(), DL, VT, B, C);
3095
3096 // fold (A-B)+(C-D) to (A+C)-(B+D) when A or C is constant
3097 if (sd_match(N0, m_OneUse(m_Sub(m_Value(A), m_Value(B)))) &&
3098 sd_match(N1, m_OneUse(m_Sub(m_Value(C), m_Value(D)))) &&
3100 return DAG.getNode(ISD::SUB, DL, VT,
3101 DAG.getNode(ISD::ADD, SDLoc(N0), VT, A, C),
3102 DAG.getNode(ISD::ADD, SDLoc(N1), VT, B, D));
3103
3104 // fold (add (umax X, C), -C) --> (usubsat X, C)
3105 if (N0.getOpcode() == ISD::UMAX && hasOperation(ISD::USUBSAT, VT)) {
3106 auto MatchUSUBSAT = [](ConstantSDNode *Max, ConstantSDNode *Op) {
3107 return (!Max && !Op) ||
3108 (Max && Op && Max->getAPIntValue() == (-Op->getAPIntValue()));
3109 };
3110 if (ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchUSUBSAT,
3111 /*AllowUndefs*/ true))
3112 return DAG.getNode(ISD::USUBSAT, DL, VT, N0.getOperand(0),
3113 N0.getOperand(1));
3114 }
3115
3117 return SDValue(N, 0);
3118
3119 if (isOneOrOneSplat(N1)) {
3120 // fold (add (xor a, -1), 1) -> (sub 0, a)
3121 if (isBitwiseNot(N0))
3122 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
3123 N0.getOperand(0));
3124
3125 // fold (add (add (xor a, -1), b), 1) -> (sub b, a)
3126 if (N0.getOpcode() == ISD::ADD) {
3127 SDValue A, Xor;
3128
3129 if (isBitwiseNot(N0.getOperand(0))) {
3130 A = N0.getOperand(1);
3131 Xor = N0.getOperand(0);
3132 } else if (isBitwiseNot(N0.getOperand(1))) {
3133 A = N0.getOperand(0);
3134 Xor = N0.getOperand(1);
3135 }
3136
3137 if (Xor)
3138 return DAG.getNode(ISD::SUB, DL, VT, A, Xor.getOperand(0));
3139 }
3140
3141 // Look for:
3142 // add (add x, y), 1
3143 // And if the target does not like this form then turn into:
3144 // sub x, (xor y, -1)
3145 if (!TLI.preferIncOfAddToSubOfNot(VT) && N0.getOpcode() == ISD::ADD &&
3146 N0.hasOneUse() &&
3147 // Limit this to after legalization if the add has wrap flags
3148 (Level >= AfterLegalizeDAG || (!N->getFlags().hasNoUnsignedWrap() &&
3149 !N->getFlags().hasNoSignedWrap()))) {
3150 SDValue Not = DAG.getNOT(DL, N0.getOperand(1), VT);
3151 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), Not);
3152 }
3153 }
3154
3155 // (x - y) + -1 -> add (xor y, -1), x
3156 if (N0.getOpcode() == ISD::SUB && N0.hasOneUse() &&
3157 isAllOnesOrAllOnesSplat(N1, /*AllowUndefs=*/true)) {
3158 SDValue Not = DAG.getNOT(DL, N0.getOperand(1), VT);
3159 return DAG.getNode(ISD::ADD, DL, VT, Not, N0.getOperand(0));
3160 }
3161
3162 // Fold add(mul(add(A, CA), CM), CB) -> add(mul(A, CM), CM*CA+CB).
3163 // This can help if the inner add has multiple uses.
3164 APInt CM, CA;
3165 if (ConstantSDNode *CB = dyn_cast<ConstantSDNode>(N1)) {
3166 if (VT.getScalarSizeInBits() <= 64) {
3168 m_ConstInt(CM)))) &&
3170 (CA * CM + CB->getAPIntValue()).getSExtValue())) {
3171 SDNodeFlags Flags;
3172 // If all the inputs are nuw, the outputs can be nuw. If all the input
3173 // are _also_ nsw the outputs can be too.
3174 if (N->getFlags().hasNoUnsignedWrap() &&
3175 N0->getFlags().hasNoUnsignedWrap() &&
3178 if (N->getFlags().hasNoSignedWrap() &&
3179 N0->getFlags().hasNoSignedWrap() &&
3182 }
3183 SDValue Mul = DAG.getNode(ISD::MUL, SDLoc(N1), VT, A,
3184 DAG.getConstant(CM, DL, VT), Flags);
3185 return DAG.getNode(
3186 ISD::ADD, DL, VT, Mul,
3187 DAG.getConstant(CA * CM + CB->getAPIntValue(), DL, VT), Flags);
3188 }
3189 // Also look in case there is an intermediate add.
3190 if (sd_match(N0, m_OneUse(m_Add(
3192 m_ConstInt(CM))),
3193 m_Value(B)))) &&
3195 (CA * CM + CB->getAPIntValue()).getSExtValue())) {
3196 SDNodeFlags Flags;
3197 // If all the inputs are nuw, the outputs can be nuw. If all the input
3198 // are _also_ nsw the outputs can be too.
3199 SDValue OMul =
3200 N0.getOperand(0) == B ? N0.getOperand(1) : N0.getOperand(0);
3201 if (N->getFlags().hasNoUnsignedWrap() &&
3202 N0->getFlags().hasNoUnsignedWrap() &&
3203 OMul->getFlags().hasNoUnsignedWrap() &&
3204 OMul.getOperand(0)->getFlags().hasNoUnsignedWrap()) {
3206 if (N->getFlags().hasNoSignedWrap() &&
3207 N0->getFlags().hasNoSignedWrap() &&
3208 OMul->getFlags().hasNoSignedWrap() &&
3209 OMul.getOperand(0)->getFlags().hasNoSignedWrap())
3211 }
3212 SDValue Mul = DAG.getNode(ISD::MUL, SDLoc(N1), VT, A,
3213 DAG.getConstant(CM, DL, VT), Flags);
3214 SDValue Add = DAG.getNode(ISD::ADD, SDLoc(N1), VT, Mul, B, Flags);
3215 return DAG.getNode(
3216 ISD::ADD, DL, VT, Add,
3217 DAG.getConstant(CA * CM + CB->getAPIntValue(), DL, VT), Flags);
3218 }
3219 }
3220 }
3221
3222 if (SDValue Combined = visitADDLikeCommutative(N0, N1, DL))
3223 return Combined;
3224
3225 if (SDValue Combined = visitADDLikeCommutative(N1, N0, DL))
3226 return Combined;
3227
3228 return SDValue();
3229}
3230
3231// Attempt to form avgfloor(A, B) from (A & B) + ((A ^ B) >> 1)
3232// Attempt to form avgfloor(A, B) from ((A >> 1) + (B >> 1)) + (A & B & 1)
3233// Attempt to form avgceil(A, B) from ((A >> 1) + (B >> 1)) + ((A | B) & 1)
3234SDValue DAGCombiner::foldAddToAvg(SDNode *N, const SDLoc &DL) {
3235 SDValue N0 = N->getOperand(0);
3236 EVT VT = N0.getValueType();
3237 SDValue A, B;
3238
3239 if ((!LegalOperations || hasOperation(ISD::AVGFLOORU, VT)) &&
3240 (sd_match(N,
3242 m_Srl(m_Xor(m_Deferred(A), m_Deferred(B)), m_One()))) ||
3245 m_Srl(m_Deferred(A), m_One()),
3246 m_Srl(m_Deferred(B), m_One()))))) {
3247 return DAG.getNode(ISD::AVGFLOORU, DL, VT, A, B);
3248 }
3249 if ((!LegalOperations || hasOperation(ISD::AVGFLOORS, VT)) &&
3250 (sd_match(N,
3252 m_Sra(m_Xor(m_Deferred(A), m_Deferred(B)), m_One()))) ||
3255 m_Sra(m_Deferred(A), m_One()),
3256 m_Sra(m_Deferred(B), m_One()))))) {
3257 return DAG.getNode(ISD::AVGFLOORS, DL, VT, A, B);
3258 }
3259
3260 if ((!LegalOperations || hasOperation(ISD::AVGCEILU, VT)) &&
3261 sd_match(N,
3263 m_Srl(m_Deferred(A), m_One()),
3264 m_Srl(m_Deferred(B), m_One())))) {
3265 return DAG.getNode(ISD::AVGCEILU, DL, VT, A, B);
3266 }
3267 if ((!LegalOperations || hasOperation(ISD::AVGCEILS, VT)) &&
3268 sd_match(N,
3270 m_Sra(m_Deferred(A), m_One()),
3271 m_Sra(m_Deferred(B), m_One())))) {
3272 return DAG.getNode(ISD::AVGCEILS, DL, VT, A, B);
3273 }
3274
3275 return SDValue();
3276}
3277
3278SDValue DAGCombiner::visitADD(SDNode *N) {
3279 SDValue N0 = N->getOperand(0);
3280 SDValue N1 = N->getOperand(1);
3281 EVT VT = N0.getValueType();
3282 SDLoc DL(N);
3283
3284 if (SDValue Combined = visitADDLike(N))
3285 return Combined;
3286
3287 if (SDValue V = foldAddSubBoolOfMaskedVal(N, DL, DAG))
3288 return V;
3289
3290 if (SDValue V = foldAddSubOfSignBit(N, DL, DAG))
3291 return V;
3292
3293 if (SDValue V = MatchRotate(N0, N1, SDLoc(N), /*FromAdd=*/true))
3294 return V;
3295
3296 // Try to match AVGFLOOR fixedwidth pattern
3297 if (SDValue V = foldAddToAvg(N, DL))
3298 return V;
3299
3300 // fold (a+b) -> (a|b) iff a and b share no bits.
3301 if ((!LegalOperations || TLI.isOperationLegal(ISD::OR, VT)) &&
3302 DAG.haveNoCommonBitsSet(N0, N1))
3303 return DAG.getNode(ISD::OR, DL, VT, N0, N1, SDNodeFlags::Disjoint);
3304
3305 // Fold (add (vscale * C0), (vscale * C1)) to (vscale * (C0 + C1)).
3306 if (N0.getOpcode() == ISD::VSCALE && N1.getOpcode() == ISD::VSCALE) {
3307 const APInt &C0 = N0->getConstantOperandAPInt(0);
3308 const APInt &C1 = N1->getConstantOperandAPInt(0);
3309 return DAG.getVScale(DL, VT, C0 + C1);
3310 }
3311
3312 // fold a+vscale(c1)+vscale(c2) -> a+vscale(c1+c2)
3313 if (N0.getOpcode() == ISD::ADD &&
3314 N0.getOperand(1).getOpcode() == ISD::VSCALE &&
3316 const APInt &VS0 = N0.getOperand(1)->getConstantOperandAPInt(0);
3317 const APInt &VS1 = N1->getConstantOperandAPInt(0);
3318 SDValue VS = DAG.getVScale(DL, VT, VS0 + VS1);
3319 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), VS);
3320 }
3321
3322 // Fold (add step_vector(c1), step_vector(c2) to step_vector(c1+c2))
3323 if (N0.getOpcode() == ISD::STEP_VECTOR &&
3324 N1.getOpcode() == ISD::STEP_VECTOR) {
3325 const APInt &C0 = N0->getConstantOperandAPInt(0);
3326 const APInt &C1 = N1->getConstantOperandAPInt(0);
3327 APInt NewStep = C0 + C1;
3328 return DAG.getStepVector(DL, VT, NewStep);
3329 }
3330
3331 // Fold a + step_vector(c1) + step_vector(c2) to a + step_vector(c1+c2)
3332 if (N0.getOpcode() == ISD::ADD &&
3334 N1.getOpcode() == ISD::STEP_VECTOR) {
3335 const APInt &SV0 = N0.getOperand(1)->getConstantOperandAPInt(0);
3336 const APInt &SV1 = N1->getConstantOperandAPInt(0);
3337 APInt NewStep = SV0 + SV1;
3338 SDValue SV = DAG.getStepVector(DL, VT, NewStep);
3339 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), SV);
3340 }
3341
3342 return SDValue();
3343}
3344
3345SDValue DAGCombiner::visitADDSAT(SDNode *N) {
3346 unsigned Opcode = N->getOpcode();
3347 SDValue N0 = N->getOperand(0);
3348 SDValue N1 = N->getOperand(1);
3349 EVT VT = N0.getValueType();
3350 bool IsSigned = Opcode == ISD::SADDSAT;
3351 SDLoc DL(N);
3352
3353 // fold (add_sat x, undef) -> -1
3354 if (N0.isUndef() || N1.isUndef())
3355 return DAG.getAllOnesConstant(DL, VT);
3356
3357 // fold (add_sat c1, c2) -> c3
3358 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1}))
3359 return C;
3360
3361 // canonicalize constant to RHS
3364 return DAG.getNode(Opcode, DL, VT, N1, N0);
3365
3366 // fold vector ops
3367 if (VT.isVector()) {
3368 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
3369 return FoldedVOp;
3370
3371 // fold (add_sat x, 0) -> x, vector edition
3373 return N0;
3374 }
3375
3376 // fold (add_sat x, 0) -> x
3377 if (isNullConstant(N1))
3378 return N0;
3379
3380 // If it cannot overflow, transform into an add.
3381 if (DAG.willNotOverflowAdd(IsSigned, N0, N1))
3382 return DAG.getNode(ISD::ADD, DL, VT, N0, N1);
3383
3384 return SDValue();
3385}
3386
3388 bool ForceCarryReconstruction = false) {
3389 bool Masked = false;
3390
3391 // First, peel away TRUNCATE/ZERO_EXTEND/AND nodes due to legalization.
3392 while (true) {
3393 if (ForceCarryReconstruction && V.getValueType() == MVT::i1)
3394 return V;
3395
3396 if (V.getOpcode() == ISD::TRUNCATE || V.getOpcode() == ISD::ZERO_EXTEND) {
3397 V = V.getOperand(0);
3398 continue;
3399 }
3400
3401 if (V.getOpcode() == ISD::AND && isOneConstant(V.getOperand(1))) {
3402 if (ForceCarryReconstruction)
3403 return V;
3404
3405 Masked = true;
3406 V = V.getOperand(0);
3407 continue;
3408 }
3409
3410 break;
3411 }
3412
3413 // If this is not a carry, return.
3414 if (V.getResNo() != 1)
3415 return SDValue();
3416
3417 if (V.getOpcode() != ISD::UADDO_CARRY && V.getOpcode() != ISD::USUBO_CARRY &&
3418 V.getOpcode() != ISD::UADDO && V.getOpcode() != ISD::USUBO)
3419 return SDValue();
3420
3421 EVT VT = V->getValueType(0);
3422 if (!TLI.isOperationLegalOrCustom(V.getOpcode(), VT))
3423 return SDValue();
3424
3425 // If the result is masked, then no matter what kind of bool it is we can
3426 // return. If it isn't, then we need to make sure the bool type is either 0 or
3427 // 1 and not other values.
3428 if (Masked ||
3429 TLI.getBooleanContents(V.getValueType()) ==
3431 return V;
3432
3433 return SDValue();
3434}
3435
3436/// Given the operands of an add/sub operation, see if the 2nd operand is a
3437/// masked 0/1 whose source operand is actually known to be 0/-1. If so, invert
3438/// the opcode and bypass the mask operation.
3439static SDValue foldAddSubMasked1(bool IsAdd, SDValue N0, SDValue N1,
3440 SelectionDAG &DAG, const SDLoc &DL) {
3441 if (N1.getOpcode() == ISD::ZERO_EXTEND)
3442 N1 = N1.getOperand(0);
3443
3444 if (N1.getOpcode() != ISD::AND || !isOneOrOneSplat(N1->getOperand(1)))
3445 return SDValue();
3446
3447 EVT VT = N0.getValueType();
3448 SDValue N10 = N1.getOperand(0);
3449 if (N10.getValueType() != VT && N10.getOpcode() == ISD::TRUNCATE)
3450 N10 = N10.getOperand(0);
3451
3452 if (N10.getValueType() != VT)
3453 return SDValue();
3454
3455 if (DAG.ComputeNumSignBits(N10) != VT.getScalarSizeInBits())
3456 return SDValue();
3457
3458 // add N0, (and (AssertSext X, i1), 1) --> sub N0, X
3459 // sub N0, (and (AssertSext X, i1), 1) --> add N0, X
3460 return DAG.getNode(IsAdd ? ISD::SUB : ISD::ADD, DL, VT, N0, N10);
3461}
3462
3463/// Helper for doing combines based on N0 and N1 being added to each other.
3464SDValue DAGCombiner::visitADDLikeCommutative(SDValue N0, SDValue N1,
3465 const SDLoc &DL) {
3466 EVT VT = N0.getValueType();
3467
3468 // fold (add x, shl(0 - y, n)) -> sub(x, shl(y, n))
3469 SDValue Y, N;
3470 if (sd_match(N1, m_Shl(m_Neg(m_Value(Y)), m_Value(N))))
3471 return DAG.getNode(ISD::SUB, DL, VT, N0,
3472 DAG.getNode(ISD::SHL, DL, VT, Y, N));
3473
3474 if (SDValue V = foldAddSubMasked1(true, N0, N1, DAG, DL))
3475 return V;
3476
3477 // Look for:
3478 // add (add x, 1), y
3479 // And if the target does not like this form then turn into:
3480 // sub x, (xor y, -1)
3481 if (!TLI.preferIncOfAddToSubOfNot(VT) && N0.getOpcode() == ISD::ADD &&
3482 N0.hasOneUse() && isOneOrOneSplat(N0.getOperand(1)) &&
3483 // Limit this to after legalization if the add has wrap flags
3484 (Level >= AfterLegalizeDAG || (!N0->getFlags().hasNoUnsignedWrap() &&
3485 !N0->getFlags().hasNoSignedWrap()))) {
3486 SDValue Not = DAG.getNOT(DL, N1, VT);
3487 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), Not);
3488 }
3489
3490 if (N0.getOpcode() == ISD::SUB && N0.hasOneUse()) {
3491 // Hoist one-use subtraction by non-opaque constant:
3492 // (x - C) + y -> (x + y) - C
3493 // This is necessary because SUB(X,C) -> ADD(X,-C) doesn't work for vectors.
3494 if (isConstantOrConstantVector(N0.getOperand(1), /*NoOpaques=*/true)) {
3495 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), N1);
3496 return DAG.getNode(ISD::SUB, DL, VT, Add, N0.getOperand(1));
3497 }
3498 // Hoist one-use subtraction from non-opaque constant:
3499 // (C - x) + y -> (y - x) + C
3500 if (isConstantOrConstantVector(N0.getOperand(0), /*NoOpaques=*/true)) {
3501 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N1, N0.getOperand(1));
3502 return DAG.getNode(ISD::ADD, DL, VT, Sub, N0.getOperand(0));
3503 }
3504 }
3505
3506 // add (mul x, C), x -> mul x, C+1
3507 if (N0.getOpcode() == ISD::MUL && N0.getOperand(0) == N1 &&
3508 isConstantOrConstantVector(N0.getOperand(1), /*NoOpaques=*/true) &&
3509 N0.hasOneUse()) {
3510 SDValue NewC = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(1),
3511 DAG.getConstant(1, DL, VT));
3512 return DAG.getNode(ISD::MUL, DL, VT, N0.getOperand(0), NewC);
3513 }
3514
3515 // If the target's bool is represented as 0/1, prefer to make this 'sub 0/1'
3516 // rather than 'add 0/-1' (the zext should get folded).
3517 // add (sext i1 Y), X --> sub X, (zext i1 Y)
3518 if (N0.getOpcode() == ISD::SIGN_EXTEND &&
3519 N0.getOperand(0).getScalarValueSizeInBits() == 1 &&
3521 SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0));
3522 return DAG.getNode(ISD::SUB, DL, VT, N1, ZExt);
3523 }
3524
3525 // add X, (sextinreg Y i1) -> sub X, (and Y 1)
3526 if (N1.getOpcode() == ISD::SIGN_EXTEND_INREG) {
3527 VTSDNode *TN = cast<VTSDNode>(N1.getOperand(1));
3528 if (TN->getVT() == MVT::i1) {
3529 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0),
3530 DAG.getConstant(1, DL, VT));
3531 return DAG.getNode(ISD::SUB, DL, VT, N0, ZExt);
3532 }
3533 }
3534
3535 // (add X, (uaddo_carry Y, 0, Carry)) -> (uaddo_carry X, Y, Carry)
3536 if (N1.getOpcode() == ISD::UADDO_CARRY && isNullConstant(N1.getOperand(1)) &&
3537 N1.getResNo() == 0)
3538 return DAG.getNode(ISD::UADDO_CARRY, DL, N1->getVTList(),
3539 N0, N1.getOperand(0), N1.getOperand(2));
3540
3541 // (add X, Carry) -> (uaddo_carry X, 0, Carry)
3543 if (SDValue Carry = getAsCarry(TLI, N1))
3544 return DAG.getNode(ISD::UADDO_CARRY, DL,
3545 DAG.getVTList(VT, Carry.getValueType()), N0,
3546 DAG.getConstant(0, DL, VT), Carry);
3547
3548 return SDValue();
3549}
3550
3551SDValue DAGCombiner::visitADDC(SDNode *N) {
3552 SDValue N0 = N->getOperand(0);
3553 SDValue N1 = N->getOperand(1);
3554 EVT VT = N0.getValueType();
3555 SDLoc DL(N);
3556
3557 // If the flag result is dead, turn this into an ADD.
3558 if (!N->hasAnyUseOfValue(1))
3559 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
3560 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
3561
3562 // canonicalize constant to RHS.
3563 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
3564 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
3565 if (N0C && !N1C)
3566 return DAG.getNode(ISD::ADDC, DL, N->getVTList(), N1, N0);
3567
3568 // fold (addc x, 0) -> x + no carry out
3569 if (isNullConstant(N1))
3570 return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE,
3571 DL, MVT::Glue));
3572
3573 // If it cannot overflow, transform into an add.
3575 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
3576 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
3577
3578 return SDValue();
3579}
3580
3581/**
3582 * Flips a boolean if it is cheaper to compute. If the Force parameters is set,
3583 * then the flip also occurs if computing the inverse is the same cost.
3584 * This function returns an empty SDValue in case it cannot flip the boolean
3585 * without increasing the cost of the computation. If you want to flip a boolean
3586 * no matter what, use DAG.getLogicalNOT.
3587 */
3589 const TargetLowering &TLI,
3590 bool Force) {
3591 if (Force && isa<ConstantSDNode>(V))
3592 return DAG.getLogicalNOT(SDLoc(V), V, V.getValueType());
3593
3594 if (V.getOpcode() != ISD::XOR)
3595 return SDValue();
3596
3597 if (DAG.isBoolConstant(V.getOperand(1)) == true)
3598 return V.getOperand(0);
3599 if (Force && isConstOrConstSplat(V.getOperand(1), false))
3600 return DAG.getLogicalNOT(SDLoc(V), V, V.getValueType());
3601 return SDValue();
3602}
3603
3604SDValue DAGCombiner::visitADDO(SDNode *N) {
3605 SDValue N0 = N->getOperand(0);
3606 SDValue N1 = N->getOperand(1);
3607 EVT VT = N0.getValueType();
3608 bool IsSigned = (ISD::SADDO == N->getOpcode());
3609
3610 EVT CarryVT = N->getValueType(1);
3611 SDLoc DL(N);
3612
3613 // If the flag result is dead, turn this into an ADD.
3614 if (!N->hasAnyUseOfValue(1))
3615 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
3616 DAG.getUNDEF(CarryVT));
3617
3618 // fold operation with constant operands.
3619 if (SDValue C = DAG.FoldConstantArithmetic(N->getOpcode(), DL, N->getVTList(),
3620 {N0, N1}))
3621 return C;
3622
3623 // canonicalize constant to RHS.
3626 return DAG.getNode(N->getOpcode(), DL, N->getVTList(), N1, N0);
3627
3628 // fold (addo x, 0) -> x + no carry out
3629 if (isNullOrNullSplat(N1))
3630 return CombineTo(N, N0, DAG.getConstant(0, DL, CarryVT));
3631
3632 // If it cannot overflow, transform into an add.
3633 if (DAG.willNotOverflowAdd(IsSigned, N0, N1))
3634 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
3635 DAG.getConstant(0, DL, CarryVT));
3636
3637 if (IsSigned) {
3638 // fold (saddo (xor a, -1), 1) -> (ssub 0, a).
3639 if (isBitwiseNot(N0) && isOneOrOneSplat(N1))
3640 return DAG.getNode(ISD::SSUBO, DL, N->getVTList(),
3641 DAG.getConstant(0, DL, VT), N0.getOperand(0));
3642 } else {
3643 // fold (uaddo (xor a, -1), 1) -> (usub 0, a) and flip carry.
3644 if (isBitwiseNot(N0) && isOneOrOneSplat(N1)) {
3645 SDValue Sub = DAG.getNode(ISD::USUBO, DL, N->getVTList(),
3646 DAG.getConstant(0, DL, VT), N0.getOperand(0));
3647 return CombineTo(
3648 N, Sub, DAG.getLogicalNOT(DL, Sub.getValue(1), Sub->getValueType(1)));
3649 }
3650
3651 if (SDValue Combined = visitUADDOLike(N0, N1, N))
3652 return Combined;
3653
3654 if (SDValue Combined = visitUADDOLike(N1, N0, N))
3655 return Combined;
3656 }
3657
3658 return SDValue();
3659}
3660
3661SDValue DAGCombiner::visitUADDOLike(SDValue N0, SDValue N1, SDNode *N) {
3662 EVT VT = N0.getValueType();
3663 if (VT.isVector())
3664 return SDValue();
3665
3666 // (uaddo X, (uaddo_carry Y, 0, Carry)) -> (uaddo_carry X, Y, Carry)
3667 // If Y + 1 cannot overflow.
3668 if (N1.getOpcode() == ISD::UADDO_CARRY && isNullConstant(N1.getOperand(1))) {
3669 SDValue Y = N1.getOperand(0);
3670 SDValue One = DAG.getConstant(1, SDLoc(N), Y.getValueType());
3672 return DAG.getNode(ISD::UADDO_CARRY, SDLoc(N), N->getVTList(), N0, Y,
3673 N1.getOperand(2));
3674 }
3675
3676 // (uaddo X, Carry) -> (uaddo_carry X, 0, Carry)
3678 if (SDValue Carry = getAsCarry(TLI, N1))
3679 return DAG.getNode(ISD::UADDO_CARRY, SDLoc(N), N->getVTList(), N0,
3680 DAG.getConstant(0, SDLoc(N), VT), Carry);
3681
3682 return SDValue();
3683}
3684
3685SDValue DAGCombiner::visitADDE(SDNode *N) {
3686 SDValue N0 = N->getOperand(0);
3687 SDValue N1 = N->getOperand(1);
3688 SDValue CarryIn = N->getOperand(2);
3689
3690 // canonicalize constant to RHS
3691 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
3692 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
3693 if (N0C && !N1C)
3694 return DAG.getNode(ISD::ADDE, SDLoc(N), N->getVTList(),
3695 N1, N0, CarryIn);
3696
3697 // fold (adde x, y, false) -> (addc x, y)
3698 if (CarryIn.getOpcode() == ISD::CARRY_FALSE)
3699 return DAG.getNode(ISD::ADDC, SDLoc(N), N->getVTList(), N0, N1);
3700
3701 return SDValue();
3702}
3703
3704SDValue DAGCombiner::visitUADDO_CARRY(SDNode *N) {
3705 SDValue N0 = N->getOperand(0);
3706 SDValue N1 = N->getOperand(1);
3707 SDValue CarryIn = N->getOperand(2);
3708 SDLoc DL(N);
3709
3710 // canonicalize constant to RHS
3711 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
3712 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
3713 if (N0C && !N1C)
3714 return DAG.getNode(ISD::UADDO_CARRY, DL, N->getVTList(), N1, N0, CarryIn);
3715
3716 // fold (uaddo_carry x, y, false) -> (uaddo x, y)
3717 if (isNullConstant(CarryIn)) {
3718 if (!LegalOperations ||
3719 TLI.isOperationLegalOrCustom(ISD::UADDO, N->getValueType(0)))
3720 return DAG.getNode(ISD::UADDO, DL, N->getVTList(), N0, N1);
3721 }
3722
3723 // fold (uaddo_carry 0, 0, X) -> (and (ext/trunc X), 1) and no carry.
3724 if (isNullConstant(N0) && isNullConstant(N1)) {
3725 EVT VT = N0.getValueType();
3726 EVT CarryVT = CarryIn.getValueType();
3727 SDValue CarryExt = DAG.getBoolExtOrTrunc(CarryIn, DL, VT, CarryVT);
3728 AddToWorklist(CarryExt.getNode());
3729 return CombineTo(N, DAG.getNode(ISD::AND, DL, VT, CarryExt,
3730 DAG.getConstant(1, DL, VT)),
3731 DAG.getConstant(0, DL, CarryVT));
3732 }
3733
3734 if (SDValue Combined = visitUADDO_CARRYLike(N0, N1, CarryIn, N))
3735 return Combined;
3736
3737 if (SDValue Combined = visitUADDO_CARRYLike(N1, N0, CarryIn, N))
3738 return Combined;
3739
3740 // We want to avoid useless duplication.
3741 // TODO: This is done automatically for binary operations. As UADDO_CARRY is
3742 // not a binary operation, this is not really possible to leverage this
3743 // existing mechanism for it. However, if more operations require the same
3744 // deduplication logic, then it may be worth generalize.
3745 SDValue Ops[] = {N1, N0, CarryIn};
3746 SDNode *CSENode =
3747 DAG.getNodeIfExists(ISD::UADDO_CARRY, N->getVTList(), Ops, N->getFlags());
3748 if (CSENode)
3749 return SDValue(CSENode, 0);
3750
3751 return SDValue();
3752}
3753
3754/**
3755 * If we are facing some sort of diamond carry propagation pattern try to
3756 * break it up to generate something like:
3757 * (uaddo_carry X, 0, (uaddo_carry A, B, Z):Carry)
3758 *
3759 * The end result is usually an increase in operation required, but because the
3760 * carry is now linearized, other transforms can kick in and optimize the DAG.
3761 *
3762 * Patterns typically look something like
3763 * (uaddo A, B)
3764 * / \
3765 * Carry Sum
3766 * | \
3767 * | (uaddo_carry *, 0, Z)
3768 * | /
3769 * \ Carry
3770 * | /
3771 * (uaddo_carry X, *, *)
3772 *
3773 * But numerous variation exist. Our goal is to identify A, B, X and Z and
3774 * produce a combine with a single path for carry propagation.
3775 */
3777 SelectionDAG &DAG, SDValue X,
3778 SDValue Carry0, SDValue Carry1,
3779 SDNode *N) {
3780 if (Carry1.getResNo() != 1 || Carry0.getResNo() != 1)
3781 return SDValue();
3782 if (Carry1.getOpcode() != ISD::UADDO)
3783 return SDValue();
3784
3785 SDValue Z;
3786
3787 /**
3788 * First look for a suitable Z. It will present itself in the form of
3789 * (uaddo_carry Y, 0, Z) or its equivalent (uaddo Y, 1) for Z=true
3790 */
3791 if (Carry0.getOpcode() == ISD::UADDO_CARRY &&
3792 isNullConstant(Carry0.getOperand(1))) {
3793 Z = Carry0.getOperand(2);
3794 } else if (Carry0.getOpcode() == ISD::UADDO &&
3795 isOneConstant(Carry0.getOperand(1))) {
3796 EVT VT = Carry0->getValueType(1);
3797 Z = DAG.getConstant(1, SDLoc(Carry0.getOperand(1)), VT);
3798 } else {
3799 // We couldn't find a suitable Z.
3800 return SDValue();
3801 }
3802
3803
3804 auto cancelDiamond = [&](SDValue A,SDValue B) {
3805 SDLoc DL(N);
3806 SDValue NewY =
3807 DAG.getNode(ISD::UADDO_CARRY, DL, Carry0->getVTList(), A, B, Z);
3808 Combiner.AddToWorklist(NewY.getNode());
3809 return DAG.getNode(ISD::UADDO_CARRY, DL, N->getVTList(), X,
3810 DAG.getConstant(0, DL, X.getValueType()),
3811 NewY.getValue(1));
3812 };
3813
3814 /**
3815 * (uaddo A, B)
3816 * |
3817 * Sum
3818 * |
3819 * (uaddo_carry *, 0, Z)
3820 */
3821 if (Carry0.getOperand(0) == Carry1.getValue(0)) {
3822 return cancelDiamond(Carry1.getOperand(0), Carry1.getOperand(1));
3823 }
3824
3825 /**
3826 * (uaddo_carry A, 0, Z)
3827 * |
3828 * Sum
3829 * |
3830 * (uaddo *, B)
3831 */
3832 if (Carry1.getOperand(0) == Carry0.getValue(0)) {
3833 return cancelDiamond(Carry0.getOperand(0), Carry1.getOperand(1));
3834 }
3835
3836 if (Carry1.getOperand(1) == Carry0.getValue(0)) {
3837 return cancelDiamond(Carry1.getOperand(0), Carry0.getOperand(0));
3838 }
3839
3840 return SDValue();
3841}
3842
3843// If we are facing some sort of diamond carry/borrow in/out pattern try to
3844// match patterns like:
3845//
3846// (uaddo A, B) CarryIn
3847// | \ |
3848// | \ |
3849// PartialSum PartialCarryOutX /
3850// | | /
3851// | ____|____________/
3852// | / |
3853// (uaddo *, *) \________
3854// | \ \
3855// | \ |
3856// | PartialCarryOutY |
3857// | \ |
3858// | \ /
3859// AddCarrySum | ______/
3860// | /
3861// CarryOut = (or *, *)
3862//
3863// And generate UADDO_CARRY (or USUBO_CARRY) with two result values:
3864//
3865// {AddCarrySum, CarryOut} = (uaddo_carry A, B, CarryIn)
3866//
3867// Our goal is to identify A, B, and CarryIn and produce UADDO_CARRY/USUBO_CARRY
3868// with a single path for carry/borrow out propagation.
3870 SDValue N0, SDValue N1, SDNode *N) {
3871 SDValue Carry0 = getAsCarry(TLI, N0);
3872 if (!Carry0)
3873 return SDValue();
3874 SDValue Carry1 = getAsCarry(TLI, N1);
3875 if (!Carry1)
3876 return SDValue();
3877
3878 unsigned Opcode = Carry0.getOpcode();
3879 if (Opcode != Carry1.getOpcode())
3880 return SDValue();
3881 if (Opcode != ISD::UADDO && Opcode != ISD::USUBO)
3882 return SDValue();
3883 // Guarantee identical type of CarryOut
3884 EVT CarryOutType = N->getValueType(0);
3885 if (CarryOutType != Carry0.getValue(1).getValueType() ||
3886 CarryOutType != Carry1.getValue(1).getValueType())
3887 return SDValue();
3888
3889 // Canonicalize the add/sub of A and B (the top node in the above ASCII art)
3890 // as Carry0 and the add/sub of the carry in as Carry1 (the middle node).
3891 if (Carry1.getNode()->isOperandOf(Carry0.getNode()))
3892 std::swap(Carry0, Carry1);
3893
3894 // Check if nodes are connected in expected way.
3895 if (Carry1.getOperand(0) != Carry0.getValue(0) &&
3896 Carry1.getOperand(1) != Carry0.getValue(0))
3897 return SDValue();
3898
3899 // The carry in value must be on the righthand side for subtraction.
3900 unsigned CarryInOperandNum =
3901 Carry1.getOperand(0) == Carry0.getValue(0) ? 1 : 0;
3902 if (Opcode == ISD::USUBO && CarryInOperandNum != 1)
3903 return SDValue();
3904 SDValue CarryIn = Carry1.getOperand(CarryInOperandNum);
3905
3906 unsigned NewOp = Opcode == ISD::UADDO ? ISD::UADDO_CARRY : ISD::USUBO_CARRY;
3907 if (!TLI.isOperationLegalOrCustom(NewOp, Carry0.getValue(0).getValueType()))
3908 return SDValue();
3909
3910 // Verify that the carry/borrow in is plausibly a carry/borrow bit.
3911 CarryIn = getAsCarry(TLI, CarryIn, true);
3912 if (!CarryIn)
3913 return SDValue();
3914
3915 SDLoc DL(N);
3916 CarryIn = DAG.getBoolExtOrTrunc(CarryIn, DL, Carry1->getValueType(1),
3917 Carry1->getValueType(0));
3918 SDValue Merged =
3919 DAG.getNode(NewOp, DL, Carry1->getVTList(), Carry0.getOperand(0),
3920 Carry0.getOperand(1), CarryIn);
3921
3922 // Please note that because we have proven that the result of the UADDO/USUBO
3923 // of A and B feeds into the UADDO/USUBO that does the carry/borrow in, we can
3924 // therefore prove that if the first UADDO/USUBO overflows, the second
3925 // UADDO/USUBO cannot. For example consider 8-bit numbers where 0xFF is the
3926 // maximum value.
3927 //
3928 // 0xFF + 0xFF == 0xFE with carry but 0xFE + 1 does not carry
3929 // 0x00 - 0xFF == 1 with a carry/borrow but 1 - 1 == 0 (no carry/borrow)
3930 //
3931 // This is important because it means that OR and XOR can be used to merge
3932 // carry flags; and that AND can return a constant zero.
3933 //
3934 // TODO: match other operations that can merge flags (ADD, etc)
3935 DAG.ReplaceAllUsesOfValueWith(Carry1.getValue(0), Merged.getValue(0));
3936 if (N->getOpcode() == ISD::AND)
3937 return DAG.getConstant(0, DL, CarryOutType);
3938 return Merged.getValue(1);
3939}
3940
3941// Reconstruct a subtract-with-borrow chain from its canonicalized icmp form:
3942// carry_out = or(icmp ult A, B, and(icmp eq A, B, carry_in))
3943// InstCombine folds usub.with.overflow chains into this, losing the
3944// USUBO_CARRY that lowers to sbb/sbcs.
3946 const TargetLowering &TLI) {
3947 SDValue A, B, CarryIn;
3950 m_Deferred(B)),
3951 m_Value(CarryIn)))))
3952 return SDValue();
3953
3954 EVT IntVT = A.getValueType();
3955 // Skip vectors: USUBO_CARRY on a vector type has no legalization path and
3956 // would crash.
3957 if (IntVT.isVector() || !TLI.isOperationLegalOrCustom(
3958 ISD::USUBO_CARRY, TLI.getLegalTypeToTransformTo(
3959 *DAG.getContext(), IntVT)))
3960 return SDValue();
3961
3962 // USUBO_CARRY's carry-in must match boolean contents, which the matched
3963 // pattern does not guarantee.
3964 // TODO: Extend to other boolean contents.
3965 if (TLI.getBooleanContents(IntVT) !=
3967 !DAG.MaskedValueIsZero(
3968 CarryIn,
3970 return SDValue();
3971
3972 SDLoc DL(N);
3973 SDVTList VTs = DAG.getVTList(IntVT, N->getValueType(0));
3974 return DAG.getNode(ISD::USUBO_CARRY, DL, VTs, A, B, CarryIn).getValue(1);
3975}
3976
3977SDValue DAGCombiner::visitUADDO_CARRYLike(SDValue N0, SDValue N1,
3978 SDValue CarryIn, SDNode *N) {
3979 // fold (uaddo_carry (xor a, -1), b, c) -> (usubo_carry b, a, !c) and flip
3980 // carry.
3981 if (isBitwiseNot(N0))
3982 if (SDValue NotC = extractBooleanFlip(CarryIn, DAG, TLI, true)) {
3983 SDLoc DL(N);
3984 SDValue Sub = DAG.getNode(ISD::USUBO_CARRY, DL, N->getVTList(), N1,
3985 N0.getOperand(0), NotC);
3986 return CombineTo(
3987 N, Sub, DAG.getLogicalNOT(DL, Sub.getValue(1), Sub->getValueType(1)));
3988 }
3989
3990 // Iff the flag result is dead:
3991 // (uaddo_carry (add|uaddo X, Y), 0, Carry) -> (uaddo_carry X, Y, Carry)
3992 // Don't do this if the Carry comes from the uaddo. It won't remove the uaddo
3993 // or the dependency between the instructions.
3994 if ((N0.getOpcode() == ISD::ADD ||
3995 (N0.getOpcode() == ISD::UADDO && N0.getResNo() == 0 &&
3996 N0.getValue(1) != CarryIn)) &&
3997 isNullConstant(N1) && !N->hasAnyUseOfValue(1))
3998 return DAG.getNode(ISD::UADDO_CARRY, SDLoc(N), N->getVTList(),
3999 N0.getOperand(0), N0.getOperand(1), CarryIn);
4000
4001 /**
4002 * When one of the uaddo_carry argument is itself a carry, we may be facing
4003 * a diamond carry propagation. In which case we try to transform the DAG
4004 * to ensure linear carry propagation if that is possible.
4005 */
4006 if (auto Y = getAsCarry(TLI, N1)) {
4007 // Because both are carries, Y and Z can be swapped.
4008 if (auto R = combineUADDO_CARRYDiamond(*this, DAG, N0, Y, CarryIn, N))
4009 return R;
4010 if (auto R = combineUADDO_CARRYDiamond(*this, DAG, N0, CarryIn, Y, N))
4011 return R;
4012 }
4013
4014 return SDValue();
4015}
4016
4017SDValue DAGCombiner::visitSADDO_CARRYLike(SDValue N0, SDValue N1,
4018 SDValue CarryIn, SDNode *N) {
4019 // fold (saddo_carry (xor a, -1), b, c) -> (ssubo_carry b, a, !c)
4020 if (isBitwiseNot(N0)) {
4021 if (SDValue NotC = extractBooleanFlip(CarryIn, DAG, TLI, true))
4022 return DAG.getNode(ISD::SSUBO_CARRY, SDLoc(N), N->getVTList(), N1,
4023 N0.getOperand(0), NotC);
4024 }
4025
4026 return SDValue();
4027}
4028
4029SDValue DAGCombiner::visitSADDO_CARRY(SDNode *N) {
4030 SDValue N0 = N->getOperand(0);
4031 SDValue N1 = N->getOperand(1);
4032 SDValue CarryIn = N->getOperand(2);
4033 SDLoc DL(N);
4034
4035 // canonicalize constant to RHS
4036 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
4037 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
4038 if (N0C && !N1C)
4039 return DAG.getNode(ISD::SADDO_CARRY, DL, N->getVTList(), N1, N0, CarryIn);
4040
4041 // fold (saddo_carry x, y, false) -> (saddo x, y)
4042 if (isNullConstant(CarryIn)) {
4043 if (!LegalOperations ||
4044 TLI.isOperationLegalOrCustom(ISD::SADDO, N->getValueType(0)))
4045 return DAG.getNode(ISD::SADDO, DL, N->getVTList(), N0, N1);
4046 }
4047
4048 if (SDValue Combined = visitSADDO_CARRYLike(N0, N1, CarryIn, N))
4049 return Combined;
4050
4051 if (SDValue Combined = visitSADDO_CARRYLike(N1, N0, CarryIn, N))
4052 return Combined;
4053
4054 return SDValue();
4055}
4056
4057// Attempt to create a USUBSAT(LHS, RHS) node with DstVT, performing a
4058// clamp/truncation if necessary.
4060 SDValue RHS, SelectionDAG &DAG,
4061 const SDLoc &DL) {
4062 assert(DstVT.getScalarSizeInBits() <= SrcVT.getScalarSizeInBits() &&
4063 "Illegal truncation");
4064
4065 if (DstVT == SrcVT)
4066 return DAG.getNode(ISD::USUBSAT, DL, DstVT, LHS, RHS);
4067
4068 // If the LHS is zero-extended then we can perform the USUBSAT as DstVT by
4069 // clamping RHS.
4071 DstVT.getScalarSizeInBits());
4072 if (!DAG.MaskedValueIsZero(LHS, UpperBits))
4073 return SDValue();
4074
4075 SDValue SatLimit =
4077 DstVT.getScalarSizeInBits()),
4078 DL, SrcVT);
4079 RHS = DAG.getNode(ISD::UMIN, DL, SrcVT, RHS, SatLimit);
4080 RHS = DAG.getNode(ISD::TRUNCATE, DL, DstVT, RHS);
4081 LHS = DAG.getNode(ISD::TRUNCATE, DL, DstVT, LHS);
4082 return DAG.getNode(ISD::USUBSAT, DL, DstVT, LHS, RHS);
4083}
4084
4085// Try to find umax(a,b) - b or a - umin(a,b) patterns that may be converted to
4086// usubsat(a,b), optionally as a truncated type.
4087SDValue DAGCombiner::foldSubToUSubSat(EVT DstVT, SDNode *N, const SDLoc &DL) {
4088 if (N->getOpcode() != ISD::SUB ||
4089 !(!LegalOperations || hasOperation(ISD::USUBSAT, DstVT)))
4090 return SDValue();
4091
4092 EVT SubVT = N->getValueType(0);
4093 SDValue Op0 = N->getOperand(0);
4094 SDValue Op1 = N->getOperand(1);
4095
4096 // Try to find umax(a,b) - b or a - umin(a,b) patterns
4097 // they may be converted to usubsat(a,b).
4098 if (Op0.getOpcode() == ISD::UMAX && Op0.hasOneUse()) {
4099 SDValue MaxLHS = Op0.getOperand(0);
4100 SDValue MaxRHS = Op0.getOperand(1);
4101 if (MaxLHS == Op1)
4102 return getTruncatedUSUBSAT(DstVT, SubVT, MaxRHS, Op1, DAG, DL);
4103 if (MaxRHS == Op1)
4104 return getTruncatedUSUBSAT(DstVT, SubVT, MaxLHS, Op1, DAG, DL);
4105 }
4106
4107 if (Op1.getOpcode() == ISD::UMIN && Op1.hasOneUse()) {
4108 SDValue MinLHS = Op1.getOperand(0);
4109 SDValue MinRHS = Op1.getOperand(1);
4110 if (MinLHS == Op0)
4111 return getTruncatedUSUBSAT(DstVT, SubVT, Op0, MinRHS, DAG, DL);
4112 if (MinRHS == Op0)
4113 return getTruncatedUSUBSAT(DstVT, SubVT, Op0, MinLHS, DAG, DL);
4114 }
4115
4116 // sub(a,trunc(umin(zext(a),b))) -> usubsat(a,trunc(umin(b,SatLimit)))
4117 if (Op1.getOpcode() == ISD::TRUNCATE &&
4118 Op1.getOperand(0).getOpcode() == ISD::UMIN &&
4119 Op1.getOperand(0).hasOneUse()) {
4120 SDValue MinLHS = Op1.getOperand(0).getOperand(0);
4121 SDValue MinRHS = Op1.getOperand(0).getOperand(1);
4122 if (MinLHS.getOpcode() == ISD::ZERO_EXTEND && MinLHS.getOperand(0) == Op0)
4123 return getTruncatedUSUBSAT(DstVT, MinLHS.getValueType(), MinLHS, MinRHS,
4124 DAG, DL);
4125 if (MinRHS.getOpcode() == ISD::ZERO_EXTEND && MinRHS.getOperand(0) == Op0)
4126 return getTruncatedUSUBSAT(DstVT, MinLHS.getValueType(), MinRHS, MinLHS,
4127 DAG, DL);
4128 }
4129
4130 return SDValue();
4131}
4132
4133// Refinement of DAG/Type Legalisation (promotion) when CTLZ is used for
4134// counting leading ones. Broadly, it replaces the substraction with a left
4135// shift.
4136//
4137// * DAG Legalisation Pattern:
4138//
4139// (sub (ctlz (zeroextend (not Src)))
4140// BitWidthDiff)
4141//
4142// if BitWidthDiff == BitWidth(Node) - BitWidth(Src)
4143// -->
4144//
4145// (ctlz_zero_poison (not (shl (anyextend Src)
4146// BitWidthDiff)))
4147//
4148// * Type Legalisation Pattern:
4149//
4150// (sub (ctlz (and (xor Src XorMask)
4151// AndMask))
4152// BitWidthDiff)
4153//
4154// if AndMask has only trailing ones
4155// and MaskBitWidth(AndMask) == BitWidth(Node) - BitWidthDiff
4156// and XorMask has more trailing ones than AndMask
4157// -->
4158//
4159// (ctlz_zero_poison (not (shl Src BitWidthDiff)))
4161 const SDLoc DL(N);
4162 SDValue N0 = N->getOperand(0);
4163 EVT VT = N0.getValueType();
4164 unsigned BitWidth = VT.getScalarSizeInBits();
4165
4166 APInt AndMask;
4167 APInt XorMask;
4168 uint64_t BitWidthDiff;
4169
4170 SDValue CtlzOp;
4171 SDValue Src;
4172
4173 if (!sd_match(N, m_Sub(m_Ctlz(m_Value(CtlzOp)), m_ConstInt(BitWidthDiff))))
4174 return SDValue();
4175
4176 if (sd_match(CtlzOp, m_ZExt(m_Not(m_Value(Src))))) {
4177 // DAG Legalisation Pattern:
4178 // (sub (ctlz (zero_extend (not Op)) BitWidthDiff))
4179 if ((BitWidth - Src.getValueType().getScalarSizeInBits()) != BitWidthDiff)
4180 return SDValue();
4181
4182 Src = DAG.getNode(ISD::ANY_EXTEND, DL, VT, Src);
4183 } else if (sd_match(CtlzOp, m_And(m_Xor(m_Value(Src), m_ConstInt(XorMask)),
4184 m_ConstInt(AndMask)))) {
4185 // Type Legalisation Pattern:
4186 // (sub (ctlz (and (xor Op XorMask) AndMask)) BitWidthDiff)
4187 if (BitWidthDiff >= BitWidth)
4188 return SDValue();
4189 unsigned AndMaskWidth = BitWidth - BitWidthDiff;
4190 if (!(AndMask.isMask(AndMaskWidth) && XorMask.countr_one() >= AndMaskWidth))
4191 return SDValue();
4192 } else
4193 return SDValue();
4194
4195 SDValue ShiftConst = DAG.getShiftAmountConstant(BitWidthDiff, VT, DL);
4196 SDValue LShift = DAG.getNode(ISD::SHL, DL, VT, Src, ShiftConst);
4197 SDValue Not =
4198 DAG.getNode(ISD::XOR, DL, VT, LShift, DAG.getAllOnesConstant(DL, VT));
4199
4200 return DAG.getNode(ISD::CTLZ_ZERO_POISON, DL, VT, Not);
4201}
4202
4203// Fold sub(x, mul(divrem(x,y)[0], y)) to divrem(x, y)[1]
4205 const SDLoc &DL) {
4206 assert(N->getOpcode() == ISD::SUB && "Node must be a SUB");
4207 SDValue Sub0 = N->getOperand(0);
4208 SDValue Sub1 = N->getOperand(1);
4209
4210 auto CheckAndFoldMulCase = [&](SDValue DivRem, SDValue MaybeY) -> SDValue {
4211 if ((DivRem.getOpcode() == ISD::SDIVREM ||
4212 DivRem.getOpcode() == ISD::UDIVREM) &&
4213 DivRem.getResNo() == 0 && DivRem.getOperand(0) == Sub0 &&
4214 DivRem.getOperand(1) == MaybeY) {
4215 return SDValue(DivRem.getNode(), 1);
4216 }
4217 return SDValue();
4218 };
4219
4220 if (Sub1.getOpcode() == ISD::MUL) {
4221 // (sub x, (mul divrem(x,y)[0], y))
4222 SDValue Mul0 = Sub1.getOperand(0);
4223 SDValue Mul1 = Sub1.getOperand(1);
4224
4225 if (SDValue Res = CheckAndFoldMulCase(Mul0, Mul1))
4226 return Res;
4227
4228 if (SDValue Res = CheckAndFoldMulCase(Mul1, Mul0))
4229 return Res;
4230
4231 } else if (Sub1.getOpcode() == ISD::SHL) {
4232 // Handle (sub x, (shl divrem(x,y)[0], C)) where y = 1 << C
4233 SDValue Shl0 = Sub1.getOperand(0);
4234 SDValue Shl1 = Sub1.getOperand(1);
4235 // Check if Shl0 is divrem(x, Y)[0]
4236 if ((Shl0.getOpcode() == ISD::SDIVREM ||
4237 Shl0.getOpcode() == ISD::UDIVREM) &&
4238 Shl0.getResNo() == 0 && Shl0.getOperand(0) == Sub0) {
4239
4240 SDValue Divisor = Shl0.getOperand(1);
4241
4242 ConstantSDNode *DivC = isConstOrConstSplat(Divisor);
4244 if (!DivC || !ShC)
4245 return SDValue();
4246
4247 if (DivC->getAPIntValue().isPowerOf2() &&
4248 DivC->getAPIntValue().logBase2() == ShC->getAPIntValue())
4249 return SDValue(Shl0.getNode(), 1);
4250 }
4251 }
4252 return SDValue();
4253}
4254
4255// Since it may not be valid to emit a fold to zero for vector initializers
4256// check if we can before folding.
4258 SelectionDAG &DAG, bool LegalOperations) {
4259 if (!VT.isVector())
4260 return DAG.getConstant(0, DL, VT);
4261 if (!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT))
4262 return DAG.getConstant(0, DL, VT);
4263 return SDValue();
4264}
4265
4266SDValue DAGCombiner::visitSUB(SDNode *N) {
4267 SDValue N0 = N->getOperand(0);
4268 SDValue N1 = N->getOperand(1);
4269 EVT VT = N0.getValueType();
4270 unsigned BitWidth = VT.getScalarSizeInBits();
4271 SDLoc DL(N);
4272
4273 if (SDValue V = foldSubCtlzNot(N, DAG))
4274 return V;
4275
4276 // fold (sub x, x) -> 0
4277 if (N0 == N1)
4278 return tryFoldToZero(DL, TLI, VT, DAG, LegalOperations);
4279
4280 // fold (sub c1, c2) -> c3
4281 if (SDValue C =
4282 DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N0, N1}, N->getFlags()))
4283 return C;
4284
4285 // fold vector ops
4286 if (VT.isVector()) {
4287 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
4288 return FoldedVOp;
4289
4290 // fold (sub x, 0) -> x, vector edition
4292 return N0;
4293 }
4294
4295 // (sub x, ([v]select (ult x, y), 0, y)) -> (umin x, (sub x, y))
4296 // (sub x, ([v]select (uge x, y), y, 0)) -> (umin x, (sub x, y))
4297 if (N1.hasOneUse() && hasUMin(VT)) {
4298 SDValue Y;
4299 auto MS0 = m_Specific(N0);
4300 auto MVY = m_Value(Y);
4301 auto MZ = m_Zero();
4302
4303 if (sd_match(N1, m_SpecificSelectCCLike(ISD::SETULT, MS0, MVY, MZ,
4304 m_Deferred(Y))) ||
4306 m_Deferred(Y), MZ)) ||
4307 sd_match(N1, m_VSelect(m_SpecificSetCC(ISD::SETULT, MS0, MVY), MZ,
4308 m_Deferred(Y))) ||
4310 m_Deferred(Y), MZ)))
4311
4312 return DAG.getNode(ISD::UMIN, DL, VT, N0,
4313 DAG.getNode(ISD::SUB, DL, VT, N0, Y));
4314 }
4315
4316 if (SDValue NewSel = foldBinOpIntoSelect(N))
4317 return NewSel;
4318
4319 // fold (sub x, c) -> (add x, -c)
4320 if (ConstantSDNode *N1C = getAsNonOpaqueConstant(N1))
4321 return DAG.getNode(ISD::ADD, DL, VT, N0,
4322 DAG.getConstant(-N1C->getAPIntValue(), DL, VT));
4323
4324 if (isNullOrNullSplat(N0)) {
4325 // Right-shifting everything out but the sign bit followed by negation is
4326 // the same as flipping arithmetic/logical shift type without the negation:
4327 // -(X >>u 31) -> (X >>s 31)
4328 // -(X >>s 31) -> (X >>u 31)
4329 if (N1->getOpcode() == ISD::SRA || N1->getOpcode() == ISD::SRL) {
4330 ConstantSDNode *ShiftAmt = isConstOrConstSplat(N1.getOperand(1));
4331 if (ShiftAmt && ShiftAmt->getAPIntValue() == (BitWidth - 1)) {
4332 auto NewSh = N1->getOpcode() == ISD::SRA ? ISD::SRL : ISD::SRA;
4333 if (!LegalOperations || TLI.isOperationLegal(NewSh, VT))
4334 return DAG.getNode(NewSh, DL, VT, N1.getOperand(0), N1.getOperand(1));
4335 }
4336 }
4337
4338 // 0 - X --> 0 if the sub is NUW.
4339 if (N->getFlags().hasNoUnsignedWrap())
4340 return N0;
4341
4343 // N1 is either 0 or the minimum signed value. If the sub is NSW, then
4344 // N1 must be 0 because negating the minimum signed value is undefined.
4345 if (N->getFlags().hasNoSignedWrap())
4346 return N0;
4347
4348 // 0 - X --> X if X is 0 or the minimum signed value.
4349 return N1;
4350 }
4351
4352 // Convert 0 - abs(x).
4353 if (ISD::isAbsOpcode(N1.getOpcode()) && N1.hasOneUse() &&
4355 if (SDValue Result = TLI.expandABS(N1.getNode(), DAG, true))
4356 return Result;
4357
4358 // Similar to the previous rule, but this time targeting an expanded abs.
4359 // (sub 0, (max X, (sub 0, X))) --> (min X, (sub 0, X))
4360 // as well as
4361 // (sub 0, (min X, (sub 0, X))) --> (max X, (sub 0, X))
4362 // Note that these two are applicable to both signed and unsigned min/max.
4363 SDValue X;
4364 SDValue S0;
4365 auto NegPat = m_Value(S0, m_Neg(m_Deferred(X)));
4366 if (sd_match(N1, m_OneUse(m_AnyOf(m_SMax(m_Value(X), NegPat),
4367 m_UMax(m_Value(X), NegPat),
4368 m_SMin(m_Value(X), NegPat),
4369 m_UMin(m_Value(X), NegPat))))) {
4370 unsigned NewOpc = ISD::getInverseMinMaxOpcode(N1->getOpcode());
4371 // If the type is legal and the existing min/max isn't supported go ahead
4372 // and do the conversion. The expansion of the new min/max will likely be
4373 // similar to the old.
4374 if (hasOperation(NewOpc, VT) ||
4375 (isTypeLegal(VT) && !hasOperation(N1->getOpcode(), VT)))
4376 return DAG.getNode(NewOpc, DL, VT, X, S0);
4377 }
4378
4379 // Fold neg(splat(neg(x)) -> splat(x)
4380 if (VT.isVector()) {
4381 SDValue N1S = DAG.getSplatValue(N1, true);
4382 if (N1S && N1S.getOpcode() == ISD::SUB &&
4383 isNullConstant(N1S.getOperand(0)))
4384 return DAG.getSplat(VT, DL, N1S.getOperand(1));
4385 }
4386
4387 // sub 0, (and x, 1) --> SIGN_EXTEND_INREG x, i1
4388 if (N1.getOpcode() == ISD::AND && N1.hasOneUse() &&
4389 isOneOrOneSplat(N1->getOperand(1))) {
4390 EVT ExtVT = VT.changeElementType(*DAG.getContext(), MVT::i1);
4393 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, N1->getOperand(0),
4394 DAG.getValueType(ExtVT));
4395 }
4396 }
4397 }
4398
4399 // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1)
4401 return DAG.getNode(ISD::XOR, DL, VT, N1, N0);
4402
4403 // fold (A - (0-B)) -> A+B
4404 if (N1.getOpcode() == ISD::SUB && isNullOrNullSplat(N1.getOperand(0)))
4405 return DAG.getNode(ISD::ADD, DL, VT, N0, N1.getOperand(1));
4406
4407 // fold A-(A-B) -> B
4408 if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(0))
4409 return N1.getOperand(1);
4410
4411 // fold (A+B)-A -> B
4412 if (N0.getOpcode() == ISD::ADD && N0.getOperand(0) == N1)
4413 return N0.getOperand(1);
4414
4415 // fold (A+B)-B -> A
4416 if (N0.getOpcode() == ISD::ADD && N0.getOperand(1) == N1)
4417 return N0.getOperand(0);
4418
4419 // fold (A+C1)-C2 -> A+(C1-C2)
4420 if (N0.getOpcode() == ISD::ADD) {
4421 SDValue N01 = N0.getOperand(1);
4422 if (SDValue NewC = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N01, N1}))
4423 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), NewC);
4424 }
4425
4426 // fold C2-(A+C1) -> (C2-C1)-A
4427 if (N1.getOpcode() == ISD::ADD) {
4428 SDValue N11 = N1.getOperand(1);
4429 if (SDValue NewC = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N0, N11}))
4430 return DAG.getNode(ISD::SUB, DL, VT, NewC, N1.getOperand(0));
4431 }
4432
4433 // fold (A-C1)-C2 -> A-(C1+C2)
4434 if (N0.getOpcode() == ISD::SUB) {
4435 SDValue N01 = N0.getOperand(1);
4436 if (SDValue NewC = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N01, N1}))
4437 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), NewC);
4438 }
4439
4440 // fold (c1-A)-c2 -> (c1-c2)-A
4441 if (N0.getOpcode() == ISD::SUB) {
4442 SDValue N00 = N0.getOperand(0);
4443 if (SDValue NewC = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N00, N1}))
4444 return DAG.getNode(ISD::SUB, DL, VT, NewC, N0.getOperand(1));
4445 }
4446
4447 SDValue A, B, C;
4448
4449 // fold ((A+(B+C))-B) -> A+C
4450 if (sd_match(N0, m_Add(m_Value(A), m_Add(m_Specific(N1), m_Value(C)))))
4451 return DAG.getNode(ISD::ADD, DL, VT, A, C);
4452
4453 // fold ((A+(B-C))-B) -> A-C
4454 if (sd_match(N0, m_Add(m_Value(A), m_Sub(m_Specific(N1), m_Value(C)))))
4455 return DAG.getNode(ISD::SUB, DL, VT, A, C);
4456
4457 // fold ((A-(B-C))-C) -> A-B
4458 if (sd_match(N0, m_Sub(m_Value(A), m_Sub(m_Value(B), m_Specific(N1)))))
4459 return DAG.getNode(ISD::SUB, DL, VT, A, B);
4460
4461 // fold (A-(B-C)) -> A+(C-B)
4462 if (sd_match(N1, m_OneUse(m_Sub(m_Value(B), m_Value(C)))))
4463 return DAG.getNode(ISD::ADD, DL, VT, N0,
4464 DAG.getNode(ISD::SUB, DL, VT, C, B));
4465
4466 // A - (A & B) -> A & (~B)
4467 if (sd_match(N1, m_And(m_Specific(N0), m_Value(B))) &&
4468 (N1.hasOneUse() || isConstantOrConstantVector(B, /*NoOpaques=*/true)))
4469 return DAG.getNode(ISD::AND, DL, VT, N0, DAG.getNOT(DL, B, VT));
4470
4471 // fold (A - (-B * C)) -> (A + (B * C))
4472 if (sd_match(N1, m_OneUse(m_Mul(m_Neg(m_Value(B)), m_Value(C)))))
4473 return DAG.getNode(ISD::ADD, DL, VT, N0,
4474 DAG.getNode(ISD::MUL, DL, VT, B, C));
4475
4476 // If either operand of a sub is undef, the result is undef
4477 if (N0.isUndef())
4478 return N0;
4479 if (N1.isUndef())
4480 return N1;
4481
4482 if (SDValue V = foldAddSubBoolOfMaskedVal(N, DL, DAG))
4483 return V;
4484
4485 if (SDValue V = foldAddSubOfSignBit(N, DL, DAG))
4486 return V;
4487
4488 // Try to match AVGCEIL fixedwidth pattern
4489 if (SDValue V = foldSubToAvg(N, DL))
4490 return V;
4491
4492 if (SDValue V = foldAddSubMasked1(false, N0, N1, DAG, DL))
4493 return V;
4494
4495 if (SDValue V = foldSubToUSubSat(VT, N, DL))
4496 return V;
4497
4498 if (SDValue V = foldRemainderIdiom(N, DAG, DL))
4499 return V;
4500
4501 // (A - B) - 1 -> add (xor B, -1), A
4503 m_One(/*AllowUndefs=*/true))))
4504 return DAG.getNode(ISD::ADD, DL, VT, A, DAG.getNOT(DL, B, VT));
4505
4506 // Look for:
4507 // sub y, (xor x, -1)
4508 // And if the target does not like this form then turn into:
4509 // add (add x, y), 1
4510 if (TLI.preferIncOfAddToSubOfNot(VT) && N1.hasOneUse() && isBitwiseNot(N1)) {
4511 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, N1.getOperand(0));
4512 return DAG.getNode(ISD::ADD, DL, VT, Add, DAG.getConstant(1, DL, VT));
4513 }
4514
4515 // Hoist one-use addition by non-opaque constant:
4516 // (x + C) - y -> (x - y) + C
4517 if (!reassociationCanBreakAddressingModePattern(ISD::SUB, DL, N, N0, N1) &&
4518 N0.getOpcode() == ISD::ADD && N0.hasOneUse() &&
4519 isConstantOrConstantVector(N0.getOperand(1), /*NoOpaques=*/true)) {
4520 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), N1);
4521 return DAG.getNode(ISD::ADD, DL, VT, Sub, N0.getOperand(1));
4522 }
4523 // y - (x + C) -> (y - x) - C
4524 if (N1.getOpcode() == ISD::ADD && N1.hasOneUse() &&
4525 isConstantOrConstantVector(N1.getOperand(1), /*NoOpaques=*/true)) {
4526 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, N1.getOperand(0));
4527 return DAG.getNode(ISD::SUB, DL, VT, Sub, N1.getOperand(1));
4528 }
4529 // (x - C) - y -> (x - y) - C
4530 // This is necessary because SUB(X,C) -> ADD(X,-C) doesn't work for vectors.
4531 if (N0.getOpcode() == ISD::SUB && N0.hasOneUse() &&
4532 isConstantOrConstantVector(N0.getOperand(1), /*NoOpaques=*/true)) {
4533 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), N1);
4534 return DAG.getNode(ISD::SUB, DL, VT, Sub, N0.getOperand(1));
4535 }
4536 // (C - x) - y -> C - (x + y)
4537 if (N0.getOpcode() == ISD::SUB && N0.hasOneUse() &&
4538 isConstantOrConstantVector(N0.getOperand(0), /*NoOpaques=*/true)) {
4539 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(1), N1);
4540 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), Add);
4541 }
4542
4543 // If the target's bool is represented as 0/-1, prefer to make this 'add 0/-1'
4544 // rather than 'sub 0/1' (the sext should get folded).
4545 // sub X, (zext i1 Y) --> add X, (sext i1 Y)
4546 if (N1.getOpcode() == ISD::ZERO_EXTEND &&
4547 N1.getOperand(0).getScalarValueSizeInBits() == 1 &&
4548 TLI.getBooleanContents(VT) ==
4550 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, N1.getOperand(0));
4551 return DAG.getNode(ISD::ADD, DL, VT, N0, SExt);
4552 }
4553
4554 // fold B = sra (A, size(A)-1); sub (xor (A, B), B) -> (abs A)
4555 if ((!LegalOperations || hasOperation(ISD::ABS, VT)) &&
4557 sd_match(N0, m_Xor(m_Specific(A), m_Specific(N1))))
4558 return DAG.getNode(ISD::ABS, DL, VT, A);
4559
4560 // If the relocation model supports it, consider symbol offsets.
4561 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N0))
4562 if (!LegalOperations && TLI.isOffsetFoldingLegal(GA)) {
4563 // fold (sub Sym+c1, Sym+c2) -> c1-c2
4564 if (GlobalAddressSDNode *GB = dyn_cast<GlobalAddressSDNode>(N1))
4565 if (GA->getGlobal() == GB->getGlobal())
4566 return DAG.getConstant(
4567 APInt(VT.getScalarSizeInBits(), GA->getOffset() - GB->getOffset(),
4568 /*isSigned=*/false, /*implicitTrunc=*/true),
4569 DL, VT);
4570 }
4571
4572 // sub X, (sextinreg Y i1) -> add X, (and Y 1)
4573 if (N1.getOpcode() == ISD::SIGN_EXTEND_INREG) {
4574 VTSDNode *TN = cast<VTSDNode>(N1.getOperand(1));
4575 if (TN->getVT() == MVT::i1) {
4576 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0),
4577 DAG.getConstant(1, DL, VT));
4578 return DAG.getNode(ISD::ADD, DL, VT, N0, ZExt);
4579 }
4580 }
4581
4582 // canonicalize (sub X, (vscale * C)) to (add X, (vscale * -C)) if this is the
4583 // only use of the vscale value or if (vscale * -C) is a valid add immediate.
4584 // avoid if ISD::MUL handling is poor and ISD::SHL isn't an option.
4585 if (N1.getOpcode() == ISD::VSCALE) {
4586 const APInt &IntVal = N1.getConstantOperandAPInt(0);
4587 if ((N1.hasOneUse() ||
4588 TLI.isLegalAddScalableImmediate(-IntVal.getSExtValue())) &&
4589 (!IntVal.isPowerOf2() ||
4590 hasOperation(ISD::MUL, N1.getOperand(0).getValueType())))
4591 return DAG.getNode(ISD::ADD, DL, VT, N0, DAG.getVScale(DL, VT, -IntVal));
4592 }
4593
4594 // canonicalize (sub X, step_vector(C)) to (add X, step_vector(-C))
4595 if (N1.getOpcode() == ISD::STEP_VECTOR && N1.hasOneUse()) {
4596 APInt NewStep = -N1.getConstantOperandAPInt(0);
4597 return DAG.getNode(ISD::ADD, DL, VT, N0,
4598 DAG.getStepVector(DL, VT, NewStep));
4599 }
4600
4601 // Prefer an add for more folding potential and possibly better codegen:
4602 // sub N0, (lshr N10, width-1) --> add N0, (ashr N10, width-1)
4603 if (!LegalOperations && N1.getOpcode() == ISD::SRL && N1.hasOneUse()) {
4604 SDValue ShAmt = N1.getOperand(1);
4605 ConstantSDNode *ShAmtC = isConstOrConstSplat(ShAmt);
4606 if (ShAmtC && ShAmtC->getAPIntValue() == (BitWidth - 1)) {
4607 SDValue SRA = DAG.getNode(ISD::SRA, DL, VT, N1.getOperand(0), ShAmt);
4608 return DAG.getNode(ISD::ADD, DL, VT, N0, SRA);
4609 }
4610 }
4611
4612 // As with the previous fold, prefer add for more folding potential.
4613 // Subtracting SMIN/0 is the same as adding SMIN/0:
4614 // N0 - (X << BW-1) --> N0 + (X << BW-1)
4615 if (N1.getOpcode() == ISD::SHL) {
4616 ConstantSDNode *ShlC = isConstOrConstSplat(N1.getOperand(1));
4617 if (ShlC && ShlC->getAPIntValue() == (BitWidth - 1))
4618 return DAG.getNode(ISD::ADD, DL, VT, N1, N0);
4619 }
4620
4621 // (sub (usubo_carry X, 0, Carry), Y) -> (usubo_carry X, Y, Carry)
4622 if (N0.getOpcode() == ISD::USUBO_CARRY && isNullConstant(N0.getOperand(1)) &&
4623 N0.getResNo() == 0 && N0.hasOneUse())
4624 return DAG.getNode(ISD::USUBO_CARRY, DL, N0->getVTList(),
4625 N0.getOperand(0), N1, N0.getOperand(2));
4626
4628 // (sub Carry, X) -> (uaddo_carry (sub 0, X), 0, Carry)
4629 if (SDValue Carry = getAsCarry(TLI, N0)) {
4630 SDValue X = N1;
4631 SDValue Zero = DAG.getConstant(0, DL, VT);
4632 SDValue NegX = DAG.getNode(ISD::SUB, DL, VT, Zero, X);
4633 return DAG.getNode(ISD::UADDO_CARRY, DL,
4634 DAG.getVTList(VT, Carry.getValueType()), NegX, Zero,
4635 Carry);
4636 }
4637 }
4638
4639 if (ConstantSDNode *C0 = isConstOrConstSplat(N0)) {
4640 const APInt &C0Val = C0->getAPIntValue();
4641
4642 // sub nuw C, x --> xor x, C when C is a mask (2^k - 1)
4643 if (N->getFlags().hasNoUnsignedWrap() && C0Val.isMask())
4644 return DAG.getNode(ISD::XOR, DL, VT, N1, N0);
4645
4646 // If there's no chance of borrowing from adjacent bits, then sub is xor:
4647 // sub C0, X --> xor X, C0
4648 if (!C0->isOpaque()) {
4649 const APInt &MaybeOnes = ~DAG.computeKnownBits(N1).Zero;
4650 if ((C0Val - MaybeOnes) == (C0Val ^ MaybeOnes))
4651 return DAG.getNode(ISD::XOR, DL, VT, N1, N0);
4652 }
4653 }
4654
4655 // smax(a,b) - smin(a,b) --> abds(a,b)
4656 if ((!LegalOperations || hasOperation(ISD::ABDS, VT)) &&
4657 sd_match(N0, m_SMaxLike(m_Value(A), m_Value(B))) &&
4659 return DAG.getNode(ISD::ABDS, DL, VT, A, B);
4660
4661 // smin(a,b) - smax(a,b) --> neg(abds(a,b))
4662 if ((!LegalOperations || hasOperation(ISD::ABDS, VT)) &&
4663 sd_match(N0, m_SMinLike(m_Value(A), m_Value(B))) &&
4665 return DAG.getNegative(DAG.getNode(ISD::ABDS, DL, VT, A, B), DL, VT);
4666
4667 // umax(a,b) - umin(a,b) --> abdu(a,b)
4668 if ((!LegalOperations || hasOperation(ISD::ABDU, VT)) &&
4669 sd_match(N0, m_UMaxLike(m_Value(A), m_Value(B))) &&
4671 return DAG.getNode(ISD::ABDU, DL, VT, A, B);
4672
4673 // umin(a,b) - umax(a,b) --> neg(abdu(a,b))
4674 if ((!LegalOperations || hasOperation(ISD::ABDU, VT)) &&
4675 sd_match(N0, m_UMinLike(m_Value(A), m_Value(B))) &&
4677 return DAG.getNegative(DAG.getNode(ISD::ABDU, DL, VT, A, B), DL, VT);
4678
4679 return SDValue();
4680}
4681
4682SDValue DAGCombiner::visitSUBSAT(SDNode *N) {
4683 unsigned Opcode = N->getOpcode();
4684 SDValue N0 = N->getOperand(0);
4685 SDValue N1 = N->getOperand(1);
4686 EVT VT = N0.getValueType();
4687 bool IsSigned = Opcode == ISD::SSUBSAT;
4688 SDLoc DL(N);
4689
4690 // fold (sub_sat x, undef) -> 0
4691 if (N0.isUndef() || N1.isUndef())
4692 return DAG.getConstant(0, DL, VT);
4693
4694 // fold (sub_sat x, x) -> 0
4695 if (N0 == N1)
4696 return DAG.getConstant(0, DL, VT);
4697
4698 // fold (sub_sat c1, c2) -> c3
4699 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1}))
4700 return C;
4701
4702 // fold vector ops
4703 if (VT.isVector()) {
4704 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
4705 return FoldedVOp;
4706
4707 // fold (sub_sat x, 0) -> x, vector edition
4709 return N0;
4710 }
4711
4712 // fold (sub_sat x, 0) -> x
4713 if (isNullConstant(N1))
4714 return N0;
4715
4716 // If it cannot overflow, transform into an sub.
4717 if (DAG.willNotOverflowSub(IsSigned, N0, N1))
4718 return DAG.getNode(ISD::SUB, DL, VT, N0, N1);
4719
4720 // Narrow a vXiN USUBSAT to a smaller type when both operands are known
4721 // to fit in fewer bits. This allows targets with native narrow USUBSAT
4722 // (e.g. vpsubusb/vpsubusw) to avoid emulation with vpmaxu + vsub.
4723 if (!IsSigned && VT.isVector() && VT.isSimple()) {
4724 unsigned ScalarBits = VT.getScalarSizeInBits();
4725 if (ScalarBits > 8 && isPowerOf2_32(ScalarBits) &&
4727 KnownBits Known0 = DAG.computeKnownBits(N0);
4728 unsigned ActiveBits = Known0.countMaxActiveBits();
4729 for (unsigned NarrowBits = PowerOf2Ceil(ActiveBits);
4730 NarrowBits != 0 && NarrowBits < ScalarBits; NarrowBits *= 2) {
4731 unsigned Scale = ScalarBits / NarrowBits;
4732 ElementCount ScaledEC = VT.getVectorElementCount() * Scale;
4733 MVT NarrowSVT = MVT::getIntegerVT(NarrowBits);
4734 EVT NarrowVT = EVT::getVectorVT(*DAG.getContext(), NarrowSVT, ScaledEC);
4735
4737 continue;
4738 KnownBits Known1 = DAG.computeKnownBits(N1);
4739 if (Known1.countMaxActiveBits() <= NarrowBits) {
4740 SDValue NarrowN0 = DAG.getBitcast(NarrowVT, N0);
4741 SDValue NarrowN1 = DAG.getBitcast(NarrowVT, N1);
4742 SDValue NarrowSub =
4743 DAG.getNode(ISD::USUBSAT, DL, NarrowVT, NarrowN0, NarrowN1);
4744 return DAG.getBitcast(VT, NarrowSub);
4745 }
4746 // TODO: If N1 doesn't fit in NarrowBits, we could OR the upper bits
4747 // of N1 with 1s to force saturation in those lanes, allowing the
4748 // narrow USUBSAT to still be used. This requires a TLI hook to check
4749 // whether the constant can be folded as a broadcast memory operand
4750 // (profitable on AVX512, not on SSE/AVX), to avoid introducing an
4751 // extra register and instruction on non-AVX512 targets.
4752 break;
4753 }
4754 }
4755 }
4756 return SDValue();
4757}
4758
4759SDValue DAGCombiner::visitSUBC(SDNode *N) {
4760 SDValue N0 = N->getOperand(0);
4761 SDValue N1 = N->getOperand(1);
4762 EVT VT = N0.getValueType();
4763 SDLoc DL(N);
4764
4765 // If the flag result is dead, turn this into an SUB.
4766 if (!N->hasAnyUseOfValue(1))
4767 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1),
4768 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
4769
4770 // fold (subc x, x) -> 0 + no borrow
4771 if (N0 == N1)
4772 return CombineTo(N, DAG.getConstant(0, DL, VT),
4773 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
4774
4775 // fold (subc x, 0) -> x + no borrow
4776 if (isNullConstant(N1))
4777 return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
4778
4779 // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1) + no borrow
4780 if (isAllOnesConstant(N0))
4781 return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0),
4782 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
4783
4784 return SDValue();
4785}
4786
4787SDValue DAGCombiner::visitSUBO(SDNode *N) {
4788 SDValue N0 = N->getOperand(0);
4789 SDValue N1 = N->getOperand(1);
4790 EVT VT = N0.getValueType();
4791 bool IsSigned = (ISD::SSUBO == N->getOpcode());
4792
4793 EVT CarryVT = N->getValueType(1);
4794 SDLoc DL(N);
4795
4796 // If the flag result is dead, turn this into an SUB.
4797 if (!N->hasAnyUseOfValue(1))
4798 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1),
4799 DAG.getUNDEF(CarryVT));
4800
4801 // fold operation with constant operands.
4802 if (SDValue C = DAG.FoldConstantArithmetic(N->getOpcode(), DL, N->getVTList(),
4803 {N0, N1}))
4804 return C;
4805
4806 // fold (subo x, x) -> 0 + no borrow
4807 if (N0 == N1)
4808 return CombineTo(N, DAG.getConstant(0, DL, VT),
4809 DAG.getConstant(0, DL, CarryVT));
4810
4811 // fold (subox, c) -> (addo x, -c)
4812 if (ConstantSDNode *N1C = getAsNonOpaqueConstant(N1))
4813 if (IsSigned && !N1C->isMinSignedValue())
4814 return DAG.getNode(ISD::SADDO, DL, N->getVTList(), N0,
4815 DAG.getConstant(-N1C->getAPIntValue(), DL, VT));
4816
4817 // fold (subo x, 0) -> x + no borrow
4818 if (isNullOrNullSplat(N1))
4819 return CombineTo(N, N0, DAG.getConstant(0, DL, CarryVT));
4820
4821 // If it cannot overflow, transform into an sub.
4822 if (DAG.willNotOverflowSub(IsSigned, N0, N1))
4823 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1),
4824 DAG.getConstant(0, DL, CarryVT));
4825
4826 // Canonicalize (usubo -1, x) -> ~x, i.e. (xor x, -1) + no borrow
4827 if (!IsSigned && isAllOnesOrAllOnesSplat(N0))
4828 return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0),
4829 DAG.getConstant(0, DL, CarryVT));
4830
4831 return SDValue();
4832}
4833
4834SDValue DAGCombiner::visitSUBE(SDNode *N) {
4835 SDValue N0 = N->getOperand(0);
4836 SDValue N1 = N->getOperand(1);
4837 SDValue CarryIn = N->getOperand(2);
4838
4839 // fold (sube x, y, false) -> (subc x, y)
4840 if (CarryIn.getOpcode() == ISD::CARRY_FALSE)
4841 return DAG.getNode(ISD::SUBC, SDLoc(N), N->getVTList(), N0, N1);
4842
4843 return SDValue();
4844}
4845
4846SDValue DAGCombiner::visitUSUBO_CARRY(SDNode *N) {
4847 SDValue N0 = N->getOperand(0);
4848 SDValue N1 = N->getOperand(1);
4849 SDValue CarryIn = N->getOperand(2);
4850
4851 // fold (usubo_carry x, y, false) -> (usubo x, y)
4852 if (isNullConstant(CarryIn)) {
4853 if (!LegalOperations ||
4854 TLI.isOperationLegalOrCustom(ISD::USUBO, N->getValueType(0)))
4855 return DAG.getNode(ISD::USUBO, SDLoc(N), N->getVTList(), N0, N1);
4856 }
4857
4858 // Iff the flag result is dead:
4859 // (usubo_carry (sub X, Y), 0, Carry) -> (usubo_carry X, Y, Carry)
4860 if (N0.getOpcode() == ISD::SUB && isNullConstant(N1) &&
4861 !N->hasAnyUseOfValue(1))
4862 return DAG.getNode(ISD::USUBO_CARRY, SDLoc(N), N->getVTList(),
4863 N0.getOperand(0), N0.getOperand(1), CarryIn);
4864
4865 return SDValue();
4866}
4867
4868SDValue DAGCombiner::visitSSUBO_CARRY(SDNode *N) {
4869 SDValue N0 = N->getOperand(0);
4870 SDValue N1 = N->getOperand(1);
4871 SDValue CarryIn = N->getOperand(2);
4872
4873 // fold (ssubo_carry x, y, false) -> (ssubo x, y)
4874 if (isNullConstant(CarryIn)) {
4875 if (!LegalOperations ||
4876 TLI.isOperationLegalOrCustom(ISD::SSUBO, N->getValueType(0)))
4877 return DAG.getNode(ISD::SSUBO, SDLoc(N), N->getVTList(), N0, N1);
4878 }
4879
4880 return SDValue();
4881}
4882
4883// Notice that "mulfix" can be any of SMULFIX, SMULFIXSAT, UMULFIX and
4884// UMULFIXSAT here.
4885SDValue DAGCombiner::visitMULFIX(SDNode *N) {
4886 SDValue N0 = N->getOperand(0);
4887 SDValue N1 = N->getOperand(1);
4888 SDValue Scale = N->getOperand(2);
4889 EVT VT = N0.getValueType();
4890
4891 // fold (mulfix x, undef, scale) -> 0
4892 if (N0.isUndef() || N1.isUndef())
4893 return DAG.getConstant(0, SDLoc(N), VT);
4894
4895 // Canonicalize constant to RHS (vector doesn't have to splat)
4898 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N1, N0, Scale);
4899
4900 // fold (mulfix x, 0, scale) -> 0
4901 if (isNullConstant(N1))
4902 return DAG.getConstant(0, SDLoc(N), VT);
4903
4904 return SDValue();
4905}
4906
4907SDValue DAGCombiner::visitMUL(SDNode *N) {
4908 SDValue N0 = N->getOperand(0);
4909 SDValue N1 = N->getOperand(1);
4910 EVT VT = N0.getValueType();
4911 unsigned BitWidth = VT.getScalarSizeInBits();
4912 SDLoc DL(N);
4913
4914 // fold (mul x, undef) -> 0
4915 if (N0.isUndef() || N1.isUndef())
4916 return DAG.getConstant(0, DL, VT);
4917
4918 // fold (mul c1, c2) -> c1*c2
4919 if (SDValue C =
4920 DAG.FoldConstantArithmetic(ISD::MUL, DL, VT, {N0, N1}, N->getFlags()))
4921 return C;
4922
4923 // canonicalize constant to RHS (vector doesn't have to splat). An opaque
4924 // constant on the RHS is treated as non-constant so that a foldable constant
4925 // still ends up on the RHS.
4926 if (DAG.isConstantIntBuildVectorOrConstantInt(N0, /*AllowOpaques=*/false) &&
4927 !DAG.isConstantIntBuildVectorOrConstantInt(N1, /*AllowOpaques=*/false))
4928 return DAG.getNode(ISD::MUL, DL, VT, N1, N0);
4929
4930 bool N1IsConst = false;
4931 bool N1IsOpaqueConst = false;
4932 APInt ConstValue1;
4933
4934 // fold vector ops
4935 if (VT.isVector()) {
4936 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
4937 return FoldedVOp;
4938
4939 N1IsConst = ISD::isConstantSplatVector(N1.getNode(), ConstValue1);
4940 assert((!N1IsConst || ConstValue1.getBitWidth() == BitWidth) &&
4941 "Splat APInt should be element width");
4942 } else {
4943 N1IsConst = isa<ConstantSDNode>(N1);
4944 if (N1IsConst) {
4945 ConstValue1 = N1->getAsAPIntVal();
4946 N1IsOpaqueConst = cast<ConstantSDNode>(N1)->isOpaque();
4947 }
4948 }
4949
4950 // fold (mul x, 0) -> 0
4951 if (N1IsConst && ConstValue1.isZero())
4952 return N1;
4953
4954 // fold (mul x, 1) -> x
4955 if (N1IsConst && ConstValue1.isOne())
4956 return N0;
4957
4958 if (SDValue NewSel = foldBinOpIntoSelect(N))
4959 return NewSel;
4960
4961 // fold (mul x, -1) -> 0-x
4962 if (N1IsConst && ConstValue1.isAllOnes())
4963 return DAG.getNegative(N0, DL, VT);
4964
4965 // fold (mul x, (1 << c)) -> x << c
4966 if (isConstantOrConstantVector(N1, /*NoOpaques*/ true) &&
4967 (!VT.isVector() || Level <= AfterLegalizeVectorOps)) {
4968 if (SDValue LogBase2 = BuildLogBase2(N1, DL)) {
4969 EVT ShiftVT = getShiftAmountTy(N0.getValueType());
4970 SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ShiftVT);
4971 SDNodeFlags Flags;
4972 Flags.setNoUnsignedWrap(N->getFlags().hasNoUnsignedWrap());
4973 // Preserve nsw when the shift amount is strictly less than BitWidth - 1,
4974 // i.e. the multiplier is not the signed minimum value.
4975 if (N->getFlags().hasNoSignedWrap() && N1IsConst &&
4976 ConstValue1.logBase2() < BitWidth - 1)
4977 Flags.setNoSignedWrap(true);
4978 return DAG.getNode(ISD::SHL, DL, VT, N0, Trunc, Flags);
4979 }
4980 }
4981
4982 // fold (mul x, -(1 << c)) -> -(x << c) or (-x) << c
4983 if (N1IsConst && !N1IsOpaqueConst && ConstValue1.isNegatedPowerOf2()) {
4984 unsigned Log2Val = (-ConstValue1).logBase2();
4985
4986 // FIXME: If the input is something that is easily negated (e.g. a
4987 // single-use add), we should put the negate there.
4988 return DAG.getNode(
4989 ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
4990 DAG.getNode(ISD::SHL, DL, VT, N0,
4991 DAG.getShiftAmountConstant(Log2Val, VT, DL)));
4992 }
4993
4994 // Attempt to reuse an existing umul_lohi/smul_lohi node, but only if the
4995 // hi result is in use in case we hit this mid-legalization.
4996 for (unsigned LoHiOpc : {ISD::UMUL_LOHI, ISD::SMUL_LOHI}) {
4997 if (!LegalOperations || TLI.isOperationLegalOrCustom(LoHiOpc, VT)) {
4998 SDVTList LoHiVT = DAG.getVTList(VT, VT);
4999 // TODO: Can we match commutable operands with getNodeIfExists?
5000 if (SDNode *LoHi = DAG.getNodeIfExists(LoHiOpc, LoHiVT, {N0, N1}))
5001 if (LoHi->hasAnyUseOfValue(1))
5002 return SDValue(LoHi, 0);
5003 if (SDNode *LoHi = DAG.getNodeIfExists(LoHiOpc, LoHiVT, {N1, N0}))
5004 if (LoHi->hasAnyUseOfValue(1))
5005 return SDValue(LoHi, 0);
5006 }
5007 }
5008
5009 // Try to transform:
5010 // (1) multiply-by-(power-of-2 +/- 1) into shift and add/sub.
5011 // mul x, (2^N + 1) --> add (shl x, N), x
5012 // mul x, (2^N - 1) --> sub (shl x, N), x
5013 // Examples: x * 33 --> (x << 5) + x
5014 // x * 15 --> (x << 4) - x
5015 // x * -33 --> -((x << 5) + x)
5016 // x * -15 --> -((x << 4) - x) ; this reduces --> x - (x << 4)
5017 // (2) multiply-by-(power-of-2 +/- power-of-2) into shifts and add/sub.
5018 // mul x, (2^N + 2^M) --> (add (shl x, N), (shl x, M))
5019 // mul x, (2^N - 2^M) --> (sub (shl x, N), (shl x, M))
5020 // Examples: x * 0x8800 --> (x << 15) + (x << 11)
5021 // x * 0xf800 --> (x << 16) - (x << 11)
5022 // x * -0x8800 --> -((x << 15) + (x << 11))
5023 // x * -0xf800 --> -((x << 16) - (x << 11)) ; (x << 11) - (x << 16)
5024 if (N1IsConst && TLI.decomposeMulByConstant(*DAG.getContext(), VT, N1)) {
5025 // TODO: We could handle more general decomposition of any constant by
5026 // having the target set a limit on number of ops and making a
5027 // callback to determine that sequence (similar to sqrt expansion).
5028 unsigned MathOp = ISD::DELETED_NODE;
5029 APInt MulC = ConstValue1.abs();
5030 // The constant `2` should be treated as (2^0 + 1).
5031 unsigned TZeros = MulC == 2 ? 0 : MulC.countr_zero();
5032 MulC.lshrInPlace(TZeros);
5033 if ((MulC - 1).isPowerOf2())
5034 MathOp = ISD::ADD;
5035 else if ((MulC + 1).isPowerOf2())
5036 MathOp = ISD::SUB;
5037
5038 if (MathOp != ISD::DELETED_NODE) {
5039 unsigned ShAmt =
5040 MathOp == ISD::ADD ? (MulC - 1).logBase2() : (MulC + 1).logBase2();
5041 ShAmt += TZeros;
5042 assert(ShAmt < BitWidth &&
5043 "multiply-by-constant generated out of bounds shift");
5044 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, N0,
5045 DAG.getShiftAmountConstant(ShAmt, VT, DL));
5046 SDValue R = N0;
5047 if (TZeros)
5048 R = DAG.getNode(ISD::SHL, DL, VT, N0,
5049 DAG.getShiftAmountConstant(TZeros, VT, DL));
5050 R = DAG.getNode(MathOp, DL, VT, Shl, R);
5051 if (ConstValue1.isNegative())
5052 R = DAG.getNegative(R, DL, VT);
5053 return R;
5054 }
5055 }
5056
5057 // (mul (shl X, c1), c2) -> (mul X, c2 << c1)
5058 {
5059 SDValue X, C1;
5060 if (sd_match(N0, m_Shl(m_Value(X), m_Value(C1))))
5061 if (SDValue C3 = DAG.FoldConstantArithmetic(ISD::SHL, DL, VT, {N1, C1}))
5062 return DAG.getNode(ISD::MUL, DL, VT, X, C3);
5063 }
5064
5065 // Change (mul (shl X, C), Y) -> (shl (mul X, Y), C) when the shift has one
5066 // use.
5067 {
5068 SDValue X, C, Y;
5069 if (sd_match(N,
5072 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, X, Y);
5073 return DAG.getNode(ISD::SHL, DL, VT, Mul, C);
5074 }
5075 }
5076
5077 // fold (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2)
5078 if (N0.getOpcode() == ISD::ADD && isConstantOrConstantVector(N1) &&
5081 return DAG.getNode(
5082 ISD::ADD, DL, VT,
5083 DAG.getNode(ISD::MUL, SDLoc(N0), VT, N0.getOperand(0), N1),
5084 DAG.getNode(ISD::MUL, SDLoc(N1), VT, N0.getOperand(1), N1));
5085
5086 // Fold (mul (vscale * C0), C1) to (vscale * (C0 * C1)).
5087 // avoid if ISD::MUL handling is poor and ISD::SHL isn't an option.
5088 ConstantSDNode *NC1 = isConstOrConstSplat(N1);
5089 if (N0.getOpcode() == ISD::VSCALE && NC1) {
5090 const APInt &C0 = N0.getConstantOperandAPInt(0);
5091 const APInt &C1 = NC1->getAPIntValue();
5092 if (!C0.isPowerOf2() || C1.isPowerOf2() ||
5093 hasOperation(ISD::MUL, NC1->getValueType(0)))
5094 return DAG.getVScale(DL, VT, C0 * C1);
5095 }
5096
5097 // Fold (mul step_vector(C0), C1) to (step_vector(C0 * C1)).
5098 APInt MulVal;
5099 if (N0.getOpcode() == ISD::STEP_VECTOR &&
5100 ISD::isConstantSplatVector(N1.getNode(), MulVal)) {
5101 const APInt &C0 = N0.getConstantOperandAPInt(0);
5102 APInt NewStep = C0 * MulVal;
5103 return DAG.getStepVector(DL, VT, NewStep);
5104 }
5105
5106 // Fold Y = sra (X, size(X)-1); mul (or (Y, 1), X) -> (abs X)
5107 SDValue X;
5108 if ((!LegalOperations || hasOperation(ISD::ABS, VT)) &&
5110 m_One()),
5111 m_Deferred(X)))) {
5112 return DAG.getNode(ISD::ABS, DL, VT, X);
5113 }
5114
5115 // Fold ((mul x, 0/undef) -> 0,
5116 // (mul x, 1) -> x) -> x)
5117 // -> and(x, mask)
5118 // We can replace vectors with '0' and '1' factors with a clearing mask.
5119 if (VT.isFixedLengthVector()) {
5120 unsigned NumElts = VT.getVectorNumElements();
5121 SmallBitVector ClearMask;
5122 ClearMask.reserve(NumElts);
5123 auto IsClearMask = [&ClearMask](ConstantSDNode *V) {
5124 if (!V || V->isZero()) {
5125 ClearMask.push_back(true);
5126 return true;
5127 }
5128 ClearMask.push_back(false);
5129 return V->isOne();
5130 };
5131 if ((!LegalOperations || TLI.isOperationLegalOrCustom(ISD::AND, VT)) &&
5132 ISD::matchUnaryPredicate(N1, IsClearMask, /*AllowUndefs*/ true)) {
5133 assert(N1.getOpcode() == ISD::BUILD_VECTOR && "Unknown constant vector");
5134 EVT LegalSVT = N1.getOperand(0).getValueType();
5135 SDValue Zero = DAG.getConstant(0, DL, LegalSVT);
5136 SDValue AllOnes = DAG.getAllOnesConstant(DL, LegalSVT);
5138 for (unsigned I = 0; I != NumElts; ++I)
5139 if (ClearMask[I])
5140 Mask[I] = Zero;
5141 return DAG.getNode(ISD::AND, DL, VT, N0, DAG.getBuildVector(VT, DL, Mask));
5142 }
5143 }
5144
5145 // reassociate mul
5146 if (SDValue RMUL = reassociateOps(ISD::MUL, DL, N0, N1, N->getFlags()))
5147 return RMUL;
5148
5149 // Fold mul(vecreduce(x), vecreduce(y)) -> vecreduce(mul(x, y))
5150 if (SDValue SD =
5151 reassociateReduction(ISD::VECREDUCE_MUL, ISD::MUL, DL, VT, N0, N1))
5152 return SD;
5153
5154 // Simplify the operands using demanded-bits information.
5156 return SDValue(N, 0);
5157
5158 return SDValue();
5159}
5160
5161/// Return true if divmod libcall is available.
5163 const SelectionDAG &DAG) {
5164 RTLIB::Libcall LC;
5165 EVT NodeType = Node->getValueType(0);
5166 if (!NodeType.isSimple())
5167 return false;
5168 switch (NodeType.getSimpleVT().SimpleTy) {
5169 default: return false; // No libcall for vector types.
5170 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
5171 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
5172 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
5173 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
5174 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
5175 }
5176
5177 return DAG.getLibcalls().getLibcallImpl(LC) != RTLIB::Unsupported;
5178}
5179
5180/// Issue divrem if both quotient and remainder are needed.
5181SDValue DAGCombiner::useDivRem(SDNode *Node) {
5182 if (Node->use_empty())
5183 return SDValue(); // This is a dead node, leave it alone.
5184
5185 unsigned Opcode = Node->getOpcode();
5186 bool isSigned = (Opcode == ISD::SDIV) || (Opcode == ISD::SREM);
5187 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
5188
5189 // DivMod lib calls can still work on non-legal types if using lib-calls.
5190 EVT VT = Node->getValueType(0);
5191 if (VT.isVector() || !VT.isInteger())
5192 return SDValue();
5193
5194 if (!TLI.isTypeLegal(VT) && !TLI.isOperationCustom(DivRemOpc, VT))
5195 return SDValue();
5196
5197 // If DIVREM is going to get expanded into a libcall,
5198 // but there is no libcall available, then don't combine.
5199 if (!TLI.isOperationLegalOrCustom(DivRemOpc, VT) &&
5201 return SDValue();
5202
5203 // If div is legal, it's better to do the normal expansion
5204 unsigned OtherOpcode = 0;
5205 if ((Opcode == ISD::SDIV) || (Opcode == ISD::UDIV)) {
5206 OtherOpcode = isSigned ? ISD::SREM : ISD::UREM;
5207 if (TLI.isOperationLegalOrCustom(Opcode, VT))
5208 return SDValue();
5209 } else {
5210 OtherOpcode = isSigned ? ISD::SDIV : ISD::UDIV;
5211 if (TLI.isOperationLegalOrCustom(OtherOpcode, VT))
5212 return SDValue();
5213 }
5214
5215 SDValue Op0 = Node->getOperand(0);
5216 SDValue Op1 = Node->getOperand(1);
5217 SDValue combined;
5218 for (SDNode *User : Op0->users()) {
5219 if (User == Node || User->getOpcode() == ISD::DELETED_NODE ||
5220 User->use_empty())
5221 continue;
5222 // Convert the other matching node(s), too;
5223 // otherwise, the DIVREM may get target-legalized into something
5224 // target-specific that we won't be able to recognize.
5225 unsigned UserOpc = User->getOpcode();
5226 if ((UserOpc == Opcode || UserOpc == OtherOpcode || UserOpc == DivRemOpc) &&
5227 User->getOperand(0) == Op0 &&
5228 User->getOperand(1) == Op1) {
5229 if (!combined) {
5230 if (UserOpc == OtherOpcode) {
5231 SDVTList VTs = DAG.getVTList(VT, VT);
5232 combined = DAG.getNode(DivRemOpc, SDLoc(Node), VTs, Op0, Op1);
5233 } else if (UserOpc == DivRemOpc) {
5234 combined = SDValue(User, 0);
5235 } else {
5236 assert(UserOpc == Opcode);
5237 continue;
5238 }
5239 }
5240 if (UserOpc == ISD::SDIV || UserOpc == ISD::UDIV)
5241 CombineTo(User, combined);
5242 else if (UserOpc == ISD::SREM || UserOpc == ISD::UREM)
5243 CombineTo(User, combined.getValue(1));
5244 }
5245 }
5246 return combined;
5247}
5248
5250 SDValue N0 = N->getOperand(0);
5251 SDValue N1 = N->getOperand(1);
5252 EVT VT = N->getValueType(0);
5253 SDLoc DL(N);
5254
5255 unsigned Opc = N->getOpcode();
5256 bool IsDiv = (ISD::SDIV == Opc) || (ISD::UDIV == Opc);
5257
5258 // X / undef -> undef
5259 // X % undef -> undef
5260 // X / 0 -> undef
5261 // X % 0 -> undef
5262 // NOTE: This includes vectors where any divisor element is zero/undef.
5263 if (DAG.isUndef(Opc, {N0, N1}))
5264 return DAG.getUNDEF(VT);
5265
5266 // undef / X -> 0
5267 // undef % X -> 0
5268 if (N0.isUndef())
5269 return DAG.getConstant(0, DL, VT);
5270
5271 // 0 / X -> 0
5272 // 0 % X -> 0
5274 if (N0C && N0C->isZero())
5275 return N0;
5276
5277 // X / X -> 1
5278 // X % X -> 0
5279 if (N0 == N1)
5280 return DAG.getConstant(IsDiv ? 1 : 0, DL, VT);
5281
5282 // X / 1 -> X
5283 // X % 1 -> 0
5284 // If this is a boolean op (single-bit element type), we can't have
5285 // division-by-zero or remainder-by-zero, so assume the divisor is 1.
5286 // TODO: Similarly, if we're zero-extending a boolean divisor, then assume
5287 // it's a 1.
5288 if (isOneOrOneSplat(N1) || (VT.getScalarType() == MVT::i1))
5289 return IsDiv ? N0 : DAG.getConstant(0, DL, VT);
5290
5291 return SDValue();
5292}
5293
5294SDValue DAGCombiner::visitSDIV(SDNode *N) {
5295 SDValue N0 = N->getOperand(0);
5296 SDValue N1 = N->getOperand(1);
5297 EVT VT = N->getValueType(0);
5298 EVT CCVT = getSetCCResultType(VT);
5299 SDLoc DL(N);
5300
5301 // fold (sdiv c1, c2) -> c1/c2
5302 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SDIV, DL, VT, {N0, N1},
5303 N->getFlags()))
5304 return C;
5305
5306 // fold vector ops
5307 if (VT.isVector())
5308 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
5309 return FoldedVOp;
5310
5311 // fold (sdiv X, -1) -> 0-X
5312 ConstantSDNode *N1C = isConstOrConstSplat(N1);
5313 if (N1C && N1C->isAllOnes())
5314 return DAG.getNegative(N0, DL, VT);
5315
5316 // fold (sdiv X, MIN_SIGNED) -> select(X == MIN_SIGNED, 1, 0)
5317 if (N1C && N1C->isMinSignedValue())
5318 return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ),
5319 DAG.getConstant(1, DL, VT),
5320 DAG.getConstant(0, DL, VT));
5321
5322 if (SDValue V = simplifyDivRem(N, DAG))
5323 return V;
5324
5325 if (SDValue NewSel = foldBinOpIntoSelect(N))
5326 return NewSel;
5327
5328 // If we know the sign bits of both operands are zero, strength reduce to a
5329 // udiv instead. Handles (X&15) /s 4 -> X&15 >> 2
5330 if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
5331 return DAG.getNode(ISD::UDIV, DL, N1.getValueType(), N0, N1);
5332
5333 if (SDValue V = visitSDIVLike(N0, N1, N)) {
5334 // If the corresponding remainder node exists, update its users with
5335 // (Dividend - (Quotient * Divisor).
5336 if (SDNode *RemNode = DAG.getNodeIfExists(ISD::SREM, N->getVTList(),
5337 { N0, N1 })) {
5338 // If the sdiv has the exact flag we shouldn't propagate it to the
5339 // remainder node.
5340 if (!N->getFlags().hasExact()) {
5341 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, V, N1);
5342 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
5343 AddToWorklist(Mul.getNode());
5344 AddToWorklist(Sub.getNode());
5345 CombineTo(RemNode, Sub);
5346 }
5347 }
5348 return V;
5349 }
5350
5351 // sdiv, srem -> sdivrem
5352 // If the divisor is constant, then return DIVREM only if isIntDivCheap() is
5353 // true. Otherwise, we break the simplification logic in visitREM().
5354 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
5355 if (!N1C || TLI.isIntDivCheap(N->getValueType(0), Attr))
5356 if (SDValue DivRem = useDivRem(N))
5357 return DivRem;
5358
5359 return SDValue();
5360}
5361
5362static bool isDivisorPowerOfTwo(SDValue Divisor) {
5363 // Helper for determining whether a value is a power-2 constant scalar or a
5364 // vector of such elements.
5365 auto IsPowerOfTwo = [](ConstantSDNode *C) {
5366 if (C->isZero() || C->isOpaque())
5367 return false;
5368 if (C->getAPIntValue().isPowerOf2())
5369 return true;
5370 if (C->getAPIntValue().isNegatedPowerOf2())
5371 return true;
5372 return false;
5373 };
5374
5375 return ISD::matchUnaryPredicate(Divisor, IsPowerOfTwo, /*AllowUndefs=*/false,
5376 /*AllowTruncation=*/true);
5377}
5378
5379SDValue DAGCombiner::visitSDIVLike(SDValue N0, SDValue N1, SDNode *N) {
5380 SDLoc DL(N);
5381 EVT VT = N->getValueType(0);
5382 EVT CCVT = getSetCCResultType(VT);
5383 unsigned BitWidth = VT.getScalarSizeInBits();
5384 unsigned MaxLegalDivRemBitWidth = TLI.getMaxDivRemBitWidthSupported();
5385
5386 // fold (sdiv X, pow2) -> simple ops after legalize
5387 // FIXME: We check for the exact bit here because the generic lowering gives
5388 // better results in that case. The target-specific lowering should learn how
5389 // to handle exact sdivs efficiently. An exception is made for large bitwidths
5390 // exceeding what the target can natively support, as division expansion was
5391 // skipped in favor of this optimization.
5392 if ((!N->getFlags().hasExact() || BitWidth > MaxLegalDivRemBitWidth) &&
5393 isDivisorPowerOfTwo(N1)) {
5394 // Target-specific implementation of sdiv x, pow2.
5395 if (SDValue Res = BuildSDIVPow2(N))
5396 return Res;
5397
5398 // Create constants that are functions of the shift amount value.
5399 EVT ShiftAmtTy = getShiftAmountTy(N0.getValueType());
5400 SDValue Bits = DAG.getConstant(BitWidth, DL, ShiftAmtTy);
5401 SDValue C1 = DAG.getNode(ISD::CTTZ, DL, VT, N1);
5402 C1 = DAG.getZExtOrTrunc(C1, DL, ShiftAmtTy);
5403 SDValue Inexact = DAG.getNode(ISD::SUB, DL, ShiftAmtTy, Bits, C1);
5404 if (!isConstantOrConstantVector(Inexact))
5405 return SDValue();
5406
5407 // Splat the sign bit into the register
5408 SDValue Sign = DAG.getNode(ISD::SRA, DL, VT, N0,
5409 DAG.getConstant(BitWidth - 1, DL, ShiftAmtTy));
5410 AddToWorklist(Sign.getNode());
5411
5412 // Add (N0 < 0) ? abs2 - 1 : 0;
5413 SDValue Srl = DAG.getNode(ISD::SRL, DL, VT, Sign, Inexact);
5414 AddToWorklist(Srl.getNode());
5415 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Srl);
5416 AddToWorklist(Add.getNode());
5417 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Add, C1);
5418 AddToWorklist(Sra.getNode());
5419
5420 // Special case: (sdiv X, 1) -> X
5421 // Special Case: (sdiv X, -1) -> 0-X
5422 SDValue One = DAG.getConstant(1, DL, VT);
5424 SDValue IsOne = DAG.getSetCC(DL, CCVT, N1, One, ISD::SETEQ);
5425 SDValue IsAllOnes = DAG.getSetCC(DL, CCVT, N1, AllOnes, ISD::SETEQ);
5426 SDValue IsOneOrAllOnes = DAG.getNode(ISD::OR, DL, CCVT, IsOne, IsAllOnes);
5427 Sra = DAG.getSelect(DL, VT, IsOneOrAllOnes, N0, Sra);
5428
5429 // If dividing by a positive value, we're done. Otherwise, the result must
5430 // be negated.
5431 SDValue Zero = DAG.getConstant(0, DL, VT);
5432 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, Zero, Sra);
5433
5434 // FIXME: Use SELECT_CC once we improve SELECT_CC constant-folding.
5435 SDValue IsNeg = DAG.getSetCC(DL, CCVT, N1, Zero, ISD::SETLT);
5436 SDValue Res = DAG.getSelect(DL, VT, IsNeg, Sub, Sra);
5437 return Res;
5438 }
5439
5440 // If integer divide is expensive and we satisfy the requirements, emit an
5441 // alternate sequence. Targets may check function attributes for size/speed
5442 // trade-offs.
5443 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
5444 if (isConstantOrConstantVector(N1, /*NoOpaques=*/false,
5445 /*AllowTruncation=*/true) &&
5446 !TLI.isIntDivCheap(N->getValueType(0), Attr))
5447 if (SDValue Op = BuildSDIV(N))
5448 return Op;
5449
5450 return SDValue();
5451}
5452
5453SDValue DAGCombiner::visitUDIV(SDNode *N) {
5454 SDValue N0 = N->getOperand(0);
5455 SDValue N1 = N->getOperand(1);
5456 EVT VT = N->getValueType(0);
5457 EVT CCVT = getSetCCResultType(VT);
5458 SDLoc DL(N);
5459
5460 // fold (udiv c1, c2) -> c1/c2
5461 if (SDValue C = DAG.FoldConstantArithmetic(ISD::UDIV, DL, VT, {N0, N1},
5462 N->getFlags()))
5463 return C;
5464
5465 // fold vector ops
5466 if (VT.isVector())
5467 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
5468 return FoldedVOp;
5469
5470 // fold (udiv X, -1) -> select(X == -1, 1, 0)
5471 ConstantSDNode *N1C = isConstOrConstSplat(N1);
5472 if (N1C && N1C->isAllOnes() && CCVT.isVector() == VT.isVector()) {
5473 return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ),
5474 DAG.getConstant(1, DL, VT),
5475 DAG.getConstant(0, DL, VT));
5476 }
5477
5478 if (SDValue V = simplifyDivRem(N, DAG))
5479 return V;
5480
5481 if (SDValue NewSel = foldBinOpIntoSelect(N))
5482 return NewSel;
5483
5484 if (SDValue V = visitUDIVLike(N0, N1, N)) {
5485 // If the corresponding remainder node exists, update its users with
5486 // (Dividend - (Quotient * Divisor).
5487 if (SDNode *RemNode = DAG.getNodeIfExists(ISD::UREM, N->getVTList(),
5488 { N0, N1 })) {
5489 // If the udiv has the exact flag we shouldn't propagate it to the
5490 // remainder node.
5491 if (!N->getFlags().hasExact()) {
5492 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, V, N1);
5493 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
5494 AddToWorklist(Mul.getNode());
5495 AddToWorklist(Sub.getNode());
5496 CombineTo(RemNode, Sub);
5497 }
5498 }
5499 return V;
5500 }
5501
5502 // sdiv, srem -> sdivrem
5503 // If the divisor is constant, then return DIVREM only if isIntDivCheap() is
5504 // true. Otherwise, we break the simplification logic in visitREM().
5505 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
5506 if (!N1C || TLI.isIntDivCheap(N->getValueType(0), Attr))
5507 if (SDValue DivRem = useDivRem(N))
5508 return DivRem;
5509
5510 // Simplify the operands using demanded-bits information.
5511 // We don't have demanded bits support for UDIV so this just enables constant
5512 // folding based on known bits.
5514 return SDValue(N, 0);
5515
5516 return SDValue();
5517}
5518
5519SDValue DAGCombiner::visitUDIVLike(SDValue N0, SDValue N1, SDNode *N) {
5520 SDLoc DL(N);
5521 EVT VT = N->getValueType(0);
5522
5523 // fold (udiv x, (1 << c)) -> x >>u c
5524 if (isConstantOrConstantVector(N1, /*NoOpaques=*/true,
5525 /*AllowTruncation=*/true)) {
5526 if (SDValue LogBase2 = BuildLogBase2(N1, DL)) {
5527 AddToWorklist(LogBase2.getNode());
5528
5529 EVT ShiftVT = getShiftAmountTy(N0.getValueType());
5530 SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ShiftVT);
5531 AddToWorklist(Trunc.getNode());
5532 return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc);
5533 }
5534 }
5535
5536 // fold (udiv x, (shl c, y)) -> x >>u (log2(c)+y) iff c is power of 2
5537 if (N1.getOpcode() == ISD::SHL) {
5538 SDValue N10 = N1.getOperand(0);
5539 if (isConstantOrConstantVector(N10, /*NoOpaques=*/true,
5540 /*AllowTruncation=*/true)) {
5541 if (SDValue LogBase2 = BuildLogBase2(N10, DL)) {
5542 AddToWorklist(LogBase2.getNode());
5543
5544 EVT ADDVT = N1.getOperand(1).getValueType();
5545 SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ADDVT);
5546 AddToWorklist(Trunc.getNode());
5547 SDValue Add = DAG.getNode(ISD::ADD, DL, ADDVT, N1.getOperand(1), Trunc);
5548 AddToWorklist(Add.getNode());
5549 return DAG.getNode(ISD::SRL, DL, VT, N0, Add);
5550 }
5551 }
5552 }
5553
5554 // fold (udiv x, c) -> alternate
5555 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
5556 if (isConstantOrConstantVector(N1, /*NoOpaques=*/false,
5557 /*AllowTruncation=*/true) &&
5558 !TLI.isIntDivCheap(N->getValueType(0), Attr))
5559 if (SDValue Op = BuildUDIV(N))
5560 return Op;
5561
5562 return SDValue();
5563}
5564
5565SDValue DAGCombiner::buildOptimizedSREM(SDValue N0, SDValue N1, SDNode *N) {
5566 if (!N->getFlags().hasExact() && isDivisorPowerOfTwo(N1) &&
5567 !DAG.doesNodeExist(ISD::SDIV, N->getVTList(), {N0, N1})) {
5568 // Target-specific implementation of srem x, pow2.
5569 if (SDValue Res = BuildSREMPow2(N))
5570 return Res;
5571 }
5572 return SDValue();
5573}
5574
5575// handles ISD::SREM and ISD::UREM
5576SDValue DAGCombiner::visitREM(SDNode *N) {
5577 unsigned Opcode = N->getOpcode();
5578 SDValue N0 = N->getOperand(0);
5579 SDValue N1 = N->getOperand(1);
5580 EVT VT = N->getValueType(0);
5581 EVT CCVT = getSetCCResultType(VT);
5582
5583 bool isSigned = (Opcode == ISD::SREM);
5584 unsigned DivOpcode = isSigned ? ISD::SDIV : ISD::UDIV;
5585 SDLoc DL(N);
5586
5587 // fold (rem c1, c2) -> c1%c2
5588 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1}))
5589 return C;
5590
5591 // fold (urem X, -1) -> select(FX == -1, 0, FX)
5592 // Freeze the numerator to avoid a miscompile with an undefined value.
5593 if (!isSigned && llvm::isAllOnesOrAllOnesSplat(N1, /*AllowUndefs*/ false) &&
5594 CCVT.isVector() == VT.isVector()) {
5595 SDValue F0 = DAG.getFreeze(N0);
5596 SDValue EqualsNeg1 = DAG.getSetCC(DL, CCVT, F0, N1, ISD::SETEQ);
5597 return DAG.getSelect(DL, VT, EqualsNeg1, DAG.getConstant(0, DL, VT), F0);
5598 }
5599
5600 if (SDValue V = simplifyDivRem(N, DAG))
5601 return V;
5602
5603 if (SDValue NewSel = foldBinOpIntoSelect(N))
5604 return NewSel;
5605
5606 if (isSigned) {
5607 // If we know the sign bits of both operands are zero, strength reduce to a
5608 // urem instead. Handles (X & 0x0FFFFFFF) %s 16 -> X&15
5609 if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
5610 return DAG.getNode(ISD::UREM, DL, VT, N0, N1);
5611 } else {
5612 if (DAG.isKnownToBeAPowerOfTwo(N1, /*OrZero=*/true)) {
5613 // fold (urem x, pow2) -> (and x, pow2-1)
5614 SDValue NegOne = DAG.getAllOnesConstant(DL, VT);
5615 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne);
5616 AddToWorklist(Add.getNode());
5617 return DAG.getNode(ISD::AND, DL, VT, N0, Add);
5618 }
5619 }
5620
5621 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
5622
5623 // If X/C can be simplified by the division-by-constant logic, lower
5624 // X%C to the equivalent of X-X/C*C.
5625 // Reuse the SDIVLike/UDIVLike combines - to avoid mangling nodes, the
5626 // speculative DIV must not cause a DIVREM conversion. We guard against this
5627 // by skipping the simplification if isIntDivCheap(). When div is not cheap,
5628 // combine will not return a DIVREM. Regardless, checking cheapness here
5629 // makes sense since the simplification results in fatter code.
5630 if (DAG.isKnownNeverZero(N1) && !TLI.isIntDivCheap(VT, Attr)) {
5631 if (isSigned) {
5632 // check if we can build faster implementation for srem
5633 if (SDValue OptimizedRem = buildOptimizedSREM(N0, N1, N))
5634 return OptimizedRem;
5635 }
5636
5637 SDValue OptimizedDiv =
5638 isSigned ? visitSDIVLike(N0, N1, N) : visitUDIVLike(N0, N1, N);
5639 if (OptimizedDiv.getNode() && OptimizedDiv.getNode() != N) {
5640 // If the equivalent Div node also exists, update its users.
5641 if (SDNode *DivNode = DAG.getNodeIfExists(DivOpcode, N->getVTList(),
5642 { N0, N1 }))
5643 CombineTo(DivNode, OptimizedDiv);
5644 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, OptimizedDiv, N1);
5645 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
5646 AddToWorklist(OptimizedDiv.getNode());
5647 AddToWorklist(Mul.getNode());
5648 return Sub;
5649 }
5650 }
5651
5652 // Fold Num % Den -> Num - (Num / Den) * Den, if (Num / Den) is already
5653 // computed. Defer for types that will be promoted and do not fold if DIVREM
5654 // is available
5655 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
5656 if (!ForCodeSize &&
5657 !TLI.isOperationLegalOrCustom(DivRemOpc, VT.getScalarType()) &&
5659 TLI.getTypeAction(*DAG.getContext(), VT) !=
5661 if (SDNode *Div =
5662 DAG.getNodeIfExists(DivOpcode, N->getVTList(), {N0, N1})) {
5663 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, SDValue(Div, 0), N1);
5664 return DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
5665 }
5666 }
5667
5668 // sdiv, srem -> sdivrem
5669 if (SDValue DivRem = useDivRem(N))
5670 return DivRem.getValue(1);
5671
5672 // fold urem(urem(A, BCst), Op1Cst) -> urem(A, Op1Cst)
5673 // iff urem(BCst, Op1Cst) == 0
5674 SDValue A;
5675 APInt Op1Cst, BCst;
5676 if (sd_match(N, m_URem(m_URem(m_Value(A), m_ConstInt(BCst)),
5677 m_ConstInt(Op1Cst))) &&
5678 BCst.urem(Op1Cst).isZero()) {
5679 return DAG.getNode(ISD::UREM, DL, VT, A, DAG.getConstant(Op1Cst, DL, VT));
5680 }
5681
5682 // fold srem(srem(A, BCst), Op1Cst) -> srem(A, Op1Cst)
5683 // iff srem(BCst, Op1Cst) == 0 && Op1Cst != 1
5684 if (sd_match(N, m_SRem(m_SRem(m_Value(A), m_ConstInt(BCst)),
5685 m_ConstInt(Op1Cst))) &&
5686 BCst.srem(Op1Cst).isZero() && !Op1Cst.isAllOnes()) {
5687 return DAG.getNode(ISD::SREM, DL, VT, A, DAG.getConstant(Op1Cst, DL, VT));
5688 }
5689
5690 return SDValue();
5691}
5692
5693SDValue DAGCombiner::visitMULHS(SDNode *N) {
5694 SDValue N0 = N->getOperand(0);
5695 SDValue N1 = N->getOperand(1);
5696 EVT VT = N->getValueType(0);
5697 SDLoc DL(N);
5698
5699 // fold (mulhs c1, c2)
5700 if (SDValue C = DAG.FoldConstantArithmetic(ISD::MULHS, DL, VT, {N0, N1}))
5701 return C;
5702
5703 // canonicalize constant to RHS.
5706 return DAG.getNode(ISD::MULHS, DL, N->getVTList(), N1, N0);
5707
5708 if (VT.isVector()) {
5709 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
5710 return FoldedVOp;
5711
5712 // fold (mulhs x, 0) -> 0
5713 // do not return N1, because undef node may exist.
5715 return DAG.getConstant(0, DL, VT);
5716 }
5717
5718 // fold (mulhs x, 0) -> 0
5719 if (isNullConstant(N1))
5720 return N1;
5721
5722 // fold (mulhs x, 1) -> (sra x, size(x)-1)
5723 if (isOneConstant(N1))
5724 return DAG.getNode(
5725 ISD::SRA, DL, VT, N0,
5727
5728 // fold (mulhs x, undef) -> 0
5729 if (N0.isUndef() || N1.isUndef())
5730 return DAG.getConstant(0, DL, VT);
5731
5732 // If the type twice as wide is legal, transform the mulhs to a wider multiply
5733 // plus a shift.
5734 if (!TLI.isOperationLegalOrCustom(ISD::MULHS, VT) && VT.isSimple() &&
5735 !VT.isVector()) {
5736 MVT Simple = VT.getSimpleVT();
5737 unsigned SimpleSize = Simple.getSizeInBits();
5738 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
5739 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
5740 N0 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N0);
5741 N1 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N1);
5742 N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
5743 N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
5744 DAG.getShiftAmountConstant(SimpleSize, NewVT, DL));
5745 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
5746 }
5747 }
5748
5749 return SDValue();
5750}
5751
5752SDValue DAGCombiner::visitMULHU(SDNode *N) {
5753 SDValue N0 = N->getOperand(0);
5754 SDValue N1 = N->getOperand(1);
5755 EVT VT = N->getValueType(0);
5756 SDLoc DL(N);
5757
5758 // fold (mulhu c1, c2)
5759 if (SDValue C = DAG.FoldConstantArithmetic(ISD::MULHU, DL, VT, {N0, N1}))
5760 return C;
5761
5762 // canonicalize constant to RHS.
5765 return DAG.getNode(ISD::MULHU, DL, N->getVTList(), N1, N0);
5766
5767 if (VT.isVector()) {
5768 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
5769 return FoldedVOp;
5770
5771 // fold (mulhu x, 0) -> 0
5772 // do not return N1, because undef node may exist.
5774 return DAG.getConstant(0, DL, VT);
5775 }
5776
5777 // fold (mulhu x, 0) -> 0
5778 if (isNullConstant(N1))
5779 return N1;
5780
5781 // fold (mulhu x, 1) -> 0
5782 if (isOneConstant(N1))
5783 return DAG.getConstant(0, DL, VT);
5784
5785 // fold (mulhu x, undef) -> 0
5786 if (N0.isUndef() || N1.isUndef())
5787 return DAG.getConstant(0, DL, VT);
5788
5789 // fold (mulhu x, (1 << c)) -> x >> (bitwidth - c)
5790 if (isConstantOrConstantVector(N1, /*NoOpaques=*/true,
5791 /*AllowTruncation=*/true) &&
5792 (!LegalOperations || hasOperation(ISD::SRL, VT))) {
5793 if (SDValue LogBase2 = BuildLogBase2(N1, DL)) {
5794 unsigned NumEltBits = VT.getScalarSizeInBits();
5795 SDValue SRLAmt = DAG.getNode(
5796 ISD::SUB, DL, VT, DAG.getConstant(NumEltBits, DL, VT), LogBase2);
5797 EVT ShiftVT = getShiftAmountTy(N0.getValueType());
5798 SDValue Trunc = DAG.getZExtOrTrunc(SRLAmt, DL, ShiftVT);
5799 return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc);
5800 }
5801 }
5802
5803 // If the type twice as wide is legal, transform the mulhu to a wider multiply
5804 // plus a shift.
5805 if (!TLI.isOperationLegalOrCustom(ISD::MULHU, VT) && VT.isSimple() &&
5806 !VT.isVector()) {
5807 MVT Simple = VT.getSimpleVT();
5808 unsigned SimpleSize = Simple.getSizeInBits();
5809 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
5810 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
5811 N0 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N0);
5812 N1 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N1);
5813 N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
5814 N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
5815 DAG.getShiftAmountConstant(SimpleSize, NewVT, DL));
5816 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
5817 }
5818 }
5819
5820 // Simplify the operands using demanded-bits information.
5821 // We don't have demanded bits support for MULHU so this just enables constant
5822 // folding based on known bits.
5824 return SDValue(N, 0);
5825
5826 return SDValue();
5827}
5828
5829SDValue DAGCombiner::visitAVGSUB(SDNode *N) {
5830 unsigned Opcode = N->getOpcode();
5831 SDValue N0 = N->getOperand(0);
5832 SDValue N1 = N->getOperand(1);
5833 EVT VT = N->getValueType(0);
5834 SDLoc DL(N);
5835
5836 // fold (avgsub c1, c2)
5837 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1}))
5838 return C;
5839
5840 if (VT.isVector())
5841 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
5842 return FoldedVOp;
5843
5844 // fold (avgsub x, x) --> 0
5845 if (N0 == N1)
5846 return DAG.getConstant(0, DL, VT);
5847
5848 // fold (avgsubfloors x, 0) -> x >> 1
5849 // fold (avgsubflooru x, 0) -> x >>u 1
5850 if (isNullOrNullSplat(N1))
5851 return DAG.getNode(Opcode == ISD::AVGSUBFLOORS ? ISD::SRA : ISD::SRL, DL,
5852 VT, N0, DAG.getShiftAmountConstant(1, VT, DL));
5853
5854 return SDValue();
5855}
5856
5857SDValue DAGCombiner::visitAVG(SDNode *N) {
5858 unsigned Opcode = N->getOpcode();
5859 SDValue N0 = N->getOperand(0);
5860 SDValue N1 = N->getOperand(1);
5861 EVT VT = N->getValueType(0);
5862 SDLoc DL(N);
5863 bool IsSigned = Opcode == ISD::AVGCEILS || Opcode == ISD::AVGFLOORS;
5864
5865 // fold (avg c1, c2)
5866 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1}))
5867 return C;
5868
5869 // canonicalize constant to RHS.
5872 return DAG.getNode(Opcode, DL, N->getVTList(), N1, N0);
5873
5874 if (VT.isVector())
5875 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
5876 return FoldedVOp;
5877
5878 // fold (avg x, undef) -> x
5879 if (N0.isUndef())
5880 return N1;
5881 if (N1.isUndef())
5882 return N0;
5883
5884 // fold (avg x, x) --> x
5885 if (N0 == N1 && Level >= AfterLegalizeTypes)
5886 return N0;
5887
5888 // fold (avgfloor x, 0) -> x >> 1
5889 SDValue X, Y;
5891 return DAG.getNode(ISD::SRA, DL, VT, X,
5892 DAG.getShiftAmountConstant(1, VT, DL));
5894 return DAG.getNode(ISD::SRL, DL, VT, X,
5895 DAG.getShiftAmountConstant(1, VT, DL));
5896
5897 // fold avgu(zext(x), zext(y)) -> zext(avgu(x, y))
5898 // fold avgs(sext(x), sext(y)) -> sext(avgs(x, y))
5899 if (!IsSigned &&
5900 sd_match(N, m_BinOp(Opcode, m_ZExt(m_Value(X)), m_ZExt(m_Value(Y)))) &&
5901 X.getValueType() == Y.getValueType() &&
5902 hasOperation(Opcode, X.getValueType())) {
5903 SDValue AvgU = DAG.getNode(Opcode, DL, X.getValueType(), X, Y);
5904 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, AvgU);
5905 }
5906 if (IsSigned &&
5907 sd_match(N, m_BinOp(Opcode, m_SExt(m_Value(X)), m_SExt(m_Value(Y)))) &&
5908 X.getValueType() == Y.getValueType() &&
5909 hasOperation(Opcode, X.getValueType())) {
5910 SDValue AvgS = DAG.getNode(Opcode, DL, X.getValueType(), X, Y);
5911 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, AvgS);
5912 }
5913
5914 // Fold avgflooru(x,y) -> avgceilu(x,y-1) iff y != 0
5915 // Fold avgflooru(x,y) -> avgceilu(x-1,y) iff x != 0
5916 // Check if avgflooru isn't legal/custom but avgceilu is.
5917 if (Opcode == ISD::AVGFLOORU && !hasOperation(ISD::AVGFLOORU, VT) &&
5918 (!LegalOperations || hasOperation(ISD::AVGCEILU, VT))) {
5919 if (DAG.isKnownNeverZero(N1))
5920 return DAG.getNode(
5921 ISD::AVGCEILU, DL, VT, N0,
5922 DAG.getNode(ISD::ADD, DL, VT, N1, DAG.getAllOnesConstant(DL, VT)));
5923 if (DAG.isKnownNeverZero(N0))
5924 return DAG.getNode(
5925 ISD::AVGCEILU, DL, VT, N1,
5926 DAG.getNode(ISD::ADD, DL, VT, N0, DAG.getAllOnesConstant(DL, VT)));
5927 }
5928
5929 // Fold avgfloor((add nw x,y), 1) -> avgceil(x,y)
5930 // Fold avgfloor((add nw x,1), y) -> avgceil(x,y)
5931 if ((Opcode == ISD::AVGFLOORU && hasOperation(ISD::AVGCEILU, VT)) ||
5932 (Opcode == ISD::AVGFLOORS && hasOperation(ISD::AVGCEILS, VT))) {
5933 SDValue Add;
5934 if (sd_match(N,
5935 m_c_BinOp(Opcode, m_Value(Add, m_Add(m_Value(X), m_Value(Y))),
5936 m_One())) ||
5937 sd_match(N, m_c_BinOp(Opcode, m_Value(Add, m_Add(m_Value(X), m_One())),
5938 m_Value(Y)))) {
5939
5940 // Check nsw flag or willNotOverflowAdd because we don't infer nsw flags
5941 // in SelectionDAG.
5942 if (IsSigned &&
5943 (Add->getFlags().hasNoSignedWrap() ||
5944 DAG.willNotOverflowAdd(/*IsSigned=*/true, Add->getOperand(0),
5945 Add->getOperand(1))))
5946 return DAG.getNode(ISD::AVGCEILS, DL, VT, X, Y);
5947
5948 // FIXME: Use willNotOverflowAdd here?
5949 if (!IsSigned && Add->getFlags().hasNoUnsignedWrap())
5950 return DAG.getNode(ISD::AVGCEILU, DL, VT, X, Y);
5951 }
5952 }
5953
5954 // Fold avgfloors(x,y) -> avgflooru(x,y) if both x and y are non-negative
5955 if (Opcode == ISD::AVGFLOORS && hasOperation(ISD::AVGFLOORU, VT)) {
5956 if (DAG.SignBitIsZero(N0) && DAG.SignBitIsZero(N1))
5957 return DAG.getNode(ISD::AVGFLOORU, DL, VT, N0, N1);
5958 }
5959
5960 return SDValue();
5961}
5962
5963SDValue DAGCombiner::visitABD(SDNode *N) {
5964 unsigned Opcode = N->getOpcode();
5965 SDValue N0 = N->getOperand(0);
5966 SDValue N1 = N->getOperand(1);
5967 EVT VT = N->getValueType(0);
5968 SDLoc DL(N);
5969
5970 // fold (abd c1, c2)
5971 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1}))
5972 return C;
5973
5974 // canonicalize constant to RHS.
5977 return DAG.getNode(Opcode, DL, N->getVTList(), N1, N0);
5978
5979 if (VT.isVector())
5980 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
5981 return FoldedVOp;
5982
5983 // fold (abd x, undef) -> 0
5984 if (N0.isUndef() || N1.isUndef())
5985 return DAG.getConstant(0, DL, VT);
5986
5987 // fold (abd x, x) -> 0
5988 if (N0 == N1)
5989 return DAG.getConstant(0, DL, VT);
5990
5991 SDValue X, Y;
5992
5993 // fold (abds x, 0) -> abs x
5995 (!LegalOperations || hasOperation(ISD::ABS, VT)))
5996 return DAG.getNode(ISD::ABS, DL, VT, X);
5997
5998 // fold (abdu x, 0) -> x
6000 return X;
6001
6002 // fold (abds x, y) -> (abdu x, y) iff both args are known positive
6003 if (Opcode == ISD::ABDS && hasOperation(ISD::ABDU, VT) &&
6004 DAG.SignBitIsZero(N0) && DAG.SignBitIsZero(N1))
6005 return DAG.getNode(ISD::ABDU, DL, VT, N1, N0);
6006
6007 // fold (abdu x, y) -> (sub x, y) iff x >= y (the sub doesn't wrap).
6008 if (Opcode == ISD::ABDU) {
6009 if (DAG.willNotOverflowSub(/*IsSigned=*/false, N0, N1))
6010 return DAG.getNode(ISD::SUB, DL, VT, N0, N1);
6011 if (DAG.willNotOverflowSub(/*IsSigned=*/false, N1, N0))
6012 return DAG.getNode(ISD::SUB, DL, VT, N1, N0);
6013 }
6014
6015 // fold (abd? (?ext x), (?ext y)) -> (zext (abd? x, y))
6018 EVT SmallVT = X.getScalarValueSizeInBits() > Y.getScalarValueSizeInBits()
6019 ? X.getValueType()
6020 : Y.getValueType();
6021 if (!LegalOperations || hasOperation(Opcode, SmallVT)) {
6022 SDValue ExtedX = DAG.getExtOrTrunc(X, SDLoc(X), SmallVT, N0->getOpcode());
6023 SDValue ExtedY = DAG.getExtOrTrunc(Y, SDLoc(Y), SmallVT, N0->getOpcode());
6024 SDValue SmallABD = DAG.getNode(Opcode, DL, SmallVT, {ExtedX, ExtedY});
6025 SDValue ZExted = DAG.getZExtOrTrunc(SmallABD, DL, VT);
6026 return ZExted;
6027 }
6028 }
6029
6030 // fold (abd? (?ext ty:x), small_const:c) -> (zext (abd? x, c))
6033 EVT SmallVT = X.getValueType();
6034 if (!LegalOperations || hasOperation(Opcode, SmallVT)) {
6035 uint64_t Bits = SmallVT.getScalarSizeInBits();
6036 unsigned RelevantBits =
6037 (Opcode == ISD::ABDS) ? DAG.ComputeMaxSignificantBits(Y)
6039 bool TruncatingYIsCheap = TLI.isTruncateFree(Y, SmallVT) ||
6041 Y,
6042 [&](auto *C) {
6043 if (!C)
6044 return true;
6045 const APInt &YConst = C->getAsAPIntVal();
6046 return (Opcode == ISD::ABDS)
6047 ? YConst.isSignedIntN(Bits)
6048 : YConst.isIntN(Bits);
6049 },
6050 /*AllowUndefs=*/true);
6051
6052 if (RelevantBits <= Bits && TruncatingYIsCheap) {
6053 SDValue NewY = DAG.getNode(ISD::TRUNCATE, SDLoc(Y), SmallVT, Y);
6054 SDValue SmallABD = DAG.getNode(Opcode, DL, SmallVT, {X, NewY});
6055 return DAG.getZExtOrTrunc(SmallABD, DL, VT);
6056 }
6057 }
6058 }
6059
6060 return SDValue();
6061}
6062
6063/// Perform optimizations common to nodes that compute two values. LoOp and HiOp
6064/// give the opcodes for the two computations that are being performed. Return
6065/// true if a simplification was made.
6066SDValue DAGCombiner::SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
6067 unsigned HiOp) {
6068 // If the high half is not needed, just compute the low half.
6069 bool HiExists = N->hasAnyUseOfValue(1);
6070 if (!HiExists && (!LegalOperations ||
6071 TLI.isOperationLegalOrCustom(LoOp, N->getValueType(0)))) {
6072 SDValue Res = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0), N->ops());
6073 return CombineTo(N, Res, Res);
6074 }
6075
6076 // If the low half is not needed, just compute the high half.
6077 bool LoExists = N->hasAnyUseOfValue(0);
6078 if (!LoExists && (!LegalOperations ||
6079 TLI.isOperationLegalOrCustom(HiOp, N->getValueType(1)))) {
6080 SDValue Res = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1), N->ops());
6081 return CombineTo(N, Res, Res);
6082 }
6083
6084 // If both halves are used, return as it is.
6085 if (LoExists && HiExists)
6086 return SDValue();
6087
6088 // If the two computed results can be simplified separately, separate them.
6089 if (LoExists) {
6090 SDValue Lo = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0), N->ops());
6091 AddToWorklist(Lo.getNode());
6092 SDValue LoOpt = combine(Lo.getNode());
6093 if (LoOpt.getNode() && LoOpt.getNode() != Lo.getNode() &&
6094 (!LegalOperations ||
6096 return CombineTo(N, LoOpt, LoOpt);
6097 }
6098
6099 if (HiExists) {
6100 SDValue Hi = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1), N->ops());
6101 AddToWorklist(Hi.getNode());
6102 SDValue HiOpt = combine(Hi.getNode());
6103 if (HiOpt.getNode() && HiOpt != Hi &&
6104 (!LegalOperations ||
6106 return CombineTo(N, HiOpt, HiOpt);
6107 }
6108
6109 return SDValue();
6110}
6111
6112SDValue DAGCombiner::visitSMUL_LOHI(SDNode *N) {
6113 if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHS))
6114 return Res;
6115
6116 SDValue N0 = N->getOperand(0);
6117 SDValue N1 = N->getOperand(1);
6118 EVT VT = N->getValueType(0);
6119 SDLoc DL(N);
6120
6121 // fold (smul_lohi c1, c2)
6122 if (SDValue C = DAG.FoldConstantArithmetic(N->getOpcode(), DL, N->getVTList(),
6123 {N0, N1}))
6124 return C;
6125
6126 // canonicalize constant to RHS (vector doesn't have to splat)
6129 return DAG.getNode(ISD::SMUL_LOHI, DL, N->getVTList(), N1, N0);
6130
6131 // If the type is twice as wide is legal, transform the mulhu to a wider
6132 // multiply plus a shift.
6133 if (VT.isSimple() && !VT.isVector()) {
6134 MVT Simple = VT.getSimpleVT();
6135 unsigned SimpleSize = Simple.getSizeInBits();
6136 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
6137 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
6138 SDValue Lo = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N0);
6139 SDValue Hi = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N1);
6140 Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
6141 // Compute the high part as N1.
6142 Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
6143 DAG.getShiftAmountConstant(SimpleSize, NewVT, DL));
6144 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
6145 // Compute the low part as N0.
6146 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
6147 return CombineTo(N, Lo, Hi);
6148 }
6149 }
6150
6151 return SDValue();
6152}
6153
6154SDValue DAGCombiner::visitUMUL_LOHI(SDNode *N) {
6155 if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHU))
6156 return Res;
6157
6158 SDValue N0 = N->getOperand(0);
6159 SDValue N1 = N->getOperand(1);
6160 EVT VT = N->getValueType(0);
6161 SDLoc DL(N);
6162
6163 // fold (umul_lohi c1, c2)
6164 if (SDValue C = DAG.FoldConstantArithmetic(N->getOpcode(), DL, N->getVTList(),
6165 {N0, N1}))
6166 return C;
6167
6168 // canonicalize constant to RHS (vector doesn't have to splat)
6171 return DAG.getNode(ISD::UMUL_LOHI, DL, N->getVTList(), N1, N0);
6172
6173 // (umul_lohi N0, 0) -> (0, 0)
6174 if (isNullConstant(N1)) {
6175 SDValue Zero = DAG.getConstant(0, DL, VT);
6176 return CombineTo(N, Zero, Zero);
6177 }
6178
6179 // (umul_lohi N0, 1) -> (N0, 0)
6180 if (isOneConstant(N1)) {
6181 SDValue Zero = DAG.getConstant(0, DL, VT);
6182 return CombineTo(N, N0, Zero);
6183 }
6184
6185 // If the type is twice as wide is legal, transform the mulhu to a wider
6186 // multiply plus a shift.
6187 if (VT.isSimple() && !VT.isVector()) {
6188 MVT Simple = VT.getSimpleVT();
6189 unsigned SimpleSize = Simple.getSizeInBits();
6190 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
6191 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
6192 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N0);
6193 SDValue Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N1);
6194 Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
6195 // Compute the high part as N1.
6196 Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
6197 DAG.getShiftAmountConstant(SimpleSize, NewVT, DL));
6198 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
6199 // Compute the low part as N0.
6200 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
6201 return CombineTo(N, Lo, Hi);
6202 }
6203 }
6204
6205 return SDValue();
6206}
6207
6208SDValue DAGCombiner::visitMULO(SDNode *N) {
6209 SDValue N0 = N->getOperand(0);
6210 SDValue N1 = N->getOperand(1);
6211 EVT VT = N0.getValueType();
6212 bool IsSigned = (ISD::SMULO == N->getOpcode());
6213
6214 EVT CarryVT = N->getValueType(1);
6215 SDLoc DL(N);
6216
6217 // fold operation with constant operands.
6218 if (SDValue C = DAG.FoldConstantArithmetic(N->getOpcode(), DL, N->getVTList(),
6219 {N0, N1}))
6220 return C;
6221
6222 ConstantSDNode *N1C = isConstOrConstSplat(N1);
6223
6224 // canonicalize constant to RHS.
6227 return DAG.getNode(N->getOpcode(), DL, N->getVTList(), N1, N0);
6228
6229 // fold (mulo x, 0) -> 0 + no carry out
6230 if (isNullOrNullSplat(N1))
6231 return CombineTo(N, DAG.getConstant(0, DL, VT),
6232 DAG.getConstant(0, DL, CarryVT));
6233
6234 // (mulo x, 2) -> (addo x, x)
6235 // FIXME: This needs a freeze.
6236 if (N1C && N1C->getAPIntValue() == 2 &&
6237 (!IsSigned || VT.getScalarSizeInBits() > 2))
6238 return DAG.getNode(IsSigned ? ISD::SADDO : ISD::UADDO, DL,
6239 N->getVTList(), N0, N0);
6240
6241 // A 1 bit SMULO overflows if both inputs are 1.
6242 if (IsSigned && VT.getScalarSizeInBits() == 1) {
6243 SDValue And = DAG.getNode(ISD::AND, DL, VT, N0, N1);
6244 SDValue Cmp = DAG.getSetCC(DL, CarryVT, And,
6245 DAG.getConstant(0, DL, VT), ISD::SETNE);
6246 return CombineTo(N, And, Cmp);
6247 }
6248
6249 // If it cannot overflow, transform into a mul.
6250 if (DAG.willNotOverflowMul(IsSigned, N0, N1))
6251 return CombineTo(N, DAG.getNode(ISD::MUL, DL, VT, N0, N1),
6252 DAG.getConstant(0, DL, CarryVT));
6253 return SDValue();
6254}
6255
6256// Function to calculate whether the Min/Max pair of SDNodes (potentially
6257// swapped around) make a signed saturate pattern, clamping to between a signed
6258// saturate of -2^(BW-1) and 2^(BW-1)-1, or an unsigned saturate of 0 and 2^BW.
6259// Returns the node being clamped and the bitwidth of the clamp in BW. Should
6260// work with both SMIN/SMAX nodes and setcc/select combo. The operands are the
6261// same as SimplifySelectCC. N0<N1 ? N2 : N3.
6263 SDValue N3, ISD::CondCode CC, unsigned &BW,
6264 bool &Unsigned, SelectionDAG &DAG) {
6265 auto isSignedMinMax = [&](SDValue N0, SDValue N1, SDValue N2, SDValue N3,
6266 ISD::CondCode CC) {
6267 // The compare and select operand should be the same or the select operands
6268 // should be truncated versions of the comparison.
6269 if (N0 != N2 && (N2.getOpcode() != ISD::TRUNCATE || N0 != N2.getOperand(0)))
6270 return 0;
6271 // The constants need to be the same or a truncated version of each other.
6274 if (!N1C || !N3C)
6275 return 0;
6276 const APInt &C1 = N1C->getAPIntValue().trunc(N1.getScalarValueSizeInBits());
6277 const APInt &C2 = N3C->getAPIntValue().trunc(N3.getScalarValueSizeInBits());
6278 if (C1.getBitWidth() < C2.getBitWidth() || C1 != C2.sext(C1.getBitWidth()))
6279 return 0;
6280 return CC == ISD::SETLT ? ISD::SMIN : (CC == ISD::SETGT ? ISD::SMAX : 0);
6281 };
6282
6283 // Check the initial value is a SMIN/SMAX equivalent.
6284 unsigned Opcode0 = isSignedMinMax(N0, N1, N2, N3, CC);
6285 if (!Opcode0)
6286 return SDValue();
6287
6288 // We could only need one range check, if the fptosi could never produce
6289 // the upper value.
6290 if (N0.getOpcode() == ISD::FP_TO_SINT && Opcode0 == ISD::SMAX) {
6291 if (isNullOrNullSplat(N3)) {
6292 EVT IntVT = N0.getValueType().getScalarType();
6293 EVT FPVT = N0.getOperand(0).getValueType().getScalarType();
6294 if (FPVT.isSimple()) {
6295 Type *InputTy = FPVT.getTypeForEVT(*DAG.getContext());
6296 const fltSemantics &Semantics = InputTy->getFltSemantics();
6297 uint32_t MinBitWidth =
6298 APFloatBase::semanticsIntSizeInBits(Semantics, /*isSigned*/ true);
6299 if (IntVT.getSizeInBits() >= MinBitWidth) {
6300 Unsigned = true;
6301 BW = PowerOf2Ceil(MinBitWidth);
6302 return N0;
6303 }
6304 }
6305 }
6306 }
6307
6308 SDValue N00, N01, N02, N03;
6309 ISD::CondCode N0CC;
6310 switch (N0.getOpcode()) {
6311 case ISD::SMIN:
6312 case ISD::SMAX:
6313 N00 = N02 = N0.getOperand(0);
6314 N01 = N03 = N0.getOperand(1);
6315 N0CC = N0.getOpcode() == ISD::SMIN ? ISD::SETLT : ISD::SETGT;
6316 break;
6317 case ISD::SELECT_CC:
6318 N00 = N0.getOperand(0);
6319 N01 = N0.getOperand(1);
6320 N02 = N0.getOperand(2);
6321 N03 = N0.getOperand(3);
6322 N0CC = N0.getOperandCondCode(4);
6323 break;
6324 case ISD::SELECT:
6325 case ISD::VSELECT:
6326 if (N0.getOperand(0).getOpcode() != ISD::SETCC)
6327 return SDValue();
6328 N00 = N0.getOperand(0).getOperand(0);
6329 N01 = N0.getOperand(0).getOperand(1);
6330 N02 = N0.getOperand(1);
6331 N03 = N0.getOperand(2);
6332 N0CC = N0.getOperand(0).getOperandCondCode(2);
6333 break;
6334 default:
6335 return SDValue();
6336 }
6337
6338 unsigned Opcode1 = isSignedMinMax(N00, N01, N02, N03, N0CC);
6339 if (!Opcode1 || Opcode0 == Opcode1)
6340 return SDValue();
6341
6342 ConstantSDNode *MinCOp = isConstOrConstSplat(Opcode0 == ISD::SMIN ? N1 : N01);
6343 ConstantSDNode *MaxCOp = isConstOrConstSplat(Opcode0 == ISD::SMIN ? N01 : N1);
6344 if (!MinCOp || !MaxCOp || MinCOp->getValueType(0) != MaxCOp->getValueType(0))
6345 return SDValue();
6346
6347 const APInt &MinC = MinCOp->getAPIntValue();
6348 const APInt &MaxC = MaxCOp->getAPIntValue();
6349 APInt MinCPlus1 = MinC + 1;
6350 if (-MaxC == MinCPlus1 && MinCPlus1.isPowerOf2()) {
6351 BW = MinCPlus1.exactLogBase2() + 1;
6352 Unsigned = false;
6353 return N02;
6354 }
6355
6356 if (MaxC == 0 && MinC != 0 && MinCPlus1.isPowerOf2()) {
6357 BW = MinCPlus1.exactLogBase2();
6358 Unsigned = true;
6359 return N02;
6360 }
6361
6362 return SDValue();
6363}
6364
6367 SelectionDAG &DAG) {
6368 unsigned BW;
6369 bool Unsigned;
6370 SDValue Fp = isSaturatingMinMax(N0, N1, N2, N3, CC, BW, Unsigned, DAG);
6371 if (!Fp || Fp.getOpcode() != ISD::FP_TO_SINT)
6372 return SDValue();
6373 EVT FPVT = Fp.getOperand(0).getValueType();
6374 EVT NewVT = FPVT.changeElementType(*DAG.getContext(),
6375 EVT::getIntegerVT(*DAG.getContext(), BW));
6376 unsigned NewOpc = Unsigned ? ISD::FP_TO_UINT_SAT : ISD::FP_TO_SINT_SAT;
6377 if (!DAG.getTargetLoweringInfo().shouldConvertFpToSat(NewOpc, FPVT, NewVT))
6378 return SDValue();
6379 SDLoc DL(Fp);
6380 SDValue Sat = DAG.getNode(NewOpc, DL, NewVT, Fp.getOperand(0),
6381 DAG.getValueType(NewVT.getScalarType()));
6382 return DAG.getExtOrTrunc(!Unsigned, Sat, DL, N2->getValueType(0));
6383}
6384
6387 SelectionDAG &DAG) {
6388 // We are looking for UMIN(FPTOUI(X), (2^n)-1), which may have come via a
6389 // select/vselect/select_cc. The two operands pairs for the select (N2/N3) may
6390 // be truncated versions of the setcc (N0/N1).
6391 if ((N0 != N2 &&
6392 (N2.getOpcode() != ISD::TRUNCATE || N0 != N2.getOperand(0))) ||
6394 return SDValue();
6397 if (!N1C || !N3C)
6398 return SDValue();
6399 const APInt &C1 = N1C->getAPIntValue();
6400 const APInt &C3 = N3C->getAPIntValue();
6401 if (!(C1 + 1).isPowerOf2() || C1.getBitWidth() < C3.getBitWidth() ||
6402 C1 != C3.zext(C1.getBitWidth()))
6403 return SDValue();
6404
6405 unsigned BW = (C1 + 1).exactLogBase2();
6406 // When BW is 0, there is no equivalent i0 VT. And since an unsigned value is
6407 // never < 0, the result is always 0.
6408 if (BW == 0)
6409 return DAG.getConstant(0, SDLoc(N0), N3.getValueType());
6410
6411 EVT FPVT = N0.getOperand(0).getValueType();
6412 EVT NewVT = FPVT.changeElementType(*DAG.getContext(),
6413 EVT::getIntegerVT(*DAG.getContext(), BW));
6415 FPVT, NewVT))
6416 return SDValue();
6417
6418 SDValue Sat =
6419 DAG.getNode(ISD::FP_TO_UINT_SAT, SDLoc(N0), NewVT, N0.getOperand(0),
6420 DAG.getValueType(NewVT.getScalarType()));
6421 return DAG.getZExtOrTrunc(Sat, SDLoc(N0), N3.getValueType());
6422}
6423
6424// Fold a NaN-guard select of fp_to_sint/fp_to_uint into the saturating
6425// variant, which returns 0 for NaN.
6427 EVT VT = N->getValueType(0);
6428 SDLoc DL(N);
6429
6430 // Match an isnan-guarded select, requiring the compare to be single-use.
6431 // The guarded value is fp_to_sint/fp_to_uint of X, optionally masked by an
6432 // AND:
6433 // select (setcc X, 0.0, uno), 0, (fp_to_sint/uint X)
6434 // select (setcc X, 0.0, ord), (fp_to_sint/uint X), 0
6435 // select (setcc X, 0.0, uno), 0, (and (fp_to_sint/uint X), M)
6436 // select (setcc X, 0.0, ord), (and (fp_to_sint/uint X), M), 0
6437 SDValue X, GuardedVal;
6439 m_AnyZeroFP())),
6440 m_Zero(), m_Value(GuardedVal))) &&
6442 m_AnyZeroFP())),
6443 m_Value(GuardedVal), m_Zero())))
6444 return SDValue();
6445
6446 // The guarded value must be fp_to_sint/fp_to_uint of the same X, optionally
6447 // masked by a (commutative) AND.
6448 SDValue Mask;
6449 unsigned NewOpc;
6450 if (sd_match(GuardedVal, m_FPToSI(m_Specific(X))) ||
6451 sd_match(GuardedVal, m_And(m_FPToSI(m_Specific(X)), m_Value(Mask))))
6452 NewOpc = ISD::FP_TO_SINT_SAT;
6453 else if (sd_match(GuardedVal, m_FPToUI(m_Specific(X))) ||
6454 sd_match(GuardedVal, m_And(m_FPToUI(m_Specific(X)), m_Value(Mask))))
6455 NewOpc = ISD::FP_TO_UINT_SAT;
6456 else
6457 return SDValue();
6458
6460 X.getValueType(), VT))
6461 return SDValue();
6462
6463 SDValue Sat =
6464 DAG.getNode(NewOpc, DL, VT, X, DAG.getValueType(VT.getScalarType()));
6465 if (Mask) {
6466 // For NaN inputs the saturating conversion yields 0, so (and 0, Mask) must
6467 // stay 0 to match the original select. A poison Mask would make it poison,
6468 // so freeze Mask to guarantee a defined value.
6469 Sat = DAG.getNode(ISD::AND, DL, VT, Sat, DAG.getFreeze(Mask));
6470 }
6471 return Sat;
6472}
6473
6474SDValue DAGCombiner::visitIMINMAX(SDNode *N) {
6475 SDValue N0 = N->getOperand(0);
6476 SDValue N1 = N->getOperand(1);
6477 EVT VT = N0.getValueType();
6478 unsigned Opcode = N->getOpcode();
6479 SDLoc DL(N);
6480
6481 // fold operation with constant operands.
6482 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1}))
6483 return C;
6484
6485 // If the operands are the same, this is a no-op.
6486 if (N0 == N1)
6487 return N0;
6488
6489 // canonicalize constant to RHS
6492 return DAG.getNode(Opcode, DL, VT, N1, N0);
6493
6494 // fold vector ops
6495 if (VT.isVector())
6496 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
6497 return FoldedVOp;
6498
6499 // reassociate minmax
6500 if (SDValue RMINMAX = reassociateOps(Opcode, DL, N0, N1, N->getFlags()))
6501 return RMINMAX;
6502
6503 // Fold sign-extension masks using arithmetic shift:
6504 // smax(X, -1) -> or(X, ashr(X, BW-1))
6505 // smin(X, 0) -> and(X, ashr(X, BW-1))
6506 // ashr(X, BW-1) sign-extends the sign bit: 0 for X>=0, -1 for X<0.
6507 // OR with X yields X (non-negative) or -1 (negative) = smax(X,-1).
6508 // AND with X yields 0 (non-negative) or X (negative) = smin(X, 0).
6509 // Both reduce to two instructions vs. a compare+cmov on x86-64.
6510 // Only fold when the target has no native SMAX/SMIN instruction for this
6511 // type (isOperationExpand), the type is legal (not needing splitting),
6512 // the operand is not a min/max chain (preserving target combine patterns
6513 // that fold smax(smin(x,C),D) into a single saturation instruction), and
6514 // for smax(X,-1) the operand is not a sign extension (doubling its use
6515 // count can cause the target to lower the extension less efficiently).
6516 APInt C;
6517 if (TLI.isTypeLegal(VT) &&
6519 sd_match(N1, m_ConstInt(C))) {
6520 if (Opcode == ISD::SMAX && TLI.isOperationExpand(ISD::SMAX, VT) &&
6521 N0.getOpcode() != ISD::SMIN && N0.getOpcode() != ISD::SIGN_EXTEND &&
6522 C.isAllOnes()) {
6523 SDValue ShiftAmt =
6525 SDValue Shift = DAG.getNode(ISD::SRA, DL, VT, N0, ShiftAmt);
6526 return DAG.getNode(ISD::OR, DL, VT, N0, Shift);
6527 }
6528 if (Opcode == ISD::SMIN && TLI.isOperationExpand(ISD::SMIN, VT) &&
6529 N0.getOpcode() != ISD::SMAX && C.isZero()) {
6530 SDValue ShiftAmt =
6532 SDValue Shift = DAG.getNode(ISD::SRA, DL, VT, N0, ShiftAmt);
6533 return DAG.getNode(ISD::AND, DL, VT, N0, Shift);
6534 }
6535 }
6536
6537 // If both operands are known to have the same sign (both non-negative or both
6538 // negative), flip between UMIN/UMAX and SMIN/SMAX.
6539 // Only do this if:
6540 // 1. The current op isn't legal and the flipped is.
6541 // 2. The saturation pattern is broken by canonicalization in InstCombine.
6542 bool IsOpIllegal = !TLI.isOperationLegal(Opcode, VT);
6543 bool IsSatBroken = Opcode == ISD::UMIN && N0.getOpcode() == ISD::SMAX;
6544
6545 if (IsSatBroken || IsOpIllegal) {
6546 auto HasKnownSameSign = [&](SDValue A, SDValue B) {
6547 if (A.isUndef() || B.isUndef())
6548 return true;
6549
6550 KnownBits KA = DAG.computeKnownBits(A);
6551 if (!KA.isNonNegative() && !KA.isNegative())
6552 return false;
6553
6554 KnownBits KB = DAG.computeKnownBits(B);
6555 if (KA.isNonNegative())
6556 return KB.isNonNegative();
6557 return KB.isNegative();
6558 };
6559
6560 if (HasKnownSameSign(N0, N1)) {
6561 unsigned AltOpcode = ISD::getOppositeSignednessMinMaxOpcode(Opcode);
6562 if ((IsSatBroken && IsOpIllegal) || TLI.isOperationLegal(AltOpcode, VT))
6563 return DAG.getNode(AltOpcode, DL, VT, N0, N1);
6564 }
6565 }
6566
6567 if (Opcode == ISD::SMIN || Opcode == ISD::SMAX)
6569 N0, N1, N0, N1, Opcode == ISD::SMIN ? ISD::SETLT : ISD::SETGT, DAG))
6570 return S;
6571 if (Opcode == ISD::UMIN)
6572 if (SDValue S = PerformUMinFpToSatCombine(N0, N1, N0, N1, ISD::SETULT, DAG))
6573 return S;
6574
6575 // Fold min/max(vecreduce(x), vecreduce(y)) -> vecreduce(min/max(x, y))
6576 auto ReductionOpcode = [](unsigned Opcode) {
6577 switch (Opcode) {
6578 case ISD::SMIN:
6579 return ISD::VECREDUCE_SMIN;
6580 case ISD::SMAX:
6581 return ISD::VECREDUCE_SMAX;
6582 case ISD::UMIN:
6583 return ISD::VECREDUCE_UMIN;
6584 case ISD::UMAX:
6585 return ISD::VECREDUCE_UMAX;
6586 default:
6587 llvm_unreachable("Unexpected opcode");
6588 }
6589 };
6590 if (SDValue SD = reassociateReduction(ReductionOpcode(Opcode), Opcode,
6591 SDLoc(N), VT, N0, N1))
6592 return SD;
6593
6594 // Fold operation with vscale operands.
6595 if (N0.getOpcode() == ISD::VSCALE && N1.getOpcode() == ISD::VSCALE) {
6598 if (Opcode == ISD::UMAX)
6599 return C0 > C1 ? N0 : N1;
6600 else if (Opcode == ISD::UMIN)
6601 return C0 > C1 ? N1 : N0;
6602 }
6603
6604 // If we know the range of vscale, see if we can fold it given a constant.
6605 if (N0.getOpcode() == ISD::VSCALE) {
6606 if (auto *C1 = dyn_cast<ConstantSDNode>(N1)) {
6607 bool ForSigned = (Opcode == ISD::SMAX || Opcode == ISD::SMIN);
6608 ConstantRange Range = DAG.computeConstantRange(N0, ForSigned);
6609
6610 const APInt &C1V = C1->getAPIntValue();
6611 if ((Opcode == ISD::UMAX && Range.getUnsignedMax().ule(C1V)) ||
6612 (Opcode == ISD::UMIN && Range.getUnsignedMin().uge(C1V)) ||
6613 (Opcode == ISD::SMAX && Range.getSignedMax().sle(C1V)) ||
6614 (Opcode == ISD::SMIN && Range.getSignedMin().sge(C1V))) {
6615 return N1;
6616 }
6617 }
6618 }
6619
6620 // Simplify the operands using demanded-bits information.
6622 return SDValue(N, 0);
6623
6624 return SDValue();
6625}
6626
6627/// If this is a bitwise logic instruction and both operands have the same
6628/// opcode, try to sink the other opcode after the logic instruction.
6629SDValue DAGCombiner::hoistLogicOpWithSameOpcodeHands(SDNode *N) {
6630 SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
6631 EVT VT = N0.getValueType();
6632 unsigned LogicOpcode = N->getOpcode();
6633 unsigned HandOpcode = N0.getOpcode();
6634 assert(ISD::isBitwiseLogicOp(LogicOpcode) && "Expected logic opcode");
6635 assert(HandOpcode == N1.getOpcode() && "Bad input!");
6636
6637 // Bail early if none of these transforms apply.
6638 if (N0.getNumOperands() == 0)
6639 return SDValue();
6640
6641 // FIXME: We should check number of uses of the operands to not increase
6642 // the instruction count for all transforms.
6643
6644 // Handle size-changing casts (or sign_extend_inreg).
6645 SDValue X = N0.getOperand(0);
6646 SDValue Y = N1.getOperand(0);
6647 EVT XVT = X.getValueType();
6648 SDLoc DL(N);
6649 if (ISD::isExtOpcode(HandOpcode) || ISD::isExtVecInRegOpcode(HandOpcode) ||
6650 (HandOpcode == ISD::SIGN_EXTEND_INREG &&
6651 N0.getOperand(1) == N1.getOperand(1))) {
6652 // If both operands have other uses, this transform would create extra
6653 // instructions without eliminating anything.
6654 if (!N0.hasOneUse() && !N1.hasOneUse())
6655 return SDValue();
6656 // We need matching integer source types.
6657 if (XVT != Y.getValueType())
6658 return SDValue();
6659 // Don't create an illegal op during or after legalization. Don't ever
6660 // create an unsupported vector op.
6661 if ((VT.isVector() || LegalOperations) &&
6662 !TLI.isOperationLegalOrCustom(LogicOpcode, XVT))
6663 return SDValue();
6664 // Avoid infinite looping with PromoteIntBinOp.
6665 // TODO: Should we apply desirable/legal constraints to all opcodes?
6666 if ((HandOpcode == ISD::ANY_EXTEND ||
6667 HandOpcode == ISD::ANY_EXTEND_VECTOR_INREG) &&
6668 LegalTypes && !TLI.isTypeDesirableForOp(LogicOpcode, XVT))
6669 return SDValue();
6670 // logic_op (hand_op X), (hand_op Y) --> hand_op (logic_op X, Y)
6671 SDNodeFlags LogicFlags;
6672 LogicFlags.setDisjoint(N->getFlags().hasDisjoint() &&
6673 ISD::isExtOpcode(HandOpcode));
6674 SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y, LogicFlags);
6675 if (HandOpcode == ISD::SIGN_EXTEND_INREG)
6676 return DAG.getNode(HandOpcode, DL, VT, Logic, N0.getOperand(1));
6677 return DAG.getNode(HandOpcode, DL, VT, Logic);
6678 }
6679
6680 // logic_op (truncate x), (truncate y) --> truncate (logic_op x, y)
6681 if (HandOpcode == ISD::TRUNCATE) {
6682 // If both operands have other uses, this transform would create extra
6683 // instructions without eliminating anything.
6684 if (!N0.hasOneUse() && !N1.hasOneUse())
6685 return SDValue();
6686 // We need matching source types.
6687 if (XVT != Y.getValueType())
6688 return SDValue();
6689 // Don't create an illegal op during or after legalization.
6690 if (LegalOperations && !TLI.isOperationLegal(LogicOpcode, XVT))
6691 return SDValue();
6692 // Be extra careful sinking truncate. If it's free, there's no benefit in
6693 // widening a binop. Also, don't create a logic op on an illegal type.
6694 if (TLI.isZExtFree(VT, XVT) && TLI.isTruncateFree(XVT, VT))
6695 return SDValue();
6696 if (!TLI.isTypeLegal(XVT))
6697 return SDValue();
6698 SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
6699 return DAG.getNode(HandOpcode, DL, VT, Logic);
6700 }
6701
6702 // For binops SHL/SRL/SRA/AND:
6703 // logic_op (OP x, z), (OP y, z) --> OP (logic_op x, y), z
6704 if ((HandOpcode == ISD::SHL || HandOpcode == ISD::SRL ||
6705 HandOpcode == ISD::SRA || HandOpcode == ISD::AND) &&
6706 N0.getOperand(1) == N1.getOperand(1)) {
6707 // If either operand has other uses, this transform is not an improvement.
6708 if (!N0.hasOneUse() || !N1.hasOneUse())
6709 return SDValue();
6710 SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
6711 return DAG.getNode(HandOpcode, DL, VT, Logic, N0.getOperand(1));
6712 }
6713
6714 // Unary ops: logic_op (bswap x), (bswap y) --> bswap (logic_op x, y)
6715 if (HandOpcode == ISD::BSWAP) {
6716 // If either operand has other uses, this transform is not an improvement.
6717 if (!N0.hasOneUse() || !N1.hasOneUse())
6718 return SDValue();
6719 SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
6720 return DAG.getNode(HandOpcode, DL, VT, Logic);
6721 }
6722
6723 // For funnel shifts FSHL/FSHR:
6724 // logic_op (OP x, x1, s), (OP y, y1, s) -->
6725 // --> OP (logic_op x, y), (logic_op, x1, y1), s
6726 if ((HandOpcode == ISD::FSHL || HandOpcode == ISD::FSHR) &&
6727 N0.getOperand(2) == N1.getOperand(2)) {
6728 if (!N0.hasOneUse() || !N1.hasOneUse())
6729 return SDValue();
6730 SDValue X1 = N0.getOperand(1);
6731 SDValue Y1 = N1.getOperand(1);
6732 SDValue S = N0.getOperand(2);
6733 SDValue Logic0 = DAG.getNode(LogicOpcode, DL, VT, X, Y);
6734 SDValue Logic1 = DAG.getNode(LogicOpcode, DL, VT, X1, Y1);
6735 return DAG.getNode(HandOpcode, DL, VT, Logic0, Logic1, S);
6736 }
6737
6738 // Simplify xor/and/or (bitcast(A), bitcast(B)) -> bitcast(op (A,B))
6739 // Only perform this optimization up until type legalization, before
6740 // LegalizeVectorOprs. LegalizeVectorOprs promotes vector operations by
6741 // adding bitcasts. For example (xor v4i32) is promoted to (v2i64), and
6742 // we don't want to undo this promotion.
6743 // We also handle SCALAR_TO_VECTOR because xor/or/and operations are cheaper
6744 // on scalars.
6745 if ((HandOpcode == ISD::BITCAST || HandOpcode == ISD::SCALAR_TO_VECTOR) &&
6746 Level <= AfterLegalizeTypes) {
6747 // Input types must be integer and the same.
6748 if (XVT.isInteger() && XVT == Y.getValueType() &&
6749 !(VT.isVector() && TLI.isTypeLegal(VT) &&
6750 !XVT.isVector() && !TLI.isTypeLegal(XVT))) {
6751 SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
6752 return DAG.getNode(HandOpcode, DL, VT, Logic);
6753 }
6754 }
6755
6756 // Xor/and/or are indifferent to the swizzle operation (shuffle of one value).
6757 // Simplify xor/and/or (shuff(A), shuff(B)) -> shuff(op (A,B))
6758 // If both shuffles use the same mask, and both shuffle within a single
6759 // vector, then it is worthwhile to move the swizzle after the operation.
6760 // The type-legalizer generates this pattern when loading illegal
6761 // vector types from memory. In many cases this allows additional shuffle
6762 // optimizations.
6763 // There are other cases where moving the shuffle after the xor/and/or
6764 // is profitable even if shuffles don't perform a swizzle.
6765 // If both shuffles use the same mask, and both shuffles have the same first
6766 // or second operand, then it might still be profitable to move the shuffle
6767 // after the xor/and/or operation.
6768 if (HandOpcode == ISD::VECTOR_SHUFFLE && Level < AfterLegalizeDAG) {
6769 auto *SVN0 = cast<ShuffleVectorSDNode>(N0);
6770 auto *SVN1 = cast<ShuffleVectorSDNode>(N1);
6771 assert(X.getValueType() == Y.getValueType() &&
6772 "Inputs to shuffles are not the same type");
6773
6774 // Check that both shuffles use the same mask. The masks are known to be of
6775 // the same length because the result vector type is the same.
6776 // Check also that shuffles have only one use to avoid introducing extra
6777 // instructions.
6778 if (!SVN0->hasOneUse() || !SVN1->hasOneUse() ||
6779 !SVN0->getMask().equals(SVN1->getMask()))
6780 return SDValue();
6781
6782 // Don't try to fold this node if it requires introducing a
6783 // build vector of all zeros that might be illegal at this stage.
6784 SDValue ShOp = N0.getOperand(1);
6785 if (LogicOpcode == ISD::XOR && !ShOp.isUndef())
6786 ShOp = tryFoldToZero(DL, TLI, VT, DAG, LegalOperations);
6787
6788 // (logic_op (shuf (A, C), shuf (B, C))) --> shuf (logic_op (A, B), C)
6789 if (N0.getOperand(1) == N1.getOperand(1) && ShOp.getNode()) {
6790 SDValue Logic = DAG.getNode(LogicOpcode, DL, VT,
6791 N0.getOperand(0), N1.getOperand(0));
6792 return DAG.getVectorShuffle(VT, DL, Logic, ShOp, SVN0->getMask());
6793 }
6794
6795 // Don't try to fold this node if it requires introducing a
6796 // build vector of all zeros that might be illegal at this stage.
6797 ShOp = N0.getOperand(0);
6798 if (LogicOpcode == ISD::XOR && !ShOp.isUndef())
6799 ShOp = tryFoldToZero(DL, TLI, VT, DAG, LegalOperations);
6800
6801 // (logic_op (shuf (C, A), shuf (C, B))) --> shuf (C, logic_op (A, B))
6802 if (N0.getOperand(0) == N1.getOperand(0) && ShOp.getNode()) {
6803 SDValue Logic = DAG.getNode(LogicOpcode, DL, VT, N0.getOperand(1),
6804 N1.getOperand(1));
6805 return DAG.getVectorShuffle(VT, DL, ShOp, Logic, SVN0->getMask());
6806 }
6807 }
6808
6809 return SDValue();
6810}
6811
6812/// Try to make (and/or setcc (LL, LR), setcc (RL, RR)) more efficient.
6813SDValue DAGCombiner::foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1,
6814 const SDLoc &DL) {
6815 SDValue LL, LR, RL, RR, N0CC, N1CC;
6816 if (!isSetCCEquivalent(N0, LL, LR, N0CC) ||
6817 !isSetCCEquivalent(N1, RL, RR, N1CC))
6818 return SDValue();
6819
6820 assert(N0.getValueType() == N1.getValueType() &&
6821 "Unexpected operand types for bitwise logic op");
6822 assert(LL.getValueType() == LR.getValueType() &&
6823 RL.getValueType() == RR.getValueType() &&
6824 "Unexpected operand types for setcc");
6825
6826 // If we're here post-legalization or the logic op type is not i1, the logic
6827 // op type must match a setcc result type. Also, all folds require new
6828 // operations on the left and right operands, so those types must match.
6829 EVT VT = N0.getValueType();
6830 EVT OpVT = LL.getValueType();
6831 if (LegalOperations || VT.getScalarType() != MVT::i1)
6832 if (VT != getSetCCResultType(OpVT))
6833 return SDValue();
6834 if (OpVT != RL.getValueType())
6835 return SDValue();
6836
6837 ISD::CondCode CC0 = cast<CondCodeSDNode>(N0CC)->get();
6838 ISD::CondCode CC1 = cast<CondCodeSDNode>(N1CC)->get();
6839 bool IsInteger = OpVT.isInteger();
6840 if (LR == RR && CC0 == CC1 && IsInteger) {
6841 bool IsZero = isNullOrNullSplat(LR);
6842 bool IsNeg1 = isAllOnesOrAllOnesSplat(LR);
6843
6844 // All bits clear?
6845 bool AndEqZero = IsAnd && CC1 == ISD::SETEQ && IsZero;
6846 // All sign bits clear?
6847 bool AndGtNeg1 = IsAnd && CC1 == ISD::SETGT && IsNeg1;
6848 // Any bits set?
6849 bool OrNeZero = !IsAnd && CC1 == ISD::SETNE && IsZero;
6850 // Any sign bits set?
6851 bool OrLtZero = !IsAnd && CC1 == ISD::SETLT && IsZero;
6852
6853 // (and (seteq X, 0), (seteq Y, 0)) --> (seteq (or X, Y), 0)
6854 // (and (setgt X, -1), (setgt Y, -1)) --> (setgt (or X, Y), -1)
6855 // (or (setne X, 0), (setne Y, 0)) --> (setne (or X, Y), 0)
6856 // (or (setlt X, 0), (setlt Y, 0)) --> (setlt (or X, Y), 0)
6857 if (AndEqZero || AndGtNeg1 || OrNeZero || OrLtZero) {
6858 SDValue Or = DAG.getNode(ISD::OR, SDLoc(N0), OpVT, LL, RL);
6859 AddToWorklist(Or.getNode());
6860 return DAG.getSetCC(DL, VT, Or, LR, CC1);
6861 }
6862
6863 // All bits set?
6864 bool AndEqNeg1 = IsAnd && CC1 == ISD::SETEQ && IsNeg1;
6865 // All sign bits set?
6866 bool AndLtZero = IsAnd && CC1 == ISD::SETLT && IsZero;
6867 // Any bits clear?
6868 bool OrNeNeg1 = !IsAnd && CC1 == ISD::SETNE && IsNeg1;
6869 // Any sign bits clear?
6870 bool OrGtNeg1 = !IsAnd && CC1 == ISD::SETGT && IsNeg1;
6871
6872 // (and (seteq X, -1), (seteq Y, -1)) --> (seteq (and X, Y), -1)
6873 // (and (setlt X, 0), (setlt Y, 0)) --> (setlt (and X, Y), 0)
6874 // (or (setne X, -1), (setne Y, -1)) --> (setne (and X, Y), -1)
6875 // (or (setgt X, -1), (setgt Y -1)) --> (setgt (and X, Y), -1)
6876 if (AndEqNeg1 || AndLtZero || OrNeNeg1 || OrGtNeg1) {
6877 SDValue And = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, LL, RL);
6878 AddToWorklist(And.getNode());
6879 return DAG.getSetCC(DL, VT, And, LR, CC1);
6880 }
6881 }
6882
6883 // (and (setne (and X, LL1), 0), (setne (and X, RL1), 0))
6884 // --> (seteq (and X, (LL1|RL1)), (LL1|RL1))
6885 // (or (seteq (and X, LL1), 0), (seteq (and X, RL1), 0))
6886 // --> (setne (and X, (LL1|RL1)), (LL1|RL1))
6887 if (LL.getOpcode() == ISD::AND && RL.getOpcode() == ISD::AND &&
6888 isNullConstant(LR) && isNullConstant(RR) && CC0 == CC1 &&
6889 (CC0 == ISD::SETNE || CC0 == ISD::SETEQ)) {
6890 SDValue LL0, LL1, RL0, RL1;
6891 LL0 = LL.getOperand(0);
6892 RL0 = RL.getOperand(0);
6893 LL1 = LL.getOperand(1);
6894 RL1 = RL.getOperand(1);
6895 if (LL0 == RL0 && DAG.isKnownToBeAPowerOfTwo(LL1) &&
6896 DAG.isKnownToBeAPowerOfTwo(RL1)) {
6897 SDValue Or = DAG.getNode(ISD::OR, SDLoc(N0), OpVT, LL1, RL1);
6898 SDValue And = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, LL0, Or);
6899 return DAG.getSetCC(DL, VT, And, Or, IsAnd ? ISD::SETEQ : ISD::SETNE);
6900 }
6901 }
6902
6903 // (and (setne X, 0), (setne X, -1)) --> (setuge (add X, 1), 2)
6904 // (or (seteq X, 0), (seteq X, -1)) --> (setult (add X, 1), 2)
6905 if (LL == RL && CC0 == CC1 && OpVT.getScalarSizeInBits() > 1 && IsInteger &&
6906 ((IsAnd && CC0 == ISD::SETNE) || (!IsAnd && CC0 == ISD::SETEQ)) &&
6907 ((isNullConstant(LR) && isAllOnesConstant(RR)) ||
6908 (isAllOnesConstant(LR) && isNullConstant(RR)))) {
6909 SDValue One = DAG.getConstant(1, DL, OpVT);
6910 SDValue Two = DAG.getConstant(2, DL, OpVT);
6911 SDValue Add = DAG.getNode(ISD::ADD, SDLoc(N0), OpVT, LL, One);
6912 AddToWorklist(Add.getNode());
6913 return DAG.getSetCC(DL, VT, Add, Two, IsAnd ? ISD::SETUGE : ISD::SETULT);
6914 }
6915
6916 // Try more general transforms if the predicates match and the only user of
6917 // the compares is the 'and' or 'or'.
6918 if (IsInteger && TLI.convertSetCCLogicToBitwiseLogic(OpVT) && CC0 == CC1 &&
6919 N0.hasOneUse() && N1.hasOneUse()) {
6920 // and (seteq A, B), (seteq C, D) --> seteq (or (xor A, B), (xor C, D)), 0
6921 // or (setne A, B), (setne C, D) --> setne (or (xor A, B), (xor C, D)), 0
6922 if ((IsAnd && CC1 == ISD::SETEQ) || (!IsAnd && CC1 == ISD::SETNE)) {
6923 SDValue XorL = DAG.getNode(ISD::XOR, SDLoc(N0), OpVT, LL, LR);
6924 SDValue XorR = DAG.getNode(ISD::XOR, SDLoc(N1), OpVT, RL, RR);
6925 SDValue Or = DAG.getNode(ISD::OR, DL, OpVT, XorL, XorR);
6926 SDValue Zero = DAG.getConstant(0, DL, OpVT);
6927 return DAG.getSetCC(DL, VT, Or, Zero, CC1);
6928 }
6929
6930 // Turn compare of constants whose difference is 1 bit into add+and+setcc.
6931 if ((IsAnd && CC1 == ISD::SETNE) || (!IsAnd && CC1 == ISD::SETEQ)) {
6932 // Match a shared variable operand and 2 non-opaque constant operands.
6933 auto MatchDiffPow2 = [&](ConstantSDNode *C0, ConstantSDNode *C1) {
6934 // The difference of the constants must be a single bit.
6935 const APInt &CMax =
6936 APIntOps::umax(C0->getAPIntValue(), C1->getAPIntValue());
6937 const APInt &CMin =
6938 APIntOps::umin(C0->getAPIntValue(), C1->getAPIntValue());
6939 return !C0->isOpaque() && !C1->isOpaque() && (CMax - CMin).isPowerOf2();
6940 };
6941 if (LL == RL && ISD::matchBinaryPredicate(LR, RR, MatchDiffPow2)) {
6942 // and/or (setcc X, CMax, ne), (setcc X, CMin, ne/eq) -->
6943 // setcc ((sub X, CMin), ~(CMax - CMin)), 0, ne/eq
6944 SDValue Max = DAG.getNode(ISD::UMAX, DL, OpVT, LR, RR);
6945 SDValue Min = DAG.getNode(ISD::UMIN, DL, OpVT, LR, RR);
6946 SDValue Offset = DAG.getNode(ISD::SUB, DL, OpVT, LL, Min);
6947 SDValue Diff = DAG.getNode(ISD::SUB, DL, OpVT, Max, Min);
6948 SDValue Mask = DAG.getNOT(DL, Diff, OpVT);
6949 SDValue And = DAG.getNode(ISD::AND, DL, OpVT, Offset, Mask);
6950 SDValue Zero = DAG.getConstant(0, DL, OpVT);
6951 return DAG.getSetCC(DL, VT, And, Zero, CC0);
6952 }
6953 }
6954 }
6955
6956 // Canonicalize equivalent operands to LL == RL.
6957 if (LL == RR && LR == RL) {
6959 std::swap(RL, RR);
6960 }
6961
6962 // (and (setcc X, Y, CC0), (setcc X, Y, CC1)) --> (setcc X, Y, NewCC)
6963 // (or (setcc X, Y, CC0), (setcc X, Y, CC1)) --> (setcc X, Y, NewCC)
6964 if (LL == RL && LR == RR) {
6965 ISD::CondCode NewCC = IsAnd ? ISD::getSetCCAndOperation(CC0, CC1, OpVT)
6966 : ISD::getSetCCOrOperation(CC0, CC1, OpVT);
6967 if (NewCC != ISD::SETCC_INVALID &&
6968 (!LegalOperations ||
6969 (TLI.isCondCodeLegal(NewCC, LL.getSimpleValueType()) &&
6971 return DAG.getSetCC(DL, VT, LL, LR, NewCC);
6972 }
6973
6974 return SDValue();
6975}
6976
6977static bool areBothOperandsNotSNan(SDValue Operand1, SDValue Operand2,
6978 SelectionDAG &DAG) {
6979 return DAG.isKnownNeverSNaN(Operand2) && DAG.isKnownNeverSNaN(Operand1);
6980}
6981
6982static bool areBothOperandsNotNan(SDValue Operand1, SDValue Operand2,
6983 SelectionDAG &DAG) {
6984 return DAG.isKnownNeverNaN(Operand2) && DAG.isKnownNeverNaN(Operand1);
6985}
6986
6987/// Returns an appropriate FP min/max opcode for clamping operations.
6988static unsigned getMinMaxOpcodeForClamp(bool IsMin, SDValue Operand1,
6989 SDValue Operand2, SelectionDAG &DAG,
6990 const TargetLowering &TLI) {
6991 EVT VT = Operand1.getValueType();
6992 unsigned IEEEOp = IsMin ? ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE;
6993 if (TLI.isOperationLegalOrCustom(IEEEOp, VT) &&
6994 areBothOperandsNotNan(Operand1, Operand2, DAG))
6995 return IEEEOp;
6996 unsigned PreferredOp = IsMin ? ISD::FMINNUM : ISD::FMAXNUM;
6997 if (TLI.isOperationLegalOrCustom(PreferredOp, VT))
6998 return PreferredOp;
6999 return ISD::DELETED_NODE;
7000}
7001
7003 SDValue Operand1, SDValue Operand2, bool SetCCNoNaNs, ISD::CondCode CC,
7004 unsigned OrAndOpcode, SelectionDAG &DAG, bool isFMAXNUMFMINNUM_IEEE,
7005 bool isFMAXNUMFMINNUM, bool isFMAXIMUMNUMFMINIMUMNUM,
7006 bool isPSEUDOFMAXFMIN) {
7007 // The optimization cannot be applied for all the predicates because
7008 // of the way the min/max opcodes handle NaNs.
7009
7010 // It is safe to use PSEUDO_FMIN/PSEUDO_FMAX, FMINIMUMNUM/FMAXIMUMNUM or
7011 // FMINNUM_IEEE/FMAXNUM_IEEE if all the operands are non NaN values.
7012 if (((CC == ISD::SETLT || CC == ISD::SETLE) && (OrAndOpcode == ISD::OR)) ||
7013 ((CC == ISD::SETGT || CC == ISD::SETGE) && (OrAndOpcode == ISD::AND))) {
7014 if (!SetCCNoNaNs && !areBothOperandsNotNan(Operand1, Operand2, DAG))
7015 return ISD::DELETED_NODE;
7016 return isPSEUDOFMAXFMIN ? ISD::PSEUDO_FMIN
7017 : isFMAXIMUMNUMFMINIMUMNUM ? ISD::FMINIMUMNUM
7018 : isFMAXNUMFMINNUM_IEEE ? ISD::FMINNUM_IEEE
7020 }
7021
7022 if (((CC == ISD::SETGT || CC == ISD::SETGE) && (OrAndOpcode == ISD::OR)) ||
7023 ((CC == ISD::SETLT || CC == ISD::SETLE) && (OrAndOpcode == ISD::AND))) {
7024 if (!SetCCNoNaNs && !areBothOperandsNotNan(Operand1, Operand2, DAG))
7025 return ISD::DELETED_NODE;
7026 return isPSEUDOFMAXFMIN ? ISD::PSEUDO_FMAX
7027 : isFMAXIMUMNUMFMINIMUMNUM ? ISD::FMAXIMUMNUM
7028 : isFMAXNUMFMINNUM_IEEE ? ISD::FMAXNUM_IEEE
7030 }
7031
7032 bool IsMin;
7033 if (((CC == ISD::SETOLT || CC == ISD::SETOLE) && (OrAndOpcode == ISD::OR)) ||
7034 ((CC == ISD::SETUGT || CC == ISD::SETUGE) && (OrAndOpcode == ISD::AND)))
7035 IsMin = true;
7036 else if (((CC == ISD::SETOGT || CC == ISD::SETOGE) &&
7037 (OrAndOpcode == ISD::OR)) ||
7038 ((CC == ISD::SETULT || CC == ISD::SETULE) &&
7039 (OrAndOpcode == ISD::AND)))
7040 IsMin = false;
7041 else
7042 return ISD::DELETED_NODE;
7043
7044 // For the above predicates, the optimization is valid if a NaN operand is
7045 // discarded. PSEUDO_FMIN/PSEUDO_FMAX return the second operand if either
7046 // operand is a NaN. FMINIMUMNUM/FMAXIMUMNUM always discard. FMINNUM/FMAXNUM
7047 // and FMINNUM_IEEE/FMAXNUM_IEEE only do so for quiet NaNs, as a signaling NaN
7048 // operand may instead produce a NaN.
7049 if (isPSEUDOFMAXFMIN && DAG.isKnownNeverNaN(Operand2))
7050 return IsMin ? ISD::PSEUDO_FMIN : ISD::PSEUDO_FMAX;
7051 if (isFMAXIMUMNUMFMINIMUMNUM)
7052 return IsMin ? ISD::FMINIMUMNUM : ISD::FMAXIMUMNUM;
7053 if (!areBothOperandsNotSNan(Operand1, Operand2, DAG))
7054 return ISD::DELETED_NODE;
7055 if (isFMAXNUMFMINNUM)
7056 return IsMin ? ISD::FMINNUM : ISD::FMAXNUM;
7057 if (isFMAXNUMFMINNUM_IEEE)
7058 return IsMin ? ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE;
7059 return ISD::DELETED_NODE;
7060}
7061
7064 assert(
7065 (LogicOp->getOpcode() == ISD::AND || LogicOp->getOpcode() == ISD::OR) &&
7066 "Invalid Op to combine SETCC with");
7067
7068 // TODO: Search past casts/truncates.
7069 SDValue LHS = LogicOp->getOperand(0);
7070 SDValue RHS = LogicOp->getOperand(1);
7071 if (LHS->getOpcode() != ISD::SETCC || RHS->getOpcode() != ISD::SETCC ||
7072 !LHS->hasOneUse() || !RHS->hasOneUse())
7073 return SDValue();
7074
7075 SDNodeFlags LHSSetCCFlags = LHS->getFlags();
7076 SDNodeFlags RHSSetCCFlags = RHS->getFlags();
7078 AndOrSETCCFoldKind TargetPreference = TLI.isDesirableToCombineLogicOpOfSETCC(
7079 LogicOp, LHS.getNode(), RHS.getNode());
7080
7081 SDValue LHS0 = LHS->getOperand(0);
7082 SDValue RHS0 = RHS->getOperand(0);
7083 SDValue LHS1 = LHS->getOperand(1);
7084 SDValue RHS1 = RHS->getOperand(1);
7085 // TODO: We don't actually need a splat here, for vectors we just need the
7086 // invariants to hold for each element.
7087 auto *LHS1C = isConstOrConstSplat(LHS1);
7088 auto *RHS1C = isConstOrConstSplat(RHS1);
7089 ISD::CondCode CCL = LHS.getOperandCondCode(2);
7090 ISD::CondCode CCR = RHS.getOperandCondCode(2);
7091 EVT VT = LogicOp->getValueType(0);
7092 EVT OpVT = LHS0.getValueType();
7093 SDLoc DL(LogicOp);
7094
7095 // Check if the operands of an and/or operation are comparisons and if they
7096 // compare against the same value. Replace the and/or-cmp-cmp sequence with
7097 // min/max cmp sequence. If LHS1 is equal to RHS1, then the or-cmp-cmp
7098 // sequence will be replaced with min-cmp sequence:
7099 // (LHS0 < LHS1) | (RHS0 < RHS1) -> min(LHS0, RHS0) < LHS1
7100 // and and-cmp-cmp will be replaced with max-cmp sequence:
7101 // (LHS0 < LHS1) & (RHS0 < RHS1) -> max(LHS0, RHS0) < LHS1
7102 // The optimization does not work for `==` or `!=` .
7103 // The two comparisons should have either the same predicate or the
7104 // predicate of one of the comparisons is the opposite of the other one.
7105 bool isFMAXNUMFMINNUM_IEEE = TLI.isOperationLegal(ISD::FMAXNUM_IEEE, OpVT) &&
7106 TLI.isOperationLegal(ISD::FMINNUM_IEEE, OpVT);
7107 bool isFMAXNUMFMINNUM = TLI.isOperationLegalOrCustom(ISD::FMAXNUM, OpVT) &&
7108 TLI.isOperationLegalOrCustom(ISD::FMINNUM, OpVT);
7109 // A custom FMINIMUMNUM/FMAXIMUMNUM may be more expensive than the compares.
7110 bool isFMAXIMUMNUMFMINIMUMNUM =
7111 TLI.isOperationLegalOrCustom(ISD::FMAXIMUMNUM, OpVT) &&
7112 TLI.isOperationLegalOrCustom(ISD::FMINIMUMNUM, OpVT) &&
7113 TLI.isProfitableToCombineMinNumMaxNum(OpVT);
7114 bool isPSEUDOFMAXFMIN =
7115 TLI.isOperationLegalOrCustom(ISD::PSEUDO_FMAX, OpVT) &&
7116 TLI.isOperationLegalOrCustom(ISD::PSEUDO_FMIN, OpVT);
7117 if (((OpVT.isInteger() && TLI.isOperationLegal(ISD::UMAX, OpVT) &&
7118 TLI.isOperationLegal(ISD::SMAX, OpVT) &&
7119 TLI.isOperationLegal(ISD::UMIN, OpVT) &&
7120 TLI.isOperationLegal(ISD::SMIN, OpVT)) ||
7121 (OpVT.isFloatingPoint() &&
7122 (isFMAXNUMFMINNUM_IEEE || isFMAXNUMFMINNUM ||
7123 isFMAXIMUMNUMFMINIMUMNUM || isPSEUDOFMAXFMIN))) &&
7125 CCL != ISD::SETFALSE && CCL != ISD::SETO && CCL != ISD::SETUO &&
7126 CCL != ISD::SETTRUE &&
7127 (CCL == CCR || CCL == ISD::getSetCCSwappedOperands(CCR))) {
7128
7129 SDValue CommonValue, Operand1, Operand2;
7131 if (CCL == CCR) {
7132 if (LHS0 == RHS0) {
7133 CommonValue = LHS0;
7134 Operand1 = LHS1;
7135 Operand2 = RHS1;
7137 } else if (LHS1 == RHS1) {
7138 CommonValue = LHS1;
7139 Operand1 = LHS0;
7140 Operand2 = RHS0;
7141 CC = CCL;
7142 }
7143 } else {
7144 assert(CCL == ISD::getSetCCSwappedOperands(CCR) && "Unexpected CC");
7145 if (LHS0 == RHS1) {
7146 CommonValue = LHS0;
7147 Operand1 = LHS1;
7148 Operand2 = RHS0;
7149 CC = CCR;
7150 } else if (RHS0 == LHS1) {
7151 CommonValue = LHS1;
7152 Operand1 = LHS0;
7153 Operand2 = RHS1;
7154 CC = CCL;
7155 }
7156 }
7157
7158 // Don't do this transform for sign bit tests. Let foldLogicOfSetCCs
7159 // handle it using OR/AND.
7160 if (CC == ISD::SETLT && isNullOrNullSplat(CommonValue))
7162 else if (CC == ISD::SETGT && isAllOnesOrAllOnesSplat(CommonValue))
7164
7165 if (CC != ISD::SETCC_INVALID) {
7166 unsigned NewOpcode = ISD::DELETED_NODE;
7167 bool IsSigned = isSignedIntSetCC(CC);
7168 if (OpVT.isInteger()) {
7169 bool IsLess = (CC == ISD::SETLE || CC == ISD::SETULE ||
7170 CC == ISD::SETLT || CC == ISD::SETULT);
7171 bool IsOr = (LogicOp->getOpcode() == ISD::OR);
7172 if (IsLess == IsOr)
7173 NewOpcode = IsSigned ? ISD::SMIN : ISD::UMIN;
7174 else
7175 NewOpcode = IsSigned ? ISD::SMAX : ISD::UMAX;
7176 } else if (OpVT.isFloatingPoint()) {
7177 // The min/max opcodes below are commutative, so move a known non-NaN
7178 // operand second, where PSEUDO_FMIN/PSEUDO_FMAX can make use of it.
7179 if (isPSEUDOFMAXFMIN && !DAG.isKnownNeverNaN(Operand2) &&
7180 DAG.isKnownNeverNaN(Operand1))
7181 std::swap(Operand1, Operand2);
7183 Operand1, Operand2,
7184 LHSSetCCFlags.hasNoNaNs() && RHSSetCCFlags.hasNoNaNs(), CC,
7185 LogicOp->getOpcode(), DAG, isFMAXNUMFMINNUM_IEEE, isFMAXNUMFMINNUM,
7186 isFMAXIMUMNUMFMINIMUMNUM, isPSEUDOFMAXFMIN);
7187 }
7188
7189 if (NewOpcode != ISD::DELETED_NODE) {
7190 // Propagate fast-math flags from setcc.
7191 SDNodeFlags Flags = LHS->getFlags() & RHS->getFlags();
7192 SDValue MinMaxValue =
7193 DAG.getNode(NewOpcode, DL, OpVT, Operand1, Operand2, Flags);
7194 return DAG.getSetCC(DL, VT, MinMaxValue, CommonValue, CC, /*Chain=*/{},
7195 /*IsSignaling=*/false, Flags);
7196 }
7197 }
7198 }
7199
7200 if (LHS0 == LHS1 && RHS0 == RHS1 && CCL == CCR &&
7201 LHS0.getValueType() == RHS0.getValueType() &&
7202 ((LogicOp->getOpcode() == ISD::AND && CCL == ISD::SETO) ||
7203 (LogicOp->getOpcode() == ISD::OR && CCL == ISD::SETUO)))
7204 return DAG.getSetCC(DL, VT, LHS0, RHS0, CCL);
7205
7206 if (TargetPreference == AndOrSETCCFoldKind::None)
7207 return SDValue();
7208
7209 if (CCL == CCR &&
7210 CCL == (LogicOp->getOpcode() == ISD::AND ? ISD::SETNE : ISD::SETEQ) &&
7211 LHS0 == RHS0 && LHS1C && RHS1C && OpVT.isInteger()) {
7212 const APInt &APLhs = LHS1C->getAPIntValue();
7213 const APInt &APRhs = RHS1C->getAPIntValue();
7214
7215 // Preference is to use ISD::ABS or we already have an ISD::ABS (in which
7216 // case this is just a compare).
7217 if (APLhs == (-APRhs) &&
7218 ((TargetPreference & AndOrSETCCFoldKind::ABS) ||
7219 DAG.doesNodeExist(ISD::ABS, DAG.getVTList(OpVT), {LHS0}))) {
7220 const APInt &C = APLhs.isNegative() ? APRhs : APLhs;
7221 // (icmp eq A, C) | (icmp eq A, -C)
7222 // -> (icmp eq Abs(A), C)
7223 // (icmp ne A, C) & (icmp ne A, -C)
7224 // -> (icmp ne Abs(A), C)
7225 SDValue AbsOp = DAG.getNode(ISD::ABS, DL, OpVT, LHS0);
7226 return DAG.getNode(ISD::SETCC, DL, VT, AbsOp,
7227 DAG.getConstant(C, DL, OpVT), LHS.getOperand(2));
7228 } else if (TargetPreference &
7230
7231 // AndOrSETCCFoldKind::AddAnd:
7232 // A == C0 | A == C1
7233 // IF IsPow2(smax(C0, C1)-smin(C0, C1))
7234 // -> ((A - smin(C0, C1)) & ~(smax(C0, C1)-smin(C0, C1))) == 0
7235 // A != C0 & A != C1
7236 // IF IsPow2(smax(C0, C1)-smin(C0, C1))
7237 // -> ((A - smin(C0, C1)) & ~(smax(C0, C1)-smin(C0, C1))) != 0
7238
7239 // AndOrSETCCFoldKind::NotAnd:
7240 // A == C0 | A == C1
7241 // IF smax(C0, C1) == -1 AND IsPow2(smax(C0, C1) - smin(C0, C1))
7242 // -> ~A & smin(C0, C1) == 0
7243 // A != C0 & A != C1
7244 // IF smax(C0, C1) == -1 AND IsPow2(smax(C0, C1) - smin(C0, C1))
7245 // -> ~A & smin(C0, C1) != 0
7246
7247 const APInt &MaxC = APIntOps::smax(APRhs, APLhs);
7248 const APInt &MinC = APIntOps::smin(APRhs, APLhs);
7249 APInt Dif = MaxC - MinC;
7250 if (!Dif.isZero() && Dif.isPowerOf2()) {
7251 if (MaxC.isAllOnes() &&
7252 (TargetPreference & AndOrSETCCFoldKind::NotAnd)) {
7253 SDValue NotOp = DAG.getNOT(DL, LHS0, OpVT);
7254 SDValue AndOp = DAG.getNode(ISD::AND, DL, OpVT, NotOp,
7255 DAG.getConstant(MinC, DL, OpVT));
7256 return DAG.getNode(ISD::SETCC, DL, VT, AndOp,
7257 DAG.getConstant(0, DL, OpVT), LHS.getOperand(2));
7258 } else if (TargetPreference & AndOrSETCCFoldKind::AddAnd) {
7259
7260 SDValue AddOp = DAG.getNode(ISD::ADD, DL, OpVT, LHS0,
7261 DAG.getConstant(-MinC, DL, OpVT));
7262 SDValue AndOp = DAG.getNode(ISD::AND, DL, OpVT, AddOp,
7263 DAG.getConstant(~Dif, DL, OpVT));
7264 return DAG.getNode(ISD::SETCC, DL, VT, AndOp,
7265 DAG.getConstant(0, DL, OpVT), LHS.getOperand(2));
7266 }
7267 }
7268 }
7269 }
7270
7271 return SDValue();
7272}
7273
7274// Combine `(select c, (X & 1), 0)` -> `(and (zext c), X)`.
7275// We canonicalize to the `select` form in the middle end, but the `and` form
7276// gets better codegen and all tested targets (arm, x86, riscv)
7278 const SDLoc &DL, SelectionDAG &DAG) {
7280 if (!isNullConstant(F))
7281 return SDValue();
7282
7283 EVT CondVT = Cond.getValueType();
7284 if (TLI.getBooleanContents(CondVT) !=
7286 return SDValue();
7287
7288 if (T.getOpcode() != ISD::AND)
7289 return SDValue();
7290
7291 if (!isOneConstant(T.getOperand(1)))
7292 return SDValue();
7293
7294 EVT OpVT = T.getValueType();
7295
7296 SDValue CondMask =
7297 OpVT == CondVT ? Cond : DAG.getBoolExtOrTrunc(Cond, DL, OpVT, CondVT);
7298 return DAG.getNode(ISD::AND, DL, OpVT, CondMask, T.getOperand(0));
7299}
7300
7301/// This contains all DAGCombine rules which reduce two values combined by
7302/// an And operation to a single value. This makes them reusable in the context
7303/// of visitSELECT(). Rules involving constants are not included as
7304/// visitSELECT() already handles those cases.
7305SDValue DAGCombiner::visitANDLike(SDValue N0, SDValue N1, SDNode *N) {
7306 EVT VT = N1.getValueType();
7307 SDLoc DL(N);
7308
7309 // fold (and x, undef) -> 0
7310 if (N0.isUndef() || N1.isUndef())
7311 return DAG.getConstant(0, DL, VT);
7312
7313 if (SDValue V = foldLogicOfSetCCs(true, N0, N1, DL))
7314 return V;
7315
7316 // Canonicalize:
7317 // and(x, add) -> and(add, x)
7318 if (N1.getOpcode() == ISD::ADD)
7319 std::swap(N0, N1);
7320
7321 // TODO: Rewrite this to return a new 'AND' instead of using CombineTo.
7322 if (N0.getOpcode() == ISD::ADD && N1.getOpcode() == ISD::SRL &&
7323 VT.isScalarInteger() && VT.getSizeInBits() <= 64 && N0->hasOneUse()) {
7324 if (ConstantSDNode *ADDI = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
7325 if (ConstantSDNode *SRLI = dyn_cast<ConstantSDNode>(N1.getOperand(1))) {
7326 // Look for (and (add x, c1), (lshr y, c2)). If C1 wasn't a legal
7327 // immediate for an add, but it is legal if its top c2 bits are set,
7328 // transform the ADD so the immediate doesn't need to be materialized
7329 // in a register.
7330 APInt ADDC = ADDI->getAPIntValue();
7331 APInt SRLC = SRLI->getAPIntValue();
7332 if (ADDC.getSignificantBits() <= 64 && SRLC.ult(VT.getSizeInBits()) &&
7333 !TLI.isLegalAddImmediate(ADDC.getSExtValue())) {
7335 SRLC.getZExtValue());
7336 if (DAG.MaskedValueIsZero(N0.getOperand(1), Mask)) {
7337 ADDC |= Mask;
7338 if (TLI.isLegalAddImmediate(ADDC.getSExtValue())) {
7339 SDLoc DL0(N0);
7340 SDValue NewAdd =
7341 DAG.getNode(ISD::ADD, DL0, VT,
7342 N0.getOperand(0), DAG.getConstant(ADDC, DL, VT));
7343 CombineTo(N0.getNode(), NewAdd);
7344 // Return N so it doesn't get rechecked!
7345 return SDValue(N, 0);
7346 }
7347 }
7348 }
7349 }
7350 }
7351 }
7352
7353 return SDValue();
7354}
7355
7356bool DAGCombiner::isAndLoadExtLoad(ConstantSDNode *AndC, LoadSDNode *LoadN,
7357 EVT LoadResultTy, EVT &ExtVT) {
7358 if (!AndC->getAPIntValue().isMask())
7359 return false;
7360
7361 unsigned ActiveBits = AndC->getAPIntValue().countr_one();
7362
7363 ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
7364 EVT LoadedVT = LoadN->getMemoryVT();
7365
7366 if (ExtVT == LoadedVT &&
7367 (!LegalOperations ||
7368 TLI.isLoadLegal(LoadResultTy, ExtVT, LoadN->getAlign(),
7369 LoadN->getAddressSpace(), ISD::ZEXTLOAD, false))) {
7370 // ZEXTLOAD will match without needing to change the size of the value being
7371 // loaded.
7372 return true;
7373 }
7374
7375 // Do not change the width of a volatile or atomic loads.
7376 if (!LoadN->isSimple())
7377 return false;
7378
7379 // Do not generate loads of non-round integer types since these can
7380 // be expensive (and would be wrong if the type is not byte sized).
7381 if (!LoadedVT.bitsGT(ExtVT) || !ExtVT.isRound())
7382 return false;
7383
7384 if (LegalOperations &&
7385 !TLI.isLoadLegal(LoadResultTy, ExtVT, LoadN->getAlign(),
7386 LoadN->getAddressSpace(), ISD::ZEXTLOAD, false))
7387 return false;
7388
7389 if (!TLI.shouldReduceLoadWidth(LoadN, ISD::ZEXTLOAD, ExtVT, /*ByteOffset=*/0))
7390 return false;
7391
7392 return true;
7393}
7394
7395bool DAGCombiner::isLegalNarrowLdSt(LSBaseSDNode *LDST,
7396 ISD::LoadExtType ExtType, EVT &MemVT,
7397 unsigned ShAmt) {
7398 if (!LDST)
7399 return false;
7400
7401 // Only allow byte offsets.
7402 if (ShAmt % 8)
7403 return false;
7404 const unsigned ByteShAmt = ShAmt / 8;
7405
7406 // Do not generate loads of non-round integer types since these can
7407 // be expensive (and would be wrong if the type is not byte sized).
7408 if (!MemVT.isRound())
7409 return false;
7410
7411 // Don't change the width of a volatile or atomic loads.
7412 if (!LDST->isSimple())
7413 return false;
7414
7415 EVT LdStMemVT = LDST->getMemoryVT();
7416
7417 // Bail out when changing the scalable property, since we can't be sure that
7418 // we're actually narrowing here.
7419 if (LdStMemVT.isScalableVector() != MemVT.isScalableVector())
7420 return false;
7421
7422 // Verify that we are actually reducing a load width here.
7423 if (LdStMemVT.bitsLT(MemVT))
7424 return false;
7425
7426 // Ensure that this isn't going to produce an unsupported memory access.
7427 if (ShAmt) {
7428 const Align LDSTAlign = LDST->getAlign();
7429 const Align NarrowAlign = commonAlignment(LDSTAlign, ByteShAmt);
7430 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), MemVT,
7431 LDST->getAddressSpace(), NarrowAlign,
7432 LDST->getMemOperand()->getFlags()))
7433 return false;
7434 }
7435
7436 // It's not possible to generate a constant of extended or untyped type.
7437 EVT PtrType = LDST->getBasePtr().getValueType();
7438 if (PtrType == MVT::Untyped || PtrType.isExtended())
7439 return false;
7440
7441 if (isa<LoadSDNode>(LDST)) {
7442 LoadSDNode *Load = cast<LoadSDNode>(LDST);
7443 // Don't transform one with multiple uses, this would require adding a new
7444 // load.
7445 if (!SDValue(Load, 0).hasOneUse())
7446 return false;
7447
7448 if (LegalOperations &&
7449 !TLI.isLoadLegal(Load->getValueType(0), MemVT, Load->getAlign(),
7450 Load->getAddressSpace(), ExtType, false))
7451 return false;
7452
7453 // For the transform to be legal, the load must produce only two values
7454 // (the value loaded and the chain). Don't transform a pre-increment
7455 // load, for example, which produces an extra value. Otherwise the
7456 // transformation is not equivalent, and the downstream logic to replace
7457 // uses gets things wrong.
7458 if (Load->getNumValues() > 2)
7459 return false;
7460
7461 // If the load that we're shrinking is an extload and we're not just
7462 // discarding the extension we can't simply shrink the load. Bail.
7463 // TODO: It would be possible to merge the extensions in some cases.
7464 if (Load->getExtensionType() != ISD::NON_EXTLOAD &&
7465 Load->getMemoryVT().getSizeInBits() < MemVT.getSizeInBits() + ShAmt)
7466 return false;
7467
7468 if (!TLI.shouldReduceLoadWidth(Load, ExtType, MemVT, ByteShAmt))
7469 return false;
7470 } else {
7471 assert(isa<StoreSDNode>(LDST) && "It is not a Load nor a Store SDNode");
7472 StoreSDNode *Store = cast<StoreSDNode>(LDST);
7473 // Can't write outside the original store
7474 if (Store->getMemoryVT().getSizeInBits() < MemVT.getSizeInBits() + ShAmt)
7475 return false;
7476
7477 if (LegalOperations &&
7478 !TLI.isTruncStoreLegal(Store->getValue().getValueType(), MemVT,
7479 Store->getAlign(), Store->getAddressSpace()))
7480 return false;
7481 }
7482 return true;
7483}
7484
7485bool DAGCombiner::SearchForAndLoads(SDNode *N,
7486 SmallVectorImpl<LoadSDNode*> &Loads,
7487 SmallPtrSetImpl<SDNode*> &NodesWithConsts,
7488 ConstantSDNode *Mask,
7489 SDNode *&NodeToMask) {
7490 // Recursively search for the operands, looking for loads which can be
7491 // narrowed.
7492 for (SDValue Op : N->op_values()) {
7493 if (Op.getValueType().isVector())
7494 return false;
7495
7496 // Some constants may need fixing up later if they are too large.
7497 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
7498 assert(ISD::isBitwiseLogicOp(N->getOpcode()) &&
7499 "Expected bitwise logic operation");
7500 if (!C->getAPIntValue().isSubsetOf(Mask->getAPIntValue()))
7501 NodesWithConsts.insert(N);
7502 continue;
7503 }
7504
7505 if (!Op.hasOneUse())
7506 return false;
7507
7508 switch(Op.getOpcode()) {
7509 case ISD::LOAD: {
7510 auto *Load = cast<LoadSDNode>(Op);
7511 EVT ExtVT;
7512 if (isAndLoadExtLoad(Mask, Load, Load->getValueType(0), ExtVT) &&
7513 isLegalNarrowLdSt(Load, ISD::ZEXTLOAD, ExtVT)) {
7514
7515 // ZEXTLOAD is already small enough.
7516 if (Load->getExtensionType() == ISD::ZEXTLOAD &&
7517 ExtVT.bitsGE(Load->getMemoryVT()))
7518 continue;
7519
7520 // Use LE to convert equal sized loads to zext.
7521 if (ExtVT.bitsLE(Load->getMemoryVT()))
7522 Loads.push_back(Load);
7523
7524 continue;
7525 }
7526 return false;
7527 }
7528 case ISD::ZERO_EXTEND:
7529 case ISD::AssertZext: {
7530 unsigned ActiveBits = Mask->getAPIntValue().countr_one();
7531 EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
7532 EVT VT = Op.getOpcode() == ISD::AssertZext ?
7533 cast<VTSDNode>(Op.getOperand(1))->getVT() :
7534 Op.getOperand(0).getValueType();
7535
7536 // We can accept extending nodes if the mask is wider or an equal
7537 // width to the original type.
7538 if (ExtVT.bitsGE(VT))
7539 continue;
7540 break;
7541 }
7542 case ISD::OR:
7543 case ISD::XOR:
7544 case ISD::AND:
7545 if (!SearchForAndLoads(Op.getNode(), Loads, NodesWithConsts, Mask,
7546 NodeToMask))
7547 return false;
7548 continue;
7549 }
7550
7551 // Allow one node which will masked along with any loads found.
7552 if (NodeToMask)
7553 return false;
7554
7555 // Also ensure that the node to be masked only produces one data result.
7556 NodeToMask = Op.getNode();
7557 if (NodeToMask->getNumValues() > 1) {
7558 bool HasValue = false;
7559 for (unsigned i = 0, e = NodeToMask->getNumValues(); i < e; ++i) {
7560 MVT VT = SDValue(NodeToMask, i).getSimpleValueType();
7561 if (VT != MVT::Glue && VT != MVT::Other) {
7562 if (HasValue) {
7563 NodeToMask = nullptr;
7564 return false;
7565 }
7566 HasValue = true;
7567 }
7568 }
7569 assert(HasValue && "Node to be masked has no data result?");
7570 }
7571 }
7572 return true;
7573}
7574
7575bool DAGCombiner::BackwardsPropagateMask(SDNode *N) {
7576 auto *Mask = dyn_cast<ConstantSDNode>(N->getOperand(1));
7577 if (!Mask)
7578 return false;
7579
7580 if (!Mask->getAPIntValue().isMask())
7581 return false;
7582
7583 // No need to do anything if the and directly uses a load.
7584 if (isa<LoadSDNode>(N->getOperand(0)))
7585 return false;
7586
7588 SmallPtrSet<SDNode*, 2> NodesWithConsts;
7589 SDNode *FixupNode = nullptr;
7590 if (SearchForAndLoads(N, Loads, NodesWithConsts, Mask, FixupNode)) {
7591 if (Loads.empty())
7592 return false;
7593
7594 LLVM_DEBUG(dbgs() << "Backwards propagate AND: "; N->dump());
7595 SDValue MaskOp = N->getOperand(1);
7596
7597 // If it exists, fixup the single node we allow in the tree that needs
7598 // masking.
7599 if (FixupNode) {
7600 LLVM_DEBUG(dbgs() << "First, need to fix up: "; FixupNode->dump());
7601 SDValue And = DAG.getNode(ISD::AND, SDLoc(FixupNode),
7602 FixupNode->getValueType(0),
7603 SDValue(FixupNode, 0), MaskOp);
7604 DAG.ReplaceAllUsesOfValueWith(SDValue(FixupNode, 0), And);
7605 if (And.getOpcode() == ISD ::AND)
7606 DAG.UpdateNodeOperands(And.getNode(), SDValue(FixupNode, 0), MaskOp);
7607 }
7608
7609 // Narrow any constants that need it.
7610 for (auto *LogicN : NodesWithConsts) {
7611 SDValue Op0 = LogicN->getOperand(0);
7612 SDValue Op1 = LogicN->getOperand(1);
7613
7614 // We only need to fix AND if both inputs are constants. And we only need
7615 // to fix one of the constants.
7616 if (LogicN->getOpcode() == ISD::AND &&
7618 continue;
7619
7620 if (isa<ConstantSDNode>(Op0) && LogicN->getOpcode() != ISD::AND)
7621 Op0 =
7622 DAG.getNode(ISD::AND, SDLoc(Op0), Op0.getValueType(), Op0, MaskOp);
7623
7624 if (isa<ConstantSDNode>(Op1))
7625 Op1 =
7626 DAG.getNode(ISD::AND, SDLoc(Op1), Op1.getValueType(), Op1, MaskOp);
7627
7628 if (isa<ConstantSDNode>(Op0) && !isa<ConstantSDNode>(Op1))
7629 std::swap(Op0, Op1);
7630
7631 DAG.UpdateNodeOperands(LogicN, Op0, Op1);
7632 }
7633
7634 // Create narrow loads.
7635 for (auto *Load : Loads) {
7636 LLVM_DEBUG(dbgs() << "Propagate AND back to: "; Load->dump());
7637 SDValue And = DAG.getNode(ISD::AND, SDLoc(Load), Load->getValueType(0),
7638 SDValue(Load, 0), MaskOp);
7640 if (And.getOpcode() == ISD ::AND)
7641 And = SDValue(
7642 DAG.UpdateNodeOperands(And.getNode(), SDValue(Load, 0), MaskOp), 0);
7643 SDValue NewLoad = reduceLoadWidth(And.getNode());
7644 assert(NewLoad &&
7645 "Shouldn't be masking the load if it can't be narrowed");
7646 CombineTo(Load, NewLoad, NewLoad.getValue(1));
7647 }
7648 DAG.ReplaceAllUsesWith(N, N->getOperand(0).getNode());
7649 return true;
7650 }
7651 return false;
7652}
7653
7654// Unfold
7655// x & (-1 'logical shift' y)
7656// To
7657// (x 'opposite logical shift' y) 'logical shift' y
7658// if it is better for performance.
7659SDValue DAGCombiner::unfoldExtremeBitClearingToShifts(SDNode *N) {
7660 assert(N->getOpcode() == ISD::AND);
7661
7662 SDValue N0 = N->getOperand(0);
7663 SDValue N1 = N->getOperand(1);
7664
7665 // Do we actually prefer shifts over mask?
7667 return SDValue();
7668
7669 // Try to match (-1 '[outer] logical shift' y)
7670 unsigned OuterShift;
7671 unsigned InnerShift; // The opposite direction to the OuterShift.
7672 SDValue Y; // Shift amount.
7673 auto matchMask = [&OuterShift, &InnerShift, &Y](SDValue M) -> bool {
7674 if (!M.hasOneUse())
7675 return false;
7676 OuterShift = M->getOpcode();
7677 if (OuterShift == ISD::SHL)
7678 InnerShift = ISD::SRL;
7679 else if (OuterShift == ISD::SRL)
7680 InnerShift = ISD::SHL;
7681 else
7682 return false;
7683 if (!isAllOnesConstant(M->getOperand(0)))
7684 return false;
7685 Y = M->getOperand(1);
7686 return true;
7687 };
7688
7689 SDValue X;
7690 if (matchMask(N1))
7691 X = N0;
7692 else if (matchMask(N0))
7693 X = N1;
7694 else
7695 return SDValue();
7696
7697 SDLoc DL(N);
7698 EVT VT = N->getValueType(0);
7699
7700 // tmp = x 'opposite logical shift' y
7701 SDValue T0 = DAG.getNode(InnerShift, DL, VT, X, Y);
7702 // ret = tmp 'logical shift' y
7703 SDValue T1 = DAG.getNode(OuterShift, DL, VT, T0, Y);
7704
7705 return T1;
7706}
7707
7708/// Try to replace shift/logic that tests if a bit is clear with mask + setcc.
7709/// For a target with a bit test, this is expected to become test + set and save
7710/// at least 1 instruction.
7712 assert(And->getOpcode() == ISD::AND && "Expected an 'and' op");
7713
7714 // Look through an optional extension.
7715 SDValue And0 = And->getOperand(0), And1 = And->getOperand(1);
7716 if (And0.getOpcode() == ISD::ANY_EXTEND && And0.hasOneUse())
7717 And0 = And0.getOperand(0);
7718 if (!isOneConstant(And1) || !And0.hasOneUse())
7719 return SDValue();
7720
7721 SDValue Src = And0;
7722
7723 // Attempt to find a 'not' op.
7724 // TODO: Should we favor test+set even without the 'not' op?
7725 bool FoundNot = false;
7726 if (isBitwiseNot(Src)) {
7727 FoundNot = true;
7728 Src = Src.getOperand(0);
7729
7730 // Look though an optional truncation. The source operand may not be the
7731 // same type as the original 'and', but that is ok because we are masking
7732 // off everything but the low bit.
7733 if (Src.getOpcode() == ISD::TRUNCATE && Src.hasOneUse())
7734 Src = Src.getOperand(0);
7735 }
7736
7737 // Match a shift-right by constant.
7738 if (Src.getOpcode() != ISD::SRL || !Src.hasOneUse())
7739 return SDValue();
7740
7741 // This is probably not worthwhile without a supported type.
7742 EVT SrcVT = Src.getValueType();
7744 if (!TLI.isTypeLegal(SrcVT))
7745 return SDValue();
7746
7747 // We might have looked through casts that make this transform invalid.
7748 unsigned BitWidth = SrcVT.getScalarSizeInBits();
7749 SDValue ShiftAmt = Src.getOperand(1);
7750 auto *ShiftAmtC = dyn_cast<ConstantSDNode>(ShiftAmt);
7751 if (!ShiftAmtC || !ShiftAmtC->getAPIntValue().ult(BitWidth))
7752 return SDValue();
7753
7754 // Set source to shift source.
7755 Src = Src.getOperand(0);
7756
7757 // Try again to find a 'not' op.
7758 // TODO: Should we favor test+set even with two 'not' ops?
7759 if (!FoundNot) {
7760 if (!isBitwiseNot(Src))
7761 return SDValue();
7762 Src = Src.getOperand(0);
7763 }
7764
7765 if (!TLI.hasBitTest(Src, ShiftAmt))
7766 return SDValue();
7767
7768 // Turn this into a bit-test pattern using mask op + setcc:
7769 // and (not (srl X, C)), 1 --> (and X, 1<<C) == 0
7770 // and (srl (not X), C)), 1 --> (and X, 1<<C) == 0
7771 SDLoc DL(And);
7772 SDValue X = DAG.getZExtOrTrunc(Src, DL, SrcVT);
7773 EVT CCVT =
7774 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), SrcVT);
7775 SDValue Mask = DAG.getConstant(
7776 APInt::getOneBitSet(BitWidth, ShiftAmtC->getZExtValue()), DL, SrcVT);
7777 SDValue NewAnd = DAG.getNode(ISD::AND, DL, SrcVT, X, Mask);
7778 SDValue Zero = DAG.getConstant(0, DL, SrcVT);
7779 SDValue Setcc = DAG.getSetCC(DL, CCVT, NewAnd, Zero, ISD::SETEQ);
7780 return DAG.getZExtOrTrunc(Setcc, DL, And->getValueType(0));
7781}
7782
7783/// For targets that support usubsat, match a bit-hack form of that operation
7784/// that ends in 'and' and convert it.
7786 EVT VT = N->getValueType(0);
7787 unsigned BitWidth = VT.getScalarSizeInBits();
7788 APInt SignMask = APInt::getSignMask(BitWidth);
7789
7790 // (i8 X ^ 128) & (i8 X s>> 7) --> usubsat X, 128
7791 // (i8 X + 128) & (i8 X s>> 7) --> usubsat X, 128
7792 // xor/add with SMIN (signmask) are logically equivalent.
7793 SDValue X;
7794 if (!sd_match(N, m_And(m_OneUse(m_Xor(m_Value(X), m_SpecificInt(SignMask))),
7796 m_SpecificInt(BitWidth - 1))))) &&
7799 m_SpecificInt(BitWidth - 1))))))
7800 return SDValue();
7801
7802 return DAG.getNode(ISD::USUBSAT, DL, VT, X,
7803 DAG.getConstant(SignMask, DL, VT));
7804}
7805
7806/// Given a bitwise logic operation N with a matching bitwise logic operand,
7807/// fold a pattern where 2 of the source operands are identically shifted
7808/// values. For example:
7809/// ((X0 << Y) | Z) | (X1 << Y) --> ((X0 | X1) << Y) | Z
7811 SelectionDAG &DAG) {
7812 unsigned LogicOpcode = N->getOpcode();
7813 assert(ISD::isBitwiseLogicOp(LogicOpcode) &&
7814 "Expected bitwise logic operation");
7815
7816 if (!LogicOp.hasOneUse() || !ShiftOp.hasOneUse())
7817 return SDValue();
7818
7819 // Match another bitwise logic op and a shift.
7820 unsigned ShiftOpcode = ShiftOp.getOpcode();
7821 if (LogicOp.getOpcode() != LogicOpcode ||
7822 !(ShiftOpcode == ISD::SHL || ShiftOpcode == ISD::SRL ||
7823 ShiftOpcode == ISD::SRA))
7824 return SDValue();
7825
7826 // Match another shift op inside the first logic operand. Handle both commuted
7827 // possibilities.
7828 // LOGIC (LOGIC (SH X0, Y), Z), (SH X1, Y) --> LOGIC (SH (LOGIC X0, X1), Y), Z
7829 // LOGIC (LOGIC Z, (SH X0, Y)), (SH X1, Y) --> LOGIC (SH (LOGIC X0, X1), Y), Z
7830 SDValue X1 = ShiftOp.getOperand(0);
7831 SDValue Y = ShiftOp.getOperand(1);
7832 SDValue X0, Z;
7833 if (LogicOp.getOperand(0).getOpcode() == ShiftOpcode &&
7834 LogicOp.getOperand(0).getOperand(1) == Y) {
7835 X0 = LogicOp.getOperand(0).getOperand(0);
7836 Z = LogicOp.getOperand(1);
7837 } else if (LogicOp.getOperand(1).getOpcode() == ShiftOpcode &&
7838 LogicOp.getOperand(1).getOperand(1) == Y) {
7839 X0 = LogicOp.getOperand(1).getOperand(0);
7840 Z = LogicOp.getOperand(0);
7841 } else {
7842 return SDValue();
7843 }
7844
7845 EVT VT = N->getValueType(0);
7846 SDLoc DL(N);
7847 SDValue LogicX = DAG.getNode(LogicOpcode, DL, VT, X0, X1);
7848 SDValue NewShift = DAG.getNode(ShiftOpcode, DL, VT, LogicX, Y);
7849 return DAG.getNode(LogicOpcode, DL, VT, NewShift, Z);
7850}
7851
7852/// Given a tree of logic operations with shape like
7853/// (LOGIC (LOGIC (X, Y), LOGIC (Z, Y)))
7854/// try to match and fold shift operations with the same shift amount.
7855/// For example:
7856/// LOGIC (LOGIC (SH X0, Y), Z), (LOGIC (SH X1, Y), W) -->
7857/// --> LOGIC (SH (LOGIC X0, X1), Y), (LOGIC Z, W)
7859 SDValue RightHand, SelectionDAG &DAG) {
7860 unsigned LogicOpcode = N->getOpcode();
7861 assert(ISD::isBitwiseLogicOp(LogicOpcode) &&
7862 "Expected bitwise logic operation");
7863 if (LeftHand.getOpcode() != LogicOpcode ||
7864 RightHand.getOpcode() != LogicOpcode)
7865 return SDValue();
7866 if (!LeftHand.hasOneUse() || !RightHand.hasOneUse())
7867 return SDValue();
7868
7869 // Try to match one of following patterns:
7870 // LOGIC (LOGIC (SH X0, Y), Z), (LOGIC (SH X1, Y), W)
7871 // LOGIC (LOGIC (SH X0, Y), Z), (LOGIC W, (SH X1, Y))
7872 // Note that foldLogicOfShifts will handle commuted versions of the left hand
7873 // itself.
7874 SDValue CombinedShifts, W;
7875 SDValue R0 = RightHand.getOperand(0);
7876 SDValue R1 = RightHand.getOperand(1);
7877 if ((CombinedShifts = foldLogicOfShifts(N, LeftHand, R0, DAG)))
7878 W = R1;
7879 else if ((CombinedShifts = foldLogicOfShifts(N, LeftHand, R1, DAG)))
7880 W = R0;
7881 else
7882 return SDValue();
7883
7884 EVT VT = N->getValueType(0);
7885 SDLoc DL(N);
7886 return DAG.getNode(LogicOpcode, DL, VT, CombinedShifts, W);
7887}
7888
7889/// Fold "masked merge" expressions like `(m & x) | (~m & y)` and its DeMorgan
7890/// variant `(~m | x) & (m | y)` into the equivalent `((x ^ y) & m) ^ y)`
7891/// pattern. This is typically a better representation for targets without a
7892/// fused "and-not" operation.
7894 const TargetLowering &TLI, const SDLoc &DL) {
7895 // Note that masked-merge variants using XOR or ADD expressions are
7896 // normalized to OR by InstCombine so we only check for OR or AND.
7897 assert((Node->getOpcode() == ISD::OR || Node->getOpcode() == ISD::AND) &&
7898 "Must be called with ISD::OR or ISD::AND node");
7899
7900 // If the target supports and-not, don't fold this.
7901 if (TLI.hasAndNot(SDValue(Node, 0)))
7902 return SDValue();
7903
7904 SDValue M, X, Y;
7905
7906 if (sd_match(Node,
7908 m_OneUse(m_And(m_Deferred(M), m_Value(X))))) ||
7909 sd_match(Node,
7911 m_OneUse(m_Or(m_Deferred(M), m_Value(Y)))))) {
7912 EVT VT = M.getValueType();
7913 SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, X, Y);
7914 SDValue And = DAG.getNode(ISD::AND, DL, VT, Xor, M);
7915 return DAG.getNode(ISD::XOR, DL, VT, And, Y);
7916 }
7917 return SDValue();
7918}
7919
7920SDValue DAGCombiner::visitAND(SDNode *N) {
7921 SDValue N0 = N->getOperand(0);
7922 SDValue N1 = N->getOperand(1);
7923 EVT VT = N1.getValueType();
7924 SDLoc DL(N);
7925
7926 // x & x --> x
7927 if (N0 == N1)
7928 return N0;
7929
7930 // fold (and c1, c2) -> c1&c2
7931 if (SDValue C = DAG.FoldConstantArithmetic(ISD::AND, DL, VT, {N0, N1}))
7932 return C;
7933
7934 // canonicalize constant to RHS
7937 return DAG.getNode(ISD::AND, DL, VT, N1, N0);
7938
7939 if (areBitwiseNotOfEachother(N0, N1))
7940 return DAG.getConstant(APInt::getZero(VT.getScalarSizeInBits()), DL, VT);
7941
7942 // fold vector ops
7943 if (VT.isVector()) {
7944 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
7945 return FoldedVOp;
7946
7947 // fold (and x, 0) -> 0, vector edition
7949 // do not return N1, because undef node may exist in N1
7951 N1.getValueType());
7952
7953 // fold (and x, -1) -> x, vector edition
7955 return N0;
7956
7957 // fold (and buildvector(x,0,-1,w), buildvector(0,y,z,w))
7958 // --> buildvector(0,0,z,w)
7959 auto *BV0 = dyn_cast<BuildVectorSDNode>(N0);
7960 auto *BV1 = dyn_cast<BuildVectorSDNode>(N1);
7961 if (BV0 && BV1 && !BV0->getSplatValue() && !BV1->getSplatValue() &&
7962 N0.hasOneUse() && N1.hasOneUse() &&
7963 BV0->getOperand(0).getValueType() ==
7964 BV1->getOperand(0).getValueType()) {
7965 SmallVector<SDValue> MergedOps;
7966 unsigned NumElts = VT.getVectorNumElements();
7967 EVT EltVT = BV0->getOperand(0).getValueType();
7968 for (unsigned I = 0; I != NumElts; ++I) {
7969 auto *C0 = dyn_cast<ConstantSDNode>(BV0->getOperand(I));
7970 auto *C1 = dyn_cast<ConstantSDNode>(BV1->getOperand(I));
7971 if (C0 && C1)
7972 MergedOps.push_back(DAG.getConstant(
7973 C0->getAPIntValue() & C1->getAPIntValue(), DL, EltVT));
7974 else if (C0 && C0->isZero())
7975 MergedOps.push_back(BV0->getOperand(I));
7976 else if (C1 && C1->isZero())
7977 MergedOps.push_back(BV1->getOperand(I));
7978 else if (C0 && C0->isAllOnes())
7979 MergedOps.push_back(BV1->getOperand(I));
7980 else if (C1 && C1->isAllOnes())
7981 MergedOps.push_back(BV0->getOperand(I));
7982 else if (BV0->getOperand(I) == BV1->getOperand(I))
7983 MergedOps.push_back(BV0->getOperand(I));
7984 else
7985 break;
7986 }
7987 if (MergedOps.size() == NumElts)
7988 return DAG.getBuildVector(VT, DL, MergedOps);
7989 }
7990
7991 // fold (and (masked_load) (splat_vec (x, ...))) to zext_masked_load
7992 bool Frozen = N0.getOpcode() == ISD::FREEZE;
7993 auto *MLoad = dyn_cast<MaskedLoadSDNode>(Frozen ? N0.getOperand(0) : N0);
7994 ConstantSDNode *Splat = isConstOrConstSplat(N1, true, true);
7995 if (MLoad && MLoad->getExtensionType() == ISD::EXTLOAD && Splat) {
7996 EVT MemVT = MLoad->getMemoryVT();
7997 if (TLI.isLoadLegal(VT, MemVT, MLoad->getAlign(),
7998 MLoad->getAddressSpace(), ISD::ZEXTLOAD, false)) {
7999 // For this AND to be a zero extension of the masked load the elements
8000 // of the BuildVec must mask the bottom bits of the extended element
8001 // type
8002 if (Splat->getAPIntValue().isMask(MemVT.getScalarSizeInBits())) {
8003 SDValue NewLoad = DAG.getMaskedLoad(
8004 VT, DL, MLoad->getChain(), MLoad->getBasePtr(),
8005 MLoad->getOffset(), MLoad->getMask(), MLoad->getPassThru(), MemVT,
8006 MLoad->getMemOperand(), MLoad->getAddressingMode(), ISD::ZEXTLOAD,
8007 MLoad->isExpandingLoad());
8008 CombineTo(N, Frozen ? N0 : NewLoad);
8009 CombineTo(MLoad, NewLoad, NewLoad.getValue(1));
8010 return SDValue(N, 0);
8011 }
8012 }
8013 }
8014 }
8015
8016 // fold (and x, -1) -> x
8017 if (isAllOnesConstant(N1))
8018 return N0;
8019
8020 // if (and x, c) is known to be zero, return 0
8021 unsigned BitWidth = VT.getScalarSizeInBits();
8022 ConstantSDNode *N1C = isConstOrConstSplat(N1);
8024 return DAG.getConstant(0, DL, VT);
8025
8026 if (SDValue R = foldAndOrOfSETCC(N, DAG))
8027 return R;
8028
8029 if (SDValue NewSel = foldBinOpIntoSelect(N))
8030 return NewSel;
8031
8032 // reassociate and
8033 if (SDValue RAND = reassociateOps(ISD::AND, DL, N0, N1, N->getFlags()))
8034 return RAND;
8035
8036 // Fold and(vecreduce(x), vecreduce(y)) -> vecreduce(and(x, y))
8037 if (SDValue SD =
8038 reassociateReduction(ISD::VECREDUCE_AND, ISD::AND, DL, VT, N0, N1))
8039 return SD;
8040
8041 // fold (and (or x, C), D) -> D if (C & D) == D
8042 auto MatchSubset = [](ConstantSDNode *LHS, ConstantSDNode *RHS) {
8043 return RHS->getAPIntValue().isSubsetOf(LHS->getAPIntValue());
8044 };
8045 if (N0.getOpcode() == ISD::OR &&
8046 ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchSubset))
8047 return N1;
8048
8049 if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) {
8050 SDValue N0Op0 = N0.getOperand(0);
8051 EVT SrcVT = N0Op0.getValueType();
8052 unsigned SrcBitWidth = SrcVT.getScalarSizeInBits();
8053 APInt Mask = ~N1C->getAPIntValue();
8054 Mask = Mask.trunc(SrcBitWidth);
8055
8056 // fold (and (any_ext V), c) -> (zero_ext V) if 'and' only clears top bits.
8057 if (DAG.MaskedValueIsZero(N0Op0, Mask))
8058 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0Op0);
8059
8060 // fold (and (any_ext V), c) -> (zero_ext (and V, c)) if profitable, when
8061 // the zext is free or the anyext costs the same as the zext.
8062 if (N1C->getAPIntValue().countLeadingZeros() >= (BitWidth - SrcBitWidth) &&
8063 (TLI.isZExtFree(SrcVT, VT) || !TLI.isAnyExtFree(SrcVT, VT)) &&
8065 TLI.isNarrowingProfitable(N, VT, SrcVT))
8066 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT,
8067 DAG.getNode(ISD::AND, DL, SrcVT, N0Op0,
8068 DAG.getZExtOrTrunc(N1, DL, SrcVT)));
8069 }
8070
8071 // fold (and (ext (and V, c1)), c2) -> (and (ext V), (and c1, (ext c2)))
8072 if (ISD::isExtOpcode(N0.getOpcode())) {
8073 unsigned ExtOpc = N0.getOpcode();
8074 SDValue N0Op0 = N0.getOperand(0);
8075 if (N0Op0.getOpcode() == ISD::AND &&
8076 (ExtOpc != ISD::ZERO_EXTEND || !TLI.isZExtFree(N0Op0, VT)) &&
8077 N0->hasOneUse() && N0Op0->hasOneUse()) {
8078 if (SDValue NewExt = DAG.FoldConstantArithmetic(ExtOpc, DL, VT,
8079 {N0Op0.getOperand(1)})) {
8080 if (SDValue NewMask =
8081 DAG.FoldConstantArithmetic(ISD::AND, DL, VT, {N1, NewExt})) {
8082 return DAG.getNode(ISD::AND, DL, VT,
8083 DAG.getNode(ExtOpc, DL, VT, N0Op0.getOperand(0)),
8084 NewMask);
8085 }
8086 }
8087 }
8088 }
8089
8090 // similarly fold (and (X (load ([non_ext|any_ext|zero_ext] V))), c) ->
8091 // (X (load ([non_ext|zero_ext] V))) if 'and' only clears top bits which must
8092 // already be zero by virtue of the width of the base type of the load.
8093 //
8094 // the 'X' node here can either be nothing or an extract_vector_elt to catch
8095 // more cases.
8096 if ((N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
8098 N0.getOperand(0).getOpcode() == ISD::LOAD &&
8099 N0.getOperand(0).getResNo() == 0) ||
8100 (N0.getOpcode() == ISD::LOAD && N0.getResNo() == 0)) {
8101 auto *Load =
8102 cast<LoadSDNode>((N0.getOpcode() == ISD::LOAD) ? N0 : N0.getOperand(0));
8103
8104 // Get the constant (if applicable) the zero'th operand is being ANDed with.
8105 // This can be a pure constant or a vector splat, in which case we treat the
8106 // vector as a scalar and use the splat value.
8107 APInt Constant = APInt::getZero(1);
8108 if (const ConstantSDNode *C = isConstOrConstSplat(
8109 N1, /*AllowUndefs=*/false, /*AllowTruncation=*/true)) {
8110 Constant = C->getAPIntValue();
8111 } else if (BuildVectorSDNode *Vector = dyn_cast<BuildVectorSDNode>(N1)) {
8112 unsigned EltBitWidth = Vector->getValueType(0).getScalarSizeInBits();
8113 APInt SplatValue, SplatUndef;
8114 unsigned SplatBitSize;
8115 bool HasAnyUndefs;
8116 // Endianness should not matter here. Code below makes sure that we only
8117 // use the result if the SplatBitSize is a multiple of the vector element
8118 // size. And after that we AND all element sized parts of the splat
8119 // together. So the end result should be the same regardless of in which
8120 // order we do those operations.
8121 const bool IsBigEndian = false;
8122 bool IsSplat =
8123 Vector->isConstantSplat(SplatValue, SplatUndef, SplatBitSize,
8124 HasAnyUndefs, EltBitWidth, IsBigEndian);
8125
8126 // Make sure that variable 'Constant' is only set if 'SplatBitSize' is a
8127 // multiple of 'BitWidth'. Otherwise, we could propagate a wrong value.
8128 if (IsSplat && (SplatBitSize % EltBitWidth) == 0) {
8129 // Undef bits can contribute to a possible optimisation if set, so
8130 // set them.
8131 SplatValue |= SplatUndef;
8132
8133 // The splat value may be something like "0x00FFFFFF", which means 0 for
8134 // the first vector value and FF for the rest, repeating. We need a mask
8135 // that will apply equally to all members of the vector, so AND all the
8136 // lanes of the constant together.
8137 Constant = APInt::getAllOnes(EltBitWidth);
8138 for (unsigned i = 0, n = (SplatBitSize / EltBitWidth); i < n; ++i)
8139 Constant &= SplatValue.extractBits(EltBitWidth, i * EltBitWidth);
8140 }
8141 }
8142
8143 // If we want to change an EXTLOAD to a ZEXTLOAD, ensure a ZEXTLOAD is
8144 // actually legal and isn't going to get expanded, else this is a false
8145 // optimisation.
8146 bool CanZextLoadProfitably = TLI.isLoadLegal(
8147 Load->getValueType(0), Load->getMemoryVT(), Load->getAlign(),
8148 Load->getAddressSpace(), ISD::ZEXTLOAD, false);
8149
8150 // Resize the constant to the same size as the original memory access before
8151 // extension. If it is still the AllOnesValue then this AND is completely
8152 // unneeded.
8153 Constant = Constant.zextOrTrunc(Load->getMemoryVT().getScalarSizeInBits());
8154
8155 bool B;
8156 switch (Load->getExtensionType()) {
8157 default: B = false; break;
8158 case ISD::EXTLOAD: B = CanZextLoadProfitably; break;
8159 case ISD::ZEXTLOAD:
8160 case ISD::NON_EXTLOAD: B = true; break;
8161 }
8162
8163 if (B && Constant.isAllOnes()) {
8164 // If the load type was an EXTLOAD, convert to ZEXTLOAD in order to
8165 // preserve semantics once we get rid of the AND.
8166 SDValue NewLoad(Load, 0);
8167
8168 // Fold the AND away. NewLoad may get replaced immediately.
8169 CombineTo(N, (N0.getNode() == Load) ? NewLoad : N0);
8170
8171 if (Load->getExtensionType() == ISD::EXTLOAD) {
8172 NewLoad = DAG.getLoad(Load->getAddressingMode(), ISD::ZEXTLOAD,
8173 Load->getValueType(0), SDLoc(Load),
8174 Load->getChain(), Load->getBasePtr(),
8175 Load->getOffset(), Load->getMemoryVT(),
8176 Load->getMemOperand());
8177 // Replace uses of the EXTLOAD with the new ZEXTLOAD.
8178 if (Load->getNumValues() == 3) {
8179 // PRE/POST_INC loads have 3 values.
8180 SDValue To[] = { NewLoad.getValue(0), NewLoad.getValue(1),
8181 NewLoad.getValue(2) };
8182 CombineTo(Load, To, 3, true);
8183 } else {
8184 CombineTo(Load, NewLoad.getValue(0), NewLoad.getValue(1));
8185 }
8186 }
8187
8188 return SDValue(N, 0); // Return N so it doesn't get rechecked!
8189 }
8190 }
8191
8192 // Try to convert a constant mask AND into a shuffle clear mask.
8193 if (VT.isVector())
8194 if (SDValue Shuffle = XformToShuffleWithZero(N))
8195 return Shuffle;
8196
8197 if (SDValue Combined = combineCarryDiamond(DAG, TLI, N0, N1, N))
8198 return Combined;
8199
8200 if (N0.getOpcode() == ISD::EXTRACT_SUBVECTOR && N0.hasOneUse() && N1C &&
8202 SDValue Ext = N0.getOperand(0);
8203 EVT ExtVT = Ext->getValueType(0);
8204 SDValue Extendee = Ext->getOperand(0);
8205
8206 unsigned ScalarWidth = Extendee.getValueType().getScalarSizeInBits();
8207 if (N1C->getAPIntValue().isMask(ScalarWidth) &&
8208 (!LegalOperations || TLI.isOperationLegal(ISD::ZERO_EXTEND, ExtVT))) {
8209 // (and (extract_subvector (zext|anyext|sext v) _) iN_mask)
8210 // => (extract_subvector (iN_zeroext v))
8211 SDValue ZeroExtExtendee =
8212 DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVT, Extendee);
8213
8214 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, ZeroExtExtendee,
8215 N0.getOperand(1));
8216 }
8217 }
8218
8219 // fold (and (masked_gather x)) -> (zext_masked_gather x)
8220 if (auto *GN0 = dyn_cast<MaskedGatherSDNode>(N0)) {
8221 EVT MemVT = GN0->getMemoryVT();
8222 EVT ScalarVT = MemVT.getScalarType();
8223
8224 if (SDValue(GN0, 0).hasOneUse() &&
8225 isConstantSplatVectorMaskForType(N1.getNode(), ScalarVT) &&
8227 SDValue Ops[] = {GN0->getChain(), GN0->getPassThru(), GN0->getMask(),
8228 GN0->getBasePtr(), GN0->getIndex(), GN0->getScale()};
8229
8230 SDValue ZExtLoad = DAG.getMaskedGather(
8231 DAG.getVTList(VT, MVT::Other), MemVT, DL, Ops, GN0->getMemOperand(),
8232 GN0->getIndexType(), ISD::ZEXTLOAD);
8233
8234 CombineTo(N, ZExtLoad);
8235 AddToWorklist(ZExtLoad.getNode());
8236 // Avoid recheck of N.
8237 return SDValue(N, 0);
8238 }
8239 }
8240
8241 // fold (and (load x), 255) -> (zextload x, i8)
8242 // fold (and (extload x, i16), 255) -> (zextload x, i8)
8243 // fold (and (freeze (load x)), 255) -> (freeze (zextload x, i8))
8244 // fold (and (freeze (extload x, i16)), 255) -> (freeze (zextload x, i8))
8245 if (N1C && !VT.isVector()) {
8246 SDValue Inner = peekThroughFreeze(N0);
8247 if (Inner.getOpcode() == ISD::LOAD)
8248 if (SDValue Res = reduceLoadWidth(N))
8249 return Res;
8250 }
8251
8252 if (LegalTypes) {
8253 // Attempt to propagate the AND back up to the leaves which, if they're
8254 // loads, can be combined to narrow loads and the AND node can be removed.
8255 // Perform after legalization so that extend nodes will already be
8256 // combined into the loads.
8257 if (BackwardsPropagateMask(N))
8258 return SDValue(N, 0);
8259 }
8260
8261 if (SDValue Combined = visitANDLike(N0, N1, N))
8262 return Combined;
8263
8264 // Simplify: (and (op x...), (op y...)) -> (op (and x, y))
8265 if (N0.getOpcode() == N1.getOpcode())
8266 if (SDValue V = hoistLogicOpWithSameOpcodeHands(N))
8267 return V;
8268
8269 if (SDValue R = foldLogicOfShifts(N, N0, N1, DAG))
8270 return R;
8271 if (SDValue R = foldLogicOfShifts(N, N1, N0, DAG))
8272 return R;
8273
8274 // Fold (and X, (bswap (not Y))) -> (and X, (not (bswap Y)))
8275 // Fold (and X, (bitreverse (not Y))) -> (and X, (not (bitreverse Y)))
8276 SDValue X, Y, Z, NotY;
8277 for (unsigned Opc : {ISD::BSWAP, ISD::BITREVERSE})
8278 if (sd_match(N,
8279 m_And(m_Value(X), m_OneUse(m_UnaryOp(Opc, m_Value(NotY))))) &&
8280 sd_match(NotY, m_Not(m_Value(Y))) &&
8281 (TLI.hasAndNot(SDValue(N, 0)) || NotY->hasOneUse()))
8282 return DAG.getNode(ISD::AND, DL, VT, X,
8283 DAG.getNOT(DL, DAG.getNode(Opc, DL, VT, Y), VT));
8284
8285 // Fold (and X, (rot (not Y), Z)) -> (and X, (not (rot Y, Z)))
8286 for (unsigned Opc : {ISD::ROTL, ISD::ROTR})
8287 if (sd_match(N, m_And(m_Value(X),
8288 m_OneUse(m_BinOp(Opc, m_Value(NotY), m_Value(Z))))) &&
8289 sd_match(NotY, m_Not(m_Value(Y))) &&
8290 (TLI.hasAndNot(SDValue(N, 0)) || NotY->hasOneUse()))
8291 return DAG.getNode(ISD::AND, DL, VT, X,
8292 DAG.getNOT(DL, DAG.getNode(Opc, DL, VT, Y, Z), VT));
8293
8294 // Fold (and X, (add (not Y), Z)) -> (and X, (not (sub Y, Z)))
8295 // Fold (and X, (sub (not Y), Z)) -> (and X, (not (add Y, Z)))
8296 if (TLI.hasAndNot(SDValue(N, 0)))
8297 if (SDValue Folded = foldBitwiseOpWithNeg(N, DL, VT))
8298 return Folded;
8299
8300 // Fold (and (srl X, C), 1) -> (srl X, BW-1) for signbit extraction
8301 // If we are shifting down an extended sign bit, see if we can simplify
8302 // this to shifting the MSB directly to expose further simplifications.
8303 // This pattern often appears after sext_inreg legalization.
8304 APInt Amt;
8305 if (sd_match(N, m_And(m_Srl(m_Value(X), m_ConstInt(Amt)), m_One())) &&
8306 Amt.ult(BitWidth - 1) && Amt.uge(BitWidth - DAG.ComputeNumSignBits(X)))
8307 return DAG.getNode(ISD::SRL, DL, VT, X,
8308 DAG.getShiftAmountConstant(BitWidth - 1, VT, DL));
8309
8310 // Masking the negated extension of a boolean is just the zero-extended
8311 // boolean:
8312 // and (sub 0, zext(bool X)), 1 --> zext(bool X)
8313 // and (sub 0, sext(bool X)), 1 --> zext(bool X)
8314 //
8315 // Note: the SimplifyDemandedBits fold below can make an information-losing
8316 // transform, and then we have no way to find this better fold.
8317 if (sd_match(N, m_And(m_Sub(m_Zero(), m_Value(X)), m_One()))) {
8318 if (X.getOpcode() == ISD::ZERO_EXTEND &&
8319 X.getOperand(0).getScalarValueSizeInBits() == 1)
8320 return X;
8321 if (X.getOpcode() == ISD::SIGN_EXTEND &&
8322 X.getOperand(0).getScalarValueSizeInBits() == 1)
8323 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, X.getOperand(0));
8324 }
8325
8326 // fold (and (sign_extend_inreg x, i16 to i32), 1) -> (and x, 1)
8327 // fold (and (sra)) -> (and (srl)) when possible.
8329 return SDValue(N, 0);
8330
8331 // fold (zext_inreg (extload x)) -> (zextload x)
8332 // fold (zext_inreg (sextload x)) -> (zextload x) iff load has one use
8333 if (ISD::isUNINDEXEDLoad(N0.getNode()) &&
8334 (ISD::isEXTLoad(N0.getNode()) ||
8335 (ISD::isSEXTLoad(N0.getNode()) && N0.hasOneUse()))) {
8336 auto *LN0 = cast<LoadSDNode>(N0);
8337 EVT MemVT = LN0->getMemoryVT();
8338 // If we zero all the possible extended bits, then we can turn this into
8339 // a zextload if we are running before legalize or the operation is legal.
8340 unsigned ExtBitSize = N1.getScalarValueSizeInBits();
8341 unsigned MemBitSize = MemVT.getScalarSizeInBits();
8342 APInt ExtBits = APInt::getHighBitsSet(ExtBitSize, ExtBitSize - MemBitSize);
8343 if (DAG.MaskedValueIsZero(N1, ExtBits) &&
8344 ((!LegalOperations && LN0->isSimple()) ||
8345 TLI.isLoadLegal(VT, MemVT, LN0->getAlign(), LN0->getAddressSpace(),
8346 ISD::ZEXTLOAD, false))) {
8347 SDValue ExtLoad =
8348 DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT, LN0->getChain(),
8349 LN0->getBasePtr(), MemVT, LN0->getMemOperand());
8350 AddToWorklist(N);
8351 CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
8352 return SDValue(N, 0); // Return N so it doesn't get rechecked!
8353 }
8354 }
8355
8356 // fold (and (or (srl N, 8), (shl N, 8)), 0xffff) -> (srl (bswap N), const)
8357 if (N1C && N1C->getAPIntValue() == 0xffff && N0.getOpcode() == ISD::OR) {
8358 if (SDValue BSwap = MatchBSwapHWordLow(N0.getNode(), N0.getOperand(0),
8359 N0.getOperand(1), false))
8360 return BSwap;
8361 }
8362
8363 if (SDValue Shifts = unfoldExtremeBitClearingToShifts(N))
8364 return Shifts;
8365
8367 return V;
8368
8369 // Recognize the following pattern:
8370 //
8371 // AndVT = (and (sign_extend NarrowVT to AndVT) #bitmask)
8372 //
8373 // where bitmask is a mask that clears the upper bits of AndVT. The
8374 // number of bits in bitmask must be a power of two.
8375 auto IsAndZeroExtMask = [](SDValue LHS, SDValue RHS) {
8376 if (LHS->getOpcode() != ISD::SIGN_EXTEND)
8377 return false;
8378
8379 auto *C = isConstOrConstSplat(RHS, false, true);
8380 if (!C)
8381 return false;
8382
8383 if (!C->getAPIntValue().isMask(
8384 LHS.getOperand(0).getValueType().getScalarSizeInBits()))
8385 return false;
8386
8387 return true;
8388 };
8389
8390 // Replace (and (sign_extend ...) #bitmask) with (zero_extend ...).
8391 if (IsAndZeroExtMask(N0, N1) &&
8392 (!LegalOperations || TLI.isOperationLegal(ISD::ZERO_EXTEND, VT)))
8393 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0));
8394
8395 if (hasOperation(ISD::USUBSAT, VT))
8396 if (SDValue V = foldAndToUsubsat(N, DAG, DL))
8397 return V;
8398
8399 // Postpone until legalization completed to avoid interference with bswap
8400 // folding
8401 if (LegalOperations || VT.isVector())
8402 if (SDValue R = foldLogicTreeOfShifts(N, N0, N1, DAG))
8403 return R;
8404
8405 if (VT.isScalarInteger() && VT != MVT::i1)
8406 if (SDValue R = foldMaskedMerge(N, DAG, TLI, DL))
8407 return R;
8408
8409 return SDValue();
8410}
8411
8412/// Match (a >> 8) | (a << 8) as (bswap a) >> 16.
8413SDValue DAGCombiner::MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
8414 bool DemandHighBits) {
8415 if (!LegalOperations)
8416 return SDValue();
8417
8418 EVT VT = N->getValueType(0);
8419 if (VT != MVT::i64 && VT != MVT::i32 && VT != MVT::i16)
8420 return SDValue();
8422 return SDValue();
8423
8424 // Recognize (and (shl a, 8), 0xff00), (and (srl a, 8), 0xff)
8425 bool LookPassAnd0 = false;
8426 bool LookPassAnd1 = false;
8427 if (N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::SRL)
8428 std::swap(N0, N1);
8429 if (N1.getOpcode() == ISD::AND && N1.getOperand(0).getOpcode() == ISD::SHL)
8430 std::swap(N0, N1);
8431 if (N0.getOpcode() == ISD::AND) {
8432 if (!N0->hasOneUse())
8433 return SDValue();
8434 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
8435 // Also handle 0xffff since the LHS is guaranteed to have zeros there.
8436 // This is needed for X86.
8437 if (!N01C || (N01C->getZExtValue() != 0xFF00 &&
8438 N01C->getZExtValue() != 0xFFFF))
8439 return SDValue();
8440 N0 = N0.getOperand(0);
8441 LookPassAnd0 = true;
8442 }
8443
8444 if (N1.getOpcode() == ISD::AND) {
8445 if (!N1->hasOneUse())
8446 return SDValue();
8447 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
8448 if (!N11C || N11C->getZExtValue() != 0xFF)
8449 return SDValue();
8450 N1 = N1.getOperand(0);
8451 LookPassAnd1 = true;
8452 }
8453
8454 if (N0.getOpcode() == ISD::SRL && N1.getOpcode() == ISD::SHL)
8455 std::swap(N0, N1);
8456 if (N0.getOpcode() != ISD::SHL || N1.getOpcode() != ISD::SRL)
8457 return SDValue();
8458 if (!N0->hasOneUse() || !N1->hasOneUse())
8459 return SDValue();
8460
8461 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
8462 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
8463 if (!N01C || !N11C)
8464 return SDValue();
8465 if (N01C->getZExtValue() != 8 || N11C->getZExtValue() != 8)
8466 return SDValue();
8467
8468 // Look for (shl (and a, 0xff), 8), (srl (and a, 0xff00), 8)
8469 SDValue N00 = N0->getOperand(0);
8470 if (!LookPassAnd0 && N00.getOpcode() == ISD::AND) {
8471 if (!N00->hasOneUse())
8472 return SDValue();
8473 ConstantSDNode *N001C = dyn_cast<ConstantSDNode>(N00.getOperand(1));
8474 if (!N001C || N001C->getZExtValue() != 0xFF)
8475 return SDValue();
8476 N00 = N00.getOperand(0);
8477 LookPassAnd0 = true;
8478 }
8479
8480 SDValue N10 = N1->getOperand(0);
8481 if (!LookPassAnd1 && N10.getOpcode() == ISD::AND) {
8482 if (!N10->hasOneUse())
8483 return SDValue();
8484 ConstantSDNode *N101C = dyn_cast<ConstantSDNode>(N10.getOperand(1));
8485 // Also allow 0xFFFF since the bits will be shifted out. This is needed
8486 // for X86.
8487 if (!N101C || (N101C->getZExtValue() != 0xFF00 &&
8488 N101C->getZExtValue() != 0xFFFF))
8489 return SDValue();
8490 N10 = N10.getOperand(0);
8491 LookPassAnd1 = true;
8492 }
8493
8494 if (N00 != N10)
8495 return SDValue();
8496
8497 // Make sure everything beyond the low halfword gets set to zero since the SRL
8498 // 16 will clear the top bits.
8499 unsigned OpSizeInBits = VT.getSizeInBits();
8500 if (OpSizeInBits > 16) {
8501 // If the left-shift isn't masked out then the only way this is a bswap is
8502 // if all bits beyond the low 8 are 0. In that case the entire pattern
8503 // reduces to a left shift anyway: leave it for other parts of the combiner.
8504 if (DemandHighBits && !LookPassAnd0)
8505 return SDValue();
8506
8507 // However, if the right shift isn't masked out then it might be because
8508 // it's not needed. See if we can spot that too. If the high bits aren't
8509 // demanded, we only need bits 23:16 to be zero. Otherwise, we need all
8510 // upper bits to be zero.
8511 if (!LookPassAnd1) {
8512 unsigned HighBit = DemandHighBits ? OpSizeInBits : 24;
8513 if (!DAG.MaskedValueIsZero(N10,
8514 APInt::getBitsSet(OpSizeInBits, 16, HighBit)))
8515 return SDValue();
8516 }
8517 }
8518
8519 SDValue Res = DAG.getNode(ISD::BSWAP, SDLoc(N), VT, N00);
8520 if (OpSizeInBits > 16) {
8521 SDLoc DL(N);
8522 Res = DAG.getNode(ISD::SRL, DL, VT, Res,
8523 DAG.getShiftAmountConstant(OpSizeInBits - 16, VT, DL));
8524 }
8525 return Res;
8526}
8527
8528/// Return true if the specified node is an element that makes up a 32-bit
8529/// packed halfword byteswap.
8530/// ((x & 0x000000ff) << 8) |
8531/// ((x & 0x0000ff00) >> 8) |
8532/// ((x & 0x00ff0000) << 8) |
8533/// ((x & 0xff000000) >> 8)
8535 if (!N->hasOneUse())
8536 return false;
8537
8538 unsigned Opc = N.getOpcode();
8539 if (Opc != ISD::AND && Opc != ISD::SHL && Opc != ISD::SRL)
8540 return false;
8541
8542 SDValue N0 = N.getOperand(0);
8543 unsigned Opc0 = N0.getOpcode();
8544 if (Opc0 != ISD::AND && Opc0 != ISD::SHL && Opc0 != ISD::SRL)
8545 return false;
8546
8547 ConstantSDNode *N1C = nullptr;
8548 // SHL or SRL: look upstream for AND mask operand
8549 if (Opc == ISD::AND)
8550 N1C = dyn_cast<ConstantSDNode>(N.getOperand(1));
8551 else if (Opc0 == ISD::AND)
8553 if (!N1C)
8554 return false;
8555
8556 unsigned MaskByteOffset;
8557 switch (N1C->getZExtValue()) {
8558 default:
8559 return false;
8560 case 0xFF: MaskByteOffset = 0; break;
8561 case 0xFF00: MaskByteOffset = 1; break;
8562 case 0xFFFF:
8563 // In case demanded bits didn't clear the bits that will be shifted out.
8564 // This is needed for X86.
8565 if (Opc == ISD::SRL || (Opc == ISD::AND && Opc0 == ISD::SHL)) {
8566 MaskByteOffset = 1;
8567 break;
8568 }
8569 return false;
8570 case 0xFF0000: MaskByteOffset = 2; break;
8571 case 0xFF000000: MaskByteOffset = 3; break;
8572 }
8573
8574 // Look for (x & 0xff) << 8 as well as ((x << 8) & 0xff00).
8575 if (Opc == ISD::AND) {
8576 if (MaskByteOffset == 0 || MaskByteOffset == 2) {
8577 // (x >> 8) & 0xff
8578 // (x >> 8) & 0xff0000
8579 if (Opc0 != ISD::SRL)
8580 return false;
8582 if (!C || C->getZExtValue() != 8)
8583 return false;
8584 } else {
8585 // (x << 8) & 0xff00
8586 // (x << 8) & 0xff000000
8587 if (Opc0 != ISD::SHL)
8588 return false;
8590 if (!C || C->getZExtValue() != 8)
8591 return false;
8592 }
8593 } else if (Opc == ISD::SHL) {
8594 // (x & 0xff) << 8
8595 // (x & 0xff0000) << 8
8596 if (MaskByteOffset != 0 && MaskByteOffset != 2)
8597 return false;
8598 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
8599 if (!C || C->getZExtValue() != 8)
8600 return false;
8601 } else { // Opc == ISD::SRL
8602 // (x & 0xff00) >> 8
8603 // (x & 0xff000000) >> 8
8604 if (MaskByteOffset != 1 && MaskByteOffset != 3)
8605 return false;
8606 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
8607 if (!C || C->getZExtValue() != 8)
8608 return false;
8609 }
8610
8611 if (Parts[MaskByteOffset])
8612 return false;
8613
8614 Parts[MaskByteOffset] = N0.getOperand(0).getNode();
8615 return true;
8616}
8617
8618// Match 2 elements of a packed halfword bswap.
8620 if (N.getOpcode() == ISD::OR)
8621 return isBSwapHWordElement(N.getOperand(0), Parts) &&
8622 isBSwapHWordElement(N.getOperand(1), Parts);
8623
8624 if (N.getOpcode() == ISD::SRL && N.getOperand(0).getOpcode() == ISD::BSWAP) {
8625 ConstantSDNode *C = isConstOrConstSplat(N.getOperand(1));
8626 if (!C || C->getAPIntValue() != 16)
8627 return false;
8628 Parts[0] = Parts[1] = N.getOperand(0).getOperand(0).getNode();
8629 return true;
8630 }
8631
8632 return false;
8633}
8634
8635// Match this pattern:
8636// (or (and (shl (A, 8)), 0xff00ff00), (and (srl (A, 8)), 0x00ff00ff))
8637// And rewrite this to:
8638// (rotr (bswap A), 16)
8640 SelectionDAG &DAG, SDNode *N, SDValue N0,
8641 SDValue N1, EVT VT) {
8642 assert(N->getOpcode() == ISD::OR && VT == MVT::i32 &&
8643 "MatchBSwapHWordOrAndAnd: expecting i32");
8644 if (!TLI.isOperationLegalOrCustom(ISD::ROTR, VT))
8645 return SDValue();
8646 if (N0.getOpcode() != ISD::AND || N1.getOpcode() != ISD::AND)
8647 return SDValue();
8648 // TODO: this is too restrictive; lifting this restriction requires more tests
8649 if (!N0->hasOneUse() || !N1->hasOneUse())
8650 return SDValue();
8653 if (!Mask0 || !Mask1)
8654 return SDValue();
8655 if (Mask0->getAPIntValue() != 0xff00ff00 ||
8656 Mask1->getAPIntValue() != 0x00ff00ff)
8657 return SDValue();
8658 SDValue Shift0 = N0.getOperand(0);
8659 SDValue Shift1 = N1.getOperand(0);
8660 if (Shift0.getOpcode() != ISD::SHL || Shift1.getOpcode() != ISD::SRL)
8661 return SDValue();
8662 ConstantSDNode *ShiftAmt0 = isConstOrConstSplat(Shift0.getOperand(1));
8663 ConstantSDNode *ShiftAmt1 = isConstOrConstSplat(Shift1.getOperand(1));
8664 if (!ShiftAmt0 || !ShiftAmt1)
8665 return SDValue();
8666 if (ShiftAmt0->getAPIntValue() != 8 || ShiftAmt1->getAPIntValue() != 8)
8667 return SDValue();
8668 if (Shift0.getOperand(0) != Shift1.getOperand(0))
8669 return SDValue();
8670
8671 SDLoc DL(N);
8672 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT, Shift0.getOperand(0));
8673 SDValue ShAmt = DAG.getShiftAmountConstant(16, VT, DL);
8674 return DAG.getNode(ISD::ROTR, DL, VT, BSwap, ShAmt);
8675}
8676
8677/// Match a 32-bit packed halfword bswap. That is
8678/// ((x & 0x000000ff) << 8) |
8679/// ((x & 0x0000ff00) >> 8) |
8680/// ((x & 0x00ff0000) << 8) |
8681/// ((x & 0xff000000) >> 8)
8682/// => (rotl (bswap x), 16)
8683SDValue DAGCombiner::MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1) {
8684 if (!LegalOperations)
8685 return SDValue();
8686
8687 EVT VT = N->getValueType(0);
8688 if (VT != MVT::i32)
8689 return SDValue();
8691 return SDValue();
8692
8693 if (SDValue BSwap = matchBSwapHWordOrAndAnd(TLI, DAG, N, N0, N1, VT))
8694 return BSwap;
8695
8696 // Try again with commuted operands.
8697 if (SDValue BSwap = matchBSwapHWordOrAndAnd(TLI, DAG, N, N1, N0, VT))
8698 return BSwap;
8699
8700
8701 // Look for either
8702 // (or (bswaphpair), (bswaphpair))
8703 // (or (or (bswaphpair), (and)), (and))
8704 // (or (or (and), (bswaphpair)), (and))
8705 SDNode *Parts[4] = {};
8706
8707 if (isBSwapHWordPair(N0, Parts)) {
8708 // (or (or (and), (and)), (or (and), (and)))
8709 if (!isBSwapHWordPair(N1, Parts))
8710 return SDValue();
8711 } else if (N0.getOpcode() == ISD::OR) {
8712 // (or (or (or (and), (and)), (and)), (and))
8713 if (!isBSwapHWordElement(N1, Parts))
8714 return SDValue();
8715 SDValue N00 = N0.getOperand(0);
8716 SDValue N01 = N0.getOperand(1);
8717 if (!(isBSwapHWordElement(N01, Parts) && isBSwapHWordPair(N00, Parts)) &&
8718 !(isBSwapHWordElement(N00, Parts) && isBSwapHWordPair(N01, Parts)))
8719 return SDValue();
8720 } else {
8721 return SDValue();
8722 }
8723
8724 // Make sure the parts are all coming from the same node.
8725 if (Parts[0] != Parts[1] || Parts[0] != Parts[2] || Parts[0] != Parts[3])
8726 return SDValue();
8727
8728 SDLoc DL(N);
8729 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT,
8730 SDValue(Parts[0], 0));
8731
8732 // Result of the bswap should be rotated by 16. If it's not legal, then
8733 // do (x << 16) | (x >> 16).
8734 SDValue ShAmt = DAG.getShiftAmountConstant(16, VT, DL);
8736 return DAG.getNode(ISD::ROTL, DL, VT, BSwap, ShAmt);
8738 return DAG.getNode(ISD::ROTR, DL, VT, BSwap, ShAmt);
8739 return DAG.getNode(ISD::OR, DL, VT,
8740 DAG.getNode(ISD::SHL, DL, VT, BSwap, ShAmt),
8741 DAG.getNode(ISD::SRL, DL, VT, BSwap, ShAmt));
8742}
8743
8744/// This contains all DAGCombine rules which reduce two values combined by
8745/// an Or operation to a single value \see visitANDLike().
8746SDValue DAGCombiner::visitORLike(SDValue N0, SDValue N1, const SDLoc &DL) {
8747 EVT VT = N1.getValueType();
8748
8749 // fold (or x, undef) -> -1
8750 if (!LegalOperations && (N0.isUndef() || N1.isUndef()))
8751 return DAG.getAllOnesConstant(DL, VT);
8752
8753 if (SDValue V = foldLogicOfSetCCs(false, N0, N1, DL))
8754 return V;
8755
8756 // (or (and X, C1), (and Y, C2)) -> (and (or X, Y), C3) if possible.
8757 if (N0.getOpcode() == ISD::AND && N1.getOpcode() == ISD::AND &&
8758 // Don't increase # computations.
8759 (N0->hasOneUse() || N1->hasOneUse())) {
8760 // We can only do this xform if we know that bits from X that are set in C2
8761 // but not in C1 are already zero. Likewise for Y.
8762 if (const ConstantSDNode *N0O1C =
8764 if (const ConstantSDNode *N1O1C =
8766 // We can only do this xform if we know that bits from X that are set in
8767 // C2 but not in C1 are already zero. Likewise for Y.
8768 const APInt &LHSMask = N0O1C->getAPIntValue();
8769 const APInt &RHSMask = N1O1C->getAPIntValue();
8770
8771 if (DAG.MaskedValueIsZero(N0.getOperand(0), RHSMask&~LHSMask) &&
8772 DAG.MaskedValueIsZero(N1.getOperand(0), LHSMask&~RHSMask)) {
8773 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT,
8774 N0.getOperand(0), N1.getOperand(0));
8775 return DAG.getNode(ISD::AND, DL, VT, X,
8776 DAG.getConstant(LHSMask | RHSMask, DL, VT));
8777 }
8778 }
8779 }
8780 }
8781
8782 // (or (and X, M), (and X, N)) -> (and X, (or M, N))
8783 if (N0.getOpcode() == ISD::AND &&
8784 N1.getOpcode() == ISD::AND &&
8785 N0.getOperand(0) == N1.getOperand(0) &&
8786 // Don't increase # computations.
8787 (N0->hasOneUse() || N1->hasOneUse())) {
8788 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT,
8789 N0.getOperand(1), N1.getOperand(1));
8790 return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), X);
8791 }
8792
8793 return SDValue();
8794}
8795
8796/// OR combines for which the commuted variant will be tried as well.
8798 SDNode *N) {
8799 EVT VT = N0.getValueType();
8800 unsigned BW = VT.getScalarSizeInBits();
8801 SDLoc DL(N);
8802
8803 auto peekThroughResize = [](SDValue V) {
8804 if (V->getOpcode() == ISD::ZERO_EXTEND || V->getOpcode() == ISD::TRUNCATE)
8805 return V->getOperand(0);
8806 return V;
8807 };
8808
8809 SDValue N0Resized = peekThroughResize(N0);
8810 if (N0Resized.getOpcode() == ISD::AND) {
8811 SDValue N1Resized = peekThroughResize(N1);
8812 SDValue N00 = N0Resized.getOperand(0);
8813 SDValue N01 = N0Resized.getOperand(1);
8814
8815 // fold or (and x, y), x --> x
8816 if (N00 == N1Resized || N01 == N1Resized)
8817 return N1;
8818
8819 // fold (or (and X, (xor Y, -1)), Y) -> (or X, Y)
8820 // TODO: Set AllowUndefs = true.
8821 if (SDValue NotOperand = getBitwiseNotOperand(N01, N00,
8822 /* AllowUndefs */ false)) {
8823 if (peekThroughResize(NotOperand) == N1Resized)
8824 return DAG.getNode(ISD::OR, DL, VT, DAG.getZExtOrTrunc(N00, DL, VT),
8825 N1);
8826 }
8827
8828 // fold (or (and (xor Y, -1), X), Y) -> (or X, Y)
8829 if (SDValue NotOperand = getBitwiseNotOperand(N00, N01,
8830 /* AllowUndefs */ false)) {
8831 if (peekThroughResize(NotOperand) == N1Resized)
8832 return DAG.getNode(ISD::OR, DL, VT, DAG.getZExtOrTrunc(N01, DL, VT),
8833 N1);
8834 }
8835 }
8836
8837 SDValue X, Y;
8838
8839 // fold or (xor X, N1), N1 --> or X, N1
8840 if (sd_match(N0, m_Xor(m_Value(X), m_Specific(N1))))
8841 return DAG.getNode(ISD::OR, DL, VT, X, N1);
8842
8843 // fold or (xor x, y), (x and/or y) --> or x, y
8844 if (sd_match(N0, m_Xor(m_Value(X), m_Value(Y))) &&
8845 (sd_match(N1, m_And(m_Specific(X), m_Specific(Y))) ||
8847 return DAG.getNode(ISD::OR, DL, VT, X, Y);
8848
8849 // fold or (not x), (and x, y) --> or (not x), y
8850 if (sd_match(N0, m_Not(m_Value(X))) &&
8852 return DAG.getNode(ISD::OR, DL, VT, N0, Y);
8853
8854 if (SDValue R = foldLogicOfShifts(N, N0, N1, DAG))
8855 return R;
8856
8857 auto peekThroughZext = [](SDValue V) {
8858 if (V->getOpcode() == ISD::ZERO_EXTEND)
8859 return V->getOperand(0);
8860 return V;
8861 };
8862
8863 if (N0.getOpcode() == ISD::FSHL && N1.getOpcode() == ISD::SHL &&
8864 peekThroughZext(N0.getOperand(2)) == peekThroughZext(N1.getOperand(1))) {
8865 // (fshl X, ?, Y) | (shl X, Y) --> fshl X, ?, Y
8866 if (N0.getOperand(0) == N1.getOperand(0))
8867 return N0;
8868 // (fshl A, X, Y) | (shl X, Y) --> fshl (A|X), X, Y
8869 if (N0.getOperand(1) == N1.getOperand(0) && N0.hasOneUse() &&
8870 N1.hasOneUse()) {
8871 SDValue A = N0.getOperand(0);
8872 SDValue X = N1.getOperand(0);
8873 SDValue NewLHS = DAG.getNode(ISD::OR, DL, VT, A, X);
8874 return DAG.getNode(ISD::FSHL, DL, VT, NewLHS, X, N0.getOperand(2));
8875 }
8876 }
8877
8878 if (N0.getOpcode() == ISD::FSHR && N1.getOpcode() == ISD::SRL &&
8879 peekThroughZext(N0.getOperand(2)) == peekThroughZext(N1.getOperand(1))) {
8880 // (fshr ?, X, Y) | (srl X, Y) --> fshr ?, X, Y
8881 if (N0.getOperand(1) == N1.getOperand(0))
8882 return N0;
8883 // (fshr X, B, Y) | (srl X, Y) --> fshr X, (X|B), Y
8884 if (N0.getOperand(0) == N1.getOperand(0) && N0.hasOneUse() &&
8885 N1.hasOneUse()) {
8886 SDValue X = N1.getOperand(0);
8887 SDValue B = N0.getOperand(1);
8888 SDValue NewRHS = DAG.getNode(ISD::OR, DL, VT, X, B);
8889 return DAG.getNode(ISD::FSHR, DL, VT, X, NewRHS, N0.getOperand(2));
8890 }
8891 }
8892
8893 // (fshl A, B, S0) | (fshr C, D, S1) --> fshl (A|C), (B|D), S0
8894 // iff S0 + S1 == bitwidth(S1)
8895 if (N0.getOpcode() == ISD::FSHL && N1.getOpcode() == ISD::FSHR &&
8896 N0.hasOneUse() && N1.hasOneUse()) {
8897 auto *S0 = dyn_cast<ConstantSDNode>(N0.getOperand(2));
8898 auto *S1 = dyn_cast<ConstantSDNode>(N1.getOperand(2));
8899 if (S0 && S1 && S0->getZExtValue() < BW && S1->getZExtValue() < BW &&
8900 S0->getZExtValue() == (BW - S1->getZExtValue())) {
8901 SDValue A = N0.getOperand(0);
8902 SDValue B = N0.getOperand(1);
8903 SDValue C = N1.getOperand(0);
8904 SDValue D = N1.getOperand(1);
8905 SDValue NewLHS = DAG.getNode(ISD::OR, DL, VT, A, C);
8906 SDValue NewRHS = DAG.getNode(ISD::OR, DL, VT, B, D);
8907 return DAG.getNode(ISD::FSHL, DL, VT, NewLHS, NewRHS, N0.getOperand(2));
8908 }
8909 }
8910
8911 // Attempt to match a legalized build_pair-esque pattern:
8912 // or(shl(aext(Hi),BW/2),zext(Lo))
8913 SDValue Lo, Hi;
8914 if (sd_match(N0,
8916 sd_match(N1, m_ZExt(m_Value(Lo))) &&
8917 Lo.getScalarValueSizeInBits() == (BW / 2) &&
8918 Lo.getValueType() == Hi.getValueType()) {
8919 // Fold build_pair(not(Lo),not(Hi)) -> not(build_pair(Lo,Hi)).
8920 SDValue NotLo, NotHi;
8921 if (sd_match(Lo, m_OneUse(m_Not(m_Value(NotLo)))) &&
8922 sd_match(Hi, m_OneUse(m_Not(m_Value(NotHi))))) {
8923 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, NotLo);
8924 Hi = DAG.getNode(ISD::ANY_EXTEND, DL, VT, NotHi);
8925 Hi = DAG.getNode(ISD::SHL, DL, VT, Hi,
8926 DAG.getShiftAmountConstant(BW / 2, VT, DL));
8927 return DAG.getNOT(DL, DAG.getNode(ISD::OR, DL, VT, Lo, Hi), VT);
8928 }
8929 }
8930
8931 return SDValue();
8932}
8933
8934SDValue DAGCombiner::visitOR(SDNode *N) {
8935 SDValue N0 = N->getOperand(0);
8936 SDValue N1 = N->getOperand(1);
8937 EVT VT = N1.getValueType();
8938 SDLoc DL(N);
8939
8940 // x | x --> x
8941 if (N0 == N1)
8942 return N0;
8943
8944 // fold (or c1, c2) -> c1|c2
8945 if (SDValue C =
8946 DAG.FoldConstantArithmetic(ISD::OR, DL, VT, {N0, N1}, N->getFlags()))
8947 return C;
8948
8949 // canonicalize constant to RHS
8952 return DAG.getNode(ISD::OR, DL, VT, N1, N0);
8953
8954 // fold vector ops
8955 if (VT.isVector()) {
8956 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
8957 return FoldedVOp;
8958
8959 // fold (or x, 0) -> x, vector edition
8961 return N0;
8962
8963 // fold (or x, -1) -> -1, vector edition
8965 // do not return N1, because undef node may exist in N1
8966 return DAG.getAllOnesConstant(DL, N1.getValueType());
8967
8968 // fold (or buildvector(x,0,-1,w), buildvector(0,y,z,w))
8969 // --> buildvector(x,y,-1,w)
8970 auto *BV0 = dyn_cast<BuildVectorSDNode>(N0);
8971 auto *BV1 = dyn_cast<BuildVectorSDNode>(N1);
8972 if (BV0 && BV1 && !BV0->getSplatValue() && !BV1->getSplatValue() &&
8973 N0.hasOneUse() && N1.hasOneUse() &&
8974 BV0->getOperand(0).getValueType() ==
8975 BV1->getOperand(0).getValueType()) {
8976 SmallVector<SDValue> MergedOps;
8977 unsigned NumElts = VT.getVectorNumElements();
8978 EVT EltVT = BV0->getOperand(0).getValueType();
8979 for (unsigned I = 0; I != NumElts; ++I) {
8980 auto *C0 = dyn_cast<ConstantSDNode>(BV0->getOperand(I));
8981 auto *C1 = dyn_cast<ConstantSDNode>(BV1->getOperand(I));
8982 if (C0 && C1)
8983 MergedOps.push_back(DAG.getConstant(
8984 C0->getAPIntValue() | C1->getAPIntValue(), DL, EltVT));
8985 else if (C0 && C0->isZero())
8986 MergedOps.push_back(BV1->getOperand(I));
8987 else if (C1 && C1->isZero())
8988 MergedOps.push_back(BV0->getOperand(I));
8989 else if (C0 && C0->isAllOnes())
8990 MergedOps.push_back(BV0->getOperand(I));
8991 else if (C1 && C1->isAllOnes())
8992 MergedOps.push_back(BV1->getOperand(I));
8993 else if (BV0->getOperand(I) == BV1->getOperand(I))
8994 MergedOps.push_back(BV0->getOperand(I));
8995 else
8996 break;
8997 }
8998 if (MergedOps.size() == NumElts)
8999 return DAG.getBuildVector(VT, DL, MergedOps);
9000 }
9001
9002 // fold (or (shuf A, V_0, MA), (shuf B, V_0, MB)) -> (shuf A, B, Mask)
9003 // Do this only if the resulting type / shuffle is legal.
9004 auto *SV0 = dyn_cast<ShuffleVectorSDNode>(N0);
9005 auto *SV1 = dyn_cast<ShuffleVectorSDNode>(N1);
9006 if (SV0 && SV1 && TLI.isTypeLegal(VT)) {
9007 bool ZeroN00 = ISD::isBuildVectorAllZeros(N0.getOperand(0).getNode());
9008 bool ZeroN01 = ISD::isBuildVectorAllZeros(N0.getOperand(1).getNode());
9009 bool ZeroN10 = ISD::isBuildVectorAllZeros(N1.getOperand(0).getNode());
9010 bool ZeroN11 = ISD::isBuildVectorAllZeros(N1.getOperand(1).getNode());
9011 // Ensure both shuffles have a zero input.
9012 if ((ZeroN00 != ZeroN01) && (ZeroN10 != ZeroN11)) {
9013 assert((!ZeroN00 || !ZeroN01) && "Both inputs zero!");
9014 assert((!ZeroN10 || !ZeroN11) && "Both inputs zero!");
9015 bool CanFold = true;
9016 int NumElts = VT.getVectorNumElements();
9017 SmallVector<int, 4> Mask(NumElts, -1);
9018
9019 for (int i = 0; i != NumElts; ++i) {
9020 int M0 = SV0->getMaskElt(i);
9021 int M1 = SV1->getMaskElt(i);
9022
9023 // Determine if either index is pointing to a zero vector.
9024 bool M0Zero = M0 < 0 || (ZeroN00 == (M0 < NumElts));
9025 bool M1Zero = M1 < 0 || (ZeroN10 == (M1 < NumElts));
9026
9027 // If one element is zero and the otherside is undef, keep undef.
9028 // This also handles the case that both are undef.
9029 if ((M0Zero && M1 < 0) || (M1Zero && M0 < 0))
9030 continue;
9031
9032 // Make sure only one of the elements is zero.
9033 if (M0Zero == M1Zero) {
9034 CanFold = false;
9035 break;
9036 }
9037
9038 assert((M0 >= 0 || M1 >= 0) && "Undef index!");
9039
9040 // We have a zero and non-zero element. If the non-zero came from
9041 // SV0 make the index a LHS index. If it came from SV1, make it
9042 // a RHS index. We need to mod by NumElts because we don't care
9043 // which operand it came from in the original shuffles.
9044 Mask[i] = M1Zero ? M0 % NumElts : (M1 % NumElts) + NumElts;
9045 }
9046
9047 if (CanFold) {
9048 SDValue NewLHS = ZeroN00 ? N0.getOperand(1) : N0.getOperand(0);
9049 SDValue NewRHS = ZeroN10 ? N1.getOperand(1) : N1.getOperand(0);
9050 SDValue LegalShuffle =
9051 TLI.buildLegalVectorShuffle(VT, DL, NewLHS, NewRHS, Mask, DAG);
9052 if (LegalShuffle)
9053 return LegalShuffle;
9054 }
9055 }
9056 }
9057 }
9058
9059 // fold (or x, 0) -> x
9060 if (isNullConstant(N1))
9061 return N0;
9062
9063 // fold (or x, -1) -> -1
9064 if (isAllOnesConstant(N1))
9065 return N1;
9066
9067 if (SDValue NewSel = foldBinOpIntoSelect(N))
9068 return NewSel;
9069
9070 // fold (or x, c) -> c iff (x & ~c) == 0
9071 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
9072 if (N1C && DAG.MaskedValueIsZero(N0, ~N1C->getAPIntValue()))
9073 return N1;
9074
9075 if (SDValue R = foldAndOrOfSETCC(N, DAG))
9076 return R;
9077
9078 if (SDValue Combined = visitORLike(N0, N1, DL))
9079 return Combined;
9080
9081 if (SDValue Combined = combineCarryDiamond(DAG, TLI, N0, N1, N))
9082 return Combined;
9083
9084 if (SDValue Combined = combineOrOfSetCCToUSUBOCarry(N, DAG, TLI))
9085 return Combined;
9086
9087 // Recognize halfword bswaps as (bswap + rotl 16) or (bswap + shl 16)
9088 if (SDValue BSwap = MatchBSwapHWord(N, N0, N1))
9089 return BSwap;
9090 if (SDValue BSwap = MatchBSwapHWordLow(N, N0, N1))
9091 return BSwap;
9092
9093 // reassociate or
9094 if (SDValue ROR = reassociateOps(ISD::OR, DL, N0, N1, N->getFlags()))
9095 return ROR;
9096
9097 // Fold or(vecreduce(x), vecreduce(y)) -> vecreduce(or(x, y))
9098 if (SDValue SD =
9099 reassociateReduction(ISD::VECREDUCE_OR, ISD::OR, DL, VT, N0, N1))
9100 return SD;
9101
9102 // Canonicalize (or (and X, c1), c2) -> (and (or X, c2), c1|c2)
9103 // iff (c1 & c2) != 0 or c1/c2 are undef.
9104 auto MatchIntersect = [](ConstantSDNode *C1, ConstantSDNode *C2) {
9105 return !C1 || !C2 || C1->getAPIntValue().intersects(C2->getAPIntValue());
9106 };
9107 if (N0.getOpcode() == ISD::AND && N0->hasOneUse() &&
9108 ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchIntersect, true)) {
9109 if (SDValue COR = DAG.FoldConstantArithmetic(ISD::OR, SDLoc(N1), VT,
9110 {N1, N0.getOperand(1)})) {
9111 SDValue IOR = DAG.getNode(ISD::OR, SDLoc(N0), VT, N0.getOperand(0), N1);
9112 AddToWorklist(IOR.getNode());
9113 return DAG.getNode(ISD::AND, DL, VT, COR, IOR);
9114 }
9115 }
9116
9117 if (SDValue Combined = visitORCommutative(DAG, N0, N1, N))
9118 return Combined;
9119 if (SDValue Combined = visitORCommutative(DAG, N1, N0, N))
9120 return Combined;
9121
9122 // Simplify: (or (op x...), (op y...)) -> (op (or x, y))
9123 if (N0.getOpcode() == N1.getOpcode())
9124 if (SDValue V = hoistLogicOpWithSameOpcodeHands(N))
9125 return V;
9126
9127 // See if this is some rotate idiom.
9128 if (SDValue Rot = MatchRotate(N0, N1, DL, /*FromAdd=*/false))
9129 return Rot;
9130
9131 if (SDValue Load = MatchLoadCombine(N))
9132 return Load;
9133
9134 // Simplify the operands using demanded-bits information.
9136 return SDValue(N, 0);
9137
9138 // If OR can be rewritten into ADD, try combines based on ADD.
9139 if ((!LegalOperations || TLI.isOperationLegal(ISD::ADD, VT)) &&
9140 DAG.isADDLike(SDValue(N, 0)))
9141 if (SDValue Combined = visitADDLike(N))
9142 return Combined;
9143
9144 // Postpone until legalization completed to avoid interference with bswap
9145 // folding
9146 if (LegalOperations || VT.isVector())
9147 if (SDValue R = foldLogicTreeOfShifts(N, N0, N1, DAG))
9148 return R;
9149
9150 if (VT.isScalarInteger() && VT != MVT::i1)
9151 if (SDValue R = foldMaskedMerge(N, DAG, TLI, DL))
9152 return R;
9153
9154 return SDValue();
9155}
9156
9158 SDValue &Mask) {
9159 if (Op.getOpcode() == ISD::AND &&
9160 DAG.isConstantIntBuildVectorOrConstantInt(Op.getOperand(1))) {
9161 Mask = Op.getOperand(1);
9162 return Op.getOperand(0);
9163 }
9164 return Op;
9165}
9166
9167/// Match "(X shl/srl V1) & V2" where V2 may not be present.
9168static bool matchRotateHalf(const SelectionDAG &DAG, SDValue Op, SDValue &Shift,
9169 SDValue &Mask) {
9170 Op = stripConstantMask(DAG, Op, Mask);
9171 if (Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SHL) {
9172 Shift = Op;
9173 return true;
9174 }
9175 return false;
9176}
9177
9178/// Helper function for visitOR to extract the needed side of a rotate idiom
9179/// from a shl/srl/mul/udiv. This is meant to handle cases where
9180/// InstCombine merged some outside op with one of the shifts from
9181/// the rotate pattern.
9182/// \returns An empty \c SDValue if the needed shift couldn't be extracted.
9183/// Otherwise, returns an expansion of \p ExtractFrom based on the following
9184/// patterns:
9185///
9186/// (or (add v v) (shrl v bitwidth-1)):
9187/// expands (add v v) -> (shl v 1)
9188///
9189/// (or (mul v c0) (shrl (mul v c1) c2)):
9190/// expands (mul v c0) -> (shl (mul v c1) c3)
9191///
9192/// (or (udiv v c0) (shl (udiv v c1) c2)):
9193/// expands (udiv v c0) -> (shrl (udiv v c1) c3)
9194///
9195/// (or (shl v c0) (shrl (shl v c1) c2)):
9196/// expands (shl v c0) -> (shl (shl v c1) c3)
9197///
9198/// (or (shrl v c0) (shl (shrl v c1) c2)):
9199/// expands (shrl v c0) -> (shrl (shrl v c1) c3)
9200///
9201/// Such that in all cases, c3+c2==bitwidth(op v c1).
9203 SDValue ExtractFrom, SDValue &Mask,
9204 const SDLoc &DL) {
9205 assert(OppShift && ExtractFrom && "Empty SDValue");
9206 if (OppShift.getOpcode() != ISD::SHL && OppShift.getOpcode() != ISD::SRL)
9207 return SDValue();
9208
9209 ExtractFrom = stripConstantMask(DAG, ExtractFrom, Mask);
9210
9211 // Value and Type of the shift.
9212 SDValue OppShiftLHS = OppShift.getOperand(0);
9213 EVT ShiftedVT = OppShiftLHS.getValueType();
9214
9215 // Amount of the existing shift.
9216 ConstantSDNode *OppShiftCst = isConstOrConstSplat(OppShift.getOperand(1));
9217
9218 // (add v v) -> (shl v 1)
9219 // TODO: Should this be a general DAG canonicalization?
9220 if (OppShift.getOpcode() == ISD::SRL && OppShiftCst &&
9221 ExtractFrom.getOpcode() == ISD::ADD &&
9222 ExtractFrom.getOperand(0) == ExtractFrom.getOperand(1) &&
9223 ExtractFrom.getOperand(0) == OppShiftLHS &&
9224 OppShiftCst->getAPIntValue() == ShiftedVT.getScalarSizeInBits() - 1)
9225 return DAG.getNode(ISD::SHL, DL, ShiftedVT, OppShiftLHS,
9226 DAG.getShiftAmountConstant(1, ShiftedVT, DL));
9227
9228 // Preconditions:
9229 // (or (op0 v c0) (shiftl/r (op0 v c1) c2))
9230 //
9231 // Find opcode of the needed shift to be extracted from (op0 v c0).
9232 unsigned Opcode = ISD::DELETED_NODE;
9233 bool IsMulOrDiv = false;
9234 // Set Opcode and IsMulOrDiv if the extract opcode matches the needed shift
9235 // opcode or its arithmetic (mul or udiv) variant.
9236 auto SelectOpcode = [&](unsigned NeededShift, unsigned MulOrDivVariant) {
9237 IsMulOrDiv = ExtractFrom.getOpcode() == MulOrDivVariant;
9238 if (!IsMulOrDiv && ExtractFrom.getOpcode() != NeededShift)
9239 return false;
9240 Opcode = NeededShift;
9241 return true;
9242 };
9243 // op0 must be either the needed shift opcode or the mul/udiv equivalent
9244 // that the needed shift can be extracted from.
9245 if ((OppShift.getOpcode() != ISD::SRL || !SelectOpcode(ISD::SHL, ISD::MUL)) &&
9246 (OppShift.getOpcode() != ISD::SHL || !SelectOpcode(ISD::SRL, ISD::UDIV)))
9247 return SDValue();
9248
9249 // op0 must be the same opcode on both sides, have the same LHS argument,
9250 // and produce the same value type.
9251 if (OppShiftLHS.getOpcode() != ExtractFrom.getOpcode() ||
9252 OppShiftLHS.getOperand(0) != ExtractFrom.getOperand(0) ||
9253 ShiftedVT != ExtractFrom.getValueType())
9254 return SDValue();
9255
9256 // Constant mul/udiv/shift amount from the RHS of the shift's LHS op.
9257 ConstantSDNode *OppLHSCst = isConstOrConstSplat(OppShiftLHS.getOperand(1));
9258 // Constant mul/udiv/shift amount from the RHS of the ExtractFrom op.
9259 ConstantSDNode *ExtractFromCst =
9260 isConstOrConstSplat(ExtractFrom.getOperand(1));
9261 // TODO: We should be able to handle non-uniform constant vectors for these values
9262 // Check that we have constant values.
9263 if (!OppShiftCst || !OppShiftCst->getAPIntValue() ||
9264 !OppLHSCst || !OppLHSCst->getAPIntValue() ||
9265 !ExtractFromCst || !ExtractFromCst->getAPIntValue())
9266 return SDValue();
9267
9268 // Compute the shift amount we need to extract to complete the rotate.
9269 const unsigned VTWidth = ShiftedVT.getScalarSizeInBits();
9270 if (OppShiftCst->getAPIntValue().ugt(VTWidth))
9271 return SDValue();
9272 APInt NeededShiftAmt = VTWidth - OppShiftCst->getAPIntValue();
9273 // Normalize the bitwidth of the two mul/udiv/shift constant operands.
9274 APInt ExtractFromAmt = ExtractFromCst->getAPIntValue();
9275 APInt OppLHSAmt = OppLHSCst->getAPIntValue();
9276 zeroExtendToMatch(ExtractFromAmt, OppLHSAmt);
9277
9278 // Now try extract the needed shift from the ExtractFrom op and see if the
9279 // result matches up with the existing shift's LHS op.
9280 if (IsMulOrDiv) {
9281 // Op to extract from is a mul or udiv by a constant.
9282 // Check:
9283 // c2 / (1 << (bitwidth(op0 v c0) - c1)) == c0
9284 // c2 % (1 << (bitwidth(op0 v c0) - c1)) == 0
9285 const APInt ExtractDiv = APInt::getOneBitSet(ExtractFromAmt.getBitWidth(),
9286 NeededShiftAmt.getZExtValue());
9287 APInt ResultAmt;
9288 APInt Rem;
9289 APInt::udivrem(ExtractFromAmt, ExtractDiv, ResultAmt, Rem);
9290 if (Rem != 0 || ResultAmt != OppLHSAmt)
9291 return SDValue();
9292 } else {
9293 // Op to extract from is a shift by a constant.
9294 // Check:
9295 // c2 - (bitwidth(op0 v c0) - c1) == c0
9296 if (OppLHSAmt != ExtractFromAmt - NeededShiftAmt.zextOrTrunc(
9297 ExtractFromAmt.getBitWidth()))
9298 return SDValue();
9299 }
9300
9301 // Return the expanded shift op that should allow a rotate to be formed.
9302 EVT ShiftVT = OppShift.getOperand(1).getValueType();
9303 EVT ResVT = ExtractFrom.getValueType();
9304 SDValue NewShiftNode = DAG.getConstant(NeededShiftAmt, DL, ShiftVT);
9305 return DAG.getNode(Opcode, DL, ResVT, OppShiftLHS, NewShiftNode);
9306}
9307
9308// Return true if we can prove that, whenever Neg and Pos are both in the
9309// range [0, EltSize), Neg == (Pos == 0 ? 0 : EltSize - Pos). This means that
9310// for two opposing shifts shift1 and shift2 and a value X with OpBits bits:
9311//
9312// (or (shift1 X, Neg), (shift2 X, Pos))
9313//
9314// reduces to a rotate in direction shift2 by Pos or (equivalently) a rotate
9315// in direction shift1 by Neg. The range [0, EltSize) means that we only need
9316// to consider shift amounts with defined behavior.
9317//
9318// The IsRotate flag should be set when the LHS of both shifts is the same.
9319// Otherwise if matching a general funnel shift, it should be clear.
9320static bool matchRotateSub(SDValue Pos, SDValue Neg, unsigned EltSize,
9321 SelectionDAG &DAG, bool IsRotate, bool FromAdd) {
9322 const auto &TLI = DAG.getTargetLoweringInfo();
9323 // If EltSize is a power of 2 then:
9324 //
9325 // (a) (Pos == 0 ? 0 : EltSize - Pos) == (EltSize - Pos) & (EltSize - 1)
9326 // (b) Neg == Neg & (EltSize - 1) whenever Neg is in [0, EltSize).
9327 //
9328 // So if EltSize is a power of 2 and Neg is (and Neg', EltSize-1), we check
9329 // for the stronger condition:
9330 //
9331 // Neg & (EltSize - 1) == (EltSize - Pos) & (EltSize - 1) [A]
9332 //
9333 // for all Neg and Pos. Since Neg & (EltSize - 1) == Neg' & (EltSize - 1)
9334 // we can just replace Neg with Neg' for the rest of the function.
9335 //
9336 // In other cases we check for the even stronger condition:
9337 //
9338 // Neg == EltSize - Pos [B]
9339 //
9340 // for all Neg and Pos. Note that the (or ...) then invokes undefined
9341 // behavior if Pos == 0 (and consequently Neg == EltSize).
9342 //
9343 // We could actually use [A] whenever EltSize is a power of 2, but the
9344 // only extra cases that it would match are those uninteresting ones
9345 // where Neg and Pos are never in range at the same time. E.g. for
9346 // EltSize == 32, using [A] would allow a Neg of the form (sub 64, Pos)
9347 // as well as (sub 32, Pos), but:
9348 //
9349 // (or (shift1 X, (sub 64, Pos)), (shift2 X, Pos))
9350 //
9351 // always invokes undefined behavior for 32-bit X.
9352 //
9353 // Below, Mask == EltSize - 1 when using [A] and is all-ones otherwise.
9354 // This allows us to peek through any operations that only affect Mask's
9355 // un-demanded bits.
9356 //
9357 // NOTE: We can only do this when matching operations which won't modify the
9358 // least Log2(EltSize) significant bits and not a general funnel shift.
9359 unsigned MaskLoBits = 0;
9360 if (IsRotate && !FromAdd && isPowerOf2_64(EltSize)) {
9361 unsigned Bits = Log2_64(EltSize);
9362 unsigned NegBits = Neg.getScalarValueSizeInBits();
9363 if (NegBits >= Bits) {
9364 APInt DemandedBits = APInt::getLowBitsSet(NegBits, Bits);
9365 if (SDValue Inner =
9366 TLI.SimplifyMultipleUseDemandedBits(Neg, DemandedBits, DAG)) {
9367 Neg = Inner;
9368 MaskLoBits = Bits;
9369 }
9370 }
9371 }
9372
9373 // Check whether Neg has the form (sub NegC, NegOp1) for some NegC and NegOp1.
9374 if (Neg.getOpcode() != ISD::SUB)
9375 return false;
9377 if (!NegC)
9378 return false;
9379 SDValue NegOp1 = Neg.getOperand(1);
9380
9381 // On the RHS of [A], if Pos is the result of operation on Pos' that won't
9382 // affect Mask's demanded bits, just replace Pos with Pos'. These operations
9383 // are redundant for the purpose of the equality.
9384 if (MaskLoBits) {
9385 unsigned PosBits = Pos.getScalarValueSizeInBits();
9386 if (PosBits >= MaskLoBits) {
9387 APInt DemandedBits = APInt::getLowBitsSet(PosBits, MaskLoBits);
9388 if (SDValue Inner =
9389 TLI.SimplifyMultipleUseDemandedBits(Pos, DemandedBits, DAG)) {
9390 Pos = Inner;
9391 }
9392 }
9393 }
9394
9395 // The condition we need is now:
9396 //
9397 // (NegC - NegOp1) & Mask == (EltSize - Pos) & Mask
9398 //
9399 // If NegOp1 == Pos then we need:
9400 //
9401 // EltSize & Mask == NegC & Mask
9402 //
9403 // (because "x & Mask" is a truncation and distributes through subtraction).
9404 //
9405 // We also need to account for a potential truncation of NegOp1 if the amount
9406 // has already been legalized to a shift amount type.
9407 APInt Width;
9408 if ((Pos == NegOp1) ||
9409 (NegOp1.getOpcode() == ISD::TRUNCATE && Pos == NegOp1.getOperand(0)))
9410 Width = NegC->getAPIntValue();
9411
9412 // Check for cases where Pos has the form (add NegOp1, PosC) for some PosC.
9413 // Then the condition we want to prove becomes:
9414 //
9415 // (NegC - NegOp1) & Mask == (EltSize - (NegOp1 + PosC)) & Mask
9416 //
9417 // which, again because "x & Mask" is a truncation, becomes:
9418 //
9419 // NegC & Mask == (EltSize - PosC) & Mask
9420 // EltSize & Mask == (NegC + PosC) & Mask
9421 else if (Pos.getOpcode() == ISD::ADD && Pos.getOperand(0) == NegOp1) {
9422 if (ConstantSDNode *PosC = isConstOrConstSplat(Pos.getOperand(1)))
9423 Width = PosC->getAPIntValue() + NegC->getAPIntValue();
9424 else
9425 return false;
9426 } else
9427 return false;
9428
9429 // Now we just need to check that EltSize & Mask == Width & Mask.
9430 if (MaskLoBits)
9431 // EltSize & Mask is 0 since Mask is EltSize - 1.
9432 return Width.getLoBits(MaskLoBits) == 0;
9433 return Width == EltSize;
9434}
9435
9436// A subroutine of MatchRotate used once we have found an OR of two opposite
9437// shifts of Shifted. If Neg == <operand size> - Pos then the OR reduces
9438// to both (PosOpcode Shifted, Pos) and (NegOpcode Shifted, Neg), with the
9439// former being preferred if supported. InnerPos and InnerNeg are Pos and
9440// Neg with outer conversions stripped away.
9441SDValue DAGCombiner::MatchRotatePosNeg(SDValue Shifted, SDValue Pos,
9442 SDValue Neg, SDValue InnerPos,
9443 SDValue InnerNeg, bool FromAdd,
9444 bool HasPos, unsigned PosOpcode,
9445 unsigned NegOpcode, const SDLoc &DL) {
9446 // fold (or/add (shl x, (*ext y)),
9447 // (srl x, (*ext (sub 32, y)))) ->
9448 // (rotl x, y) or (rotr x, (sub 32, y))
9449 //
9450 // fold (or/add (shl x, (*ext (sub 32, y))),
9451 // (srl x, (*ext y))) ->
9452 // (rotr x, y) or (rotl x, (sub 32, y))
9453 EVT VT = Shifted.getValueType();
9454 if (matchRotateSub(InnerPos, InnerNeg, VT.getScalarSizeInBits(), DAG,
9455 /*IsRotate*/ true, FromAdd))
9456 return DAG.getNode(HasPos ? PosOpcode : NegOpcode, DL, VT, Shifted,
9457 HasPos ? Pos : Neg);
9458
9459 return SDValue();
9460}
9461
9462// A subroutine of MatchRotate used once we have found an OR of two opposite
9463// shifts of N0 + N1. If Neg == <operand size> - Pos then the OR reduces
9464// to both (PosOpcode N0, N1, Pos) and (NegOpcode N0, N1, Neg), with the
9465// former being preferred if supported. InnerPos and InnerNeg are Pos and
9466// Neg with outer conversions stripped away.
9467// TODO: Merge with MatchRotatePosNeg.
9468SDValue DAGCombiner::MatchFunnelPosNeg(SDValue N0, SDValue N1, SDValue Pos,
9469 SDValue Neg, SDValue InnerPos,
9470 SDValue InnerNeg, bool FromAdd,
9471 bool HasPos, unsigned PosOpcode,
9472 unsigned NegOpcode, const SDLoc &DL) {
9473 EVT VT = N0.getValueType();
9474 unsigned EltBits = VT.getScalarSizeInBits();
9475
9476 // fold (or/add (shl x0, (*ext y)),
9477 // (srl x1, (*ext (sub 32, y)))) ->
9478 // (fshl x0, x1, y) or (fshr x0, x1, (sub 32, y))
9479 //
9480 // fold (or/add (shl x0, (*ext (sub 32, y))),
9481 // (srl x1, (*ext y))) ->
9482 // (fshr x0, x1, y) or (fshl x0, x1, (sub 32, y))
9483 if (matchRotateSub(InnerPos, InnerNeg, EltBits, DAG, /*IsRotate*/ N0 == N1,
9484 FromAdd))
9485 return DAG.getNode(HasPos ? PosOpcode : NegOpcode, DL, VT, N0, N1,
9486 HasPos ? Pos : Neg);
9487
9488 // Matching the shift+xor cases, we can't easily use the xor'd shift amount
9489 // so for now just use the PosOpcode case if its legal.
9490 // TODO: When can we use the NegOpcode case?
9491 if (PosOpcode == ISD::FSHL && isPowerOf2_32(EltBits)) {
9492 SDValue X;
9493 // fold (or/add (shl x0, y), (srl (srl x1, 1), (xor y, 31)))
9494 // -> (fshl x0, x1, y)
9495 if (sd_match(N1, m_Srl(m_Value(X), m_One())) &&
9496 sd_match(InnerNeg,
9497 m_Xor(m_Specific(InnerPos), m_SpecificInt(EltBits - 1))) &&
9499 return DAG.getNode(ISD::FSHL, DL, VT, N0, X, Pos);
9500 }
9501
9502 // fold (or/add (shl (shl x0, 1), (xor y, 31)), (srl x1, y))
9503 // -> (fshr x0, x1, y)
9504 if (sd_match(N0, m_Shl(m_Value(X), m_One())) &&
9505 sd_match(InnerPos,
9506 m_Xor(m_Specific(InnerNeg), m_SpecificInt(EltBits - 1))) &&
9508 return DAG.getNode(ISD::FSHR, DL, VT, X, N1, Neg);
9509 }
9510
9511 // fold (or/add (shl (add x0, x0), (xor y, 31)), (srl x1, y))
9512 // -> (fshr x0, x1, y)
9513 // TODO: Should add(x,x) -> shl(x,1) be a general DAG canonicalization?
9514 if (sd_match(N0, m_Add(m_Value(X), m_Deferred(X))) &&
9515 sd_match(InnerPos,
9516 m_Xor(m_Specific(InnerNeg), m_SpecificInt(EltBits - 1))) &&
9518 return DAG.getNode(ISD::FSHR, DL, VT, X, N1, Neg);
9519 }
9520 }
9521
9522 return SDValue();
9523}
9524
9525// MatchRotate - Handle an 'or' or 'add' of two operands. If this is one of the
9526// many idioms for rotate, and if the target supports rotation instructions,
9527// generate a rot[lr]. This also matches funnel shift patterns, similar to
9528// rotation but with different shifted sources.
9529SDValue DAGCombiner::MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL,
9530 bool FromAdd) {
9531 EVT VT = LHS.getValueType();
9532
9533 // The target must have at least one rotate/funnel flavor.
9534 // We still try to match rotate by constant pre-legalization.
9535 // TODO: Support pre-legalization funnel-shift by constant.
9536 bool HasROTL = hasOperation(ISD::ROTL, VT);
9537 bool HasROTR = hasOperation(ISD::ROTR, VT);
9538 bool HasFSHL = hasOperation(ISD::FSHL, VT);
9539 bool HasFSHR = hasOperation(ISD::FSHR, VT);
9540
9541 // If the type is going to be promoted and the target has enabled custom
9542 // lowering for rotate, allow matching rotate by non-constants. Only allow
9543 // this for scalar types.
9544 if (VT.isScalarInteger() && TLI.getTypeAction(*DAG.getContext(), VT) ==
9548 }
9549
9550 if (LegalOperations && !HasROTL && !HasROTR && !HasFSHL && !HasFSHR)
9551 return SDValue();
9552
9553 // Check for truncated rotate.
9554 if (LHS.getOpcode() == ISD::TRUNCATE && RHS.getOpcode() == ISD::TRUNCATE &&
9555 LHS.getOperand(0).getValueType() == RHS.getOperand(0).getValueType()) {
9556 assert(LHS.getValueType() == RHS.getValueType());
9557 if (SDValue Rot =
9558 MatchRotate(LHS.getOperand(0), RHS.getOperand(0), DL, FromAdd))
9559 return DAG.getNode(ISD::TRUNCATE, SDLoc(LHS), LHS.getValueType(), Rot);
9560 }
9561
9562 // Match "(X shl/srl V1) & V2" where V2 may not be present.
9563 SDValue LHSShift; // The shift.
9564 SDValue LHSMask; // AND value if any.
9565 matchRotateHalf(DAG, LHS, LHSShift, LHSMask);
9566
9567 SDValue RHSShift; // The shift.
9568 SDValue RHSMask; // AND value if any.
9569 matchRotateHalf(DAG, RHS, RHSShift, RHSMask);
9570
9571 // If neither side matched a rotate half, bail
9572 if (!LHSShift && !RHSShift)
9573 return SDValue();
9574
9575 // InstCombine may have combined a constant shl, srl, mul, or udiv with one
9576 // side of the rotate, so try to handle that here. In all cases we need to
9577 // pass the matched shift from the opposite side to compute the opcode and
9578 // needed shift amount to extract. We still want to do this if both sides
9579 // matched a rotate half because one half may be a potential overshift that
9580 // can be broken down (ie if InstCombine merged two shl or srl ops into a
9581 // single one).
9582
9583 // Have LHS side of the rotate, try to extract the needed shift from the RHS.
9584 if (LHSShift)
9585 if (SDValue NewRHSShift =
9586 extractShiftForRotate(DAG, LHSShift, RHS, RHSMask, DL))
9587 RHSShift = NewRHSShift;
9588 // Have RHS side of the rotate, try to extract the needed shift from the LHS.
9589 if (RHSShift)
9590 if (SDValue NewLHSShift =
9591 extractShiftForRotate(DAG, RHSShift, LHS, LHSMask, DL))
9592 LHSShift = NewLHSShift;
9593
9594 // If a side is still missing, nothing else we can do.
9595 if (!RHSShift || !LHSShift)
9596 return SDValue();
9597
9598 // At this point we've matched or extracted a shift op on each side.
9599
9600 if (LHSShift.getOpcode() == RHSShift.getOpcode())
9601 return SDValue(); // Shifts must disagree.
9602
9603 // Canonicalize shl to left side in a shl/srl pair.
9604 if (RHSShift.getOpcode() == ISD::SHL) {
9605 std::swap(LHS, RHS);
9606 std::swap(LHSShift, RHSShift);
9607 std::swap(LHSMask, RHSMask);
9608 }
9609
9610 // Something has gone wrong - we've lost the shl/srl pair - bail.
9611 if (LHSShift.getOpcode() != ISD::SHL || RHSShift.getOpcode() != ISD::SRL)
9612 return SDValue();
9613
9614 unsigned EltSizeInBits = VT.getScalarSizeInBits();
9615 SDValue LHSShiftArg = LHSShift.getOperand(0);
9616 SDValue LHSShiftAmt = LHSShift.getOperand(1);
9617 SDValue RHSShiftArg = RHSShift.getOperand(0);
9618 SDValue RHSShiftAmt = RHSShift.getOperand(1);
9619
9620 auto MatchRotateSum = [EltSizeInBits](ConstantSDNode *LHS,
9621 ConstantSDNode *RHS) {
9622 return (LHS->getAPIntValue() + RHS->getAPIntValue()) == EltSizeInBits;
9623 };
9624
9625 auto ApplyMasks = [&](SDValue Res) {
9626 // If there is an AND of either shifted operand, apply it to the result.
9627 if (LHSMask.getNode() || RHSMask.getNode()) {
9630
9631 if (LHSMask.getNode()) {
9632 SDValue RHSBits = DAG.getNode(ISD::SRL, DL, VT, AllOnes, RHSShiftAmt);
9633 Mask = DAG.getNode(ISD::AND, DL, VT, Mask,
9634 DAG.getNode(ISD::OR, DL, VT, LHSMask, RHSBits));
9635 }
9636 if (RHSMask.getNode()) {
9637 SDValue LHSBits = DAG.getNode(ISD::SHL, DL, VT, AllOnes, LHSShiftAmt);
9638 Mask = DAG.getNode(ISD::AND, DL, VT, Mask,
9639 DAG.getNode(ISD::OR, DL, VT, RHSMask, LHSBits));
9640 }
9641
9642 Res = DAG.getNode(ISD::AND, DL, VT, Res, Mask);
9643 }
9644
9645 return Res;
9646 };
9647
9648 // TODO: Support pre-legalization funnel-shift by constant.
9649 bool IsRotate = LHSShiftArg == RHSShiftArg;
9650 if (!IsRotate && !(HasFSHL || HasFSHR)) {
9651 if (TLI.isTypeLegal(VT) && LHS.hasOneUse() && RHS.hasOneUse() &&
9652 ISD::matchBinaryPredicate(LHSShiftAmt, RHSShiftAmt, MatchRotateSum)) {
9653 // Look for a disguised rotate by constant.
9654 // The common shifted operand X may be hidden inside another 'or'.
9655 SDValue X, Y;
9656 auto matchOr = [&X, &Y](SDValue Or, SDValue CommonOp) {
9657 if (!Or.hasOneUse() || Or.getOpcode() != ISD::OR)
9658 return false;
9659 if (CommonOp == Or.getOperand(0)) {
9660 X = CommonOp;
9661 Y = Or.getOperand(1);
9662 return true;
9663 }
9664 if (CommonOp == Or.getOperand(1)) {
9665 X = CommonOp;
9666 Y = Or.getOperand(0);
9667 return true;
9668 }
9669 return false;
9670 };
9671
9672 SDValue Res;
9673 if (matchOr(LHSShiftArg, RHSShiftArg)) {
9674 // (shl (X | Y), C1) | (srl X, C2) --> (rotl X, C1) | (shl Y, C1)
9675 SDValue RotX = DAG.getNode(ISD::ROTL, DL, VT, X, LHSShiftAmt);
9676 SDValue ShlY = DAG.getNode(ISD::SHL, DL, VT, Y, LHSShiftAmt);
9677 Res = DAG.getNode(ISD::OR, DL, VT, RotX, ShlY);
9678 } else if (matchOr(RHSShiftArg, LHSShiftArg)) {
9679 // (shl X, C1) | (srl (X | Y), C2) --> (rotl X, C1) | (srl Y, C2)
9680 SDValue RotX = DAG.getNode(ISD::ROTL, DL, VT, X, LHSShiftAmt);
9681 SDValue SrlY = DAG.getNode(ISD::SRL, DL, VT, Y, RHSShiftAmt);
9682 Res = DAG.getNode(ISD::OR, DL, VT, RotX, SrlY);
9683 } else {
9684 return SDValue();
9685 }
9686
9687 return ApplyMasks(Res);
9688 }
9689
9690 return SDValue(); // Requires funnel shift support.
9691 }
9692
9693 // fold (or/add (shl x, C1), (srl x, C2)) -> (rotl x, C1)
9694 // fold (or/add (shl x, C1), (srl x, C2)) -> (rotr x, C2)
9695 // fold (or/add (shl x, C1), (srl y, C2)) -> (fshl x, y, C1)
9696 // fold (or/add (shl x, C1), (srl y, C2)) -> (fshr x, y, C2)
9697 // iff C1+C2 == EltSizeInBits
9698 if (ISD::matchBinaryPredicate(LHSShiftAmt, RHSShiftAmt, MatchRotateSum)) {
9699 SDValue Res;
9700 if (IsRotate && (HasROTL || HasROTR || !(HasFSHL || HasFSHR))) {
9701 bool UseROTL = !LegalOperations || HasROTL;
9702 Res = DAG.getNode(UseROTL ? ISD::ROTL : ISD::ROTR, DL, VT, LHSShiftArg,
9703 UseROTL ? LHSShiftAmt : RHSShiftAmt);
9704 } else {
9705 bool UseFSHL = !LegalOperations || HasFSHL;
9706 Res = DAG.getNode(UseFSHL ? ISD::FSHL : ISD::FSHR, DL, VT, LHSShiftArg,
9707 RHSShiftArg, UseFSHL ? LHSShiftAmt : RHSShiftAmt);
9708 }
9709
9710 return ApplyMasks(Res);
9711 }
9712
9713 // Even pre-legalization, we can't easily rotate/funnel-shift by a variable
9714 // shift.
9715 if (!HasROTL && !HasROTR && !HasFSHL && !HasFSHR)
9716 return SDValue();
9717
9718 // If there is a mask here, and we have a variable shift, we can't be sure
9719 // that we're masking out the right stuff.
9720 if (LHSMask.getNode() || RHSMask.getNode())
9721 return SDValue();
9722
9723 // If the shift amount is sign/zext/any-extended just peel it off.
9724 SDValue LExtOp0 = LHSShiftAmt;
9725 SDValue RExtOp0 = RHSShiftAmt;
9726 if ((LHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND ||
9727 LHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND ||
9728 LHSShiftAmt.getOpcode() == ISD::ANY_EXTEND ||
9729 LHSShiftAmt.getOpcode() == ISD::TRUNCATE) &&
9730 (RHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND ||
9731 RHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND ||
9732 RHSShiftAmt.getOpcode() == ISD::ANY_EXTEND ||
9733 RHSShiftAmt.getOpcode() == ISD::TRUNCATE)) {
9734 LExtOp0 = LHSShiftAmt.getOperand(0);
9735 RExtOp0 = RHSShiftAmt.getOperand(0);
9736 }
9737
9738 if (IsRotate && (HasROTL || HasROTR)) {
9739 if (SDValue TryL = MatchRotatePosNeg(LHSShiftArg, LHSShiftAmt, RHSShiftAmt,
9740 LExtOp0, RExtOp0, FromAdd, HasROTL,
9742 return TryL;
9743
9744 if (SDValue TryR = MatchRotatePosNeg(RHSShiftArg, RHSShiftAmt, LHSShiftAmt,
9745 RExtOp0, LExtOp0, FromAdd, HasROTR,
9747 return TryR;
9748 }
9749
9750 if (SDValue TryL = MatchFunnelPosNeg(LHSShiftArg, RHSShiftArg, LHSShiftAmt,
9751 RHSShiftAmt, LExtOp0, RExtOp0, FromAdd,
9752 HasFSHL, ISD::FSHL, ISD::FSHR, DL))
9753 return TryL;
9754
9755 if (SDValue TryR = MatchFunnelPosNeg(LHSShiftArg, RHSShiftArg, RHSShiftAmt,
9756 LHSShiftAmt, RExtOp0, LExtOp0, FromAdd,
9757 HasFSHR, ISD::FSHR, ISD::FSHL, DL))
9758 return TryR;
9759
9760 return SDValue();
9761}
9762
9763/// Recursively traverses the expression calculating the origin of the requested
9764/// byte of the given value. Returns std::nullopt if the provider can't be
9765/// calculated.
9766///
9767/// For all the values except the root of the expression, we verify that the
9768/// value has exactly one use and if not then return std::nullopt. This way if
9769/// the origin of the byte is returned it's guaranteed that the values which
9770/// contribute to the byte are not used outside of this expression.
9771
9772/// However, there is a special case when dealing with vector loads -- we allow
9773/// more than one use if the load is a vector type. Since the values that
9774/// contribute to the byte ultimately come from the ExtractVectorElements of the
9775/// Load, we don't care if the Load has uses other than ExtractVectorElements,
9776/// because those operations are independent from the pattern to be combined.
9777/// For vector loads, we simply care that the ByteProviders are adjacent
9778/// positions of the same vector, and their index matches the byte that is being
9779/// provided. This is captured by the \p VectorIndex algorithm. \p VectorIndex
9780/// is the index used in an ExtractVectorElement, and \p StartingIndex is the
9781/// byte position we are trying to provide for the LoadCombine. If these do
9782/// not match, then we can not combine the vector loads. \p Index uses the
9783/// byte position we are trying to provide for and is matched against the
9784/// shl and load size. The \p Index algorithm ensures the requested byte is
9785/// provided for by the pattern, and the pattern does not over provide bytes.
9786///
9787///
9788/// The supported LoadCombine pattern for vector loads is as follows
9789/// or
9790/// / \
9791/// or shl
9792/// / \ |
9793/// or shl zext
9794/// / \ | |
9795/// shl zext zext EVE*
9796/// | | | |
9797/// zext EVE* EVE* LOAD
9798/// | | |
9799/// EVE* LOAD LOAD
9800/// |
9801/// LOAD
9802///
9803/// *ExtractVectorElement
9805
9806static std::optional<SDByteProvider>
9807calculateByteProvider(SDValue Op, unsigned Index, unsigned Depth,
9808 std::optional<uint64_t> VectorIndex,
9809 unsigned StartingIndex = 0,
9810 MutableArrayRef<uint8_t> ByteMask = {}) {
9811
9812 // Typical i64 by i8 pattern requires recursion up to 8 calls depth
9813 if (Depth == 10)
9814 return std::nullopt;
9815
9816 // Only allow multiple uses if the instruction is a vector load (in which
9817 // case we will use the load for every ExtractVectorElement)
9818 if (Depth && !Op.hasOneUse() &&
9819 (Op.getOpcode() != ISD::LOAD || !Op.getValueType().isVector()))
9820 return std::nullopt;
9821
9822 // Fail to combine if we have encountered anything but a LOAD after handling
9823 // an ExtractVectorElement.
9824 if (Op.getOpcode() != ISD::LOAD && VectorIndex.has_value())
9825 return std::nullopt;
9826
9827 unsigned BitWidth = Op.getScalarValueSizeInBits();
9828 if (BitWidth % 8 != 0)
9829 return std::nullopt;
9830 unsigned ByteWidth = BitWidth / 8;
9831 assert(Index < ByteWidth && "invalid index requested");
9832 (void) ByteWidth;
9833
9834 switch (Op.getOpcode()) {
9835 case ISD::OR: {
9836 auto LHS = calculateByteProvider(Op->getOperand(0), Index, Depth + 1,
9837 VectorIndex, StartingIndex, ByteMask);
9838 if (!LHS)
9839 return std::nullopt;
9840 auto RHS = calculateByteProvider(Op->getOperand(1), Index, Depth + 1,
9841 VectorIndex, StartingIndex, ByteMask);
9842 if (!RHS)
9843 return std::nullopt;
9844
9845 if (LHS->isConstantZero())
9846 return RHS;
9847 if (RHS->isConstantZero())
9848 return LHS;
9849 return std::nullopt;
9850 }
9851 case ISD::SHL: {
9852 auto ShiftOp = dyn_cast<ConstantSDNode>(Op->getOperand(1));
9853 if (!ShiftOp)
9854 return std::nullopt;
9855
9856 uint64_t BitShift = ShiftOp->getZExtValue();
9857
9858 if (BitShift % 8 != 0)
9859 return std::nullopt;
9860 uint64_t ByteShift = BitShift / 8;
9861
9862 // If we are shifting by an amount greater than the index we are trying to
9863 // provide, then do not provide anything. Otherwise, subtract the index by
9864 // the amount we shifted by.
9865 return Index < ByteShift
9867 : calculateByteProvider(Op->getOperand(0), Index - ByteShift,
9868 Depth + 1, VectorIndex, Index, ByteMask);
9869 }
9870 case ISD::ANY_EXTEND:
9871 case ISD::SIGN_EXTEND:
9872 case ISD::ZERO_EXTEND: {
9873 SDValue NarrowOp = Op->getOperand(0);
9874 unsigned NarrowBitWidth = NarrowOp.getScalarValueSizeInBits();
9875 if (NarrowBitWidth % 8 != 0)
9876 return std::nullopt;
9877 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
9878
9879 if (Index >= NarrowByteWidth)
9880 return Op.getOpcode() == ISD::ZERO_EXTEND
9881 ? std::optional<SDByteProvider>(
9883 : std::nullopt;
9884 return calculateByteProvider(NarrowOp, Index, Depth + 1, VectorIndex,
9885 StartingIndex, ByteMask);
9886 }
9887 case ISD::BSWAP:
9888 return calculateByteProvider(Op->getOperand(0), ByteWidth - Index - 1,
9889 Depth + 1, VectorIndex, StartingIndex,
9890 ByteMask);
9891 case ISD::AND: {
9892 // Constants are canonicalized to the RHS of AND, so only operand 1 needs
9893 // to be checked.
9894 auto *MaskOp = dyn_cast<ConstantSDNode>(Op->getOperand(1));
9895 if (!MaskOp)
9896 return std::nullopt;
9897
9898 uint8_t MaskByte =
9899 MaskOp->getAPIntValue().extractBitsAsZExtValue(8, Index * 8);
9900
9901 if (MaskByte == 0x00)
9903
9904 auto Result = calculateByteProvider(Op->getOperand(0), Index, Depth + 1,
9905 VectorIndex, StartingIndex, ByteMask);
9906 if (!Result)
9907 return std::nullopt;
9908
9909 // Only record the mask if this byte is actually provided (not zero).
9910 // A ConstantZero result may be discarded by the OR handler in favor of
9911 // the other operand, so writing the mask here would corrupt ByteMask.
9912 if (MaskByte != 0xFF && !ByteMask.empty() && !Result->isConstantZero())
9913 ByteMask[StartingIndex] &= MaskByte;
9914
9915 return Result;
9916 }
9918 auto OffsetOp = dyn_cast<ConstantSDNode>(Op->getOperand(1));
9919 if (!OffsetOp)
9920 return std::nullopt;
9921
9922 VectorIndex = OffsetOp->getZExtValue();
9923
9924 SDValue NarrowOp = Op->getOperand(0);
9925 unsigned NarrowBitWidth = NarrowOp.getScalarValueSizeInBits();
9926 if (NarrowBitWidth % 8 != 0)
9927 return std::nullopt;
9928 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
9929 // EXTRACT_VECTOR_ELT can extend the element type to the width of the return
9930 // type, leaving the high bits undefined.
9931 if (Index >= NarrowByteWidth)
9932 return std::nullopt;
9933
9934 // Check to see if the position of the element in the vector corresponds
9935 // with the byte we are trying to provide for. In the case of a vector of
9936 // i8, this simply means the VectorIndex == StartingIndex. For non i8 cases,
9937 // the element will provide a range of bytes. For example, if we have a
9938 // vector of i16s, each element provides two bytes (V[1] provides byte 2 and
9939 // 3).
9940 if (*VectorIndex * NarrowByteWidth > StartingIndex)
9941 return std::nullopt;
9942 if ((*VectorIndex + 1) * NarrowByteWidth <= StartingIndex)
9943 return std::nullopt;
9944
9945 return calculateByteProvider(Op->getOperand(0), Index, Depth + 1,
9946 VectorIndex, StartingIndex, ByteMask);
9947 }
9948 case ISD::LOAD: {
9949 auto L = cast<LoadSDNode>(Op.getNode());
9950 if (!L->isSimple() || L->isIndexed())
9951 return std::nullopt;
9952
9953 unsigned NarrowBitWidth = L->getMemoryVT().getScalarSizeInBits();
9954 if (NarrowBitWidth % 8 != 0)
9955 return std::nullopt;
9956 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
9957
9958 // If the width of the load does not reach byte we are trying to provide for
9959 // and it is not a ZEXTLOAD, then the load does not provide for the byte in
9960 // question
9961 if (Index >= NarrowByteWidth)
9962 return L->getExtensionType() == ISD::ZEXTLOAD
9963 ? std::optional<SDByteProvider>(
9965 : std::nullopt;
9966
9967 unsigned BPVectorIndex = VectorIndex.value_or(0U);
9968 return SDByteProvider::getSrc(L, Index, BPVectorIndex);
9969 }
9970 }
9971
9972 return std::nullopt;
9973}
9974
9975static unsigned littleEndianByteAt(unsigned BW, unsigned i) {
9976 return i;
9977}
9978
9979static unsigned bigEndianByteAt(unsigned BW, unsigned i) {
9980 return BW - i - 1;
9981}
9982
9983// Check if the bytes offsets we are looking at match with either big or
9984// little endian value loaded. Return true for big endian, false for little
9985// endian, and std::nullopt if match failed.
9986static std::optional<bool> isBigEndian(ArrayRef<int64_t> ByteOffsets,
9987 int64_t FirstOffset) {
9988 // The endian can be decided only when it is 2 bytes at least.
9989 unsigned Width = ByteOffsets.size();
9990 if (Width < 2)
9991 return std::nullopt;
9992
9993 bool BigEndian = true, LittleEndian = true;
9994 for (unsigned i = 0; i < Width; i++) {
9995 int64_t CurrentByteOffset = ByteOffsets[i] - FirstOffset;
9996 LittleEndian &= CurrentByteOffset == littleEndianByteAt(Width, i);
9997 BigEndian &= CurrentByteOffset == bigEndianByteAt(Width, i);
9998 if (!BigEndian && !LittleEndian)
9999 return std::nullopt;
10000 }
10001
10002 assert((BigEndian != LittleEndian) && "It should be either big endian or"
10003 "little endian");
10004 return BigEndian;
10005}
10006
10007// Look through one layer of truncate or extend.
10009 switch (Value.getOpcode()) {
10010 case ISD::TRUNCATE:
10011 case ISD::ZERO_EXTEND:
10012 case ISD::SIGN_EXTEND:
10013 case ISD::ANY_EXTEND:
10014 return Value.getOperand(0);
10015 }
10016 return SDValue();
10017}
10018
10019/// Match a pattern where a wide type scalar value is stored by several narrow
10020/// stores. Fold it into a single store or a BSWAP and a store if the targets
10021/// supports it.
10022///
10023/// Assuming little endian target:
10024/// i8 *p = ...
10025/// i32 val = ...
10026/// p[0] = (val >> 0) & 0xFF;
10027/// p[1] = (val >> 8) & 0xFF;
10028/// p[2] = (val >> 16) & 0xFF;
10029/// p[3] = (val >> 24) & 0xFF;
10030/// =>
10031/// *((i32)p) = val;
10032///
10033/// i8 *p = ...
10034/// i32 val = ...
10035/// p[0] = (val >> 24) & 0xFF;
10036/// p[1] = (val >> 16) & 0xFF;
10037/// p[2] = (val >> 8) & 0xFF;
10038/// p[3] = (val >> 0) & 0xFF;
10039/// =>
10040/// *((i32)p) = BSWAP(val);
10041SDValue DAGCombiner::mergeTruncStores(StoreSDNode *N) {
10042 // The matching looks for "store (trunc x)" patterns that appear early but are
10043 // likely to be replaced by truncating store nodes during combining.
10044 // TODO: If there is evidence that running this later would help, this
10045 // limitation could be removed. Legality checks may need to be added
10046 // for the created store and optional bswap/rotate.
10047 if (LegalOperations || OptLevel == CodeGenOptLevel::None)
10048 return SDValue();
10049
10050 // We only handle merging simple stores of 1-4 bytes.
10051 // TODO: Allow unordered atomics when wider type is legal (see D66309)
10052 EVT MemVT = N->getMemoryVT();
10053 if (!(MemVT == MVT::i8 || MemVT == MVT::i16 || MemVT == MVT::i32) ||
10054 !N->isSimple() || N->isIndexed())
10055 return SDValue();
10056
10057 // Collect all of the stores in the chain, upto the maximum store width (i64).
10058 SDValue Chain = N->getChain();
10060 unsigned NarrowNumBits = MemVT.getScalarSizeInBits();
10061 unsigned MaxWideNumBits = 64;
10062 unsigned MaxStores = MaxWideNumBits / NarrowNumBits;
10063 while (auto *Store = dyn_cast<StoreSDNode>(Chain)) {
10064 // All stores must be the same size to ensure that we are writing all of the
10065 // bytes in the wide value.
10066 // This store should have exactly one use as a chain operand for another
10067 // store in the merging set. If there are other chain uses, then the
10068 // transform may not be safe because order of loads/stores outside of this
10069 // set may not be preserved.
10070 // TODO: We could allow multiple sizes by tracking each stored byte.
10071 // Keep a differently-sized store in the incoming chain and try to merge
10072 // the stores collected after it.
10073 if (Store->getMemoryVT() != MemVT)
10074 break;
10075 if (!Store->isSimple() || Store->isIndexed() || !Store->hasOneUse())
10076 return SDValue();
10077 Stores.push_back(Store);
10078 Chain = Store->getChain();
10079 if (MaxStores < Stores.size())
10080 return SDValue();
10081 }
10082 // There is no reason to continue if we do not have at least a pair of stores.
10083 if (Stores.size() < 2)
10084 return SDValue();
10085
10086 // Handle simple types only.
10087 LLVMContext &Context = *DAG.getContext();
10088 unsigned NumStores = Stores.size();
10089 unsigned WideNumBits = NumStores * NarrowNumBits;
10090 if (WideNumBits != 16 && WideNumBits != 32 && WideNumBits != 64)
10091 return SDValue();
10092
10093 // Check if all bytes of the source value that we are looking at are stored
10094 // to the same base address. Collect offsets from Base address into OffsetMap.
10095 SDValue SourceValue;
10096 SmallVector<int64_t, 8> OffsetMap(NumStores, INT64_MAX);
10097 int64_t FirstOffset = INT64_MAX;
10098 StoreSDNode *FirstStore = nullptr;
10099 std::optional<BaseIndexOffset> Base;
10100 for (auto *Store : Stores) {
10101 // All the stores store different parts of the CombinedValue. A truncate is
10102 // required to get the partial value.
10103 SDValue Trunc = Store->getValue();
10104 if (Trunc.getOpcode() != ISD::TRUNCATE)
10105 return SDValue();
10106 // Other than the first/last part, a shift operation is required to get the
10107 // offset.
10108 int64_t Offset = 0;
10109 SDValue WideVal = Trunc.getOperand(0);
10110 if ((WideVal.getOpcode() == ISD::SRL || WideVal.getOpcode() == ISD::SRA) &&
10111 isa<ConstantSDNode>(WideVal.getOperand(1))) {
10112 // The shift amount must be a constant multiple of the narrow type.
10113 // It is translated to the offset address in the wide source value "y".
10114 //
10115 // x = srl y, ShiftAmtC
10116 // i8 z = trunc x
10117 // store z, ...
10118 uint64_t ShiftAmtC = WideVal.getConstantOperandVal(1);
10119 if (ShiftAmtC % NarrowNumBits != 0)
10120 return SDValue();
10121
10122 // Make sure we aren't reading bits that are shifted in.
10123 if (ShiftAmtC > WideVal.getScalarValueSizeInBits() - NarrowNumBits)
10124 return SDValue();
10125
10126 Offset = ShiftAmtC / NarrowNumBits;
10127 WideVal = WideVal.getOperand(0);
10128 }
10129
10130 // Stores must share the same source value with different offsets.
10131 if (!SourceValue)
10132 SourceValue = WideVal;
10133 else if (SourceValue != WideVal) {
10134 // Truncate and extends can be stripped to see if the values are related.
10135 if (stripTruncAndExt(SourceValue) != WideVal &&
10136 stripTruncAndExt(WideVal) != SourceValue)
10137 return SDValue();
10138
10139 if (WideVal.getScalarValueSizeInBits() >
10140 SourceValue.getScalarValueSizeInBits())
10141 SourceValue = WideVal;
10142
10143 // Give up if the source value type is smaller than the store size.
10144 if (SourceValue.getScalarValueSizeInBits() < WideNumBits)
10145 return SDValue();
10146 }
10147
10148 // Stores must share the same base address.
10149 BaseIndexOffset Ptr = BaseIndexOffset::match(Store, DAG);
10150 int64_t ByteOffsetFromBase = 0;
10151 if (!Base)
10152 Base = Ptr;
10153 else if (!Base->equalBaseIndex(Ptr, DAG, ByteOffsetFromBase))
10154 return SDValue();
10155
10156 // Remember the first store.
10157 if (ByteOffsetFromBase < FirstOffset) {
10158 FirstStore = Store;
10159 FirstOffset = ByteOffsetFromBase;
10160 }
10161 // Map the offset in the store and the offset in the combined value, and
10162 // early return if it has been set before.
10163 if (Offset < 0 || Offset >= NumStores || OffsetMap[Offset] != INT64_MAX)
10164 return SDValue();
10165 OffsetMap[Offset] = ByteOffsetFromBase;
10166 }
10167
10168 EVT WideVT = EVT::getIntegerVT(Context, WideNumBits);
10169
10170 assert(FirstOffset != INT64_MAX && "First byte offset must be set");
10171 assert(FirstStore && "First store must be set");
10172
10173 // Check that a store of the wide type is both allowed and fast on the target
10174 const DataLayout &Layout = DAG.getDataLayout();
10175 unsigned Fast = 0;
10176 bool Allowed = TLI.allowsMemoryAccess(Context, Layout, WideVT,
10177 *FirstStore->getMemOperand(), &Fast);
10178 if (!Allowed || !Fast)
10179 return SDValue();
10180
10181 // Check if the pieces of the value are going to the expected places in memory
10182 // to merge the stores.
10183 auto checkOffsets = [&](bool MatchLittleEndian) {
10184 if (MatchLittleEndian) {
10185 for (unsigned i = 0; i != NumStores; ++i)
10186 if (OffsetMap[i] != i * (NarrowNumBits / 8) + FirstOffset)
10187 return false;
10188 } else { // MatchBigEndian by reversing loop counter.
10189 for (unsigned i = 0, j = NumStores - 1; i != NumStores; ++i, --j)
10190 if (OffsetMap[j] != i * (NarrowNumBits / 8) + FirstOffset)
10191 return false;
10192 }
10193 return true;
10194 };
10195
10196 // Check if the offsets line up for the native data layout of this target.
10197 bool NeedBswap = false;
10198 bool NeedRotate = false;
10199 if (!checkOffsets(Layout.isLittleEndian())) {
10200 // Special-case: check if byte offsets line up for the opposite endian.
10201 if (NarrowNumBits == 8 && checkOffsets(Layout.isBigEndian()))
10202 NeedBswap = true;
10203 else if (NumStores == 2 && checkOffsets(Layout.isBigEndian()))
10204 NeedRotate = true;
10205 else
10206 return SDValue();
10207 }
10208
10209 SDLoc DL(N);
10210 if (WideVT != SourceValue.getValueType()) {
10211 assert(SourceValue.getValueType().getScalarSizeInBits() > WideNumBits &&
10212 "Unexpected store value to merge");
10213 SourceValue = DAG.getNode(ISD::TRUNCATE, DL, WideVT, SourceValue);
10214 }
10215
10216 // Before legalize we can introduce illegal bswaps/rotates which will be later
10217 // converted to an explicit bswap sequence. This way we end up with a single
10218 // store and byte shuffling instead of several stores and byte shuffling.
10219 if (NeedBswap) {
10220 SourceValue = DAG.getNode(ISD::BSWAP, DL, WideVT, SourceValue);
10221 } else if (NeedRotate) {
10222 assert(WideNumBits % 2 == 0 && "Unexpected type for rotate");
10223 SDValue RotAmt = DAG.getConstant(WideNumBits / 2, DL, WideVT);
10224 SourceValue = DAG.getNode(ISD::ROTR, DL, WideVT, SourceValue, RotAmt);
10225 }
10226
10227 const MDNode *MemCacheHint = getCommonMemCacheHint(Stores);
10228
10229 SDValue NewStore =
10230 DAG.getStore(Chain, DL, SourceValue, FirstStore->getBasePtr(),
10231 FirstStore->getPointerInfo(), FirstStore->getAlign(),
10233 MMOMetadata(AAMDNodes(), nullptr, MemCacheHint));
10234
10235 // Rely on other DAG combine rules to remove the other individual stores.
10236 DAG.ReplaceAllUsesWith(N, NewStore.getNode());
10237 return NewStore;
10238}
10239
10240/// Match a pattern where a wide type scalar value is loaded by several narrow
10241/// loads and combined by shifts and ors. Fold it into a single load or a load
10242/// and a BSWAP if the targets supports it.
10243///
10244/// Assuming little endian target:
10245/// i8 *a = ...
10246/// i32 val = a[0] | (a[1] << 8) | (a[2] << 16) | (a[3] << 24)
10247/// =>
10248/// i32 val = *((i32)a)
10249///
10250/// i8 *a = ...
10251/// i32 val = (a[0] << 24) | (a[1] << 16) | (a[2] << 8) | a[3]
10252/// =>
10253/// i32 val = BSWAP(*((i32)a))
10254///
10255/// TODO: This rule matches complex patterns with OR node roots and doesn't
10256/// interact well with the worklist mechanism. When a part of the pattern is
10257/// updated (e.g. one of the loads) its direct users are put into the worklist,
10258/// but the root node of the pattern which triggers the load combine is not
10259/// necessarily a direct user of the changed node. For example, once the address
10260/// of t28 load is reassociated load combine won't be triggered:
10261/// t25: i32 = add t4, Constant:i32<2>
10262/// t26: i64 = sign_extend t25
10263/// t27: i64 = add t2, t26
10264/// t28: i8,ch = load<LD1[%tmp9]> t0, t27, undef:i64
10265/// t29: i32 = zero_extend t28
10266/// t32: i32 = shl t29, Constant:i8<8>
10267/// t33: i32 = or t23, t32
10268/// As a possible fix visitLoad can check if the load can be a part of a load
10269/// combine pattern and add corresponding OR roots to the worklist.
10270SDValue DAGCombiner::MatchLoadCombine(SDNode *N) {
10271 assert(N->getOpcode() == ISD::OR &&
10272 "Can only match load combining against OR nodes");
10273
10274 // Handles simple types only
10275 EVT VT = N->getValueType(0);
10276 if (VT != MVT::i16 && VT != MVT::i32 && VT != MVT::i64)
10277 return SDValue();
10278 unsigned ByteWidth = VT.getSizeInBits() / 8;
10279
10280 bool IsBigEndianTarget = DAG.getDataLayout().isBigEndian();
10281 auto MemoryByteOffset = [&](SDByteProvider P) {
10282 assert(P.hasSrc() && "Must be a memory byte provider");
10283 auto *Load = cast<LoadSDNode>(P.Src.value());
10284
10285 unsigned LoadBitWidth = Load->getMemoryVT().getScalarSizeInBits();
10286
10287 assert(LoadBitWidth % 8 == 0 &&
10288 "can only analyze providers for individual bytes not bit");
10289 unsigned LoadByteWidth = LoadBitWidth / 8;
10290 return IsBigEndianTarget ? bigEndianByteAt(LoadByteWidth, P.DestOffset)
10291 : littleEndianByteAt(LoadByteWidth, P.DestOffset);
10292 };
10293
10294 std::optional<BaseIndexOffset> Base;
10295 SDValue Chain;
10296
10297 SmallPtrSet<LoadSDNode *, 8> Loads;
10298 std::optional<SDByteProvider> FirstByteProvider;
10299 int64_t FirstOffset = INT64_MAX;
10300
10301 // Check if all the bytes of the OR we are looking at are loaded from the same
10302 // base address. Collect bytes offsets from Base address in ByteOffsets.
10303 SmallVector<int64_t, 8> ByteOffsets(ByteWidth);
10304 SmallVector<uint8_t, 8> ByteMasks(ByteWidth, 0xFF);
10305 unsigned ZeroExtendedBytes = 0;
10306 for (int i = ByteWidth - 1; i >= 0; --i) {
10307 auto P =
10308 calculateByteProvider(SDValue(N, 0), i, 0, /*VectorIndex*/ std::nullopt,
10309 /*StartingIndex*/ i, ByteMasks);
10310 if (!P)
10311 return SDValue();
10312
10313 if (P->isConstantZero()) {
10314 // It's OK for the N most significant bytes to be 0, we can just
10315 // zero-extend the load.
10316 if (++ZeroExtendedBytes != (ByteWidth - static_cast<unsigned>(i)))
10317 return SDValue();
10318 continue;
10319 }
10320 assert(P->hasSrc() && "provenance should either be memory or zero");
10321 auto *L = cast<LoadSDNode>(P->Src.value());
10322
10323 // All loads must share the same chain
10324 SDValue LChain = L->getChain();
10325 if (!Chain)
10326 Chain = LChain;
10327 else if (Chain != LChain)
10328 return SDValue();
10329
10330 // Loads must share the same base address
10331 BaseIndexOffset Ptr = BaseIndexOffset::match(L, DAG);
10332 int64_t ByteOffsetFromBase = 0;
10333
10334 // For vector loads, the expected load combine pattern will have an
10335 // ExtractElement for each index in the vector. While each of these
10336 // ExtractElements will be accessing the same base address as determined
10337 // by the load instruction, the actual bytes they interact with will differ
10338 // due to different ExtractElement indices. To accurately determine the
10339 // byte position of an ExtractElement, we offset the base load ptr with
10340 // the index multiplied by the byte size of each element in the vector.
10341 if (L->getMemoryVT().isVector()) {
10342 unsigned LoadWidthInBit = L->getMemoryVT().getScalarSizeInBits();
10343 if (LoadWidthInBit % 8 != 0)
10344 return SDValue();
10345 unsigned ByteOffsetFromVector = P->SrcOffset * LoadWidthInBit / 8;
10346 Ptr.addToOffset(ByteOffsetFromVector);
10347 }
10348
10349 if (!Base)
10350 Base = Ptr;
10351
10352 else if (!Base->equalBaseIndex(Ptr, DAG, ByteOffsetFromBase))
10353 return SDValue();
10354
10355 // Calculate the offset of the current byte from the base address
10356 ByteOffsetFromBase += MemoryByteOffset(*P);
10357 ByteOffsets[i] = ByteOffsetFromBase;
10358
10359 // Remember the first byte load
10360 if (ByteOffsetFromBase < FirstOffset) {
10361 FirstByteProvider = P;
10362 FirstOffset = ByteOffsetFromBase;
10363 }
10364
10365 Loads.insert(L);
10366 }
10367
10368 assert(!Loads.empty() && "All the bytes of the value must be loaded from "
10369 "memory, so there must be at least one load which produces the value");
10370 assert(Base && "Base address of the accessed memory location must be set");
10371 assert(FirstOffset != INT64_MAX && "First byte offset must be set");
10372
10373 bool NeedsZext = ZeroExtendedBytes > 0;
10374
10375 EVT MemVT =
10376 EVT::getIntegerVT(*DAG.getContext(), (ByteWidth - ZeroExtendedBytes) * 8);
10377
10378 if (!MemVT.isSimple())
10379 return SDValue();
10380
10381 // Check if the bytes of the OR we are looking at match with either big or
10382 // little endian value load
10383 std::optional<bool> IsBigEndian = isBigEndian(
10384 ArrayRef(ByteOffsets).drop_back(ZeroExtendedBytes), FirstOffset);
10385 if (!IsBigEndian)
10386 return SDValue();
10387
10388 assert(FirstByteProvider && "must be set");
10389
10390 // Ensure that the first byte is loaded from zero offset of the first load.
10391 // So the combined value can be loaded from the first load address.
10392 if (MemoryByteOffset(*FirstByteProvider) != 0)
10393 return SDValue();
10394 auto *FirstLoad = cast<LoadSDNode>(FirstByteProvider->Src.value());
10395
10396 // Before legalization we allow introducing loads that are wider than legal,
10397 // which will later be split into legally sized loads. This enables us to
10398 // combine, for example, i8 loads forming an i64 into an i64 load, which get
10399 // then gets split up into couple of i32 loads on 32 bit targets.
10400 if (LegalOperations &&
10401 !TLI.isLoadLegal(VT, MemVT, FirstLoad->getAlign(),
10402 FirstLoad->getAddressSpace(),
10403 NeedsZext ? ISD::ZEXTLOAD : ISD::NON_EXTLOAD, false))
10404 return SDValue();
10405
10406 // The node we are looking at matches with the pattern, check if we can
10407 // replace it with a single (possibly zero-extended) load and bswap + shift if
10408 // needed.
10409
10410 // If the load needs byte swap check if the target supports it
10411 bool NeedsBswap = IsBigEndianTarget != *IsBigEndian;
10412
10413 // Before legalize we can introduce illegal bswaps which will be later
10414 // converted to an explicit bswap sequence. This way we end up with a single
10415 // load and byte shuffling instead of several loads and byte shuffling.
10416 // We do not introduce illegal bswaps when zero-extending as this tends to
10417 // introduce too many arithmetic instructions.
10418 if (NeedsBswap && (LegalOperations || NeedsZext) &&
10420 return SDValue();
10421
10422 // If we need to bswap and zero extend, we have to insert a shift. Check that
10423 // it is legal.
10424 if (NeedsBswap && NeedsZext && LegalOperations &&
10426 return SDValue();
10427
10428 // Check that a load of the wide type is both allowed and fast on the target
10429 unsigned Fast = 0;
10430 bool Allowed =
10431 TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), MemVT,
10432 *FirstLoad->getMemOperand(), &Fast);
10433 if (!Allowed || !Fast)
10434 return SDValue();
10435
10436 const MDNode *MemCacheHint = getCommonMemCacheHint(Loads);
10437
10438 SDValue NewLoad = DAG.getExtLoad(
10439 NeedsZext ? ISD::ZEXTLOAD : ISD::NON_EXTLOAD, SDLoc(N), VT, Chain,
10440 FirstLoad->getBasePtr(), FirstLoad->getPointerInfo(), MemVT,
10441 FirstLoad->getAlign(), MachineMemOperand::MONone,
10442 MMOMetadata(AAMDNodes(), nullptr, MemCacheHint));
10443
10444 // Transfer chain users from old loads to the new load.
10445 for (LoadSDNode *L : Loads)
10446 DAG.makeEquivalentMemoryOrdering(L, NewLoad);
10447
10448 // Apply combined mask if any bytes were partially masked by AND operations.
10449 bool HasPartialMask = false;
10450 uint64_t CombinedMask = 0;
10451 for (unsigned i = 0; i < ByteWidth; ++i) {
10452 CombinedMask |= (uint64_t)ByteMasks[i] << (i * 8);
10453 if (ByteMasks[i] != 0xFF)
10454 HasPartialMask = true;
10455 }
10456
10457 if (!NeedsBswap) {
10458 if (HasPartialMask)
10459 NewLoad = DAG.getNode(ISD::AND, SDLoc(N), VT, NewLoad,
10460 DAG.getConstant(CombinedMask, SDLoc(N), VT));
10461 return NewLoad;
10462 }
10463
10464 SDValue ShiftedLoad =
10465 NeedsZext ? DAG.getNode(ISD::SHL, SDLoc(N), VT, NewLoad,
10466 DAG.getShiftAmountConstant(ZeroExtendedBytes * 8,
10467 VT, SDLoc(N)))
10468 : NewLoad;
10469 SDValue Result = DAG.getNode(ISD::BSWAP, SDLoc(N), VT, ShiftedLoad);
10470
10471 // The mask is built in final-result byte order (ByteMasks[i] corresponds to
10472 // byte i of the result), so it is correct to apply after the bswap.
10473 if (HasPartialMask)
10474 Result = DAG.getNode(ISD::AND, SDLoc(N), VT, Result,
10475 DAG.getConstant(CombinedMask, SDLoc(N), VT));
10476
10477 return Result;
10478}
10479
10480// If the target has andn, bsl, or a similar bit-select instruction,
10481// we want to unfold masked merge, with canonical pattern of:
10482// | A | |B|
10483// ((x ^ y) & m) ^ y
10484// | D |
10485// Into:
10486// (x & m) | (y & ~m)
10487// If y is a constant, m is not a 'not', and the 'andn' does not work with
10488// immediates, we unfold into a different pattern:
10489// ~(~x & m) & (m | y)
10490// If x is a constant, m is a 'not', and the 'andn' does not work with
10491// immediates, we unfold into a different pattern:
10492// (x | ~m) & ~(~m & ~y)
10493// NOTE: we don't unfold the pattern if 'xor' is actually a 'not', because at
10494// the very least that breaks andnpd / andnps patterns, and because those
10495// patterns are simplified in IR and shouldn't be created in the DAG
10496SDValue DAGCombiner::unfoldMaskedMerge(SDNode *N) {
10497 assert(N->getOpcode() == ISD::XOR);
10498
10499 // Don't touch 'not' (i.e. where y = -1).
10500 if (isAllOnesOrAllOnesSplat(N->getOperand(1)))
10501 return SDValue();
10502
10503 EVT VT = N->getValueType(0);
10504
10505 // There are 3 commutable operators in the pattern,
10506 // so we have to deal with 8 possible variants of the basic pattern.
10507 SDValue X, Y, M;
10508 auto matchAndXor = [&X, &Y, &M](SDValue And, unsigned XorIdx, SDValue Other) {
10509 if (And.getOpcode() != ISD::AND || !And.hasOneUse())
10510 return false;
10511 SDValue Xor = And.getOperand(XorIdx);
10512 if (Xor.getOpcode() != ISD::XOR || !Xor.hasOneUse())
10513 return false;
10514 SDValue Xor0 = Xor.getOperand(0);
10515 SDValue Xor1 = Xor.getOperand(1);
10516 // Don't touch 'not' (i.e. where y = -1).
10517 if (isAllOnesOrAllOnesSplat(Xor1))
10518 return false;
10519 if (Other == Xor0)
10520 std::swap(Xor0, Xor1);
10521 if (Other != Xor1)
10522 return false;
10523 X = Xor0;
10524 Y = Xor1;
10525 M = And.getOperand(XorIdx ? 0 : 1);
10526 return true;
10527 };
10528
10529 SDValue N0 = N->getOperand(0);
10530 SDValue N1 = N->getOperand(1);
10531 if (!matchAndXor(N0, 0, N1) && !matchAndXor(N0, 1, N1) &&
10532 !matchAndXor(N1, 0, N0) && !matchAndXor(N1, 1, N0))
10533 return SDValue();
10534
10535 // Don't do anything if the mask is constant. This should not be reachable.
10536 // InstCombine should have already unfolded this pattern, and DAGCombiner
10537 // probably shouldn't produce it, too.
10538 if (isa<ConstantSDNode>(M.getNode()))
10539 return SDValue();
10540
10541 // We can transform if the target has AndNot
10542 if (!TLI.hasAndNot(M))
10543 return SDValue();
10544
10545 SDLoc DL(N);
10546
10547 // If Y is a constant, check that 'andn' works with immediates. Unless M is
10548 // a bitwise not that would already allow ANDN to be used.
10549 if (!TLI.hasAndNot(Y) && !isBitwiseNot(M)) {
10550 assert(TLI.hasAndNot(X) && "Only mask is a variable? Unreachable.");
10551 // If not, we need to do a bit more work to make sure andn is still used.
10552 SDValue NotX = DAG.getNOT(DL, X, VT);
10553 SDValue LHS = DAG.getNode(ISD::AND, DL, VT, NotX, M);
10554 SDValue NotLHS = DAG.getNOT(DL, LHS, VT);
10555 SDValue RHS = DAG.getNode(ISD::OR, DL, VT, M, Y);
10556 return DAG.getNode(ISD::AND, DL, VT, NotLHS, RHS);
10557 }
10558
10559 // If X is a constant and M is a bitwise not, check that 'andn' works with
10560 // immediates.
10561 if (!TLI.hasAndNot(X) && isBitwiseNot(M)) {
10562 assert(TLI.hasAndNot(Y) && "Only mask is a variable? Unreachable.");
10563 // If not, we need to do a bit more work to make sure andn is still used.
10564 SDValue NotM = M.getOperand(0);
10565 SDValue LHS = DAG.getNode(ISD::OR, DL, VT, X, NotM);
10566 SDValue NotY = DAG.getNOT(DL, Y, VT);
10567 SDValue RHS = DAG.getNode(ISD::AND, DL, VT, NotM, NotY);
10568 SDValue NotRHS = DAG.getNOT(DL, RHS, VT);
10569 return DAG.getNode(ISD::AND, DL, VT, LHS, NotRHS);
10570 }
10571
10572 SDValue LHS = DAG.getNode(ISD::AND, DL, VT, X, M);
10573 SDValue NotM = DAG.getNOT(DL, M, VT);
10574 SDValue RHS = DAG.getNode(ISD::AND, DL, VT, Y, NotM);
10575
10576 return DAG.getNode(ISD::OR, DL, VT, LHS, RHS);
10577}
10578
10579SDValue DAGCombiner::visitXOR(SDNode *N) {
10580 SDValue N0 = N->getOperand(0);
10581 SDValue N1 = N->getOperand(1);
10582 EVT VT = N0.getValueType();
10583 SDLoc DL(N);
10584
10585 // fold (xor undef, undef) -> 0. This is a common idiom (misuse).
10586 if (N0.isUndef() && N1.isUndef())
10587 return DAG.getConstant(0, DL, VT);
10588
10589 // fold (xor x, undef) -> undef
10590 if (N0.isUndef())
10591 return N0;
10592 if (N1.isUndef())
10593 return N1;
10594
10595 // fold (xor c1, c2) -> c1^c2
10596 if (SDValue C = DAG.FoldConstantArithmetic(ISD::XOR, DL, VT, {N0, N1}))
10597 return C;
10598
10599 // canonicalize constant to RHS
10602 return DAG.getNode(ISD::XOR, DL, VT, N1, N0);
10603
10604 // fold vector ops
10605 if (VT.isVector()) {
10606 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
10607 return FoldedVOp;
10608
10609 // fold (xor x, 0) -> x, vector edition
10611 return N0;
10612 }
10613
10614 // fold (xor x, 0) -> x
10615 if (isNullConstant(N1))
10616 return N0;
10617
10618 if (SDValue NewSel = foldBinOpIntoSelect(N))
10619 return NewSel;
10620
10621 // reassociate xor
10622 if (SDValue RXOR = reassociateOps(ISD::XOR, DL, N0, N1, N->getFlags()))
10623 return RXOR;
10624
10625 // Fold xor(vecreduce(x), vecreduce(y)) -> vecreduce(xor(x, y))
10626 if (SDValue SD =
10627 reassociateReduction(ISD::VECREDUCE_XOR, ISD::XOR, DL, VT, N0, N1))
10628 return SD;
10629
10630 // fold (a^b) -> (a|b) iff a and b share no bits.
10631 if ((!LegalOperations || TLI.isOperationLegal(ISD::OR, VT)) &&
10632 DAG.haveNoCommonBitsSet(N0, N1))
10633 return DAG.getNode(ISD::OR, DL, VT, N0, N1, SDNodeFlags::Disjoint);
10634
10635 // look for 'add-like' folds:
10636 // XOR(N0,MIN_SIGNED_VALUE) == ADD(N0,MIN_SIGNED_VALUE)
10637 if ((!LegalOperations || TLI.isOperationLegal(ISD::ADD, VT)) &&
10639 if (SDValue Combined = visitADDLike(N))
10640 return Combined;
10641
10642 // fold not (setcc x, y, cc) -> setcc x y !cc
10643 // Avoid breaking: and (not(setcc x, y, cc), z) -> andn for vec
10644 unsigned N0Opcode = N0.getOpcode();
10645 SDValue LHS, RHS, CC;
10646 if (TLI.isConstTrueVal(N1) &&
10647 isSetCCEquivalent(N0, LHS, RHS, CC, /*MatchStrict*/ true) &&
10648 !(VT.isVector() && TLI.hasAndNot(SDValue(N, 0)) && N->hasOneUse() &&
10649 N->use_begin()->getUser()->getOpcode() == ISD::AND)) {
10651 LHS.getValueType());
10652 if (!LegalOperations ||
10653 TLI.isCondCodeLegal(NotCC, LHS.getSimpleValueType())) {
10654 // Propagate fast-math-flags.
10655 SDNodeFlags Flags = N0->getFlags();
10656 switch (N0Opcode) {
10657 default:
10658 llvm_unreachable("Unhandled SetCC Equivalent!");
10659 case ISD::SETCC:
10660 return DAG.getSetCC(SDLoc(N0), VT, LHS, RHS, NotCC, SDValue(),
10661 /*IsSignaling=*/false, Flags);
10662 case ISD::SELECT_CC:
10663 return DAG.getSelectCC(SDLoc(N0), LHS, RHS, N0.getOperand(2),
10664 N0.getOperand(3), NotCC, Flags);
10665 case ISD::STRICT_FSETCC:
10666 case ISD::STRICT_FSETCCS: {
10667 if (N0.hasOneUse()) {
10668 // FIXME Can we handle multiple uses? Could we token factor the chain
10669 // results from the new/old setcc?
10670 SDValue SetCC =
10671 DAG.getSetCC(SDLoc(N0), VT, LHS, RHS, NotCC, N0.getOperand(0),
10672 N0Opcode == ISD::STRICT_FSETCCS, Flags);
10673 CombineTo(N, SetCC);
10674 DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), SetCC.getValue(1));
10675 recursivelyDeleteUnusedNodes(N0.getNode());
10676 return SDValue(N, 0); // Return N so it doesn't get rechecked!
10677 }
10678 break;
10679 }
10680 }
10681 }
10682 }
10683
10684 // fold (not (zext (setcc x, y))) -> (zext (not (setcc x, y)))
10685 if (isOneConstant(N1) && N0Opcode == ISD::ZERO_EXTEND && N0.hasOneUse() &&
10686 isSetCCEquivalent(N0.getOperand(0), LHS, RHS, CC)){
10687 SDValue V = N0.getOperand(0);
10688 SDLoc DL0(N0);
10689 V = DAG.getNode(ISD::XOR, DL0, V.getValueType(), V,
10690 DAG.getConstant(1, DL0, V.getValueType()));
10691 AddToWorklist(V.getNode());
10692 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, V);
10693 }
10694
10695 // fold (not (or x, y)) -> (and (not x), (not y)) iff x or y are setcc
10696 // fold (not (and x, y)) -> (or (not x), (not y)) iff x or y are setcc
10697 if (isOneConstant(N1) && VT == MVT::i1 && N0.hasOneUse() &&
10698 (N0Opcode == ISD::OR || N0Opcode == ISD::AND)) {
10699 SDValue N00 = N0.getOperand(0), N01 = N0.getOperand(1);
10700 if (isOneUseSetCC(N01) || isOneUseSetCC(N00)) {
10701 unsigned NewOpcode = N0Opcode == ISD::AND ? ISD::OR : ISD::AND;
10702 N00 = DAG.getNode(ISD::XOR, SDLoc(N00), VT, N00, N1); // N00 = ~N00
10703 N01 = DAG.getNode(ISD::XOR, SDLoc(N01), VT, N01, N1); // N01 = ~N01
10704 AddToWorklist(N00.getNode()); AddToWorklist(N01.getNode());
10705 return DAG.getNode(NewOpcode, DL, VT, N00, N01);
10706 }
10707 }
10708 // fold (not (or x, y)) -> (and (not x), (not y)) iff x or y are constants
10709 // fold (not (and x, y)) -> (or (not x), (not y)) iff x or y are constants
10710 if (isAllOnesConstant(N1) && N0.hasOneUse() &&
10711 (N0Opcode == ISD::OR || N0Opcode == ISD::AND)) {
10712 SDValue N00 = N0.getOperand(0), N01 = N0.getOperand(1);
10713 if (isa<ConstantSDNode>(N01) || isa<ConstantSDNode>(N00)) {
10714 unsigned NewOpcode = N0Opcode == ISD::AND ? ISD::OR : ISD::AND;
10715 N00 = DAG.getNode(ISD::XOR, SDLoc(N00), VT, N00, N1); // N00 = ~N00
10716 N01 = DAG.getNode(ISD::XOR, SDLoc(N01), VT, N01, N1); // N01 = ~N01
10717 AddToWorklist(N00.getNode()); AddToWorklist(N01.getNode());
10718 return DAG.getNode(NewOpcode, DL, VT, N00, N01);
10719 }
10720 }
10721
10722 // fold (not (sub Y, X)) -> (add X, ~Y) if Y is a constant
10723 if (N0.getOpcode() == ISD::SUB && isAllOnesConstant(N1)) {
10724 SDValue Y = N0.getOperand(0);
10725 SDValue X = N0.getOperand(1);
10726
10727 if (auto *YConst = dyn_cast<ConstantSDNode>(Y)) {
10728 APInt NotYValue = ~YConst->getAPIntValue();
10729 SDValue NotY = DAG.getConstant(NotYValue, DL, VT);
10730 return DAG.getNode(ISD::ADD, DL, VT, X, NotY, N->getFlags());
10731 }
10732 }
10733
10734 // fold (not (add X, -1)) -> (neg X)
10735 if (N0.getOpcode() == ISD::ADD && N0.hasOneUse() && isAllOnesConstant(N1) &&
10737 return DAG.getNegative(N0.getOperand(0), DL, VT);
10738 }
10739
10740 // fold (xor (and x, y), y) -> (and (not x), y)
10741 if (N0Opcode == ISD::AND && N0.hasOneUse() && N0->getOperand(1) == N1) {
10742 SDValue X = N0.getOperand(0);
10743 SDValue NotX = DAG.getNOT(SDLoc(X), X, VT);
10744 AddToWorklist(NotX.getNode());
10745 return DAG.getNode(ISD::AND, DL, VT, NotX, N1);
10746 }
10747
10748 // fold Y = sra (X, size(X)-1); xor (add (X, Y), Y) -> (abs X)
10749 if (!LegalOperations || hasOperation(ISD::ABS, VT)) {
10750 SDValue A = N0Opcode == ISD::ADD ? N0 : N1;
10751 SDValue S = N0Opcode == ISD::SRA ? N0 : N1;
10752 if (A.getOpcode() == ISD::ADD && S.getOpcode() == ISD::SRA) {
10753 SDValue A0 = A.getOperand(0), A1 = A.getOperand(1);
10754 SDValue S0 = S.getOperand(0);
10755 if ((A0 == S && A1 == S0) || (A1 == S && A0 == S0))
10756 if (ConstantSDNode *C = isConstOrConstSplat(S.getOperand(1)))
10757 if (C->getAPIntValue() == (VT.getScalarSizeInBits() - 1))
10758 return DAG.getNode(ISD::ABS, DL, VT, S0);
10759 }
10760 }
10761
10762 // fold (xor x, x) -> 0
10763 if (N0 == N1)
10764 return tryFoldToZero(DL, TLI, VT, DAG, LegalOperations);
10765
10766 // fold (xor (shl 1, x), -1) -> (rotl ~1, x)
10767 // Here is a concrete example of this equivalence:
10768 // i16 x == 14
10769 // i16 shl == 1 << 14 == 16384 == 0b0100000000000000
10770 // i16 xor == ~(1 << 14) == 49151 == 0b1011111111111111
10771 //
10772 // =>
10773 //
10774 // i16 ~1 == 0b1111111111111110
10775 // i16 rol(~1, 14) == 0b1011111111111111
10776 //
10777 // Some additional tips to help conceptualize this transform:
10778 // - Try to see the operation as placing a single zero in a value of all ones.
10779 // - There exists no value for x which would allow the result to contain zero.
10780 // - Values of x larger than the bitwidth are undefined and do not require a
10781 // consistent result.
10782 // - Pushing the zero left requires shifting one bits in from the right.
10783 // A rotate left of ~1 is a nice way of achieving the desired result.
10784 if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT) && N0Opcode == ISD::SHL &&
10786 return DAG.getNode(ISD::ROTL, DL, VT, DAG.getSignedConstant(~1, DL, VT),
10787 N0.getOperand(1));
10788 }
10789
10790 // Simplify: xor (op x...), (op y...) -> (op (xor x, y))
10791 if (N0Opcode == N1.getOpcode())
10792 if (SDValue V = hoistLogicOpWithSameOpcodeHands(N))
10793 return V;
10794
10795 if (SDValue R = foldLogicOfShifts(N, N0, N1, DAG))
10796 return R;
10797 if (SDValue R = foldLogicOfShifts(N, N1, N0, DAG))
10798 return R;
10799 if (SDValue R = foldLogicTreeOfShifts(N, N0, N1, DAG))
10800 return R;
10801
10802 // Unfold ((x ^ y) & m) ^ y into (x & m) | (y & ~m) if profitable
10803 if (SDValue MM = unfoldMaskedMerge(N))
10804 return MM;
10805
10806 // Simplify the expression using non-local knowledge.
10808 return SDValue(N, 0);
10809
10810 if (SDValue Combined = combineCarryDiamond(DAG, TLI, N0, N1, N))
10811 return Combined;
10812
10813 // fold (xor (smin(x, C), C)) -> select (x < C), xor(x, C), 0
10814 // fold (xor (smax(x, C), C)) -> select (x > C), xor(x, C), 0
10815 // fold (xor (umin(x, C), C)) -> select (x < C), xor(x, C), 0
10816 // fold (xor (umax(x, C), C)) -> select (x > C), xor(x, C), 0
10817 SDValue Op0;
10818 if (sd_match(N0, m_OneUse(m_AnyOf(m_SMin(m_Value(Op0), m_Specific(N1)),
10819 m_SMax(m_Value(Op0), m_Specific(N1)),
10820 m_UMin(m_Value(Op0), m_Specific(N1)),
10821 m_UMax(m_Value(Op0), m_Specific(N1)))))) {
10822
10823 if (isa<ConstantSDNode>(N1) ||
10825 // For vectors, only optimize when the constant is zero or all-ones to
10826 // avoid generating more instructions
10827 if (VT.isVector()) {
10828 ConstantSDNode *N1C = isConstOrConstSplat(N1);
10829 if (!N1C || (!N1C->isZero() && !N1C->isAllOnes()))
10830 return SDValue();
10831 }
10832
10833 // Avoid the fold if the minmax operation is legal and select is expensive
10834 if (TLI.isOperationLegal(N0.getOpcode(), VT) &&
10836 return SDValue();
10837
10838 EVT CCVT = getSetCCResultType(VT);
10840 switch (N0.getOpcode()) {
10841 case ISD::SMIN:
10842 CC = ISD::SETLT;
10843 break;
10844 case ISD::SMAX:
10845 CC = ISD::SETGT;
10846 break;
10847 case ISD::UMIN:
10848 CC = ISD::SETULT;
10849 break;
10850 case ISD::UMAX:
10851 CC = ISD::SETUGT;
10852 break;
10853 }
10854 SDValue FN1 = DAG.getFreeze(N1);
10855 SDValue Cmp = DAG.getSetCC(DL, CCVT, Op0, FN1, CC);
10856 SDValue XorXC = DAG.getNode(ISD::XOR, DL, VT, Op0, FN1);
10857 SDValue Zero = DAG.getConstant(0, DL, VT);
10858 return DAG.getSelect(DL, VT, Cmp, XorXC, Zero);
10859 }
10860 }
10861
10862 return SDValue();
10863}
10864
10865/// If we have a shift-by-constant of a bitwise logic op that itself has a
10866/// shift-by-constant operand with identical opcode, we may be able to convert
10867/// that into 2 independent shifts followed by the logic op. This is a
10868/// throughput improvement.
10870 // Match a one-use bitwise logic op.
10871 SDValue LogicOp = Shift->getOperand(0);
10872 if (!LogicOp.hasOneUse())
10873 return SDValue();
10874
10875 unsigned LogicOpcode = LogicOp.getOpcode();
10876 if (LogicOpcode != ISD::AND && LogicOpcode != ISD::OR &&
10877 LogicOpcode != ISD::XOR)
10878 return SDValue();
10879
10880 // Find a matching one-use shift by constant.
10881 unsigned ShiftOpcode = Shift->getOpcode();
10882 SDValue C1 = Shift->getOperand(1);
10884 assert(C1Node && "Expected a shift with constant operand");
10885 const APInt &C1Val = C1Node->getAPIntValue();
10886 auto matchFirstShift = [&](SDValue V, SDValue &ShiftOp,
10887 const APInt *&ShiftAmtVal) {
10888 if (V.getOpcode() != ShiftOpcode || !V.hasOneUse())
10889 return false;
10890
10891 ConstantSDNode *ShiftCNode = isConstOrConstSplat(V.getOperand(1));
10892 if (!ShiftCNode)
10893 return false;
10894
10895 // Capture the shifted operand and shift amount value.
10896 ShiftOp = V.getOperand(0);
10897 ShiftAmtVal = &ShiftCNode->getAPIntValue();
10898
10899 // Shift amount types do not have to match their operand type, so check that
10900 // the constants are the same width.
10901 if (ShiftAmtVal->getBitWidth() != C1Val.getBitWidth())
10902 return false;
10903
10904 // The fold is not valid if the sum of the shift values doesn't fit in the
10905 // given shift amount type.
10906 bool Overflow = false;
10907 APInt NewShiftAmt = C1Val.uadd_ov(*ShiftAmtVal, Overflow);
10908 if (Overflow)
10909 return false;
10910
10911 // The fold is not valid if the sum of the shift values exceeds bitwidth.
10912 if (NewShiftAmt.uge(V.getScalarValueSizeInBits()))
10913 return false;
10914
10915 return true;
10916 };
10917
10918 // Logic ops are commutative, so check each operand for a match.
10919 SDValue X, Y;
10920 const APInt *C0Val;
10921 if (matchFirstShift(LogicOp.getOperand(0), X, C0Val))
10922 Y = LogicOp.getOperand(1);
10923 else if (matchFirstShift(LogicOp.getOperand(1), X, C0Val))
10924 Y = LogicOp.getOperand(0);
10925 else
10926 return SDValue();
10927
10928 // shift (logic (shift X, C0), Y), C1 -> logic (shift X, C0+C1), (shift Y, C1)
10929 SDLoc DL(Shift);
10930 EVT VT = Shift->getValueType(0);
10931 EVT ShiftAmtVT = Shift->getOperand(1).getValueType();
10932 SDValue ShiftSumC = DAG.getConstant(*C0Val + C1Val, DL, ShiftAmtVT);
10933 SDValue NewShift1 = DAG.getNode(ShiftOpcode, DL, VT, X, ShiftSumC);
10934 SDValue NewShift2 = DAG.getNode(ShiftOpcode, DL, VT, Y, C1);
10935 return DAG.getNode(LogicOpcode, DL, VT, NewShift1, NewShift2,
10936 LogicOp->getFlags());
10937}
10938
10939/// Handle transforms common to the three shifts, when the shift amount is a
10940/// constant.
10941/// We are looking for: (shift being one of shl/sra/srl)
10942/// shift (binop X, C0), C1
10943/// And want to transform into:
10944/// binop (shift X, C1), (shift C0, C1)
10945SDValue DAGCombiner::visitShiftByConstant(SDNode *N) {
10946 assert(isConstOrConstSplat(N->getOperand(1)) && "Expected constant operand");
10947
10948 // Do not turn a 'not' into a regular xor.
10949 if (isBitwiseNot(N->getOperand(0)))
10950 return SDValue();
10951
10952 // The inner binop must be one-use, since we want to replace it.
10953 SDValue LHS = N->getOperand(0);
10954 if (!LHS.hasOneUse() || !TLI.isDesirableToCommuteWithShift(N, Level))
10955 return SDValue();
10956
10957 // Fold shift(bitop(shift(x,c1),y), c2) -> bitop(shift(x,c1+c2),shift(y,c2)).
10958 if (SDValue R = combineShiftOfShiftedLogic(N, DAG))
10959 return R;
10960
10961 // We want to pull some binops through shifts, so that we have (and (shift))
10962 // instead of (shift (and)), likewise for add, or, xor, etc. This sort of
10963 // thing happens with address calculations, so it's important to canonicalize
10964 // it.
10965 switch (LHS.getOpcode()) {
10966 default:
10967 return SDValue();
10968 case ISD::OR:
10969 case ISD::XOR:
10970 case ISD::AND:
10971 break;
10972 case ISD::ADD:
10973 if (N->getOpcode() != ISD::SHL)
10974 return SDValue(); // only shl(add) not sr[al](add).
10975 break;
10976 }
10977
10978 // FIXME: disable this unless the input to the binop is a shift by a constant
10979 // or is copy/select. Enable this in other cases when figure out it's exactly
10980 // profitable.
10981 SDValue BinOpLHSVal = LHS.getOperand(0);
10982 bool IsShiftByConstant = (BinOpLHSVal.getOpcode() == ISD::SHL ||
10983 BinOpLHSVal.getOpcode() == ISD::SRA ||
10984 BinOpLHSVal.getOpcode() == ISD::SRL) &&
10985 isa<ConstantSDNode>(BinOpLHSVal.getOperand(1));
10986 bool IsCopyOrSelect = BinOpLHSVal.getOpcode() == ISD::CopyFromReg ||
10987 BinOpLHSVal.getOpcode() == ISD::SELECT;
10988
10989 if (!IsShiftByConstant && !IsCopyOrSelect)
10990 return SDValue();
10991
10992 if (IsCopyOrSelect && N->hasOneUse())
10993 return SDValue();
10994
10995 // Attempt to fold the constants, shifting the binop RHS by the shift amount.
10996 SDLoc DL(N);
10997 EVT VT = N->getValueType(0);
10998 if (SDValue NewRHS = DAG.FoldConstantArithmetic(
10999 N->getOpcode(), DL, VT, {LHS.getOperand(1), N->getOperand(1)})) {
11000 SDValue NewShift = DAG.getNode(N->getOpcode(), DL, VT, LHS.getOperand(0),
11001 N->getOperand(1));
11002 return DAG.getNode(LHS.getOpcode(), DL, VT, NewShift, NewRHS);
11003 }
11004
11005 return SDValue();
11006}
11007
11008SDValue DAGCombiner::distributeTruncateThroughAnd(SDNode *N) {
11009 assert(N->getOpcode() == ISD::TRUNCATE);
11010 assert(N->getOperand(0).getOpcode() == ISD::AND);
11011
11012 // (truncate:TruncVT (and N00, N01C)) -> (and (truncate:TruncVT N00), TruncC)
11013 EVT TruncVT = N->getValueType(0);
11014 if (N->hasOneUse() && N->getOperand(0).hasOneUse() &&
11015 TLI.isTypeDesirableForOp(ISD::AND, TruncVT)) {
11016 SDValue N01 = N->getOperand(0).getOperand(1);
11017 if (isConstantOrConstantVector(N01, /* NoOpaques */ true)) {
11018 SDLoc DL(N);
11019 SDValue N00 = N->getOperand(0).getOperand(0);
11020 SDValue Trunc00 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N00);
11021 SDValue Trunc01 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N01);
11022 AddToWorklist(Trunc00.getNode());
11023 AddToWorklist(Trunc01.getNode());
11024 return DAG.getNode(ISD::AND, DL, TruncVT, Trunc00, Trunc01);
11025 }
11026 }
11027
11028 return SDValue();
11029}
11030
11031SDValue DAGCombiner::visitRotate(SDNode *N) {
11032 SDLoc dl(N);
11033 SDValue N0 = N->getOperand(0);
11034 SDValue N1 = N->getOperand(1);
11035 EVT VT = N->getValueType(0);
11036 unsigned Bitsize = VT.getScalarSizeInBits();
11037
11038 // fold (rot x, 0) -> x
11039 if (isNullOrNullSplat(N1))
11040 return N0;
11041
11042 // fold (rot x, c) -> x iff (c % BitSize) == 0
11043 if (isPowerOf2_32(Bitsize) && Bitsize > 1) {
11044 APInt ModuloMask(N1.getScalarValueSizeInBits(), Bitsize - 1);
11045 if (DAG.MaskedValueIsZero(N1, ModuloMask))
11046 return N0;
11047 }
11048
11049 // fold (rot x, c) -> (rot x, c % BitSize)
11050 bool OutOfRange = false;
11051 auto MatchOutOfRange = [Bitsize, &OutOfRange](ConstantSDNode *C) {
11052 OutOfRange |= C->getAPIntValue().uge(Bitsize);
11053 return true;
11054 };
11055 if (ISD::matchUnaryPredicate(N1, MatchOutOfRange) && OutOfRange) {
11056 EVT AmtVT = N1.getValueType();
11057 SDValue Bits = DAG.getConstant(Bitsize, dl, AmtVT);
11058 if (SDValue Amt =
11059 DAG.FoldConstantArithmetic(ISD::UREM, dl, AmtVT, {N1, Bits}))
11060 return DAG.getNode(N->getOpcode(), dl, VT, N0, Amt);
11061 }
11062
11063 // rot i16 X, 8 --> bswap X
11064 auto *RotAmtC = isConstOrConstSplat(N1);
11065 if (RotAmtC && RotAmtC->getAPIntValue() == 8 &&
11066 VT.getScalarSizeInBits() == 16 && hasOperation(ISD::BSWAP, VT))
11067 return DAG.getNode(ISD::BSWAP, dl, VT, N0);
11068
11069 // Simplify the operands using demanded-bits information.
11071 return SDValue(N, 0);
11072
11073 // fold (rot* x, (trunc (and y, c))) -> (rot* x, (and (trunc y), (trunc c))).
11074 if (N1.getOpcode() == ISD::TRUNCATE &&
11075 N1.getOperand(0).getOpcode() == ISD::AND) {
11076 if (SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode()))
11077 return DAG.getNode(N->getOpcode(), dl, VT, N0, NewOp1);
11078 }
11079
11080 unsigned NextOp = N0.getOpcode();
11081
11082 // fold (rot* (rot* x, c2), c1)
11083 // -> (rot* x, ((c1 % bitsize) +- (c2 % bitsize) + bitsize) % bitsize)
11084 if (NextOp == ISD::ROTL || NextOp == ISD::ROTR) {
11087 if (C1 && C2 && N1.getValueType() == N0.getOperand(1).getValueType()) {
11088 EVT ShiftVT = N1.getValueType();
11089 bool SameSide = (N->getOpcode() == NextOp);
11090 unsigned CombineOp = SameSide ? ISD::ADD : ISD::SUB;
11091 SDValue BitsizeC = DAG.getConstant(Bitsize, dl, ShiftVT);
11092 SDValue Norm1 = DAG.FoldConstantArithmetic(ISD::UREM, dl, ShiftVT,
11093 {N1, BitsizeC});
11094 SDValue Norm2 = DAG.FoldConstantArithmetic(ISD::UREM, dl, ShiftVT,
11095 {N0.getOperand(1), BitsizeC});
11096 if (Norm1 && Norm2)
11097 if (SDValue CombinedShift = DAG.FoldConstantArithmetic(
11098 CombineOp, dl, ShiftVT, {Norm1, Norm2})) {
11099 CombinedShift = DAG.FoldConstantArithmetic(ISD::ADD, dl, ShiftVT,
11100 {CombinedShift, BitsizeC});
11101 SDValue CombinedShiftNorm = DAG.FoldConstantArithmetic(
11102 ISD::UREM, dl, ShiftVT, {CombinedShift, BitsizeC});
11103 return DAG.getNode(N->getOpcode(), dl, VT, N0->getOperand(0),
11104 CombinedShiftNorm);
11105 }
11106 }
11107 }
11108 return SDValue();
11109}
11110
11111SDValue DAGCombiner::visitSHL(SDNode *N) {
11112 SDValue N0 = N->getOperand(0);
11113 SDValue N1 = N->getOperand(1);
11114 if (SDValue V = DAG.simplifyShift(N0, N1))
11115 return V;
11116
11117 SDLoc DL(N);
11118 EVT VT = N0.getValueType();
11119 EVT ShiftVT = N1.getValueType();
11120 unsigned OpSizeInBits = VT.getScalarSizeInBits();
11121
11122 // fold (shl c1, c2) -> c1<<c2
11123 if (SDValue C =
11124 DAG.FoldConstantArithmetic(ISD::SHL, DL, VT, {N0, N1}, N->getFlags()))
11125 return C;
11126
11127 // fold vector ops
11128 if (VT.isVector()) {
11129 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
11130 return FoldedVOp;
11131
11132 BuildVectorSDNode *N1CV = dyn_cast<BuildVectorSDNode>(N1);
11133 // If setcc produces all-one true value then:
11134 // (shl (and (setcc) N01CV) N1CV) -> (and (setcc) N01CV<<N1CV)
11135 if (N1CV && N1CV->isConstant()) {
11136 if (N0.getOpcode() == ISD::AND) {
11137 SDValue N00 = N0->getOperand(0);
11138 SDValue N01 = N0->getOperand(1);
11139 BuildVectorSDNode *N01CV = dyn_cast<BuildVectorSDNode>(N01);
11140
11141 if (N01CV && N01CV->isConstant() && N00.getOpcode() == ISD::SETCC &&
11144 if (SDValue C =
11145 DAG.FoldConstantArithmetic(ISD::SHL, DL, VT, {N01, N1}))
11146 return DAG.getNode(ISD::AND, DL, VT, N00, C);
11147 }
11148 }
11149 }
11150 }
11151
11152 if (SDValue NewSel = foldBinOpIntoSelect(N))
11153 return NewSel;
11154
11155 // if (shl x, c) is known to be zero, return 0
11156 if (DAG.MaskedValueIsZero(SDValue(N, 0), APInt::getAllOnes(OpSizeInBits)))
11157 return DAG.getConstant(0, DL, VT);
11158
11159 // fold (shl x, (trunc (and y, c))) -> (shl x, (and (trunc y), (trunc c))).
11160 if (N1.getOpcode() == ISD::TRUNCATE &&
11161 N1.getOperand(0).getOpcode() == ISD::AND) {
11162 if (SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode()))
11163 return DAG.getNode(ISD::SHL, DL, VT, N0, NewOp1);
11164 }
11165
11166 // fold (shl (shl x, c1), c2) -> 0 or (shl x, (add c1, c2))
11167 if (N0.getOpcode() == ISD::SHL) {
11168 auto MatchOutOfRange = [OpSizeInBits](ConstantSDNode *LHS,
11169 ConstantSDNode *RHS) {
11170 APInt c1 = LHS->getAPIntValue();
11171 APInt c2 = RHS->getAPIntValue();
11172 zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
11173 return (c1 + c2).uge(OpSizeInBits);
11174 };
11175 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchOutOfRange))
11176 return DAG.getConstant(0, DL, VT);
11177
11178 auto MatchInRange = [OpSizeInBits](ConstantSDNode *LHS,
11179 ConstantSDNode *RHS) {
11180 APInt c1 = LHS->getAPIntValue();
11181 APInt c2 = RHS->getAPIntValue();
11182 zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
11183 return (c1 + c2).ult(OpSizeInBits);
11184 };
11185 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchInRange)) {
11186 SDValue Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, N1, N0.getOperand(1));
11187 return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Sum);
11188 }
11189 }
11190
11191 // fold (shl (ext (shl x, c1)), c2) -> (shl (ext x), (add c1, c2))
11192 // For this to be valid, the second form must not preserve any of the bits
11193 // that are shifted out by the inner shift in the first form. This means
11194 // the outer shift size must be >= the number of bits added by the ext.
11195 // As a corollary, we don't care what kind of ext it is.
11196 if ((N0.getOpcode() == ISD::ZERO_EXTEND ||
11197 N0.getOpcode() == ISD::ANY_EXTEND ||
11198 N0.getOpcode() == ISD::SIGN_EXTEND) &&
11199 N0.getOperand(0).getOpcode() == ISD::SHL) {
11200 SDValue N0Op0 = N0.getOperand(0);
11201 SDValue InnerShiftAmt = N0Op0.getOperand(1);
11202 EVT InnerVT = N0Op0.getValueType();
11203 uint64_t InnerBitwidth = InnerVT.getScalarSizeInBits();
11204
11205 auto MatchOutOfRange = [OpSizeInBits, InnerBitwidth](ConstantSDNode *LHS,
11206 ConstantSDNode *RHS) {
11207 APInt c1 = LHS->getAPIntValue();
11208 APInt c2 = RHS->getAPIntValue();
11209 zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
11210 return c2.uge(OpSizeInBits - InnerBitwidth) &&
11211 (c1 + c2).uge(OpSizeInBits);
11212 };
11213 if (ISD::matchBinaryPredicate(InnerShiftAmt, N1, MatchOutOfRange,
11214 /*AllowUndefs*/ false,
11215 /*AllowTypeMismatch*/ true))
11216 return DAG.getConstant(0, DL, VT);
11217
11218 auto MatchInRange = [OpSizeInBits, InnerBitwidth](ConstantSDNode *LHS,
11219 ConstantSDNode *RHS) {
11220 APInt c1 = LHS->getAPIntValue();
11221 APInt c2 = RHS->getAPIntValue();
11222 zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
11223 return c2.uge(OpSizeInBits - InnerBitwidth) &&
11224 (c1 + c2).ult(OpSizeInBits);
11225 };
11226 if (ISD::matchBinaryPredicate(InnerShiftAmt, N1, MatchInRange,
11227 /*AllowUndefs*/ false,
11228 /*AllowTypeMismatch*/ true)) {
11229 SDValue Ext = DAG.getNode(N0.getOpcode(), DL, VT, N0Op0.getOperand(0));
11230 SDValue Sum = DAG.getZExtOrTrunc(InnerShiftAmt, DL, ShiftVT);
11231 Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, Sum, N1);
11232 return DAG.getNode(ISD::SHL, DL, VT, Ext, Sum);
11233 }
11234 }
11235
11236 // fold (shl (zext (srl x, C)), C) -> (zext (shl (srl x, C), C))
11237 // Only fold this if the inner zext has no other uses to avoid increasing
11238 // the total number of instructions.
11239 if (N0.getOpcode() == ISD::ZERO_EXTEND && N0.hasOneUse() &&
11240 N0.getOperand(0).getOpcode() == ISD::SRL) {
11241 SDValue N0Op0 = N0.getOperand(0);
11242 SDValue InnerShiftAmt = N0Op0.getOperand(1);
11243
11244 auto MatchEqual = [VT](ConstantSDNode *LHS, ConstantSDNode *RHS) {
11245 APInt c1 = LHS->getAPIntValue();
11246 APInt c2 = RHS->getAPIntValue();
11247 zeroExtendToMatch(c1, c2);
11248 return c1.ult(VT.getScalarSizeInBits()) && (c1 == c2);
11249 };
11250 if (ISD::matchBinaryPredicate(InnerShiftAmt, N1, MatchEqual,
11251 /*AllowUndefs*/ false,
11252 /*AllowTypeMismatch*/ true)) {
11253 EVT InnerShiftAmtVT = N0Op0.getOperand(1).getValueType();
11254 SDValue NewSHL = DAG.getZExtOrTrunc(N1, DL, InnerShiftAmtVT);
11255 NewSHL = DAG.getNode(ISD::SHL, DL, N0Op0.getValueType(), N0Op0, NewSHL);
11256 AddToWorklist(NewSHL.getNode());
11257 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N0), VT, NewSHL);
11258 }
11259 }
11260
11261 if (N0.getOpcode() == ISD::SRL || N0.getOpcode() == ISD::SRA) {
11262 auto MatchShiftAmount = [OpSizeInBits](ConstantSDNode *LHS,
11263 ConstantSDNode *RHS) {
11264 const APInt &LHSC = LHS->getAPIntValue();
11265 const APInt &RHSC = RHS->getAPIntValue();
11266 return LHSC.ult(OpSizeInBits) && RHSC.ult(OpSizeInBits) &&
11267 LHSC.getZExtValue() <= RHSC.getZExtValue();
11268 };
11269
11270 // fold (shl (sr[la] exact X, C1), C2) -> (shl X, (C2-C1)) if C1 <= C2
11271 // fold (shl (sr[la] exact X, C1), C2) -> (sr[la] X, (C2-C1)) if C1 >= C2
11272 if (N0->getFlags().hasExact()) {
11273 if (ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchShiftAmount,
11274 /*AllowUndefs*/ false,
11275 /*AllowTypeMismatch*/ true)) {
11276 SDValue N01 = DAG.getZExtOrTrunc(N0.getOperand(1), DL, ShiftVT);
11277 SDValue Diff = DAG.getNode(ISD::SUB, DL, ShiftVT, N1, N01);
11278 return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Diff);
11279 }
11280 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchShiftAmount,
11281 /*AllowUndefs*/ false,
11282 /*AllowTypeMismatch*/ true)) {
11283 SDValue N01 = DAG.getZExtOrTrunc(N0.getOperand(1), DL, ShiftVT);
11284 SDValue Diff = DAG.getNode(ISD::SUB, DL, ShiftVT, N01, N1);
11285 return DAG.getNode(N0.getOpcode(), DL, VT, N0.getOperand(0), Diff);
11286 }
11287 }
11288
11289 // fold (shl (srl x, c1), c2) -> (and (shl x, (sub c2, c1), MASK) or
11290 // (and (srl x, (sub c1, c2), MASK)
11291 // Only fold this if the inner shift has no other uses -- if it does,
11292 // folding this will increase the total number of instructions.
11293 if (N0.getOpcode() == ISD::SRL &&
11294 (N0.getOperand(1) == N1 || N0.hasOneUse()) &&
11296 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchShiftAmount,
11297 /*AllowUndefs*/ false,
11298 /*AllowTypeMismatch*/ true)) {
11299 SDValue N01 = DAG.getZExtOrTrunc(N0.getOperand(1), DL, ShiftVT);
11300 SDValue Diff = DAG.getNode(ISD::SUB, DL, ShiftVT, N01, N1);
11301 SDValue Mask = DAG.getAllOnesConstant(DL, VT);
11302 Mask = DAG.getNode(ISD::SHL, DL, VT, Mask, N01);
11303 Mask = DAG.getNode(ISD::SRL, DL, VT, Mask, Diff);
11304 SDValue Shift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), Diff);
11305 return DAG.getNode(ISD::AND, DL, VT, Shift, Mask);
11306 }
11307 if (ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchShiftAmount,
11308 /*AllowUndefs*/ false,
11309 /*AllowTypeMismatch*/ true)) {
11310 SDValue N01 = DAG.getZExtOrTrunc(N0.getOperand(1), DL, ShiftVT);
11311 SDValue Diff = DAG.getNode(ISD::SUB, DL, ShiftVT, N1, N01);
11312 SDValue Mask = DAG.getAllOnesConstant(DL, VT);
11313 Mask = DAG.getNode(ISD::SHL, DL, VT, Mask, N1);
11314 SDValue Shift = DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Diff);
11315 return DAG.getNode(ISD::AND, DL, VT, Shift, Mask);
11316 }
11317 }
11318 }
11319
11320 // fold (shl (sra x, c1), c1) -> (and x, (shl -1, c1))
11321 if (N0.getOpcode() == ISD::SRA && N1 == N0.getOperand(1) &&
11322 isConstantOrConstantVector(N1, /* No Opaques */ true)) {
11323 SDValue AllBits = DAG.getAllOnesConstant(DL, VT);
11324 SDValue HiBitsMask = DAG.getNode(ISD::SHL, DL, VT, AllBits, N1);
11325 return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), HiBitsMask);
11326 }
11327
11328 // fold (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2)
11329 // fold (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2)
11330 // Variant of version done on multiply, except mul by a power of 2 is turned
11331 // into a shift.
11332 if ((N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::OR) &&
11334 SDValue N01 = N0.getOperand(1);
11335 if (SDValue Shl1 =
11336 DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N1), VT, {N01, N1})) {
11337 SDValue Shl0 = DAG.getNode(ISD::SHL, SDLoc(N0), VT, N0.getOperand(0), N1);
11338 AddToWorklist(Shl0.getNode());
11339 SDNodeFlags Flags;
11340 // Preserve the disjoint flag for Or.
11341 if (N0.getOpcode() == ISD::OR && N0->getFlags().hasDisjoint())
11343 return DAG.getNode(N0.getOpcode(), DL, VT, Shl0, Shl1, Flags);
11344 }
11345 }
11346
11347 // fold (shl (sext (add_nsw x, c1)), c2) -> (add (shl (sext x), c2), c1 << c2)
11348 // TODO: Add zext/add_nuw variant with suitable test coverage
11349 // TODO: Should we limit this with isLegalAddImmediate?
11350 if (N0.getOpcode() == ISD::SIGN_EXTEND &&
11351 N0.getOperand(0).getOpcode() == ISD::ADD &&
11352 N0.getOperand(0)->getFlags().hasNoSignedWrap() &&
11354 SDValue Add = N0.getOperand(0);
11355 SDLoc DL(N0);
11356 if (SDValue ExtC = DAG.FoldConstantArithmetic(N0.getOpcode(), DL, VT,
11357 {Add.getOperand(1)})) {
11358 if (SDValue ShlC =
11359 DAG.FoldConstantArithmetic(ISD::SHL, DL, VT, {ExtC, N1})) {
11360 SDValue ExtX = DAG.getNode(N0.getOpcode(), DL, VT, Add.getOperand(0));
11361 SDValue ShlX = DAG.getNode(ISD::SHL, DL, VT, ExtX, N1);
11362 return DAG.getNode(ISD::ADD, DL, VT, ShlX, ShlC);
11363 }
11364 }
11365 }
11366
11367 // fold (shl (mul x, c1), c2) -> (mul x, c1 << c2)
11368 if (N0.getOpcode() == ISD::MUL && N0->hasOneUse()) {
11369 SDValue N01 = N0.getOperand(1);
11370 if (SDValue Shl =
11371 DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N1), VT, {N01, N1}))
11372 return DAG.getNode(ISD::MUL, DL, VT, N0.getOperand(0), Shl);
11373 }
11374
11375 ConstantSDNode *N1C = isConstOrConstSplat(N1);
11376 if (N1C && !N1C->isOpaque())
11377 if (SDValue NewSHL = visitShiftByConstant(N))
11378 return NewSHL;
11379
11380 // fold (shl X, cttz(Y)) -> (mul (Y & -Y), X) if cttz is unsupported on the
11381 // target.
11382 if (((N1.getOpcode() == ISD::CTTZ &&
11383 VT.getScalarSizeInBits() <= ShiftVT.getScalarSizeInBits()) ||
11385 N1.hasOneUse() && !TLI.isOperationLegalOrCustom(ISD::CTTZ, ShiftVT) &&
11387 SDValue Y = N1.getOperand(0);
11388 SDLoc DL(N);
11389 SDValue NegY = DAG.getNegative(Y, DL, ShiftVT);
11390 SDValue And =
11391 DAG.getZExtOrTrunc(DAG.getNode(ISD::AND, DL, ShiftVT, Y, NegY), DL, VT);
11392 return DAG.getNode(ISD::MUL, DL, VT, And, N0);
11393 }
11394
11396 return SDValue(N, 0);
11397
11398 // Fold (shl (vscale * C0), C1) to (vscale * (C0 << C1)).
11399 if (N0.getOpcode() == ISD::VSCALE && N1C) {
11400 const APInt &C0 = N0.getConstantOperandAPInt(0);
11401 const APInt &C1 = N1C->getAPIntValue();
11402 return DAG.getVScale(DL, VT, C0 << C1);
11403 }
11404
11405 SDValue X;
11406 APInt VS0;
11407
11408 // fold (shl (X * vscale(VS0)), C1) -> (X * vscale(VS0 << C1))
11409 if (N1C && sd_match(N0, m_Mul(m_Value(X), m_VScale(m_ConstInt(VS0))))) {
11410 SDNodeFlags Flags;
11411 Flags.setNoUnsignedWrap(N->getFlags().hasNoUnsignedWrap() &&
11412 N0->getFlags().hasNoUnsignedWrap());
11413
11414 SDValue VScale = DAG.getVScale(DL, VT, VS0 << N1C->getAPIntValue());
11415 return DAG.getNode(ISD::MUL, DL, VT, X, VScale, Flags);
11416 }
11417
11418 // Fold (shl step_vector(C0), C1) to (step_vector(C0 << C1)).
11419 APInt ShlVal;
11420 if (N0.getOpcode() == ISD::STEP_VECTOR &&
11421 ISD::isConstantSplatVector(N1.getNode(), ShlVal)) {
11422 const APInt &C0 = N0.getConstantOperandAPInt(0);
11423 if (ShlVal.ult(C0.getBitWidth())) {
11424 APInt NewStep = C0 << ShlVal;
11425 return DAG.getStepVector(DL, VT, NewStep);
11426 }
11427 }
11428
11429 return SDValue();
11430}
11431
11432// Transform a right shift of a multiply into a multiply-high.
11433// Examples:
11434// (srl (mul (zext i32:$a to i64), (zext i32:$a to i64)), 32) -> (mulhu $a, $b)
11435// (sra (mul (sext i32:$a to i64), (sext i32:$a to i64)), 32) -> (mulhs $a, $b)
11437 const TargetLowering &TLI) {
11438 assert((N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) &&
11439 "SRL or SRA node is required here!");
11440
11441 // Check the shift amount. Proceed with the transformation if the shift
11442 // amount is constant.
11443 ConstantSDNode *ShiftAmtSrc = isConstOrConstSplat(N->getOperand(1));
11444 if (!ShiftAmtSrc)
11445 return SDValue();
11446
11447 // The operation feeding into the shift must be a multiply.
11448 SDValue ShiftOperand = N->getOperand(0);
11449 if (ShiftOperand.getOpcode() != ISD::MUL)
11450 return SDValue();
11451
11452 // Both operands must be equivalent extend nodes.
11453 SDValue LeftOp = ShiftOperand.getOperand(0);
11454 SDValue RightOp = ShiftOperand.getOperand(1);
11455
11456 if (LeftOp.getOpcode() != ISD::SIGN_EXTEND &&
11457 LeftOp.getOpcode() != ISD::ZERO_EXTEND)
11458 std::swap(LeftOp, RightOp);
11459
11460 bool IsSignExt = LeftOp.getOpcode() == ISD::SIGN_EXTEND;
11461 bool IsZeroExt = LeftOp.getOpcode() == ISD::ZERO_EXTEND;
11462
11463 if (!IsSignExt && !IsZeroExt)
11464 return SDValue();
11465
11466 EVT NarrowVT = LeftOp.getOperand(0).getValueType();
11467 unsigned NarrowVTSize = NarrowVT.getScalarSizeInBits();
11468
11469 // return true if U may use the lower bits of its operands
11470 auto UserOfLowerBits = [NarrowVTSize](SDNode *U) {
11471 if (U->getOpcode() != ISD::SRL && U->getOpcode() != ISD::SRA) {
11472 return true;
11473 }
11474 ConstantSDNode *UShiftAmtSrc = isConstOrConstSplat(U->getOperand(1));
11475 if (!UShiftAmtSrc) {
11476 return true;
11477 }
11478 unsigned UShiftAmt = UShiftAmtSrc->getZExtValue();
11479 return UShiftAmt < NarrowVTSize;
11480 };
11481
11482 // If the lower part of the MUL is also used and MUL_LOHI is supported
11483 // do not introduce the MULH in favor of MUL_LOHI
11484 unsigned MulLoHiOp = IsSignExt ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
11485 if (!ShiftOperand.hasOneUse() &&
11486 TLI.isOperationLegalOrCustom(MulLoHiOp, NarrowVT) &&
11487 llvm::any_of(ShiftOperand->users(), UserOfLowerBits)) {
11488 return SDValue();
11489 }
11490
11491 SDValue MulhRightOp;
11492 if (LeftOp.getOpcode() != RightOp.getOpcode()) {
11493 if (IsZeroExt && ShiftOperand.hasOneUse() &&
11494 DAG.computeKnownBits(RightOp).countMaxActiveBits() <= NarrowVTSize) {
11495 MulhRightOp = DAG.getNode(ISD::TRUNCATE, DL, NarrowVT, RightOp);
11496 } else if (IsSignExt && ShiftOperand.hasOneUse() &&
11497 DAG.ComputeMaxSignificantBits(RightOp) <= NarrowVTSize) {
11498 MulhRightOp = DAG.getNode(ISD::TRUNCATE, DL, NarrowVT, RightOp);
11499 } else {
11500 return SDValue();
11501 }
11502 } else {
11503 // Check that the two extend nodes are the same type.
11504 if (NarrowVT != RightOp.getOperand(0).getValueType())
11505 return SDValue();
11506 MulhRightOp = RightOp.getOperand(0);
11507 }
11508
11509 EVT WideVT = LeftOp.getValueType();
11510 // Proceed with the transformation if the wide types match.
11511 assert((WideVT == RightOp.getValueType()) &&
11512 "Cannot have a multiply node with two different operand types.");
11513
11514 // Proceed with the transformation if the wide type is twice as large
11515 // as the narrow type.
11516 if (WideVT.getScalarSizeInBits() != 2 * NarrowVTSize)
11517 return SDValue();
11518
11519 // Check the shift amount with the narrow type size.
11520 // Proceed with the transformation if the shift amount is the width
11521 // of the narrow type.
11522 unsigned ShiftAmt = ShiftAmtSrc->getZExtValue();
11523 if (ShiftAmt != NarrowVTSize)
11524 return SDValue();
11525
11526 // If the operation feeding into the MUL is a sign extend (sext),
11527 // we use mulhs. Othewise, zero extends (zext) use mulhu.
11528 unsigned MulhOpcode = IsSignExt ? ISD::MULHS : ISD::MULHU;
11529
11530 // Combine to mulh if mulh is legal/custom for the narrow type on the target
11531 // or if it is a vector type then we could transform to an acceptable type and
11532 // rely on legalization to split/combine the result.
11533 EVT TransformVT = NarrowVT;
11534 if (NarrowVT.isVector()) {
11535 TransformVT = TLI.getLegalTypeToTransformTo(*DAG.getContext(), NarrowVT);
11536 if (TransformVT.getScalarType() != NarrowVT.getScalarType())
11537 return SDValue();
11538 }
11539 if (!TLI.isOperationLegalOrCustom(MulhOpcode, TransformVT))
11540 return SDValue();
11541
11542 SDValue Result =
11543 DAG.getNode(MulhOpcode, DL, NarrowVT, LeftOp.getOperand(0), MulhRightOp);
11544 bool IsSigned = N->getOpcode() == ISD::SRA;
11545 return DAG.getExtOrTrunc(IsSigned, Result, DL, WideVT);
11546}
11547
11548// fold (bswap (logic_op(bswap(x),y))) -> logic_op(x,bswap(y))
11549// This helper function accept SDNode with opcode ISD::BSWAP and ISD::BITREVERSE
11551 unsigned Opcode = N->getOpcode();
11552 if (Opcode != ISD::BSWAP && Opcode != ISD::BITREVERSE)
11553 return SDValue();
11554
11555 SDValue N0 = N->getOperand(0);
11556 EVT VT = N->getValueType(0);
11557 SDLoc DL(N);
11558 SDValue X, Y;
11559
11560 // If both operands are bswap/bitreverse, ignore the multiuse
11562 m_UnaryOp(Opcode, m_Value(Y))))))
11563 return DAG.getNode(N0.getOpcode(), DL, VT, X, Y);
11564
11565 // Otherwise need to ensure logic_op and bswap/bitreverse(x) have one use.
11567 m_OneUse(m_UnaryOp(Opcode, m_Value(X))), m_Value(Y))))) {
11568 SDValue NewBitReorder = DAG.getNode(Opcode, DL, VT, Y);
11569 return DAG.getNode(N0.getOpcode(), DL, VT, X, NewBitReorder);
11570 }
11571
11572 return SDValue();
11573}
11574
11575SDValue DAGCombiner::visitSRA(SDNode *N) {
11576 SDValue N0 = N->getOperand(0);
11577 SDValue N1 = N->getOperand(1);
11578 if (SDValue V = DAG.simplifyShift(N0, N1))
11579 return V;
11580
11581 SDLoc DL(N);
11582 EVT VT = N0.getValueType();
11583 unsigned OpSizeInBits = VT.getScalarSizeInBits();
11584
11585 // fold (sra c1, c2) -> (sra c1, c2)
11586 if (SDValue C =
11587 DAG.FoldConstantArithmetic(ISD::SRA, DL, VT, {N0, N1}, N->getFlags()))
11588 return C;
11589
11590 // Arithmetic shifting an all-sign-bit value is a no-op.
11591 // fold (sra 0, x) -> 0
11592 // fold (sra -1, x) -> -1
11593 if (DAG.ComputeNumSignBits(N0) == OpSizeInBits)
11594 return N0;
11595
11596 // fold vector ops
11597 if (VT.isVector())
11598 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
11599 return FoldedVOp;
11600
11601 if (SDValue NewSel = foldBinOpIntoSelect(N))
11602 return NewSel;
11603
11604 ConstantSDNode *N1C = isConstOrConstSplat(N1);
11605
11606 // fold (sra (sra x, c1), c2) -> (sra x, (add c1, c2))
11607 // clamp (add c1, c2) to max shift.
11608 if (N0.getOpcode() == ISD::SRA) {
11609 EVT ShiftVT = N1.getValueType();
11610 EVT ShiftSVT = ShiftVT.getScalarType();
11611 SmallVector<SDValue, 16> ShiftValues;
11612
11613 auto SumOfShifts = [&](ConstantSDNode *LHS, ConstantSDNode *RHS) {
11614 APInt c1 = LHS->getAPIntValue();
11615 APInt c2 = RHS->getAPIntValue();
11616 zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
11617 APInt Sum = c1 + c2;
11618 unsigned ShiftSum =
11619 Sum.uge(OpSizeInBits) ? (OpSizeInBits - 1) : Sum.getZExtValue();
11620 ShiftValues.push_back(DAG.getConstant(ShiftSum, DL, ShiftSVT));
11621 return true;
11622 };
11623 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), SumOfShifts)) {
11624 SDValue ShiftValue;
11625 if (N1.getOpcode() == ISD::BUILD_VECTOR)
11626 ShiftValue = DAG.getBuildVector(ShiftVT, DL, ShiftValues);
11627 else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {
11628 assert(ShiftValues.size() == 1 &&
11629 "Expected matchBinaryPredicate to return one element for "
11630 "SPLAT_VECTORs");
11631 ShiftValue = DAG.getSplatVector(ShiftVT, DL, ShiftValues[0]);
11632 } else
11633 ShiftValue = ShiftValues[0];
11634 return DAG.getNode(ISD::SRA, DL, VT, N0.getOperand(0), ShiftValue);
11635 }
11636 }
11637
11638 // fold (sra (xor (sra x, c1), -1), c2) -> (xor (sra x, c3), -1)
11639 // This allows merging two arithmetic shifts even when there's a NOT in
11640 // between.
11641 SDValue X;
11642 APInt C1;
11643 if (N1C && sd_match(N0, m_OneUse(m_Not(
11644 m_OneUse(m_Sra(m_Value(X), m_ConstInt(C1))))))) {
11645 APInt C2 = N1C->getAPIntValue();
11646 zeroExtendToMatch(C1, C2, 1 /* Overflow Bit */);
11647 APInt Sum = C1 + C2;
11648 unsigned ShiftSum = Sum.getLimitedValue(OpSizeInBits - 1);
11649 SDValue NewShift = DAG.getNode(
11650 ISD::SRA, DL, VT, X, DAG.getShiftAmountConstant(ShiftSum, VT, DL));
11651 return DAG.getNOT(DL, NewShift, VT);
11652 }
11653
11654 // fold (sra (shl X, m), (sub result_size, n))
11655 // -> (sign_extend (trunc (shl X, (sub (sub result_size, n), m)))) for
11656 // result_size - n != m.
11657 // If truncate is free for the target sext(shl) is likely to result in better
11658 // code.
11659 if (N0.getOpcode() == ISD::SHL && N1C) {
11660 // Get the two constants of the shifts, CN0 = m, CN = n.
11661 const ConstantSDNode *N01C = isConstOrConstSplat(N0.getOperand(1));
11662 if (N01C) {
11663 LLVMContext &Ctx = *DAG.getContext();
11664 // Determine what the truncate's result bitsize and type would be.
11665 EVT TruncVT = VT.changeElementType(
11666 Ctx, EVT::getIntegerVT(Ctx, OpSizeInBits - N1C->getZExtValue()));
11667
11668 // Determine the residual right-shift amount.
11669 int ShiftAmt = N1C->getZExtValue() - N01C->getZExtValue();
11670
11671 // If the shift is not a no-op (in which case this should be just a sign
11672 // extend already), the truncated to type is legal, sign_extend is legal
11673 // on that type, and the truncate to that type is both legal and free,
11674 // perform the transform.
11675 if ((ShiftAmt > 0) &&
11678 TLI.isTruncateFree(VT, TruncVT)) {
11679 SDValue Amt = DAG.getShiftAmountConstant(ShiftAmt, VT, DL);
11680 SDValue Shift = DAG.getNode(ISD::SRL, DL, VT,
11681 N0.getOperand(0), Amt);
11682 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, TruncVT,
11683 Shift);
11684 return DAG.getNode(ISD::SIGN_EXTEND, DL,
11685 N->getValueType(0), Trunc);
11686 }
11687 }
11688 }
11689
11690 // We convert trunc/ext to opposing shifts in IR, but casts may be cheaper.
11691 // sra (add (shl X, N1C), AddC), N1C -->
11692 // sext (add (trunc X to (width - N1C)), AddC')
11693 // sra (sub AddC, (shl X, N1C)), N1C -->
11694 // sext (sub AddC1',(trunc X to (width - N1C)))
11695 if ((N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB) && N1C &&
11696 N0.hasOneUse()) {
11697 bool IsAdd = N0.getOpcode() == ISD::ADD;
11698 SDValue Shl = N0.getOperand(IsAdd ? 0 : 1);
11699 if (Shl.getOpcode() == ISD::SHL && Shl.getOperand(1) == N1 &&
11700 Shl.hasOneUse()) {
11701 // TODO: AddC does not need to be a splat.
11702 if (ConstantSDNode *AddC =
11703 isConstOrConstSplat(N0.getOperand(IsAdd ? 1 : 0))) {
11704 // Determine what the truncate's type would be and ask the target if
11705 // that is a free operation.
11706 LLVMContext &Ctx = *DAG.getContext();
11707 unsigned ShiftAmt = N1C->getZExtValue();
11708 EVT TruncVT = VT.changeElementType(
11709 Ctx, EVT::getIntegerVT(Ctx, OpSizeInBits - ShiftAmt));
11710
11711 // TODO: The simple type check probably belongs in the default hook
11712 // implementation and/or target-specific overrides (because
11713 // non-simple types likely require masking when legalized), but
11714 // that restriction may conflict with other transforms.
11715 if (TruncVT.isSimple() && isTypeLegal(TruncVT) &&
11716 TLI.isTruncateFree(VT, TruncVT)) {
11717 SDValue Trunc = DAG.getZExtOrTrunc(Shl.getOperand(0), DL, TruncVT);
11718 SDValue ShiftC =
11719 DAG.getConstant(AddC->getAPIntValue().lshr(ShiftAmt).trunc(
11720 TruncVT.getScalarSizeInBits()),
11721 DL, TruncVT);
11722 SDValue Add;
11723 if (IsAdd)
11724 Add = DAG.getNode(ISD::ADD, DL, TruncVT, Trunc, ShiftC);
11725 else
11726 Add = DAG.getNode(ISD::SUB, DL, TruncVT, ShiftC, Trunc);
11727 return DAG.getSExtOrTrunc(Add, DL, VT);
11728 }
11729 }
11730 }
11731 }
11732
11733 // fold (sra x, (trunc (and y, c))) -> (sra x, (and (trunc y), (trunc c))).
11734 if (N1.getOpcode() == ISD::TRUNCATE &&
11735 N1.getOperand(0).getOpcode() == ISD::AND) {
11736 if (SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode()))
11737 return DAG.getNode(ISD::SRA, DL, VT, N0, NewOp1);
11738 }
11739
11740 // fold (sra (trunc (sra x, c1)), c2) -> (trunc (sra x, c1 + c2))
11741 // fold (sra (trunc (srl x, c1)), c2) -> (trunc (sra x, c1 + c2))
11742 // if c1 is equal to the number of bits the trunc removes
11743 // TODO - support non-uniform vector shift amounts.
11744 if (N0.getOpcode() == ISD::TRUNCATE &&
11745 (N0.getOperand(0).getOpcode() == ISD::SRL ||
11746 N0.getOperand(0).getOpcode() == ISD::SRA) &&
11747 N0.getOperand(0).hasOneUse() &&
11748 N0.getOperand(0).getOperand(1).hasOneUse() && N1C) {
11749 SDValue N0Op0 = N0.getOperand(0);
11750 if (ConstantSDNode *LargeShift = isConstOrConstSplat(N0Op0.getOperand(1))) {
11751 EVT LargeVT = N0Op0.getValueType();
11752 unsigned TruncBits = LargeVT.getScalarSizeInBits() - OpSizeInBits;
11753 if (LargeShift->getAPIntValue() == TruncBits) {
11754 EVT LargeShiftVT = getShiftAmountTy(LargeVT);
11755 SDValue Amt = DAG.getZExtOrTrunc(N1, DL, LargeShiftVT);
11756 Amt = DAG.getNode(ISD::ADD, DL, LargeShiftVT, Amt,
11757 DAG.getConstant(TruncBits, DL, LargeShiftVT));
11758 SDValue SRA =
11759 DAG.getNode(ISD::SRA, DL, LargeVT, N0Op0.getOperand(0), Amt);
11760 return DAG.getNode(ISD::TRUNCATE, DL, VT, SRA);
11761 }
11762 }
11763 }
11764
11765 // fold (sra (add nsw X, C), D) -> (add nsw (sra X, D), C s>> D)
11766 // when C has D trailing zeros (so C s>> D is exact).
11767 if (N1C && N0.hasOneUse() && N0.getOpcode() == ISD::ADD &&
11768 N0->getFlags().hasNoSignedWrap()) {
11769 if (ConstantSDNode *AddC = isConstOrConstSplat(N0.getOperand(1))) {
11770 const APInt &ShAmt = N1C->getAPIntValue();
11771 const APInt &AddVal = AddC->getAPIntValue();
11772 if (ShAmt.ult(AddVal.countr_zero())) {
11773 SDNodeFlags ShiftFlags = N->getFlags();
11774 SDValue NewSra =
11775 DAG.getNode(ISD::SRA, DL, VT, N0.getOperand(0), N1, ShiftFlags);
11776 SDValue NewC = DAG.getConstant(AddVal.ashr(ShAmt), DL, VT);
11777 SDNodeFlags AddFlags = N0->getFlags();
11778 return DAG.getNode(ISD::ADD, DL, VT, NewSra, NewC, AddFlags);
11779 }
11780 }
11781 }
11782
11783 // Simplify, based on bits shifted out of the LHS.
11785 return SDValue(N, 0);
11786
11787 // If the sign bit is known to be zero, switch this to a SRL.
11788 if (DAG.SignBitIsZero(N0))
11789 return DAG.getNode(ISD::SRL, DL, VT, N0, N1);
11790
11791 if (N1C && !N1C->isOpaque())
11792 if (SDValue NewSRA = visitShiftByConstant(N))
11793 return NewSRA;
11794
11795 // Try to transform this shift into a multiply-high if
11796 // it matches the appropriate pattern detected in combineShiftToMULH.
11797 if (SDValue MULH = combineShiftToMULH(N, DL, DAG, TLI))
11798 return MULH;
11799
11800 // Attempt to convert a sra of a load into a narrower sign-extending load.
11801 if (SDValue NarrowLoad = reduceLoadWidth(N))
11802 return NarrowLoad;
11803
11804 if (SDValue AVG = foldShiftToAvg(N, DL))
11805 return AVG;
11806
11807 return SDValue();
11808}
11809
11810SDValue DAGCombiner::visitSRL(SDNode *N) {
11811 SDValue N0 = N->getOperand(0);
11812 SDValue N1 = N->getOperand(1);
11813 if (SDValue V = DAG.simplifyShift(N0, N1))
11814 return V;
11815
11816 SDLoc DL(N);
11817 EVT VT = N0.getValueType();
11818 EVT ShiftVT = N1.getValueType();
11819 unsigned OpSizeInBits = VT.getScalarSizeInBits();
11820
11821 // fold (srl c1, c2) -> c1 >>u c2
11822 if (SDValue C =
11823 DAG.FoldConstantArithmetic(ISD::SRL, DL, VT, {N0, N1}, N->getFlags()))
11824 return C;
11825
11826 // fold vector ops
11827 if (VT.isVector())
11828 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
11829 return FoldedVOp;
11830
11831 if (SDValue NewSel = foldBinOpIntoSelect(N))
11832 return NewSel;
11833
11834 // if (srl x, c) is known to be zero, return 0
11835 ConstantSDNode *N1C = isConstOrConstSplat(N1);
11836 if (N1C &&
11837 DAG.MaskedValueIsZero(SDValue(N, 0), APInt::getAllOnes(OpSizeInBits)))
11838 return DAG.getConstant(0, DL, VT);
11839
11840 // fold (srl (srl x, c1), c2) -> 0 or (srl x, (add c1, c2))
11841 if (N0.getOpcode() == ISD::SRL) {
11842 auto MatchOutOfRange = [OpSizeInBits](ConstantSDNode *LHS,
11843 ConstantSDNode *RHS) {
11844 APInt c1 = LHS->getAPIntValue();
11845 APInt c2 = RHS->getAPIntValue();
11846 zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
11847 return (c1 + c2).uge(OpSizeInBits);
11848 };
11849 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchOutOfRange))
11850 return DAG.getConstant(0, DL, VT);
11851
11852 auto MatchInRange = [OpSizeInBits](ConstantSDNode *LHS,
11853 ConstantSDNode *RHS) {
11854 APInt c1 = LHS->getAPIntValue();
11855 APInt c2 = RHS->getAPIntValue();
11856 zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
11857 return (c1 + c2).ult(OpSizeInBits);
11858 };
11859 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchInRange)) {
11860 SDValue Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, N1, N0.getOperand(1));
11861 return DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), Sum);
11862 }
11863 }
11864
11865 if (N1C && N0.getOpcode() == ISD::TRUNCATE &&
11866 N0.getOperand(0).getOpcode() == ISD::SRL) {
11867 SDValue InnerShift = N0.getOperand(0);
11868 // TODO - support non-uniform vector shift amounts.
11869 if (auto *N001C = isConstOrConstSplat(InnerShift.getOperand(1))) {
11870 uint64_t c1 = N001C->getZExtValue();
11871 uint64_t c2 = N1C->getZExtValue();
11872 EVT InnerShiftVT = InnerShift.getValueType();
11873 EVT ShiftAmtVT = InnerShift.getOperand(1).getValueType();
11874 uint64_t InnerShiftSize = InnerShiftVT.getScalarSizeInBits();
11875 // srl (trunc (srl x, c1)), c2 --> 0 or (trunc (srl x, (add c1, c2)))
11876 // This is only valid if the OpSizeInBits + c1 = size of inner shift.
11877 if (c1 + OpSizeInBits == InnerShiftSize) {
11878 if (c1 + c2 >= InnerShiftSize)
11879 return DAG.getConstant(0, DL, VT);
11880 SDValue NewShiftAmt = DAG.getConstant(c1 + c2, DL, ShiftAmtVT);
11881 SDValue NewShift = DAG.getNode(ISD::SRL, DL, InnerShiftVT,
11882 InnerShift.getOperand(0), NewShiftAmt);
11883 return DAG.getNode(ISD::TRUNCATE, DL, VT, NewShift);
11884 }
11885 // In the more general case, we can clear the high bits after the shift:
11886 // srl (trunc (srl x, c1)), c2 --> trunc (and (srl x, (c1+c2)), Mask)
11887 if (N0.hasOneUse() && InnerShift.hasOneUse() &&
11888 c1 + c2 < InnerShiftSize) {
11889 SDValue NewShiftAmt = DAG.getConstant(c1 + c2, DL, ShiftAmtVT);
11890 SDValue NewShift = DAG.getNode(ISD::SRL, DL, InnerShiftVT,
11891 InnerShift.getOperand(0), NewShiftAmt);
11892 SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(InnerShiftSize,
11893 OpSizeInBits - c2),
11894 DL, InnerShiftVT);
11895 SDValue And = DAG.getNode(ISD::AND, DL, InnerShiftVT, NewShift, Mask);
11896 return DAG.getNode(ISD::TRUNCATE, DL, VT, And);
11897 }
11898 }
11899 }
11900
11901 if (N0.getOpcode() == ISD::SHL) {
11902 // fold (srl (shl nuw x, c), c) -> x
11903 if (N0.getOperand(1) == N1 && N0->getFlags().hasNoUnsignedWrap())
11904 return N0.getOperand(0);
11905
11906 // fold (srl (shl x, c1), c2) -> (and (shl x, (sub c1, c2), MASK) or
11907 // (and (srl x, (sub c2, c1), MASK)
11908 if ((N0.getOperand(1) == N1 || N0->hasOneUse()) &&
11910 auto MatchShiftAmount = [OpSizeInBits](ConstantSDNode *LHS,
11911 ConstantSDNode *RHS) {
11912 const APInt &LHSC = LHS->getAPIntValue();
11913 const APInt &RHSC = RHS->getAPIntValue();
11914 return LHSC.ult(OpSizeInBits) && RHSC.ult(OpSizeInBits) &&
11915 LHSC.getZExtValue() <= RHSC.getZExtValue();
11916 };
11917 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchShiftAmount,
11918 /*AllowUndefs*/ false,
11919 /*AllowTypeMismatch*/ true)) {
11920 SDValue N01 = DAG.getZExtOrTrunc(N0.getOperand(1), DL, ShiftVT);
11921 SDValue Diff = DAG.getNode(ISD::SUB, DL, ShiftVT, N01, N1);
11922 SDValue Mask = DAG.getAllOnesConstant(DL, VT);
11923 Mask = DAG.getNode(ISD::SRL, DL, VT, Mask, N01);
11924 Mask = DAG.getNode(ISD::SHL, DL, VT, Mask, Diff);
11925 SDValue Shift = DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Diff);
11926 return DAG.getNode(ISD::AND, DL, VT, Shift, Mask);
11927 }
11928 if (ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchShiftAmount,
11929 /*AllowUndefs*/ false,
11930 /*AllowTypeMismatch*/ true)) {
11931 SDValue N01 = DAG.getZExtOrTrunc(N0.getOperand(1), DL, ShiftVT);
11932 SDValue Diff = DAG.getNode(ISD::SUB, DL, ShiftVT, N1, N01);
11933 SDValue Mask = DAG.getAllOnesConstant(DL, VT);
11934 Mask = DAG.getNode(ISD::SRL, DL, VT, Mask, N1);
11935 SDValue Shift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), Diff);
11936 return DAG.getNode(ISD::AND, DL, VT, Shift, Mask);
11937 }
11938 }
11939 }
11940
11941 // fold (srl (anyextend x), c) -> (and (anyextend (srl x, c)), mask)
11942 // TODO - support non-uniform vector shift amounts.
11943 if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) {
11944 // Shifting in all undef bits?
11945 EVT SmallVT = N0.getOperand(0).getValueType();
11946 unsigned BitSize = SmallVT.getScalarSizeInBits();
11947 if (N1C->getAPIntValue().uge(BitSize))
11948 return DAG.getUNDEF(VT);
11949
11950 if (!LegalTypes || TLI.isTypeDesirableForOp(ISD::SRL, SmallVT)) {
11951 uint64_t ShiftAmt = N1C->getZExtValue();
11952 SDLoc DL0(N0);
11953 SDValue SmallShift =
11954 DAG.getNode(ISD::SRL, DL0, SmallVT, N0.getOperand(0),
11955 DAG.getShiftAmountConstant(ShiftAmt, SmallVT, DL0));
11956 AddToWorklist(SmallShift.getNode());
11957 APInt Mask = APInt::getLowBitsSet(OpSizeInBits, OpSizeInBits - ShiftAmt);
11958 return DAG.getNode(ISD::AND, DL, VT,
11959 DAG.getNode(ISD::ANY_EXTEND, DL, VT, SmallShift),
11960 DAG.getConstant(Mask, DL, VT));
11961 }
11962 }
11963
11964 // fold (srl (sra X, Y), 31) -> (srl X, 31). This srl only looks at the sign
11965 // bit, which is unmodified by sra.
11966 if (N1C && N1C->getAPIntValue() == (OpSizeInBits - 1)) {
11967 if (N0.getOpcode() == ISD::SRA)
11968 return DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), N1);
11969 }
11970
11971 // fold (srl (ctlz x), "5") -> x iff x has one bit set (the low bit), and x has a power
11972 // of two bitwidth. The "5" represents (log2 (bitwidth x)).
11973 if (N1C && N0.getOpcode() == ISD::CTLZ &&
11974 isPowerOf2_32(OpSizeInBits) &&
11975 N1C->getAPIntValue() == Log2_32(OpSizeInBits)) {
11976 KnownBits Known = DAG.computeKnownBits(N0.getOperand(0));
11977
11978 // If any of the input bits are KnownOne, then the input couldn't be all
11979 // zeros, thus the result of the srl will always be zero.
11980 if (Known.One.getBoolValue()) return DAG.getConstant(0, SDLoc(N0), VT);
11981
11982 // If all of the bits input the to ctlz node are known to be zero, then
11983 // the result of the ctlz is "32" and the result of the shift is one.
11984 APInt UnknownBits = ~Known.Zero;
11985 if (UnknownBits == 0) return DAG.getConstant(1, SDLoc(N0), VT);
11986
11987 // Otherwise, check to see if there is exactly one bit input to the ctlz.
11988 if (UnknownBits.isPowerOf2()) {
11989 // Okay, we know that only that the single bit specified by UnknownBits
11990 // could be set on input to the CTLZ node. If this bit is set, the SRL
11991 // will return 0, if it is clear, it returns 1. Change the CTLZ/SRL pair
11992 // to an SRL/XOR pair, which is likely to simplify more.
11993 unsigned ShAmt = UnknownBits.countr_zero();
11994 SDValue Op = N0.getOperand(0);
11995
11996 if (ShAmt) {
11997 SDLoc DL(N0);
11998 Op = DAG.getNode(ISD::SRL, DL, VT, Op,
11999 DAG.getShiftAmountConstant(ShAmt, VT, DL));
12000 AddToWorklist(Op.getNode());
12001 }
12002 return DAG.getNode(ISD::XOR, DL, VT, Op, DAG.getConstant(1, DL, VT));
12003 }
12004 }
12005
12006 // fold (srl x, (trunc (and y, c))) -> (srl x, (and (trunc y), (trunc c))).
12007 if (N1.getOpcode() == ISD::TRUNCATE &&
12008 N1.getOperand(0).getOpcode() == ISD::AND) {
12009 if (SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode()))
12010 return DAG.getNode(ISD::SRL, DL, VT, N0, NewOp1);
12011 }
12012
12013 // fold (srl (logic_op x, (shl (zext y), c1)), c1)
12014 // -> (logic_op (srl x, c1), (zext y))
12015 // c1 <= leadingzeros(zext(y))
12016 // TODO: Replace c1 with valuetracking?
12017 SDValue X, ZExtY;
12018 if (sd_match(
12019 N0,
12021 m_Value(X),
12023 m_Specific(N1))))))) {
12024 unsigned NumLeadingZeros = ZExtY.getScalarValueSizeInBits() -
12026 if (N1C && N1C->getZExtValue() <= NumLeadingZeros)
12027 return DAG.getNode(N0.getOpcode(), SDLoc(N0), VT,
12028 DAG.getNode(ISD::SRL, SDLoc(N0), VT, X, N1), ZExtY);
12029 }
12030
12031 // fold (srl (bitcast (build_vector e1, ..., eN)), (N-1) * eltsize)
12032 // -> (zext eN)
12033 if (N1C && VT.isScalarInteger() && DAG.getDataLayout().isLittleEndian()) {
12035 if (BV.getOpcode() == ISD::BUILD_VECTOR) {
12036 EVT BVVT = BV.getValueType();
12037 unsigned EltSizeInBits = BVVT.getScalarSizeInBits();
12038 unsigned NumElts = BVVT.getVectorNumElements();
12039 if (N1C->getZExtValue() == (NumElts - 1) * EltSizeInBits) {
12040 SDValue LastElt = BV.getOperand(NumElts - 1);
12041 assert(LastElt.getScalarValueSizeInBits() >= EltSizeInBits &&
12042 "Expected BUILD_VECTOR operand as wide as element type");
12043 EVT IntEltVT = LastElt.getValueType().changeTypeToInteger();
12044 if (!LegalTypes || TLI.isTypeLegal(IntEltVT)) {
12045 LastElt = DAG.getBitcast(IntEltVT, LastElt);
12046 SDValue Ext = DAG.getZExtOrTrunc(LastElt, DL, VT);
12047 APInt Mask = APInt::getLowBitsSet(VT.getSizeInBits(), EltSizeInBits);
12048 return DAG.getNode(ISD::AND, DL, VT, Ext,
12049 DAG.getConstant(Mask, DL, VT));
12050 }
12051 }
12052 }
12053 }
12054
12055 // fold (srl (add nuw X, C), D) -> (add nuw (srl X, D), C u>> D)
12056 // when C has D trailing zeros (so C >> D is exact).
12057 if (N1C && N0.hasOneUse() && N0.getOpcode() == ISD::ADD &&
12058 N0->getFlags().hasNoUnsignedWrap()) {
12059 if (ConstantSDNode *AddC = isConstOrConstSplat(N0.getOperand(1))) {
12060 const APInt &ShAmt = N1C->getAPIntValue();
12061 const APInt &AddVal = AddC->getAPIntValue();
12062 if (ShAmt.ult(AddVal.countr_zero())) {
12063 SDNodeFlags ShiftFlags = N->getFlags();
12064 SDValue NewSrl =
12065 DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), N1, ShiftFlags);
12066 SDValue NewC = DAG.getConstant(AddVal.lshr(ShAmt), DL, VT);
12067 SDNodeFlags AddFlags = N0->getFlags();
12068 return DAG.getNode(ISD::ADD, DL, VT, NewSrl, NewC, AddFlags);
12069 }
12070 }
12071 }
12072
12073 // fold operands of srl based on knowledge that the low bits are not
12074 // demanded.
12076 return SDValue(N, 0);
12077
12078 if (N1C && !N1C->isOpaque())
12079 if (SDValue NewSRL = visitShiftByConstant(N))
12080 return NewSRL;
12081
12082 // Attempt to convert a srl of a load into a narrower zero-extending load.
12083 if (SDValue NarrowLoad = reduceLoadWidth(N))
12084 return NarrowLoad;
12085
12086 // Here is a common situation. We want to optimize:
12087 //
12088 // %a = ...
12089 // %b = and i32 %a, 2
12090 // %c = srl i32 %b, 1
12091 // brcond i32 %c ...
12092 //
12093 // into
12094 //
12095 // %a = ...
12096 // %b = and %a, 2
12097 // %c = setcc eq %b, 0
12098 // brcond %c ...
12099 //
12100 // However when after the source operand of SRL is optimized into AND, the SRL
12101 // itself may not be optimized further. Look for it and add the BRCOND into
12102 // the worklist.
12103 //
12104 // The also tends to happen for binary operations when SimplifyDemandedBits
12105 // is involved.
12106 //
12107 // FIXME: This is unecessary if we process the DAG in topological order,
12108 // which we plan to do. This workaround can be removed once the DAG is
12109 // processed in topological order.
12110 if (N->hasOneUse()) {
12111 SDNode *User = *N->user_begin();
12112
12113 // Look pass the truncate.
12114 if (User->getOpcode() == ISD::TRUNCATE && User->hasOneUse())
12115 User = *User->user_begin();
12116
12117 if (User->getOpcode() == ISD::BRCOND || User->getOpcode() == ISD::AND ||
12118 User->getOpcode() == ISD::OR || User->getOpcode() == ISD::XOR)
12119 AddToWorklist(User);
12120 }
12121
12122 // Try to transform this shift into a multiply-high if
12123 // it matches the appropriate pattern detected in combineShiftToMULH.
12124 if (SDValue MULH = combineShiftToMULH(N, DL, DAG, TLI))
12125 return MULH;
12126
12127 if (SDValue AVG = foldShiftToAvg(N, DL))
12128 return AVG;
12129
12130 SDValue Y;
12131 if (VT.getScalarSizeInBits() % 2 == 0 && N1C) {
12132 // Fold clmul(zext(x), zext(y)) >> (BW - 1 | BW) -> clmul(r|h)(x, y).
12133 unsigned HalfBW = VT.getScalarSizeInBits() / 2;
12134 if (sd_match(N0, m_Clmul(m_ZExt(m_Value(X)), m_ZExt(m_Value(Y)))) &&
12135 X.getScalarValueSizeInBits() == HalfBW &&
12136 Y.getScalarValueSizeInBits() == HalfBW) {
12137 if (N1C->getZExtValue() == HalfBW - 1 &&
12138 (!LegalOperations ||
12139 TLI.isOperationLegalOrCustom(ISD::CLMULR, X.getValueType())))
12140 return DAG.getNode(
12141 ISD::ZERO_EXTEND, DL, VT,
12142 DAG.getNode(ISD::CLMULR, DL, X.getValueType(), X, Y));
12143 if (N1C->getZExtValue() == HalfBW &&
12144 (!LegalOperations ||
12145 TLI.isOperationLegalOrCustom(ISD::CLMULH, X.getValueType())))
12146 return DAG.getNode(
12147 ISD::ZERO_EXTEND, DL, VT,
12148 DAG.getNode(ISD::CLMULH, DL, X.getValueType(), X, Y));
12149 }
12150 }
12151
12152 // Fold bitreverse(clmul(bitreverse(x), bitreverse(y))) >> 1 ->
12153 // clmulh(x, y).
12154 if (N1C && N1C->getZExtValue() == 1 &&
12156 m_BitReverse(m_Value(Y))))))
12157 return DAG.getNode(ISD::CLMULH, DL, VT, X, Y);
12158
12159 return SDValue();
12160}
12161
12162SDValue DAGCombiner::visitFunnelShift(SDNode *N) {
12163 EVT VT = N->getValueType(0);
12164 SDValue N0 = N->getOperand(0);
12165 SDValue N1 = N->getOperand(1);
12166 SDValue N2 = N->getOperand(2);
12167 bool IsFSHL = N->getOpcode() == ISD::FSHL;
12168 unsigned BitWidth = VT.getScalarSizeInBits();
12169 SDLoc DL(N);
12170
12171 // fold (fshl/fshr C0, C1, C2) -> C3
12172 if (SDValue C =
12173 DAG.FoldConstantArithmetic(N->getOpcode(), DL, VT, {N0, N1, N2}))
12174 return C;
12175
12176 // fold (fshl N0, N1, 0) -> N0
12177 // fold (fshr N0, N1, 0) -> N1
12179 if (DAG.MaskedValueIsZero(
12180 N2, APInt(N2.getScalarValueSizeInBits(), BitWidth - 1)))
12181 return IsFSHL ? N0 : N1;
12182
12183 auto IsUndefOrZero = [](SDValue V) {
12184 return V.isUndef() || isNullOrNullSplat(V, /*AllowUndefs*/ true);
12185 };
12186
12187 // TODO - support non-uniform vector shift amounts.
12188 if (ConstantSDNode *Cst = isConstOrConstSplat(N2)) {
12189 EVT ShAmtTy = N2.getValueType();
12190
12191 // fold (fsh* N0, N1, c) -> (fsh* N0, N1, c % BitWidth)
12192 if (Cst->getAPIntValue().uge(BitWidth)) {
12193 uint64_t RotAmt = Cst->getAPIntValue().urem(BitWidth);
12194 return DAG.getNode(N->getOpcode(), DL, VT, N0, N1,
12195 DAG.getConstant(RotAmt, DL, ShAmtTy));
12196 }
12197
12198 unsigned ShAmt = Cst->getZExtValue();
12199 if (ShAmt == 0)
12200 return IsFSHL ? N0 : N1;
12201
12202 // fold fshl(undef_or_zero, N1, C) -> lshr(N1, BW-C)
12203 // fold fshr(undef_or_zero, N1, C) -> lshr(N1, C)
12204 // fold fshl(N0, undef_or_zero, C) -> shl(N0, C)
12205 // fold fshr(N0, undef_or_zero, C) -> shl(N0, BW-C)
12206 if (IsUndefOrZero(N0))
12207 return DAG.getNode(
12208 ISD::SRL, DL, VT, N1,
12209 DAG.getConstant(IsFSHL ? BitWidth - ShAmt : ShAmt, DL, ShAmtTy));
12210 if (IsUndefOrZero(N1))
12211 return DAG.getNode(
12212 ISD::SHL, DL, VT, N0,
12213 DAG.getConstant(IsFSHL ? ShAmt : BitWidth - ShAmt, DL, ShAmtTy));
12214
12215 // fold fshl(N0, N1, c) -> x and fshr(N0, N1, c) -> x
12216 // where N0 is any node that contributes "x >> C0" to the result:
12217 // lshr(x, C0) | fshr(_, x, C0) | fshl(_, x, C1)
12218 // and N1 is any node that contributes "x << C1" to the result:
12219 // shl(x, C1) | fshl(x, _, C1) | fshr(x, _, C0)
12220 // with C0 = IsFSHL ? amnt : BW-amnt, C1 = BW - C0
12221
12222 // ShAmt == 0 was handled above; uge(BitWidth) was reduced via modulo above.
12223 assert(ShAmt >= 1 && ShAmt < BitWidth &&
12224 "ShAmt must be in [1, BW-1] for the identity fold to be valid");
12225 SDValue Val;
12226 unsigned C0Expected = IsFSHL ? ShAmt : BitWidth - ShAmt;
12227 unsigned C1Expected = IsFSHL ? BitWidth - ShAmt : ShAmt;
12228
12229 if ((sd_match(N0, m_Srl(m_Value(Val), m_SpecificInt(C0Expected))) ||
12230 sd_match(N0,
12231 m_FShR(m_Value(), m_Value(Val), m_SpecificInt(C0Expected))) ||
12232 sd_match(
12233 N0, m_FShL(m_Value(), m_Value(Val), m_SpecificInt(C1Expected)))) &&
12234 (sd_match(N1, m_Shl(m_Specific(Val), m_SpecificInt(C1Expected))) ||
12235 sd_match(N1, m_FShL(m_Specific(Val), m_Value(),
12236 m_SpecificInt(C1Expected))) ||
12237 sd_match(N1, m_FShR(m_Specific(Val), m_Value(),
12238 m_SpecificInt(C0Expected)))))
12239 return Val;
12240
12241 // fold (fshl ld1, ld0, c) -> (ld0[ofs]) iff ld0 and ld1 are consecutive.
12242 // fold (fshr ld1, ld0, c) -> (ld0[ofs]) iff ld0 and ld1 are consecutive.
12243 // TODO - bigendian support once we have test coverage.
12244 // TODO - can we merge this with CombineConseutiveLoads/MatchLoadCombine?
12245 // TODO - permit LHS EXTLOAD if extensions are shifted out.
12246 if ((BitWidth % 8) == 0 && (ShAmt % 8) == 0 && !VT.isVector() &&
12247 !DAG.getDataLayout().isBigEndian()) {
12248 auto *LHS = dyn_cast<LoadSDNode>(N0);
12249 auto *RHS = dyn_cast<LoadSDNode>(N1);
12250 if (LHS && RHS && LHS->isSimple() && RHS->isSimple() &&
12251 LHS->getAddressSpace() == RHS->getAddressSpace() &&
12252 (LHS->hasNUsesOfValue(1, 0) || RHS->hasNUsesOfValue(1, 0)) &&
12254 if (DAG.areNonVolatileConsecutiveLoads(LHS, RHS, BitWidth / 8, 1)) {
12255 SDLoc DL(RHS);
12256 uint64_t PtrOff =
12257 IsFSHL ? (((BitWidth - ShAmt) % BitWidth) / 8) : (ShAmt / 8);
12258 Align NewAlign = commonAlignment(RHS->getAlign(), PtrOff);
12259 unsigned Fast = 0;
12260 if (TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT,
12261 RHS->getAddressSpace(), NewAlign,
12262 RHS->getMemOperand()->getFlags(), &Fast) &&
12263 Fast) {
12264 SDValue NewPtr = DAG.getMemBasePlusOffset(
12265 RHS->getBasePtr(), TypeSize::getFixed(PtrOff), DL);
12266 AddToWorklist(NewPtr.getNode());
12267 const MDNode *MemCacheHint = getCommonMemCacheHint({LHS, RHS});
12268 SDValue Load = DAG.getLoad(
12269 VT, DL, RHS->getChain(), NewPtr,
12270 RHS->getPointerInfo().getWithOffset(PtrOff), NewAlign,
12271 RHS->getMemOperand()->getFlags(),
12272 MMOMetadata(RHS->getAAInfo(), nullptr, MemCacheHint));
12273 DAG.makeEquivalentMemoryOrdering(LHS, Load.getValue(1));
12274 DAG.makeEquivalentMemoryOrdering(RHS, Load.getValue(1));
12275 return Load;
12276 }
12277 }
12278 }
12279 }
12280 }
12281
12282 // fold fshr(undef_or_zero, N1, N2) -> lshr(N1, N2)
12283 // fold fshl(N0, undef_or_zero, N2) -> shl(N0, N2)
12284 // iff We know the shift amount is in range.
12285 // TODO: when is it worth doing SUB(BW, N2) as well?
12286 if (isPowerOf2_32(BitWidth)) {
12287 APInt ModuloBits(N2.getScalarValueSizeInBits(), BitWidth - 1);
12288 if (IsUndefOrZero(N0) && !IsFSHL && DAG.MaskedValueIsZero(N2, ~ModuloBits))
12289 return DAG.getNode(ISD::SRL, DL, VT, N1, N2);
12290 if (IsUndefOrZero(N1) && IsFSHL && DAG.MaskedValueIsZero(N2, ~ModuloBits))
12291 return DAG.getNode(ISD::SHL, DL, VT, N0, N2);
12292 }
12293
12294 // fold (fshl N0, N0, N2) -> (rotl N0, N2)
12295 // fold (fshr N0, N0, N2) -> (rotr N0, N2)
12296 // TODO: Investigate flipping this rotate if only one is legal.
12297 // If funnel shift is legal as well we might be better off avoiding
12298 // non-constant (BW - N2).
12299 unsigned RotOpc = IsFSHL ? ISD::ROTL : ISD::ROTR;
12300 if (N0 == N1 && hasOperation(RotOpc, VT))
12301 return DAG.getNode(RotOpc, DL, VT, N0, N2);
12302
12303 // Simplify, based on bits shifted out of N0/N1.
12305 return SDValue(N, 0);
12306
12307 return SDValue();
12308}
12309
12310SDValue DAGCombiner::visitSHLSAT(SDNode *N) {
12311 SDValue N0 = N->getOperand(0);
12312 SDValue N1 = N->getOperand(1);
12313 if (SDValue V = DAG.simplifyShift(N0, N1))
12314 return V;
12315
12316 SDLoc DL(N);
12317 EVT VT = N0.getValueType();
12318
12319 // fold (*shlsat c1, c2) -> c1<<c2
12320 if (SDValue C = DAG.FoldConstantArithmetic(N->getOpcode(), DL, VT, {N0, N1}))
12321 return C;
12322
12323 ConstantSDNode *N1C = isConstOrConstSplat(N1);
12324
12325 if (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::SHL, VT)) {
12326 // fold (sshlsat x, c) -> (shl x, c)
12327 if (N->getOpcode() == ISD::SSHLSAT && N1C &&
12328 N1C->getAPIntValue().ult(DAG.ComputeNumSignBits(N0)))
12329 return DAG.getNode(ISD::SHL, DL, VT, N0, N1);
12330
12331 // fold (ushlsat x, c) -> (shl x, c)
12332 if (N->getOpcode() == ISD::USHLSAT && N1C &&
12333 N1C->getAPIntValue().ule(
12335 return DAG.getNode(ISD::SHL, DL, VT, N0, N1);
12336 }
12337
12338 return SDValue();
12339}
12340
12341// Given a ABS node, detect the following patterns:
12342// (ABS (SUB (EXTEND a), (EXTEND b))).
12343// (TRUNC (ABS (SUB (EXTEND a), (EXTEND b)))).
12344// Generates UABD/SABD instruction.
12345SDValue DAGCombiner::foldABSToABD(SDNode *N, const SDLoc &DL) {
12346 EVT SrcVT = N->getValueType(0);
12347
12348 if (N->getOpcode() == ISD::TRUNCATE)
12349 N = N->getOperand(0).getNode();
12350
12351 EVT VT = N->getValueType(0);
12352 SDValue Op0, Op1;
12353
12354 if (!sd_match(N, m_Abs(m_AnyOf(m_Sub(m_Value(Op0), m_Value(Op1)),
12355 m_Add(m_Value(Op0), m_Value(Op1))))))
12356 return SDValue();
12357
12358 SDValue AbsOp0 = N->getOperand(0);
12359 bool IsAdd = AbsOp0.getOpcode() == ISD::ADD;
12360 // Make sure (abs B) is positive.
12361 if (IsAdd) {
12362 // Elements of Op1 must be constant and != VT.minSignedValue() (or undef)
12363 auto IsNotMinSignedInt = [VT](ConstantSDNode *C) {
12364 if (C == nullptr)
12365 return true;
12366 return !C->getAPIntValue()
12367 .trunc(VT.getScalarSizeInBits())
12368 .isMinSignedValue();
12369 };
12370
12371 if (!ISD::matchUnaryPredicate(Op1, IsNotMinSignedInt, /*AllowUndefs=*/true,
12372 /*AllowTruncation=*/true))
12373 return SDValue();
12374 }
12375
12376 unsigned Opc0 = Op0.getOpcode();
12377
12378 // Check if the operands of the sub are (zero|sign)-extended, otherwise
12379 // fallback to ValueTracking.
12380 if (Opc0 != Op1.getOpcode() ||
12381 (Opc0 != ISD::ZERO_EXTEND && Opc0 != ISD::SIGN_EXTEND &&
12382 Opc0 != ISD::SIGN_EXTEND_INREG)) {
12383
12384 auto CreateZextedAbd = [&](unsigned AbdOpc) {
12385 if (IsAdd)
12386 Op1 = DAG.getNegative(Op1, SDLoc(Op1), VT);
12387 SDValue ABD = DAG.getNode(AbdOpc, DL, VT, Op0, Op1);
12388 return DAG.getZExtOrTrunc(ABD, DL, SrcVT);
12389 };
12390
12391 // fold (abs (sub nsw x, y)) -> abds(x, y)
12392 // fold (abs (add nsw x, -y)) -> abds(x, y)
12393 // Don't fold this for unsupported types as we lose the NSW handling.
12394 if (hasOperation(ISD::ABDS, VT) && TLI.preferABDSToABSWithNSW(VT) &&
12395 (AbsOp0->getFlags().hasNoSignedWrap() ||
12396 (IsAdd ? DAG.willNotOverflowAdd(/*IsSigned=*/true, Op0, Op1)
12397 : DAG.willNotOverflowSub(/*IsSigned=*/true, Op0, Op1))))
12398 return CreateZextedAbd(ISD::ABDS);
12399
12400 // fold (abs (sub x, y)) -> abdu(x, y)
12401 if (!IsAdd && hasOperation(ISD::ABDU, VT) && DAG.SignBitIsZero(Op0) &&
12402 DAG.SignBitIsZero(Op1))
12403 return CreateZextedAbd(ISD::ABDU);
12404
12405 return SDValue();
12406 }
12407
12408 // The IsAdd case explicitly checks for const/bv-of-const. This implies either
12409 // (Opc0 != Op1.getOpcode() || Opc0 is not in {zext/sext/sign_ext_inreg}. This
12410 // implies it was alrady handled by the above if statement.
12411 assert(!IsAdd && "Unexpected abs(add(x,y)) pattern");
12412
12413 EVT VT0, VT1;
12414 if (Opc0 == ISD::SIGN_EXTEND_INREG) {
12415 VT0 = cast<VTSDNode>(Op0.getOperand(1))->getVT();
12416 VT1 = cast<VTSDNode>(Op1.getOperand(1))->getVT();
12417 } else {
12418 VT0 = Op0.getOperand(0).getValueType();
12419 VT1 = Op1.getOperand(0).getValueType();
12420 }
12421 unsigned ABDOpcode = (Opc0 == ISD::ZERO_EXTEND) ? ISD::ABDU : ISD::ABDS;
12422
12423 // fold abs(sext(x) - sext(y)) -> zext(abds(x, y))
12424 // fold abs(zext(x) - zext(y)) -> zext(abdu(x, y))
12425 EVT MaxVT = VT0.bitsGT(VT1) ? VT0 : VT1;
12426 if ((VT0 == MaxVT || Op0->hasOneUse()) &&
12427 (VT1 == MaxVT || Op1->hasOneUse()) &&
12428 (!LegalTypes || hasOperation(ABDOpcode, MaxVT))) {
12429 SDValue ABD = DAG.getNode(ABDOpcode, DL, MaxVT,
12430 DAG.getNode(ISD::TRUNCATE, DL, MaxVT, Op0),
12431 DAG.getNode(ISD::TRUNCATE, DL, MaxVT, Op1));
12432 ABD = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, ABD);
12433 return DAG.getZExtOrTrunc(ABD, DL, SrcVT);
12434 }
12435
12436 // fold abs(sext(x) - sext(y)) -> abds(sext(x), sext(y))
12437 // fold abs(zext(x) - zext(y)) -> abdu(zext(x), zext(y))
12438 if (!LegalOperations || hasOperation(ABDOpcode, VT)) {
12439 SDValue ABD = DAG.getNode(ABDOpcode, DL, VT, Op0, Op1);
12440 return DAG.getZExtOrTrunc(ABD, DL, SrcVT);
12441 }
12442
12443 return SDValue();
12444}
12445
12446SDValue DAGCombiner::visitABS(SDNode *N) {
12447 SDValue N0 = N->getOperand(0);
12448 EVT VT = N->getValueType(0);
12449 SDLoc DL(N);
12450
12451 // fold (abs c1) -> c2
12452 if (SDValue C = DAG.FoldConstantArithmetic(ISD::ABS, DL, VT, {N0}))
12453 return C;
12454 // fold (abs (abs x)) -> (abs x)
12455 // fold (abs (abs_min_poison x)) -> (abs_min_poison x)
12456 if (ISD::isAbsOpcode(N0.getOpcode()))
12457 return N0;
12458 // fold (abs x) -> x iff not-negative
12459 if (DAG.SignBitIsZero(N0))
12460 return N0;
12461
12462 if (SDValue ABD = foldABSToABD(N, DL))
12463 return ABD;
12464
12465 // fold (abs (sign_extend_inreg x)) -> (zero_extend (abs (truncate x)))
12466 // iff zero_extend/truncate are free.
12467 if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG) {
12468 EVT ExtVT = cast<VTSDNode>(N0.getOperand(1))->getVT();
12469 if (TLI.isTruncateFree(VT, ExtVT) && TLI.isZExtFree(ExtVT, VT) &&
12471 hasOperation(ISD::ABS, ExtVT)) {
12472 return DAG.getNode(
12473 ISD::ZERO_EXTEND, DL, VT,
12474 DAG.getNode(ISD::ABS, DL, ExtVT,
12475 DAG.getNode(ISD::TRUNCATE, DL, ExtVT, N0.getOperand(0))));
12476 }
12477 }
12478
12479 return SDValue();
12480}
12481
12482SDValue DAGCombiner::visitABS_MIN_POISON(SDNode *N) {
12483 SDValue N0 = N->getOperand(0);
12484 EVT VT = N->getValueType(0);
12485 SDLoc DL(N);
12486
12487 // fold (abs_min_poison c1) -> c2 (or poison if c1 == INT_MIN)
12489 return C;
12490 // fold (abs_min_poison (abs_min_poison x)) -> (abs_min_poison x)
12491 // fold (abs_min_poison (abs x)) -> (abs x)
12492 // fold (abs_min_poison (freeze (abs x))) -> (freeze (abs x))
12493 // fold (abs_min_poison (freeze (abs_min_poison x))) ->
12494 // (freeze (abs_min_poison x))
12495 //
12496 // Freeze case is valid because: for x != INT_MIN both sides equal abs(x);
12497 // for x == INT_MIN both forms produce a non-deterministic but well-defined
12498 // value since freeze already consumed the poison.
12500 return N0;
12501 // fold (abs_min_poison x) -> x iff not-negative
12502 if (DAG.SignBitIsZero(N0))
12503 return N0;
12504
12505 if (SDValue ABD = foldABSToABD(N, DL))
12506 return ABD;
12507
12508 // fold (abs_min_poison (sign_extend_inreg x)) ->
12509 // (zero_extend (abs (truncate x)))
12510 // iff zero_extend/truncate are free. The sign_extend_inreg keeps the value
12511 // in the narrow type's range, so the wide abs_min_poison is never actually
12512 // poison.
12513 if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG) {
12514 EVT ExtVT = cast<VTSDNode>(N0.getOperand(1))->getVT();
12515 if (TLI.isTruncateFree(VT, ExtVT) && TLI.isZExtFree(ExtVT, VT) &&
12517 hasOperation(ISD::ABS, ExtVT)) {
12518 return DAG.getNode(
12519 ISD::ZERO_EXTEND, DL, VT,
12520 DAG.getNode(ISD::ABS, DL, ExtVT,
12521 DAG.getNode(ISD::TRUNCATE, DL, ExtVT, N0.getOperand(0))));
12522 }
12523 }
12524
12525 return SDValue();
12526}
12527
12528SDValue DAGCombiner::visitCLMUL(SDNode *N) {
12529 unsigned Opcode = N->getOpcode();
12530 SDValue N0 = N->getOperand(0);
12531 SDValue N1 = N->getOperand(1);
12532 EVT VT = N->getValueType(0);
12533 SDLoc DL(N);
12534
12535 // fold (clmul c1, c2)
12536 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1}))
12537 return C;
12538
12539 // canonicalize constant to RHS
12542 return DAG.getNode(Opcode, DL, VT, N1, N0);
12543
12544 // fold (clmul x, 0) -> 0
12546 return DAG.getConstant(0, DL, VT);
12547
12548 // fold (clmul x, c_pow2) -> (shl x, log2(c_pow2))
12549 // This also handles (clmul x, 1) -> x since (shl x, 0) simplifies to x.
12550 if (Opcode == ISD::CLMUL) {
12551 if (ConstantSDNode *C = isConstOrConstSplat(N1)) {
12552 APInt CV = C->getAPIntValue().trunc(VT.getScalarSizeInBits());
12553 if (CV.isPowerOf2() &&
12554 (!LegalOperations || TLI.isOperationLegal(ISD::SHL, VT)))
12555 return DAG.getNode(ISD::SHL, DL, VT, N0,
12556 DAG.getShiftAmountConstant(CV.logBase2(), VT, DL));
12557 }
12558 }
12559
12560 return SDValue();
12561}
12562
12563SDValue DAGCombiner::visitPEXT(SDNode *N) {
12564 EVT VT = N->getValueType(0);
12565 SDValue N0 = N->getOperand(0);
12566 SDValue N1 = N->getOperand(1);
12567 SDLoc DL(N);
12568
12569 // pext(x, 0) -> 0
12570 if (isNullOrNullSplat(N1))
12571 return DAG.getConstant(0, DL, VT);
12572 // pext(x, -1) -> x (all bits selected, packed into low positions = x)
12574 return N0;
12575 // fold pext(c1, c2) -> c3
12576 if (SDValue C = DAG.FoldConstantArithmetic(ISD::PEXT, DL, VT, {N0, N1}))
12577 return C;
12578 return SDValue();
12579}
12580
12581SDValue DAGCombiner::visitPDEP(SDNode *N) {
12582 EVT VT = N->getValueType(0);
12583 SDValue N0 = N->getOperand(0);
12584 SDValue N1 = N->getOperand(1);
12585 SDLoc DL(N);
12586
12587 // pdep(x, 0) -> 0
12588 if (isNullOrNullSplat(N1))
12589 return DAG.getConstant(0, DL, VT);
12590
12591 // pdep(x, -1) -> x (all positions selected, bits deposited at identity)
12593 return N0;
12594
12595 // fold pdep(c1, c2) -> c3
12596 if (SDValue C = DAG.FoldConstantArithmetic(ISD::PDEP, DL, VT, {N0, N1}))
12597 return C;
12598
12600 return SDValue(N, 0);
12601
12602 return SDValue();
12603}
12604
12605SDValue DAGCombiner::visitBSWAP(SDNode *N) {
12606 SDValue N0 = N->getOperand(0);
12607 EVT VT = N->getValueType(0);
12608 SDLoc DL(N);
12609
12610 // fold (bswap c1) -> c2
12611 if (SDValue C = DAG.FoldConstantArithmetic(ISD::BSWAP, DL, VT, {N0}))
12612 return C;
12613 // fold (bswap (bswap x)) -> x
12614 if (N0.getOpcode() == ISD::BSWAP)
12615 return N0.getOperand(0);
12616
12617 // Canonicalize bswap(bitreverse(x)) -> bitreverse(bswap(x)). If bitreverse
12618 // isn't supported, it will be expanded to bswap followed by a manual reversal
12619 // of bits in each byte. By placing bswaps before bitreverse, we can remove
12620 // the two bswaps if the bitreverse gets expanded.
12621 if (N0.getOpcode() == ISD::BITREVERSE && N0.hasOneUse()) {
12622 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT, N0.getOperand(0));
12623 return DAG.getNode(ISD::BITREVERSE, DL, VT, BSwap);
12624 }
12625
12626 unsigned BW = VT.getScalarSizeInBits();
12627 // fold (bswap shl(x,c)) -> (zext(bswap(trunc(shl(x,sub(c,bw/2))))))
12628 // iff x >= bw/2 (i.e. lower half is known zero)
12629 if (BW >= 32 && N0.getOpcode() == ISD::SHL && N0.hasOneUse()) {
12630 auto *ShAmt = dyn_cast<ConstantSDNode>(N0.getOperand(1));
12631 EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), BW / 2);
12632 if (ShAmt && ShAmt->getAPIntValue().ult(BW) &&
12633 ShAmt->getZExtValue() >= (BW / 2) && (ShAmt->getZExtValue() % 8) == 0 &&
12634 TLI.isTypeLegal(HalfVT) && TLI.isTruncateFree(VT, HalfVT) &&
12635 (!LegalOperations || hasOperation(ISD::BSWAP, HalfVT))) {
12636 SDValue Res = N0.getOperand(0);
12637 if (uint64_t NewShAmt = (ShAmt->getZExtValue() - (BW / 2)))
12638 Res = DAG.getNode(ISD::SHL, DL, VT, Res,
12639 DAG.getShiftAmountConstant(NewShAmt, VT, DL));
12640 Res = DAG.getZExtOrTrunc(Res, DL, HalfVT);
12641 Res = DAG.getNode(ISD::BSWAP, DL, HalfVT, Res);
12642 return DAG.getZExtOrTrunc(Res, DL, VT);
12643 }
12644 }
12645
12646 // Try to canonicalize bswap-of-logical-shift-by-8-bit-multiple as
12647 // inverse-shift-of-bswap:
12648 // bswap (X u<< C) --> (bswap X) u>> C
12649 // bswap (X u>> C) --> (bswap X) u<< C
12650 if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL) &&
12651 N0.hasOneUse()) {
12652 auto *ShAmt = dyn_cast<ConstantSDNode>(N0.getOperand(1));
12653 if (ShAmt && ShAmt->getAPIntValue().ult(BW) &&
12654 ShAmt->getZExtValue() % 8 == 0) {
12655 SDValue NewSwap = DAG.getNode(ISD::BSWAP, DL, VT, N0.getOperand(0));
12656 unsigned InverseShift = N0.getOpcode() == ISD::SHL ? ISD::SRL : ISD::SHL;
12657 return DAG.getNode(InverseShift, DL, VT, NewSwap, N0.getOperand(1));
12658 }
12659 }
12660
12661 if (SDValue V = foldBitOrderCrossLogicOp(N, DAG))
12662 return V;
12663
12664 // Folds that depend on computeKnownBits of the operand.
12665 KnownBits Known = DAG.computeKnownBits(N0);
12666 // bswap(0) = 0. Catch cases that computeKnownBits can prove are zero but
12667 // that structural combines haven't simplified to a constant yet
12668 // (e.g. and of disjoint byte masks).
12669 if (Known.isZero())
12670 return DAG.getConstant(0, DL, VT);
12671 // If only one byte of the operand may be nonzero, bswap becomes a shift
12672 // to the mirror byte.
12673 unsigned TZ = alignDown(Known.countMinTrailingZeros(), 8);
12674 unsigned LZ = alignDown(Known.countMinLeadingZeros(), 8);
12675 if (BW - (LZ + TZ) == 8) {
12676 unsigned Opc = LZ > TZ ? ISD::SHL : ISD::SRL;
12677 // Skip if the target would re-expand the produced shift post-legalize.
12678 // Targets that custom-lower byte-multiple shifts via bswap (e.g. MSP430
12679 // for shl i16) would loop with this combine.
12680 if (!LegalOperations || hasOperation(Opc, VT)) {
12681 unsigned Amt = AbsoluteDifference(LZ, TZ);
12682 SDNodeFlags Flags =
12684 return DAG.getNode(Opc, DL, VT, N0,
12685 DAG.getShiftAmountConstant(Amt, VT, DL), Flags);
12686 }
12687 }
12688
12689 return SDValue();
12690}
12691
12692SDValue DAGCombiner::visitBITREVERSE(SDNode *N) {
12693 SDValue N0 = N->getOperand(0);
12694 EVT VT = N->getValueType(0);
12695 SDLoc DL(N);
12696
12697 // fold (bitreverse c1) -> c2
12698 if (SDValue C = DAG.FoldConstantArithmetic(ISD::BITREVERSE, DL, VT, {N0}))
12699 return C;
12700
12701 // fold (bitreverse (bitreverse x)) -> x
12702 if (N0.getOpcode() == ISD::BITREVERSE)
12703 return N0.getOperand(0);
12704
12705 SDValue X, Y;
12706
12707 // fold (bitreverse (lshr (bitreverse x), y)) -> (shl x, y)
12708 if ((!LegalOperations || TLI.isOperationLegal(ISD::SHL, VT)) &&
12710 return DAG.getNode(ISD::SHL, DL, VT, X, Y);
12711
12712 // fold (bitreverse (shl (bitreverse x), y)) -> (lshr x, y)
12713 if ((!LegalOperations || TLI.isOperationLegal(ISD::SRL, VT)) &&
12715 return DAG.getNode(ISD::SRL, DL, VT, X, Y);
12716
12717 // fold bitreverse(clmul(bitreverse(x), bitreverse(y))) -> clmulr(x, y)
12718 if ((!LegalOperations || TLI.isOperationLegalOrCustom(ISD::CLMULR, VT)) &&
12720 return DAG.getNode(ISD::CLMULR, DL, VT, X, Y);
12721
12722 return SDValue();
12723}
12724
12725// Fold (ctlz (xor x, (sra x, bitwidth-1))) -> (add (ctls x), 1).
12726// Fold (ctlz (or (shl (xor x, (sra x, bitwidth-1)), 1), 1) -> (ctls x)
12727SDValue DAGCombiner::foldCTLZToCTLS(SDValue Src, const SDLoc &DL) {
12728 EVT VT = Src.getValueType();
12729
12730 auto LK = TLI.getTypeConversion(*DAG.getContext(), VT);
12731 if ((LK.first != TargetLoweringBase::TypeLegal &&
12734 return SDValue();
12735
12736 unsigned BitWidth = VT.getScalarSizeInBits();
12737
12738 bool NeedAdd = true;
12739
12740 SDValue X;
12741 if (sd_match(Src,
12743 NeedAdd = false;
12744 Src = X;
12745 }
12746
12747 if (!sd_match(Src,
12750 m_SpecificInt(BitWidth - 1)))))))
12751 return SDValue();
12752
12753 SDValue Res = DAG.getNode(ISD::CTLS, DL, VT, X);
12754 if (!NeedAdd)
12755 return Res;
12756
12757 return DAG.getNode(ISD::ADD, DL, VT, Res, DAG.getConstant(1, DL, VT));
12758}
12759
12760SDValue DAGCombiner::visitCTLZ(SDNode *N) {
12761 SDValue N0 = N->getOperand(0);
12762 EVT VT = N->getValueType(0);
12763 SDLoc DL(N);
12764
12765 // fold (ctlz c1) -> c2
12766 if (SDValue C = DAG.FoldConstantArithmetic(ISD::CTLZ, DL, VT, {N0}))
12767 return C;
12768
12769 // If the value is known never to be zero, switch to the poison version.
12770 if (!LegalOperations || TLI.isOperationLegal(ISD::CTLZ_ZERO_POISON, VT))
12771 if (DAG.isKnownNeverZero(N0))
12772 return DAG.getNode(ISD::CTLZ_ZERO_POISON, DL, VT, N0);
12773
12774 if (SDValue V = foldCTLZToCTLS(N0, DL))
12775 return V;
12776
12777 return SDValue();
12778}
12779
12780SDValue DAGCombiner::visitCTLZ_ZERO_POISON(SDNode *N) {
12781 SDValue N0 = N->getOperand(0);
12782 EVT VT = N->getValueType(0);
12783 SDLoc DL(N);
12784
12785 // fold (ctlz_zero_poison c1) -> c2
12786 if (SDValue C =
12788 return C;
12789
12790 if (SDValue V = foldCTLZToCTLS(N0, DL))
12791 return V;
12792
12793 return SDValue();
12794}
12795
12796SDValue DAGCombiner::visitCTTZ(SDNode *N) {
12797 SDValue N0 = N->getOperand(0);
12798 EVT VT = N->getValueType(0);
12799 SDLoc DL(N);
12800
12801 // fold (cttz c1) -> c2
12802 if (SDValue C = DAG.FoldConstantArithmetic(ISD::CTTZ, DL, VT, {N0}))
12803 return C;
12804
12805 // If the value is known never to be zero, switch to the poison version.
12806 if (!LegalOperations || TLI.isOperationLegal(ISD::CTTZ_ZERO_POISON, VT))
12807 if (DAG.isKnownNeverZero(N0))
12808 return DAG.getNode(ISD::CTTZ_ZERO_POISON, DL, VT, N0);
12809
12810 return SDValue();
12811}
12812
12813SDValue DAGCombiner::visitCTTZ_ZERO_POISON(SDNode *N) {
12814 SDValue N0 = N->getOperand(0);
12815 EVT VT = N->getValueType(0);
12816 SDLoc DL(N);
12817
12818 // fold (cttz_zero_poison c1) -> c2
12819 if (SDValue C =
12821 return C;
12822 return SDValue();
12823}
12824
12825SDValue DAGCombiner::visitPARITY(SDNode *N) {
12826 SDValue N0 = N->getOperand(0);
12827 EVT VT = N->getValueType(0);
12828
12829 // fold (parity c1) -> c2
12830 if (SDValue C = DAG.FoldConstantArithmetic(ISD::PARITY, SDLoc(N), VT, {N0}))
12831 return C;
12832
12833 return SDValue();
12834}
12835
12836SDValue DAGCombiner::visitCTPOP(SDNode *N) {
12837 SDValue N0 = N->getOperand(0);
12838 EVT VT = N->getValueType(0);
12839 unsigned NumBits = VT.getScalarSizeInBits();
12840 SDLoc DL(N);
12841
12842 // fold (ctpop c1) -> c2
12843 if (SDValue C = DAG.FoldConstantArithmetic(ISD::CTPOP, DL, VT, {N0}))
12844 return C;
12845
12846 // If the source is being shifted, but doesn't affect any active bits,
12847 // then we can call CTPOP on the shift source directly.
12848 if (N0.getOpcode() == ISD::SRL || N0.getOpcode() == ISD::SHL) {
12849 if (ConstantSDNode *AmtC = isConstOrConstSplat(N0.getOperand(1))) {
12850 const APInt &Amt = AmtC->getAPIntValue();
12851 if (Amt.ult(NumBits)) {
12852 KnownBits KnownSrc = DAG.computeKnownBits(N0.getOperand(0));
12853 if ((N0.getOpcode() == ISD::SRL &&
12854 Amt.ule(KnownSrc.countMinTrailingZeros())) ||
12855 (N0.getOpcode() == ISD::SHL &&
12856 Amt.ule(KnownSrc.countMinLeadingZeros()))) {
12857 return DAG.getNode(ISD::CTPOP, DL, VT, N0.getOperand(0));
12858 }
12859 }
12860 }
12861 }
12862
12863 // If the upper bits are known to be zero, then see if its profitable to
12864 // only count the lower bits.
12865 if (VT.isScalarInteger() && NumBits > 8 && (NumBits & 1) == 0) {
12866 EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), NumBits / 2);
12867 if (hasOperation(ISD::CTPOP, HalfVT) &&
12869 TLI.isTruncateFree(N0, HalfVT) && TLI.isZExtFree(HalfVT, VT)) {
12870 APInt UpperBits = APInt::getHighBitsSet(NumBits, NumBits / 2);
12871 if (DAG.MaskedValueIsZero(N0, UpperBits)) {
12872 SDValue PopCnt = DAG.getNode(ISD::CTPOP, DL, HalfVT,
12873 DAG.getZExtOrTrunc(N0, DL, HalfVT));
12874 return DAG.getZExtOrTrunc(PopCnt, DL, VT);
12875 }
12876 }
12877 }
12878
12879 return SDValue();
12880}
12881
12883 SDValue RHS,
12884 const SDNodeFlags SelectFlags,
12885 const SDNodeFlags CmpFlags,
12886 const TargetLowering &TLI) {
12887 EVT VT = LHS.getValueType();
12888 if (!VT.isFloatingPoint())
12889 return false;
12890
12891 return SelectFlags.hasNoSignedZeros() &&
12892 TLI.isProfitableToCombineMinNumMaxNum(VT) &&
12893 (SelectFlags.hasNoNaNs() || CmpFlags.hasNoNaNs() ||
12894 (DAG.isKnownNeverNaN(RHS) && DAG.isKnownNeverNaN(LHS)));
12895}
12896
12900 const TargetLowering &TLI,
12901 SelectionDAG &DAG) {
12902 EVT TransformVT = TLI.getLegalTypeToTransformTo(*DAG.getContext(), VT);
12903
12904 // We have checked nnan and nsz as pre-conditions for the transform.
12906
12907 switch (CC) {
12908 case ISD::SETOLT:
12909 case ISD::SETOLE:
12910 case ISD::SETLT:
12911 case ISD::SETLE:
12912 case ISD::SETULT:
12913 case ISD::SETULE: {
12914 // Since it's known never nan to get here already, either fminnum or
12915 // fminnum_ieee are OK. Try the ieee version first, since it's fminnum is
12916 // expanded in terms of it.
12917 unsigned IEEEOpcode = (LHS == True) ? ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE;
12918 if (TLI.isOperationLegalOrCustom(IEEEOpcode, VT))
12919 return DAG.getNode(IEEEOpcode, DL, VT, LHS, RHS, Flags);
12920
12921 unsigned Opcode = (LHS == True) ? ISD::FMINNUM : ISD::FMAXNUM;
12922 if (TLI.isOperationLegalOrCustom(Opcode, TransformVT))
12923 return DAG.getNode(Opcode, DL, VT, LHS, RHS, Flags);
12924 return SDValue();
12925 }
12926 case ISD::SETOGT:
12927 case ISD::SETOGE:
12928 case ISD::SETGT:
12929 case ISD::SETGE:
12930 case ISD::SETUGT:
12931 case ISD::SETUGE: {
12932 unsigned IEEEOpcode = (LHS == True) ? ISD::FMAXNUM_IEEE : ISD::FMINNUM_IEEE;
12933 if (TLI.isOperationLegalOrCustom(IEEEOpcode, VT))
12934 return DAG.getNode(IEEEOpcode, DL, VT, LHS, RHS, Flags);
12935
12936 unsigned Opcode = (LHS == True) ? ISD::FMAXNUM : ISD::FMINNUM;
12937 if (TLI.isOperationLegalOrCustom(Opcode, TransformVT))
12938 return DAG.getNode(Opcode, DL, VT, LHS, RHS, Flags);
12939 return SDValue();
12940 }
12941 default:
12942 return SDValue();
12943 }
12944}
12945
12946// Convert (sr[al] (add n[su]w x, y)) -> (avgfloor[su] x, y)
12947// Convert (sr[al] (sub n[su]w x, y)) -> (avgsubfloor[su] x, y)
12948SDValue DAGCombiner::foldShiftToAvg(SDNode *N, const SDLoc &DL) {
12949 const unsigned Opcode = N->getOpcode();
12950 if (Opcode != ISD::SRA && Opcode != ISD::SRL)
12951 return SDValue();
12952
12953 EVT VT = N->getValueType(0);
12954 bool IsUnsigned = Opcode == ISD::SRL;
12955
12956 // Captured values.
12957 SDValue A, B;
12958
12959 // Match floor average as it is common to both floor/ceil avgs, ensure the add
12960 // doesn't wrap.
12961 SDNodeFlags Flags =
12963 if (sd_match(N, m_BinOp(Opcode,
12964 m_c_BinOp(ISD::ADD, m_Value(A), m_Value(B), Flags),
12965 m_One()))) {
12966 // Decide whether signed or unsigned.
12967 unsigned FloorISD = IsUnsigned ? ISD::AVGFLOORU : ISD::AVGFLOORS;
12968 if (hasOperation(FloorISD, VT))
12969 return DAG.getNode(FloorISD, DL, VT, {A, B});
12970 }
12971
12972 // Match the subtract form, ensuring the sub doesn't wrap.
12973 if (sd_match(N,
12974 m_BinOp(Opcode, m_BinOp(ISD::SUB, m_Value(A), m_Value(B), Flags),
12975 m_One()))) {
12976 unsigned SubISD = IsUnsigned ? ISD::AVGSUBFLOORU : ISD::AVGSUBFLOORS;
12977 if (hasOperation(SubISD, VT))
12978 return DAG.getNode(SubISD, DL, VT, {A, B});
12979 }
12980
12981 return SDValue();
12982}
12983
12984SDValue DAGCombiner::foldBitwiseOpWithNeg(SDNode *N, const SDLoc &DL, EVT VT) {
12985 unsigned Opc = N->getOpcode();
12986 SDValue X, Y, Z;
12987 if (sd_match(
12989 return DAG.getNode(Opc, DL, VT, X,
12990 DAG.getNOT(DL, DAG.getNode(ISD::SUB, DL, VT, Y, Z), VT));
12991
12993 m_Value(Z)))))
12994 return DAG.getNode(Opc, DL, VT, X,
12995 DAG.getNOT(DL, DAG.getNode(ISD::ADD, DL, VT, Y, Z), VT));
12996
12997 return SDValue();
12998}
12999
13000/// Generate Min/Max node
13001SDValue DAGCombiner::combineMinNumMaxNum(const SDLoc &DL, EVT VT, SDValue LHS,
13004 if ((LHS == True && RHS == False) || (LHS == False && RHS == True))
13005 return combineMinNumMaxNumImpl(DL, VT, LHS, RHS, True, False, CC, TLI, DAG);
13006
13007 // If we can't directly match this, try to see if we can pull an fneg out of
13008 // the select.
13010 True, DAG, LegalOperations, ForCodeSize);
13011 if (!NegTrue)
13012 return SDValue();
13013
13014 HandleSDNode NegTrueHandle(NegTrue);
13015
13016 // Try to unfold an fneg from the select if we are comparing the negated
13017 // constant.
13018 //
13019 // select (setcc x, K) (fneg x), -K -> fneg(minnum(x, K))
13020 //
13021 // TODO: Handle fabs
13022 if (LHS == NegTrue) {
13023 // If we can't directly match this, try to see if we can pull an fneg out of
13024 // the select.
13026 RHS, DAG, LegalOperations, ForCodeSize);
13027 if (NegRHS) {
13028 HandleSDNode NegRHSHandle(NegRHS);
13029 if (NegRHS == False) {
13030 SDValue Combined = combineMinNumMaxNumImpl(DL, VT, LHS, RHS, NegTrue,
13031 False, CC, TLI, DAG);
13032 if (Combined)
13033 return DAG.getNode(ISD::FNEG, DL, VT, Combined);
13034 }
13035 }
13036 }
13037
13038 return SDValue();
13039}
13040
13041/// If a (v)select has a condition value that is a sign-bit test, try to smear
13042/// the condition operand sign-bit across the value width and use it as a mask.
13044 SelectionDAG &DAG) {
13045 SDValue Cond = N->getOperand(0);
13046 SDValue C1 = N->getOperand(1);
13047 SDValue C2 = N->getOperand(2);
13049 return SDValue();
13050
13051 EVT VT = N->getValueType(0);
13052 if (Cond.getOpcode() != ISD::SETCC || !Cond.hasOneUse() ||
13053 VT != Cond.getOperand(0).getValueType())
13054 return SDValue();
13055
13056 // The inverted-condition + commuted-select variants of these patterns are
13057 // canonicalized to these forms in IR.
13058 SDValue X = Cond.getOperand(0);
13059 SDValue CondC = Cond.getOperand(1);
13060 ISD::CondCode CC = Cond.getOperandCondCode(2);
13061 if (CC == ISD::SETGT && isAllOnesOrAllOnesSplat(CondC) &&
13063 // i32 X > -1 ? C1 : -1 --> (X >>s 31) | C1
13064 SDValue ShAmtC = DAG.getConstant(X.getScalarValueSizeInBits() - 1, DL, VT);
13065 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, X, ShAmtC);
13066 return DAG.getNode(ISD::OR, DL, VT, Sra, C1);
13067 }
13068 if (CC == ISD::SETLT && isNullOrNullSplat(CondC) && isNullOrNullSplat(C2)) {
13069 // i8 X < 0 ? C1 : 0 --> (X >>s 7) & C1
13070 SDValue ShAmtC = DAG.getConstant(X.getScalarValueSizeInBits() - 1, DL, VT);
13071 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, X, ShAmtC);
13072 return DAG.getNode(ISD::AND, DL, VT, Sra, C1);
13073 }
13074 return SDValue();
13075}
13076
13078 const TargetLowering &TLI) {
13079 if (!TLI.convertSelectOfConstantsToMath(VT))
13080 return false;
13081
13082 if (Cond.getOpcode() != ISD::SETCC || !Cond->hasOneUse())
13083 return true;
13084 if (!TLI.isOperationLegalOrCustom(ISD::SELECT_CC, VT))
13085 return true;
13086
13087 ISD::CondCode CC = Cond.getOperandCondCode(2);
13088 if (CC == ISD::SETLT && isNullOrNullSplat(Cond.getOperand(1)))
13089 return true;
13090 if (CC == ISD::SETGT && isAllOnesOrAllOnesSplat(Cond.getOperand(1)))
13091 return true;
13092
13093 return false;
13094}
13095
13096SDValue DAGCombiner::foldSelectOfConstants(SDNode *N) {
13097 SDValue Cond = N->getOperand(0);
13098 SDValue N1 = N->getOperand(1);
13099 SDValue N2 = N->getOperand(2);
13100 EVT VT = N->getValueType(0);
13101 EVT CondVT = Cond.getValueType();
13102 SDLoc DL(N);
13103
13104 if (!VT.isInteger())
13105 return SDValue();
13106
13107 auto *C1 = dyn_cast<ConstantSDNode>(N1);
13108 auto *C2 = dyn_cast<ConstantSDNode>(N2);
13109 if (!C1 || !C2)
13110 return SDValue();
13111
13112 if (CondVT != MVT::i1 || LegalOperations) {
13113 // We can't do this reliably if integer based booleans have different contents
13114 // to floating point based booleans. This is because we can't tell whether we
13115 // have an integer-based boolean or a floating-point-based boolean unless we
13116 // can find the SETCC that produced it and inspect its operands. This is
13117 // fairly easy if C is the SETCC node, but it can potentially be
13118 // undiscoverable (or not reasonably discoverable). For example, it could be
13119 // in another basic block or it could require searching a complicated
13120 // expression.
13121 if (CondVT.isInteger() &&
13122 TLI.getBooleanContents(/*isVec*/false, /*isFloat*/true) ==
13124 TLI.getBooleanContents(/*isVec*/false, /*isFloat*/false) ==
13126 // fold (select Cond, 0, 1) -> (xor Cond, 1)
13127 if (C1->isZero() && C2->isOne()) {
13128 SDValue NotCond = DAG.getNode(ISD::XOR, DL, CondVT, Cond,
13129 DAG.getConstant(1, DL, CondVT));
13130 if (VT.bitsEq(CondVT))
13131 return NotCond;
13132 return DAG.getZExtOrTrunc(NotCond, DL, VT);
13133 }
13134
13135 // fold (select Cond, 1, 0) -> Cond
13136 if (C1->isOne() && C2->isZero() && CondVT == VT)
13137 return Cond;
13138 }
13139
13140 return SDValue();
13141 }
13142
13143 // Only do this before legalization to avoid conflicting with target-specific
13144 // transforms in the other direction (create a select from a zext/sext). There
13145 // is also a target-independent combine here in DAGCombiner in the other
13146 // direction for (select Cond, -1, 0) when the condition is not i1.
13147 assert(CondVT == MVT::i1 && !LegalOperations);
13148
13149 // select Cond, 1, 0 --> zext (Cond)
13150 if (C1->isOne() && C2->isZero())
13151 return DAG.getZExtOrTrunc(Cond, DL, VT);
13152
13153 // select Cond, -1, 0 --> sext (Cond)
13154 if (C1->isAllOnes() && C2->isZero())
13155 return DAG.getSExtOrTrunc(Cond, DL, VT);
13156
13157 // select Cond, 0, 1 --> zext (!Cond)
13158 if (C1->isZero() && C2->isOne()) {
13159 SDValue NotCond = DAG.getNOT(DL, Cond, MVT::i1);
13160 NotCond = DAG.getZExtOrTrunc(NotCond, DL, VT);
13161 return NotCond;
13162 }
13163
13164 // select Cond, 0, -1 --> sext (!Cond)
13165 if (C1->isZero() && C2->isAllOnes()) {
13166 SDValue NotCond = DAG.getNOT(DL, Cond, MVT::i1);
13167 NotCond = DAG.getSExtOrTrunc(NotCond, DL, VT);
13168 return NotCond;
13169 }
13170
13171 // Use a target hook because some targets may prefer to transform in the
13172 // other direction.
13174 return SDValue();
13175
13176 // For any constants that differ by 1, we can transform the select into
13177 // an extend and add.
13178 const APInt &C1Val = C1->getAPIntValue();
13179 const APInt &C2Val = C2->getAPIntValue();
13180
13181 // select Cond, C1, C1-1 --> add (zext Cond), C1-1
13182 if (C1Val - 1 == C2Val) {
13183 Cond = DAG.getZExtOrTrunc(Cond, DL, VT);
13184 return DAG.getNode(ISD::ADD, DL, VT, Cond, N2);
13185 }
13186
13187 // select Cond, C1, C1+1 --> add (sext Cond), C1+1
13188 if (C1Val + 1 == C2Val) {
13189 Cond = DAG.getSExtOrTrunc(Cond, DL, VT);
13190 return DAG.getNode(ISD::ADD, DL, VT, Cond, N2);
13191 }
13192
13193 // select Cond, Pow2, 0 --> (zext Cond) << log2(Pow2)
13194 if (C1Val.isPowerOf2() && C2Val.isZero()) {
13195 Cond = DAG.getZExtOrTrunc(Cond, DL, VT);
13196 SDValue ShAmtC =
13197 DAG.getShiftAmountConstant(C1Val.exactLogBase2(), VT, DL);
13198 return DAG.getNode(ISD::SHL, DL, VT, Cond, ShAmtC);
13199 }
13200
13201 // select Cond, -1, C --> or (sext Cond), C
13202 if (C1->isAllOnes()) {
13203 Cond = DAG.getSExtOrTrunc(Cond, DL, VT);
13204 return DAG.getNode(ISD::OR, DL, VT, Cond, N2);
13205 }
13206
13207 // select Cond, C, -1 --> or (sext (not Cond)), C
13208 if (C2->isAllOnes()) {
13209 SDValue NotCond = DAG.getNOT(DL, Cond, MVT::i1);
13210 NotCond = DAG.getSExtOrTrunc(NotCond, DL, VT);
13211 return DAG.getNode(ISD::OR, DL, VT, NotCond, N1);
13212 }
13213
13215 return V;
13216
13217 return SDValue();
13218}
13219
13221 SelectionDAG &DAG) {
13222 assert((N->getOpcode() == ISD::SELECT || N->getOpcode() == ISD::VSELECT) &&
13223 "Expected a (v)select");
13224 SDValue Cond = N->getOperand(0);
13225 SDValue T = N->getOperand(1), F = N->getOperand(2);
13226 EVT VT = N->getValueType(0);
13227
13228 if (VT != Cond.getValueType() || VT.getScalarSizeInBits() != 1)
13229 return SDValue();
13230
13231 // select Cond, Cond, F --> or Cond, freeze(F)
13232 // select Cond, 1, F --> or Cond, freeze(F)
13233 if (Cond == T || isOneOrOneSplat(T, /* AllowUndefs */ true))
13234 return DAG.getNode(ISD::OR, DL, VT, Cond, DAG.getFreeze(F));
13235
13236 // select Cond, T, Cond --> and Cond, freeze(T)
13237 // select Cond, T, 0 --> and Cond, freeze(T)
13238 if (Cond == F || isNullOrNullSplat(F, /* AllowUndefs */ true))
13239 return DAG.getNode(ISD::AND, DL, VT, Cond, DAG.getFreeze(T));
13240
13241 // select Cond, T, 1 --> or (not Cond), freeze(T)
13242 if (isOneOrOneSplat(F, /* AllowUndefs */ true)) {
13243 SDValue NotCond =
13244 DAG.getNode(ISD::XOR, DL, VT, Cond, DAG.getAllOnesConstant(DL, VT));
13245 return DAG.getNode(ISD::OR, DL, VT, NotCond, DAG.getFreeze(T));
13246 }
13247
13248 // select Cond, 0, F --> and (not Cond), freeze(F)
13249 if (isNullOrNullSplat(T, /* AllowUndefs */ true)) {
13250 SDValue NotCond =
13251 DAG.getNode(ISD::XOR, DL, VT, Cond, DAG.getAllOnesConstant(DL, VT));
13252 return DAG.getNode(ISD::AND, DL, VT, NotCond, DAG.getFreeze(F));
13253 }
13254
13255 return SDValue();
13256}
13257
13259 SDValue N0 = N->getOperand(0);
13260 SDValue N1 = N->getOperand(1);
13261 SDValue N2 = N->getOperand(2);
13262 EVT VT = N->getValueType(0);
13263 unsigned EltSizeInBits = VT.getScalarSizeInBits();
13264
13265 SDValue Cond0, Cond1;
13267 if (!sd_match(N0, m_OneUse(m_SetCC(CC, m_Value(Cond0), m_Value(Cond1)))) ||
13268 VT != Cond0.getValueType())
13269 return SDValue();
13270
13271 // Match a signbit check of Cond0 as "Cond0 s<0". Swap select operands if the
13272 // compare is inverted from that pattern ("Cond0 s> -1").
13273 if (CC == ISD::SETLT && isNullOrNullSplat(Cond1))
13274 ; // This is the pattern we are looking for.
13275 else if (CC == ISD::SETGT && isAllOnesOrAllOnesSplat(Cond1))
13276 std::swap(N1, N2);
13277 else
13278 return SDValue();
13279
13280 // (Cond0 s< 0) ? N1 : 0 --> (Cond0 s>> BW-1) & freeze(N1)
13281 if (isNullOrNullSplat(N2)) {
13282 SDLoc DL(N);
13283 SDValue ShiftAmt = DAG.getShiftAmountConstant(EltSizeInBits - 1, VT, DL);
13284 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Cond0, ShiftAmt);
13285 return DAG.getNode(ISD::AND, DL, VT, Sra, DAG.getFreeze(N1));
13286 }
13287
13288 // (Cond0 s< 0) ? -1 : N2 --> (Cond0 s>> BW-1) | freeze(N2)
13289 if (isAllOnesOrAllOnesSplat(N1)) {
13290 SDLoc DL(N);
13291 SDValue ShiftAmt = DAG.getShiftAmountConstant(EltSizeInBits - 1, VT, DL);
13292 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Cond0, ShiftAmt);
13293 return DAG.getNode(ISD::OR, DL, VT, Sra, DAG.getFreeze(N2));
13294 }
13295
13296 // If we have to invert the sign bit mask, only do that transform if the
13297 // target has a bitwise 'and not' instruction (the invert is free).
13298 // (Cond0 s< -0) ? 0 : N2 --> ~(Cond0 s>> BW-1) & freeze(N2)
13300 if (isNullOrNullSplat(N1) && TLI.hasAndNot(N1)) {
13301 SDLoc DL(N);
13302 SDValue ShiftAmt = DAG.getShiftAmountConstant(EltSizeInBits - 1, VT, DL);
13303 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Cond0, ShiftAmt);
13304 SDValue Not = DAG.getNOT(DL, Sra, VT);
13305 return DAG.getNode(ISD::AND, DL, VT, Not, DAG.getFreeze(N2));
13306 }
13307
13308 // TODO: There's another pattern in this family, but it may require
13309 // implementing hasOrNot() to check for profitability:
13310 // (Cond0 s> -1) ? -1 : N2 --> ~(Cond0 s>> BW-1) | freeze(N2)
13311
13312 return SDValue();
13313}
13314
13315// Match SELECTs with absolute difference patterns.
13316// (select (setcc a, b, set?gt), (sub a, b), (sub b, a)) --> (abd? a, b)
13317// (select (setcc a, b, set?ge), (sub a, b), (sub b, a)) --> (abd? a, b)
13318// (select (setcc a, b, set?lt), (sub b, a), (sub a, b)) --> (abd? a, b)
13319// (select (setcc a, b, set?le), (sub b, a), (sub a, b)) --> (abd? a, b)
13320SDValue DAGCombiner::foldSelectToABD(SDValue LHS, SDValue RHS, SDValue True,
13322 const SDLoc &DL) {
13323 bool IsSigned = isSignedIntSetCC(CC);
13324 unsigned ABDOpc = IsSigned ? ISD::ABDS : ISD::ABDU;
13325 EVT VT = LHS.getValueType();
13326
13327 if (LegalOperations && !hasOperation(ABDOpc, VT))
13328 return SDValue();
13329
13330 // (setcc 0, b set???) --> (setcc b, 0, set???)
13331 if (isZeroOrZeroSplat(LHS)) {
13332 std::swap(LHS, RHS);
13334 }
13335
13336 // (setcc (add nsw A, Const), 0, sets??) --> (setcc A, -Const, sets??)
13337 // Ensure -Const doesn't overflow.
13338 SDValue A, B;
13339 if (ISD::isSignedIntSetCC(CC) && LHS->getFlags().hasNoSignedWrap() &&
13341 ISD::matchUnaryPredicate(B, [](ConstantSDNode *C) {
13342 return !C->getAPIntValue().isMinSignedValue();
13343 })) {
13344 RHS = DAG.getNegative(B, LHS, B.getValueType());
13345 LHS = A;
13346 }
13347
13348 bool IsTypeLegalOrPromote =
13349 TLI.isTypeLegal(VT) || TLI.getTypeAction(*DAG.getContext(), VT) ==
13351
13352 switch (CC) {
13353 case ISD::SETGT:
13354 case ISD::SETGE:
13355 case ISD::SETUGT:
13356 case ISD::SETUGE:
13361 return DAG.getNode(ABDOpc, DL, VT, LHS, RHS);
13366 IsTypeLegalOrPromote)
13367 return DAG.getNegative(DAG.getNode(ABDOpc, DL, VT, LHS, RHS), DL, VT);
13368 break;
13369 case ISD::SETLT:
13370 case ISD::SETLE:
13371 case ISD::SETULT:
13372 case ISD::SETULE:
13377 return DAG.getNode(ABDOpc, DL, VT, LHS, RHS);
13382 IsTypeLegalOrPromote)
13383 return DAG.getNegative(DAG.getNode(ABDOpc, DL, VT, LHS, RHS), DL, VT);
13384 break;
13385 default:
13386 break;
13387 }
13388
13389 return SDValue();
13390}
13391
13392// ([v]select (ugt x, C), (add x, ~C), x) -> (umin (add x, ~C), x)
13393// ([v]select (ult x, C), x, (add x, -C)) -> (umin x, (add x, -C))
13394SDValue DAGCombiner::foldSelectToUMin(SDValue LHS, SDValue RHS, SDValue True,
13396 const SDLoc &DL) {
13397 APInt C;
13398 EVT VT = True.getValueType();
13399 if (sd_match(RHS, m_ConstInt(C)) && hasUMin(VT)) {
13400 if (CC == ISD::SETUGT && LHS == False &&
13402 SDValue AddC = DAG.getConstant(~C, DL, VT);
13403 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, False, AddC);
13404 return DAG.getNode(ISD::UMIN, DL, VT, Add, False);
13405 }
13406 if (CC == ISD::SETULT && LHS == True &&
13408 SDValue AddC = DAG.getConstant(-C, DL, VT);
13409 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, True, AddC);
13410 return DAG.getNode(ISD::UMIN, DL, VT, True, Add);
13411 }
13412 }
13413 return SDValue();
13414}
13415
13416// Combine x olt y ? x : y to pseudo_fmin and x ogt y ? x : y to pseudo_fmax.
13417// Op0/Op1 are the setcc operands, LHS/RHS are the select operands, Flags are
13418// from the select.
13419// The return value is the opcode and its operands.
13420static std::tuple<unsigned, SDValue, SDValue> combineSelectCCToPseudoMinMax(
13421 SelectionDAG &DAG, const SDLoc &DL, ISD::CondCode CC, SDValue Op0,
13422 SDValue Op1, SDValue LHS, SDValue RHS, SDNodeFlags Flags, bool IsStrict) {
13423 std::tuple<unsigned, SDValue, SDValue> Invalid(0, {}, {});
13425 EVT VT = LHS.getValueType();
13426 if (!VT.isFloatingPoint())
13427 return Invalid;
13428
13429 // Check for x CC y ? x : y.
13430 if (!DAG.isEqualTo(LHS, Op0) || !DAG.isEqualTo(RHS, Op1)) {
13431 if (!DAG.isEqualTo(LHS, Op1) || !DAG.isEqualTo(RHS, Op0))
13432 return Invalid;
13433
13434 // Convert x CC y ? y : x to x inv(CC) y ? x : y.
13435 CC = ISD::getSetCCInverse(CC, VT);
13436 std::swap(LHS, RHS);
13437 }
13438
13439 // Convert x CC y ? x : y to y swap(inv(CC)) x ? y : x
13440 // to convert an unordered into an ordered comparison.
13441 if (ISD::getUnorderedFlavor(CC) == 1) {
13443 std::swap(LHS, RHS);
13444 }
13445
13446 unsigned Opcode = 0;
13447 switch (CC) {
13448 default:
13449 break;
13450 case ISD::SETOLE:
13451 // Converting this to a min would handle comparisons between positive
13452 // and negative zero incorrectly.
13453 if (!Flags.hasNoSignedZeros() && !DAG.isKnownNeverLogicalZero(LHS) &&
13455 break;
13456 Opcode = ISD::PSEUDO_FMIN;
13457 break;
13458 case ISD::SETLE:
13459 // Convert setle to setlt via inv+swap.
13460 std::swap(LHS, RHS);
13461 [[fallthrough]];
13462 case ISD::SETOLT:
13463 case ISD::SETLT:
13464 Opcode = ISD::PSEUDO_FMIN;
13465 break;
13466
13467 case ISD::SETOGE:
13468 // Converting this to a max would handle comparisons between positive
13469 // and negative zero incorrectly.
13470 if (!Flags.hasNoSignedZeros() && !DAG.isKnownNeverLogicalZero(LHS) &&
13472 break;
13473 Opcode = ISD::PSEUDO_FMAX;
13474 break;
13475 case ISD::SETGE:
13476 // Convert setge to setgt via inv+swap.
13477 std::swap(LHS, RHS);
13478 [[fallthrough]];
13479 case ISD::SETOGT:
13480 case ISD::SETGT:
13481 Opcode = ISD::PSEUDO_FMAX;
13482 break;
13483 }
13484
13485 if (!Opcode)
13486 return Invalid;
13487
13488 if (IsStrict)
13489 Opcode = Opcode == ISD::PSEUDO_FMIN ? ISD::STRICT_PSEUDO_FMIN
13491 if (!TLI.isOperationLegalOrCustom(Opcode, VT))
13492 return Invalid;
13493
13494 return {Opcode, LHS, RHS};
13495}
13496
13498 SDLoc DL(N);
13499 SDValue Cond = N->getOperand(0);
13500 SDValue LHS = N->getOperand(1);
13501 SDValue RHS = N->getOperand(2);
13502 EVT VT = LHS.getValueType();
13503 if ((Cond.getOpcode() != ISD::SETCC &&
13504 Cond.getOpcode() != ISD::STRICT_FSETCCS))
13505 return SDValue();
13506
13507 bool IsStrict = Cond->isStrictFPOpcode();
13508 ISD::CondCode CC = Cond.getOperandCondCode(IsStrict ? 3 : 2);
13509 SDValue Op0 = Cond.getOperand(IsStrict ? 1 : 0);
13510 SDValue Op1 = Cond.getOperand(IsStrict ? 2 : 1);
13511 auto [Opcode, NewLHS, NewRHS] = combineSelectCCToPseudoMinMax(
13512 DAG, DL, CC, Op0, Op1, LHS, RHS, N->getFlags(), IsStrict);
13513 if (!Opcode)
13514 return SDValue();
13515
13516 // Propagate fast-math-flags.
13517 SelectionDAG::FlagInserter FlagsInserter(DAG, N->getFlags());
13518 if (IsStrict) {
13519 SDValue Ret = DAG.getNode(Opcode, DL, {VT, MVT::Other},
13520 {Cond.getOperand(0), NewLHS, NewRHS});
13521 DAG.ReplaceAllUsesOfValueWith(Cond.getValue(1), Ret.getValue(1));
13522 return Ret;
13523 }
13524 return DAG.getNode(Opcode, DL, VT, NewLHS, NewRHS);
13525}
13526
13527/// Fold:
13528/// select_cc (select C, TV, FV), CmpC, TrueV, FalseV, seteq
13529/// -> select C, TrueV, FalseV
13530/// select_cc (select C, TV, FV), CmpC, TrueV, FalseV, setne
13531/// -> select C, FalseV, TrueV
13532/// and the same with CmpC on the LHS of the comparison. TV and FV must be
13533/// distinct integer constants. Also used for select (setcc ...).
13535 SDValue TrueV, SDValue FalseV,
13536 const SDLoc &DL, EVT VT, SelectionDAG &DAG,
13537 SDNodeFlags Flags) {
13538 if (CC != ISD::SETEQ && CC != ISD::SETNE)
13539 return SDValue();
13540
13541 SDValue InnerSel;
13542 SDValue CmpC;
13543 if (LHS.getOpcode() == ISD::SELECT) {
13544 InnerSel = LHS;
13545 CmpC = RHS;
13546 } else if (RHS.getOpcode() == ISD::SELECT) {
13547 InnerSel = RHS;
13548 CmpC = LHS;
13549 } else
13550 return SDValue();
13551
13552 SDValue Cond = InnerSel.getOperand(0);
13553 SDValue InnerTV = InnerSel.getOperand(1);
13554 SDValue InnerFV = InnerSel.getOperand(2);
13555
13556 auto *CTV = dyn_cast<ConstantSDNode>(InnerTV);
13557 auto *CFV = dyn_cast<ConstantSDNode>(InnerFV);
13558 auto *CCnst = dyn_cast<ConstantSDNode>(CmpC);
13559 if (!CTV || !CFV || !CCnst)
13560 return SDValue();
13561
13562 // If one of the constants is opaque, the SDNodes may differ while the values
13563 // are the same. Check APInt to avoid miscompiles.
13564 if (CTV->getAPIntValue() == CFV->getAPIntValue())
13565 return SDValue();
13566
13567 const APInt &CmpVal = CCnst->getAPIntValue();
13568 bool MatchesTV = CmpVal == CTV->getAPIntValue();
13569 bool MatchesFV = CmpVal == CFV->getAPIntValue();
13570 if (!MatchesTV && !MatchesFV)
13571 return SDValue();
13572
13573 SDValue SelTrueV = TrueV;
13574 SDValue SelFalseV = FalseV;
13575 if (CC == ISD::SETEQ) {
13576 if (MatchesFV)
13577 std::swap(SelTrueV, SelFalseV);
13578 } else {
13579 if (MatchesTV)
13580 std::swap(SelTrueV, SelFalseV);
13581 }
13582
13583 return DAG.getSelect(DL, VT, Cond, SelTrueV, SelFalseV, Flags);
13584}
13585
13587 ISD::CondCode CC = N->getOperandCondCode(4);
13588 return foldSelectOfSelectCmp(N->getOperand(0), N->getOperand(1), CC,
13589 N->getOperand(2), N->getOperand(3), SDLoc(N),
13590 N->getValueType(0), DAG, N->getFlags());
13591}
13592
13593SDValue DAGCombiner::visitSELECT(SDNode *N) {
13594 SDValue N0 = N->getOperand(0);
13595 SDValue N1 = N->getOperand(1);
13596 SDValue N2 = N->getOperand(2);
13597 EVT VT = N->getValueType(0);
13598 EVT VT0 = N0.getValueType();
13599 SDLoc DL(N);
13600 SDNodeFlags Flags = N->getFlags();
13601
13602 if (SDValue V = DAG.simplifySelect(N0, N1, N2))
13603 return V;
13604
13605 if (SDValue V = foldBoolSelectToLogic(N, DL, DAG))
13606 return V;
13607
13608 // select (not Cond), N1, N2 -> select Cond, N2, N1
13609 if (SDValue F = extractBooleanFlip(N0, DAG, TLI, false))
13610 return DAG.getSelect(DL, VT, F, N2, N1, Flags);
13611
13612 if (SDValue V = foldSelectOfConstants(N))
13613 return V;
13614
13615 // select (setcc (select C, TV, FV), CmpC, cc), TrueV, FalseV
13616 // -> select C, TrueV, FalseV (or swapped FalseV/TrueV)
13617 if (N0.getOpcode() == ISD::SETCC) {
13620 CC, N1, N2, DL, VT, DAG, Flags))
13621 return R;
13622 }
13623
13624 // If we can fold this based on the true/false value, do so.
13625 if (SimplifySelectOps(N, N1, N2))
13626 return SDValue(N, 0); // Don't revisit N.
13627
13628 if (VT0 == MVT::i1) {
13629 // The code in this block deals with the following 2 equivalences:
13630 // select(C0|C1, x, y) <=> select(C0, x, select(C1, x, y))
13631 // select(C0&C1, x, y) <=> select(C0, select(C1, x, y), y)
13632 // The target can specify its preferred form with the
13633 // shouldNormalizeToSelectSequence() callback. However we always transform
13634 // to the right anyway if we find the inner select exists in the DAG anyway
13635 // and we always transform to the left side if we know that we can further
13636 // optimize the combination of the conditions.
13637 bool normalizeToSequence =
13639 // select (and Cond0, Cond1), X, Y
13640 // -> select Cond0, (select Cond1, X, Y), Y
13641 if (N0->getOpcode() == ISD::AND && N0->hasOneUse()) {
13642 SDValue Cond0 = N0->getOperand(0);
13643 SDValue Cond1 = N0->getOperand(1);
13644 SDValue InnerSelect =
13645 DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond1, N1, N2, Flags);
13646 if (normalizeToSequence || !InnerSelect.use_empty())
13647 return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond0,
13648 InnerSelect, N2, Flags);
13649 // Cleanup on failure.
13650 if (InnerSelect.use_empty())
13651 recursivelyDeleteUnusedNodes(InnerSelect.getNode());
13652 }
13653 // select (or Cond0, Cond1), X, Y -> select Cond0, X, (select Cond1, X, Y)
13654 if (N0->getOpcode() == ISD::OR && N0->hasOneUse()) {
13655 SDValue Cond0 = N0->getOperand(0);
13656 SDValue Cond1 = N0->getOperand(1);
13657 SDValue InnerSelect = DAG.getNode(ISD::SELECT, DL, N1.getValueType(),
13658 Cond1, N1, N2, Flags);
13659 if (normalizeToSequence || !InnerSelect.use_empty())
13660 return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond0, N1,
13661 InnerSelect, Flags);
13662 // Cleanup on failure.
13663 if (InnerSelect.use_empty())
13664 recursivelyDeleteUnusedNodes(InnerSelect.getNode());
13665 }
13666
13667 // select Cond0, (select Cond1, X, Y), Y -> select (and Cond0, Cond1), X, Y
13668 if (N1->getOpcode() == ISD::SELECT && N1->hasOneUse()) {
13669 SDValue N1_0 = N1->getOperand(0);
13670 SDValue N1_1 = N1->getOperand(1);
13671 SDValue N1_2 = N1->getOperand(2);
13672 if (N1_2 == N2 && N0.getValueType() == N1_0.getValueType()) {
13673 // Create the actual and node if we can generate good code for it.
13674 if (!normalizeToSequence) {
13675 SDValue And = DAG.getNode(ISD::AND, DL, N0.getValueType(), N0, N1_0);
13676 return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), And, N1_1,
13677 N2, Flags);
13678 }
13679 // Otherwise see if we can optimize the "and" to a better pattern.
13680 if (SDValue Combined = visitANDLike(N0, N1_0, N)) {
13681 return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Combined, N1_1,
13682 N2, Flags);
13683 }
13684 }
13685 }
13686 // select Cond0, X, (select Cond1, X, Y) -> select (or Cond0, Cond1), X, Y
13687 if (N2->getOpcode() == ISD::SELECT && N2->hasOneUse()) {
13688 SDValue N2_0 = N2->getOperand(0);
13689 SDValue N2_1 = N2->getOperand(1);
13690 SDValue N2_2 = N2->getOperand(2);
13691 if (N2_1 == N1 && N0.getValueType() == N2_0.getValueType()) {
13692 // Create the actual or node if we can generate good code for it.
13693 if (!normalizeToSequence) {
13694 SDValue Or = DAG.getNode(ISD::OR, DL, N0.getValueType(), N0, N2_0);
13695 return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Or, N1,
13696 N2_2, Flags);
13697 }
13698 // Otherwise see if we can optimize to a better pattern.
13699 if (SDValue Combined = visitORLike(N0, N2_0, DL))
13700 return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Combined, N1,
13701 N2_2, Flags);
13702 }
13703 }
13704
13705 // select usubo(x, y).overflow, (sub y, x), (usubo x, y) -> abdu(x, y)
13706 if (N0.getOpcode() == ISD::USUBO && N0.getResNo() == 1 &&
13707 N2.getNode() == N0.getNode() && N2.getResNo() == 0 &&
13708 N1.getOpcode() == ISD::SUB && N2.getOperand(0) == N1.getOperand(1) &&
13709 N2.getOperand(1) == N1.getOperand(0) &&
13710 (!LegalOperations || TLI.isOperationLegal(ISD::ABDU, VT)))
13711 return DAG.getNode(ISD::ABDU, DL, VT, N0.getOperand(0), N0.getOperand(1));
13712
13713 // select usubo(x, y).overflow, (usubo x, y), (sub y, x) -> neg (abdu x, y)
13714 if (N0.getOpcode() == ISD::USUBO && N0.getResNo() == 1 &&
13715 N1.getNode() == N0.getNode() && N1.getResNo() == 0 &&
13716 N2.getOpcode() == ISD::SUB && N2.getOperand(0) == N1.getOperand(1) &&
13717 N2.getOperand(1) == N1.getOperand(0) &&
13718 (!LegalOperations || TLI.isOperationLegal(ISD::ABDU, VT)))
13719 return DAG.getNegative(
13720 DAG.getNode(ISD::ABDU, DL, VT, N0.getOperand(0), N0.getOperand(1)),
13721 DL, VT);
13722 }
13723
13724 // Fold selects based on a setcc into other things, such as min/max/abs.
13725 if (N0.getOpcode() == ISD::SETCC) {
13726 SDValue Cond0 = N0.getOperand(0), Cond1 = N0.getOperand(1);
13728
13729 // select (fcmp lt x, y), x, y -> fminnum x, y
13730 // select (fcmp gt x, y), x, y -> fmaxnum x, y
13731 //
13732 // This is OK if we don't care what happens if either operand is a NaN.
13733 if (N0.hasOneUse() &&
13734 isLegalToCombineMinNumMaxNum(DAG, N1, N2, Flags, N0->getFlags(), TLI))
13735 if (SDValue FMinMax =
13736 combineMinNumMaxNum(DL, VT, Cond0, Cond1, N1, N2, CC))
13737 return FMinMax;
13738
13739 // Use 'unsigned add with overflow' to optimize an unsigned saturating add.
13740 // This is conservatively limited to pre-legal-operations to give targets
13741 // a chance to reverse the transform if they want to do that. Also, it is
13742 // unlikely that the pattern would be formed late, so it's probably not
13743 // worth going through the other checks.
13744 if (!LegalOperations && TLI.isOperationLegalOrCustom(ISD::UADDO, VT) &&
13745 CC == ISD::SETUGT && N0.hasOneUse() && isAllOnesConstant(N1) &&
13746 N2.getOpcode() == ISD::ADD && Cond0 == N2.getOperand(0)) {
13747 auto *C = dyn_cast<ConstantSDNode>(N2.getOperand(1));
13748 auto *NotC = dyn_cast<ConstantSDNode>(Cond1);
13749 if (C && NotC && C->getAPIntValue() == ~NotC->getAPIntValue()) {
13750 // select (setcc Cond0, ~C, ugt), -1, (add Cond0, C) -->
13751 // uaddo Cond0, C; select uaddo.1, -1, uaddo.0
13752 //
13753 // The IR equivalent of this transform would have this form:
13754 // %a = add %x, C
13755 // %c = icmp ugt %x, ~C
13756 // %r = select %c, -1, %a
13757 // =>
13758 // %u = call {iN,i1} llvm.uadd.with.overflow(%x, C)
13759 // %u0 = extractvalue %u, 0
13760 // %u1 = extractvalue %u, 1
13761 // %r = select %u1, -1, %u0
13762 SDVTList VTs = DAG.getVTList(VT, VT0);
13763 SDValue UAO = DAG.getNode(ISD::UADDO, DL, VTs, Cond0, N2.getOperand(1));
13764 return DAG.getSelect(DL, VT, UAO.getValue(1), N1, UAO.getValue(0));
13765 }
13766 }
13767
13769 return S;
13770
13772 (!LegalOperations &&
13774 // Any flags available in a select/setcc fold will be on the setcc as they
13775 // migrated from fcmp
13776 return DAG.getNode(ISD::SELECT_CC, DL, VT, Cond0, Cond1, N1, N2,
13777 N0.getOperand(2), N0->getFlags());
13778 }
13779
13780 if (SDValue ABD = foldSelectToABD(Cond0, Cond1, N1, N2, CC, DL))
13781 return ABD;
13782
13783 if (SDValue NewSel = SimplifySelect(DL, N0, N1, N2))
13784 return NewSel;
13785
13786 // (select (ugt x, C), (add x, ~C), x) -> (umin (add x, ~C), x)
13787 // (select (ult x, C), x, (add x, -C)) -> (umin x, (add x, -C))
13788 if (SDValue UMin = foldSelectToUMin(Cond0, Cond1, N1, N2, CC, DL))
13789 return UMin;
13790 }
13791
13792 if (!VT.isVector())
13793 if (SDValue BinOp = foldSelectOfBinops(N))
13794 return BinOp;
13795
13796 if (SDValue R = combineSelectAsExtAnd(N0, N1, N2, DL, DAG))
13797 return R;
13798
13800 return R;
13801
13802 return SDValue();
13803}
13804
13805// This function assumes all the vselect's arguments are CONCAT_VECTOR
13806// nodes and that the condition is a BV of ConstantSDNodes (or undefs).
13808 SDLoc DL(N);
13809 SDValue Cond = N->getOperand(0);
13810 SDValue LHS = N->getOperand(1);
13811 SDValue RHS = N->getOperand(2);
13812 EVT VT = N->getValueType(0);
13813 int NumElems = VT.getVectorNumElements();
13814 assert(LHS.getOpcode() == ISD::CONCAT_VECTORS &&
13815 RHS.getOpcode() == ISD::CONCAT_VECTORS &&
13816 Cond.getOpcode() == ISD::BUILD_VECTOR);
13817
13818 // CONCAT_VECTOR can take an arbitrary number of arguments. We only care about
13819 // binary ones here.
13820 if (LHS->getNumOperands() != 2 || RHS->getNumOperands() != 2)
13821 return SDValue();
13822
13823 // We're sure we have an even number of elements due to the
13824 // concat_vectors we have as arguments to vselect.
13825 // Skip BV elements until we find one that's not an UNDEF
13826 // After we find an UNDEF element, keep looping until we get to half the
13827 // length of the BV and see if all the non-undef nodes are the same.
13828 ConstantSDNode *BottomHalf = nullptr;
13829 for (int i = 0; i < NumElems / 2; ++i) {
13830 if (Cond->getOperand(i)->isUndef())
13831 continue;
13832
13833 if (BottomHalf == nullptr)
13834 BottomHalf = cast<ConstantSDNode>(Cond.getOperand(i));
13835 else if (Cond->getOperand(i).getNode() != BottomHalf)
13836 return SDValue();
13837 }
13838
13839 // Do the same for the second half of the BuildVector
13840 ConstantSDNode *TopHalf = nullptr;
13841 for (int i = NumElems / 2; i < NumElems; ++i) {
13842 if (Cond->getOperand(i)->isUndef())
13843 continue;
13844
13845 if (TopHalf == nullptr)
13846 TopHalf = cast<ConstantSDNode>(Cond.getOperand(i));
13847 else if (Cond->getOperand(i).getNode() != TopHalf)
13848 return SDValue();
13849 }
13850
13851 assert(TopHalf && BottomHalf &&
13852 "One half of the selector was all UNDEFs and the other was all the "
13853 "same value. This should have been addressed before this function.");
13854 return DAG.getNode(
13856 BottomHalf->isZero() ? RHS->getOperand(0) : LHS->getOperand(0),
13857 TopHalf->isZero() ? RHS->getOperand(1) : LHS->getOperand(1));
13858}
13859
13860bool refineUniformBase(SDValue &BasePtr, SDValue &Index, bool IndexIsScaled,
13861 SelectionDAG &DAG, const SDLoc &DL) {
13862
13863 // Only perform the transformation when existing operands can be reused.
13864 if (IndexIsScaled)
13865 return false;
13866
13867 if (!isNullConstant(BasePtr) && !Index.hasOneUse())
13868 return false;
13869
13870 EVT VT = BasePtr.getValueType();
13871
13872 if (SDValue SplatVal = DAG.getSplatValue(Index);
13873 SplatVal && !isNullConstant(SplatVal) &&
13874 SplatVal.getValueType() == VT) {
13875 BasePtr = DAG.getNode(ISD::ADD, DL, VT, BasePtr, SplatVal);
13876 Index = DAG.getSplat(Index.getValueType(), DL, DAG.getConstant(0, DL, VT));
13877 return true;
13878 }
13879
13880 if (Index.getOpcode() != ISD::ADD)
13881 return false;
13882
13883 if (SDValue SplatVal = DAG.getSplatValue(Index.getOperand(0));
13884 SplatVal && SplatVal.getValueType() == VT) {
13885 BasePtr = DAG.getNode(ISD::ADD, DL, VT, BasePtr, SplatVal);
13886 Index = Index.getOperand(1);
13887 return true;
13888 }
13889 if (SDValue SplatVal = DAG.getSplatValue(Index.getOperand(1));
13890 SplatVal && SplatVal.getValueType() == VT) {
13891 BasePtr = DAG.getNode(ISD::ADD, DL, VT, BasePtr, SplatVal);
13892 Index = Index.getOperand(0);
13893 return true;
13894 }
13895 return false;
13896}
13897
13898// Fold sext/zext of index into index type.
13899bool refineIndexType(SDValue &Index, ISD::MemIndexType &IndexType, EVT DataVT,
13900 SelectionDAG &DAG) {
13902
13903 // It's always safe to look through zero extends.
13904 if (Index.getOpcode() == ISD::ZERO_EXTEND) {
13905 if (TLI.shouldRemoveExtendFromGSIndex(Index, DataVT)) {
13906 IndexType = ISD::UNSIGNED_SCALED;
13907 Index = Index.getOperand(0);
13908 return true;
13909 }
13910 if (ISD::isIndexTypeSigned(IndexType)) {
13911 IndexType = ISD::UNSIGNED_SCALED;
13912 return true;
13913 }
13914 }
13915
13916 // It's only safe to look through sign extends when Index is signed.
13917 if (Index.getOpcode() == ISD::SIGN_EXTEND &&
13918 ISD::isIndexTypeSigned(IndexType) &&
13919 TLI.shouldRemoveExtendFromGSIndex(Index, DataVT)) {
13920 Index = Index.getOperand(0);
13921 return true;
13922 }
13923
13924 return false;
13925}
13926
13927SDValue DAGCombiner::visitVPSCATTER(SDNode *N) {
13928 VPScatterSDNode *MSC = cast<VPScatterSDNode>(N);
13929 SDValue Mask = MSC->getMask();
13930 SDValue Chain = MSC->getChain();
13931 SDValue Index = MSC->getIndex();
13932 SDValue Scale = MSC->getScale();
13933 SDValue StoreVal = MSC->getValue();
13934 SDValue BasePtr = MSC->getBasePtr();
13935 SDValue VL = MSC->getVectorLength();
13936 ISD::MemIndexType IndexType = MSC->getIndexType();
13937 SDLoc DL(N);
13938
13939 // Zap scatters with a zero mask.
13941 return Chain;
13942
13943 if (refineUniformBase(BasePtr, Index, MSC->isIndexScaled(), DAG, DL)) {
13944 SDValue Ops[] = {Chain, StoreVal, BasePtr, Index, Scale, Mask, VL};
13945 return DAG.getScatterVP(DAG.getVTList(MVT::Other), MSC->getMemoryVT(),
13946 DL, Ops, MSC->getMemOperand(), IndexType);
13947 }
13948
13949 if (refineIndexType(Index, IndexType, StoreVal.getValueType(), DAG)) {
13950 SDValue Ops[] = {Chain, StoreVal, BasePtr, Index, Scale, Mask, VL};
13951 return DAG.getScatterVP(DAG.getVTList(MVT::Other), MSC->getMemoryVT(),
13952 DL, Ops, MSC->getMemOperand(), IndexType);
13953 }
13954
13955 return SDValue();
13956}
13957
13958SDValue DAGCombiner::visitMSCATTER(SDNode *N) {
13959 MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(N);
13960 SDValue Mask = MSC->getMask();
13961 SDValue Chain = MSC->getChain();
13962 SDValue Index = MSC->getIndex();
13963 SDValue Scale = MSC->getScale();
13964 SDValue StoreVal = MSC->getValue();
13965 SDValue BasePtr = MSC->getBasePtr();
13966 ISD::MemIndexType IndexType = MSC->getIndexType();
13967 SDLoc DL(N);
13968
13969 // Zap scatters with a zero mask.
13971 return Chain;
13972
13973 if (refineUniformBase(BasePtr, Index, MSC->isIndexScaled(), DAG, DL)) {
13974 SDValue Ops[] = {Chain, StoreVal, Mask, BasePtr, Index, Scale};
13975 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), MSC->getMemoryVT(),
13976 DL, Ops, MSC->getMemOperand(), IndexType,
13977 MSC->isTruncatingStore());
13978 }
13979
13980 if (refineIndexType(Index, IndexType, StoreVal.getValueType(), DAG)) {
13981 SDValue Ops[] = {Chain, StoreVal, Mask, BasePtr, Index, Scale};
13982 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), MSC->getMemoryVT(),
13983 DL, Ops, MSC->getMemOperand(), IndexType,
13984 MSC->isTruncatingStore());
13985 }
13986
13987 return SDValue();
13988}
13989
13990/// Check if Mask defines a known constant set of enabled lanes, where only the
13991/// first N lanes are enabled. N is returned if so.
13993 // We expect masks for masked load/store to be i1 predicates.
13994 if (Mask.getValueType().getScalarSizeInBits() != 1)
13995 return 0;
13996
13997 if (Mask.getOpcode() == ISD::BUILD_VECTOR) {
13998 unsigned NumOnes = 0;
13999 auto Op = Mask->op_begin();
14000 for (; Op != Mask->op_end(); ++Op) {
14001 if (!isOneConstant(*Op))
14002 break;
14003 ++NumOnes;
14004 }
14005
14006 if (NumOnes == 0)
14007 return 0;
14008
14009 for (; Op != Mask->op_end(); ++Op)
14010 if (!isNullConstant(*Op))
14011 return 0;
14012 return NumOnes;
14013 }
14014
14015 if (Mask.getOpcode() == ISD::GET_ACTIVE_LANE_MASK &&
14016 isNullConstant(Mask.getOperand(0)) &&
14017 isa<ConstantSDNode>(Mask.getOperand(1)) &&
14018 Mask.getOperand(1).getValueType().getSizeInBits() <= 64)
14019 return Mask.getConstantOperandVal(1);
14020
14021 return 0;
14022}
14023
14024SDValue DAGCombiner::visitMSTORE(SDNode *N) {
14025 MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
14026 SDValue Mask = MST->getMask();
14027 SDValue Chain = MST->getChain();
14028 SDValue Value = MST->getValue();
14029 SDValue Ptr = MST->getBasePtr();
14030 EVT VT = Value.getValueType();
14031
14032 // Zap masked stores with a zero mask.
14034 return Chain;
14035
14036 // Remove a masked store if base pointers and masks are equal.
14037 if (MaskedStoreSDNode *MST1 = dyn_cast<MaskedStoreSDNode>(Chain)) {
14038 if (MST->isUnindexed() && MST->isSimple() && MST1->isUnindexed() &&
14039 MST1->isSimple() && MST1->getBasePtr() == Ptr &&
14040 !MST->getBasePtr().isUndef() &&
14041 ((Mask == MST1->getMask() && MST->getMemoryVT().getStoreSize() ==
14042 MST1->getMemoryVT().getStoreSize()) ||
14044 TypeSize::isKnownLE(MST1->getMemoryVT().getStoreSize(),
14045 MST->getMemoryVT().getStoreSize())) {
14046 CombineTo(MST1, MST1->getChain());
14047 if (N->getOpcode() != ISD::DELETED_NODE)
14048 AddToWorklist(N);
14049 return SDValue(N, 0);
14050 }
14051 }
14052
14053 // If this is a masked store with an all ones mask, we can use a unmasked
14054 // store.
14055 // FIXME: Can we do this for indexed, compressing, or truncating stores?
14056 if (MST->isUnindexed() && !MST->isCompressingStore() &&
14057 !MST->isTruncatingStore()) {
14059 return DAG.getStore(MST->getChain(), SDLoc(N), MST->getValue(),
14060 MST->getBasePtr(), MST->getPointerInfo(),
14061 MST->getBaseAlign(), MST->getMemOperand()->getFlags(),
14063
14064 // Convert a masked_store with constant getactivelanemask input mask to a
14065 // standard store.
14066 if (uint64_t Lanes = calculateConstantLowMaskLanes(Mask)) {
14067 if (Lanes < VT.getVectorMinNumElements() && isPowerOf2_32(Lanes)) {
14068 EVT SubVT = VT.changeElementCount(*DAG.getContext(),
14069 ElementCount::getFixed(Lanes));
14070 unsigned IsFast = 0;
14071 if ((!LegalTypes || isTypeLegal(SubVT)) &&
14073 SubVT, MST->getAddressSpace(),
14074 MST->getBaseAlign(),
14075 MST->getMemOperand()->getFlags(), &IsFast) &&
14076 IsFast) {
14077 SDLoc DL(N);
14078 SDValue Ext = DAG.getExtractSubvector(DL, SubVT, MST->getValue(), 0);
14079 return DAG.getStore(MST->getChain(), DL, Ext, MST->getBasePtr(),
14080 MST->getPointerInfo(), MST->getBaseAlign(),
14081 MST->getMemOperand()->getFlags(),
14083 }
14084 }
14085 }
14086 }
14087
14088 // Try transforming N to an indexed store.
14089 if (CombineToPreIndexedLoadStore(N) || CombineToPostIndexedLoadStore(N))
14090 return SDValue(N, 0);
14091
14092 if (MST->isTruncatingStore() && MST->isUnindexed() && VT.isInteger() &&
14094 !cast<ConstantSDNode>(Value)->isOpaque())) {
14095 APInt TruncDemandedBits =
14096 APInt::getLowBitsSet(Value.getScalarValueSizeInBits(),
14098
14099 // See if we can simplify the operation with
14100 // SimplifyDemandedBits, which only works if the value has a single use.
14101 if (SimplifyDemandedBits(Value, TruncDemandedBits)) {
14102 // Re-visit the store if anything changed and the store hasn't been merged
14103 // with another node (N is deleted) SimplifyDemandedBits will add Value's
14104 // node back to the worklist if necessary, but we also need to re-visit
14105 // the Store node itself.
14106 if (N->getOpcode() != ISD::DELETED_NODE)
14107 AddToWorklist(N);
14108 return SDValue(N, 0);
14109 }
14110 }
14111
14112 // If this is a TRUNC followed by a masked store, fold this into a masked
14113 // truncating store. We can do this even if this is already a masked
14114 // truncstore.
14115 // TODO: Try combine to masked compress store if possiable.
14116 if ((Value.getOpcode() == ISD::TRUNCATE) && Value->hasOneUse() &&
14117 MST->isUnindexed() && !MST->isCompressingStore() &&
14118 TLI.canCombineTruncStore(Value.getOperand(0).getValueType(),
14119 MST->getMemoryVT(), MST->getAlign(),
14120 MST->getAddressSpace(), LegalOperations)) {
14121 auto Mask = TLI.promoteTargetBoolean(DAG, MST->getMask(),
14122 Value.getOperand(0).getValueType());
14123 return DAG.getMaskedStore(Chain, SDLoc(N), Value.getOperand(0), Ptr,
14124 MST->getOffset(), Mask, MST->getMemoryVT(),
14125 MST->getMemOperand(), MST->getAddressingMode(),
14126 /*IsTruncating=*/true);
14127 }
14128
14129 return SDValue();
14130}
14131
14132SDValue DAGCombiner::visitVP_STRIDED_STORE(SDNode *N) {
14133 auto *SST = cast<VPStridedStoreSDNode>(N);
14134 EVT EltVT = SST->getValue().getValueType().getVectorElementType();
14135 // Combine strided stores with unit-stride to a regular VP store.
14136 if (auto *CStride = dyn_cast<ConstantSDNode>(SST->getStride());
14137 CStride && CStride->getZExtValue() == EltVT.getStoreSize()) {
14138 return DAG.getStoreVP(SST->getChain(), SDLoc(N), SST->getValue(),
14139 SST->getBasePtr(), SST->getOffset(), SST->getMask(),
14140 SST->getVectorLength(), SST->getMemoryVT(),
14141 SST->getMemOperand(), SST->getAddressingMode(),
14142 SST->isTruncatingStore(), SST->isCompressingStore());
14143 }
14144 return SDValue();
14145}
14146
14147SDValue DAGCombiner::visitVECTOR_COMPRESS(SDNode *N) {
14148 SDLoc DL(N);
14149 SDValue Vec = N->getOperand(0);
14150 SDValue Mask = N->getOperand(1);
14151 SDValue Passthru = N->getOperand(2);
14152 EVT VecVT = Vec.getValueType();
14153
14154 bool HasPassthru = !Passthru.isUndef();
14155
14156 APInt SplatVal;
14157 if (ISD::isConstantSplatVector(Mask.getNode(), SplatVal))
14158 return TLI.isConstTrueVal(Mask) ? Vec : Passthru;
14159
14160 if (Vec.isUndef() || Mask.isUndef())
14161 return Passthru;
14162
14163 // No need for potentially expensive compress if the mask is constant.
14166 EVT ScalarVT = VecVT.getVectorElementType();
14167 unsigned NumSelected = 0;
14168 unsigned NumElmts = VecVT.getVectorNumElements();
14169 for (unsigned I = 0; I < NumElmts; ++I) {
14170 SDValue MaskI = Mask.getOperand(I);
14171 // We treat undef mask entries as "false".
14172 if (MaskI.isUndef())
14173 continue;
14174
14175 if (TLI.isConstTrueVal(MaskI)) {
14176 SDValue VecI = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ScalarVT, Vec,
14177 DAG.getVectorIdxConstant(I, DL));
14178 Ops.push_back(VecI);
14179 NumSelected++;
14180 }
14181 }
14182 for (unsigned Rest = NumSelected; Rest < NumElmts; ++Rest) {
14183 SDValue Val =
14184 HasPassthru
14185 ? DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ScalarVT, Passthru,
14186 DAG.getVectorIdxConstant(Rest, DL))
14187 : DAG.getUNDEF(ScalarVT);
14188 Ops.push_back(Val);
14189 }
14190 return DAG.getBuildVector(VecVT, DL, Ops);
14191 }
14192
14193 return SDValue();
14194}
14195
14196SDValue DAGCombiner::visitVPGATHER(SDNode *N) {
14197 VPGatherSDNode *MGT = cast<VPGatherSDNode>(N);
14198 SDValue Mask = MGT->getMask();
14199 SDValue Chain = MGT->getChain();
14200 SDValue Index = MGT->getIndex();
14201 SDValue Scale = MGT->getScale();
14202 SDValue BasePtr = MGT->getBasePtr();
14203 SDValue VL = MGT->getVectorLength();
14204 ISD::MemIndexType IndexType = MGT->getIndexType();
14205 SDLoc DL(N);
14206
14207 if (refineUniformBase(BasePtr, Index, MGT->isIndexScaled(), DAG, DL)) {
14208 SDValue Ops[] = {Chain, BasePtr, Index, Scale, Mask, VL};
14209 return DAG.getGatherVP(
14210 DAG.getVTList(N->getValueType(0), MVT::Other), MGT->getMemoryVT(), DL,
14211 Ops, MGT->getMemOperand(), IndexType);
14212 }
14213
14214 if (refineIndexType(Index, IndexType, N->getValueType(0), DAG)) {
14215 SDValue Ops[] = {Chain, BasePtr, Index, Scale, Mask, VL};
14216 return DAG.getGatherVP(
14217 DAG.getVTList(N->getValueType(0), MVT::Other), MGT->getMemoryVT(), DL,
14218 Ops, MGT->getMemOperand(), IndexType);
14219 }
14220
14221 return SDValue();
14222}
14223
14224SDValue DAGCombiner::visitMGATHER(SDNode *N) {
14225 MaskedGatherSDNode *MGT = cast<MaskedGatherSDNode>(N);
14226 SDValue Mask = MGT->getMask();
14227 SDValue Chain = MGT->getChain();
14228 SDValue Index = MGT->getIndex();
14229 SDValue Scale = MGT->getScale();
14230 SDValue PassThru = MGT->getPassThru();
14231 SDValue BasePtr = MGT->getBasePtr();
14232 ISD::MemIndexType IndexType = MGT->getIndexType();
14233 SDLoc DL(N);
14234
14235 // Zap gathers with a zero mask.
14237 return CombineTo(N, PassThru, MGT->getChain());
14238
14239 if (refineUniformBase(BasePtr, Index, MGT->isIndexScaled(), DAG, DL)) {
14240 SDValue Ops[] = {Chain, PassThru, Mask, BasePtr, Index, Scale};
14241 return DAG.getMaskedGather(
14242 DAG.getVTList(N->getValueType(0), MVT::Other), MGT->getMemoryVT(), DL,
14243 Ops, MGT->getMemOperand(), IndexType, MGT->getExtensionType());
14244 }
14245
14246 if (refineIndexType(Index, IndexType, N->getValueType(0), DAG)) {
14247 SDValue Ops[] = {Chain, PassThru, Mask, BasePtr, Index, Scale};
14248 return DAG.getMaskedGather(
14249 DAG.getVTList(N->getValueType(0), MVT::Other), MGT->getMemoryVT(), DL,
14250 Ops, MGT->getMemOperand(), IndexType, MGT->getExtensionType());
14251 }
14252
14253 return SDValue();
14254}
14255
14256SDValue DAGCombiner::visitMLOAD(SDNode *N) {
14257 MaskedLoadSDNode *MLD = cast<MaskedLoadSDNode>(N);
14258 SDValue Mask = MLD->getMask();
14259
14260 // Zap masked loads with a zero mask.
14262 return CombineTo(N, MLD->getPassThru(), MLD->getChain());
14263
14264 // If this is a masked load with an all ones mask, we can use a unmasked load.
14265 // FIXME: Can we do this for indexed, expanding, or extending loads?
14266 if (ISD::isConstantSplatVectorAllOnes(Mask.getNode()) && MLD->isUnindexed() &&
14267 !MLD->isExpandingLoad() && MLD->getExtensionType() == ISD::NON_EXTLOAD) {
14268 SDValue NewLd =
14269 DAG.getLoad(N->getValueType(0), SDLoc(N), MLD->getChain(),
14270 MLD->getBasePtr(), MLD->getPointerInfo(),
14271 MLD->getBaseAlign(), MLD->getMemOperand()->getFlags(),
14272 MMOMetadata(MLD->getAAInfo(), MLD->getRanges(),
14273 MLD->getMemCacheHint()));
14274 return CombineTo(N, NewLd, NewLd.getValue(1));
14275 }
14276
14277 // Try transforming N to an indexed load.
14278 if (CombineToPreIndexedLoadStore(N) || CombineToPostIndexedLoadStore(N))
14279 return SDValue(N, 0);
14280
14281 return SDValue();
14282}
14283
14284SDValue DAGCombiner::visitMHISTOGRAM(SDNode *N) {
14285 MaskedHistogramSDNode *HG = cast<MaskedHistogramSDNode>(N);
14286 SDValue Chain = HG->getChain();
14287 SDValue Inc = HG->getInc();
14288 SDValue Mask = HG->getMask();
14289 SDValue BasePtr = HG->getBasePtr();
14290 SDValue Index = HG->getIndex();
14291 SDLoc DL(HG);
14292
14293 EVT MemVT = HG->getMemoryVT();
14294 EVT DataVT = Index.getValueType();
14295 MachineMemOperand *MMO = HG->getMemOperand();
14296 ISD::MemIndexType IndexType = HG->getIndexType();
14297
14299 return Chain;
14300
14301 if (refineUniformBase(BasePtr, Index, HG->isIndexScaled(), DAG, DL) ||
14302 refineIndexType(Index, IndexType, DataVT, DAG)) {
14303 SDValue Ops[] = {Chain, Inc, Mask, BasePtr, Index,
14304 HG->getScale(), HG->getIntID()};
14305 return DAG.getMaskedHistogram(DAG.getVTList(MVT::Other), MemVT, DL, Ops,
14306 MMO, IndexType);
14307 }
14308
14309 return SDValue();
14310}
14311
14312SDValue DAGCombiner::visitPARTIAL_REDUCE_MLA(SDNode *N) {
14313 if (SDValue Res = foldPartialReduceMLAMulOp(N))
14314 return Res;
14315 if (SDValue Res = foldPartialReduceAdd(N))
14316 return Res;
14317 return SDValue();
14318}
14319
14320// partial_reduce_*mla(acc, mul(*ext(a), *ext(b)), splat(1))
14321// -> partial_reduce_*mla(acc, a, b)
14322//
14323// partial_reduce_*mla(acc, mul(*ext(x), splat(C)), splat(1))
14324// -> partial_reduce_*mla(acc, x, splat(C))
14325//
14326// partial_reduce_*mla(acc, sel(p, mul(*ext(a), *ext(b)), splat(0)), splat(1))
14327// -> partial_reduce_*mla(acc, sel(p, a, splat(0)), b)
14328//
14329// partial_reduce_*mla(acc, sel(p, mul(*ext(a), splat(C)), splat(0)), splat(1))
14330// -> partial_reduce_*mla(acc, sel(p, a, splat(0)), splat(C))
14331//
14332// `sel` could either be VSELECT or VP_MERGE.
14333SDValue DAGCombiner::foldPartialReduceMLAMulOp(SDNode *N) {
14334 SDLoc DL(N);
14335 auto *Context = DAG.getContext();
14336 SDValue Tmp;
14337 SDValue Acc = N->getOperand(0);
14338 SDValue Op1 = N->getOperand(1);
14339 SDValue OrigOp1 = Op1;
14340 SDValue Op2 = N->getOperand(2);
14341 unsigned Opc = Op1->getOpcode();
14342
14343 // Handle predication by moving the VSELECT / VP_MERGE into the operand of the
14344 // MUL.
14345 SDValue Pred;
14346 if ((Opc == ISD::VSELECT || Opc == ISD::VP_MERGE) &&
14347 (isZeroOrZeroSplat(Op1->getOperand(2)) ||
14348 isZeroOrZeroSplatFP(Op1->getOperand(2)))) {
14349 Pred = Op1->getOperand(0);
14350 Op1 = Op1->getOperand(1);
14351 Opc = Op1->getOpcode();
14352 }
14353
14354 // Handle negation (sub-reduction).
14355 bool IsMLS = false;
14356 if (sd_match(Op1, m_Neg(m_Value(Tmp)))) {
14357 Op1 = Tmp;
14358 Opc = Op1->getOpcode();
14359 IsMLS = true;
14360 }
14361
14362 if (Opc != ISD::MUL && Opc != ISD::FMUL && Opc != ISD::SHL)
14363 return SDValue();
14364
14365 SDValue LHS = Op1->getOperand(0);
14366 SDValue RHS = Op1->getOperand(1);
14367
14368 // After instcombine, negation for FP operations is on the RHS, so implement:
14369 // fmul(fpext(a), fneg(fpext(b)))
14370 //-> fmul(fpext(a), fpext(fneg(b)))
14371 if (sd_match(RHS, m_FNeg(m_Value(Tmp)))) {
14372 RHS = Tmp;
14373 IsMLS = true;
14374 }
14375
14376 // Try to treat (shl %a, %c) as (mul %a, (1 << %c)) for constant %c.
14377 if (Opc == ISD::SHL) {
14378 APInt C;
14379 if (!ISD::isConstantSplatVector(RHS.getNode(), C))
14380 return SDValue();
14381
14382 RHS =
14383 DAG.getSplatVector(RHS.getValueType(), DL,
14384 DAG.getConstant(APInt(C.getBitWidth(), 1).shl(C), DL,
14385 RHS.getValueType().getScalarType()));
14386 Opc = ISD::MUL;
14387 }
14388
14389 if (!(Opc == ISD::MUL && llvm::isOneOrOneSplat(Op2)) &&
14391 return SDValue();
14392
14393 auto IsIntOrFPExtOpcode = [](unsigned int Opcode) {
14394 return (ISD::isExtOpcode(Opcode) || Opcode == ISD::FP_EXTEND);
14395 };
14396
14397 unsigned LHSOpcode = LHS->getOpcode();
14398 if (!IsIntOrFPExtOpcode(LHSOpcode))
14399 return SDValue();
14400
14401 SDValue LHSExtOp = LHS->getOperand(0);
14402 EVT LHSExtOpVT = LHSExtOp.getValueType();
14403
14404 // When Pred is non-zero, set Op = select(Pred, Op, splat(0)) and freeze
14405 // OtherOp to keep the same semantics when moving the selects into the MUL
14406 // operands.
14407 auto ApplyPredicate = [&](SDValue &Op, SDValue &OtherOp) {
14408 if (Pred) {
14409 EVT OpVT = Op.getValueType();
14410 SDValue Zero = OpVT.isFloatingPoint() ? DAG.getConstantFP(0.0, DL, OpVT)
14411 : DAG.getConstant(0, DL, OpVT);
14412 if (OrigOp1.getOpcode() == ISD::VP_MERGE)
14413 Op = DAG.getNode(ISD::VP_MERGE, DL, OpVT, Pred, Op, Zero,
14414 OrigOp1.getOperand(3));
14415 else
14416 Op = DAG.getSelect(DL, OpVT, Pred, Op, Zero);
14417 OtherOp = DAG.getFreeze(OtherOp);
14418 }
14419 };
14420
14421 // Generate an MLA or MLS.
14422 auto GetMLA = [&](unsigned Opc, SDValue Acc, SDValue LHS,
14423 SDValue RHS) -> SDValue {
14424 EVT AccVT = Acc.getValueType();
14425 return IsMLS ? DAG.getPartialReduceMLS(Opc, DL, Acc, LHS, RHS)
14426 : DAG.getNode(Opc, DL, AccVT, Acc, LHS, RHS);
14427 };
14428
14429 // partial_reduce_*mla(acc, mul(ext(x), splat(C)), splat(1))
14430 // -> partial_reduce_*mla(acc, x, C)
14431 APInt C;
14432 if (ISD::isConstantSplatVector(RHS.getNode(), C)) {
14433 // TODO: Make use of partial_reduce_sumla here
14434 APInt CTrunc = C.trunc(LHSExtOpVT.getScalarSizeInBits());
14435 unsigned LHSBits = LHS.getValueType().getScalarSizeInBits();
14436 if ((LHSOpcode != ISD::ZERO_EXTEND || CTrunc.zext(LHSBits) != C) &&
14437 (LHSOpcode != ISD::SIGN_EXTEND || CTrunc.sext(LHSBits) != C))
14438 return SDValue();
14439
14440 unsigned NewOpcode = LHSOpcode == ISD::SIGN_EXTEND
14443
14444 // Only perform these combines if the target supports folding
14445 // the extends into the operation.
14447 NewOpcode, TLI.getTypeToTransformTo(*Context, N->getValueType(0)),
14448 TLI.getTypeToTransformTo(*Context, LHSExtOpVT)))
14449 return SDValue();
14450
14451 SDValue C = DAG.getConstant(CTrunc, DL, LHSExtOpVT);
14452 ApplyPredicate(C, LHSExtOp);
14453 return GetMLA(NewOpcode, Acc, LHSExtOp, C);
14454 }
14455
14456 unsigned RHSOpcode = RHS->getOpcode();
14457 if (!IsIntOrFPExtOpcode(RHSOpcode))
14458 return SDValue();
14459
14460 SDValue RHSExtOp = RHS->getOperand(0);
14461 if (LHSExtOpVT != RHSExtOp.getValueType())
14462 return SDValue();
14463
14464 unsigned NewOpc;
14465 if (LHSOpcode == ISD::SIGN_EXTEND && RHSOpcode == ISD::SIGN_EXTEND)
14466 NewOpc = ISD::PARTIAL_REDUCE_SMLA;
14467 else if (LHSOpcode == ISD::ZERO_EXTEND && RHSOpcode == ISD::ZERO_EXTEND)
14468 NewOpc = ISD::PARTIAL_REDUCE_UMLA;
14469 else if (LHSOpcode == ISD::SIGN_EXTEND && RHSOpcode == ISD::ZERO_EXTEND)
14471 else if (LHSOpcode == ISD::ZERO_EXTEND && RHSOpcode == ISD::SIGN_EXTEND) {
14473 std::swap(LHSExtOp, RHSExtOp);
14474 } else if (LHSOpcode == ISD::FP_EXTEND && RHSOpcode == ISD::FP_EXTEND) {
14475 NewOpc = ISD::PARTIAL_REDUCE_FMLA;
14476 } else
14477 return SDValue();
14478 // For a 2-stage extend the signedness of both of the extends must match
14479 // If the mul has the same type, there is no outer extend, and thus we
14480 // can simply use the inner extends to pick the result node.
14481 // TODO: extend to handle nonneg zext as sext
14482 EVT AccElemVT = Acc.getValueType().getVectorElementType();
14483 if (Op1.getValueType().getVectorElementType() != AccElemVT &&
14484 NewOpc != N->getOpcode())
14485 return SDValue();
14486
14487 // Only perform these combines if the target supports folding
14488 // the extends into the operation.
14490 NewOpc, TLI.getTypeToTransformTo(*Context, N->getValueType(0)),
14491 TLI.getTypeToTransformTo(*Context, LHSExtOpVT)))
14492 return SDValue();
14493
14494 ApplyPredicate(RHSExtOp, LHSExtOp);
14495 return GetMLA(NewOpc, Acc, LHSExtOp, RHSExtOp);
14496}
14497
14498// partial.reduce.*mla(acc, *ext(op), splat(1))
14499// -> partial.reduce.*mla(acc, op, splat(trunc(1)))
14500// partial.reduce.sumla(acc, sext(op), splat(1))
14501// -> partial.reduce.smla(acc, op, splat(trunc(1)))
14502//
14503// partial.reduce.*mla(acc, sel(p, *ext(op), splat(0)), splat(1))
14504// -> partial.reduce.*mla(acc, sel(p, op, splat(0)), splat(trunc(1)))
14505SDValue DAGCombiner::foldPartialReduceAdd(SDNode *N) {
14506 SDLoc DL(N);
14507 SDValue Tmp;
14508 SDValue Acc = N->getOperand(0);
14509 SDValue Op1 = N->getOperand(1);
14510 SDValue Op2 = N->getOperand(2);
14511
14513 return SDValue();
14514
14515 SDValue Pred;
14516 unsigned Op1Opcode = Op1.getOpcode();
14517 if (Op1Opcode == ISD::VSELECT && (isZeroOrZeroSplat(Op1->getOperand(2)) ||
14518 isZeroOrZeroSplatFP(Op1->getOperand(2)))) {
14519 Pred = Op1->getOperand(0);
14520 Op1 = Op1->getOperand(1);
14521 Op1Opcode = Op1->getOpcode();
14522 }
14523
14524 // Handle negation (sub-reduction).
14525 bool IsMLS = false;
14526 if (sd_match(Op1, m_AnyOf(m_Neg(m_Value(Tmp)), m_FNeg(m_Value(Tmp))))) {
14527 Op1 = Tmp;
14528 Op1Opcode = Op1.getOpcode();
14529 IsMLS = true;
14530 }
14531
14532 if (!ISD::isExtOpcode(Op1Opcode) && Op1Opcode != ISD::FP_EXTEND)
14533 return SDValue();
14534
14535 bool Op1IsSigned =
14536 Op1Opcode == ISD::SIGN_EXTEND || Op1Opcode == ISD::FP_EXTEND;
14537 bool NodeIsSigned = N->getOpcode() != ISD::PARTIAL_REDUCE_UMLA;
14538 EVT AccElemVT = Acc.getValueType().getVectorElementType();
14539 if (Op1IsSigned != NodeIsSigned &&
14540 Op1.getValueType().getVectorElementType() != AccElemVT)
14541 return SDValue();
14542
14543 unsigned NewOpcode = N->getOpcode() == ISD::PARTIAL_REDUCE_FMLA
14545 : Op1IsSigned ? ISD::PARTIAL_REDUCE_SMLA
14547
14548 SDValue UnextOp1 = Op1.getOperand(0);
14549 EVT UnextOp1VT = UnextOp1.getValueType();
14550 auto *Context = DAG.getContext();
14551 EVT PromOp1VT = TLI.getTypeToTransformTo(*Context, UnextOp1VT);
14553 NewOpcode, TLI.getTypeToTransformTo(*Context, N->getValueType(0)),
14554 PromOp1VT))
14555 return SDValue();
14556
14557 // The multiplier below is built at the operand type, where a splat of 1 in i1
14558 // sign extends to -1. Extend i1 masks to the promoted type first.
14559 if (Op1IsSigned && UnextOp1VT.getVectorElementType() == MVT::i1) {
14560 if (PromOp1VT == UnextOp1VT)
14561 return SDValue();
14562 UnextOp1VT = PromOp1VT;
14563 UnextOp1 = DAG.getNode(ISD::SIGN_EXTEND, DL, UnextOp1VT, UnextOp1);
14564 }
14565
14566 SDValue Constant = N->getOpcode() == ISD::PARTIAL_REDUCE_FMLA
14567 ? DAG.getConstantFP(1, DL, UnextOp1VT)
14568 : DAG.getConstant(1, DL, UnextOp1VT);
14569
14570 if (Pred) {
14571 SDValue Zero = N->getOpcode() == ISD::PARTIAL_REDUCE_FMLA
14572 ? DAG.getConstantFP(0, DL, UnextOp1VT)
14573 : DAG.getConstant(0, DL, UnextOp1VT);
14574 Constant = DAG.getSelect(DL, UnextOp1VT, Pred, Constant, Zero);
14575 }
14576 EVT AccVT = Acc.getValueType();
14577 return IsMLS ? DAG.getPartialReduceMLS(NewOpcode, DL, Acc, UnextOp1, Constant)
14578 : DAG.getNode(NewOpcode, DL, AccVT, Acc, UnextOp1, Constant);
14579}
14580
14581SDValue DAGCombiner::visitLOOP_DEPENDENCE_MASK(SDNode *N) {
14582 SDLoc DL(N);
14583 EVT VT = N->getValueType(0);
14584 unsigned LaneOffset = N->getConstantOperandVal(3);
14585
14586 // The first lane is always active, so v1i1 => true.
14587 if (LaneOffset == 0 &&
14589 return DAG.getBoolConstant(true, DL, VT, VT);
14590
14591 return SDValue();
14592}
14593
14594SDValue DAGCombiner::visitVP_STRIDED_LOAD(SDNode *N) {
14595 auto *SLD = cast<VPStridedLoadSDNode>(N);
14596 EVT EltVT = SLD->getValueType(0).getVectorElementType();
14597 // Combine strided loads with unit-stride to a regular VP load.
14598 if (auto *CStride = dyn_cast<ConstantSDNode>(SLD->getStride());
14599 CStride && CStride->getZExtValue() == EltVT.getStoreSize()) {
14600 SDValue NewLd = DAG.getLoadVP(
14601 SLD->getAddressingMode(), SLD->getExtensionType(), SLD->getValueType(0),
14602 SDLoc(N), SLD->getChain(), SLD->getBasePtr(), SLD->getOffset(),
14603 SLD->getMask(), SLD->getVectorLength(), SLD->getMemoryVT(),
14604 SLD->getMemOperand(), SLD->isExpandingLoad());
14605 return CombineTo(N, NewLd, NewLd.getValue(1));
14606 }
14607 return SDValue();
14608}
14609
14610/// A vector select of 2 constant vectors can be simplified to math/logic to
14611/// avoid a variable select instruction and possibly avoid constant loads.
14612SDValue DAGCombiner::foldVSelectOfConstants(SDNode *N) {
14613 SDValue Cond = N->getOperand(0);
14614 SDValue N1 = N->getOperand(1);
14615 SDValue N2 = N->getOperand(2);
14616 EVT VT = N->getValueType(0);
14617 if (!Cond.hasOneUse() || Cond.getScalarValueSizeInBits() != 1 ||
14621 return SDValue();
14622
14623 // Check if we can use the condition value to increment/decrement a single
14624 // constant value. This simplifies a select to an add and removes a constant
14625 // load/materialization from the general case.
14626 bool AllAddOne = true;
14627 bool AllSubOne = true;
14628 unsigned Elts = VT.getVectorNumElements();
14629 for (unsigned i = 0; i != Elts; ++i) {
14630 SDValue N1Elt = N1.getOperand(i);
14631 SDValue N2Elt = N2.getOperand(i);
14632 if (N1Elt.isUndef())
14633 continue;
14634 // N2 should not contain undef values since it will be reused in the fold.
14635 if (N2Elt.isUndef() || N1Elt.getValueType() != N2Elt.getValueType()) {
14636 AllAddOne = false;
14637 AllSubOne = false;
14638 break;
14639 }
14640
14641 const APInt &C1 = N1Elt->getAsAPIntVal();
14642 const APInt &C2 = N2Elt->getAsAPIntVal();
14643 if (C1 != C2 + 1)
14644 AllAddOne = false;
14645 if (C1 != C2 - 1)
14646 AllSubOne = false;
14647 }
14648
14649 // Further simplifications for the extra-special cases where the constants are
14650 // all 0 or all -1 should be implemented as folds of these patterns.
14651 SDLoc DL(N);
14652 if (AllAddOne || AllSubOne) {
14653 // vselect <N x i1> Cond, C+1, C --> add (zext Cond), C
14654 // vselect <N x i1> Cond, C-1, C --> add (sext Cond), C
14655 auto ExtendOpcode = AllAddOne ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND;
14656 SDValue ExtendedCond = DAG.getNode(ExtendOpcode, DL, VT, Cond);
14657 return DAG.getNode(ISD::ADD, DL, VT, ExtendedCond, N2);
14658 }
14659
14660 // select Cond, Pow2C, 0 --> (zext Cond) << log2(Pow2C)
14661 APInt Pow2C;
14662 if (ISD::isConstantSplatVector(N1.getNode(), Pow2C) && Pow2C.isPowerOf2() &&
14663 isNullOrNullSplat(N2)) {
14664 SDValue ZextCond = DAG.getZExtOrTrunc(Cond, DL, VT);
14665 SDValue ShAmtC = DAG.getConstant(Pow2C.exactLogBase2(), DL, VT);
14666 return DAG.getNode(ISD::SHL, DL, VT, ZextCond, ShAmtC);
14667 }
14668
14670 return V;
14671
14672 // The general case for select-of-constants:
14673 // vselect <N x i1> Cond, C1, C2 --> xor (and (sext Cond), (C1^C2)), C2
14674 // ...but that only makes sense if a vselect is slower than 2 logic ops, so
14675 // leave that to a machine-specific pass.
14676 return SDValue();
14677}
14678
14680 SDValue FVal,
14681 const TargetLowering &TLI,
14682 SelectionDAG &DAG,
14683 const SDLoc &DL) {
14684 EVT VT = TVal.getValueType();
14685 if (!TLI.isTypeLegal(VT))
14686 return SDValue();
14687
14688 EVT CondVT = Cond.getValueType();
14689 assert(CondVT.isVector() && "Vector select expects a vector selector!");
14690
14691 bool IsTAllZero = ISD::isConstantSplatVectorAllZeros(TVal.getNode());
14692 bool IsTAllOne = ISD::isConstantSplatVectorAllOnes(TVal.getNode());
14693 bool IsFAllZero = ISD::isConstantSplatVectorAllZeros(FVal.getNode());
14694 bool IsFAllOne = ISD::isConstantSplatVectorAllOnes(FVal.getNode());
14695
14696 // no vselect(cond, 0/-1, X) or vselect(cond, X, 0/-1), return
14697 if (!IsTAllZero && !IsTAllOne && !IsFAllZero && !IsFAllOne)
14698 return SDValue();
14699
14700 // select Cond, 0, 0 → 0
14701 if (IsTAllZero && IsFAllZero) {
14702 return VT.isFloatingPoint() ? DAG.getConstantFP(0.0, DL, VT)
14703 : DAG.getConstant(0, DL, VT);
14704 }
14705
14706 // check select(setgt lhs, -1), 1, -1 --> or (sra lhs, bitwidth - 1), 1
14707 APInt TValAPInt;
14708 if (Cond.getOpcode() == ISD::SETCC &&
14709 Cond.getOperand(2) == DAG.getCondCode(ISD::SETGT) &&
14710 Cond.getOperand(0).getValueType() == VT && VT.isSimple() &&
14711 ISD::isConstantSplatVector(TVal.getNode(), TValAPInt) &&
14712 TValAPInt.isOne() &&
14713 ISD::isConstantSplatVectorAllOnes(Cond.getOperand(1).getNode()) &&
14715 !TLI.shouldAvoidTransformToShift(VT, VT.getScalarSizeInBits() - 1)) {
14716 SDValue LHS = Cond.getOperand(0);
14717 SDValue ShiftC =
14719 SDValue Shift = DAG.getNode(ISD::SRA, DL, VT, LHS, ShiftC);
14720 return DAG.getNode(ISD::OR, DL, VT, Shift, TVal);
14721 }
14722
14723 // To use the condition operand as a bitwise mask, it must have elements that
14724 // are the same size as the select elements. i.e, the condition operand must
14725 // have already been promoted from the IR select condition type <N x i1>.
14726 // Don't check if the types themselves are equal because that excludes
14727 // vector floating-point selects.
14728 if (CondVT.getScalarSizeInBits() != VT.getScalarSizeInBits())
14729 return SDValue();
14730
14731 // Cond value must be 'sign splat' to be converted to a logical op.
14732 if (DAG.ComputeNumSignBits(Cond) != CondVT.getScalarSizeInBits())
14733 return SDValue();
14734
14735 // Try inverting Cond and swapping T/F if it gives all-ones/all-zeros form
14736 if (!IsTAllOne && !IsFAllZero && Cond.hasOneUse() &&
14737 Cond.getOpcode() == ISD::SETCC &&
14738 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT) ==
14739 CondVT) {
14740 if (IsTAllZero || IsFAllOne) {
14741 SDValue CC = Cond.getOperand(2);
14743 cast<CondCodeSDNode>(CC)->get(), Cond.getOperand(0).getValueType());
14744 Cond = DAG.getSetCC(DL, CondVT, Cond.getOperand(0), Cond.getOperand(1),
14745 InverseCC);
14746 std::swap(TVal, FVal);
14747 std::swap(IsTAllOne, IsFAllOne);
14748 std::swap(IsTAllZero, IsFAllZero);
14749 }
14750 }
14751
14753 "Select condition no longer all-sign bits");
14754
14755 // select Cond, -1, 0 → bitcast Cond
14756 if (IsTAllOne && IsFAllZero)
14757 return DAG.getBitcast(VT, Cond);
14758
14759 // select Cond, -1, x → or Cond, x
14760 if (IsTAllOne) {
14761 SDValue X = DAG.getBitcast(CondVT, DAG.getFreeze(FVal));
14762 SDValue Or = DAG.getNode(ISD::OR, DL, CondVT, Cond, X);
14763 return DAG.getBitcast(VT, Or);
14764 }
14765
14766 // select Cond, x, 0 → and Cond, x
14767 if (IsFAllZero) {
14768 SDValue X = DAG.getBitcast(CondVT, DAG.getFreeze(TVal));
14769 SDValue And = DAG.getNode(ISD::AND, DL, CondVT, Cond, X);
14770 return DAG.getBitcast(VT, And);
14771 }
14772
14773 // select Cond, 0, x -> and not(Cond), x
14774 if (IsTAllZero &&
14775 (isBitwiseNot(peekThroughBitcasts(Cond)) || TLI.hasAndNot(Cond))) {
14776 SDValue X = DAG.getBitcast(CondVT, DAG.getFreeze(FVal));
14777 SDValue And =
14778 DAG.getNode(ISD::AND, DL, CondVT, DAG.getNOT(DL, Cond, CondVT), X);
14779 return DAG.getBitcast(VT, And);
14780 }
14781
14782 return SDValue();
14783}
14784
14785SDValue DAGCombiner::visitVSELECT(SDNode *N) {
14786 SDValue N0 = N->getOperand(0);
14787 SDValue N1 = N->getOperand(1);
14788 SDValue N2 = N->getOperand(2);
14789 EVT VT = N->getValueType(0);
14790 SDLoc DL(N);
14791
14792 if (SDValue V = DAG.simplifySelect(N0, N1, N2))
14793 return V;
14794
14795 if (SDValue V = foldBoolSelectToLogic(N, DL, DAG))
14796 return V;
14797
14798 // vselect (not Cond), N1, N2 -> vselect Cond, N2, N1
14800 if (SDValue F = extractBooleanFlip(N0, DAG, TLI, false))
14801 return DAG.getSelect(DL, VT, F, N2, N1, N->getFlags());
14802
14803 // select (sext m), (add X, C), X --> (add X, (and C, (sext m))))
14804 if (N1.getOpcode() == ISD::ADD && N1.getOperand(0) == N2 && N1->hasOneUse() &&
14809 return DAG.getNode(
14810 ISD::ADD, DL, N1.getValueType(), N2,
14811 DAG.getNode(ISD::AND, DL, N0.getValueType(), N1.getOperand(1), N0));
14812 }
14813
14814 // Canonicalize integer abs.
14815 // vselect (setg[te] X, 0), X, -X ->
14816 // vselect (setgt X, -1), X, -X ->
14817 // vselect (setl[te] X, 0), -X, X ->
14818 // Y = sra (X, size(X)-1); xor (add (X, Y), Y)
14819 if (N0.getOpcode() == ISD::SETCC) {
14820 SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1);
14822 bool isAbs = false;
14823 bool RHSIsAllZeros = ISD::isBuildVectorAllZeros(RHS.getNode());
14824
14825 if (((RHSIsAllZeros && (CC == ISD::SETGT || CC == ISD::SETGE)) ||
14826 (ISD::isBuildVectorAllOnes(RHS.getNode()) && CC == ISD::SETGT)) &&
14827 N1 == LHS && N2.getOpcode() == ISD::SUB && N1 == N2.getOperand(1))
14829 else if ((RHSIsAllZeros && (CC == ISD::SETLT || CC == ISD::SETLE)) &&
14830 N2 == LHS && N1.getOpcode() == ISD::SUB && N2 == N1.getOperand(1))
14832
14833 if (isAbs) {
14835 return DAG.getNode(ISD::ABS, DL, VT, LHS);
14836
14837 SDValue Shift = DAG.getNode(
14838 ISD::SRA, DL, VT, LHS,
14839 DAG.getShiftAmountConstant(VT.getScalarSizeInBits() - 1, VT, DL));
14840 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, LHS, Shift);
14841 AddToWorklist(Shift.getNode());
14842 AddToWorklist(Add.getNode());
14843 return DAG.getNode(ISD::XOR, DL, VT, Add, Shift);
14844 }
14845
14846 // vselect x, y (fcmp lt x, y) -> fminnum x, y
14847 // vselect x, y (fcmp gt x, y) -> fmaxnum x, y
14848 //
14849 // This is OK if we don't care about what happens if either operand is a
14850 // NaN.
14851 //
14852 if (N0.hasOneUse() &&
14853 isLegalToCombineMinNumMaxNum(DAG, LHS, RHS, N->getFlags(),
14854 N0->getFlags(), TLI)) {
14855 if (SDValue FMinMax = combineMinNumMaxNum(DL, VT, LHS, RHS, N1, N2, CC))
14856 return FMinMax;
14857 }
14858
14859 if (SDValue S = PerformMinMaxFpToSatCombine(LHS, RHS, N1, N2, CC, DAG))
14860 return S;
14861 if (SDValue S = PerformUMinFpToSatCombine(LHS, RHS, N1, N2, CC, DAG))
14862 return S;
14864 return S;
14865
14866 // If this select has a condition (setcc) with narrower operands than the
14867 // select, try to widen the compare to match the select width.
14868 // TODO: This should be extended to handle any constant.
14869 // TODO: This could be extended to handle non-loading patterns, but that
14870 // requires thorough testing to avoid regressions.
14871 if (isNullOrNullSplat(RHS)) {
14872 EVT NarrowVT = LHS.getValueType();
14874 EVT SetCCVT = getSetCCResultType(LHS.getValueType());
14875 unsigned SetCCWidth = SetCCVT.getScalarSizeInBits();
14876 unsigned WideWidth = WideVT.getScalarSizeInBits();
14877 bool IsSigned = isSignedIntSetCC(CC);
14878 auto LoadExtOpcode = IsSigned ? ISD::SEXTLOAD : ISD::ZEXTLOAD;
14879 if (LHS.getOpcode() == ISD::LOAD && LHS.hasOneUse() && SetCCWidth != 1 &&
14880 SetCCWidth < WideWidth &&
14882 LoadSDNode *Ld = cast<LoadSDNode>(LHS);
14883
14884 if (TLI.isLoadLegalOrCustom(WideVT, NarrowVT, Ld->getAlign(),
14885 Ld->getAddressSpace(), LoadExtOpcode,
14886 false)) {
14887 // Both compare operands can be widened for free. The LHS can use an
14888 // extended load, and the RHS is a constant:
14889 // vselect (ext (setcc load(X), C)), N1, N2 -->
14890 // vselect (setcc extload(X), C'), N1, N2
14891 auto ExtOpcode = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
14892 SDValue WideLHS = DAG.getNode(ExtOpcode, DL, WideVT, LHS);
14893 SDValue WideRHS = DAG.getNode(ExtOpcode, DL, WideVT, RHS);
14894 EVT WideSetCCVT = getSetCCResultType(WideVT);
14895 SDValue WideSetCC =
14896 DAG.getSetCC(DL, WideSetCCVT, WideLHS, WideRHS, CC);
14897 return DAG.getSelect(DL, N1.getValueType(), WideSetCC, N1, N2);
14898 }
14899 }
14900 }
14901
14902 if (SDValue ABD = foldSelectToABD(LHS, RHS, N1, N2, CC, DL))
14903 return ABD;
14904
14905 // Match VSELECTs into add with unsigned saturation.
14906 if (hasOperation(ISD::UADDSAT, VT)) {
14907 // Check if one of the arms of the VSELECT is vector with all bits set.
14908 // If it's on the left side invert the predicate to simplify logic below.
14909 SDValue Other;
14910 ISD::CondCode SatCC = CC;
14912 Other = N2;
14913 SatCC = ISD::getSetCCInverse(SatCC, VT.getScalarType());
14914 } else if (ISD::isConstantSplatVectorAllOnes(N2.getNode())) {
14915 Other = N1;
14916 }
14917
14918 if (Other && Other.getOpcode() == ISD::ADD) {
14919 SDValue CondLHS = LHS, CondRHS = RHS;
14920 SDValue OpLHS = Other.getOperand(0), OpRHS = Other.getOperand(1);
14921
14922 // Canonicalize condition operands.
14923 if (SatCC == ISD::SETUGE) {
14924 std::swap(CondLHS, CondRHS);
14925 SatCC = ISD::SETULE;
14926 }
14927
14928 // We can test against either of the addition operands.
14929 // x <= x+y ? x+y : ~0 --> uaddsat x, y
14930 // x+y >= x ? x+y : ~0 --> uaddsat x, y
14931 if (SatCC == ISD::SETULE && Other == CondRHS &&
14932 (OpLHS == CondLHS || OpRHS == CondLHS))
14933 return DAG.getNode(ISD::UADDSAT, DL, VT, OpLHS, OpRHS);
14934
14935 if (OpRHS.getOpcode() == CondRHS.getOpcode() &&
14936 (OpRHS.getOpcode() == ISD::BUILD_VECTOR ||
14937 OpRHS.getOpcode() == ISD::SPLAT_VECTOR) &&
14938 CondLHS == OpLHS) {
14939 // If the RHS is a constant we have to reverse the const
14940 // canonicalization.
14941 // x >= ~C ? x+C : ~0 --> uaddsat x, C
14942 auto MatchUADDSAT = [](ConstantSDNode *Op, ConstantSDNode *Cond) {
14943 return Cond->getAPIntValue() == ~Op->getAPIntValue();
14944 };
14945 if (SatCC == ISD::SETULE &&
14946 ISD::matchBinaryPredicate(OpRHS, CondRHS, MatchUADDSAT))
14947 return DAG.getNode(ISD::UADDSAT, DL, VT, OpLHS, OpRHS);
14948 }
14949 }
14950 }
14951
14952 // Match VSELECTs into sub with unsigned saturation.
14953 if (hasOperation(ISD::USUBSAT, VT)) {
14954 // Check if one of the arms of the VSELECT is a zero vector. If it's on
14955 // the left side invert the predicate to simplify logic below.
14956 SDValue Other;
14957 ISD::CondCode SatCC = CC;
14959 Other = N2;
14960 SatCC = ISD::getSetCCInverse(SatCC, VT.getScalarType());
14962 Other = N1;
14963 }
14964
14965 // zext(x) >= y ? trunc(zext(x) - y) : 0
14966 // --> usubsat(trunc(zext(x)),trunc(umin(y,SatLimit)))
14967 // zext(x) > y ? trunc(zext(x) - y) : 0
14968 // --> usubsat(trunc(zext(x)),trunc(umin(y,SatLimit)))
14969 if (Other && Other.getOpcode() == ISD::TRUNCATE &&
14970 Other.getOperand(0).getOpcode() == ISD::SUB &&
14971 (SatCC == ISD::SETUGE || SatCC == ISD::SETUGT)) {
14972 SDValue OpLHS = Other.getOperand(0).getOperand(0);
14973 SDValue OpRHS = Other.getOperand(0).getOperand(1);
14974 if (LHS == OpLHS && RHS == OpRHS && LHS.getOpcode() == ISD::ZERO_EXTEND)
14975 if (SDValue R = getTruncatedUSUBSAT(VT, LHS.getValueType(), LHS, RHS,
14976 DAG, DL))
14977 return R;
14978 }
14979
14980 if (Other && Other.getNumOperands() == 2) {
14981 SDValue CondRHS = RHS;
14982 SDValue OpLHS = Other.getOperand(0), OpRHS = Other.getOperand(1);
14983
14984 if (OpLHS == LHS) {
14985 // Look for a general sub with unsigned saturation first.
14986 // x >= y ? x-y : 0 --> usubsat x, y
14987 // x > y ? x-y : 0 --> usubsat x, y
14988 if ((SatCC == ISD::SETUGE || SatCC == ISD::SETUGT) &&
14989 Other.getOpcode() == ISD::SUB && OpRHS == CondRHS)
14990 return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS);
14991
14992 if (OpRHS.getOpcode() == ISD::BUILD_VECTOR ||
14993 OpRHS.getOpcode() == ISD::SPLAT_VECTOR) {
14994 if (CondRHS.getOpcode() == ISD::BUILD_VECTOR ||
14995 CondRHS.getOpcode() == ISD::SPLAT_VECTOR) {
14996 // If the RHS is a constant we have to reverse the const
14997 // canonicalization.
14998 // x > C-1 ? x+-C : 0 --> usubsat x, C
14999 auto MatchUSUBSAT = [](ConstantSDNode *Op, ConstantSDNode *Cond) {
15000 return (!Op && !Cond) ||
15001 (Op && Cond &&
15002 Cond->getAPIntValue() == (-Op->getAPIntValue() - 1));
15003 };
15004 if (SatCC == ISD::SETUGT && Other.getOpcode() == ISD::ADD &&
15005 ISD::matchBinaryPredicate(OpRHS, CondRHS, MatchUSUBSAT,
15006 /*AllowUndefs*/ true)) {
15007 OpRHS = DAG.getNegative(OpRHS, DL, VT);
15008 return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS);
15009 }
15010
15011 // Another special case: If C was a sign bit, the sub has been
15012 // canonicalized into a xor.
15013 // FIXME: Would it be better to use computeKnownBits to
15014 // determine whether it's safe to decanonicalize the xor?
15015 // x s< 0 ? x^C : 0 --> usubsat x, C
15016 APInt SplatValue;
15017 if (SatCC == ISD::SETLT && Other.getOpcode() == ISD::XOR &&
15018 ISD::isConstantSplatVector(OpRHS.getNode(), SplatValue) &&
15020 SplatValue.isSignMask()) {
15021 // Note that we have to rebuild the RHS constant here to
15022 // ensure we don't rely on particular values of undef lanes.
15023 OpRHS = DAG.getConstant(SplatValue, DL, VT);
15024 return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS);
15025 }
15026 }
15027 }
15028 }
15029 }
15030 }
15031
15032 // (vselect (ugt x, C), (add x, ~C), x) -> (umin (add x, ~C), x)
15033 // (vselect (ult x, C), x, (add x, -C)) -> (umin x, (add x, -C))
15034 if (SDValue UMin = foldSelectToUMin(LHS, RHS, N1, N2, CC, DL))
15035 return UMin;
15036 }
15037
15038 if (SimplifySelectOps(N, N1, N2))
15039 return SDValue(N, 0); // Don't revisit N.
15040
15041 // Fold (vselect all_ones, N1, N2) -> N1
15043 return N1;
15044 // Fold (vselect all_zeros, N1, N2) -> N2
15046 return N2;
15047
15048 // The ConvertSelectToConcatVector function is assuming both the above
15049 // checks for (vselect (build_vector all{ones,zeros) ...) have been made
15050 // and addressed.
15051 if (N1.getOpcode() == ISD::CONCAT_VECTORS &&
15054 if (SDValue CV = ConvertSelectToConcatVector(N, DAG))
15055 return CV;
15056 }
15057
15058 if (SDValue V = foldVSelectOfConstants(N))
15059 return V;
15060
15061 if (hasOperation(ISD::SRA, VT))
15063 return V;
15064
15066 return SDValue(N, 0);
15067
15068 if (SDValue V = combineVSelectWithAllOnesOrZeros(N0, N1, N2, TLI, DAG, DL))
15069 return V;
15070
15072 return R;
15073
15074 return SDValue();
15075}
15076
15077SDValue DAGCombiner::visitSELECT_CC(SDNode *N) {
15078 SDValue N0 = N->getOperand(0);
15079 SDValue N1 = N->getOperand(1);
15080 SDValue N2 = N->getOperand(2);
15081 SDValue N3 = N->getOperand(3);
15083 SDLoc DL(N);
15084
15085 // fold select_cc lhs, rhs, x, x, cc -> x
15086 if (N2 == N3)
15087 return N2;
15088
15089 if (SDValue R = foldSelectCCOfSelect(N, DAG))
15090 return R;
15091
15092 // select_cc bool, 0, x, y, seteq -> select bool, y, x
15093 if (CC == ISD::SETEQ && !LegalTypes && N0.getValueType() == MVT::i1 &&
15094 isNullConstant(N1))
15095 return DAG.getSelect(DL, N2.getValueType(), N0, N3, N2);
15096
15097 // Determine if the condition we're dealing with is constant
15098 if (SDValue SCC = SimplifySetCC(getSetCCResultType(N0.getValueType()), N0, N1,
15099 CC, DL, false)) {
15100 AddToWorklist(SCC.getNode());
15101
15102 // cond always true -> true val
15103 // cond always false -> false val
15104 if (auto *SCCC = dyn_cast<ConstantSDNode>(SCC.getNode()))
15105 return SCCC->isZero() ? N3 : N2;
15106
15107 // When the condition is UNDEF, just return the first operand. This is
15108 // coherent the DAG creation, no setcc node is created in this case
15109 if (SCC->isUndef())
15110 return N2;
15111
15112 // Fold to a simpler select_cc
15113 if (SCC.getOpcode() == ISD::SETCC) {
15114 return DAG.getNode(ISD::SELECT_CC, DL, N2.getValueType(),
15115 SCC.getOperand(0), SCC.getOperand(1), N2, N3,
15116 SCC.getOperand(2), SCC->getFlags());
15117 }
15118 }
15119
15120 // If we can fold this based on the true/false value, do so.
15121 if (SimplifySelectOps(N, N2, N3))
15122 return SDValue(N, 0); // Don't revisit N.
15123
15124 auto [Opcode, NewLHS, NewRHS] = combineSelectCCToPseudoMinMax(
15125 DAG, DL, CC, N0, N1, N2, N3, N->getFlags(), /*IsStrict=*/false);
15126 if (Opcode)
15127 return DAG.getNode(Opcode, DL, N->getValueType(0), NewLHS, NewRHS,
15128 N->getFlags());
15129
15130 // fold select_cc into other things, such as min/max/abs
15131 return SimplifySelectCC(DL, N0, N1, N2, N3, CC);
15132}
15133
15134SDValue DAGCombiner::visitSETCC(SDNode *N) {
15135 // setcc is very commonly used as an argument to brcond or cond_loop. This
15136 // pattern also lend itself to numerous combines and, as a result, it is
15137 // desired we keep the argument to a brcond as a setcc as much as possible.
15138 bool PreferSetCC =
15139 N->hasOneUse() && (N->user_begin()->getOpcode() == ISD::BRCOND ||
15140 N->user_begin()->getOpcode() == ISD::COND_LOOP);
15141
15142 ISD::CondCode Cond = N->getOperandCondCode(2);
15143 EVT VT = N->getValueType(0);
15144 SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
15145 SDLoc DL(N);
15146
15147 if (SDValue Combined = SimplifySetCC(VT, N0, N1, Cond, DL, !PreferSetCC)) {
15148 // If we prefer to have a setcc, and we don't, we'll try our best to
15149 // recreate one using rebuildSetCC.
15150 if (PreferSetCC && Combined.getOpcode() != ISD::SETCC) {
15151 SDValue NewSetCC = rebuildSetCC(Combined);
15152
15153 // We don't have anything interesting to combine to.
15154 if (NewSetCC.getNode() == N)
15155 return SDValue();
15156
15157 if (NewSetCC)
15158 return NewSetCC;
15159 }
15160 return Combined;
15161 }
15162
15163 // Optimize
15164 // 1) (icmp eq/ne (and X, C0), (shift X, C1))
15165 // or
15166 // 2) (icmp eq/ne X, (rotate X, C1))
15167 // If C0 is a mask or shifted mask and the shift amt (C1) isolates the
15168 // remaining bits (i.e something like `(x64 & UINT32_MAX) == (x64 >> 32)`)
15169 // Then:
15170 // If C1 divides the bit width, then the rotate and shift+and versions are
15171 // equivalent, so we can interchange them depending on target preference.
15172 // Otherwise, if we have the shift+and version we can interchange srl/shl
15173 // which inturn affects the constant C0. We can use this to get better
15174 // constants again determined by target preference.
15175 if (Cond == ISD::SETNE || Cond == ISD::SETEQ) {
15176 auto IsAndWithShift = [](SDValue A, SDValue B) {
15177 return A.getOpcode() == ISD::AND &&
15178 (B.getOpcode() == ISD::SRL || B.getOpcode() == ISD::SHL) &&
15179 A.getOperand(0) == B.getOperand(0);
15180 };
15181 auto IsRotateWithOp = [](SDValue A, SDValue B) {
15182 return (B.getOpcode() == ISD::ROTL || B.getOpcode() == ISD::ROTR) &&
15183 B.getOperand(0) == A;
15184 };
15185 SDValue AndOrOp = SDValue(), ShiftOrRotate = SDValue();
15186 bool IsRotate = false;
15187
15188 // Find either shift+and or rotate pattern.
15189 if (IsAndWithShift(N0, N1)) {
15190 AndOrOp = N0;
15191 ShiftOrRotate = N1;
15192 } else if (IsAndWithShift(N1, N0)) {
15193 AndOrOp = N1;
15194 ShiftOrRotate = N0;
15195 } else if (IsRotateWithOp(N0, N1)) {
15196 IsRotate = true;
15197 AndOrOp = N0;
15198 ShiftOrRotate = N1;
15199 } else if (IsRotateWithOp(N1, N0)) {
15200 IsRotate = true;
15201 AndOrOp = N1;
15202 ShiftOrRotate = N0;
15203 }
15204
15205 if (AndOrOp && ShiftOrRotate && ShiftOrRotate.hasOneUse() &&
15206 (IsRotate || AndOrOp.hasOneUse())) {
15207 EVT OpVT = N0.getValueType();
15208 // Get constant shift/rotate amount and possibly mask (if its shift+and
15209 // variant).
15210 auto GetAPIntValue = [](SDValue Op) -> std::optional<APInt> {
15211 ConstantSDNode *CNode = isConstOrConstSplat(Op, /*AllowUndefs*/ false,
15212 /*AllowTrunc*/ false);
15213 if (CNode == nullptr)
15214 return std::nullopt;
15215 return CNode->getAPIntValue();
15216 };
15217 std::optional<APInt> AndCMask =
15218 IsRotate ? std::nullopt : GetAPIntValue(AndOrOp.getOperand(1));
15219 std::optional<APInt> ShiftCAmt =
15220 GetAPIntValue(ShiftOrRotate.getOperand(1));
15221 unsigned NumBits = OpVT.getScalarSizeInBits();
15222
15223 // We found constants.
15224 if (ShiftCAmt && (IsRotate || AndCMask) && ShiftCAmt->ult(NumBits)) {
15225 unsigned ShiftOpc = ShiftOrRotate.getOpcode();
15226 // Check that the constants meet the constraints.
15227 bool CanTransform = IsRotate;
15228 if (!CanTransform) {
15229 // Check that mask and shift compliment eachother
15230 CanTransform = *ShiftCAmt == (~*AndCMask).popcount();
15231 // Check that we are comparing all bits
15232 CanTransform &= (*ShiftCAmt + AndCMask->popcount()) == NumBits;
15233 // Check that the and mask is correct for the shift
15234 CanTransform &=
15235 ShiftOpc == ISD::SHL ? (~*AndCMask).isMask() : AndCMask->isMask();
15236 }
15237
15238 // The rotate and shift+and forms are only equivalent if the shift
15239 // amount divides the bit width.
15240 bool MayTransformRotate =
15241 !ShiftCAmt->isZero() && NumBits % ShiftCAmt->getZExtValue() == 0;
15242 // See if target prefers another shift/rotate opcode.
15243 unsigned NewShiftOpc = TLI.preferedOpcodeForCmpEqPiecesOfOperand(
15244 OpVT, ShiftOpc, MayTransformRotate, *ShiftCAmt, AndCMask);
15245 // Transform is valid and we have a new preference.
15246 if (CanTransform && NewShiftOpc != ShiftOpc) {
15247 SDValue NewShiftOrRotate =
15248 DAG.getNode(NewShiftOpc, DL, OpVT, ShiftOrRotate.getOperand(0),
15249 ShiftOrRotate.getOperand(1));
15250 SDValue NewAndOrOp = SDValue();
15251
15252 if (NewShiftOpc == ISD::SHL || NewShiftOpc == ISD::SRL) {
15253 APInt NewMask =
15254 NewShiftOpc == ISD::SHL
15255 ? APInt::getHighBitsSet(NumBits,
15256 NumBits - ShiftCAmt->getZExtValue())
15257 : APInt::getLowBitsSet(NumBits,
15258 NumBits - ShiftCAmt->getZExtValue());
15259 NewAndOrOp =
15260 DAG.getNode(ISD::AND, DL, OpVT, ShiftOrRotate.getOperand(0),
15261 DAG.getConstant(NewMask, DL, OpVT));
15262 } else {
15263 NewAndOrOp = ShiftOrRotate.getOperand(0);
15264 }
15265
15266 return DAG.getSetCC(DL, VT, NewAndOrOp, NewShiftOrRotate, Cond);
15267 }
15268 }
15269 }
15270 }
15271 return SDValue();
15272}
15273
15274SDValue DAGCombiner::visitSETCCCARRY(SDNode *N) {
15275 SDValue LHS = N->getOperand(0);
15276 SDValue RHS = N->getOperand(1);
15277 SDValue Carry = N->getOperand(2);
15278 SDValue Cond = N->getOperand(3);
15279
15280 // If Carry is false, fold to a regular SETCC.
15281 if (isNullConstant(Carry))
15282 return DAG.getNode(ISD::SETCC, SDLoc(N), N->getVTList(), LHS, RHS, Cond);
15283
15284 return SDValue();
15285}
15286
15287/// Check if N satisfies:
15288/// N is used once.
15289/// N is a Load.
15290/// The load is compatible with ExtOpcode. It means
15291/// If load has explicit zero/sign extension, ExpOpcode must have the same
15292/// extension.
15293/// Otherwise returns true.
15294static bool isCompatibleLoad(SDValue N, unsigned ExtOpcode) {
15295 if (!N.hasOneUse())
15296 return false;
15297
15298 if (!isa<LoadSDNode>(N))
15299 return false;
15300
15302 ISD::LoadExtType LoadExt = Load->getExtensionType();
15303 if (LoadExt == ISD::NON_EXTLOAD || LoadExt == ISD::EXTLOAD)
15304 return true;
15305
15306 // Now LoadExt is either SEXTLOAD or ZEXTLOAD, ExtOpcode must have the same
15307 // extension.
15308 if ((LoadExt == ISD::SEXTLOAD && ExtOpcode != ISD::SIGN_EXTEND) ||
15309 (LoadExt == ISD::ZEXTLOAD && ExtOpcode != ISD::ZERO_EXTEND))
15310 return false;
15311
15312 return true;
15313}
15314
15315/// Fold
15316/// (sext (select c, load x, load y)) -> (select c, sextload x, sextload y)
15317/// (zext (select c, load x, load y)) -> (select c, zextload x, zextload y)
15318/// (aext (select c, load x, load y)) -> (select c, extload x, extload y)
15319/// This function is called by the DAGCombiner when visiting sext/zext/aext
15320/// dag nodes (see for example method DAGCombiner::visitSIGN_EXTEND).
15322 SelectionDAG &DAG, const SDLoc &DL,
15323 CombineLevel Level) {
15324 unsigned Opcode = N->getOpcode();
15325 SDValue N0 = N->getOperand(0);
15326 EVT VT = N->getValueType(0);
15327 assert((Opcode == ISD::SIGN_EXTEND || Opcode == ISD::ZERO_EXTEND ||
15328 Opcode == ISD::ANY_EXTEND) &&
15329 "Expected EXTEND dag node in input!");
15330
15331 SDValue Cond, Op1, Op2;
15333 m_Value(Op2)))))
15334 return SDValue();
15335
15336 if (!isCompatibleLoad(Op1, Opcode) || !isCompatibleLoad(Op2, Opcode))
15337 return SDValue();
15338
15339 auto ExtLoadOpcode = ISD::EXTLOAD;
15340 if (Opcode == ISD::SIGN_EXTEND)
15341 ExtLoadOpcode = ISD::SEXTLOAD;
15342 else if (Opcode == ISD::ZERO_EXTEND)
15343 ExtLoadOpcode = ISD::ZEXTLOAD;
15344
15345 // Illegal VSELECT may ISel fail if happen after legalization (DAG
15346 // Combine2), so we should conservatively check the OperationAction.
15347 LoadSDNode *Load1 = cast<LoadSDNode>(Op1);
15348 LoadSDNode *Load2 = cast<LoadSDNode>(Op2);
15349 if (!TLI.isLoadLegal(VT, Load1->getMemoryVT(), Load1->getAlign(),
15350 Load1->getAddressSpace(), ExtLoadOpcode, false) ||
15351 !TLI.isLoadLegal(VT, Load2->getMemoryVT(), Load2->getAlign(),
15352 Load2->getAddressSpace(), ExtLoadOpcode, false) ||
15353 (N0->getOpcode() == ISD::VSELECT && Level >= AfterLegalizeTypes &&
15354 TLI.getOperationAction(ISD::VSELECT, VT) != TargetLowering::Legal))
15355 return SDValue();
15356
15357 SDValue Ext1 = DAG.getNode(Opcode, DL, VT, Op1);
15358 SDValue Ext2 = DAG.getNode(Opcode, DL, VT, Op2);
15359 return DAG.getSelect(DL, VT, Cond, Ext1, Ext2);
15360}
15361
15362/// Try to fold a sext/zext/aext dag node into a ConstantSDNode or
15363/// a build_vector of constants.
15364/// This function is called by the DAGCombiner when visiting sext/zext/aext
15365/// dag nodes (see for example method DAGCombiner::visitSIGN_EXTEND).
15366/// Vector extends are not folded if operations are legal; this is to
15367/// avoid introducing illegal build_vector dag nodes.
15369 const TargetLowering &TLI,
15370 SelectionDAG &DAG, bool LegalTypes) {
15371 unsigned Opcode = N->getOpcode();
15372 SDValue N0 = N->getOperand(0);
15373 EVT VT = N->getValueType(0);
15374
15375 assert((ISD::isExtOpcode(Opcode) || ISD::isExtVecInRegOpcode(Opcode)) &&
15376 "Expected EXTEND dag node in input!");
15377
15378 // fold (sext c1) -> c1
15379 // fold (zext c1) -> c1
15380 // fold (aext c1) -> c1
15381 if (isa<ConstantSDNode>(N0))
15382 return DAG.getNode(Opcode, DL, VT, N0);
15383
15384 // fold (sext (select cond, c1, c2)) -> (select cond, sext c1, sext c2)
15385 // fold (zext (select cond, c1, c2)) -> (select cond, zext c1, zext c2)
15386 // fold (aext (select cond, c1, c2)) -> (select cond, sext c1, sext c2)
15387 if (N0->getOpcode() == ISD::SELECT) {
15388 SDValue Op1 = N0->getOperand(1);
15389 SDValue Op2 = N0->getOperand(2);
15390 if (isa<ConstantSDNode>(Op1) && isa<ConstantSDNode>(Op2) &&
15391 (Opcode != ISD::ZERO_EXTEND || !TLI.isZExtFree(N0.getValueType(), VT))) {
15392 // For any_extend, choose sign extension of the constants to allow a
15393 // possible further transform to sign_extend_inreg.i.e.
15394 //
15395 // t1: i8 = select t0, Constant:i8<-1>, Constant:i8<0>
15396 // t2: i64 = any_extend t1
15397 // -->
15398 // t3: i64 = select t0, Constant:i64<-1>, Constant:i64<0>
15399 // -->
15400 // t4: i64 = sign_extend_inreg t3
15401 unsigned FoldOpc = Opcode;
15402 if (FoldOpc == ISD::ANY_EXTEND)
15403 FoldOpc = ISD::SIGN_EXTEND;
15404 return DAG.getSelect(DL, VT, N0->getOperand(0),
15405 DAG.getNode(FoldOpc, DL, VT, Op1),
15406 DAG.getNode(FoldOpc, DL, VT, Op2));
15407 }
15408 }
15409
15410 // fold (sext (build_vector AllConstants) -> (build_vector AllConstants)
15411 // fold (zext (build_vector AllConstants) -> (build_vector AllConstants)
15412 // fold (aext (build_vector AllConstants) -> (build_vector AllConstants)
15413 EVT SVT = VT.getScalarType();
15414 if (!(VT.isVector() && (!LegalTypes || TLI.isTypeLegal(SVT)) &&
15416 return SDValue();
15417
15418 // We can fold this node into a build_vector.
15419 unsigned VTBits = SVT.getSizeInBits();
15420 unsigned EVTBits = N0->getValueType(0).getScalarSizeInBits();
15422 unsigned NumElts = VT.getVectorNumElements();
15423
15424 // Extend constant bool vectors to match the target's BooleanContent.
15425 bool SExt = Opcode == ISD::SIGN_EXTEND ||
15427 (Opcode == ISD::ANY_EXTEND && EVTBits == 1 &&
15428 TLI.getBooleanContents(VT) ==
15430
15431 for (unsigned i = 0; i != NumElts; ++i) {
15432 SDValue Op = N0.getOperand(i);
15433 if (Op.isUndef()) {
15434 if (Opcode == ISD::ANY_EXTEND || Opcode == ISD::ANY_EXTEND_VECTOR_INREG)
15435 Elts.push_back(DAG.getUNDEF(SVT));
15436 else
15437 Elts.push_back(DAG.getConstant(0, DL, SVT));
15438 continue;
15439 }
15440
15441 SDLoc DL(Op);
15442 // Get the constant value and if needed trunc it to the size of the type.
15443 // Nodes like build_vector might have constants wider than the scalar type.
15444 APInt C = Op->getAsAPIntVal().zextOrTrunc(EVTBits);
15445 if (SExt)
15446 Elts.push_back(DAG.getConstant(C.sext(VTBits), DL, SVT));
15447 else
15448 Elts.push_back(DAG.getConstant(C.zext(VTBits), DL, SVT));
15449 }
15450
15451 return DAG.getBuildVector(VT, DL, Elts);
15452}
15453
15454// ExtendUsesToFormExtLoad - Trying to extend uses of a load to enable this:
15455// "fold ({s|z|a}ext (load x)) -> ({s|z|a}ext (truncate ({s|z|a}extload x)))"
15456// transformation. Returns true if extension are possible and the above
15457// mentioned transformation is profitable.
15459 unsigned ExtOpc,
15460 SmallVectorImpl<SDNode *> &ExtendNodes,
15461 const TargetLowering &TLI) {
15462 bool HasCopyToRegUses = false;
15463 bool isTruncFree = TLI.isTruncateFree(VT, N0.getValueType());
15464 for (SDUse &Use : N0->uses()) {
15465 SDNode *User = Use.getUser();
15466 if (User == N)
15467 continue;
15468 if (Use.getResNo() != N0.getResNo())
15469 continue;
15470 // FIXME: Only extend SETCC N, N and SETCC N, c for now.
15471 if (ExtOpc != ISD::ANY_EXTEND && User->getOpcode() == ISD::SETCC) {
15472 ISD::CondCode CC = User->getOperandCondCode(2);
15473 if (ExtOpc == ISD::ZERO_EXTEND && ISD::isSignedIntSetCC(CC))
15474 // Sign bits will be lost after a zext.
15475 return false;
15476 bool Add = false;
15477 for (unsigned i = 0; i != 2; ++i) {
15478 SDValue UseOp = User->getOperand(i);
15479 if (UseOp == N0)
15480 continue;
15481 if (!isa<ConstantSDNode>(UseOp))
15482 return false;
15483 Add = true;
15484 }
15485 if (Add)
15486 ExtendNodes.push_back(User);
15487 continue;
15488 }
15489 // If truncates aren't free and there are users we can't
15490 // extend, it isn't worthwhile.
15491 if (!isTruncFree)
15492 return false;
15493 // Remember if this value is live-out.
15494 if (User->getOpcode() == ISD::CopyToReg)
15495 HasCopyToRegUses = true;
15496 }
15497
15498 if (HasCopyToRegUses) {
15499 bool BothLiveOut = false;
15500 for (SDUse &Use : N->uses()) {
15501 if (Use.getResNo() == 0 && Use.getUser()->getOpcode() == ISD::CopyToReg) {
15502 BothLiveOut = true;
15503 break;
15504 }
15505 }
15506 if (BothLiveOut)
15507 // Both unextended and extended values are live out. There had better be
15508 // a good reason for the transformation.
15509 return !ExtendNodes.empty();
15510 }
15511 return true;
15512}
15513
15514void DAGCombiner::ExtendSetCCUses(const SmallVectorImpl<SDNode *> &SetCCs,
15515 SDValue OrigLoad, SDValue ExtLoad,
15516 ISD::NodeType ExtType) {
15517 // Extend SetCC uses if necessary.
15518 SDLoc DL(ExtLoad);
15519 for (SDNode *SetCC : SetCCs) {
15521
15522 for (unsigned j = 0; j != 2; ++j) {
15523 SDValue SOp = SetCC->getOperand(j);
15524 if (SOp == OrigLoad)
15525 Ops.push_back(ExtLoad);
15526 else
15527 Ops.push_back(DAG.getNode(ExtType, DL, ExtLoad->getValueType(0), SOp));
15528 }
15529
15530 Ops.push_back(SetCC->getOperand(2));
15531 CombineTo(SetCC, DAG.getNode(ISD::SETCC, DL, SetCC->getValueType(0), Ops));
15532 }
15533}
15534
15535// FIXME: Bring more similar combines here, common to sext/zext (maybe aext?).
15536SDValue DAGCombiner::CombineExtLoad(SDNode *N) {
15537 SDValue N0 = N->getOperand(0);
15538 EVT DstVT = N->getValueType(0);
15539 EVT SrcVT = N0.getValueType();
15540
15541 assert((N->getOpcode() == ISD::SIGN_EXTEND ||
15542 N->getOpcode() == ISD::ZERO_EXTEND) &&
15543 "Unexpected node type (not an extend)!");
15544
15545 // fold (sext (load x)) to multiple smaller sextloads; same for zext.
15546 // For example, on a target with legal v4i32, but illegal v8i32, turn:
15547 // (v8i32 (sext (v8i16 (load x))))
15548 // into:
15549 // (v8i32 (concat_vectors (v4i32 (sextload x)),
15550 // (v4i32 (sextload (x + 16)))))
15551 // Where uses of the original load, i.e.:
15552 // (v8i16 (load x))
15553 // are replaced with:
15554 // (v8i16 (truncate
15555 // (v8i32 (concat_vectors (v4i32 (sextload x)),
15556 // (v4i32 (sextload (x + 16)))))))
15557 //
15558 // This combine is only applicable to illegal, but splittable, vectors.
15559 // All legal types, and illegal non-vector types, are handled elsewhere.
15560 // This combine is controlled by TargetLowering::isVectorLoadExtDesirable.
15561 //
15562 if (N0->getOpcode() != ISD::LOAD)
15563 return SDValue();
15564
15565 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
15566
15567 if (!ISD::isNON_EXTLoad(LN0) || !ISD::isUNINDEXEDLoad(LN0) ||
15568 !N0.hasOneUse() || !LN0->isSimple() ||
15569 !DstVT.isVector() || !DstVT.isPow2VectorType() ||
15571 return SDValue();
15572
15574 if (!ExtendUsesToFormExtLoad(DstVT, N, N0, N->getOpcode(), SetCCs, TLI))
15575 return SDValue();
15576
15577 ISD::LoadExtType ExtType =
15578 N->getOpcode() == ISD::SIGN_EXTEND ? ISD::SEXTLOAD : ISD::ZEXTLOAD;
15579
15580 // Try to split the vector types to get down to legal types.
15581 EVT SplitSrcVT = SrcVT;
15582 EVT SplitDstVT = DstVT;
15583 while (!TLI.isLoadLegalOrCustom(SplitDstVT, SplitSrcVT, LN0->getAlign(),
15584 LN0->getAddressSpace(), ExtType, false) &&
15585 SplitSrcVT.getVectorNumElements() > 1) {
15586 SplitDstVT = DAG.GetSplitDestVTs(SplitDstVT).first;
15587 SplitSrcVT = DAG.GetSplitDestVTs(SplitSrcVT).first;
15588 }
15589
15590 if (!TLI.isLoadLegalOrCustom(SplitDstVT, SplitSrcVT, LN0->getAlign(),
15591 LN0->getAddressSpace(), ExtType, false))
15592 return SDValue();
15593
15594 assert(!DstVT.isScalableVector() && "Unexpected scalable vector type");
15595
15596 SDLoc DL(N);
15597 const unsigned NumSplits =
15598 DstVT.getVectorNumElements() / SplitDstVT.getVectorNumElements();
15599 const unsigned Stride = SplitSrcVT.getStoreSize();
15602
15603 SDValue BasePtr = LN0->getBasePtr();
15604 for (unsigned Idx = 0; Idx < NumSplits; Idx++) {
15605 const unsigned Offset = Idx * Stride;
15606
15608 ExtType, SDLoc(LN0), SplitDstVT, LN0->getChain(), BasePtr,
15609 LN0->getPointerInfo().getWithOffset(Offset), SplitSrcVT,
15610 LN0->getBaseAlign(), LN0->getMemOperand()->getFlags(),
15612
15613 BasePtr = DAG.getMemBasePlusOffset(BasePtr, TypeSize::getFixed(Stride), DL);
15614
15615 Loads.push_back(SplitLoad.getValue(0));
15616 Chains.push_back(SplitLoad.getValue(1));
15617 }
15618
15619 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
15620 SDValue NewValue = DAG.getNode(ISD::CONCAT_VECTORS, DL, DstVT, Loads);
15621
15622 // Simplify TF.
15623 AddToWorklist(NewChain.getNode());
15624
15625 CombineTo(N, NewValue);
15626
15627 // Replace uses of the original load (before extension)
15628 // with a truncate of the concatenated sextloaded vectors.
15629 SDValue Trunc =
15630 DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), NewValue);
15631 ExtendSetCCUses(SetCCs, N0, NewValue, (ISD::NodeType)N->getOpcode());
15632 CombineTo(N0.getNode(), Trunc, NewChain);
15633 return SDValue(N, 0); // Return N so it doesn't get rechecked!
15634}
15635
15636// fold (zext (and/or/xor (shl/shr (load x), cst), cst)) ->
15637// (and/or/xor (shl/shr (zextload x), (zext cst)), (zext cst))
15638SDValue DAGCombiner::CombineZExtLogicopShiftLoad(SDNode *N) {
15639 assert(N->getOpcode() == ISD::ZERO_EXTEND);
15640 EVT VT = N->getValueType(0);
15641 EVT OrigVT = N->getOperand(0).getValueType();
15642 if (TLI.isZExtFree(OrigVT, VT))
15643 return SDValue();
15644
15645 // and/or/xor
15646 SDValue N0 = N->getOperand(0);
15647 if (!ISD::isBitwiseLogicOp(N0.getOpcode()) ||
15648 N0.getOperand(1).getOpcode() != ISD::Constant ||
15649 (LegalOperations && !TLI.isOperationLegal(N0.getOpcode(), VT)))
15650 return SDValue();
15651
15652 // shl/shr
15653 SDValue N1 = N0->getOperand(0);
15654 if (!(N1.getOpcode() == ISD::SHL || N1.getOpcode() == ISD::SRL) ||
15655 N1.getOperand(1).getOpcode() != ISD::Constant ||
15656 (LegalOperations && !TLI.isOperationLegal(N1.getOpcode(), VT)))
15657 return SDValue();
15658
15659 // load
15660 if (!isa<LoadSDNode>(N1.getOperand(0)))
15661 return SDValue();
15662 LoadSDNode *Load = cast<LoadSDNode>(N1.getOperand(0));
15663 EVT MemVT = Load->getMemoryVT();
15664 if (!TLI.isLoadLegal(VT, MemVT, Load->getAlign(), Load->getAddressSpace(),
15665 ISD::ZEXTLOAD, false) ||
15666 Load->getExtensionType() == ISD::SEXTLOAD || Load->isIndexed())
15667 return SDValue();
15668
15669
15670 // If the shift op is SHL, the logic op must be AND, otherwise the result
15671 // will be wrong.
15672 if (N1.getOpcode() == ISD::SHL && N0.getOpcode() != ISD::AND)
15673 return SDValue();
15674
15675 if (!N0.hasOneUse() || !N1.hasOneUse())
15676 return SDValue();
15677
15679 if (!ExtendUsesToFormExtLoad(VT, N1.getNode(), N1.getOperand(0),
15680 ISD::ZERO_EXTEND, SetCCs, TLI))
15681 return SDValue();
15682
15683 // Actually do the transformation.
15684 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(Load), VT,
15685 Load->getChain(), Load->getBasePtr(),
15686 Load->getMemoryVT(), Load->getMemOperand());
15687
15688 SDLoc DL1(N1);
15689 SDValue Shift = DAG.getNode(N1.getOpcode(), DL1, VT, ExtLoad,
15690 N1.getOperand(1));
15691
15692 APInt Mask = N0.getConstantOperandAPInt(1).zext(VT.getSizeInBits());
15693 SDLoc DL0(N0);
15694 SDValue And = DAG.getNode(N0.getOpcode(), DL0, VT, Shift,
15695 DAG.getConstant(Mask, DL0, VT));
15696
15697 ExtendSetCCUses(SetCCs, N1.getOperand(0), ExtLoad, ISD::ZERO_EXTEND);
15698 CombineTo(N, And);
15699 if (SDValue(Load, 0).hasOneUse()) {
15700 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), ExtLoad.getValue(1));
15701 } else {
15702 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(Load),
15703 Load->getValueType(0), ExtLoad);
15704 CombineTo(Load, Trunc, ExtLoad.getValue(1));
15705 }
15706
15707 // N0 is dead at this point.
15708 recursivelyDeleteUnusedNodes(N0.getNode());
15709
15710 return SDValue(N,0); // Return N so it doesn't get rechecked!
15711}
15712
15713/// If we're narrowing or widening the result of a vector select and the final
15714/// size is the same size as a setcc (compare) feeding the select, then try to
15715/// apply the cast operation to the select's operands because matching vector
15716/// sizes for a select condition and other operands should be more efficient.
15717SDValue DAGCombiner::matchVSelectOpSizesWithSetCC(SDNode *Cast) {
15718 unsigned CastOpcode = Cast->getOpcode();
15719 assert((CastOpcode == ISD::SIGN_EXTEND || CastOpcode == ISD::ZERO_EXTEND ||
15720 CastOpcode == ISD::TRUNCATE || CastOpcode == ISD::FP_EXTEND ||
15721 CastOpcode == ISD::FP_ROUND) &&
15722 "Unexpected opcode for vector select narrowing/widening");
15723
15724 // We only do this transform before legal ops because the pattern may be
15725 // obfuscated by target-specific operations after legalization. Do not create
15726 // an illegal select op, however, because that may be difficult to lower.
15727 EVT VT = Cast->getValueType(0);
15728 if (LegalOperations || !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT))
15729 return SDValue();
15730
15731 SDValue VSel = Cast->getOperand(0);
15732 if (VSel.getOpcode() != ISD::VSELECT || !VSel.hasOneUse() ||
15733 VSel.getOperand(0).getOpcode() != ISD::SETCC)
15734 return SDValue();
15735
15736 // Does the setcc have the same vector size as the casted select?
15737 SDValue SetCC = VSel.getOperand(0);
15738 EVT SetCCVT = getSetCCResultType(SetCC.getOperand(0).getValueType());
15739 if (SetCCVT.getSizeInBits() != VT.getSizeInBits())
15740 return SDValue();
15741
15742 // cast (vsel (setcc X), A, B) --> vsel (setcc X), (cast A), (cast B)
15743 SDValue A = VSel.getOperand(1);
15744 SDValue B = VSel.getOperand(2);
15745 SDValue CastA, CastB;
15746 SDLoc DL(Cast);
15747 if (CastOpcode == ISD::FP_ROUND) {
15748 // FP_ROUND (fptrunc) has an extra flag operand to pass along.
15749 CastA = DAG.getNode(CastOpcode, DL, VT, A, Cast->getOperand(1));
15750 CastB = DAG.getNode(CastOpcode, DL, VT, B, Cast->getOperand(1));
15751 } else {
15752 CastA = DAG.getNode(CastOpcode, DL, VT, A);
15753 CastB = DAG.getNode(CastOpcode, DL, VT, B);
15754 }
15755 return DAG.getNode(ISD::VSELECT, DL, VT, SetCC, CastA, CastB);
15756}
15757
15758// fold ([s|z]ext ([s|z]extload x)) -> ([s|z]ext (truncate ([s|z]extload x)))
15759// fold ([s|z]ext ( extload x)) -> ([s|z]ext (truncate ([s|z]extload x)))
15761 const TargetLowering &TLI, EVT VT,
15762 bool LegalOperations, SDNode *N,
15763 SDValue N0, ISD::LoadExtType ExtLoadType) {
15764 bool Frozen = N0.getOpcode() == ISD::FREEZE;
15765 auto *OldExtLoad = dyn_cast<LoadSDNode>(Frozen ? N0.getOperand(0) : N0);
15766 if (!OldExtLoad)
15767 return SDValue();
15768
15769 bool isAExtLoad = (ExtLoadType == ISD::SEXTLOAD)
15770 ? ISD::isSEXTLoad(OldExtLoad)
15771 : ISD::isZEXTLoad(OldExtLoad);
15772 if ((!isAExtLoad && !ISD::isEXTLoad(OldExtLoad)) ||
15773 !ISD::isUNINDEXEDLoad(OldExtLoad) || !OldExtLoad->hasNUsesOfValue(1, 0))
15774 return SDValue();
15775
15776 EVT MemVT = OldExtLoad->getMemoryVT();
15777 if ((LegalOperations || !OldExtLoad->isSimple() || VT.isVector()) &&
15778 !TLI.isLoadLegal(VT, MemVT, OldExtLoad->getAlign(),
15779 OldExtLoad->getAddressSpace(), ExtLoadType, false))
15780 return SDValue();
15781
15782 SDLoc DL(OldExtLoad);
15783 SDValue ExtLoad = DAG.getExtLoad(ExtLoadType, DL, VT, OldExtLoad->getChain(),
15784 OldExtLoad->getBasePtr(), MemVT,
15785 OldExtLoad->getMemOperand());
15786 SDValue Res = ExtLoad;
15787 if (Frozen) {
15788 Res = DAG.getFreeze(ExtLoad);
15789 Res = DAG.getNode(
15790 ExtLoadType == ISD::SEXTLOAD ? ISD::AssertSext : ISD::AssertZext, DL,
15791 Res.getValueType(), Res,
15792 DAG.getValueType(OldExtLoad->getValueType(0).getScalarType()));
15793 }
15794 Combiner.CombineTo(N, Res);
15795 DAG.ReplaceAllUsesOfValueWith(SDValue(OldExtLoad, 1), ExtLoad.getValue(1));
15796 if (N0->use_empty())
15797 Combiner.recursivelyDeleteUnusedNodes(N0.getNode());
15798 return SDValue(N, 0); // Return N so it doesn't get rechecked!
15799}
15800
15801// fold ([s|z]ext (load x)) -> ([s|z]ext (truncate ([s|z]extload x)))
15802// Only generate vector extloads when 1) they're legal, and 2) they are
15803// deemed desirable by the target. NonNegZExt can be set to true if a zero
15804// extend has the nonneg flag to allow use of sextload if profitable.
15806 const TargetLowering &TLI, EVT VT,
15807 bool LegalOperations, SDNode *N, SDValue N0,
15808 ISD::LoadExtType ExtLoadType,
15809 ISD::NodeType ExtOpc,
15810 bool NonNegZExt = false) {
15811
15812 bool Frozen = N0.getOpcode() == ISD::FREEZE;
15813 SDValue Freeze = Frozen ? N0 : SDValue();
15814 auto *Load = dyn_cast<LoadSDNode>(Frozen ? N0.getOperand(0) : N0);
15815 // TODO: Support multiple uses of the load when frozen.
15817 (Frozen && !Load->hasNUsesOfValue(1, 0)))
15818 return {};
15819
15820 // If this is zext nneg, see if it would make sense to treat it as a sext.
15821 if (NonNegZExt) {
15822 assert(ExtLoadType == ISD::ZEXTLOAD && ExtOpc == ISD::ZERO_EXTEND &&
15823 "Unexpected load type or opcode");
15824 for (SDNode *User : Load->users()) {
15825 if (User->getOpcode() == ISD::SETCC) {
15826 ISD::CondCode CC = User->getOperandCondCode(2);
15828 ExtLoadType = ISD::SEXTLOAD;
15829 ExtOpc = ISD::SIGN_EXTEND;
15830 break;
15831 }
15832 }
15833 }
15834 }
15835
15836 // TODO: isFixedLengthVector() should be removed and any negative effects on
15837 // code generation being the result of that target's implementation of
15838 // isVectorLoadExtDesirable().
15839 if ((LegalOperations || VT.isFixedLengthVector() || !Load->isSimple()) &&
15840 !TLI.isLoadLegal(VT, Load->getValueType(0), Load->getAlign(),
15841 Load->getAddressSpace(), ExtLoadType, false))
15842 return {};
15843
15844 bool DoXform = true;
15846 if (!N0->hasOneUse())
15847 DoXform = ExtendUsesToFormExtLoad(VT, N, Frozen ? Freeze : SDValue(Load, 0),
15848 ExtOpc, SetCCs, TLI);
15849 if (VT.isVector())
15850 DoXform &= TLI.isVectorLoadExtDesirable(SDValue(N, 0));
15851 if (!DoXform)
15852 return {};
15853
15854 SDLoc DL(Load);
15855
15856 auto SalvageDbgValue = [&](SDDbgValue *Dbg, SDValue Old, SDValue New,
15857 unsigned OldBits, unsigned NewBits,
15858 bool IsSigned) {
15859 SmallVector<SDDbgOperand> Locs = Dbg->copyLocationOps();
15860 bool Changed = false;
15861
15862 bool IsVariadic = Dbg->isVariadic();
15863 SmallVector<unsigned, 2> AffectedArgs;
15864
15865 for (unsigned I = 0, E = Locs.size(); I != E; ++I) {
15866 SDDbgOperand &Op = Locs[I];
15867 if (Op.getKind() != SDDbgOperand::SDNODE)
15868 continue;
15869
15870 if (Op.getSDNode() == Old.getNode() && Op.getResNo() == Old.getResNo()) {
15871 Op = SDDbgOperand::fromNode(New.getNode(), New.getResNo());
15872 Changed = true;
15873
15874 if (IsVariadic)
15875 AffectedArgs.push_back(I);
15876 }
15877 }
15878
15879 if (!Changed)
15880 return;
15881
15882 const DIExpression *OldExpr = Dbg->getExpression();
15883 const DIExpression *NewExpr = nullptr;
15884
15885 if (!IsVariadic) {
15886 // Do not introduce DW_OP_LLVM_arg into ordinary single-location
15887 // DBG_VALUEs.
15888 NewExpr = DIExpression::appendExt(OldExpr, NewBits, OldBits, IsSigned);
15889 } else {
15890 auto ExtOps = DIExpression::getExtOps(NewBits, OldBits, IsSigned);
15891
15893
15894 for (unsigned ArgNo : AffectedArgs)
15896 /*StackValue=*/false);
15897 }
15898
15899 SDDbgValue *NewDV = DAG.getDbgValueList(
15900 Dbg->getVariable(), const_cast<DIExpression *>(NewExpr), Locs,
15901 Dbg->getAdditionalDependencies(), Dbg->isIndirect(), Dbg->getDebugLoc(),
15902 Dbg->getOrder(), Dbg->isVariadic());
15903
15904 Dbg->setIsInvalidated();
15905 Dbg->setIsEmitted();
15906 DAG.AddDbgValue(NewDV, /*isParameter=*/false);
15907 };
15908
15909 // Because we are replacing a load and a s|z ext with a load-s|z ext
15910 // instruction, the dbg_value attached to the load will be of a smaller bit
15911 // width, and we have to add a DW_OP_LLVM_convert expression to get the
15912 // correct size.
15913 auto SalvageToOldLoadSize = [&](SDValue Old, SDValue New, bool IsSigned) {
15915 DAG.GetDbgValues(Old.getNode()).begin(),
15916 DAG.GetDbgValues(Old.getNode()).end());
15917
15918 unsigned VarBitsOld = Old.getValueSizeInBits();
15919 unsigned VarBitsNew = New.getValueSizeInBits();
15920
15921 for (SDDbgValue *Dbg : DbgVals) {
15922 if (Dbg->isInvalidated())
15923 continue;
15924
15925 SalvageDbgValue(Dbg, Old, New, VarBitsOld, VarBitsNew, IsSigned);
15926 }
15927 };
15928
15929 SDValue ExtLoad =
15930 DAG.getExtLoad(ExtLoadType, DL, VT, Load->getChain(), Load->getBasePtr(),
15931 Load->getValueType(0), Load->getMemOperand());
15932 SDValue Res = ExtLoad;
15933 if (Frozen) {
15934 Res = DAG.getFreeze(ExtLoad);
15935 Res = DAG.getNode(ExtLoadType == ISD::SEXTLOAD ? ISD::AssertSext
15937 DL, Res.getValueType(), Res,
15938 DAG.getValueType(Load->getValueType(0).getScalarType()));
15939 }
15940 Combiner.ExtendSetCCUses(SetCCs, N0, Res, ExtOpc);
15941 // If the load value is used only by N, replace it via CombineTo N.
15942 bool NoReplaceTrunc = N0.hasOneUse();
15943 if (N->getHasDebugValue()) {
15944 SDValue OldExtValue(N, 0);
15945 DAG.transferDbgValues(OldExtValue, ExtLoad);
15946 }
15947 if (NoReplaceTrunc) {
15948 bool IsSigned = N->getOpcode() == ISD::SIGN_EXTEND;
15949 if (Load->getHasDebugValue()) {
15950 SDValue OldLoadVal(Load, 0);
15951 SalvageToOldLoadSize(OldLoadVal, ExtLoad, IsSigned);
15952 }
15953 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), ExtLoad.getValue(1));
15954 Combiner.CombineTo(N, Res);
15955 Combiner.recursivelyDeleteUnusedNodes(N0.getNode());
15956 } else {
15957 Combiner.CombineTo(N, Res);
15958 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, Load->getValueType(0), Res);
15959 if (Frozen) {
15960 Combiner.CombineTo(Freeze.getNode(), Trunc);
15961 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), ExtLoad.getValue(1));
15962 } else {
15963 Combiner.CombineTo(Load, Trunc, ExtLoad.getValue(1));
15964 }
15965 }
15966 return SDValue(N, 0); // Return N so it doesn't get rechecked!
15967}
15968
15969static SDValue
15971 bool LegalOperations, SDNode *N, SDValue N0,
15972 ISD::LoadExtType ExtLoadType, ISD::NodeType ExtOpc) {
15973 if (!N0.hasOneUse())
15974 return SDValue();
15975
15977 if (!Ld || Ld->getExtensionType() != ISD::NON_EXTLOAD)
15978 return SDValue();
15979
15980 if ((LegalOperations || !cast<MaskedLoadSDNode>(N0)->isSimple()) &&
15981 !TLI.isLoadLegalOrCustom(VT, Ld->getValueType(0), Ld->getAlign(),
15982 Ld->getAddressSpace(), ExtLoadType, false))
15983 return SDValue();
15984
15985 if (!TLI.isVectorLoadExtDesirable(SDValue(N, 0)))
15986 return SDValue();
15987
15988 SDLoc dl(Ld);
15989 SDValue PassThru = DAG.getNode(ExtOpc, dl, VT, Ld->getPassThru());
15990 SDValue NewLoad = DAG.getMaskedLoad(
15991 VT, dl, Ld->getChain(), Ld->getBasePtr(), Ld->getOffset(), Ld->getMask(),
15992 PassThru, Ld->getMemoryVT(), Ld->getMemOperand(), Ld->getAddressingMode(),
15993 ExtLoadType, Ld->isExpandingLoad());
15994 DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1), SDValue(NewLoad.getNode(), 1));
15995 return NewLoad;
15996}
15997
15998// fold ([s|z]ext (atomic_load)) -> ([s|z]ext (truncate ([s|z]ext atomic_load)))
16000 const TargetLowering &TLI, EVT VT,
16001 SDValue N0,
16002 ISD::LoadExtType ExtLoadType) {
16003 auto *ALoad = dyn_cast<AtomicSDNode>(N0);
16004 if (!ALoad || ALoad->getOpcode() != ISD::ATOMIC_LOAD)
16005 return {};
16006 EVT MemoryVT = ALoad->getMemoryVT();
16007 if (!TLI.isLoadLegal(VT, MemoryVT, ALoad->getAlign(),
16008 ALoad->getAddressSpace(), ExtLoadType, true))
16009 return {};
16010 // Can't fold into ALoad if it is already extending differently.
16011 ISD::LoadExtType ALoadExtTy = ALoad->getExtensionType();
16012 if ((ALoadExtTy == ISD::ZEXTLOAD && ExtLoadType == ISD::SEXTLOAD) ||
16013 (ALoadExtTy == ISD::SEXTLOAD && ExtLoadType == ISD::ZEXTLOAD))
16014 return {};
16015
16016 EVT OrigVT = ALoad->getValueType(0);
16017 assert(OrigVT.getSizeInBits() < VT.getSizeInBits() && "VT should be wider.");
16018 auto *NewALoad = cast<AtomicSDNode>(DAG.getAtomicLoad(
16019 ExtLoadType, SDLoc(ALoad), MemoryVT, VT, ALoad->getChain(),
16020 ALoad->getBasePtr(), ALoad->getMemOperand()));
16022 SDValue(ALoad, 0),
16023 DAG.getNode(ISD::TRUNCATE, SDLoc(ALoad), OrigVT, SDValue(NewALoad, 0)));
16024 // Update the chain uses.
16025 DAG.ReplaceAllUsesOfValueWith(SDValue(ALoad, 1), SDValue(NewALoad, 1));
16026 return SDValue(NewALoad, 0);
16027}
16028
16030 bool LegalOperations) {
16031 assert((N->getOpcode() == ISD::SIGN_EXTEND ||
16032 N->getOpcode() == ISD::ZERO_EXTEND) && "Expected sext or zext");
16033
16034 SDValue SetCC = N->getOperand(0);
16035 if (LegalOperations || SetCC.getOpcode() != ISD::SETCC ||
16036 !SetCC.hasOneUse() || SetCC.getValueType() != MVT::i1)
16037 return SDValue();
16038
16039 SDValue X = SetCC.getOperand(0);
16040 SDValue Ones = SetCC.getOperand(1);
16042 EVT VT = N->getValueType(0);
16043 EVT XVT = X.getValueType();
16044 // setge X, C is canonicalized to setgt, so we do not need to match that
16045 // pattern. The setlt sibling is folded in SimplifySelectCC() because it does
16046 // not require the 'not' op.
16047 if (CC == ISD::SETGT && isAllOnesConstant(Ones) && VT == XVT) {
16048 // Invert and smear/shift the sign bit:
16049 // sext i1 (setgt iN X, -1) --> sra (not X), (N - 1)
16050 // zext i1 (setgt iN X, -1) --> srl (not X), (N - 1)
16051 SDLoc DL(N);
16052 unsigned ShCt = VT.getSizeInBits() - 1;
16054 if (!TLI.shouldAvoidTransformToShift(VT, ShCt)) {
16055 SDValue NotX = DAG.getNOT(DL, X, VT);
16056 SDValue ShiftAmount = DAG.getConstant(ShCt, DL, VT);
16057 auto ShiftOpcode =
16058 N->getOpcode() == ISD::SIGN_EXTEND ? ISD::SRA : ISD::SRL;
16059 return DAG.getNode(ShiftOpcode, DL, VT, NotX, ShiftAmount);
16060 }
16061 }
16062 return SDValue();
16063}
16064
16065SDValue DAGCombiner::foldSextSetcc(SDNode *N) {
16066 SDValue N0 = N->getOperand(0);
16067 if (N0.getOpcode() != ISD::SETCC)
16068 return SDValue();
16069
16070 SDValue N00 = N0.getOperand(0);
16071 SDValue N01 = N0.getOperand(1);
16073 EVT VT = N->getValueType(0);
16074 EVT N00VT = N00.getValueType();
16075 SDLoc DL(N);
16076
16077 // Propagate fast-math-flags.
16078 SDNodeFlags Flags = N0->getFlags();
16079
16080 // On some architectures (such as SSE/NEON/etc) the SETCC result type is
16081 // the same size as the compared operands. Try to optimize sext(setcc())
16082 // if this is the case.
16083 if (VT.isVector() && !LegalOperations &&
16084 TLI.getBooleanContents(N00VT) ==
16086 EVT SVT = getSetCCResultType(N00VT);
16087
16088 // If we already have the desired type, don't change it.
16089 if (SVT != N0.getValueType()) {
16090 // We know that the # elements of the results is the same as the
16091 // # elements of the compare (and the # elements of the compare result
16092 // for that matter). Check to see that they are the same size. If so,
16093 // we know that the element size of the sext'd result matches the
16094 // element size of the compare operands.
16095 if (VT.getSizeInBits() == SVT.getSizeInBits())
16096 return DAG.getSetCC(DL, VT, N00, N01, CC, /*Chain=*/{},
16097 /*Signaling=*/false, Flags);
16098
16099 // If the desired elements are smaller or larger than the source
16100 // elements, we can use a matching integer vector type and then
16101 // truncate/sign extend.
16102 EVT MatchingVecType = N00VT.changeVectorElementTypeToInteger();
16103 if (SVT == MatchingVecType) {
16104 SDValue VsetCC = DAG.getSetCC(DL, MatchingVecType, N00, N01, CC,
16105 /*Chain=*/{}, /*Signaling=*/false, Flags);
16106 return DAG.getSExtOrTrunc(VsetCC, DL, VT);
16107 }
16108 }
16109
16110 // Try to eliminate the sext of a setcc by zexting the compare operands.
16113 bool IsSignedCmp = ISD::isSignedIntSetCC(CC);
16114 unsigned LoadOpcode = IsSignedCmp ? ISD::SEXTLOAD : ISD::ZEXTLOAD;
16115 unsigned ExtOpcode = IsSignedCmp ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
16116
16117 // We have an unsupported narrow vector compare op that would be legal
16118 // if extended to the destination type. See if the compare operands
16119 // can be freely extended to the destination type.
16120 auto IsFreeToExtend = [&](SDValue V) {
16121 if (isConstantOrConstantVector(V, /*NoOpaques*/ true))
16122 return true;
16123 // Match a simple, non-extended load that can be converted to a
16124 // legal {z/s}ext-load.
16125 // TODO: Allow widening of an existing {z/s}ext-load?
16126 if (!(ISD::isNON_EXTLoad(V.getNode()) &&
16127 ISD::isUNINDEXEDLoad(V.getNode())))
16128 return false;
16129
16130 LoadSDNode *Ld = cast<LoadSDNode>(V.getNode());
16131
16132 if (!Ld->isSimple() ||
16133 !TLI.isLoadLegal(VT, V.getValueType(), Ld->getAlign(),
16134 Ld->getAddressSpace(), LoadOpcode, false))
16135 return false;
16136
16137 // Non-chain users of this value must either be the setcc in this
16138 // sequence or extends that can be folded into the new {z/s}ext-load.
16139 for (SDUse &Use : V->uses()) {
16140 // Skip uses of the chain and the setcc.
16141 SDNode *User = Use.getUser();
16142 if (Use.getResNo() != 0 || User == N0.getNode())
16143 continue;
16144 // Extra users must have exactly the same cast we are about to create.
16145 // TODO: This restriction could be eased if ExtendUsesToFormExtLoad()
16146 // is enhanced similarly.
16147 if (User->getOpcode() != ExtOpcode || User->getValueType(0) != VT)
16148 return false;
16149 }
16150 return true;
16151 };
16152
16153 if (IsFreeToExtend(N00) && IsFreeToExtend(N01)) {
16154 SDValue Ext0 = DAG.getNode(ExtOpcode, DL, VT, N00);
16155 SDValue Ext1 = DAG.getNode(ExtOpcode, DL, VT, N01);
16156 return DAG.getSetCC(DL, VT, Ext0, Ext1, CC, /*Chain=*/{},
16157 /*Signaling=*/false, Flags);
16158 }
16159 }
16160 }
16161
16162 // sext(setcc x, y, cc) -> (select (setcc x, y, cc), T, 0)
16163 // Here, T can be 1 or -1, depending on the type of the setcc and
16164 // getBooleanContents().
16165 unsigned SetCCWidth = N0.getScalarValueSizeInBits();
16166
16167 // To determine the "true" side of the select, we need to know the high bit
16168 // of the value returned by the setcc if it evaluates to true.
16169 // If the type of the setcc is i1, then the true case of the select is just
16170 // sext(i1 1), that is, -1.
16171 // If the type of the setcc is larger (say, i8) then the value of the high
16172 // bit depends on getBooleanContents(), so ask TLI for a real "true" value
16173 // of the appropriate width.
16174 SDValue ExtTrueVal = (SetCCWidth == 1)
16175 ? DAG.getAllOnesConstant(DL, VT)
16176 : DAG.getBoolConstant(true, DL, VT, N00VT);
16177 SDValue Zero = DAG.getConstant(0, DL, VT);
16178 if (SDValue SCC = SimplifySelectCC(DL, N00, N01, ExtTrueVal, Zero, CC, true))
16179 return SCC;
16180
16181 if (!VT.isVector() && !shouldConvertSelectOfConstantsToMath(N0, VT, TLI)) {
16182 EVT SetCCVT = getSetCCResultType(N00VT);
16183 // Don't do this transform for i1 because there's a select transform
16184 // that would reverse it.
16185 // TODO: We should not do this transform at all without a target hook
16186 // because a sext is likely cheaper than a select?
16187 if (SetCCVT.getScalarSizeInBits() != 1 &&
16188 (!LegalOperations || TLI.isOperationLegal(ISD::SETCC, N00VT))) {
16189 SDValue SetCC = DAG.getSetCC(DL, SetCCVT, N00, N01, CC, /*Chain=*/{},
16190 /*Signaling=*/false, Flags);
16191 return DAG.getSelect(DL, VT, SetCC, ExtTrueVal, Zero, Flags);
16192 }
16193 }
16194
16195 return SDValue();
16196}
16197
16198SDValue DAGCombiner::visitSIGN_EXTEND(SDNode *N) {
16199 SDValue N0 = N->getOperand(0);
16200 EVT VT = N->getValueType(0);
16201 SDLoc DL(N);
16202
16203 if (VT.isVector())
16204 if (SDValue FoldedVOp = SimplifyVCastOp(N, DL))
16205 return FoldedVOp;
16206
16207 // sext(undef) = 0 because the top bit will all be the same.
16208 if (N0.isUndef())
16209 return DAG.getConstant(0, DL, VT);
16210
16211 if (SDValue Res = tryToFoldExtendOfConstant(N, DL, TLI, DAG, LegalTypes))
16212 return Res;
16213
16214 // fold (sext (sext x)) -> (sext x)
16215 // fold (sext (aext x)) -> (sext x)
16216 if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND)
16217 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, N0.getOperand(0));
16218
16219 // fold (sext (aext_extend_vector_inreg x)) -> (sext_extend_vector_inreg x)
16220 // fold (sext (sext_extend_vector_inreg x)) -> (sext_extend_vector_inreg x)
16223 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, SDLoc(N), VT,
16224 N0.getOperand(0));
16225
16226 if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG) {
16227 SDValue N00 = N0.getOperand(0);
16228 EVT ExtVT = cast<VTSDNode>(N0->getOperand(1))->getVT();
16229 if (N00.getOpcode() == ISD::TRUNCATE || TLI.isTruncateFree(N00, ExtVT)) {
16230 // fold (sext (sext_inreg x)) -> (sext (trunc x))
16231 if ((!LegalTypes || TLI.isTypeLegal(ExtVT))) {
16232 SDValue T = DAG.getNode(ISD::TRUNCATE, DL, ExtVT, N00);
16233 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, T);
16234 }
16235
16236 // If the trunc wasn't legal, try to fold to (sext_inreg (anyext x))
16237 if (!LegalTypes || TLI.isTypeLegal(VT)) {
16238 SDValue ExtSrc = DAG.getAnyExtOrTrunc(N00, DL, VT);
16239 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, ExtSrc,
16240 N0->getOperand(1));
16241 }
16242 }
16243 }
16244
16245 if (N0.getOpcode() == ISD::TRUNCATE) {
16246 // fold (sext (truncate (load x))) -> (sext (smaller load x))
16247 // fold (sext (truncate (srl (load x), c))) -> (sext (smaller load (x+c/n)))
16248 if (SDValue NarrowLoad = reduceLoadWidth(N0.getNode())) {
16249 SDNode *oye = N0.getOperand(0).getNode();
16250 if (NarrowLoad.getNode() != N0.getNode()) {
16251 CombineTo(N0.getNode(), NarrowLoad);
16252 // CombineTo deleted the truncate, if needed, but not what's under it.
16253 AddToWorklist(oye);
16254 }
16255 return SDValue(N, 0); // Return N so it doesn't get rechecked!
16256 }
16257
16258 // See if the value being truncated is already sign extended. If so, just
16259 // eliminate the trunc/sext pair.
16260 SDValue Op = N0.getOperand(0);
16261 unsigned OpBits = Op.getScalarValueSizeInBits();
16262 unsigned MidBits = N0.getScalarValueSizeInBits();
16263 unsigned DestBits = VT.getScalarSizeInBits();
16264
16265 if (N0->getFlags().hasNoSignedWrap() ||
16266 DAG.ComputeNumSignBits(Op) > OpBits - MidBits) {
16267 if (OpBits == DestBits) {
16268 // Op is i32, Mid is i8, and Dest is i32. If Op has more than 24 sign
16269 // bits, it is already ready.
16270 return Op;
16271 }
16272
16273 if (OpBits < DestBits) {
16274 // Op is i32, Mid is i8, and Dest is i64. If Op has more than 24 sign
16275 // bits, just sext from i32.
16276 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op);
16277 }
16278
16279 // Op is i64, Mid is i8, and Dest is i32. If Op has more than 56 sign
16280 // bits, just truncate to i32.
16281 SDNodeFlags Flags;
16282 Flags.setNoSignedWrap(true);
16283 Flags.setNoUnsignedWrap(N0->getFlags().hasNoUnsignedWrap());
16284 return DAG.getNode(ISD::TRUNCATE, DL, VT, Op, Flags);
16285 }
16286
16287 // fold (sext (truncate x)) -> (sextinreg x).
16288 if (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG,
16289 N0.getValueType())) {
16290 if (OpBits < DestBits)
16291 Op = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N0), VT, Op);
16292 else if (OpBits > DestBits)
16293 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N0), VT, Op);
16294 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Op,
16295 DAG.getValueType(N0.getValueType()));
16296 }
16297 }
16298
16299 // Try to simplify (sext (load x)).
16300 if (SDValue foldedExt =
16301 tryToFoldExtOfLoad(DAG, *this, TLI, VT, LegalOperations, N, N0,
16303 return foldedExt;
16304
16305 if (SDValue foldedExt =
16306 tryToFoldExtOfMaskedLoad(DAG, TLI, VT, LegalOperations, N, N0,
16308 return foldedExt;
16309
16310 // fold (sext (load x)) to multiple smaller sextloads.
16311 // Only on illegal but splittable vectors.
16312 if (SDValue ExtLoad = CombineExtLoad(N))
16313 return ExtLoad;
16314
16315 // Try to simplify (sext (sextload x)).
16316 if (SDValue foldedExt = tryToFoldExtOfExtload(
16317 DAG, *this, TLI, VT, LegalOperations, N, N0, ISD::SEXTLOAD))
16318 return foldedExt;
16319
16320 // Try to simplify (sext (atomic_load x)).
16321 if (SDValue foldedExt =
16323 return foldedExt;
16324
16325 // fold (sext (and/or/xor (load x), cst)) ->
16326 // (and/or/xor (sextload x), (sext cst))
16327 if (ISD::isBitwiseLogicOp(N0.getOpcode()) &&
16328 isa<LoadSDNode>(N0.getOperand(0)) &&
16329 N0.getOperand(1).getOpcode() == ISD::Constant &&
16330 (!LegalOperations && TLI.isOperationLegal(N0.getOpcode(), VT))) {
16331 LoadSDNode *LN00 = cast<LoadSDNode>(N0.getOperand(0));
16332 EVT MemVT = LN00->getMemoryVT();
16333 if (TLI.isLoadLegal(VT, MemVT, LN00->getAlign(), LN00->getAddressSpace(),
16334 ISD::SEXTLOAD, false) &&
16335 LN00->getExtensionType() != ISD::ZEXTLOAD && LN00->isUnindexed()) {
16337 bool DoXform = ExtendUsesToFormExtLoad(VT, N0.getNode(), N0.getOperand(0),
16338 ISD::SIGN_EXTEND, SetCCs, TLI);
16339 if (DoXform) {
16340 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(LN00), VT,
16341 LN00->getChain(), LN00->getBasePtr(),
16342 LN00->getMemoryVT(),
16343 LN00->getMemOperand());
16344 APInt Mask = N0.getConstantOperandAPInt(1).sext(VT.getSizeInBits());
16345 SDValue And = DAG.getNode(N0.getOpcode(), DL, VT,
16346 ExtLoad, DAG.getConstant(Mask, DL, VT));
16347 ExtendSetCCUses(SetCCs, N0.getOperand(0), ExtLoad, ISD::SIGN_EXTEND);
16348 bool NoReplaceTruncAnd = !N0.hasOneUse();
16349 bool NoReplaceTrunc = SDValue(LN00, 0).hasOneUse();
16350 CombineTo(N, And);
16351 // If N0 has multiple uses, change other uses as well.
16352 if (NoReplaceTruncAnd) {
16353 SDValue TruncAnd =
16355 CombineTo(N0.getNode(), TruncAnd);
16356 }
16357 if (NoReplaceTrunc) {
16358 DAG.ReplaceAllUsesOfValueWith(SDValue(LN00, 1), ExtLoad.getValue(1));
16359 } else {
16360 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(LN00),
16361 LN00->getValueType(0), ExtLoad);
16362 CombineTo(LN00, Trunc, ExtLoad.getValue(1));
16363 }
16364 return SDValue(N,0); // Return N so it doesn't get rechecked!
16365 }
16366 }
16367 }
16368
16369 if (SDValue V = foldExtendedSignBitTest(N, DAG, LegalOperations))
16370 return V;
16371
16372 if (SDValue V = foldSextSetcc(N))
16373 return V;
16374
16375 // fold (sext x) -> (zext x) if the sign bit is known zero.
16376 if (!TLI.isSExtCheaperThanZExt(N0.getValueType(), VT) &&
16377 (!LegalOperations || TLI.isOperationLegal(ISD::ZERO_EXTEND, VT)) &&
16378 DAG.SignBitIsZero(N0))
16379 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0, SDNodeFlags::NonNeg);
16380
16381 if (SDValue NewVSel = matchVSelectOpSizesWithSetCC(N))
16382 return NewVSel;
16383
16384 // Eliminate this sign extend by doing a negation in the destination type:
16385 // sext i32 (0 - (zext i8 X to i32)) to i64 --> 0 - (zext i8 X to i64)
16386 if (N0.getOpcode() == ISD::SUB && N0.hasOneUse() &&
16390 SDValue Zext = DAG.getZExtOrTrunc(N0.getOperand(1).getOperand(0), DL, VT);
16391 return DAG.getNegative(Zext, DL, VT);
16392 }
16393 // Eliminate this sign extend by doing a decrement in the destination type:
16394 // sext i32 ((zext i8 X to i32) + (-1)) to i64 --> (zext i8 X to i64) + (-1)
16395 if (N0.getOpcode() == ISD::ADD && N0.hasOneUse() &&
16399 SDValue Zext = DAG.getZExtOrTrunc(N0.getOperand(0).getOperand(0), DL, VT);
16400 return DAG.getNode(ISD::ADD, DL, VT, Zext, DAG.getAllOnesConstant(DL, VT));
16401 }
16402
16403 // fold sext (not i1 X) -> add (zext i1 X), -1
16404 // TODO: This could be extended to handle bool vectors.
16405 if (N0.getValueType() == MVT::i1 && isBitwiseNot(N0) && N0.hasOneUse() &&
16406 (!LegalOperations || (TLI.isOperationLegal(ISD::ZERO_EXTEND, VT) &&
16407 TLI.isOperationLegal(ISD::ADD, VT)))) {
16408 // If we can eliminate the 'not', the sext form should be better
16409 if (SDValue NewXor = visitXOR(N0.getNode())) {
16410 // Returning N0 is a form of in-visit replacement that may have
16411 // invalidated N0.
16412 if (NewXor.getNode() == N0.getNode()) {
16413 // Return SDValue here as the xor should have already been replaced in
16414 // this sext.
16415 return SDValue();
16416 }
16417
16418 // Return a new sext with the new xor.
16419 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, NewXor);
16420 }
16421
16422 SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0));
16423 return DAG.getNode(ISD::ADD, DL, VT, Zext, DAG.getAllOnesConstant(DL, VT));
16424 }
16425
16426 if (SDValue Res = tryToFoldExtendSelectLoad(N, TLI, DAG, DL, Level))
16427 return Res;
16428
16429 return SDValue();
16430}
16431
16432/// Given an extending node with a pop-count operand, if the target does not
16433/// support a pop-count in the narrow source type but does support it in the
16434/// destination type, widen the pop-count to the destination type.
16435static SDValue widenCtPop(SDNode *Extend, SelectionDAG &DAG, const SDLoc &DL) {
16436 assert((Extend->getOpcode() == ISD::ZERO_EXTEND ||
16437 Extend->getOpcode() == ISD::ANY_EXTEND) &&
16438 "Expected extend op");
16439
16440 SDValue CtPop = Extend->getOperand(0);
16441 if (CtPop.getOpcode() != ISD::CTPOP || !CtPop.hasOneUse())
16442 return SDValue();
16443
16444 EVT VT = Extend->getValueType(0);
16446 if (TLI.isOperationLegalOrCustom(ISD::CTPOP, CtPop.getValueType()) ||
16447 !TLI.isOperationLegalOrCustom(ISD::CTPOP, VT))
16448 return SDValue();
16449
16450 // zext (ctpop X) --> ctpop (zext X)
16451 SDValue NewZext = DAG.getZExtOrTrunc(CtPop.getOperand(0), DL, VT);
16452 return DAG.getNode(ISD::CTPOP, DL, VT, NewZext);
16453}
16454
16455// If we have (zext (abs X)) where X is a type that will be promoted by type
16456// legalization, convert to (abs_min_poison (sext X)). But do not extend
16457// past a legal type.
16458static SDValue widenAbs(SDNode *Extend, SelectionDAG &DAG) {
16459 assert(Extend->getOpcode() == ISD::ZERO_EXTEND && "Expected zero extend.");
16460
16461 EVT VT = Extend->getValueType(0);
16462 if (VT.isVector())
16463 return SDValue();
16464
16465 SDValue Abs = Extend->getOperand(0);
16466 if (!ISD::isAbsOpcode(Abs.getOpcode()) || !Abs.hasOneUse())
16467 return SDValue();
16468
16469 EVT AbsVT = Abs.getValueType();
16471 if (TLI.getTypeAction(*DAG.getContext(), AbsVT) !=
16473 return SDValue();
16474
16475 EVT LegalVT = TLI.getTypeToTransformTo(*DAG.getContext(), AbsVT);
16476
16477 SDValue SExt =
16478 DAG.getNode(ISD::SIGN_EXTEND, SDLoc(Abs), LegalVT, Abs.getOperand(0));
16479 SDValue NewAbs = DAG.getNode(ISD::ABS_MIN_POISON, SDLoc(Abs), LegalVT, SExt);
16480 return DAG.getZExtOrTrunc(NewAbs, SDLoc(Extend), VT);
16481}
16482
16483SDValue DAGCombiner::visitZERO_EXTEND(SDNode *N) {
16484 SDValue N0 = N->getOperand(0);
16485 EVT VT = N->getValueType(0);
16486 SDLoc DL(N);
16487
16488 if (VT.isVector())
16489 if (SDValue FoldedVOp = SimplifyVCastOp(N, DL))
16490 return FoldedVOp;
16491
16492 // zext(undef) = 0
16493 if (N0.isUndef())
16494 return DAG.getConstant(0, DL, VT);
16495
16496 if (SDValue Res = tryToFoldExtendOfConstant(N, DL, TLI, DAG, LegalTypes))
16497 return Res;
16498
16499 // fold (zext (zext x)) -> (zext x)
16500 // fold (zext (aext x)) -> (zext x)
16501 if (N0.getOpcode() == ISD::ZERO_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND) {
16502 SDNodeFlags Flags;
16503 if (N0.getOpcode() == ISD::ZERO_EXTEND)
16504 Flags.setNonNeg(N0->getFlags().hasNonNeg());
16505 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0), Flags);
16506 }
16507
16508 // fold (zext (aext_extend_vector_inreg x)) -> (zext_extend_vector_inreg x)
16509 // fold (zext (zext_extend_vector_inreg x)) -> (zext_extend_vector_inreg x)
16512 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, N0.getOperand(0));
16513
16514 // fold (zext (truncate x)) -> (zext x) or
16515 // (zext (truncate x)) -> (truncate x)
16516 // This is valid when the truncated bits of x are already zero.
16517 SDValue Op;
16518 KnownBits Known;
16519 if (isTruncateOf(DAG, N0, Op, Known)) {
16520 APInt TruncatedBits =
16521 (Op.getScalarValueSizeInBits() == N0.getScalarValueSizeInBits()) ?
16522 APInt(Op.getScalarValueSizeInBits(), 0) :
16523 APInt::getBitsSet(Op.getScalarValueSizeInBits(),
16524 N0.getScalarValueSizeInBits(),
16525 std::min(Op.getScalarValueSizeInBits(),
16526 VT.getScalarSizeInBits()));
16527 if (TruncatedBits.isSubsetOf(Known.Zero)) {
16528 SDValue ZExtOrTrunc = DAG.getZExtOrTrunc(Op, DL, VT);
16529 DAG.salvageDebugInfo(*N0.getNode());
16530
16531 return ZExtOrTrunc;
16532 }
16533 }
16534
16535 // fold (zext (truncate x)) -> (and x, mask)
16536 if (N0.getOpcode() == ISD::TRUNCATE) {
16537 // fold (zext (truncate (load x))) -> (zext (smaller load x))
16538 // fold (zext (truncate (srl (load x), c))) -> (zext (smaller load (x+c/n)))
16539 if (SDValue NarrowLoad = reduceLoadWidth(N0.getNode())) {
16540 SDNode *oye = N0.getOperand(0).getNode();
16541 if (NarrowLoad.getNode() != N0.getNode()) {
16542 CombineTo(N0.getNode(), NarrowLoad);
16543 // CombineTo deleted the truncate, if needed, but not what's under it.
16544 AddToWorklist(oye);
16545 }
16546 return SDValue(N, 0); // Return N so it doesn't get rechecked!
16547 }
16548
16549 EVT SrcVT = N0.getOperand(0).getValueType();
16550 EVT MinVT = N0.getValueType();
16551
16552 if (N->getFlags().hasNonNeg()) {
16553 SDValue Op = N0.getOperand(0);
16554 unsigned OpBits = SrcVT.getScalarSizeInBits();
16555 unsigned MidBits = MinVT.getScalarSizeInBits();
16556 unsigned DestBits = VT.getScalarSizeInBits();
16557
16558 if (N0->getFlags().hasNoSignedWrap() ||
16559 DAG.ComputeNumSignBits(Op) > OpBits - MidBits) {
16560 if (OpBits == DestBits) {
16561 // Op is i32, Mid is i8, and Dest is i32. If Op has more than 24 sign
16562 // bits, it is already ready.
16563 return Op;
16564 }
16565
16566 if (OpBits < DestBits) {
16567 // Op is i32, Mid is i8, and Dest is i64. If Op has more than 24 sign
16568 // bits, just sext from i32.
16569 // FIXME: This can probably be ZERO_EXTEND nneg?
16570 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op);
16571 }
16572
16573 // Op is i64, Mid is i8, and Dest is i32. If Op has more than 56 sign
16574 // bits, just truncate to i32.
16575 SDNodeFlags Flags;
16576 Flags.setNoSignedWrap(true);
16577 Flags.setNoUnsignedWrap(true);
16578 return DAG.getNode(ISD::TRUNCATE, DL, VT, Op, Flags);
16579 }
16580 }
16581
16582 // Try to mask before the extension to avoid having to generate a larger mask,
16583 // possibly over several sub-vectors.
16584 if (SrcVT.bitsLT(VT) && VT.isVector()) {
16585 if (!LegalOperations || (TLI.isOperationLegal(ISD::AND, SrcVT) &&
16587 SDValue Op = N0.getOperand(0);
16588 Op = DAG.getZeroExtendInReg(Op, DL, MinVT);
16589 AddToWorklist(Op.getNode());
16590 SDValue ZExtOrTrunc = DAG.getZExtOrTrunc(Op, DL, VT);
16591 // Transfer the debug info; the new node is equivalent to N0.
16592 DAG.transferDbgValues(N0, ZExtOrTrunc);
16593 return ZExtOrTrunc;
16594 }
16595 }
16596
16597 if (!LegalOperations || TLI.isOperationLegal(ISD::AND, VT)) {
16598 SDValue Op = DAG.getAnyExtOrTrunc(N0.getOperand(0), DL, VT);
16599 AddToWorklist(Op.getNode());
16600 SDValue And = DAG.getZeroExtendInReg(Op, DL, MinVT);
16601 // We may safely transfer the debug info describing the truncate node over
16602 // to the equivalent and operation.
16603 DAG.transferDbgValues(N0, And);
16604 return And;
16605 }
16606 }
16607
16608 // Fold (zext (and (trunc x), cst)) -> (and x, cst),
16609 // if either of the casts is not free.
16610 // Also handles (zext (and (bitcast (extract_subvector vNi1, 0)) cst))
16611 // by treating the bitcast+extract as equivalent to a truncate of the
16612 // wider bitcast, e.g. on AVX512DQ where v8i1 extract replaces truncate.
16613 if (N0.getOpcode() == ISD::AND &&
16614 N0.getOperand(1).getOpcode() == ISD::Constant) {
16615 SDValue AndSrc = N0.getOperand(0);
16616 SDValue X;
16617 if (AndSrc.getOpcode() == ISD::TRUNCATE) {
16618 X = AndSrc.getOperand(0);
16619 } else if (AndSrc.getOpcode() == ISD::BITCAST &&
16621 AndSrc.getOperand(0).getConstantOperandVal(1) == 0) {
16622 // (bitcast (extract_subvector vNi1, 0) -> iK) is equivalent to
16623 // (truncate (bitcast vNi1 -> iN) -> iK); use the wider vNi1 as X.
16624 SDValue Src = AndSrc.getOperand(0).getOperand(0);
16625 EVT SrcVT = Src.getValueType();
16626 if (SrcVT.isFixedLengthVectorOf(MVT::i1)) {
16627 EVT WideIntVT =
16629 if (TLI.isTypeLegal(WideIntVT))
16630 X = DAG.getBitcast(WideIntVT, Src);
16631 }
16632 }
16633 if (X && (!TLI.isTruncateFree(X, N0.getValueType()) ||
16634 !TLI.isZExtFree(N0.getValueType(), VT))) {
16635 X = DAG.getAnyExtOrTrunc(X, SDLoc(X), VT);
16636 APInt Mask = N0.getConstantOperandAPInt(1).zext(VT.getSizeInBits());
16637 return DAG.getNode(ISD::AND, DL, VT, X, DAG.getConstant(Mask, DL, VT));
16638 }
16639 }
16640
16641 // Try to simplify (zext (load x)).
16642 if (SDValue foldedExt = tryToFoldExtOfLoad(
16643 DAG, *this, TLI, VT, LegalOperations, N, N0, ISD::ZEXTLOAD,
16644 ISD::ZERO_EXTEND, N->getFlags().hasNonNeg()))
16645 return foldedExt;
16646
16647 if (SDValue foldedExt =
16648 tryToFoldExtOfMaskedLoad(DAG, TLI, VT, LegalOperations, N, N0,
16650 return foldedExt;
16651
16652 // fold (zext (load x)) to multiple smaller zextloads.
16653 // Only on illegal but splittable vectors.
16654 if (SDValue ExtLoad = CombineExtLoad(N))
16655 return ExtLoad;
16656
16657 // Try to simplify (zext (atomic_load x)).
16658 if (SDValue foldedExt =
16660 return foldedExt;
16661
16662 // fold (zext (and/or/xor (load x), cst)) ->
16663 // (and/or/xor (zextload x), (zext cst))
16664 // Unless (and (load x) cst) will match as a zextload already and has
16665 // additional users, or the zext is already free.
16666 if (ISD::isBitwiseLogicOp(N0.getOpcode()) && !TLI.isZExtFree(N0, VT) &&
16667 isa<LoadSDNode>(N0.getOperand(0)) &&
16668 N0.getOperand(1).getOpcode() == ISD::Constant &&
16669 (!LegalOperations && TLI.isOperationLegal(N0.getOpcode(), VT))) {
16670 LoadSDNode *LN00 = cast<LoadSDNode>(N0.getOperand(0));
16671 EVT MemVT = LN00->getMemoryVT();
16672 if (TLI.isLoadLegal(VT, MemVT, LN00->getAlign(), LN00->getAddressSpace(),
16673 ISD::ZEXTLOAD, false) &&
16674 LN00->getExtensionType() != ISD::SEXTLOAD && LN00->isUnindexed()) {
16675 bool DoXform = true;
16677 if (!N0.hasOneUse()) {
16678 if (N0.getOpcode() == ISD::AND) {
16679 auto *AndC = cast<ConstantSDNode>(N0.getOperand(1));
16680 EVT LoadResultTy = AndC->getValueType(0);
16681 EVT ExtVT;
16682 if (isAndLoadExtLoad(AndC, LN00, LoadResultTy, ExtVT))
16683 DoXform = false;
16684 }
16685 }
16686 if (DoXform)
16687 DoXform = ExtendUsesToFormExtLoad(VT, N0.getNode(), N0.getOperand(0),
16688 ISD::ZERO_EXTEND, SetCCs, TLI);
16689 if (DoXform) {
16690 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(LN00), VT,
16691 LN00->getChain(), LN00->getBasePtr(),
16692 LN00->getMemoryVT(),
16693 LN00->getMemOperand());
16694 APInt Mask = N0.getConstantOperandAPInt(1).zext(VT.getSizeInBits());
16695 SDValue And = DAG.getNode(N0.getOpcode(), DL, VT,
16696 ExtLoad, DAG.getConstant(Mask, DL, VT));
16697 ExtendSetCCUses(SetCCs, N0.getOperand(0), ExtLoad, ISD::ZERO_EXTEND);
16698 bool NoReplaceTruncAnd = !N0.hasOneUse();
16699 bool NoReplaceTrunc = SDValue(LN00, 0).hasOneUse();
16700 CombineTo(N, And);
16701 // If N0 has multiple uses, change other uses as well.
16702 if (NoReplaceTruncAnd) {
16703 SDValue TruncAnd =
16705 CombineTo(N0.getNode(), TruncAnd);
16706 }
16707 if (NoReplaceTrunc) {
16708 DAG.ReplaceAllUsesOfValueWith(SDValue(LN00, 1), ExtLoad.getValue(1));
16709 } else {
16710 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(LN00),
16711 LN00->getValueType(0), ExtLoad);
16712 CombineTo(LN00, Trunc, ExtLoad.getValue(1));
16713 }
16714 return SDValue(N,0); // Return N so it doesn't get rechecked!
16715 }
16716 }
16717 }
16718
16719 // fold (zext (and/or/xor (shl/shr (load x), cst), cst)) ->
16720 // (and/or/xor (shl/shr (zextload x), (zext cst)), (zext cst))
16721 if (SDValue ZExtLoad = CombineZExtLogicopShiftLoad(N))
16722 return ZExtLoad;
16723
16724 // Try to simplify (zext (zextload x)).
16725 if (SDValue foldedExt = tryToFoldExtOfExtload(
16726 DAG, *this, TLI, VT, LegalOperations, N, N0, ISD::ZEXTLOAD))
16727 return foldedExt;
16728
16729 if (SDValue V = foldExtendedSignBitTest(N, DAG, LegalOperations))
16730 return V;
16731
16732 if (N0.getOpcode() == ISD::SETCC) {
16733 // Propagate fast-math-flags.
16734 SelectionDAG::FlagInserter FlagsInserter(DAG, N0->getFlags());
16735
16736 // Only do this before legalize for now.
16737 if (!LegalOperations && VT.isVector() &&
16738 N0.getValueType().getVectorElementType() == MVT::i1) {
16739 EVT N00VT = N0.getOperand(0).getValueType();
16740 if (getSetCCResultType(N00VT) == N0.getValueType())
16741 return SDValue();
16742
16743 // We know that the # elements of the results is the same as the #
16744 // elements of the compare (and the # elements of the compare result for
16745 // that matter). Check to see that they are the same size. If so, we know
16746 // that the element size of the sext'd result matches the element size of
16747 // the compare operands.
16748 if (VT.getSizeInBits() == N00VT.getSizeInBits()) {
16749 // zext(setcc) -> zext_in_reg(vsetcc) for vectors.
16750 SDValue VSetCC = DAG.getNode(ISD::SETCC, DL, VT, N0.getOperand(0),
16751 N0.getOperand(1), N0.getOperand(2));
16752 return DAG.getZeroExtendInReg(VSetCC, DL, N0.getValueType());
16753 }
16754
16755 // If the desired elements are smaller or larger than the source
16756 // elements we can use a matching integer vector type and then
16757 // truncate/any extend followed by zext_in_reg.
16758 EVT MatchingVectorType = N00VT.changeVectorElementTypeToInteger();
16759 SDValue VsetCC =
16760 DAG.getNode(ISD::SETCC, DL, MatchingVectorType, N0.getOperand(0),
16761 N0.getOperand(1), N0.getOperand(2));
16762 return DAG.getZeroExtendInReg(DAG.getAnyExtOrTrunc(VsetCC, DL, VT), DL,
16763 N0.getValueType());
16764 }
16765
16766 // zext(setcc x,y,cc) -> zext(select x, y, true, false, cc)
16767 EVT N0VT = N0.getValueType();
16768 EVT N00VT = N0.getOperand(0).getValueType();
16769 if (SDValue SCC =
16770 SimplifySelectCC(DL, N0.getOperand(0), N0.getOperand(1),
16771 DAG.getBoolConstant(true, DL, N0VT, N00VT),
16772 DAG.getBoolConstant(false, DL, N0VT, N00VT),
16773 N0.getOperandCondCode(2), true))
16774 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, SCC);
16775 }
16776
16777 // (zext (shl (zext x), cst)) -> (shl (zext x), cst)
16778 if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL) &&
16779 !TLI.isZExtFree(N0, VT)) {
16780 SDValue ShVal = N0.getOperand(0);
16781 SDValue ShAmt = N0.getOperand(1);
16782 if (auto *ShAmtC = dyn_cast<ConstantSDNode>(ShAmt)) {
16783 if (ShVal.getOpcode() == ISD::ZERO_EXTEND && N0.hasOneUse()) {
16784 if (N0.getOpcode() == ISD::SHL) {
16785 // If the original shl may be shifting out bits, do not perform this
16786 // transformation.
16787 unsigned KnownZeroBits = ShVal.getValueSizeInBits() -
16788 ShVal.getOperand(0).getValueSizeInBits();
16789 if (ShAmtC->getAPIntValue().ugt(KnownZeroBits)) {
16790 // If the shift is too large, then see if we can deduce that the
16791 // shift is safe anyway.
16792
16793 // Check if the bits being shifted out are known to be zero.
16794 KnownBits KnownShVal = DAG.computeKnownBits(ShVal);
16795 if (ShAmtC->getAPIntValue().ugt(KnownShVal.countMinLeadingZeros()))
16796 return SDValue();
16797 }
16798 }
16799
16800 // Ensure that the shift amount is wide enough for the shifted value.
16801 if (Log2_32_Ceil(VT.getSizeInBits()) > ShAmt.getValueSizeInBits())
16802 ShAmt = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, ShAmt);
16803
16804 return DAG.getNode(N0.getOpcode(), DL, VT,
16805 DAG.getNode(ISD::ZERO_EXTEND, DL, VT, ShVal), ShAmt);
16806 }
16807 }
16808 }
16809
16810 if (SDValue NewVSel = matchVSelectOpSizesWithSetCC(N))
16811 return NewVSel;
16812
16813 if (SDValue NewCtPop = widenCtPop(N, DAG, DL))
16814 return NewCtPop;
16815
16816 if (SDValue V = widenAbs(N, DAG))
16817 return V;
16818
16819 if (SDValue Res = tryToFoldExtendSelectLoad(N, TLI, DAG, DL, Level))
16820 return Res;
16821
16822 // CSE zext nneg with sext if the zext is not free.
16823 if (N->getFlags().hasNonNeg() && !TLI.isZExtFree(N0.getValueType(), VT)) {
16824 SDNode *CSENode = DAG.getNodeIfExists(ISD::SIGN_EXTEND, N->getVTList(), N0);
16825 if (CSENode)
16826 return SDValue(CSENode, 0);
16827 }
16828
16829 return SDValue();
16830}
16831
16832/// For a unary operation, propagate poison or undef from the operand \p N0 to
16833/// the result type \p VT. Returns an empty SDValue if \p N0 is neither.
16835 if (N0.getOpcode() == ISD::POISON)
16836 return DAG.getPOISON(VT);
16837 if (N0.getOpcode() == ISD::UNDEF)
16838 return DAG.getUNDEF(VT);
16839 return SDValue();
16840}
16841
16842SDValue DAGCombiner::visitANY_EXTEND(SDNode *N) {
16843 SDValue N0 = N->getOperand(0);
16844 EVT VT = N->getValueType(0);
16845 SDLoc DL(N);
16846
16847 // aext(undef) = undef, aext(poison) = poison
16848 if (SDValue R = propagateUnaryUndef(DAG, N0, VT))
16849 return R;
16850
16851 if (SDValue Res = tryToFoldExtendOfConstant(N, DL, TLI, DAG, LegalTypes))
16852 return Res;
16853
16854 // fold (aext (aext x)) -> (aext x)
16855 // fold (aext (zext x)) -> (zext x)
16856 // fold (aext (sext x)) -> (sext x)
16857 if (N0.getOpcode() == ISD::ANY_EXTEND || N0.getOpcode() == ISD::ZERO_EXTEND ||
16858 N0.getOpcode() == ISD::SIGN_EXTEND) {
16859 SDNodeFlags Flags;
16860 if (N0.getOpcode() == ISD::ZERO_EXTEND)
16861 Flags.setNonNeg(N0->getFlags().hasNonNeg());
16862 return DAG.getNode(N0.getOpcode(), DL, VT, N0.getOperand(0), Flags);
16863 }
16864
16865 // fold (aext (aext_extend_vector_inreg x)) -> (aext_extend_vector_inreg x)
16866 // fold (aext (zext_extend_vector_inreg x)) -> (zext_extend_vector_inreg x)
16867 // fold (aext (sext_extend_vector_inreg x)) -> (sext_extend_vector_inreg x)
16871 return DAG.getNode(N0.getOpcode(), DL, VT, N0.getOperand(0));
16872
16873 // fold (aext (truncate (load x))) -> (aext (smaller load x))
16874 // fold (aext (truncate (srl (load x), c))) -> (aext (small load (x+c/n)))
16875 if (N0.getOpcode() == ISD::TRUNCATE) {
16876 if (SDValue NarrowLoad = reduceLoadWidth(N0.getNode())) {
16877 SDNode *oye = N0.getOperand(0).getNode();
16878 if (NarrowLoad.getNode() != N0.getNode()) {
16879 CombineTo(N0.getNode(), NarrowLoad);
16880 // CombineTo deleted the truncate, if needed, but not what's under it.
16881 AddToWorklist(oye);
16882 }
16883 return SDValue(N, 0); // Return N so it doesn't get rechecked!
16884 }
16885 }
16886
16887 // fold (aext (truncate x))
16888 if (N0.getOpcode() == ISD::TRUNCATE)
16889 return DAG.getAnyExtOrTrunc(N0.getOperand(0), DL, VT);
16890
16891 // Fold (aext (and (trunc x), cst)) -> (and x, cst)
16892 // if either of the casts is not free, and sign-extending the narrow type is
16893 // not cheaper than zero-extending it (which would indicate the target prefers
16894 // to keep operations at the narrower width).
16895 // Also handles (aext (and (bitcast (extract_subvector vNi1, 0)) cst))
16896 // which arises on AVX512DQ where v8i1 extract replaces truncate.
16897 if (N0.getOpcode() == ISD::AND &&
16898 N0.getOperand(1).getOpcode() == ISD::Constant) {
16899 SDValue AndSrc = N0.getOperand(0);
16900 SDValue X;
16901 if (AndSrc.getOpcode() == ISD::TRUNCATE) {
16902 X = AndSrc.getOperand(0);
16903 } else if (AndSrc.getOpcode() == ISD::BITCAST &&
16905 AndSrc.getOperand(0).getConstantOperandVal(1) == 0) {
16906 SDValue Src = AndSrc.getOperand(0).getOperand(0);
16907 EVT SrcVT = Src.getValueType();
16908 if (SrcVT.isFixedLengthVectorOf(MVT::i1)) {
16909 EVT WideIntVT =
16911 if (TLI.isTypeLegal(WideIntVT))
16912 X = DAG.getBitcast(WideIntVT, Src);
16913 }
16914 }
16915 if (X && (!TLI.isTruncateFree(X, N0.getValueType()) ||
16916 (!TLI.isZExtFree(N0.getValueType(), VT) &&
16917 !TLI.isSExtCheaperThanZExt(N0.getValueType(), VT)))) {
16918 X = DAG.getAnyExtOrTrunc(X, DL, VT);
16919 APInt Mask = N0.getConstantOperandAPInt(1).zext(VT.getSizeInBits());
16920 return DAG.getNode(ISD::AND, DL, VT, X, DAG.getConstant(Mask, DL, VT));
16921 }
16922 }
16923
16924 // fold (aext (load x)) -> (aext (truncate (extload x)))
16925 // fold (aext (freeze (load x))) -> (aext (truncate (freeze (extload x))))
16926 // None of the supported targets knows how to perform load and any_ext
16927 // on vectors in one instruction, so attempt to fold to zext instead.
16928 if (VT.isVector()) {
16929 // Try to simplify (zext (load x)).
16930 if (SDValue foldedExt =
16931 tryToFoldExtOfLoad(DAG, *this, TLI, VT, LegalOperations, N, N0,
16933 return foldedExt;
16934 } else {
16935 // TODO: Support multiple uses of the load when frozen.
16936 bool Frozen = N0.getOpcode() == ISD::FREEZE && N0.getOperand(0).hasOneUse();
16937 SDValue LoadOp = Frozen ? N0.getOperand(0) : N0;
16938 EVT LoadVT = LoadOp.getValueType();
16939 if (ISD::isNON_EXTLoad(LoadOp.getNode()) &&
16940 ISD::isUNINDEXEDLoad(LoadOp.getNode())) {
16941 LoadSDNode *LN0 = cast<LoadSDNode>(LoadOp);
16942 if (TLI.isLoadLegalOrCustom(VT, LoadVT, LN0->getAlign(),
16944 false)) {
16945 bool DoXform = true;
16947 // N0 is a Load and has multiple uses.
16948 if (!Frozen && !N0.hasOneUse())
16949 DoXform =
16950 ExtendUsesToFormExtLoad(VT, N, N0, ISD::ANY_EXTEND, SetCCs, TLI);
16951 if (DoXform) {
16952 SDValue ExtLoad =
16953 DAG.getExtLoad(ISD::EXTLOAD, DL, VT, LN0->getChain(),
16954 LN0->getBasePtr(), LoadVT, LN0->getMemOperand());
16955 SDValue Res = Frozen ? DAG.getFreeze(ExtLoad) : ExtLoad;
16956 ExtendSetCCUses(SetCCs, N0, Res, ISD::ANY_EXTEND);
16957 // If the load or freeze value is used only by N, replace it via
16958 // CombineTo N.
16959 bool NoReplaceTrunc = N0.hasOneUse();
16960 CombineTo(N, Res);
16961 if (NoReplaceTrunc) {
16962 DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1));
16963 recursivelyDeleteUnusedNodes(N0.getNode());
16964 } else {
16965 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0), LoadVT, Res);
16966 CombineTo(LN0, Trunc, ExtLoad.getValue(1));
16967 }
16968 return SDValue(N, 0); // Return N so it doesn't get rechecked!
16969 }
16970 }
16971 }
16972 }
16973
16974 // fold (aext (zextload x)) -> (aext (truncate (zextload x)))
16975 // fold (aext (sextload x)) -> (aext (truncate (sextload x)))
16976 // fold (aext ( extload x)) -> (aext (truncate (extload x)))
16977 if (N0.getOpcode() == ISD::LOAD && !ISD::isNON_EXTLoad(N0.getNode()) &&
16978 ISD::isUNINDEXEDLoad(N0.getNode()) && N0.hasOneUse()) {
16979 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
16980 ISD::LoadExtType ExtType = LN0->getExtensionType();
16981 EVT MemVT = LN0->getMemoryVT();
16982 if (!LegalOperations ||
16983 TLI.isLoadLegal(VT, MemVT, LN0->getAlign(), LN0->getAddressSpace(),
16984 ExtType, false)) {
16985 SDValue ExtLoad =
16986 DAG.getExtLoad(ExtType, DL, VT, LN0->getChain(), LN0->getBasePtr(),
16987 MemVT, LN0->getMemOperand());
16988 CombineTo(N, ExtLoad);
16989 DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1));
16990 recursivelyDeleteUnusedNodes(LN0);
16991 return SDValue(N, 0); // Return N so it doesn't get rechecked!
16992 }
16993 }
16994
16995 if (N0.getOpcode() == ISD::SETCC) {
16996 // Propagate fast-math-flags.
16997 SDNodeFlags Flags = N0->getFlags();
16998 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
16999
17000 // For vectors:
17001 // aext(setcc) -> vsetcc
17002 // aext(setcc) -> truncate(vsetcc)
17003 // aext(setcc) -> aext(vsetcc)
17004 // Only do this before legalize for now.
17005 if (VT.isVector() && !LegalOperations) {
17006 EVT N00VT = N0.getOperand(0).getValueType();
17007 if (getSetCCResultType(N00VT) == N0.getValueType())
17008 return SDValue();
17009
17010 // We know that the # elements of the results is the same as the
17011 // # elements of the compare (and the # elements of the compare result
17012 // for that matter). Check to see that they are the same size. If so,
17013 // we know that the element size of the sext'd result matches the
17014 // element size of the compare operands.
17015 if (VT.getSizeInBits() == N00VT.getSizeInBits())
17016 return DAG.getSetCC(DL, VT, N0.getOperand(0), N0.getOperand(1),
17017 N0.getOperandCondCode(2),
17018 /*Chain=*/{}, /*Signaling=*/false, Flags);
17019
17020 // If the desired elements are smaller or larger than the source
17021 // elements we can use a matching integer vector type and then
17022 // truncate/any extend
17023 EVT MatchingVectorType = N00VT.changeVectorElementTypeToInteger();
17024 SDValue VsetCC =
17025 DAG.getSetCC(DL, MatchingVectorType, N0.getOperand(0),
17026 N0.getOperand(1), N0.getOperandCondCode(2), /*Chain=*/{},
17027 /*Signaling=*/false, Flags);
17028 return DAG.getAnyExtOrTrunc(VsetCC, DL, VT);
17029 }
17030
17031 // aext(setcc x,y,cc) -> select_cc x, y, 1, 0, cc
17032 if (SDValue SCC = SimplifySelectCC(
17033 DL, N0.getOperand(0), N0.getOperand(1), DAG.getConstant(1, DL, VT),
17034 DAG.getConstant(0, DL, VT), N0.getOperandCondCode(2), true))
17035 return SCC;
17036 }
17037
17038 if (SDValue NewCtPop = widenCtPop(N, DAG, DL))
17039 return NewCtPop;
17040
17041 if (SDValue Res = tryToFoldExtendSelectLoad(N, TLI, DAG, DL, Level))
17042 return Res;
17043
17044 return SDValue();
17045}
17046
17047SDValue DAGCombiner::visitAssertExt(SDNode *N) {
17048 unsigned Opcode = N->getOpcode();
17049 SDValue N0 = N->getOperand(0);
17050 SDValue N1 = N->getOperand(1);
17051 EVT AssertVT = cast<VTSDNode>(N1)->getVT();
17052
17053 // fold (assert?ext (assert?ext x, vt), vt) -> (assert?ext x, vt)
17054 if (N0.getOpcode() == Opcode &&
17055 AssertVT == cast<VTSDNode>(N0.getOperand(1))->getVT())
17056 return N0;
17057
17058 // fold (assert?ext c, vt) -> c
17059 if (isa<ConstantSDNode>(N0))
17060 return N0;
17061
17062 if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() &&
17063 N0.getOperand(0).getOpcode() == Opcode) {
17064 // We have an assert, truncate, assert sandwich. Make one stronger assert
17065 // by asserting on the smallest asserted type to the larger source type.
17066 // This eliminates the later assert:
17067 // assert (trunc (assert X, i8) to iN), i1 --> trunc (assert X, i1) to iN
17068 // assert (trunc (assert X, i1) to iN), i8 --> trunc (assert X, i1) to iN
17069 SDLoc DL(N);
17070 SDValue BigA = N0.getOperand(0);
17071 EVT BigA_AssertVT = cast<VTSDNode>(BigA.getOperand(1))->getVT();
17072 EVT MinAssertVT = AssertVT.bitsLT(BigA_AssertVT) ? AssertVT : BigA_AssertVT;
17073 SDValue MinAssertVTVal = DAG.getValueType(MinAssertVT);
17074 SDValue NewAssert = DAG.getNode(Opcode, DL, BigA.getValueType(),
17075 BigA.getOperand(0), MinAssertVTVal);
17076 return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewAssert);
17077 }
17078
17079 // If we have (AssertZext (truncate (AssertSext X, iX)), iY) and Y is smaller
17080 // than X. Just move the AssertZext in front of the truncate and drop the
17081 // AssertSExt.
17082 if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() &&
17084 Opcode == ISD::AssertZext) {
17085 SDValue BigA = N0.getOperand(0);
17086 EVT BigA_AssertVT = cast<VTSDNode>(BigA.getOperand(1))->getVT();
17087 if (AssertVT.bitsLT(BigA_AssertVT)) {
17088 SDLoc DL(N);
17089 SDValue NewAssert = DAG.getNode(Opcode, DL, BigA.getValueType(),
17090 BigA.getOperand(0), N1);
17091 return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewAssert);
17092 }
17093 }
17094
17095 if (Opcode == ISD::AssertZext && N0.getOpcode() == ISD::AND &&
17097 const APInt &Mask = N0.getConstantOperandAPInt(1);
17098
17099 // If we have (AssertZext (and (AssertSext X, iX), M), iY) and Y is smaller
17100 // than X, and the And doesn't change the lower iX bits, we can move the
17101 // AssertZext in front of the And and drop the AssertSext.
17102 if (N0.getOperand(0).getOpcode() == ISD::AssertSext && N0.hasOneUse()) {
17103 SDValue BigA = N0.getOperand(0);
17104 EVT BigA_AssertVT = cast<VTSDNode>(BigA.getOperand(1))->getVT();
17105 if (AssertVT.bitsLT(BigA_AssertVT) &&
17106 Mask.countr_one() >= BigA_AssertVT.getScalarSizeInBits()) {
17107 SDLoc DL(N);
17108 SDValue NewAssert =
17109 DAG.getNode(Opcode, DL, N->getValueType(0), BigA.getOperand(0), N1);
17110 return DAG.getNode(ISD::AND, DL, N->getValueType(0), NewAssert,
17111 N0.getOperand(1));
17112 }
17113 }
17114
17115 // Remove AssertZext entirely if the mask guarantees the assertion cannot
17116 // fail.
17117 // TODO: Use KB countMinLeadingZeros to handle non-constant masks?
17118 if (Mask.isIntN(AssertVT.getScalarSizeInBits()))
17119 return N0;
17120 }
17121
17122 return SDValue();
17123}
17124
17125SDValue DAGCombiner::visitAssertAlign(SDNode *N) {
17126 SDLoc DL(N);
17127
17128 Align AL = cast<AssertAlignSDNode>(N)->getAlign();
17129 SDValue N0 = N->getOperand(0);
17130
17131 // Fold (assertalign (assertalign x, AL0), AL1) ->
17132 // (assertalign x, max(AL0, AL1))
17133 if (auto *AAN = dyn_cast<AssertAlignSDNode>(N0))
17134 return DAG.getAssertAlign(DL, N0.getOperand(0),
17135 std::max(AL, AAN->getAlign()));
17136
17137 // In rare cases, there are trivial arithmetic ops in source operands. Sink
17138 // this assert down to source operands so that those arithmetic ops could be
17139 // exposed to the DAG combining.
17140 switch (N0.getOpcode()) {
17141 default:
17142 break;
17143 case ISD::ADD:
17144 case ISD::PTRADD:
17145 case ISD::SUB: {
17146 unsigned AlignShift = Log2(AL);
17147 SDValue LHS = N0.getOperand(0);
17148 SDValue RHS = N0.getOperand(1);
17149 unsigned LHSAlignShift = DAG.computeKnownBits(LHS).countMinTrailingZeros();
17150 unsigned RHSAlignShift = DAG.computeKnownBits(RHS).countMinTrailingZeros();
17151 if (LHSAlignShift >= AlignShift || RHSAlignShift >= AlignShift) {
17152 if (LHSAlignShift < AlignShift)
17153 LHS = DAG.getAssertAlign(DL, LHS, AL);
17154 if (RHSAlignShift < AlignShift)
17155 RHS = DAG.getAssertAlign(DL, RHS, AL);
17156 return DAG.getNode(N0.getOpcode(), DL, N0.getValueType(), LHS, RHS);
17157 }
17158 break;
17159 }
17160 }
17161
17162 return SDValue();
17163}
17164
17165SDValue DAGCombiner::visitIS_FPCLASS(SDNode *N) {
17166 SDValue Src = N->getOperand(0);
17167 FPClassTest Mask = static_cast<FPClassTest>(N->getConstantOperandVal(1));
17168 EVT VT = N->getValueType(0);
17169 SDLoc DL(N);
17170
17171 // is.fpclass(poison, mask) -> poison
17172 if (Src.getOpcode() == ISD::POISON)
17173 return DAG.getPOISON(VT);
17174
17175 KnownFPClass Known = DAG.computeKnownFPClass(Src, Mask);
17176
17177 // All possible classes are within the mask: result is always true.
17178 if ((~Mask & Known.getKnownFPClasses()) == fcNone)
17179 return DAG.getBoolConstant(true, DL, VT, Src.getValueType());
17180
17181 // Clear test bits we know must be false from the source value.
17182 // fp_class (nnan x), qnan|snan|other -> fp_class (nnan x), other
17183 // fp_class (ninf x), ninf|pinf|other -> fp_class (ninf x), other
17184 if ((Mask & Known.getKnownFPClasses()) != Mask) {
17185 return DAG.getNode(
17186 ISD::IS_FPCLASS, DL, VT, Src,
17187 DAG.getTargetConstant(Mask & Known.getKnownFPClasses(), DL, MVT::i32),
17188 N->getFlags());
17189 }
17190
17191 return SDValue();
17192}
17193
17194/// If the result of a load is shifted/masked/truncated to an effectively
17195/// narrower type, try to transform the load to a narrower type and/or
17196/// use an extending load.
17197SDValue DAGCombiner::reduceLoadWidth(SDNode *N) {
17198 unsigned Opc = N->getOpcode();
17199
17201 SDValue N0 = N->getOperand(0);
17202 EVT VT = N->getValueType(0);
17203 EVT ExtVT = VT;
17204
17205 // This transformation isn't valid for vector loads.
17206 if (VT.isVector())
17207 return SDValue();
17208
17209 // The ShAmt variable is used to indicate that we've consumed a right
17210 // shift. I.e. we want to narrow the width of the load by skipping to load the
17211 // ShAmt least significant bits.
17212 unsigned ShAmt = 0;
17213 // A special case is when the least significant bits from the load are masked
17214 // away, but using an AND rather than a right shift. HasShiftedOffset is used
17215 // to indicate that the narrowed load should be left-shifted ShAmt bits to get
17216 // the result.
17217 unsigned ShiftedOffset = 0;
17218 // Special case: SIGN_EXTEND_INREG is basically truncating to ExtVT then
17219 // extended to VT.
17220 if (Opc == ISD::SIGN_EXTEND_INREG) {
17221 ExtType = ISD::SEXTLOAD;
17222 ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT();
17223 } else if (Opc == ISD::SRL || Opc == ISD::SRA) {
17224 // Another special-case: SRL/SRA is basically zero/sign-extending a narrower
17225 // value, or it may be shifting a higher subword, half or byte into the
17226 // lowest bits.
17227
17228 // Only handle shift with constant shift amount, and the shiftee must be a
17229 // load.
17230 auto *LN = dyn_cast<LoadSDNode>(N0);
17231 auto *N1C = dyn_cast<ConstantSDNode>(N->getOperand(1));
17232 if (!N1C || !LN)
17233 return SDValue();
17234 // If the shift amount is larger than the memory type then we're not
17235 // accessing any of the loaded bytes.
17236 ShAmt = N1C->getZExtValue();
17237 uint64_t MemoryWidth = LN->getMemoryVT().getScalarSizeInBits();
17238 if (MemoryWidth <= ShAmt)
17239 return SDValue();
17240 // Attempt to fold away the SRL by using ZEXTLOAD and SRA by using SEXTLOAD.
17241 ExtType = Opc == ISD::SRL ? ISD::ZEXTLOAD : ISD::SEXTLOAD;
17242 ExtVT = EVT::getIntegerVT(*DAG.getContext(), MemoryWidth - ShAmt);
17243 // If original load is a SEXTLOAD then we can't simply replace it by a
17244 // ZEXTLOAD (we could potentially replace it by a more narrow SEXTLOAD
17245 // followed by a ZEXT, but that is not handled at the moment). Similarly if
17246 // the original load is a ZEXTLOAD and we want to use a SEXTLOAD.
17247 if ((LN->getExtensionType() == ISD::SEXTLOAD ||
17248 LN->getExtensionType() == ISD::ZEXTLOAD) &&
17249 LN->getExtensionType() != ExtType)
17250 return SDValue();
17251 } else if (Opc == ISD::AND) {
17252 // An AND with a constant mask is the same as a truncate + zero-extend.
17253 auto AndC = dyn_cast<ConstantSDNode>(N->getOperand(1));
17254 if (!AndC)
17255 return SDValue();
17256
17257 const APInt &Mask = AndC->getAPIntValue();
17258 unsigned ActiveBits = 0;
17259 if (Mask.isMask()) {
17260 ActiveBits = Mask.countr_one();
17261 } else if (Mask.isShiftedMask(ShAmt, ActiveBits)) {
17262 ShiftedOffset = ShAmt;
17263 } else {
17264 return SDValue();
17265 }
17266
17267 ExtType = ISD::ZEXTLOAD;
17268 ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
17269 }
17270
17271 // In case Opc==SRL we've already prepared ExtVT/ExtType/ShAmt based on doing
17272 // a right shift. Here we redo some of those checks, to possibly adjust the
17273 // ExtVT even further based on "a masking AND". We could also end up here for
17274 // other reasons (e.g. based on Opc==TRUNCATE) and that is why some checks
17275 // need to be done here as well.
17276 if (Opc == ISD::SRL || N0.getOpcode() == ISD::SRL) {
17277 SDValue SRL = Opc == ISD::SRL ? SDValue(N, 0) : N0;
17278 // Bail out when the SRL has more than one use. This is done for historical
17279 // (undocumented) reasons. Maybe intent was to guard the AND-masking below
17280 // check below? And maybe it could be non-profitable to do the transform in
17281 // case the SRL has multiple uses and we get here with Opc!=ISD::SRL?
17282 // FIXME: Can't we just skip this check for the Opc==ISD::SRL case.
17283 if (!SRL.hasOneUse())
17284 return SDValue();
17285
17286 // Only handle shift with constant shift amount, and the shiftee must be a
17287 // load.
17288 auto *LN = dyn_cast<LoadSDNode>(SRL.getOperand(0));
17289 auto *SRL1C = dyn_cast<ConstantSDNode>(SRL.getOperand(1));
17290 if (!SRL1C || !LN)
17291 return SDValue();
17292
17293 // If the shift amount is larger than the input type then we're not
17294 // accessing any of the loaded bytes. If the load was a zextload/extload
17295 // then the result of the shift+trunc is zero/undef (handled elsewhere).
17296 ShAmt = SRL1C->getZExtValue();
17297 uint64_t MemoryWidth = LN->getMemoryVT().getSizeInBits();
17298 if (ShAmt >= MemoryWidth)
17299 return SDValue();
17300
17301 // Because a SRL must be assumed to *need* to zero-extend the high bits
17302 // (as opposed to anyext the high bits), we can't combine the zextload
17303 // lowering of SRL and an sextload.
17304 if (LN->getExtensionType() == ISD::SEXTLOAD)
17305 return SDValue();
17306
17307 // Avoid reading outside the memory accessed by the original load (could
17308 // happened if we only adjust the load base pointer by ShAmt). Instead we
17309 // try to narrow the load even further. The typical scenario here is:
17310 // (i64 (truncate (i96 (srl (load x), 64)))) ->
17311 // (i64 (truncate (i96 (zextload (load i32 + offset) from i32))))
17312 if (ExtVT.getScalarSizeInBits() > MemoryWidth - ShAmt) {
17313 // Don't replace sextload by zextload.
17314 if (ExtType == ISD::SEXTLOAD)
17315 return SDValue();
17316 // Narrow the load.
17317 ExtType = ISD::ZEXTLOAD;
17318 ExtVT = EVT::getIntegerVT(*DAG.getContext(), MemoryWidth - ShAmt);
17319 }
17320
17321 // If the SRL is only used by a masking AND, we may be able to adjust
17322 // the ExtVT to make the AND redundant.
17323 SDNode *Mask = *(SRL->user_begin());
17324 if (SRL.hasOneUse() && Mask->getOpcode() == ISD::AND &&
17325 isa<ConstantSDNode>(Mask->getOperand(1))) {
17326 unsigned Offset, ActiveBits;
17327 const APInt& ShiftMask = Mask->getConstantOperandAPInt(1);
17328 if (ShiftMask.isMask()) {
17329 EVT MaskedVT =
17330 EVT::getIntegerVT(*DAG.getContext(), ShiftMask.countr_one());
17331 // If the mask is smaller, recompute the type.
17332 if ((ExtVT.getScalarSizeInBits() > MaskedVT.getScalarSizeInBits()) &&
17333 TLI.isLoadLegal(SRL.getValueType(), MaskedVT, LN->getAlign(),
17334 LN->getAddressSpace(), ExtType, false))
17335 ExtVT = MaskedVT;
17336 } else if (ExtType == ISD::ZEXTLOAD &&
17337 ShiftMask.isShiftedMask(Offset, ActiveBits) &&
17338 (Offset + ShAmt) < VT.getScalarSizeInBits()) {
17339 EVT MaskedVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
17340 // If the mask is shifted we can use a narrower load and a shl to insert
17341 // the trailing zeros.
17342 if (((Offset + ActiveBits) <= ExtVT.getScalarSizeInBits()) &&
17343 TLI.isLoadLegal(SRL.getValueType(), MaskedVT, LN->getAlign(),
17344 LN->getAddressSpace(), ExtType, false)) {
17345 ExtVT = MaskedVT;
17346 ShAmt = Offset + ShAmt;
17347 ShiftedOffset = Offset;
17348 }
17349 }
17350 }
17351
17352 N0 = SRL.getOperand(0);
17353 }
17354
17355 // If the load is shifted left (and the result isn't shifted back right), we
17356 // can fold a truncate through the shift. The typical scenario is that N
17357 // points at a TRUNCATE here so the attempted fold is:
17358 // (truncate (shl (load x), c))) -> (shl (narrow load x), c)
17359 // ShLeftAmt will indicate how much a narrowed load should be shifted left.
17360 unsigned ShLeftAmt = 0;
17361 if (ShAmt == 0 && N0.getOpcode() == ISD::SHL && N0.hasOneUse() &&
17362 ExtVT == VT && TLI.isNarrowingProfitable(N, N0.getValueType(), VT)) {
17363 if (ConstantSDNode *N01 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
17364 ShLeftAmt = N01->getZExtValue();
17365 N0 = N0.getOperand(0);
17366 }
17367 }
17368
17369 // Look through a freeze if present between the operation and the load.
17370 // The freeze will be preserved on the narrowed result.
17371 SDValue FreezeNode;
17372 if (N0.getOpcode() == ISD::FREEZE) {
17373 FreezeNode = N0;
17374 N0 = N0.getOperand(0);
17375 }
17376
17377 // If we haven't found a load, we can't narrow it.
17378 if (!isa<LoadSDNode>(N0))
17379 return SDValue();
17380
17381 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
17382 // Reducing the width of a volatile load is illegal. For atomics, we may be
17383 // able to reduce the width provided we never widen again. (see D66309)
17384 if (!LN0->isSimple() ||
17385 !isLegalNarrowLdSt(LN0, ExtType, ExtVT, ShAmt))
17386 return SDValue();
17387
17388 // Bail early when looking through a multi-use freeze, since other users of
17389 // the freeze can depend on the full load value. But its still safe to change
17390 // the extension type from anyext to zext.
17391 if (FreezeNode && !FreezeNode.hasOneUse() &&
17392 (LN0->getMemoryVT().bitsGT(ExtVT) || ExtType != ISD::ZEXTLOAD ||
17393 (LN0->getExtensionType() != ISD::EXTLOAD &&
17394 LN0->getExtensionType() != ISD::ZEXTLOAD)))
17395 return SDValue();
17396
17397 auto AdjustBigEndianShift = [&](unsigned ShAmt) {
17398 unsigned LVTStoreBits =
17400 unsigned EVTStoreBits = ExtVT.getStoreSizeInBits().getFixedValue();
17401 return LVTStoreBits - EVTStoreBits - ShAmt;
17402 };
17403
17404 // We need to adjust the pointer to the load by ShAmt bits in order to load
17405 // the correct bytes.
17406 unsigned PtrAdjustmentInBits =
17407 DAG.getDataLayout().isBigEndian() ? AdjustBigEndianShift(ShAmt) : ShAmt;
17408
17409 uint64_t PtrOff = PtrAdjustmentInBits / 8;
17410 SDLoc DL(LN0);
17411 // The original load itself didn't wrap, so an offset within it doesn't.
17412 SDValue NewPtr =
17415 AddToWorklist(NewPtr.getNode());
17416
17417 SDValue Load;
17418 if (ExtType == ISD::NON_EXTLOAD) {
17419 const MDNode *OldRanges = LN0->getRanges();
17420 const MDNode *NewRanges = nullptr;
17421 // If LSBs are loaded and the truncated ConstantRange for the OldRanges
17422 // metadata is not the full-set for the new width then create a NewRanges
17423 // metadata for the truncated load
17424 if (ShAmt == 0 && OldRanges) {
17425 ConstantRange CR = getConstantRangeFromMetadata(*OldRanges);
17426 unsigned BitSize = VT.getScalarSizeInBits();
17427
17428 // It is possible for an 8-bit extending load with 8-bit range
17429 // metadata to be narrowed to an 8-bit load. This guard is necessary to
17430 // ensure that truncation is strictly smaller.
17431 if (CR.getBitWidth() > BitSize) {
17432 ConstantRange TruncatedCR = CR.truncate(BitSize);
17433 if (!TruncatedCR.isFullSet()) {
17434 Metadata *Bounds[2] = {
17436 ConstantInt::get(*DAG.getContext(), TruncatedCR.getLower())),
17438 ConstantInt::get(*DAG.getContext(), TruncatedCR.getUpper()))};
17439 NewRanges = MDNode::get(*DAG.getContext(), Bounds);
17440 }
17441 } else if (CR.getBitWidth() == BitSize)
17442 NewRanges = OldRanges;
17443 }
17444 Load = DAG.getLoad(
17445 VT, DL, LN0->getChain(), NewPtr,
17446 LN0->getPointerInfo().getWithOffset(PtrOff), LN0->getBaseAlign(),
17447 LN0->getMemOperand()->getFlags(),
17448 MMOMetadata(LN0->getAAInfo(), NewRanges, LN0->getMemCacheHint()));
17449 } else
17450 Load = DAG.getExtLoad(ExtType, DL, VT, LN0->getChain(), NewPtr,
17451 LN0->getPointerInfo().getWithOffset(PtrOff), ExtVT,
17452 LN0->getBaseAlign(), LN0->getMemOperand()->getFlags(),
17454
17455 // Replace the old load's chain with the new load's chain.
17456 WorklistRemover DeadNodes(*this);
17458
17459 // Replace old load value for multi-use freeze so all users benefit.
17460 if (FreezeNode && !FreezeNode.hasOneUse())
17462
17463 // If we looked through a freeze, rewrap the narrowed result and add an
17464 // Assert node so downstream analyses can see the range.
17466 if (FreezeNode) {
17467 Result = DAG.getNode(ISD::FREEZE, DL, VT, Result);
17468 if (ExtType == ISD::ZEXTLOAD)
17469 Result =
17470 DAG.getNode(ISD::AssertZext, DL, VT, Result, DAG.getValueType(ExtVT));
17471 else if (ExtType == ISD::SEXTLOAD)
17472 Result =
17473 DAG.getNode(ISD::AssertSext, DL, VT, Result, DAG.getValueType(ExtVT));
17474 }
17475
17476 // Shift the result left, if we've swallowed a left shift.
17477 if (ShLeftAmt != 0) {
17478 // If the shift amount is as large as the result size (but, presumably,
17479 // no larger than the source) then the useful bits of the result are
17480 // zero; we can't simply return the shortened shift, because the result
17481 // of that operation is undefined.
17482 if (ShLeftAmt >= VT.getScalarSizeInBits())
17483 Result = DAG.getConstant(0, DL, VT);
17484 else
17485 Result = DAG.getNode(ISD::SHL, DL, VT, Result,
17486 DAG.getShiftAmountConstant(ShLeftAmt, VT, DL));
17487 }
17488
17489 if (ShiftedOffset != 0) {
17490 // We're using a shifted mask, so the load now has an offset. This means
17491 // that data has been loaded into the lower bytes than it would have been
17492 // before, so we need to shl the loaded data into the correct position in the
17493 // register.
17494 SDValue ShiftC = DAG.getConstant(ShiftedOffset, DL, VT);
17495 Result = DAG.getNode(ISD::SHL, DL, VT, Result, ShiftC);
17496 DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result);
17497 }
17498
17499 // Return the new loaded value.
17500 return Result;
17501}
17502
17503SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) {
17504 SDValue N0 = N->getOperand(0);
17505 SDValue N1 = N->getOperand(1);
17506 EVT VT = N->getValueType(0);
17507 EVT ExtVT = cast<VTSDNode>(N1)->getVT();
17508 unsigned VTBits = VT.getScalarSizeInBits();
17509 unsigned ExtVTBits = ExtVT.getScalarSizeInBits();
17510 SDLoc DL(N);
17511
17512 // sext_vector_inreg(undef) = 0 because the top bit will all be the same.
17513 if (N0.isUndef())
17514 return DAG.getConstant(0, DL, VT);
17515
17516 // fold (sext_in_reg c1) -> c1
17517 if (SDValue C =
17519 return C;
17520
17521 // If the input is already sign extended, just drop the extension.
17522 if (ExtVTBits >= DAG.ComputeMaxSignificantBits(N0))
17523 return N0;
17524
17525 // fold (sext_in_reg (sext_in_reg x, VT2), VT1) -> (sext_in_reg x, minVT) pt2
17526 if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
17527 ExtVT.bitsLT(cast<VTSDNode>(N0.getOperand(1))->getVT()))
17528 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, N0.getOperand(0), N1);
17529
17530 // fold (sext_in_reg (sext x)) -> (sext x)
17531 // fold (sext_in_reg (aext x)) -> (sext x)
17532 // if x is small enough or if we know that x has more than 1 sign bit and the
17533 // sign_extend_inreg is extending from one of them.
17534 if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND) {
17535 SDValue N00 = N0.getOperand(0);
17536 unsigned N00Bits = N00.getScalarValueSizeInBits();
17537 if ((N00Bits <= ExtVTBits ||
17538 DAG.ComputeMaxSignificantBits(N00) <= ExtVTBits) &&
17539 (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND, VT)))
17540 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, N00);
17541 }
17542
17543 // fold (sext_in_reg (*_extend_vector_inreg x)) -> (sext_vector_inreg x)
17544 // if x is small enough or if we know that x has more than 1 sign bit and the
17545 // sign_extend_inreg is extending from one of them.
17547 SDValue N00 = N0.getOperand(0);
17548 unsigned N00Bits = N00.getScalarValueSizeInBits();
17549 bool IsZext = N0.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG;
17550 if ((N00Bits == ExtVTBits ||
17551 (!IsZext && (N00Bits < ExtVTBits ||
17552 DAG.ComputeMaxSignificantBits(N00) <= ExtVTBits))) &&
17553 (!LegalOperations ||
17555 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, N00);
17556 }
17557
17558 // fold (sext_in_reg (zext x)) -> (sext x)
17559 // iff we are extending the source sign bit.
17560 if (N0.getOpcode() == ISD::ZERO_EXTEND) {
17561 SDValue N00 = N0.getOperand(0);
17562 if (N00.getScalarValueSizeInBits() == ExtVTBits &&
17563 (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND, VT)))
17564 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, N00);
17565 }
17566
17567 // fold (sext_in_reg x) -> (zext_in_reg x) if the sign bit is known zero.
17568 if (DAG.MaskedValueIsZero(N0, APInt::getOneBitSet(VTBits, ExtVTBits - 1)))
17569 return DAG.getZeroExtendInReg(N0, DL, ExtVT);
17570
17571 // fold operands of sext_in_reg based on knowledge that the top bits are not
17572 // demanded.
17574 return SDValue(N, 0);
17575
17576 // fold (sext_in_reg (load x)) -> (smaller sextload x)
17577 // fold (sext_in_reg (srl (load x), c)) -> (smaller sextload (x+c/evtbits))
17578 if (SDValue NarrowLoad = reduceLoadWidth(N))
17579 return NarrowLoad;
17580
17581 // fold (sext_in_reg (srl X, 24), i8) -> (sra X, 24)
17582 // fold (sext_in_reg (srl X, 23), i8) -> (sra X, 23) iff possible.
17583 // We already fold "(sext_in_reg (srl X, 25), i8) -> srl X, 25" above.
17584 if (N0.getOpcode() == ISD::SRL) {
17585 if (auto *ShAmt = dyn_cast<ConstantSDNode>(N0.getOperand(1)))
17586 if (ShAmt->getAPIntValue().ule(VTBits - ExtVTBits)) {
17587 // We can turn this into an SRA iff the input to the SRL is already sign
17588 // extended enough.
17589 unsigned InSignBits = DAG.ComputeNumSignBits(N0.getOperand(0));
17590 if (((VTBits - ExtVTBits) - ShAmt->getZExtValue()) < InSignBits)
17591 return DAG.getNode(ISD::SRA, DL, VT, N0.getOperand(0),
17592 N0.getOperand(1));
17593 }
17594 }
17595
17596 // fold (sext_inreg (extload x)) -> (sextload x)
17597 // If sextload is not supported by target, we can only do the combine when
17598 // load has one use. Doing otherwise can block folding the extload with other
17599 // extends that the target does support.
17601 auto *LN0 = cast<LoadSDNode>(N0);
17602 if (ExtVT == LN0->getMemoryVT() &&
17603 ((!LegalOperations && LN0->isSimple() && N0.hasOneUse()) ||
17604 TLI.isLoadLegal(VT, ExtVT, LN0->getAlign(), LN0->getAddressSpace(),
17605 ISD::SEXTLOAD, false))) {
17606 SDValue ExtLoad =
17607 DAG.getExtLoad(ISD::SEXTLOAD, DL, VT, LN0->getChain(),
17608 LN0->getBasePtr(), ExtVT, LN0->getMemOperand());
17609 CombineTo(N, ExtLoad);
17610 CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
17611 AddToWorklist(ExtLoad.getNode());
17612 return SDValue(N, 0); // Return N so it doesn't get rechecked!
17613 }
17614 }
17615
17616 // fold (sext_inreg (zextload x)) -> (sextload x) iff load has one use
17618 auto *LN0 = cast<LoadSDNode>(N0);
17619
17620 if (N0.hasOneUse() && ExtVT == LN0->getMemoryVT() &&
17621 ((!LegalOperations && LN0->isSimple()) &&
17622 TLI.isLoadLegal(VT, ExtVT, LN0->getAlign(), LN0->getAddressSpace(),
17623 ISD::SEXTLOAD, false))) {
17624 SDValue ExtLoad =
17625 DAG.getExtLoad(ISD::SEXTLOAD, DL, VT, LN0->getChain(),
17626 LN0->getBasePtr(), ExtVT, LN0->getMemOperand());
17627 CombineTo(N, ExtLoad);
17628 CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
17629 return SDValue(N, 0); // Return N so it doesn't get rechecked!
17630 }
17631 }
17632
17633 // fold (sext_inreg (masked_load x)) -> (sext_masked_load x)
17634 // ignore it if the masked load is already sign extended
17635 bool Frozen = N0.getOpcode() == ISD::FREEZE && N0.hasOneUse();
17636 if (auto *Ld = dyn_cast<MaskedLoadSDNode>(Frozen ? N0.getOperand(0) : N0)) {
17637 if (ExtVT == Ld->getMemoryVT() && Ld->hasNUsesOfValue(1, 0) &&
17638 Ld->getExtensionType() != ISD::LoadExtType::NON_EXTLOAD &&
17639 TLI.isLoadLegal(VT, ExtVT, Ld->getAlign(), Ld->getAddressSpace(),
17640 ISD::SEXTLOAD, false)) {
17641 SDValue ExtMaskedLoad = DAG.getMaskedLoad(
17642 VT, DL, Ld->getChain(), Ld->getBasePtr(), Ld->getOffset(),
17643 Ld->getMask(), Ld->getPassThru(), ExtVT, Ld->getMemOperand(),
17644 Ld->getAddressingMode(), ISD::SEXTLOAD, Ld->isExpandingLoad());
17645 CombineTo(N, Frozen ? N0 : ExtMaskedLoad);
17646 CombineTo(Ld, ExtMaskedLoad, ExtMaskedLoad.getValue(1));
17647 return SDValue(N, 0); // Return N so it doesn't get rechecked!
17648 }
17649 }
17650
17651 // fold (sext_inreg (masked_gather x)) -> (sext_masked_gather x)
17652 if (auto *GN0 = dyn_cast<MaskedGatherSDNode>(N0)) {
17653 if (SDValue(GN0, 0).hasOneUse() && ExtVT == GN0->getMemoryVT() &&
17655 SDValue Ops[] = {GN0->getChain(), GN0->getPassThru(), GN0->getMask(),
17656 GN0->getBasePtr(), GN0->getIndex(), GN0->getScale()};
17657
17658 SDValue ExtLoad = DAG.getMaskedGather(
17659 DAG.getVTList(VT, MVT::Other), ExtVT, DL, Ops, GN0->getMemOperand(),
17660 GN0->getIndexType(), ISD::SEXTLOAD);
17661
17662 CombineTo(N, ExtLoad);
17663 CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
17664 AddToWorklist(ExtLoad.getNode());
17665 return SDValue(N, 0); // Return N so it doesn't get rechecked!
17666 }
17667 }
17668
17669 // Form (sext_inreg (bswap >> 16)) or (sext_inreg (rotl (bswap) 16))
17670 if (ExtVTBits <= 16 && N0.getOpcode() == ISD::OR) {
17671 if (SDValue BSwap = MatchBSwapHWordLow(N0.getNode(), N0.getOperand(0),
17672 N0.getOperand(1), false))
17673 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, BSwap, N1);
17674 }
17675
17676 // Fold (iM_signext_inreg
17677 // (extract_subvector (zext|anyext|sext iN_v to _) _)
17678 // from iN)
17679 // -> (extract_subvector (signext iN_v to iM))
17680 if (N0.getOpcode() == ISD::EXTRACT_SUBVECTOR && N0.hasOneUse() &&
17682 SDValue InnerExt = N0.getOperand(0);
17683 EVT InnerExtVT = InnerExt->getValueType(0);
17684 SDValue Extendee = InnerExt->getOperand(0);
17685
17686 if (ExtVTBits == Extendee.getValueType().getScalarSizeInBits() &&
17687 (!LegalOperations ||
17688 TLI.isOperationLegal(ISD::SIGN_EXTEND, InnerExtVT))) {
17689 SDValue SignExtExtendee =
17690 DAG.getNode(ISD::SIGN_EXTEND, DL, InnerExtVT, Extendee);
17691 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, SignExtExtendee,
17692 N0.getOperand(1));
17693 }
17694 }
17695
17696 return SDValue();
17697}
17698
17700 SDNode *N, const SDLoc &DL, const TargetLowering &TLI, SelectionDAG &DAG,
17701 bool LegalOperations) {
17702 unsigned InregOpcode = N->getOpcode();
17703 unsigned Opcode = DAG.getOpcode_EXTEND(InregOpcode);
17704
17705 SDValue Src = N->getOperand(0);
17706 EVT VT = N->getValueType(0);
17707 EVT SrcVT = VT.changeVectorElementType(
17708 *DAG.getContext(), Src.getValueType().getVectorElementType());
17709
17710 assert(ISD::isExtVecInRegOpcode(InregOpcode) &&
17711 "Expected EXTEND_VECTOR_INREG dag node in input!");
17712
17713 // Profitability check: our operand must be an one-use CONCAT_VECTORS.
17714 // FIXME: one-use check may be overly restrictive
17715 if (!Src.hasOneUse() || Src.getOpcode() != ISD::CONCAT_VECTORS)
17716 return SDValue();
17717
17718 // Profitability check: we must be extending exactly one of it's operands.
17719 // FIXME: this is probably overly restrictive.
17720 Src = Src.getOperand(0);
17721 if (Src.getValueType() != SrcVT)
17722 return SDValue();
17723
17724 if (LegalOperations && !TLI.isOperationLegal(Opcode, VT))
17725 return SDValue();
17726
17727 return DAG.getNode(Opcode, DL, VT, Src);
17728}
17729
17730SDValue DAGCombiner::visitEXTEND_VECTOR_INREG(SDNode *N) {
17731 SDValue N0 = N->getOperand(0);
17732 EVT VT = N->getValueType(0);
17733 SDLoc DL(N);
17734
17735 if (N0.isUndef()) {
17736 // aext_vector_inreg(undef) = undef because the top bits are undefined.
17737 // {s/z}ext_vector_inreg(undef) = 0 because the top bits must be the same.
17738 return N->getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG
17739 ? DAG.getUNDEF(VT)
17740 : DAG.getConstant(0, DL, VT);
17741 }
17742
17743 if (SDValue Res = tryToFoldExtendOfConstant(N, DL, TLI, DAG, LegalTypes))
17744 return Res;
17745
17747 return SDValue(N, 0);
17748
17750 LegalOperations))
17751 return R;
17752
17753 return SDValue();
17754}
17755
17756SDValue DAGCombiner::visitTRUNCATE_USAT_U(SDNode *N) {
17757 EVT VT = N->getValueType(0);
17758 SDValue N0 = N->getOperand(0);
17759
17760 SDValue FPVal;
17761 if (sd_match(N0, m_FPToUI(m_Value(FPVal))) &&
17763 ISD::FP_TO_UINT_SAT, FPVal.getValueType(), VT))
17764 return DAG.getNode(ISD::FP_TO_UINT_SAT, SDLoc(N0), VT, FPVal,
17765 DAG.getValueType(VT.getScalarType()));
17766
17767 return SDValue();
17768}
17769
17770/// Detect patterns of truncation with unsigned saturation:
17771///
17772/// (truncate (umin (x, unsigned_max_of_dest_type)) to dest_type).
17773/// Return the source value x to be truncated or SDValue() if the pattern was
17774/// not matched.
17775///
17777 unsigned NumDstBits = VT.getScalarSizeInBits();
17778 unsigned NumSrcBits = In.getScalarValueSizeInBits();
17779 // Saturation with truncation. We truncate from InVT to VT.
17780 assert(NumSrcBits > NumDstBits && "Unexpected types for truncate operation");
17781
17782 SDValue Min;
17783 APInt UnsignedMax = APInt::getMaxValue(NumDstBits).zext(NumSrcBits);
17784 if (sd_match(In, m_UMin(m_Value(Min), m_SpecificInt(UnsignedMax))))
17785 return Min;
17786
17787 return SDValue();
17788}
17789
17790/// Detect patterns of truncation with signed saturation:
17791/// (truncate (smin (smax (x, signed_min_of_dest_type),
17792/// signed_max_of_dest_type)) to dest_type)
17793/// or:
17794/// (truncate (smax (smin (x, signed_max_of_dest_type),
17795/// signed_min_of_dest_type)) to dest_type).
17796///
17797/// Return the source value to be truncated or SDValue() if the pattern was not
17798/// matched.
17800 unsigned NumDstBits = VT.getScalarSizeInBits();
17801 unsigned NumSrcBits = In.getScalarValueSizeInBits();
17802 // Saturation with truncation. We truncate from InVT to VT.
17803 assert(NumSrcBits > NumDstBits && "Unexpected types for truncate operation");
17804
17805 SDValue Val;
17806 APInt SignedMax = APInt::getSignedMaxValue(NumDstBits).sext(NumSrcBits);
17807 APInt SignedMin = APInt::getSignedMinValue(NumDstBits).sext(NumSrcBits);
17808
17809 if (sd_match(In, m_SMin(m_SMax(m_Value(Val), m_SpecificInt(SignedMin)),
17810 m_SpecificInt(SignedMax))))
17811 return Val;
17812
17813 if (sd_match(In, m_SMax(m_SMin(m_Value(Val), m_SpecificInt(SignedMax)),
17814 m_SpecificInt(SignedMin))))
17815 return Val;
17816
17817 return SDValue();
17818}
17819
17820/// Detect patterns of truncation with unsigned saturation:
17822 const SDLoc &DL) {
17823 unsigned NumDstBits = VT.getScalarSizeInBits();
17824 unsigned NumSrcBits = In.getScalarValueSizeInBits();
17825 // Saturation with truncation. We truncate from InVT to VT.
17826 assert(NumSrcBits > NumDstBits && "Unexpected types for truncate operation");
17827
17828 SDValue Val;
17829 APInt UnsignedMax = APInt::getMaxValue(NumDstBits).zext(NumSrcBits);
17830 // Min == 0, Max is unsigned max of destination type.
17831 if (sd_match(In, m_SMax(m_SMin(m_Value(Val), m_SpecificInt(UnsignedMax)),
17832 m_Zero())))
17833 return Val;
17834
17835 if (sd_match(In, m_SMin(m_SMax(m_Value(Val), m_Zero()),
17836 m_SpecificInt(UnsignedMax))))
17837 return Val;
17838
17839 if (sd_match(In, m_UMin(m_SMax(m_Value(Val), m_Zero()),
17840 m_SpecificInt(UnsignedMax))))
17841 return Val;
17842
17843 return SDValue();
17844}
17845
17846static SDValue foldToSaturated(SDNode *N, EVT &VT, SDValue &Src, EVT &SrcVT,
17847 SDLoc &DL, const TargetLowering &TLI,
17848 SelectionDAG &DAG) {
17849 auto AllowedTruncateSat = [&](unsigned Opc, EVT SrcVT, EVT VT) -> bool {
17850 return (TLI.isOperationLegalOrCustom(Opc, SrcVT) &&
17851 TLI.isTypeDesirableForOp(Opc, VT));
17852 };
17853
17854 if (Src.getOpcode() == ISD::SMIN || Src.getOpcode() == ISD::SMAX) {
17855 if (AllowedTruncateSat(ISD::TRUNCATE_SSAT_S, SrcVT, VT))
17856 if (SDValue SSatVal = detectSSatSPattern(Src, VT))
17857 return DAG.getNode(ISD::TRUNCATE_SSAT_S, DL, VT, SSatVal);
17858 if (AllowedTruncateSat(ISD::TRUNCATE_SSAT_U, SrcVT, VT))
17859 if (SDValue SSatVal = detectSSatUPattern(Src, VT, DAG, DL))
17860 return DAG.getNode(ISD::TRUNCATE_SSAT_U, DL, VT, SSatVal);
17861 } else if (Src.getOpcode() == ISD::UMIN) {
17862 if (AllowedTruncateSat(ISD::TRUNCATE_SSAT_U, SrcVT, VT))
17863 if (SDValue SSatVal = detectSSatUPattern(Src, VT, DAG, DL))
17864 return DAG.getNode(ISD::TRUNCATE_SSAT_U, DL, VT, SSatVal);
17865 if (AllowedTruncateSat(ISD::TRUNCATE_USAT_U, SrcVT, VT))
17866 if (SDValue USatVal = detectUSatUPattern(Src, VT))
17867 return DAG.getNode(ISD::TRUNCATE_USAT_U, DL, VT, USatVal);
17868 }
17869
17870 return SDValue();
17871}
17872
17873SDValue DAGCombiner::visitTRUNCATE(SDNode *N) {
17874 SDValue N0 = N->getOperand(0);
17875 EVT VT = N->getValueType(0);
17876 EVT SrcVT = N0.getValueType();
17877 bool isLE = DAG.getDataLayout().isLittleEndian();
17878 SDLoc DL(N);
17879
17880 // trunc(undef) = undef, trunc(poison) = poison
17881 if (SDValue R = propagateUnaryUndef(DAG, N0, VT))
17882 return R;
17883
17884 // fold (truncate (truncate x)) -> (truncate x)
17885 if (N0.getOpcode() == ISD::TRUNCATE)
17886 return DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(0));
17887
17888 // fold saturated truncate
17889 if (SDValue SaturatedTR = foldToSaturated(N, VT, N0, SrcVT, DL, TLI, DAG))
17890 return SaturatedTR;
17891
17892 // fold (truncate c1) -> c1
17893 if (SDValue C = DAG.FoldConstantArithmetic(ISD::TRUNCATE, DL, VT, {N0},
17894 N->getFlags()))
17895 return C;
17896
17897 // fold (truncate (ext x)) -> (ext x) or (truncate x) or x
17898 if (N0.getOpcode() == ISD::ZERO_EXTEND ||
17899 N0.getOpcode() == ISD::SIGN_EXTEND ||
17900 N0.getOpcode() == ISD::ANY_EXTEND) {
17901 // if the source is smaller than the dest, we still need an extend.
17902 if (N0.getOperand(0).getValueType().bitsLT(VT)) {
17903 SDNodeFlags Flags;
17904 if (N0.getOpcode() == ISD::ZERO_EXTEND)
17905 Flags.setNonNeg(N0->getFlags().hasNonNeg());
17906 return DAG.getNode(N0.getOpcode(), DL, VT, N0.getOperand(0), Flags);
17907 }
17908 // if the source is larger than the dest, than we just need the truncate.
17909 if (N0.getOperand(0).getValueType().bitsGT(VT))
17910 return DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(0));
17911 // if the source and dest are the same type, we can drop both the extend
17912 // and the truncate.
17913 return N0.getOperand(0);
17914 }
17915
17916 // Try to narrow a truncate-of-sext_in_reg to the destination type:
17917 // trunc (sign_ext_inreg X, iM) to iN --> sign_ext_inreg (trunc X to iN), iM
17918 if (!LegalTypes && N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
17919 N0.hasOneUse()) {
17920 SDValue X = N0.getOperand(0);
17921 SDValue ExtVal = N0.getOperand(1);
17922 EVT ExtVT = cast<VTSDNode>(ExtVal)->getVT();
17923 if (ExtVT.bitsLT(VT) && TLI.preferSextInRegOfTruncate(VT, SrcVT, ExtVT)) {
17924 SDValue TrX = DAG.getNode(ISD::TRUNCATE, DL, VT, X);
17925 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, TrX, ExtVal);
17926 }
17927 }
17928
17929 // If this is anyext(trunc), don't fold it, allow ourselves to be folded.
17930 if (N->hasOneUse() && (N->user_begin()->getOpcode() == ISD::ANY_EXTEND))
17931 return SDValue();
17932
17933 // Fold extract-and-trunc into a narrow extract. For example:
17934 // i64 x = EXTRACT_VECTOR_ELT(v2i64 val, i32 1)
17935 // i32 y = TRUNCATE(i64 x)
17936 // -- becomes --
17937 // v16i8 b = BITCAST (v2i64 val)
17938 // i8 x = EXTRACT_VECTOR_ELT(v16i8 b, i32 8)
17939 //
17940 // Note: We only run this optimization after type legalization (which often
17941 // creates this pattern) and before operation legalization after which
17942 // we need to be more careful about the vector instructions that we generate.
17943 if (LegalTypes && !LegalOperations && VT.isScalarInteger() && VT != MVT::i1 &&
17944 N0->hasOneUse()) {
17945 EVT TrTy = N->getValueType(0);
17946 SDValue Src = N0;
17947
17948 // Check for cases where we shift down an upper element before truncation.
17949 int EltOffset = 0;
17950 if (Src.getOpcode() == ISD::SRL && Src.getOperand(0)->hasOneUse()) {
17951 if (auto ShAmt = DAG.getValidShiftAmount(Src)) {
17952 if ((*ShAmt % TrTy.getSizeInBits()) == 0) {
17953 Src = Src.getOperand(0);
17954 EltOffset = *ShAmt / TrTy.getSizeInBits();
17955 }
17956 }
17957 }
17958
17959 if (Src.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
17960 EVT VecTy = Src.getOperand(0).getValueType();
17961 EVT ExTy = Src.getValueType();
17962
17963 auto EltCnt = VecTy.getVectorElementCount();
17964 unsigned SizeRatio = ExTy.getSizeInBits() / TrTy.getSizeInBits();
17965 auto NewEltCnt = EltCnt * SizeRatio;
17966
17967 EVT NVT = EVT::getVectorVT(*DAG.getContext(), TrTy, NewEltCnt);
17968 assert(NVT.getSizeInBits() == VecTy.getSizeInBits() && "Invalid Size");
17969
17970 SDValue EltNo = Src->getOperand(1);
17971 if (isa<ConstantSDNode>(EltNo) && isTypeLegal(NVT)) {
17972 int Elt = EltNo->getAsZExtVal();
17973 int Index = isLE ? (Elt * SizeRatio + EltOffset)
17974 : (Elt * SizeRatio + (SizeRatio - 1) - EltOffset);
17975 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, TrTy,
17976 DAG.getBitcast(NVT, Src.getOperand(0)),
17977 DAG.getVectorIdxConstant(Index, DL));
17978 }
17979 }
17980 }
17981
17982 // trunc (select c, a, b) -> select c, (trunc a), (trunc b)
17983 if (N0.getOpcode() == ISD::SELECT && N0.hasOneUse() &&
17984 TLI.isTruncateFree(SrcVT, VT)) {
17985 if (!LegalOperations ||
17986 (TLI.isOperationLegal(ISD::SELECT, SrcVT) &&
17987 TLI.isNarrowingProfitable(N0.getNode(), SrcVT, VT))) {
17988 SDLoc SL(N0);
17989 SDValue Cond = N0.getOperand(0);
17990 SDValue TruncOp0 = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(1));
17991 SDValue TruncOp1 = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(2));
17992 return DAG.getNode(ISD::SELECT, DL, VT, Cond, TruncOp0, TruncOp1);
17993 }
17994 }
17995
17996 // trunc (shl x, K) -> shl (trunc x), K => K < VT.getScalarSizeInBits()
17997 if (N0.getOpcode() == ISD::SHL && N0.hasOneUse() &&
17998 (!LegalOperations || TLI.isOperationLegal(ISD::SHL, VT)) &&
18000 SDValue Amt = N0.getOperand(1);
18001 KnownBits Known = DAG.computeKnownBits(Amt);
18002 unsigned Size = VT.getScalarSizeInBits();
18003 if (Known.countMaxActiveBits() <= Log2_32(Size)) {
18004 EVT AmtVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
18005 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(0));
18006 if (AmtVT != Amt.getValueType()) {
18007 Amt = DAG.getZExtOrTrunc(Amt, DL, AmtVT);
18008 AddToWorklist(Amt.getNode());
18009 }
18010 return DAG.getNode(ISD::SHL, DL, VT, Trunc, Amt);
18011 }
18012 }
18013
18014 if (SDValue V = foldSubToUSubSat(VT, N0.getNode(), DL))
18015 return V;
18016
18017 if (SDValue ABD = foldABSToABD(N, DL))
18018 return ABD;
18019
18020 // Attempt to pre-truncate BUILD_VECTOR sources.
18021 if (N0.getOpcode() == ISD::BUILD_VECTOR && !LegalOperations &&
18022 N0.hasOneUse() &&
18023 // Avoid creating illegal types if running after type legalizer.
18024 (!LegalTypes || TLI.isTypeLegal(VT.getScalarType()))) {
18025 if (TLI.isTruncateFree(SrcVT.getScalarType(), VT.getScalarType()))
18026 return DAG.UnrollVectorOp(N);
18027
18028 // trunc(build_vector(ext(x), ext(x)) -> build_vector(x,x)
18029 if (SDValue SplatVal = DAG.getSplatValue(N0)) {
18030 if (ISD::isExtOpcode(SplatVal.getOpcode()) &&
18031 SrcVT.getScalarType() == SplatVal.getValueType())
18032 return DAG.UnrollVectorOp(N);
18033 }
18034 }
18035
18036 // trunc (splat_vector x) -> splat_vector (trunc x)
18037 if (N0.getOpcode() == ISD::SPLAT_VECTOR &&
18038 (!LegalTypes || TLI.isTypeLegal(VT.getScalarType())) &&
18039 (!LegalOperations || TLI.isOperationLegal(ISD::SPLAT_VECTOR, VT))) {
18040 EVT SVT = VT.getScalarType();
18041 return DAG.getSplatVector(
18042 VT, DL, DAG.getNode(ISD::TRUNCATE, DL, SVT, N0->getOperand(0)));
18043 }
18044
18045 // Fold a series of buildvector, bitcast, and truncate if possible.
18046 // For example fold
18047 // (2xi32 trunc (bitcast ((4xi32)buildvector x, x, y, y) 2xi64)) to
18048 // (2xi32 (buildvector x, y)).
18049 if (Level == AfterLegalizeVectorOps && VT.isVector() &&
18050 N0.getOpcode() == ISD::BITCAST && N0.hasOneUse() &&
18052 N0.getOperand(0).hasOneUse()) {
18053 SDValue BuildVect = N0.getOperand(0);
18054 EVT BuildVectEltTy = BuildVect.getValueType().getVectorElementType();
18055 EVT TruncVecEltTy = VT.getVectorElementType();
18056
18057 // Check that the element types match.
18058 if (BuildVectEltTy == TruncVecEltTy) {
18059 // Now we only need to compute the offset of the truncated elements.
18060 unsigned BuildVecNumElts = BuildVect.getNumOperands();
18061 unsigned TruncVecNumElts = VT.getVectorNumElements();
18062 unsigned TruncEltOffset = BuildVecNumElts / TruncVecNumElts;
18063 unsigned FirstElt = isLE ? 0 : (TruncEltOffset - 1);
18064
18065 assert((BuildVecNumElts % TruncVecNumElts) == 0 &&
18066 "Invalid number of elements");
18067
18069 for (unsigned i = FirstElt, e = BuildVecNumElts; i < e;
18070 i += TruncEltOffset)
18071 Opnds.push_back(BuildVect.getOperand(i));
18072
18073 return DAG.getBuildVector(VT, DL, Opnds);
18074 }
18075 }
18076
18077 // fold (truncate (load x)) -> (smaller load x)
18078 // fold (truncate (srl (load x), c)) -> (smaller load (x+c/evtbits))
18079 if (!LegalTypes || TLI.isTypeDesirableForOp(N0.getNode(), VT)) {
18080 if (SDValue Reduced = reduceLoadWidth(N))
18081 return Reduced;
18082
18083 // Handle the case where the truncated result is at least as wide as the
18084 // loaded type.
18085 if (N0.hasOneUse() && ISD::isUNINDEXEDLoad(N0.getNode())) {
18086 auto *LN0 = cast<LoadSDNode>(N0);
18087 if (LN0->isSimple() && LN0->getMemoryVT().bitsLE(VT)) {
18088 SDValue NewLoad = DAG.getExtLoad(
18089 LN0->getExtensionType(), SDLoc(LN0), VT, LN0->getChain(),
18090 LN0->getBasePtr(), LN0->getMemoryVT(), LN0->getMemOperand());
18091 DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), NewLoad.getValue(1));
18092 return NewLoad;
18093 }
18094 }
18095 }
18096
18097 // fold (trunc (concat ... x ...)) -> (concat ..., (trunc x), ...)),
18098 // where ... are all 'undef'.
18099 if (N0.getOpcode() == ISD::CONCAT_VECTORS && !LegalTypes) {
18101 SDValue V;
18102 unsigned Idx = 0;
18103 unsigned NumDefs = 0;
18104
18105 for (unsigned i = 0, e = N0.getNumOperands(); i != e; ++i) {
18106 SDValue X = N0.getOperand(i);
18107 if (!X.isUndef()) {
18108 V = X;
18109 Idx = i;
18110 NumDefs++;
18111 }
18112 // Stop if more than one members are non-undef.
18113 if (NumDefs > 1)
18114 break;
18115
18118 X.getValueType().getVectorElementCount()));
18119 }
18120
18121 if (NumDefs == 0)
18122 return DAG.getUNDEF(VT);
18123
18124 if (NumDefs == 1) {
18125 assert(V.getNode() && "The single defined operand is empty!");
18127 for (unsigned i = 0, e = VTs.size(); i != e; ++i) {
18128 if (i != Idx) {
18129 Opnds.push_back(DAG.getUNDEF(VTs[i]));
18130 continue;
18131 }
18132 SDValue NV = DAG.getNode(ISD::TRUNCATE, SDLoc(V), VTs[i], V);
18133 AddToWorklist(NV.getNode());
18134 Opnds.push_back(NV);
18135 }
18136 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Opnds);
18137 }
18138 }
18139
18140 // Fold truncate of a bitcast of a vector to an extract of the low vector
18141 // element.
18142 //
18143 // e.g. trunc (i64 (bitcast v2i32:x)) -> extract_vector_elt v2i32:x, idx
18144 if (N0.getOpcode() == ISD::BITCAST && !VT.isVector()) {
18145 SDValue VecSrc = N0.getOperand(0);
18146 EVT VecSrcVT = VecSrc.getValueType();
18147 if (VecSrcVT.isVectorOf(VT) &&
18148 (!LegalOperations ||
18150 unsigned Idx = isLE ? 0 : VecSrcVT.getVectorNumElements() - 1;
18151 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, VecSrc,
18152 DAG.getVectorIdxConstant(Idx, DL));
18153 }
18154 }
18155
18156 // Simplify the operands using demanded-bits information.
18158 return SDValue(N, 0);
18159
18160 // fold (truncate (extract_subvector(ext x))) ->
18161 // (extract_subvector x)
18162 // TODO: This can be generalized to cover cases where the truncate and extract
18163 // do not fully cancel each other out.
18164 if (!LegalTypes && N0.getOpcode() == ISD::EXTRACT_SUBVECTOR) {
18165 SDValue N00 = N0.getOperand(0);
18166 if (N00.getOpcode() == ISD::SIGN_EXTEND ||
18167 N00.getOpcode() == ISD::ZERO_EXTEND ||
18168 N00.getOpcode() == ISD::ANY_EXTEND) {
18169 if (N00.getOperand(0)->getValueType(0).getVectorElementType() ==
18171 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N0->getOperand(0)), VT,
18172 N00.getOperand(0), N0.getOperand(1));
18173 }
18174 }
18175
18176 if (SDValue NewVSel = matchVSelectOpSizesWithSetCC(N))
18177 return NewVSel;
18178
18179 // Narrow a suitable binary operation with a non-opaque constant operand by
18180 // moving it ahead of the truncate. This is limited to pre-legalization
18181 // because targets may prefer a wider type during later combines and invert
18182 // this transform.
18183 switch (N0.getOpcode()) {
18184 case ISD::ADD:
18185 case ISD::SUB:
18186 case ISD::MUL:
18187 case ISD::AND:
18188 case ISD::OR:
18189 case ISD::XOR:
18190 if (!LegalOperations && N0.hasOneUse() &&
18191 (N0.getOperand(0) == N0.getOperand(1) ||
18193 isConstantOrConstantVector(N0.getOperand(1), true))) {
18194 // TODO: We already restricted this to pre-legalization, but for vectors
18195 // we are extra cautious to not create an unsupported operation.
18196 // Target-specific changes are likely needed to avoid regressions here.
18197 if (VT.isScalarInteger() || TLI.isOperationLegal(N0.getOpcode(), VT)) {
18198 SDValue NarrowL = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(0));
18199 SDValue NarrowR = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(1));
18200 SDNodeFlags Flags;
18201 // Propagate nuw for sub.
18202 if (N0->getOpcode() == ISD::SUB && N0->getFlags().hasNoUnsignedWrap() &&
18204 N0->getOperand(0),
18206 VT.getScalarSizeInBits())))
18207 Flags.setNoUnsignedWrap(true);
18208 return DAG.getNode(N0.getOpcode(), DL, VT, NarrowL, NarrowR, Flags);
18209 }
18210 }
18211 break;
18212 case ISD::ADDE:
18213 case ISD::UADDO_CARRY:
18214 // (trunc adde(X, Y, Carry)) -> (adde trunc(X), trunc(Y), Carry)
18215 // (trunc uaddo_carry(X, Y, Carry)) ->
18216 // (uaddo_carry trunc(X), trunc(Y), Carry)
18217 // When the adde's carry is not used.
18218 // We only do for uaddo_carry before legalize operation
18219 if (((!LegalOperations && N0.getOpcode() == ISD::UADDO_CARRY) ||
18220 TLI.isOperationLegal(N0.getOpcode(), VT)) &&
18221 N0.hasOneUse() && !N0->hasAnyUseOfValue(1)) {
18222 SDValue X = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(0));
18223 SDValue Y = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(1));
18224 SDVTList VTs = DAG.getVTList(VT, N0->getValueType(1));
18225 return DAG.getNode(N0.getOpcode(), DL, VTs, X, Y, N0.getOperand(2));
18226 }
18227 break;
18228 case ISD::USUBSAT:
18229 // Truncate the USUBSAT only if LHS is a known zero-extension, its not
18230 // enough to know that the upper bits are zero we must ensure that we don't
18231 // introduce an extra truncate.
18232 if (!LegalOperations && N0.hasOneUse() &&
18235 VT.getScalarSizeInBits() &&
18236 hasOperation(N0.getOpcode(), VT)) {
18237 return getTruncatedUSUBSAT(VT, SrcVT, N0.getOperand(0), N0.getOperand(1),
18238 DAG, DL);
18239 }
18240 break;
18241 case ISD::AVGCEILS:
18242 case ISD::AVGCEILU:
18243 // trunc (avgceilu (sext (x), sext (y))) -> avgceils(x, y)
18244 // trunc (avgceils (zext (x), zext (y))) -> avgceilu(x, y)
18245 if (N0.hasOneUse()) {
18246 SDValue Op0 = N0.getOperand(0);
18247 SDValue Op1 = N0.getOperand(1);
18248 if (N0.getOpcode() == ISD::AVGCEILU) {
18250 Op0.getOpcode() == ISD::SIGN_EXTEND &&
18251 Op1.getOpcode() == ISD::SIGN_EXTEND &&
18252 Op0.getOperand(0).getValueType() == VT &&
18253 Op1.getOperand(0).getValueType() == VT)
18254 return DAG.getNode(ISD::AVGCEILS, DL, VT, Op0.getOperand(0),
18255 Op1.getOperand(0));
18256 } else {
18258 Op0.getOpcode() == ISD::ZERO_EXTEND &&
18259 Op1.getOpcode() == ISD::ZERO_EXTEND &&
18260 Op0.getOperand(0).getValueType() == VT &&
18261 Op1.getOperand(0).getValueType() == VT)
18262 return DAG.getNode(ISD::AVGCEILU, DL, VT, Op0.getOperand(0),
18263 Op1.getOperand(0));
18264 }
18265 }
18266 [[fallthrough]];
18267 case ISD::AVGFLOORS:
18268 case ISD::AVGFLOORU:
18269 case ISD::ABDS:
18270 case ISD::ABDU:
18271 // (trunc (avg a, b)) -> (avg (trunc a), (trunc b))
18272 // (trunc (abdu/abds a, b)) -> (abdu/abds (trunc a), (trunc b))
18273 if (!LegalOperations && N0.hasOneUse() &&
18274 TLI.isOperationLegal(N0.getOpcode(), VT)) {
18275 EVT TruncVT = VT;
18276 unsigned SrcBits = SrcVT.getScalarSizeInBits();
18277 unsigned TruncBits = TruncVT.getScalarSizeInBits();
18278
18279 SDValue A = N0.getOperand(0);
18280 SDValue B = N0.getOperand(1);
18281 bool CanFold = false;
18282
18283 if (N0.getOpcode() == ISD::AVGFLOORU || N0.getOpcode() == ISD::AVGCEILU ||
18284 N0.getOpcode() == ISD::ABDU) {
18285 APInt UpperBits = APInt::getBitsSetFrom(SrcBits, TruncBits);
18286 CanFold = DAG.MaskedValueIsZero(B, UpperBits) &&
18287 DAG.MaskedValueIsZero(A, UpperBits);
18288 } else {
18289 unsigned NeededBits = SrcBits - TruncBits;
18290 CanFold = DAG.ComputeNumSignBits(B) > NeededBits &&
18291 DAG.ComputeNumSignBits(A) > NeededBits;
18292 }
18293
18294 if (CanFold) {
18295 SDValue NewA = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, A);
18296 SDValue NewB = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, B);
18297 return DAG.getNode(N0.getOpcode(), DL, TruncVT, NewA, NewB);
18298 }
18299 }
18300 break;
18301 }
18302
18303 return SDValue();
18304}
18305
18306static SDNode *getBuildPairElt(SDNode *N, unsigned i) {
18307 SDValue Elt = N->getOperand(i);
18308 if (Elt.getOpcode() != ISD::MERGE_VALUES)
18309 return Elt.getNode();
18310 return Elt.getOperand(Elt.getResNo()).getNode();
18311}
18312
18313/// build_pair (load, load) -> load
18314/// if load locations are consecutive.
18315SDValue DAGCombiner::CombineConsecutiveLoads(SDNode *N, EVT VT) {
18316 assert(N->getOpcode() == ISD::BUILD_PAIR);
18317
18318 auto *LD1 = dyn_cast<LoadSDNode>(getBuildPairElt(N, 0));
18319 auto *LD2 = dyn_cast<LoadSDNode>(getBuildPairElt(N, 1));
18320
18321 // A BUILD_PAIR is always having the least significant part in elt 0 and the
18322 // most significant part in elt 1. So when combining into one large load, we
18323 // need to consider the endianness.
18324 if (DAG.getDataLayout().isBigEndian())
18325 std::swap(LD1, LD2);
18326
18327 if (!LD1 || !LD2 || !ISD::isNON_EXTLoad(LD1) || !ISD::isNON_EXTLoad(LD2) ||
18328 !LD1->hasOneUse() || !LD2->hasOneUse() ||
18329 LD1->getAddressSpace() != LD2->getAddressSpace())
18330 return SDValue();
18331
18332 unsigned LD1Fast = 0;
18333 EVT LD1VT = LD1->getValueType(0);
18334 unsigned LD1Bytes = LD1VT.getStoreSize();
18335 if ((!LegalOperations || TLI.isOperationLegal(ISD::LOAD, VT)) &&
18336 DAG.areNonVolatileConsecutiveLoads(LD2, LD1, LD1Bytes, 1) &&
18338 *LD1->getMemOperand(), &LD1Fast) && LD1Fast)
18339 return DAG.getLoad(
18340 VT, SDLoc(N), LD1->getChain(), LD1->getBasePtr(), LD1->getPointerInfo(),
18341 LD1->getAlign(), MachineMemOperand::MONone,
18342 MMOMetadata(AAMDNodes(), nullptr, getCommonMemCacheHint({LD1, LD2})));
18343
18344 return SDValue();
18345}
18346
18347static unsigned getPPCf128HiElementSelector(const SelectionDAG &DAG) {
18348 // On little-endian machines, bitcasting from ppcf128 to i128 does swap the Hi
18349 // and Lo parts; on big-endian machines it doesn't.
18350 return DAG.getDataLayout().isBigEndian() ? 1 : 0;
18351}
18352
18353SDValue DAGCombiner::foldBitcastedFPLogic(SDNode *N, SelectionDAG &DAG,
18354 const TargetLowering &TLI) {
18355 // If this is not a bitcast to an FP type or if the target doesn't have
18356 // IEEE754-compliant FP logic, we're done.
18357 EVT VT = N->getValueType(0);
18358 SDValue N0 = N->getOperand(0);
18359 EVT SourceVT = N0.getValueType();
18360
18361 if (!VT.isFloatingPoint())
18362 return SDValue();
18363
18364 // TODO: Handle cases where the integer constant is a different scalar
18365 // bitwidth to the FP.
18366 if (VT.getScalarSizeInBits() != SourceVT.getScalarSizeInBits())
18367 return SDValue();
18368
18369 unsigned FPOpcode;
18370 APInt SignMask;
18371 switch (N0.getOpcode()) {
18372 case ISD::AND:
18373 FPOpcode = ISD::FABS;
18374 SignMask = ~APInt::getSignMask(SourceVT.getScalarSizeInBits());
18375 break;
18376 case ISD::XOR:
18377 FPOpcode = ISD::FNEG;
18378 SignMask = APInt::getSignMask(SourceVT.getScalarSizeInBits());
18379 break;
18380 case ISD::OR:
18381 FPOpcode = ISD::FABS;
18382 SignMask = APInt::getSignMask(SourceVT.getScalarSizeInBits());
18383 break;
18384 default:
18385 return SDValue();
18386 }
18387
18388 if (LegalOperations && !TLI.isOperationLegal(FPOpcode, VT))
18389 return SDValue();
18390
18391 // This needs to be the inverse of logic in foldSignChangeInBitcast.
18392 // FIXME: I don't think looking for bitcast intrinsically makes sense, but
18393 // removing this would require more changes.
18394 auto IsBitCastOrFree = [&TLI, FPOpcode](SDValue Op, EVT VT) {
18395 if (sd_match(Op, m_BitCast(m_SpecificVT(VT))))
18396 return true;
18397
18398 return FPOpcode == ISD::FABS ? TLI.isFAbsFree(VT) : TLI.isFNegFree(VT);
18399 };
18400
18401 // Fold (bitcast int (and (bitcast fp X to int), 0x7fff...) to fp) -> fabs X
18402 // Fold (bitcast int (xor (bitcast fp X to int), 0x8000...) to fp) -> fneg X
18403 // Fold (bitcast int (or (bitcast fp X to int), 0x8000...) to fp) ->
18404 // fneg (fabs X)
18405 SDValue LogicOp0 = N0.getOperand(0);
18406 ConstantSDNode *LogicOp1 = isConstOrConstSplat(N0.getOperand(1), true);
18407 if (LogicOp1 && LogicOp1->getAPIntValue() == SignMask &&
18408 IsBitCastOrFree(LogicOp0, VT)) {
18409 SDValue CastOp0 = DAG.getNode(ISD::BITCAST, SDLoc(N), VT, LogicOp0);
18410 SDValue FPOp = DAG.getNode(FPOpcode, SDLoc(N), VT, CastOp0);
18411 NumFPLogicOpsConv++;
18412 if (N0.getOpcode() == ISD::OR)
18413 return DAG.getNode(ISD::FNEG, SDLoc(N), VT, FPOp);
18414 return FPOp;
18415 }
18416
18417 return SDValue();
18418}
18419
18420SDValue DAGCombiner::visitBITCAST(SDNode *N) {
18421 SDValue N0 = N->getOperand(0);
18422 EVT VT = N->getValueType(0);
18423
18424 if (SDValue R = propagateUnaryUndef(DAG, N0, VT))
18425 return R;
18426
18427 // If the input is a BUILD_VECTOR with all constant elements, fold this now.
18428 // Only do this before legalize types, unless both types are integer and the
18429 // scalar type is legal. Only do this before legalize ops, since the target
18430 // maybe depending on the bitcast.
18431 // First check to see if this is all constant.
18432 // TODO: Support FP bitcasts after legalize types.
18433 if (VT.isVector() &&
18434 (!LegalTypes ||
18435 (!LegalOperations && VT.isInteger() && N0.getValueType().isInteger() &&
18437 N0.getOpcode() == ISD::BUILD_VECTOR && N0->hasOneUse() &&
18438 cast<BuildVectorSDNode>(N0)->isConstant())
18439 return DAG.FoldConstantBuildVector(cast<BuildVectorSDNode>(N0), SDLoc(N),
18441
18442 // If the input is a constant, let getNode fold it.
18443 if (isIntOrFPConstant(N0)) {
18444 // If we can't allow illegal operations, we need to check that this is just
18445 // a fp -> int or int -> conversion and that the resulting operation will
18446 // be legal.
18447 if (!LegalOperations ||
18448 (isa<ConstantSDNode>(N0) && VT.isFloatingPoint() && !VT.isVector() &&
18450 (isa<ConstantFPSDNode>(N0) && VT.isInteger() && !VT.isVector() &&
18452 SDValue C = DAG.getBitcast(VT, N0);
18453 if (C.getNode() != N)
18454 return C;
18455 }
18456 }
18457
18458 // (conv (conv x, t1), t2) -> (conv x, t2)
18459 if (N0.getOpcode() == ISD::BITCAST)
18460 return DAG.getBitcast(VT, N0.getOperand(0));
18461
18462 // fold (conv (logicop (conv x), (c))) -> (logicop x, (conv c))
18463 // iff the current bitwise logicop type isn't legal
18464 if (ISD::isBitwiseLogicOp(N0.getOpcode()) && VT.isInteger() &&
18466 auto IsFreeBitcast = [VT](SDValue V) {
18467 return (V.getOpcode() == ISD::BITCAST &&
18468 V.getOperand(0).getValueType() == VT) ||
18470 V->hasOneUse());
18471 };
18472 if (IsFreeBitcast(N0.getOperand(0)) && IsFreeBitcast(N0.getOperand(1)))
18473 return DAG.getNode(N0.getOpcode(), SDLoc(N), VT,
18474 DAG.getBitcast(VT, N0.getOperand(0)),
18475 DAG.getBitcast(VT, N0.getOperand(1)));
18476 }
18477
18478 // fold (conv (load x)) -> (load (conv*)x)
18479 // fold (conv (freeze (load x))) -> (freeze (load (conv*)x))
18480 // If the resultant load doesn't need a higher alignment than the original!
18481 auto CastLoad = [this, &VT](SDValue N0, const SDLoc &DL) {
18482 // Peek through scalar_to_vector if the scalar is same size as VT - often a
18483 // leftover from legalization.
18484 if (N0.getOpcode() == ISD::SCALAR_TO_VECTOR && N0.hasOneUse() &&
18486 N0 = N0.getOperand(0);
18487 if (N0.getOpcode() == ISD::AssertNoFPClass)
18488 N0 = N0.getOperand(0);
18489 if (!ISD::isNormalLoad(N0.getNode()) || !N0.hasOneUse())
18490 return SDValue();
18491
18492 // Do not remove the cast if the types differ in endian layout.
18495 return SDValue();
18496
18497 // If the load is volatile, we only want to change the load type if the
18498 // resulting load is legal. Otherwise we might increase the number of
18499 // memory accesses. We don't care if the original type was legal or not
18500 // as we assume software couldn't rely on the number of accesses of an
18501 // illegal type.
18502 auto *LN0 = cast<LoadSDNode>(N0);
18503 if ((LegalOperations || !LN0->isSimple()) &&
18505 return SDValue();
18506
18507 if (!TLI.isLoadBitCastBeneficial(N0.getValueType(), VT, DAG,
18508 *LN0->getMemOperand()))
18509 return SDValue();
18510
18511 // If the range metadata type does not match the new memory
18512 // operation type, remove the range metadata.
18513 if (const MDNode *MD = LN0->getRanges()) {
18514 ConstantInt *Lower = mdconst::extract<ConstantInt>(MD->getOperand(0));
18515 if (Lower->getBitWidth() != VT.getScalarSizeInBits() || !VT.isInteger()) {
18516 LN0->getMemOperand()->clearRanges();
18517 }
18518 }
18519 SDValue Load = DAG.getLoad(VT, DL, LN0->getChain(), LN0->getBasePtr(),
18520 LN0->getMemOperand());
18522 return Load;
18523 };
18524
18525 if (SDValue NewLd = CastLoad(N0, SDLoc(N)))
18526 return NewLd;
18527
18528 if (N0.getOpcode() == ISD::FREEZE && N0.hasOneUse())
18529 if (SDValue NewLd = CastLoad(N0.getOperand(0), SDLoc(N)))
18530 return DAG.getFreeze(NewLd);
18531
18532 if (SDValue V = foldBitcastedFPLogic(N, DAG, TLI))
18533 return V;
18534
18535 // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
18536 // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
18537 //
18538 // For ppc_fp128:
18539 // fold (bitcast (fneg x)) ->
18540 // flipbit = signbit
18541 // (xor (bitcast x) (build_pair flipbit, flipbit))
18542 //
18543 // fold (bitcast (fabs x)) ->
18544 // flipbit = (and (extract_element (bitcast x), 0), signbit)
18545 // (xor (bitcast x) (build_pair flipbit, flipbit))
18546 // This often reduces constant pool loads.
18547 if (((N0.getOpcode() == ISD::FNEG && !TLI.isFNegFree(N0.getValueType())) ||
18548 (N0.getOpcode() == ISD::FABS && !TLI.isFAbsFree(N0.getValueType()))) &&
18549 N0->hasOneUse() && VT.isInteger() && !VT.isVector() &&
18550 !N0.getValueType().isVector()) {
18551 SDValue NewConv = DAG.getBitcast(VT, N0.getOperand(0));
18552 AddToWorklist(NewConv.getNode());
18553
18554 SDLoc DL(N);
18555 if (N0.getValueType() == MVT::ppcf128 && !LegalTypes) {
18556 assert(VT.getSizeInBits() == 128);
18557 SDValue SignBit = DAG.getConstant(
18558 APInt::getSignMask(VT.getSizeInBits() / 2), SDLoc(N0), MVT::i64);
18559 SDValue FlipBit;
18560 if (N0.getOpcode() == ISD::FNEG) {
18561 FlipBit = SignBit;
18562 AddToWorklist(FlipBit.getNode());
18563 } else {
18564 assert(N0.getOpcode() == ISD::FABS);
18565 SDValue Hi =
18566 DAG.getNode(ISD::EXTRACT_ELEMENT, SDLoc(NewConv), MVT::i64, NewConv,
18568 SDLoc(NewConv)));
18569 AddToWorklist(Hi.getNode());
18570 FlipBit = DAG.getNode(ISD::AND, SDLoc(N0), MVT::i64, Hi, SignBit);
18571 AddToWorklist(FlipBit.getNode());
18572 }
18573 SDValue FlipBits =
18574 DAG.getNode(ISD::BUILD_PAIR, SDLoc(N0), VT, FlipBit, FlipBit);
18575 AddToWorklist(FlipBits.getNode());
18576 return DAG.getNode(ISD::XOR, DL, VT, NewConv, FlipBits);
18577 }
18578 APInt SignBit = APInt::getSignMask(VT.getSizeInBits());
18579 if (N0.getOpcode() == ISD::FNEG)
18580 return DAG.getNode(ISD::XOR, DL, VT,
18581 NewConv, DAG.getConstant(SignBit, DL, VT));
18582 assert(N0.getOpcode() == ISD::FABS);
18583 return DAG.getNode(ISD::AND, DL, VT,
18584 NewConv, DAG.getConstant(~SignBit, DL, VT));
18585 }
18586
18587 // fold (bitconvert (fcopysign cst, x)) ->
18588 // (or (and (bitconvert x), sign), (and cst, (not sign)))
18589 // Note that we don't handle (copysign x, cst) because this can always be
18590 // folded to an fneg or fabs.
18591 //
18592 // For ppc_fp128:
18593 // fold (bitcast (fcopysign cst, x)) ->
18594 // flipbit = (and (extract_element
18595 // (xor (bitcast cst), (bitcast x)), 0),
18596 // signbit)
18597 // (xor (bitcast cst) (build_pair flipbit, flipbit))
18598 if (N0.getOpcode() == ISD::FCOPYSIGN && N0->hasOneUse() &&
18600 !VT.isVector()) {
18601 unsigned OrigXWidth = N0.getOperand(1).getValueSizeInBits();
18602 EVT IntXVT = EVT::getIntegerVT(*DAG.getContext(), OrigXWidth);
18603 if (isTypeLegal(IntXVT)) {
18604 SDValue X = DAG.getBitcast(IntXVT, N0.getOperand(1));
18605 AddToWorklist(X.getNode());
18606
18607 // If X has a different width than the result/lhs, sext it or truncate it.
18608 unsigned VTWidth = VT.getSizeInBits();
18609 if (OrigXWidth < VTWidth) {
18610 X = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, X);
18611 AddToWorklist(X.getNode());
18612 } else if (OrigXWidth > VTWidth) {
18613 // To get the sign bit in the right place, we have to shift it right
18614 // before truncating.
18615 SDLoc DL(X);
18616 X = DAG.getNode(ISD::SRL, DL,
18617 X.getValueType(), X,
18618 DAG.getConstant(OrigXWidth-VTWidth, DL,
18619 X.getValueType()));
18620 AddToWorklist(X.getNode());
18621 X = DAG.getNode(ISD::TRUNCATE, SDLoc(X), VT, X);
18622 AddToWorklist(X.getNode());
18623 }
18624
18625 if (N0.getValueType() == MVT::ppcf128 && !LegalTypes) {
18626 APInt SignBit = APInt::getSignMask(VT.getSizeInBits() / 2);
18627 SDValue Cst = DAG.getBitcast(VT, N0.getOperand(0));
18628 AddToWorklist(Cst.getNode());
18629 SDValue X = DAG.getBitcast(VT, N0.getOperand(1));
18630 AddToWorklist(X.getNode());
18631 SDValue XorResult = DAG.getNode(ISD::XOR, SDLoc(N0), VT, Cst, X);
18632 AddToWorklist(XorResult.getNode());
18633 SDValue XorResult64 = DAG.getNode(
18634 ISD::EXTRACT_ELEMENT, SDLoc(XorResult), MVT::i64, XorResult,
18636 SDLoc(XorResult)));
18637 AddToWorklist(XorResult64.getNode());
18638 SDValue FlipBit =
18639 DAG.getNode(ISD::AND, SDLoc(XorResult64), MVT::i64, XorResult64,
18640 DAG.getConstant(SignBit, SDLoc(XorResult64), MVT::i64));
18641 AddToWorklist(FlipBit.getNode());
18642 SDValue FlipBits =
18643 DAG.getNode(ISD::BUILD_PAIR, SDLoc(N0), VT, FlipBit, FlipBit);
18644 AddToWorklist(FlipBits.getNode());
18645 return DAG.getNode(ISD::XOR, SDLoc(N), VT, Cst, FlipBits);
18646 }
18647 APInt SignBit = APInt::getSignMask(VT.getSizeInBits());
18648 X = DAG.getNode(ISD::AND, SDLoc(X), VT,
18649 X, DAG.getConstant(SignBit, SDLoc(X), VT));
18650 AddToWorklist(X.getNode());
18651
18652 SDValue Cst = DAG.getBitcast(VT, N0.getOperand(0));
18653 Cst = DAG.getNode(ISD::AND, SDLoc(Cst), VT,
18654 Cst, DAG.getConstant(~SignBit, SDLoc(Cst), VT));
18655 AddToWorklist(Cst.getNode());
18656
18657 return DAG.getNode(ISD::OR, SDLoc(N), VT, X, Cst);
18658 }
18659 }
18660
18661 // bitconvert(build_pair(ld, ld)) -> ld iff load locations are consecutive.
18662 if (N0.getOpcode() == ISD::BUILD_PAIR)
18663 if (SDValue CombineLD = CombineConsecutiveLoads(N0.getNode(), VT))
18664 return CombineLD;
18665
18666 // int_vt (bitcast (vec_vt (scalar_to_vector elt_vt:x)))
18667 // => int_vt (any_extend elt_vt:x)
18668 if (DAG.getDataLayout().isLittleEndian() &&
18670 SDValue SrcScalar = N0.getOperand(0);
18671 EVT SrcVT = SrcScalar.getValueType();
18672 if (SrcVT.isScalarInteger() && VT.bitsGT(SrcVT))
18673 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, SrcScalar);
18674 }
18675
18676 // vt (bitcast (scalar_to_vector vt:x)) -> x
18677 if (N0.getOpcode() == ISD::SCALAR_TO_VECTOR &&
18678 N0.getOperand(0).getValueType() == VT)
18679 return N0.getOperand(0);
18680
18681 // Remove double bitcasts from shuffles - this is often a legacy of
18682 // XformToShuffleWithZero being used to combine bitmaskings (of
18683 // float vectors bitcast to integer vectors) into shuffles.
18684 // bitcast(shuffle(bitcast(s0),bitcast(s1))) -> shuffle(s0,s1)
18685 if (Level < AfterLegalizeDAG && TLI.isTypeLegal(VT) && VT.isVector() &&
18686 N0->getOpcode() == ISD::VECTOR_SHUFFLE && N0.hasOneUse() &&
18689 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N0);
18690
18691 // If operands are a bitcast, peek through if it casts the original VT.
18692 // If operands are a constant, just bitcast back to original VT.
18693 auto PeekThroughBitcast = [&](SDValue Op) {
18694 if (Op.getOpcode() == ISD::BITCAST &&
18695 Op.getOperand(0).getValueType() == VT)
18696 return SDValue(Op.getOperand(0));
18697 if (Op.isUndef() || isAnyConstantBuildVector(Op))
18698 return DAG.getBitcast(VT, Op);
18699 return SDValue();
18700 };
18701
18702 // FIXME: If either input vector is bitcast, try to convert the shuffle to
18703 // the result type of this bitcast. This would eliminate at least one
18704 // bitcast. See the transform in InstCombine.
18705 SDValue SV0 = PeekThroughBitcast(N0->getOperand(0));
18706 SDValue SV1 = PeekThroughBitcast(N0->getOperand(1));
18707 if (!(SV0 && SV1))
18708 return SDValue();
18709
18710 int MaskScale =
18712 SmallVector<int, 8> NewMask;
18713 for (int M : SVN->getMask())
18714 for (int i = 0; i != MaskScale; ++i)
18715 NewMask.push_back(M < 0 ? -1 : M * MaskScale + i);
18716
18717 SDValue LegalShuffle =
18718 TLI.buildLegalVectorShuffle(VT, SDLoc(N), SV0, SV1, NewMask, DAG);
18719 if (LegalShuffle)
18720 return LegalShuffle;
18721 }
18722
18723 return SDValue();
18724}
18725
18726SDValue DAGCombiner::visitBUILD_PAIR(SDNode *N) {
18727 EVT VT = N->getValueType(0);
18728 return CombineConsecutiveLoads(N, VT);
18729}
18730
18731SDValue DAGCombiner::visitFREEZE(SDNode *N) {
18732 SDValue N0 = N->getOperand(0);
18733
18735 return N0;
18736
18737 // If we have frozen and unfrozen users of N0, update so everything uses N.
18738 if (!N0.isUndef() && !N0.hasOneUse()) {
18739 SDValue FrozenN0(N, 0);
18740 // Unfreeze all (possibly nested) uses of N to avoid double deleting N from
18741 // the CSE map.
18742 while (!N->use_empty())
18743 DAG.ReplaceAllUsesOfValueWith(FrozenN0, N0);
18744 DAG.ReplaceAllUsesOfValueWith(N0, FrozenN0);
18745 // ReplaceAllUsesOfValueWith will have also updated the use in N, thus
18746 // creating a cycle in a DAG. Let's undo that by mutating the freeze.
18747 assert(N->getOperand(0) == FrozenN0 && "Expected cycle in DAG");
18748 DAG.UpdateNodeOperands(N, N0);
18749 // Revisit the node.
18750 AddToWorklist(N);
18751 return FrozenN0;
18752 }
18753
18754 // We currently avoid folding freeze over SRL, due to the problems seen
18755 // with (freeze (assert ext)) blocking simplifications of SRL. See for
18756 // example https://reviews.llvm.org/D136529#4120959.
18757 if (N0.getOpcode() == ISD::SRL)
18758 return SDValue();
18759
18760 // Fold freeze(op(x, ...)) -> op(freeze(x), ...).
18761 // Try to push freeze through instructions that propagate but don't produce
18762 // poison as far as possible. If an operand of freeze follows three
18763 // conditions 1) one-use, 2) does not produce poison, and 3) has all but one
18764 // guaranteed-non-poison operands (or is a BUILD_VECTOR or similar) then push
18765 // the freeze through to the operands that are not guaranteed non-poison.
18766 // NOTE: we will strip poison-generating flags, so ignore them here.
18768 /*ConsiderFlags*/ false) ||
18769 N0->getNumValues() != 1 || !N0->hasOneUse())
18770 return SDValue();
18771
18772 // TOOD: we should always allow multiple operands, however this increases the
18773 // likelihood of infinite loops due to the ReplaceAllUsesOfValueWith call
18774 // below causing later nodes that share frozen operands to fold again and no
18775 // longer being able to confirm other operands are not poison due to recursion
18776 // depth limits on isGuaranteedNotToBeUndefOrPoison.
18777 bool AllowMultipleMaybePoisonOperands =
18778 N0.getOpcode() == ISD::SELECT_CC || N0.getOpcode() == ISD::SETCC ||
18779 N0.getOpcode() == ISD::BUILD_VECTOR ||
18781 N0.getOpcode() == ISD::BUILD_PAIR ||
18784
18785 // Avoid turning a BUILD_VECTOR that can be recognized as "all zeros", "all
18786 // ones" or "constant" into something that depends on FrozenUndef. We can
18787 // instead pick undef values to keep those properties, while at the same time
18788 // folding away the freeze.
18789 // If we implement a more general solution for folding away freeze(undef) in
18790 // the future, then this special handling can be removed.
18791 if (N0.getOpcode() == ISD::BUILD_VECTOR) {
18792 SDLoc DL(N0);
18793 EVT VT = N0.getValueType();
18795 return DAG.getAllOnesConstant(DL, VT);
18798 for (const SDValue &Op : N0->op_values())
18799 NewVecC.push_back(
18800 Op.isUndef() ? DAG.getConstant(0, DL, Op.getValueType()) : Op);
18801 return DAG.getBuildVector(VT, DL, NewVecC);
18802 }
18803 }
18804
18805 SmallSet<SDValue, 8> MaybePoisonOperands;
18806 SmallVector<unsigned, 8> MaybePoisonOperandNumbers;
18807 for (auto [OpNo, Op] : enumerate(N0->ops())) {
18810 continue;
18811 bool HadMaybePoisonOperands = !MaybePoisonOperands.empty();
18812 bool IsNewMaybePoisonOperand = MaybePoisonOperands.insert(Op).second;
18813 if (IsNewMaybePoisonOperand)
18814 MaybePoisonOperandNumbers.push_back(OpNo);
18815 if (!HadMaybePoisonOperands)
18816 continue;
18817 if (IsNewMaybePoisonOperand && !AllowMultipleMaybePoisonOperands) {
18818 // Multiple maybe-poison ops when not allowed - bail out.
18819 return SDValue();
18820 }
18821 }
18822 // NOTE: the whole op may be not guaranteed to not be undef or poison because
18823 // it could create undef or poison due to it's poison-generating flags.
18824 // So not finding any maybe-poison operands is fine.
18825
18826 for (unsigned OpNo : MaybePoisonOperandNumbers) {
18827 // N0 can mutate during iteration, so make sure to refetch the maybe poison
18828 // operands via the operand numbers. The typical scenario is that we have
18829 // something like this
18830 // t262: i32 = freeze t181
18831 // t150: i32 = ctlz_zero_poison t262
18832 // t184: i32 = ctlz_zero_poison t181
18833 // t268: i32 = select_cc t181, Constant:i32<0>, t184, t186, setne:ch
18834 // When freezing the t181 operand we get t262 back, and then the
18835 // ReplaceAllUsesOfValueWith call will not only replace t181 by t262, but
18836 // also recursively replace t184 by t150.
18837 SDValue MaybePoisonOperand = N->getOperand(0).getOperand(OpNo);
18838 // Don't replace every single UNDEF everywhere with frozen UNDEF, though.
18839 if (MaybePoisonOperand.isUndef())
18840 continue;
18841 // First, freeze each offending operand.
18842 SDValue FrozenMaybePoisonOperand = DAG.getFreeze(MaybePoisonOperand);
18843 // Then, change all other uses of unfrozen operand to use frozen operand.
18844 DAG.ReplaceAllUsesOfValueWith(MaybePoisonOperand, FrozenMaybePoisonOperand);
18845 if (FrozenMaybePoisonOperand.getOpcode() == ISD::FREEZE &&
18846 FrozenMaybePoisonOperand.getOperand(0) == FrozenMaybePoisonOperand) {
18847 // But, that also updated the use in the freeze we just created, thus
18848 // creating a cycle in a DAG. Let's undo that by mutating the freeze.
18849 DAG.UpdateNodeOperands(FrozenMaybePoisonOperand.getNode(),
18850 MaybePoisonOperand);
18851 }
18852
18853 // This node has been merged with another.
18854 if (N->getOpcode() == ISD::DELETED_NODE)
18855 return SDValue(N, 0);
18856 }
18857
18858 assert(N->getOpcode() != ISD::DELETED_NODE && "Node was deleted!");
18859
18860 // The whole node may have been updated, so the value we were holding
18861 // may no longer be valid. Re-fetch the operand we're `freeze`ing.
18862 N0 = N->getOperand(0);
18863
18864 // Finally, recreate the node, it's operands were updated to use
18865 // frozen operands, so we just need to use it's "original" operands.
18867 // TODO: ISD::UNDEF and ISD::POISON should get separate handling, but best
18868 // leave for a future patch.
18869 for (SDValue &Op : Ops) {
18870 if (Op.isUndef())
18871 Op = DAG.getFreeze(Op);
18872 }
18873
18874 SDLoc DL(N0);
18875
18876 // Special case handling for ShuffleVectorSDNode nodes.
18877 if (auto *SVN = dyn_cast<ShuffleVectorSDNode>(N0))
18878 return DAG.getVectorShuffle(N0.getValueType(), DL, Ops[0], Ops[1],
18879 SVN->getMask());
18880
18881 // NOTE: this strips poison generating flags.
18882 // Folding freeze(op(x, ...)) -> op(freeze(x), ...) does not require nnan,
18883 // ninf, nsz, or fast.
18884 // However, contract, reassoc, afn, and arcp should be preserved,
18885 // as these fast-math flags do not introduce poison values.
18886 SDNodeFlags SrcFlags = N0->getFlags();
18887 SDNodeFlags SafeFlags;
18888 SafeFlags.setAllowContract(SrcFlags.hasAllowContract());
18889 SafeFlags.setAllowReassociation(SrcFlags.hasAllowReassociation());
18890 SafeFlags.setApproximateFuncs(SrcFlags.hasApproximateFuncs());
18891 SafeFlags.setAllowReciprocal(SrcFlags.hasAllowReciprocal());
18892 return DAG.getNode(N0.getOpcode(), DL, N0->getVTList(), Ops, SafeFlags);
18893}
18894
18895// Returns true if floating point contraction is allowed on the FMUL-SDValue
18896// `N`
18898 assert(N.getOpcode() == ISD::FMUL);
18899
18900 return N->getFlags().hasAllowContract();
18901}
18902
18903/// Try to perform FMA combining on a given FADD node.
18904SDValue DAGCombiner::visitFADDForFMACombine(SDNode *N) {
18905 SDValue N0 = N->getOperand(0);
18906 SDValue N1 = N->getOperand(1);
18907 EVT VT = N->getValueType(0);
18908 SDLoc SL(N);
18909
18910 // Floating-point multiply-add with intermediate rounding.
18911 bool HasFMAD = (LegalOperations && TLI.isFMADLegal(DAG, N));
18912
18913 // Floating-point multiply-add without intermediate rounding.
18914 bool HasFMA =
18915 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FMA, VT)) &&
18917
18918 // No valid opcode, do not combine.
18919 if (!HasFMAD && !HasFMA)
18920 return SDValue();
18921
18922 // FMAD (with intermediate rounding) is always safe to form; FMA requires the
18923 // contract fast-math flag.
18924 bool AllowFusionGlobally = HasFMAD;
18925 // If the addition is not contractable, do not combine.
18926 if (!AllowFusionGlobally && !N->getFlags().hasAllowContract())
18927 return SDValue();
18928
18929 // Folding fadd (fmul x, y), (fmul x, y) -> fma x, y, (fmul x, y) is never
18930 // beneficial. It does not reduce latency. It increases register pressure. It
18931 // replaces an fadd with an fma which is a more complex instruction, so is
18932 // likely to have a larger encoding, use more functional units, etc.
18933 if (N0 == N1)
18934 return SDValue();
18935
18936 if (TLI.generateFMAsInMachineCombiner(VT, OptLevel))
18937 return SDValue();
18938
18939 // Always prefer FMAD to FMA for precision.
18940 unsigned PreferredFusedOpcode = HasFMAD ? ISD::FMAD : ISD::FMA;
18942
18943 auto isFusedOp = [&](SDValue N) {
18944 unsigned Opcode = N.getOpcode();
18945 return Opcode == ISD::FMA || Opcode == ISD::FMAD;
18946 };
18947
18948 // Is the node an FMUL and contractable either due to global flags or
18949 // SDNodeFlags.
18950 auto isContractableFMUL = [AllowFusionGlobally](SDValue N) {
18951 if (N.getOpcode() != ISD::FMUL)
18952 return false;
18953 return AllowFusionGlobally || N->getFlags().hasAllowContract();
18954 };
18955 // If we have two choices trying to fold (fadd (fmul u, v), (fmul x, y)),
18956 // prefer to fold the multiply with fewer uses.
18958 if (N0->use_size() > N1->use_size())
18959 std::swap(N0, N1);
18960 }
18961
18962 // fold (fadd (fmul x, y), z) -> (fma x, y, z)
18963 if (isContractableFMUL(N0) && (Aggressive || N0->hasOneUse())) {
18964 return DAG.getNode(PreferredFusedOpcode, SL, VT, N0.getOperand(0),
18965 N0.getOperand(1), N1);
18966 }
18967
18968 // fold (fadd x, (fmul y, z)) -> (fma y, z, x)
18969 // Note: Commutes FADD operands.
18970 if (isContractableFMUL(N1) && (Aggressive || N1->hasOneUse())) {
18971 return DAG.getNode(PreferredFusedOpcode, SL, VT, N1.getOperand(0),
18972 N1.getOperand(1), N0);
18973 }
18974
18975 // fadd (fma A, B, (fmul C, D)), E --> fma A, B, (fma C, D, E)
18976 // fadd E, (fma A, B, (fmul C, D)) --> fma A, B, (fma C, D, E)
18977 // This also works with nested fma instructions:
18978 // fadd (fma A, B, (fma (C, D, (fmul (E, F))))), G -->
18979 // fma A, B, (fma C, D, fma (E, F, G))
18980 // fadd (G, (fma A, B, (fma (C, D, (fmul (E, F)))))) -->
18981 // fma A, B, (fma C, D, fma (E, F, G)).
18982 // This requires reassociation because it changes the order of operations.
18983 bool CanReassociate = N->getFlags().hasAllowReassociation();
18984 if (CanReassociate) {
18985 SDValue FMA, E;
18986 if (isFusedOp(N0) && N0.hasOneUse()) {
18987 FMA = N0;
18988 E = N1;
18989 } else if (isFusedOp(N1) && N1.hasOneUse()) {
18990 FMA = N1;
18991 E = N0;
18992 }
18993
18994 SDValue TmpFMA = FMA;
18995 while (E && isFusedOp(TmpFMA) && TmpFMA.hasOneUse()) {
18996 SDValue FMul = TmpFMA->getOperand(2);
18997 if (FMul.getOpcode() == ISD::FMUL && FMul.hasOneUse()) {
18998 SDValue C = FMul.getOperand(0);
18999 SDValue D = FMul.getOperand(1);
19000 SDValue CDE = DAG.getNode(PreferredFusedOpcode, SL, VT, C, D, E);
19002 // Replacing the inner FMul could cause the outer FMA to be simplified
19003 // away.
19004 return FMA.getOpcode() == ISD::DELETED_NODE ? SDValue(N, 0) : FMA;
19005 }
19006
19007 TmpFMA = TmpFMA->getOperand(2);
19008 }
19009 }
19010
19011 // Look through FP_EXTEND nodes to do more combining.
19012
19013 // fold (fadd (fpext (fmul x, y)), z) -> (fma (fpext x), (fpext y), z)
19014 if (N0.getOpcode() == ISD::FP_EXTEND) {
19015 SDValue N00 = N0.getOperand(0);
19016 if (isContractableFMUL(N00) &&
19017 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
19018 N00.getValueType())) {
19019 return DAG.getNode(PreferredFusedOpcode, SL, VT,
19020 DAG.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(0)),
19021 DAG.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(1)),
19022 N1);
19023 }
19024 }
19025
19026 // fold (fadd x, (fpext (fmul y, z))) -> (fma (fpext y), (fpext z), x)
19027 // Note: Commutes FADD operands.
19028 if (N1.getOpcode() == ISD::FP_EXTEND) {
19029 SDValue N10 = N1.getOperand(0);
19030 if (isContractableFMUL(N10) &&
19031 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
19032 N10.getValueType())) {
19033 return DAG.getNode(PreferredFusedOpcode, SL, VT,
19034 DAG.getNode(ISD::FP_EXTEND, SL, VT, N10.getOperand(0)),
19035 DAG.getNode(ISD::FP_EXTEND, SL, VT, N10.getOperand(1)),
19036 N0);
19037 }
19038 }
19039
19040 // More folding opportunities when target permits.
19041 if (Aggressive) {
19042 // fold (fadd (fma x, y, (fpext (fmul u, v))), z)
19043 // -> (fma x, y, (fma (fpext u), (fpext v), z))
19044 auto FoldFAddFMAFPExtFMul = [&](SDValue X, SDValue Y, SDValue U, SDValue V,
19045 SDValue Z) {
19046 return DAG.getNode(PreferredFusedOpcode, SL, VT, X, Y,
19047 DAG.getNode(PreferredFusedOpcode, SL, VT,
19048 DAG.getNode(ISD::FP_EXTEND, SL, VT, U),
19049 DAG.getNode(ISD::FP_EXTEND, SL, VT, V),
19050 Z));
19051 };
19052 if (isFusedOp(N0)) {
19053 SDValue N02 = N0.getOperand(2);
19054 if (N02.getOpcode() == ISD::FP_EXTEND) {
19055 SDValue N020 = N02.getOperand(0);
19056 if (isContractableFMUL(N020) &&
19057 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
19058 N020.getValueType())) {
19059 return FoldFAddFMAFPExtFMul(N0.getOperand(0), N0.getOperand(1),
19060 N020.getOperand(0), N020.getOperand(1),
19061 N1);
19062 }
19063 }
19064 }
19065
19066 // fold (fadd (fpext (fma x, y, (fmul u, v))), z)
19067 // -> (fma (fpext x), (fpext y), (fma (fpext u), (fpext v), z))
19068 // FIXME: This turns two single-precision and one double-precision
19069 // operation into two double-precision operations, which might not be
19070 // interesting for all targets, especially GPUs.
19071 auto FoldFAddFPExtFMAFMul = [&](SDValue X, SDValue Y, SDValue U, SDValue V,
19072 SDValue Z) {
19073 return DAG.getNode(
19074 PreferredFusedOpcode, SL, VT, DAG.getNode(ISD::FP_EXTEND, SL, VT, X),
19075 DAG.getNode(ISD::FP_EXTEND, SL, VT, Y),
19076 DAG.getNode(PreferredFusedOpcode, SL, VT,
19077 DAG.getNode(ISD::FP_EXTEND, SL, VT, U),
19078 DAG.getNode(ISD::FP_EXTEND, SL, VT, V), Z));
19079 };
19080 if (N0.getOpcode() == ISD::FP_EXTEND) {
19081 SDValue N00 = N0.getOperand(0);
19082 if (isFusedOp(N00)) {
19083 SDValue N002 = N00.getOperand(2);
19084 if (isContractableFMUL(N002) &&
19085 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
19086 N00.getValueType())) {
19087 return FoldFAddFPExtFMAFMul(N00.getOperand(0), N00.getOperand(1),
19088 N002.getOperand(0), N002.getOperand(1),
19089 N1);
19090 }
19091 }
19092 }
19093
19094 // fold (fadd x, (fma y, z, (fpext (fmul u, v)))
19095 // -> (fma y, z, (fma (fpext u), (fpext v), x))
19096 if (isFusedOp(N1)) {
19097 SDValue N12 = N1.getOperand(2);
19098 if (N12.getOpcode() == ISD::FP_EXTEND) {
19099 SDValue N120 = N12.getOperand(0);
19100 if (isContractableFMUL(N120) &&
19101 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
19102 N120.getValueType())) {
19103 return FoldFAddFMAFPExtFMul(N1.getOperand(0), N1.getOperand(1),
19104 N120.getOperand(0), N120.getOperand(1),
19105 N0);
19106 }
19107 }
19108 }
19109
19110 // fold (fadd x, (fpext (fma y, z, (fmul u, v)))
19111 // -> (fma (fpext y), (fpext z), (fma (fpext u), (fpext v), x))
19112 // FIXME: This turns two single-precision and one double-precision
19113 // operation into two double-precision operations, which might not be
19114 // interesting for all targets, especially GPUs.
19115 if (N1.getOpcode() == ISD::FP_EXTEND) {
19116 SDValue N10 = N1.getOperand(0);
19117 if (isFusedOp(N10)) {
19118 SDValue N102 = N10.getOperand(2);
19119 if (isContractableFMUL(N102) &&
19120 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
19121 N10.getValueType())) {
19122 return FoldFAddFPExtFMAFMul(N10.getOperand(0), N10.getOperand(1),
19123 N102.getOperand(0), N102.getOperand(1),
19124 N0);
19125 }
19126 }
19127 }
19128 }
19129
19130 return SDValue();
19131}
19132
19133/// Try to perform FMA combining on a given FSUB node.
19134SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
19135 SDValue N0 = N->getOperand(0);
19136 SDValue N1 = N->getOperand(1);
19137 EVT VT = N->getValueType(0);
19138 SDLoc SL(N);
19139
19140 // Floating-point multiply-add with intermediate rounding.
19141 bool HasFMAD = (LegalOperations && TLI.isFMADLegal(DAG, N));
19142
19143 // Floating-point multiply-add without intermediate rounding.
19144 bool HasFMA =
19145 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FMA, VT)) &&
19147
19148 // No valid opcode, do not combine.
19149 if (!HasFMAD && !HasFMA)
19150 return SDValue();
19151
19152 const SDNodeFlags Flags = N->getFlags();
19153 // FMAD (with intermediate rounding) is always safe to form; FMA requires the
19154 // contract fast-math flag.
19155 bool AllowFusionGlobally = HasFMAD;
19156
19157 // If the subtraction is not contractable, do not combine.
19158 if (!AllowFusionGlobally && !N->getFlags().hasAllowContract())
19159 return SDValue();
19160
19161 if (TLI.generateFMAsInMachineCombiner(VT, OptLevel))
19162 return SDValue();
19163
19164 // Always prefer FMAD to FMA for precision.
19165 unsigned PreferredFusedOpcode = HasFMAD ? ISD::FMAD : ISD::FMA;
19167 bool NoSignedZero = Flags.hasNoSignedZeros();
19168
19169 // Is the node an FMUL and contractable either due to global flags or
19170 // SDNodeFlags.
19171 auto isContractableFMUL = [AllowFusionGlobally](SDValue N) {
19172 if (N.getOpcode() != ISD::FMUL)
19173 return false;
19174 return AllowFusionGlobally || N->getFlags().hasAllowContract();
19175 };
19176
19177 // fold (fsub (fmul x, y), z) -> (fma x, y, (fneg z))
19178 auto tryToFoldXYSubZ = [&](SDValue XY, SDValue Z) {
19179 if (isContractableFMUL(XY) && (Aggressive || XY->hasOneUse())) {
19180 return DAG.getNode(PreferredFusedOpcode, SL, VT, XY.getOperand(0),
19181 XY.getOperand(1), DAG.getNode(ISD::FNEG, SL, VT, Z));
19182 }
19183 return SDValue();
19184 };
19185
19186 // fold (fsub x, (fmul y, z)) -> (fma (fneg y), z, x)
19187 // Note: Commutes FSUB operands.
19188 auto tryToFoldXSubYZ = [&](SDValue X, SDValue YZ) {
19189 if (isContractableFMUL(YZ) && (Aggressive || YZ->hasOneUse())) {
19190 return DAG.getNode(PreferredFusedOpcode, SL, VT,
19191 DAG.getNode(ISD::FNEG, SL, VT, YZ.getOperand(0)),
19192 YZ.getOperand(1), X);
19193 }
19194 return SDValue();
19195 };
19196
19197 // If we have two choices trying to fold (fsub (fmul u, v), (fmul x, y)),
19198 // prefer to fold the multiply with fewer uses.
19199 if (isContractableFMUL(N0) && isContractableFMUL(N1) &&
19200 (N0->use_size() > N1->use_size())) {
19201 // fold (fsub (fmul a, b), (fmul c, d)) -> (fma (fneg c), d, (fmul a, b))
19202 if (SDValue V = tryToFoldXSubYZ(N0, N1))
19203 return V;
19204 // fold (fsub (fmul a, b), (fmul c, d)) -> (fma a, b, (fneg (fmul c, d)))
19205 if (SDValue V = tryToFoldXYSubZ(N0, N1))
19206 return V;
19207 } else {
19208 // fold (fsub (fmul x, y), z) -> (fma x, y, (fneg z))
19209 if (SDValue V = tryToFoldXYSubZ(N0, N1))
19210 return V;
19211 // fold (fsub x, (fmul y, z)) -> (fma (fneg y), z, x)
19212 if (SDValue V = tryToFoldXSubYZ(N0, N1))
19213 return V;
19214 }
19215
19216 // fold (fsub (fneg (fmul, x, y)), z) -> (fma (fneg x), y, (fneg z))
19217 if (N0.getOpcode() == ISD::FNEG && isContractableFMUL(N0.getOperand(0)) &&
19218 (Aggressive || (N0->hasOneUse() && N0.getOperand(0).hasOneUse()))) {
19219 SDValue N00 = N0.getOperand(0).getOperand(0);
19220 SDValue N01 = N0.getOperand(0).getOperand(1);
19221 return DAG.getNode(PreferredFusedOpcode, SL, VT,
19222 DAG.getNode(ISD::FNEG, SL, VT, N00), N01,
19223 DAG.getNode(ISD::FNEG, SL, VT, N1));
19224 }
19225
19226 // Look through FP_EXTEND nodes to do more combining.
19227
19228 // fold (fsub (fpext (fmul x, y)), z)
19229 // -> (fma (fpext x), (fpext y), (fneg z))
19230 if (N0.getOpcode() == ISD::FP_EXTEND) {
19231 SDValue N00 = N0.getOperand(0);
19232 if (isContractableFMUL(N00) &&
19233 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
19234 N00.getValueType())) {
19235 return DAG.getNode(PreferredFusedOpcode, SL, VT,
19236 DAG.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(0)),
19237 DAG.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(1)),
19238 DAG.getNode(ISD::FNEG, SL, VT, N1));
19239 }
19240 }
19241
19242 // fold (fsub x, (fpext (fmul y, z)))
19243 // -> (fma (fneg (fpext y)), (fpext z), x)
19244 // Note: Commutes FSUB operands.
19245 if (N1.getOpcode() == ISD::FP_EXTEND) {
19246 SDValue N10 = N1.getOperand(0);
19247 if (isContractableFMUL(N10) &&
19248 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
19249 N10.getValueType())) {
19250 return DAG.getNode(
19251 PreferredFusedOpcode, SL, VT,
19252 DAG.getNode(ISD::FNEG, SL, VT,
19253 DAG.getNode(ISD::FP_EXTEND, SL, VT, N10.getOperand(0))),
19254 DAG.getNode(ISD::FP_EXTEND, SL, VT, N10.getOperand(1)), N0);
19255 }
19256 }
19257
19258 // fold (fsub (fpext (fneg (fmul, x, y))), z)
19259 // -> (fneg (fma (fpext x), (fpext y), z))
19260 // Note: This could be removed with appropriate canonicalization of the
19261 // input expression into (fneg (fadd (fpext (fmul, x, y)), z)). However, the
19262 // command line flag -fp-contract=fast and fast-math flag contract prevent
19263 // from implementing the canonicalization in visitFSUB.
19264 if (N0.getOpcode() == ISD::FP_EXTEND) {
19265 SDValue N00 = N0.getOperand(0);
19266 if (N00.getOpcode() == ISD::FNEG) {
19267 SDValue N000 = N00.getOperand(0);
19268 if (isContractableFMUL(N000) &&
19269 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
19270 N00.getValueType())) {
19271 return DAG.getNode(
19272 ISD::FNEG, SL, VT,
19273 DAG.getNode(PreferredFusedOpcode, SL, VT,
19274 DAG.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(0)),
19275 DAG.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(1)),
19276 N1));
19277 }
19278 }
19279 }
19280
19281 // fold (fsub (fneg (fpext (fmul, x, y))), z)
19282 // -> (fneg (fma (fpext x)), (fpext y), z)
19283 // Note: This could be removed with appropriate canonicalization of the
19284 // input expression into (fneg (fadd (fpext (fmul, x, y)), z). However, the
19285 // command line flag -fp-contract=fast and fast-math flag contract prevent
19286 // from implementing the canonicalization in visitFSUB.
19287 if (N0.getOpcode() == ISD::FNEG) {
19288 SDValue N00 = N0.getOperand(0);
19289 if (N00.getOpcode() == ISD::FP_EXTEND) {
19290 SDValue N000 = N00.getOperand(0);
19291 if (isContractableFMUL(N000) &&
19292 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
19293 N000.getValueType())) {
19294 return DAG.getNode(
19295 ISD::FNEG, SL, VT,
19296 DAG.getNode(PreferredFusedOpcode, SL, VT,
19297 DAG.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(0)),
19298 DAG.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(1)),
19299 N1));
19300 }
19301 }
19302 }
19303
19304 auto isContractableAndReassociableFMUL = [&isContractableFMUL](SDValue N) {
19305 return isContractableFMUL(N) && N->getFlags().hasAllowReassociation();
19306 };
19307
19308 auto isFusedOp = [&](SDValue N) {
19309 unsigned Opcode = N.getOpcode();
19310 return Opcode == ISD::FMA || Opcode == ISD::FMAD;
19311 };
19312
19313 // More folding opportunities when target permits.
19314 if (Aggressive && N->getFlags().hasAllowReassociation()) {
19315 bool CanFuse = N->getFlags().hasAllowContract();
19316 // fold (fsub (fma x, y, (fmul u, v)), z)
19317 // -> (fma x, y (fma u, v, (fneg z)))
19318 if (CanFuse && isFusedOp(N0) &&
19319 isContractableAndReassociableFMUL(N0.getOperand(2)) &&
19320 N0->hasOneUse() && N0.getOperand(2)->hasOneUse()) {
19321 return DAG.getNode(PreferredFusedOpcode, SL, VT, N0.getOperand(0),
19322 N0.getOperand(1),
19323 DAG.getNode(PreferredFusedOpcode, SL, VT,
19324 N0.getOperand(2).getOperand(0),
19325 N0.getOperand(2).getOperand(1),
19326 DAG.getNode(ISD::FNEG, SL, VT, N1)));
19327 }
19328
19329 // fold (fsub x, (fma y, z, (fmul u, v)))
19330 // -> (fma (fneg y), z, (fma (fneg u), v, x))
19331 if (CanFuse && isFusedOp(N1) &&
19332 isContractableAndReassociableFMUL(N1.getOperand(2)) &&
19333 N1->hasOneUse() && NoSignedZero) {
19334 SDValue N20 = N1.getOperand(2).getOperand(0);
19335 SDValue N21 = N1.getOperand(2).getOperand(1);
19336 return DAG.getNode(
19337 PreferredFusedOpcode, SL, VT,
19338 DAG.getNode(ISD::FNEG, SL, VT, N1.getOperand(0)), N1.getOperand(1),
19339 DAG.getNode(PreferredFusedOpcode, SL, VT,
19340 DAG.getNode(ISD::FNEG, SL, VT, N20), N21, N0));
19341 }
19342
19343 // fold (fsub (fma x, y, (fpext (fmul u, v))), z)
19344 // -> (fma x, y (fma (fpext u), (fpext v), (fneg z)))
19345 if (isFusedOp(N0) && N0->hasOneUse()) {
19346 SDValue N02 = N0.getOperand(2);
19347 if (N02.getOpcode() == ISD::FP_EXTEND) {
19348 SDValue N020 = N02.getOperand(0);
19349 if (isContractableAndReassociableFMUL(N020) &&
19350 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
19351 N020.getValueType())) {
19352 return DAG.getNode(
19353 PreferredFusedOpcode, SL, VT, N0.getOperand(0), N0.getOperand(1),
19354 DAG.getNode(
19355 PreferredFusedOpcode, SL, VT,
19356 DAG.getNode(ISD::FP_EXTEND, SL, VT, N020.getOperand(0)),
19357 DAG.getNode(ISD::FP_EXTEND, SL, VT, N020.getOperand(1)),
19358 DAG.getNode(ISD::FNEG, SL, VT, N1)));
19359 }
19360 }
19361 }
19362
19363 // fold (fsub (fpext (fma x, y, (fmul u, v))), z)
19364 // -> (fma (fpext x), (fpext y),
19365 // (fma (fpext u), (fpext v), (fneg z)))
19366 // FIXME: This turns two single-precision and one double-precision
19367 // operation into two double-precision operations, which might not be
19368 // interesting for all targets, especially GPUs.
19369 if (N0.getOpcode() == ISD::FP_EXTEND) {
19370 SDValue N00 = N0.getOperand(0);
19371 if (isFusedOp(N00)) {
19372 SDValue N002 = N00.getOperand(2);
19373 if (isContractableAndReassociableFMUL(N002) &&
19374 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
19375 N00.getValueType())) {
19376 return DAG.getNode(
19377 PreferredFusedOpcode, SL, VT,
19378 DAG.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(0)),
19379 DAG.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(1)),
19380 DAG.getNode(
19381 PreferredFusedOpcode, SL, VT,
19382 DAG.getNode(ISD::FP_EXTEND, SL, VT, N002.getOperand(0)),
19383 DAG.getNode(ISD::FP_EXTEND, SL, VT, N002.getOperand(1)),
19384 DAG.getNode(ISD::FNEG, SL, VT, N1)));
19385 }
19386 }
19387 }
19388
19389 // fold (fsub x, (fma y, z, (fpext (fmul u, v))))
19390 // -> (fma (fneg y), z, (fma (fneg (fpext u)), (fpext v), x))
19391 if (isFusedOp(N1) && N1.getOperand(2).getOpcode() == ISD::FP_EXTEND &&
19392 N1->hasOneUse()) {
19393 SDValue N120 = N1.getOperand(2).getOperand(0);
19394 if (isContractableAndReassociableFMUL(N120) &&
19395 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
19396 N120.getValueType())) {
19397 SDValue N1200 = N120.getOperand(0);
19398 SDValue N1201 = N120.getOperand(1);
19399 return DAG.getNode(
19400 PreferredFusedOpcode, SL, VT,
19401 DAG.getNode(ISD::FNEG, SL, VT, N1.getOperand(0)), N1.getOperand(1),
19402 DAG.getNode(PreferredFusedOpcode, SL, VT,
19403 DAG.getNode(ISD::FNEG, SL, VT,
19404 DAG.getNode(ISD::FP_EXTEND, SL, VT, N1200)),
19405 DAG.getNode(ISD::FP_EXTEND, SL, VT, N1201), N0));
19406 }
19407 }
19408
19409 // fold (fsub x, (fpext (fma y, z, (fmul u, v))))
19410 // -> (fma (fneg (fpext y)), (fpext z),
19411 // (fma (fneg (fpext u)), (fpext v), x))
19412 // FIXME: This turns two single-precision and one double-precision
19413 // operation into two double-precision operations, which might not be
19414 // interesting for all targets, especially GPUs.
19415 if (N1.getOpcode() == ISD::FP_EXTEND && isFusedOp(N1.getOperand(0))) {
19416 SDValue CvtSrc = N1.getOperand(0);
19417 SDValue N100 = CvtSrc.getOperand(0);
19418 SDValue N101 = CvtSrc.getOperand(1);
19419 SDValue N102 = CvtSrc.getOperand(2);
19420 if (isContractableAndReassociableFMUL(N102) &&
19421 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
19422 CvtSrc.getValueType())) {
19423 SDValue N1020 = N102.getOperand(0);
19424 SDValue N1021 = N102.getOperand(1);
19425 return DAG.getNode(
19426 PreferredFusedOpcode, SL, VT,
19427 DAG.getNode(ISD::FNEG, SL, VT,
19428 DAG.getNode(ISD::FP_EXTEND, SL, VT, N100)),
19429 DAG.getNode(ISD::FP_EXTEND, SL, VT, N101),
19430 DAG.getNode(PreferredFusedOpcode, SL, VT,
19431 DAG.getNode(ISD::FNEG, SL, VT,
19432 DAG.getNode(ISD::FP_EXTEND, SL, VT, N1020)),
19433 DAG.getNode(ISD::FP_EXTEND, SL, VT, N1021), N0));
19434 }
19435 }
19436 }
19437
19438 return SDValue();
19439}
19440
19441/// Try to perform FMA combining on a given FMUL node based on the distributive
19442/// law x * (y + 1) = x * y + x and variants thereof (commuted versions,
19443/// subtraction instead of addition).
19444SDValue DAGCombiner::visitFMULForFMADistributiveCombine(SDNode *N) {
19445 SDValue N0 = N->getOperand(0);
19446 SDValue N1 = N->getOperand(1);
19447 EVT VT = N->getValueType(0);
19448 SDLoc SL(N);
19449
19450 assert(N->getOpcode() == ISD::FMUL && "Expected FMUL Operation");
19451
19452 // The transforms below are incorrect when x == 0 and y == inf, because the
19453 // intermediate multiplication produces a nan.
19454 SDValue FAdd = N0.getOpcode() == ISD::FADD ? N0 : N1;
19455 if (!FAdd->getFlags().hasNoInfs())
19456 return SDValue();
19457
19458 // Floating-point multiply-add without intermediate rounding.
19459 bool HasFMA =
19461 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FMA, VT)) &&
19463
19464 // Floating-point multiply-add with intermediate rounding. This can result
19465 // in a less precise result due to the changed rounding order.
19466 bool HasFMAD = LegalOperations && TLI.isFMADLegal(DAG, N);
19467
19468 // No valid opcode, do not combine.
19469 if (!HasFMAD && !HasFMA)
19470 return SDValue();
19471
19472 // Always prefer FMAD to FMA for precision.
19473 unsigned PreferredFusedOpcode = HasFMAD ? ISD::FMAD : ISD::FMA;
19475
19476 // fold (fmul (fadd x0, +1.0), y) -> (fma x0, y, y)
19477 // fold (fmul (fadd x0, -1.0), y) -> (fma x0, y, (fneg y))
19478 auto FuseFADD = [&](SDValue X, SDValue Y) {
19479 if (X.getOpcode() == ISD::FADD && (Aggressive || X->hasOneUse())) {
19480 if (auto *C = isConstOrConstSplatFP(X.getOperand(1), true)) {
19481 if (C->isOne())
19482 return DAG.getNode(PreferredFusedOpcode, SL, VT, X.getOperand(0), Y,
19483 Y);
19484 if (C->isMinusOne())
19485 return DAG.getNode(PreferredFusedOpcode, SL, VT, X.getOperand(0), Y,
19486 DAG.getNode(ISD::FNEG, SL, VT, Y));
19487 }
19488 }
19489 return SDValue();
19490 };
19491
19492 if (SDValue FMA = FuseFADD(N0, N1))
19493 return FMA;
19494 if (SDValue FMA = FuseFADD(N1, N0))
19495 return FMA;
19496
19497 // fold (fmul (fsub +1.0, x1), y) -> (fma (fneg x1), y, y)
19498 // fold (fmul (fsub -1.0, x1), y) -> (fma (fneg x1), y, (fneg y))
19499 // fold (fmul (fsub x0, +1.0), y) -> (fma x0, y, (fneg y))
19500 // fold (fmul (fsub x0, -1.0), y) -> (fma x0, y, y)
19501 auto FuseFSUB = [&](SDValue X, SDValue Y) {
19502 if (X.getOpcode() == ISD::FSUB && (Aggressive || X->hasOneUse())) {
19503 if (auto *C0 = isConstOrConstSplatFP(X.getOperand(0), true)) {
19504 if (C0->isOne())
19505 return DAG.getNode(PreferredFusedOpcode, SL, VT,
19506 DAG.getNode(ISD::FNEG, SL, VT, X.getOperand(1)), Y,
19507 Y);
19508 if (C0->isMinusOne())
19509 return DAG.getNode(PreferredFusedOpcode, SL, VT,
19510 DAG.getNode(ISD::FNEG, SL, VT, X.getOperand(1)), Y,
19511 DAG.getNode(ISD::FNEG, SL, VT, Y));
19512 }
19513 if (auto *C1 = isConstOrConstSplatFP(X.getOperand(1), true)) {
19514 if (C1->isOne())
19515 return DAG.getNode(PreferredFusedOpcode, SL, VT, X.getOperand(0), Y,
19516 DAG.getNode(ISD::FNEG, SL, VT, Y));
19517 if (C1->isMinusOne())
19518 return DAG.getNode(PreferredFusedOpcode, SL, VT, X.getOperand(0), Y,
19519 Y);
19520 }
19521 }
19522 return SDValue();
19523 };
19524
19525 if (SDValue FMA = FuseFSUB(N0, N1))
19526 return FMA;
19527 if (SDValue FMA = FuseFSUB(N1, N0))
19528 return FMA;
19529
19530 return SDValue();
19531}
19532
19533SDValue DAGCombiner::visitFADD(SDNode *N) {
19534 SDValue N0 = N->getOperand(0);
19535 SDValue N1 = N->getOperand(1);
19536 bool N0CFP = DAG.isConstantFPBuildVectorOrConstantFP(N0);
19537 bool N1CFP = DAG.isConstantFPBuildVectorOrConstantFP(N1);
19538 EVT VT = N->getValueType(0);
19539 SDLoc DL(N);
19540 SDNodeFlags Flags = N->getFlags();
19541 SelectionDAG::FlagInserter FlagsInserter(DAG, N);
19542
19543 if (SDValue R = DAG.simplifyFPBinop(N->getOpcode(), N0, N1, Flags))
19544 return R;
19545
19546 // fold (fadd c1, c2) -> c1 + c2
19547 if (SDValue C =
19548 DAG.FoldConstantArithmetic(ISD::FADD, DL, VT, {N0, N1}, Flags))
19549 return C;
19550
19551 // canonicalize constant to RHS
19552 if (N0CFP && !N1CFP)
19553 return DAG.getNode(ISD::FADD, DL, VT, N1, N0);
19554
19555 // fold vector ops
19556 if (VT.isVector())
19557 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
19558 return FoldedVOp;
19559
19560 // N0 + -0.0 --> N0 (also allowed with +0.0 and fast-math)
19561 ConstantFPSDNode *N1C = isConstOrConstSplatFP(N1, true);
19562 if (N1C && N1C->isZero())
19563 if (N1C->isNegative() || Flags.hasNoSignedZeros() ||
19565 return N0;
19566
19567 if (SDValue NewSel = foldBinOpIntoSelect(N))
19568 return NewSel;
19569
19570 // fold (fadd A, (fneg B)) -> (fsub A, B)
19571 if (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FSUB, VT))
19573 N1, DAG, LegalOperations, ForCodeSize))
19574 return DAG.getNode(ISD::FSUB, DL, VT, N0, NegN1);
19575
19576 // fold (fadd (fneg A), B) -> (fsub B, A)
19577 if (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FSUB, VT))
19579 N0, DAG, LegalOperations, ForCodeSize))
19580 return DAG.getNode(ISD::FSUB, DL, VT, N1, NegN0);
19581
19582 auto isFMulNegTwo = [](SDValue FMul) {
19583 if (!FMul.hasOneUse() || FMul.getOpcode() != ISD::FMUL)
19584 return false;
19585 auto *C = isConstOrConstSplatFP(FMul.getOperand(1), true);
19586 return C && C->isExactlyValue(-2.0);
19587 };
19588
19589 // fadd (fmul B, -2.0), A --> fsub A, (fadd B, B)
19590 if (isFMulNegTwo(N0)) {
19591 SDValue B = N0.getOperand(0);
19592 SDValue Add = DAG.getNode(ISD::FADD, DL, VT, B, B);
19593 return DAG.getNode(ISD::FSUB, DL, VT, N1, Add);
19594 }
19595 // fadd A, (fmul B, -2.0) --> fsub A, (fadd B, B)
19596 if (isFMulNegTwo(N1)) {
19597 SDValue B = N1.getOperand(0);
19598 SDValue Add = DAG.getNode(ISD::FADD, DL, VT, B, B);
19599 return DAG.getNode(ISD::FSUB, DL, VT, N0, Add);
19600 }
19601
19602 // No FP constant should be created after legalization as Instruction
19603 // Selection pass has a hard time dealing with FP constants.
19604 bool AllowNewConst = (Level < AfterLegalizeDAG);
19605
19606 // If nnan is enabled, fold lots of things.
19607 if (Flags.hasNoNaNs() && AllowNewConst) {
19608 // If allowed, fold (fadd (fneg x), x) -> 0.0
19609 if (N0.getOpcode() == ISD::FNEG && N0.getOperand(0) == N1)
19610 return DAG.getConstantFP(0.0, DL, VT);
19611
19612 // If allowed, fold (fadd x, (fneg x)) -> 0.0
19613 if (N1.getOpcode() == ISD::FNEG && N1.getOperand(0) == N0)
19614 return DAG.getConstantFP(0.0, DL, VT);
19615 }
19616
19617 // If reassoc and nsz, fold lots of things.
19618 // TODO: break out portions of the transformations below for which Unsafe is
19619 // considered and which do not require both nsz and reassoc
19620 if (Flags.hasAllowReassociation() && Flags.hasNoSignedZeros() &&
19621 AllowNewConst) {
19622 // fadd (fadd x, c1), c2 -> fadd x, c1 + c2
19623 if (N1CFP && N0.getOpcode() == ISD::FADD &&
19625 SDValue NewC = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1), N1);
19626 return DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(0), NewC);
19627 }
19628
19629 // We can fold chains of FADD's of the same value into multiplications.
19630 // This transform is not safe in general because we are reducing the number
19631 // of rounding steps.
19632 if ((!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FMUL, VT)) &&
19633 !N0CFP && !N1CFP) {
19634 if (N0.getOpcode() == ISD::FMUL)