LLVM 24.0.0git
CombinerHelper.cpp
Go to the documentation of this file.
1//===-- lib/CodeGen/GlobalISel/GICombinerHelper.cpp -----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
9#include "llvm/ADT/APFloat.h"
11#include "llvm/ADT/STLExtras.h"
12#include "llvm/ADT/SetVector.h"
37#include "llvm/IR/DataLayout.h"
38#include "llvm/IR/InstrTypes.h"
45#include <cmath>
46#include <optional>
47#include <tuple>
48
49#define DEBUG_TYPE "gi-combiner"
50
51using namespace llvm;
52using namespace MIPatternMatch;
53
54// Option to allow testing of the combiner while no targets know about indexed
55// addressing.
56static cl::opt<bool>
57 ForceLegalIndexing("force-legal-indexing", cl::Hidden, cl::init(false),
58 cl::desc("Force all indexed operations to be "
59 "legal for the GlobalISel combiner"));
60
65 const LegalizerInfo *LI)
66 : Builder(B), MRI(Builder.getMF().getRegInfo()), Observer(Observer), VT(VT),
68 TII(Builder.getMF().getSubtarget().getInstrInfo()),
69 RBI(Builder.getMF().getSubtarget().getRegBankInfo()),
70 TRI(Builder.getMF().getSubtarget().getRegisterInfo()) {
71 (void)this->VT;
72}
73
75 return *Builder.getMF().getSubtarget().getTargetLowering();
76}
77
79 return Builder.getMF();
80}
81
82LLVMContext &CombinerHelper::getContext() const { return Builder.getContext(); }
83
84/// \returns The little endian in-memory byte position of byte \p I in a
85/// \p ByteWidth bytes wide type.
86///
87/// E.g. Given a 4-byte type x, x[0] -> byte 0
88static unsigned littleEndianByteAt(const unsigned ByteWidth, const unsigned I) {
89 assert(I < ByteWidth && "I must be in [0, ByteWidth)");
90 return I;
91}
92
93/// Determines the LogBase2 value for a non-null input value using the
94/// transform: LogBase2(V) = (EltBits - 1) - ctlz(V).
96 auto &MRI = *MIB.getMRI();
97 LLT Ty = MRI.getType(V);
98 auto Ctlz = MIB.buildCTLZ(Ty, V);
99 auto Base = MIB.buildConstant(Ty, Ty.getScalarSizeInBits() - 1);
100 return MIB.buildSub(Ty, Base, Ctlz).getReg(0);
101}
102
103/// \returns The big endian in-memory byte position of byte \p I in a
104/// \p ByteWidth bytes wide type.
105///
106/// E.g. Given a 4-byte type x, x[0] -> byte 3
107static unsigned bigEndianByteAt(const unsigned ByteWidth, const unsigned I) {
108 assert(I < ByteWidth && "I must be in [0, ByteWidth)");
109 return ByteWidth - I - 1;
110}
111
112/// Given a map from byte offsets in memory to indices in a load/store,
113/// determine if that map corresponds to a little or big endian byte pattern.
114///
115/// \param MemOffset2Idx maps memory offsets to address offsets.
116/// \param LowestIdx is the lowest index in \p MemOffset2Idx.
117///
118/// \returns true if the map corresponds to a big endian byte pattern, false if
119/// it corresponds to a little endian byte pattern, and std::nullopt otherwise.
120///
121/// E.g. given a 32-bit type x, and x[AddrOffset], the in-memory byte patterns
122/// are as follows:
123///
124/// AddrOffset Little endian Big endian
125/// 0 0 3
126/// 1 1 2
127/// 2 2 1
128/// 3 3 0
129static std::optional<bool>
131 int64_t LowestIdx) {
132 // Need at least two byte positions to decide on endianness.
133 unsigned Width = MemOffset2Idx.size();
134 if (Width < 2)
135 return std::nullopt;
136 bool BigEndian = true, LittleEndian = true;
137 for (unsigned MemOffset = 0; MemOffset < Width; ++ MemOffset) {
138 auto MemOffsetAndIdx = MemOffset2Idx.find(MemOffset);
139 if (MemOffsetAndIdx == MemOffset2Idx.end())
140 return std::nullopt;
141 const int64_t Idx = MemOffsetAndIdx->second - LowestIdx;
142 assert(Idx >= 0 && "Expected non-negative byte offset?");
143 LittleEndian &= Idx == littleEndianByteAt(Width, MemOffset);
144 BigEndian &= Idx == bigEndianByteAt(Width, MemOffset);
145 if (!BigEndian && !LittleEndian)
146 return std::nullopt;
147 }
148
149 assert((BigEndian != LittleEndian) &&
150 "Pattern cannot be both big and little endian!");
151 return BigEndian;
152}
153
155
156bool CombinerHelper::isLegal(const LegalityQuery &Query) const {
157 assert(LI && "Must have LegalizerInfo to query isLegal!");
158 return LI->getAction(Query).Action == LegalizeActions::Legal;
159}
160
162 const LegalityQuery &Query) const {
163 return isPreLegalize() || isLegal(Query);
164}
165
167 return isLegal(Query) ||
168 LI->getAction(Query).Action == LegalizeActions::WidenScalar;
169}
170
172 const LegalityQuery &Query) const {
173 LegalizeAction Action = LI->getAction(Query).Action;
174 return Action == LegalizeActions::Legal ||
176}
177
178bool CombinerHelper::isImmConstantLegalOrBeforeLegalizer(unsigned Opc,
179 const LLT Ty) const {
180 if (!Ty.isVector())
181 return isLegalOrBeforeLegalizer({Opc, {Ty}});
182 if (isPreLegalize())
183 return true;
184 // Scalable vectors cannot be constant folded here.
185 if (!Ty.isFixedVector())
186 return false;
187 LLT EltTy = Ty.getElementType();
188 return isLegal({TargetOpcode::G_BUILD_VECTOR, {Ty, EltTy}}) &&
189 isLegal({Opc, {EltTy}});
190}
191
193 return isImmConstantLegalOrBeforeLegalizer(TargetOpcode::G_CONSTANT, Ty);
194}
195
197 return isImmConstantLegalOrBeforeLegalizer(TargetOpcode::G_FCONSTANT, Ty);
198}
199
201 Register ToReg) const {
202 Observer.changingAllUsesOfReg(MRI, FromReg);
203
204 if (MRI.constrainRegAttrs(ToReg, FromReg))
205 MRI.replaceRegWith(FromReg, ToReg);
206 else
207 Builder.buildCopy(FromReg, ToReg);
208
209 Observer.finishedChangingAllUsesOfReg();
210}
211
213 MachineOperand &FromRegOp,
214 Register ToReg) const {
215 assert(FromRegOp.getParent() && "Expected an operand in an MI");
216 Observer.changingInstr(*FromRegOp.getParent());
217
218 FromRegOp.setReg(ToReg);
219
220 Observer.changedInstr(*FromRegOp.getParent());
221}
222
224 unsigned ToOpcode) const {
225 Observer.changingInstr(FromMI);
226
227 FromMI.setDesc(Builder.getTII().get(ToOpcode));
228
229 Observer.changedInstr(FromMI);
230}
231
233 return RBI->getRegBank(Reg, MRI, *TRI);
234}
235
237 const RegisterBank *RegBank) const {
238 if (RegBank)
239 MRI.setRegBank(Reg, *RegBank);
240}
241
243 if (MI.getOpcode() != TargetOpcode::COPY)
244 return false;
245 Register DstReg = MI.getOperand(0).getReg();
246 Register SrcReg = MI.getOperand(1).getReg();
247 return canReplaceReg(DstReg, SrcReg, MRI);
248}
250 Register DstReg = MI.getOperand(0).getReg();
251 Register SrcReg = MI.getOperand(1).getReg();
252 replaceRegWith(MRI, DstReg, SrcReg);
253 MI.eraseFromParent();
254}
255
257 MachineInstr &MI, BuildFnTy &MatchInfo) const {
258 assert(MI.getOpcode() == TargetOpcode::G_FREEZE && "Invalid instruction");
259
260 // Ported from InstCombinerImpl::pushFreezeToPreventPoisonFromPropagating.
261 Register DstOp = MI.getOperand(0).getReg();
262 Register OrigOp = MI.getOperand(1).getReg();
263
264 if (!MRI.hasOneNonDBGUse(OrigOp))
265 return false;
266
267 MachineInstr *OrigDef;
268 if (!mi_match(OrigOp, MRI, m_MInstr(OrigDef)))
269 return false;
270 // Even if only a single operand of the PHI is not guaranteed non-poison,
271 // moving freeze() backwards across a PHI can cause optimization issues for
272 // other users of that operand.
273 //
274 // Moving freeze() from one of the output registers of a G_UNMERGE_VALUES to
275 // the source register is unprofitable because it makes the freeze() more
276 // strict than is necessary (it would affect the whole register instead of
277 // just the subreg being frozen).
278 if (OrigDef->isPHI() || isa<GUnmerge>(OrigDef))
279 return false;
280
281 if (canCreateUndefOrPoison(OrigOp, MRI,
282 /*ConsiderFlagsAndMetadata=*/false))
283 return false;
284
285 std::optional<MachineOperand> MaybePoisonOperand;
286 for (MachineOperand &Operand : OrigDef->uses()) {
287 if (!Operand.isReg())
288 return false;
289
290 if (isGuaranteedNotToBeUndefOrPoison(Operand.getReg(), MRI))
291 continue;
292
293 if (!MaybePoisonOperand)
294 MaybePoisonOperand = Operand;
295 else {
296 // We have more than one maybe-poison operand. Moving the freeze is
297 // unsafe.
298 return false;
299 }
300 }
301
302 // Eliminate freeze if all operands are guaranteed non-poison.
303 if (!MaybePoisonOperand) {
304 MatchInfo = [=](MachineIRBuilder &B) {
305 Observer.changingInstr(*OrigDef);
306 cast<GenericMachineInstr>(OrigDef)->dropPoisonGeneratingFlags();
307 Observer.changedInstr(*OrigDef);
308 B.buildCopy(DstOp, OrigOp);
309 };
310 return true;
311 }
312
313 Register MaybePoisonOperandReg = MaybePoisonOperand->getReg();
314 LLT MaybePoisonOperandRegTy = MRI.getType(MaybePoisonOperandReg);
315
317 {TargetOpcode::G_FREEZE, {MaybePoisonOperandRegTy}}))
318 return false;
319
320 MatchInfo = [=](MachineIRBuilder &B) mutable {
321 Observer.changingInstr(*OrigDef);
322 cast<GenericMachineInstr>(OrigDef)->dropPoisonGeneratingFlags();
323 Observer.changedInstr(*OrigDef);
324 B.setInsertPt(*OrigDef->getParent(), OrigDef->getIterator());
325 auto Freeze = B.buildFreeze(MaybePoisonOperandRegTy, MaybePoisonOperandReg);
327 MRI, *OrigDef->findRegisterUseOperand(MaybePoisonOperandReg, TRI),
328 Freeze.getReg(0));
329 replaceRegWith(MRI, DstOp, OrigOp);
330 };
331 return true;
332}
333
336 assert(MI.getOpcode() == TargetOpcode::G_CONCAT_VECTORS &&
337 "Invalid instruction");
338 bool IsUndef = true;
339 MachineInstr *Undef = nullptr;
340
341 // Walk over all the operands of concat vectors and check if they are
342 // build_vector themselves or undef.
343 // Then collect their operands in Ops.
344 for (const MachineOperand &MO : MI.uses()) {
345 Register Reg = MO.getReg();
346 MachineInstr *Def;
347 if (!mi_match(Reg, MRI, m_MInstr(Def)))
348 return false;
349 if (!MRI.hasOneNonDBGUse(Reg))
350 return false;
351 switch (Def->getOpcode()) {
352 case TargetOpcode::G_BUILD_VECTOR:
353 IsUndef = false;
354 // Remember the operands of the build_vector to fold
355 // them into the yet-to-build flattened concat vectors.
356 for (const MachineOperand &BuildVecMO : Def->uses())
357 Ops.push_back(BuildVecMO.getReg());
358 break;
359 case TargetOpcode::G_IMPLICIT_DEF: {
360 LLT OpType = MRI.getType(Reg);
361 // Keep one undef value for all the undef operands.
362 if (!Undef) {
363 Builder.setInsertPt(*MI.getParent(), MI);
364 Undef = Builder.buildUndef(OpType.getScalarType());
365 }
366 assert(MRI.getType(Undef->getOperand(0).getReg()) ==
367 OpType.getScalarType() &&
368 "All undefs should have the same type");
369 // Break the undef vector in as many scalar elements as needed
370 // for the flattening.
371 for (unsigned EltIdx = 0, EltEnd = OpType.getNumElements();
372 EltIdx != EltEnd; ++EltIdx)
373 Ops.push_back(Undef->getOperand(0).getReg());
374 break;
375 }
376 default:
377 return false;
378 }
379 }
380
381 // Check if the combine is illegal
382 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
384 {TargetOpcode::G_BUILD_VECTOR, {DstTy, MRI.getType(Ops[0])}})) {
385 return false;
386 }
387
388 if (IsUndef)
389 Ops.clear();
390
391 return true;
392}
395 // We determined that the concat_vectors can be flatten.
396 // Generate the flattened build_vector.
397 Register DstReg = MI.getOperand(0).getReg();
398 Builder.setInsertPt(*MI.getParent(), MI);
399 Register NewDstReg = MRI.cloneVirtualRegister(DstReg);
400
401 // Note: IsUndef is sort of redundant. We could have determine it by
402 // checking that at all Ops are undef. Alternatively, we could have
403 // generate a build_vector of undefs and rely on another combine to
404 // clean that up. For now, given we already gather this information
405 // in matchCombineConcatVectors, just save compile time and issue the
406 // right thing.
407 if (Ops.empty())
408 Builder.buildUndef(NewDstReg);
409 else
410 Builder.buildBuildVector(NewDstReg, Ops);
411 replaceRegWith(MRI, DstReg, NewDstReg);
412 MI.eraseFromParent();
413}
414
417 auto &BV = cast<GBuildVector>(MI);
418
419 // Look at the first operand for a unmerge(bitcast) from a scalar type.
420 GUnmerge *Unmerge = getOpcodeDef<GUnmerge>(BV.getSourceReg(0), MRI);
421 if (!Unmerge || Unmerge->getReg(0) != BV.getSourceReg(0))
422 return false;
423 Register BCSrc;
424 if (!mi_match(Unmerge->getSourceReg(), MRI, m_GBitcast(m_Reg(BCSrc))))
425 return false;
426 LLT InputTy = MRI.getType(BCSrc);
427 unsigned Factor = Unmerge->getNumDefs();
428 if (!InputTy.isScalar() || BV.getNumSources() % Factor != 0)
429 return false;
430
431 // Check if the build_vector is legal
432 LLT BVDstTy = LLT::fixed_vector(BV.getNumSources() / Factor, InputTy);
433 if (!isLegal({TargetOpcode::G_BUILD_VECTOR, {BVDstTy, InputTy}}))
434 return false;
435
436 // Check all other operands are bitcasts or undef.
437 for (unsigned Idx = 0; Idx < BV.getNumSources(); Idx += Factor) {
438 GUnmerge *Unmerge = getOpcodeDef<GUnmerge>(BV.getSourceReg(Idx), MRI);
439 if (!all_of(iota_range<unsigned>(0, Factor, false), [&](unsigned J) {
440 if (mi_match(BV.getSourceReg(Idx + J), MRI, m_GImplicitDef()))
441 return true;
442 return Unmerge && BV.getSourceReg(Idx + J) == Unmerge->getReg(J);
443 }))
444 return false;
445 if (!Unmerge)
446 Ops.push_back(0);
447 else {
448 Register BCSrc;
449 if (!mi_match(
450 Unmerge->getSourceReg(), MRI,
451 m_GBitcast(m_all_of(m_Reg(BCSrc), m_SpecificType(InputTy)))))
452 return false;
453 Ops.push_back(BCSrc);
454 }
455 }
456
457 return true;
458}
459
462 LLT SrcTy = MRI.getType(Ops[0]);
463 // Build undef if any operations require it.
464 Register Undef = 0;
465 for (Register &Op : Ops) {
466 if (!Op) {
467 if (!Undef)
468 Undef = Builder.buildUndef(SrcTy).getReg(0);
469 Op = Undef;
470 }
471 }
472
473 LLT BVDstTy = LLT::fixed_vector(Ops.size(), SrcTy);
474 auto BV = Builder.buildBuildVector(BVDstTy, Ops);
475 Builder.buildBitcast(MI.getOperand(0).getReg(), BV);
476 MI.eraseFromParent();
477}
478
480 auto &Shuffle = cast<GShuffleVector>(MI);
481
482 Register SrcVec1 = Shuffle.getSrc1Reg();
483 Register SrcVec2 = Shuffle.getSrc2Reg();
484 LLT EltTy = MRI.getType(SrcVec1).getElementType();
485 int Width = MRI.getType(SrcVec1).getNumElements();
486
487 auto Unmerge1 = Builder.buildUnmerge(EltTy, SrcVec1);
488 auto Unmerge2 = Builder.buildUnmerge(EltTy, SrcVec2);
489
490 SmallVector<Register> Extracts;
491 // Select only applicable elements from unmerged values.
492 for (int Val : Shuffle.getMask()) {
493 if (Val == -1)
494 Extracts.push_back(Builder.buildUndef(EltTy).getReg(0));
495 else if (Val < Width)
496 Extracts.push_back(Unmerge1.getReg(Val));
497 else
498 Extracts.push_back(Unmerge2.getReg(Val - Width));
499 }
500 assert(Extracts.size() > 0 && "Expected at least one element in the shuffle");
501 if (Extracts.size() == 1)
502 Builder.buildCopy(MI.getOperand(0).getReg(), Extracts[0]);
503 else
504 Builder.buildBuildVector(MI.getOperand(0).getReg(), Extracts);
505 MI.eraseFromParent();
506}
507
509 auto &Shuffle = cast<GShuffleVector>(MI);
510 Register SrcVec1 = Shuffle.getSrc1Reg();
511 Register SrcVec2 = Shuffle.getSrc2Reg();
512 auto *BuildMI1 = getOpcodeDef<GBuildVector>(SrcVec1, MRI);
513 auto *BuildMI2 = getOpcodeDef<GBuildVector>(SrcVec2, MRI);
514 assert((BuildMI1 && (BuildMI2 || MRI.getVRegDef(SrcVec2)->getOpcode() ==
515 TargetOpcode::G_IMPLICIT_DEF)) &&
516 "Unexpected input to applyCombineShuffleOfBuildVector");
517 LLT EltTy = MRI.getType(SrcVec1).getElementType();
518 int NumElts = MRI.getType(SrcVec1).getNumElements();
519
520 SmallVector<Register> Extracts;
521 for (int Val : Shuffle.getMask()) {
522 if (Val == -1)
523 Extracts.push_back(Builder.buildUndef(EltTy).getReg(0));
524 else if (Val < NumElts)
525 Extracts.push_back(BuildMI1->getSourceReg(Val));
526 else if (BuildMI2)
527 Extracts.push_back(BuildMI2->getSourceReg(Val - NumElts));
528 else
529 Extracts.push_back(Builder.buildUndef(EltTy).getReg(0));
530 }
531
532 if (Extracts.size() == 1)
533 Builder.buildCopy(MI.getOperand(0).getReg(), Extracts[0]);
534 else
535 Builder.buildBuildVector(MI.getOperand(0).getReg(), Extracts);
536 MI.eraseFromParent();
537}
538
541 ArrayRef<int> Mask = MI.getOperand(3).getShuffleMask();
542 GConcatVectors *ConcatMI1, *ConcatMI2;
543 if (!mi_match(MI.getOperand(1).getReg(), MRI, m_GConcatVectors(ConcatMI1)) ||
544 !mi_match(MI.getOperand(2).getReg(), MRI, m_GConcatVectors(ConcatMI2)))
545 return false;
546
547 // Check that the sources of the Concat instructions have the same type
548 if (MRI.getType(ConcatMI1->getSourceReg(0)) !=
549 MRI.getType(ConcatMI2->getSourceReg(0)))
550 return false;
551
552 LLT ConcatSrcTy = MRI.getType(ConcatMI1->getReg(1));
553 LLT ShuffleSrcTy1 = MRI.getType(MI.getOperand(1).getReg());
554 unsigned ConcatSrcNumElt = ConcatSrcTy.getNumElements();
555 for (unsigned i = 0; i < Mask.size(); i += ConcatSrcNumElt) {
556 // Check if the index takes a whole source register from G_CONCAT_VECTORS
557 // Assumes that all Sources of G_CONCAT_VECTORS are the same type
558 if (Mask[i] == -1) {
559 for (unsigned j = 1; j < ConcatSrcNumElt; j++) {
560 if (i + j >= Mask.size())
561 return false;
562 if (Mask[i + j] != -1)
563 return false;
564 }
566 {TargetOpcode::G_IMPLICIT_DEF, {ConcatSrcTy}}))
567 return false;
568 Ops.push_back(0);
569 } else if (Mask[i] % ConcatSrcNumElt == 0) {
570 for (unsigned j = 1; j < ConcatSrcNumElt; j++) {
571 if (i + j >= Mask.size())
572 return false;
573 if (Mask[i + j] != Mask[i] + static_cast<int>(j))
574 return false;
575 }
576 // Retrieve the source register from its respective G_CONCAT_VECTORS
577 // instruction
578 if (Mask[i] < ShuffleSrcTy1.getNumElements()) {
579 Ops.push_back(ConcatMI1->getSourceReg(Mask[i] / ConcatSrcNumElt));
580 } else {
581 Ops.push_back(ConcatMI2->getSourceReg(Mask[i] / ConcatSrcNumElt -
582 ConcatMI1->getNumSources()));
583 }
584 } else {
585 return false;
586 }
587 }
588
590 {TargetOpcode::G_CONCAT_VECTORS,
591 {MRI.getType(MI.getOperand(0).getReg()), ConcatSrcTy}}))
592 return false;
593
594 return !Ops.empty();
595}
596
599 LLT SrcTy;
600 for (Register &Reg : Ops) {
601 if (Reg != 0)
602 SrcTy = MRI.getType(Reg);
603 }
604 assert(SrcTy.isValid() && "Unexpected full undef vector in concat combine");
605
606 Register UndefReg = 0;
607
608 for (Register &Reg : Ops) {
609 if (Reg == 0) {
610 if (UndefReg == 0)
611 UndefReg = Builder.buildUndef(SrcTy).getReg(0);
612 Reg = UndefReg;
613 }
614 }
615
616 if (Ops.size() > 1)
617 Builder.buildConcatVectors(MI.getOperand(0).getReg(), Ops);
618 else
619 Builder.buildCopy(MI.getOperand(0).getReg(), Ops[0]);
620 MI.eraseFromParent();
621}
622
625 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR &&
626 "Invalid instruction kind");
627 LLT DstType = MRI.getType(MI.getOperand(0).getReg());
628 Register Src1 = MI.getOperand(1).getReg();
629 LLT SrcType = MRI.getType(Src1);
630
631 unsigned DstNumElts = DstType.getNumElements();
632 unsigned SrcNumElts = SrcType.getNumElements();
633
634 // If the resulting vector is smaller than the size of the source
635 // vectors being concatenated, we won't be able to replace the
636 // shuffle vector into a concat_vectors.
637 //
638 // Note: We may still be able to produce a concat_vectors fed by
639 // extract_vector_elt and so on. It is less clear that would
640 // be better though, so don't bother for now.
641 //
642 // If the destination is a scalar, the size of the sources doesn't
643 // matter. we will lower the shuffle to a plain copy. This will
644 // work only if the source and destination have the same size. But
645 // that's covered by the next condition.
646 //
647 // TODO: If the size between the source and destination don't match
648 // we could still emit an extract vector element in that case.
649 if (DstNumElts < 2 * SrcNumElts)
650 return false;
651
652 // Check that the shuffle mask can be broken evenly between the
653 // different sources.
654 if (DstNumElts % SrcNumElts != 0)
655 return false;
656
657 // Mask length is a multiple of the source vector length.
658 // Check if the shuffle is some kind of concatenation of the input
659 // vectors.
660 unsigned NumConcat = DstNumElts / SrcNumElts;
661 SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
662 ArrayRef<int> Mask = MI.getOperand(3).getShuffleMask();
663 for (unsigned i = 0; i != DstNumElts; ++i) {
664 int Idx = Mask[i];
665 // Undef value.
666 if (Idx < 0)
667 continue;
668 // Ensure the indices in each SrcType sized piece are sequential and that
669 // the same source is used for the whole piece.
670 if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
671 (ConcatSrcs[i / SrcNumElts] >= 0 &&
672 ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts)))
673 return false;
674 // Remember which source this index came from.
675 ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
676 }
677
678 // The shuffle is concatenating multiple vectors together.
679 // Collect the different operands for that.
680 Register UndefReg;
681 Register Src2 = MI.getOperand(2).getReg();
682 for (auto Src : ConcatSrcs) {
683 if (Src < 0) {
684 if (!UndefReg) {
685 Builder.setInsertPt(*MI.getParent(), MI);
686 UndefReg = Builder.buildUndef(SrcType).getReg(0);
687 }
688 Ops.push_back(UndefReg);
689 } else if (Src == 0)
690 Ops.push_back(Src1);
691 else
692 Ops.push_back(Src2);
693 }
694 return true;
695}
696
698 ArrayRef<Register> Ops) const {
699 Register DstReg = MI.getOperand(0).getReg();
700 Builder.setInsertPt(*MI.getParent(), MI);
701 Register NewDstReg = MRI.cloneVirtualRegister(DstReg);
702
703 if (Ops.size() == 1)
704 Builder.buildCopy(NewDstReg, Ops[0]);
705 else
706 Builder.buildMergeLikeInstr(NewDstReg, Ops);
707
708 replaceRegWith(MRI, DstReg, NewDstReg);
709 MI.eraseFromParent();
710}
711
712namespace {
713
714/// Select a preference between two uses. CurrentUse is the current preference
715/// while *ForCandidate is attributes of the candidate under consideration.
716PreferredTuple ChoosePreferredUse(MachineInstr &LoadMI,
717 PreferredTuple &CurrentUse,
718 const LLT TyForCandidate,
719 unsigned OpcodeForCandidate,
720 MachineInstr *MIForCandidate) {
721 if (!CurrentUse.Ty.isValid()) {
722 if (CurrentUse.ExtendOpcode == OpcodeForCandidate ||
723 CurrentUse.ExtendOpcode == TargetOpcode::G_ANYEXT)
724 return {TyForCandidate, OpcodeForCandidate, MIForCandidate};
725 return CurrentUse;
726 }
727
728 // We permit the extend to hoist through basic blocks but this is only
729 // sensible if the target has extending loads. If you end up lowering back
730 // into a load and extend during the legalizer then the end result is
731 // hoisting the extend up to the load.
732
733 // Prefer defined extensions to undefined extensions as these are more
734 // likely to reduce the number of instructions.
735 if (OpcodeForCandidate == TargetOpcode::G_ANYEXT &&
736 CurrentUse.ExtendOpcode != TargetOpcode::G_ANYEXT)
737 return CurrentUse;
738 else if (CurrentUse.ExtendOpcode == TargetOpcode::G_ANYEXT &&
739 OpcodeForCandidate != TargetOpcode::G_ANYEXT)
740 return {TyForCandidate, OpcodeForCandidate, MIForCandidate};
741
742 // Prefer sign extensions to zero extensions as sign-extensions tend to be
743 // more expensive. Don't do this if the load is already a zero-extend load
744 // though, otherwise we'll rewrite a zero-extend load into a sign-extend
745 // later.
746 if (!isa<GZExtLoad>(LoadMI) && CurrentUse.Ty == TyForCandidate) {
747 if (CurrentUse.ExtendOpcode == TargetOpcode::G_SEXT &&
748 OpcodeForCandidate == TargetOpcode::G_ZEXT)
749 return CurrentUse;
750 else if (CurrentUse.ExtendOpcode == TargetOpcode::G_ZEXT &&
751 OpcodeForCandidate == TargetOpcode::G_SEXT)
752 return {TyForCandidate, OpcodeForCandidate, MIForCandidate};
753 }
754
755 // This is potentially target specific. We've chosen the largest type
756 // because G_TRUNC is usually free. One potential catch with this is that
757 // some targets have a reduced number of larger registers than smaller
758 // registers and this choice potentially increases the live-range for the
759 // larger value.
760 if (TyForCandidate.getSizeInBits() > CurrentUse.Ty.getSizeInBits()) {
761 return {TyForCandidate, OpcodeForCandidate, MIForCandidate};
762 }
763 return CurrentUse;
764}
765
766/// Find a suitable place to insert some instructions and insert them. This
767/// function accounts for special cases like inserting before a PHI node.
768/// The current strategy for inserting before PHI's is to duplicate the
769/// instructions for each predecessor. However, while that's ok for G_TRUNC
770/// on most targets since it generally requires no code, other targets/cases may
771/// want to try harder to find a dominating block.
772static void InsertInsnsWithoutSideEffectsBeforeUse(
775 MachineOperand &UseMO)>
776 Inserter) {
777 MachineInstr &UseMI = *UseMO.getParent();
778
779 MachineBasicBlock *InsertBB = UseMI.getParent();
780
781 // If the use is a PHI then we want the predecessor block instead.
782 if (UseMI.isPHI()) {
783 MachineOperand *PredBB = std::next(&UseMO);
784 InsertBB = PredBB->getMBB();
785 }
786
787 // If the block is the same block as the def then we want to insert just after
788 // the def instead of at the start of the block.
789 if (InsertBB == DefMI.getParent()) {
791 Inserter(InsertBB, std::next(InsertPt), UseMO);
792 return;
793 }
794
795 // Otherwise we want the start of the BB
796 Inserter(InsertBB, InsertBB->getFirstNonPHI(), UseMO);
797}
798} // end anonymous namespace
799
801 PreferredTuple Preferred;
802 if (matchCombineExtendingLoads(MI, Preferred)) {
803 applyCombineExtendingLoads(MI, Preferred);
804 return true;
805 }
806 return false;
807}
808
809static unsigned getExtLoadOpcForExtend(unsigned ExtOpc) {
810 unsigned CandidateLoadOpc;
811 switch (ExtOpc) {
812 case TargetOpcode::G_ANYEXT:
813 CandidateLoadOpc = TargetOpcode::G_LOAD;
814 break;
815 case TargetOpcode::G_SEXT:
816 CandidateLoadOpc = TargetOpcode::G_SEXTLOAD;
817 break;
818 case TargetOpcode::G_ZEXT:
819 CandidateLoadOpc = TargetOpcode::G_ZEXTLOAD;
820 break;
821 default:
822 llvm_unreachable("Unexpected extend opc");
823 }
824 return CandidateLoadOpc;
825}
826
828 MachineInstr &MI, PreferredTuple &Preferred) const {
829 // We match the loads and follow the uses to the extend instead of matching
830 // the extends and following the def to the load. This is because the load
831 // must remain in the same position for correctness (unless we also add code
832 // to find a safe place to sink it) whereas the extend is freely movable.
833 // It also prevents us from duplicating the load for the volatile case or just
834 // for performance.
835 GAnyLoad *LoadMI = dyn_cast<GAnyLoad>(&MI);
836 if (!LoadMI)
837 return false;
838
839 Register LoadReg = LoadMI->getDstReg();
840
841 LLT LoadValueTy = MRI.getType(LoadReg);
842 if (!LoadValueTy.isScalar())
843 return false;
844
845 // Most architectures are going to legalize <s8 loads into at least a 1 byte
846 // load, and the MMOs can only describe memory accesses in multiples of bytes.
847 // If we try to perform extload combining on those, we can end up with
848 // %a(s8) = extload %ptr (load 1 byte from %ptr)
849 // ... which is an illegal extload instruction.
850 if (LoadValueTy.getSizeInBits() < 8)
851 return false;
852
853 // For non power-of-2 types, they will very likely be legalized into multiple
854 // loads. Don't bother trying to match them into extending loads.
856 return false;
857
858 // Find the preferred type aside from the any-extends (unless it's the only
859 // one) and non-extending ops. We'll emit an extending load to that type and
860 // and emit a variant of (extend (trunc X)) for the others according to the
861 // relative type sizes. At the same time, pick an extend to use based on the
862 // extend involved in the chosen type.
863 unsigned PreferredOpcode =
864 isa<GLoad>(&MI)
865 ? TargetOpcode::G_ANYEXT
866 : isa<GSExtLoad>(&MI) ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
867 Preferred = {LLT(), PreferredOpcode, nullptr};
868 for (auto &UseMI : MRI.use_nodbg_instructions(LoadReg)) {
869 if (UseMI.getOpcode() == TargetOpcode::G_SEXT ||
870 UseMI.getOpcode() == TargetOpcode::G_ZEXT ||
871 (UseMI.getOpcode() == TargetOpcode::G_ANYEXT)) {
872 const auto &MMO = LoadMI->getMMO();
873 // Don't do anything for atomics.
874 if (MMO.isAtomic())
875 continue;
876 // Check for legality.
877 if (!isPreLegalize()) {
878 LegalityQuery::MemDesc MMDesc(MMO);
879 unsigned CandidateLoadOpc = getExtLoadOpcForExtend(UseMI.getOpcode());
880 LLT UseTy = MRI.getType(UseMI.getOperand(0).getReg());
881 LLT SrcTy = MRI.getType(LoadMI->getPointerReg());
882 if (LI->getAction({CandidateLoadOpc, {UseTy, SrcTy}, {MMDesc}})
883 .Action != LegalizeActions::Legal)
884 continue;
885 }
886 Preferred = ChoosePreferredUse(MI, Preferred,
887 MRI.getType(UseMI.getOperand(0).getReg()),
888 UseMI.getOpcode(), &UseMI);
889 }
890 }
891
892 // There were no extends
893 if (!Preferred.MI)
894 return false;
895 // It should be impossible to chose an extend without selecting a different
896 // type since by definition the result of an extend is larger.
897 assert(Preferred.Ty != LoadValueTy && "Extending to same type?");
898
899 LLVM_DEBUG(dbgs() << "Preferred use is: " << *Preferred.MI);
900 return true;
901}
902
904 MachineInstr &MI, PreferredTuple &Preferred) const {
905 // Rewrite the load to the chosen extending load.
906 Register ChosenDstReg = Preferred.MI->getOperand(0).getReg();
907
908 // Inserter to insert a truncate back to the original type at a given point
909 // with some basic CSE to limit truncate duplication to one per BB.
911 auto InsertTruncAt = [&](MachineBasicBlock *InsertIntoBB,
912 MachineBasicBlock::iterator InsertBefore,
913 MachineOperand &UseMO) {
914 MachineInstr *PreviouslyEmitted = EmittedInsns.lookup(InsertIntoBB);
915 if (PreviouslyEmitted) {
916 Observer.changingInstr(*UseMO.getParent());
917 UseMO.setReg(PreviouslyEmitted->getOperand(0).getReg());
918 Observer.changedInstr(*UseMO.getParent());
919 return;
920 }
921
922 Builder.setInsertPt(*InsertIntoBB, InsertBefore);
923 Register NewDstReg = MRI.cloneVirtualRegister(MI.getOperand(0).getReg());
924 MachineInstr *NewMI = Builder.buildTrunc(NewDstReg, ChosenDstReg);
925 EmittedInsns[InsertIntoBB] = NewMI;
926 replaceRegOpWith(MRI, UseMO, NewDstReg);
927 };
928
929 Observer.changingInstr(MI);
930 unsigned LoadOpc = getExtLoadOpcForExtend(Preferred.ExtendOpcode);
931 MI.setDesc(Builder.getTII().get(LoadOpc));
932
933 // Rewrite all the uses to fix up the types.
934 auto &LoadValue = MI.getOperand(0);
936 llvm::make_pointer_range(MRI.use_operands(LoadValue.getReg())));
937
938 for (auto *UseMO : Uses) {
939 MachineInstr *UseMI = UseMO->getParent();
940
941 // If the extend is compatible with the preferred extend then we should fix
942 // up the type and extend so that it uses the preferred use.
943 if (UseMI->getOpcode() == Preferred.ExtendOpcode ||
944 UseMI->getOpcode() == TargetOpcode::G_ANYEXT) {
945 Register UseDstReg = UseMI->getOperand(0).getReg();
946 MachineOperand &UseSrcMO = UseMI->getOperand(1);
947 const LLT UseDstTy = MRI.getType(UseDstReg);
948 if (UseDstReg != ChosenDstReg) {
949 if (Preferred.Ty == UseDstTy) {
950 // If the use has the same type as the preferred use, then merge
951 // the vregs and erase the extend. For example:
952 // %1:_(s8) = G_LOAD ...
953 // %2:_(s32) = G_SEXT %1(s8)
954 // %3:_(s32) = G_ANYEXT %1(s8)
955 // ... = ... %3(s32)
956 // rewrites to:
957 // %2:_(s32) = G_SEXTLOAD ...
958 // ... = ... %2(s32)
959 replaceRegWith(MRI, UseDstReg, ChosenDstReg);
960 Observer.erasingInstr(*UseMO->getParent());
961 UseMO->getParent()->eraseFromParent();
962 } else if (Preferred.Ty.getSizeInBits() < UseDstTy.getSizeInBits()) {
963 // If the preferred size is smaller, then keep the extend but extend
964 // from the result of the extending load. For example:
965 // %1:_(s8) = G_LOAD ...
966 // %2:_(s32) = G_SEXT %1(s8)
967 // %3:_(s64) = G_ANYEXT %1(s8)
968 // ... = ... %3(s64)
969 /// rewrites to:
970 // %2:_(s32) = G_SEXTLOAD ...
971 // %3:_(s64) = G_ANYEXT %2:_(s32)
972 // ... = ... %3(s64)
973 replaceRegOpWith(MRI, UseSrcMO, ChosenDstReg);
974 } else {
975 // If the preferred size is large, then insert a truncate. For
976 // example:
977 // %1:_(s8) = G_LOAD ...
978 // %2:_(s64) = G_SEXT %1(s8)
979 // %3:_(s32) = G_ZEXT %1(s8)
980 // ... = ... %3(s32)
981 /// rewrites to:
982 // %2:_(s64) = G_SEXTLOAD ...
983 // %4:_(s8) = G_TRUNC %2:_(s32)
984 // %3:_(s64) = G_ZEXT %2:_(s8)
985 // ... = ... %3(s64)
986 InsertInsnsWithoutSideEffectsBeforeUse(Builder, MI, *UseMO,
987 InsertTruncAt);
988 }
989 continue;
990 }
991 // The use is (one of) the uses of the preferred use we chose earlier.
992 // We're going to update the load to def this value later so just erase
993 // the old extend.
994 Observer.erasingInstr(*UseMO->getParent());
995 UseMO->getParent()->eraseFromParent();
996 continue;
997 }
998
999 // The use isn't an extend. Truncate back to the type we originally loaded.
1000 // This is free on many targets.
1001 InsertInsnsWithoutSideEffectsBeforeUse(Builder, MI, *UseMO, InsertTruncAt);
1002 }
1003
1004 MI.getOperand(0).setReg(ChosenDstReg);
1005 Observer.changedInstr(MI);
1006}
1007
1009 BuildFnTy &MatchInfo) const {
1010 assert(MI.getOpcode() == TargetOpcode::G_AND);
1011
1012 // If we have the following code:
1013 // %mask = G_CONSTANT 255
1014 // %ld = G_LOAD %ptr, (load s16)
1015 // %and = G_AND %ld, %mask
1016 //
1017 // Try to fold it into
1018 // %ld = G_ZEXTLOAD %ptr, (load s8)
1019
1020 Register Dst = MI.getOperand(0).getReg();
1021 if (MRI.getType(Dst).isVector())
1022 return false;
1023
1024 auto MaybeMask =
1025 getIConstantVRegValWithLookThrough(MI.getOperand(2).getReg(), MRI);
1026 if (!MaybeMask)
1027 return false;
1028
1029 APInt MaskVal = MaybeMask->Value;
1030
1031 if (!MaskVal.isMask())
1032 return false;
1033
1034 Register SrcReg = MI.getOperand(1).getReg();
1035 // Don't use getOpcodeDef() here since intermediate instructions may have
1036 // multiple users.
1037 GAnyLoad *LoadMI;
1038 Register PtrReg;
1039 const MachineMemOperand *MMO;
1040 if (!mi_match(SrcReg, MRI, m_GAnyLoad(LoadMI, m_Reg(PtrReg), m_MMO(MMO))))
1041 return false;
1042
1043 Register LoadReg = LoadMI->getDstReg();
1044 LLT RegTy = MRI.getType(LoadReg);
1045 unsigned RegSize = RegTy.getSizeInBits();
1046 unsigned LoadSizeBits = MMO->getSizeInBits().getValue();
1047 unsigned MaskSizeBits = MaskVal.countr_one();
1048
1049 if ((isa<GSExtLoad>(LoadMI) || MaskSizeBits < LoadSizeBits) &&
1050 !MRI.hasOneNonDBGUse(LoadReg))
1051 return false;
1052
1053 // The mask may not be larger than the in-memory type, as it might cover sign
1054 // extended bits
1055 if (MaskSizeBits > LoadSizeBits)
1056 return false;
1057
1058 // If the mask covers the whole destination register, there's nothing to
1059 // extend
1060 if (MaskSizeBits >= RegSize)
1061 return false;
1062
1063 // Most targets cannot deal with loads of size < 8 and need to re-legalize to
1064 // at least byte loads. Avoid creating such loads here
1065 if (MaskSizeBits < 8 || !isPowerOf2_32(MaskSizeBits))
1066 return false;
1067
1068 LegalityQuery::MemDesc MemDesc(*MMO);
1069
1070 // Don't modify the memory access size if this is atomic/volatile, but we can
1071 // still adjust the opcode to indicate the high bit behavior.
1072 if (!MMO->isAtomic() && !MMO->isVolatile())
1073 MemDesc.MemoryTy = LLT::scalar(MaskSizeBits);
1074 else if (LoadSizeBits > MaskSizeBits || LoadSizeBits == RegSize)
1075 return false;
1076
1077 // TODO: Could check if it's legal with the reduced or original memory size.
1079 {TargetOpcode::G_ZEXTLOAD, {RegTy, MRI.getType(PtrReg)}, {MemDesc}}))
1080 return false;
1081
1082 MatchInfo = [=](MachineIRBuilder &B) {
1083 B.setInstrAndDebugLoc(*LoadMI);
1084 auto &MF = B.getMF();
1085 auto PtrInfo = MMO->getPointerInfo();
1086 auto *NewMMO = MF.getMachineMemOperand(MMO, PtrInfo, MemDesc.MemoryTy);
1087 B.buildLoadInstr(TargetOpcode::G_ZEXTLOAD, Dst, PtrReg, *NewMMO);
1088 replaceRegWith(MRI, LoadReg, Dst);
1089 LoadMI->eraseFromParent();
1090 };
1091 return true;
1092}
1093
1095 const MachineInstr &UseMI) const {
1096 assert(!DefMI.isDebugInstr() && !UseMI.isDebugInstr() &&
1097 "shouldn't consider debug uses");
1098 assert(DefMI.getParent() == UseMI.getParent());
1099 if (&DefMI == &UseMI)
1100 return true;
1101 const MachineBasicBlock &MBB = *DefMI.getParent();
1102 auto DefOrUse = find_if(MBB, [&DefMI, &UseMI](const MachineInstr &MI) {
1103 return &MI == &DefMI || &MI == &UseMI;
1104 });
1105 if (DefOrUse == MBB.end())
1106 llvm_unreachable("Block must contain both DefMI and UseMI!");
1107 return &*DefOrUse == &DefMI;
1108}
1109
1111 const MachineInstr &UseMI) const {
1112 assert(!DefMI.isDebugInstr() && !UseMI.isDebugInstr() &&
1113 "shouldn't consider debug uses");
1114 if (MDT)
1115 return MDT->dominates(&DefMI, &UseMI);
1116 else if (DefMI.getParent() != UseMI.getParent())
1117 return false;
1118
1119 return isPredecessor(DefMI, UseMI);
1120}
1121
1123 assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
1124 Register SrcReg = MI.getOperand(1).getReg();
1125 Register LoadUser = SrcReg;
1126
1127 if (MRI.getType(SrcReg).isVector())
1128 return false;
1129
1130 Register TruncSrc;
1131 if (mi_match(SrcReg, MRI, m_GTrunc(m_Reg(TruncSrc))))
1132 LoadUser = TruncSrc;
1133
1134 uint64_t SizeInBits = MI.getOperand(2).getImm();
1135 // If the source is a G_SEXTLOAD from the same bit width, then we don't
1136 // need any extend at all, just a truncate.
1137 if (auto *LoadMI = getOpcodeDef<GSExtLoad>(LoadUser, MRI)) {
1138 // If truncating more than the original extended value, abort.
1139 auto LoadSizeBits = LoadMI->getMemSizeInBits();
1140 if (TruncSrc &&
1141 MRI.getType(TruncSrc).getSizeInBits() < LoadSizeBits.getValue())
1142 return false;
1143 if (LoadSizeBits == SizeInBits)
1144 return true;
1145 }
1146 return false;
1147}
1148
1150 MachineInstr &MI, std::tuple<Register, unsigned> &MatchInfo) const {
1151 assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
1152
1153 Register DstReg = MI.getOperand(0).getReg();
1154 LLT RegTy = MRI.getType(DstReg);
1155
1156 // Only supports scalars for now.
1157 if (RegTy.isVector())
1158 return false;
1159
1160 Register SrcReg = MI.getOperand(1).getReg();
1161 Register PtrReg;
1162 const MachineMemOperand *MMO;
1163 if (!mi_match(SrcReg, MRI, m_GLoad(m_Reg(PtrReg), m_MMO(MMO))))
1164 return false;
1165
1166 uint64_t MemBits = MMO->getSizeInBits().getValue();
1167 uint64_t ExtFrom = MI.getOperand(2).getImm();
1168
1169 if (MemBits > ExtFrom && !MRI.hasOneNonDBGUse(SrcReg))
1170 return false;
1171
1172 // If the sign extend extends from a narrower width than the load's width,
1173 // then we can narrow the load width when we combine to a G_SEXTLOAD.
1174 // Avoid widening the load at all.
1175 unsigned NewSizeBits = std::min(ExtFrom, MemBits);
1176
1177 // Don't generate G_SEXTLOADs with a < 1 byte width.
1178 if (NewSizeBits < 8)
1179 return false;
1180 // Don't bother creating a non-power-2 sextload, it will likely be broken up
1181 // anyway for most targets.
1182 if (!isPowerOf2_32(NewSizeBits))
1183 return false;
1184
1185 LegalityQuery::MemDesc MMDesc(*MMO);
1186
1187 // Don't modify the memory access size if this is atomic/volatile, but we can
1188 // still adjust the opcode to indicate the high bit behavior.
1189 if (!MMO->isAtomic() && !MMO->isVolatile())
1190 MMDesc.MemoryTy = LLT::scalar(NewSizeBits);
1191 else if (MemBits > NewSizeBits || MemBits == RegTy.getSizeInBits())
1192 return false;
1193
1194 // TODO: Could check if it's legal with the reduced or original memory size.
1196 {TargetOpcode::G_SEXTLOAD, {RegTy, MRI.getType(PtrReg)}, {MMDesc}}))
1197 return false;
1198
1199 MatchInfo = std::make_tuple(SrcReg, NewSizeBits);
1200 return true;
1201}
1202
1204 MachineInstr &MI, std::tuple<Register, unsigned> &MatchInfo) const {
1205 assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
1206 Register LoadReg;
1207 unsigned ScalarSizeBits;
1208 std::tie(LoadReg, ScalarSizeBits) = MatchInfo;
1209 GLoad *LoadDef = cast<GLoad>(MRI.getVRegDef(LoadReg));
1210
1211 // If we have the following:
1212 // %ld = G_LOAD %ptr, (load 2)
1213 // %ext = G_SEXT_INREG %ld, 8
1214 // ==>
1215 // %ld = G_SEXTLOAD %ptr (load 1)
1216
1217 auto &MMO = LoadDef->getMMO();
1218 Builder.setInstrAndDebugLoc(*LoadDef);
1219 auto &MF = Builder.getMF();
1220 auto PtrInfo = MMO.getPointerInfo();
1221 auto *NewMMO = MF.getMachineMemOperand(&MMO, PtrInfo, ScalarSizeBits / 8);
1222 Builder.buildLoadInstr(TargetOpcode::G_SEXTLOAD, MI.getOperand(0).getReg(),
1223 LoadDef->getPointerReg(), *NewMMO);
1224 replaceRegWith(MRI, LoadReg, MI.getOperand(0).getReg());
1225 MI.eraseFromParent();
1226
1227 // Not all loads can be deleted, so make sure the old one is removed.
1228 LoadDef->eraseFromParent();
1229}
1230
1231/// Return true if 'MI' is a load or a store that may be fold it's address
1232/// operand into the load / store addressing mode.
1236 auto *MF = MI->getMF();
1237 auto *Addr = getOpcodeDef<GPtrAdd>(MI->getPointerReg(), MRI);
1238 if (!Addr)
1239 return false;
1240
1241 AM.HasBaseReg = true;
1242 if (auto CstOff = getIConstantVRegVal(Addr->getOffsetReg(), MRI))
1243 AM.BaseOffs = CstOff->getSExtValue(); // [reg +/- imm]
1244 else
1245 AM.Scale = 1; // [reg +/- reg]
1246
1247 return TLI.isLegalAddressingMode(
1248 MF->getDataLayout(), AM,
1249 getTypeForLLT(MI->getMMO().getMemoryType(),
1250 MF->getFunction().getContext()),
1251 MI->getMMO().getAddrSpace());
1252}
1253
1254static unsigned getIndexedOpc(unsigned LdStOpc) {
1255 switch (LdStOpc) {
1256 case TargetOpcode::G_LOAD:
1257 return TargetOpcode::G_INDEXED_LOAD;
1258 case TargetOpcode::G_STORE:
1259 return TargetOpcode::G_INDEXED_STORE;
1260 case TargetOpcode::G_ZEXTLOAD:
1261 return TargetOpcode::G_INDEXED_ZEXTLOAD;
1262 case TargetOpcode::G_SEXTLOAD:
1263 return TargetOpcode::G_INDEXED_SEXTLOAD;
1264 default:
1265 llvm_unreachable("Unexpected opcode");
1266 }
1267}
1268
1269bool CombinerHelper::isIndexedLoadStoreLegal(GLoadStore &LdSt) const {
1270 // Check for legality.
1271 LLT PtrTy = MRI.getType(LdSt.getPointerReg());
1272 LLT Ty = MRI.getType(LdSt.getReg(0));
1273 LLT MemTy = LdSt.getMMO().getMemoryType();
1275 {{MemTy, MemTy.getSizeInBits().getKnownMinValue(),
1277 unsigned IndexedOpc = getIndexedOpc(LdSt.getOpcode());
1278 SmallVector<LLT> OpTys;
1279 if (IndexedOpc == TargetOpcode::G_INDEXED_STORE)
1280 OpTys = {PtrTy, Ty, Ty};
1281 else
1282 OpTys = {Ty, PtrTy}; // For G_INDEXED_LOAD, G_INDEXED_[SZ]EXTLOAD
1283
1284 LegalityQuery Q(IndexedOpc, OpTys, MemDescrs);
1285 return isLegal(Q);
1286}
1287
1289 "post-index-use-threshold", cl::Hidden, cl::init(32),
1290 cl::desc("Number of uses of a base pointer to check before it is no longer "
1291 "considered for post-indexing."));
1292
1293bool CombinerHelper::findPostIndexCandidate(GLoadStore &LdSt, Register &Addr,
1295 bool &RematOffset) const {
1296 // We're looking for the following pattern, for either load or store:
1297 // %baseptr:_(p0) = ...
1298 // G_STORE %val(s64), %baseptr(p0)
1299 // %offset:_(s64) = G_CONSTANT i64 -256
1300 // %new_addr:_(p0) = G_PTR_ADD %baseptr, %offset(s64)
1301 const auto &TLI = getTargetLowering();
1302
1303 Register Ptr = LdSt.getPointerReg();
1304 // If the store is the only use, don't bother.
1305 if (MRI.hasOneNonDBGUse(Ptr))
1306 return false;
1307
1308 if (!isIndexedLoadStoreLegal(LdSt))
1309 return false;
1310
1311 if (getOpcodeDef(TargetOpcode::G_FRAME_INDEX, Ptr, MRI))
1312 return false;
1313
1314 MachineInstr *StoredValDef = getDefIgnoringCopies(LdSt.getReg(0), MRI);
1315 MachineInstr *PtrDef;
1316 if (!mi_match(Ptr, MRI, m_MInstr(PtrDef)))
1317 return false;
1318
1319 unsigned NumUsesChecked = 0;
1320 for (auto &Use : MRI.use_nodbg_instructions(Ptr)) {
1321 if (++NumUsesChecked > PostIndexUseThreshold)
1322 return false; // Try to avoid exploding compile time.
1323
1324 auto *PtrAdd = dyn_cast<GPtrAdd>(&Use);
1325 // The use itself might be dead. This can happen during combines if DCE
1326 // hasn't had a chance to run yet. Don't allow it to form an indexed op.
1327 if (!PtrAdd || MRI.use_nodbg_empty(PtrAdd->getReg(0)))
1328 continue;
1329
1330 // Check the user of this isn't the store, otherwise we'd be generate a
1331 // indexed store defining its own use.
1332 if (StoredValDef == &Use)
1333 continue;
1334
1335 Offset = PtrAdd->getOffsetReg();
1336 if (!ForceLegalIndexing &&
1337 !TLI.isIndexingLegal(LdSt, PtrAdd->getBaseReg(), Offset,
1338 /*IsPre*/ false, MRI))
1339 continue;
1340
1341 // Make sure the offset calculation is before the potentially indexed op.
1342 MachineInstr *OffsetDef;
1343 if (!mi_match(Offset, MRI, m_MInstr(OffsetDef)))
1344 continue;
1345 RematOffset = false;
1346 if (!dominates(*OffsetDef, LdSt)) {
1347 // If the offset however is just a G_CONSTANT, we can always just
1348 // rematerialize it where we need it.
1349 if (OffsetDef->getOpcode() != TargetOpcode::G_CONSTANT)
1350 continue;
1351 RematOffset = true;
1352 }
1353
1354 for (auto &BasePtrUse : MRI.use_nodbg_instructions(PtrAdd->getBaseReg())) {
1355 if (&BasePtrUse == PtrDef)
1356 continue;
1357
1358 // If the user is a later load/store that can be post-indexed, then don't
1359 // combine this one.
1360 auto *BasePtrLdSt = dyn_cast<GLoadStore>(&BasePtrUse);
1361 if (BasePtrLdSt && BasePtrLdSt != &LdSt &&
1362 dominates(LdSt, *BasePtrLdSt) &&
1363 isIndexedLoadStoreLegal(*BasePtrLdSt))
1364 return false;
1365
1366 // Now we're looking for the key G_PTR_ADD instruction, which contains
1367 // the offset add that we want to fold.
1368 if (auto *BasePtrUseDef = dyn_cast<GPtrAdd>(&BasePtrUse)) {
1369 Register PtrAddDefReg = BasePtrUseDef->getReg(0);
1370 for (auto &BaseUseUse : MRI.use_nodbg_instructions(PtrAddDefReg)) {
1371 // If the use is in a different block, then we may produce worse code
1372 // due to the extra register pressure.
1373 if (BaseUseUse.getParent() != LdSt.getParent())
1374 return false;
1375
1376 if (auto *UseUseLdSt = dyn_cast<GLoadStore>(&BaseUseUse))
1377 if (canFoldInAddressingMode(UseUseLdSt, TLI, MRI))
1378 return false;
1379 }
1380 if (!dominates(LdSt, BasePtrUse))
1381 return false; // All use must be dominated by the load/store.
1382 }
1383 }
1384
1385 Addr = PtrAdd->getReg(0);
1386 Base = PtrAdd->getBaseReg();
1387 return true;
1388 }
1389
1390 return false;
1391}
1392
1393bool CombinerHelper::findPreIndexCandidate(GLoadStore &LdSt, Register &Addr,
1394 Register &Base,
1395 Register &Offset) const {
1396 auto &MF = *LdSt.getParent()->getParent();
1397 const auto &TLI = *MF.getSubtarget().getTargetLowering();
1398
1399 Addr = LdSt.getPointerReg();
1400 if (!mi_match(Addr, MRI, m_GPtrAdd(m_Reg(Base), m_Reg(Offset))) ||
1401 MRI.hasOneNonDBGUse(Addr))
1402 return false;
1403
1404 if (!ForceLegalIndexing &&
1405 !TLI.isIndexingLegal(LdSt, Base, Offset, /*IsPre*/ true, MRI))
1406 return false;
1407
1408 if (!isIndexedLoadStoreLegal(LdSt))
1409 return false;
1410
1411 MachineInstr *BaseDef = getDefIgnoringCopies(Base, MRI);
1412 if (BaseDef->getOpcode() == TargetOpcode::G_FRAME_INDEX)
1413 return false;
1414
1415 if (auto *St = dyn_cast<GStore>(&LdSt)) {
1416 // Would require a copy.
1417 if (Base == St->getValueReg())
1418 return false;
1419
1420 // We're expecting one use of Addr in MI, but it could also be the
1421 // value stored, which isn't actually dominated by the instruction.
1422 if (St->getValueReg() == Addr)
1423 return false;
1424 }
1425
1426 // Avoid increasing cross-block register pressure.
1427 for (auto &AddrUse : MRI.use_nodbg_instructions(Addr))
1428 if (AddrUse.getParent() != LdSt.getParent())
1429 return false;
1430
1431 // FIXME: check whether all uses of the base pointer are constant PtrAdds.
1432 // That might allow us to end base's liveness here by adjusting the constant.
1433 bool RealUse = false;
1434 for (auto &AddrUse : MRI.use_nodbg_instructions(Addr)) {
1435 if (!dominates(LdSt, AddrUse))
1436 return false; // All use must be dominated by the load/store.
1437
1438 // If Ptr may be folded in addressing mode of other use, then it's
1439 // not profitable to do this transformation.
1440 if (auto *UseLdSt = dyn_cast<GLoadStore>(&AddrUse)) {
1441 if (!canFoldInAddressingMode(UseLdSt, TLI, MRI))
1442 RealUse = true;
1443 } else {
1444 RealUse = true;
1445 }
1446 }
1447 return RealUse;
1448}
1449
1451 MachineInstr &MI, BuildFnTy &MatchInfo) const {
1452 assert(MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT);
1453
1454 // Check if there is a load that defines the vector being extracted from.
1455 auto *LoadMI = getOpcodeDef<GLoad>(MI.getOperand(1).getReg(), MRI);
1456 if (!LoadMI)
1457 return false;
1458
1459 Register Vector = MI.getOperand(1).getReg();
1460 LLT VecEltTy = MRI.getType(Vector).getElementType();
1461
1462 assert(MRI.getType(MI.getOperand(0).getReg()) == VecEltTy);
1463
1464 // Checking whether we should reduce the load width.
1465 if (!MRI.hasOneNonDBGUse(Vector))
1466 return false;
1467
1468 // Check if the defining load is simple.
1469 if (!LoadMI->isSimple())
1470 return false;
1471
1472 // If the vector element type is not a multiple of a byte then we are unable
1473 // to correctly compute an address to load only the extracted element as a
1474 // scalar.
1475 if (!VecEltTy.isByteSized())
1476 return false;
1477
1478 // Check for load fold barriers between the extraction and the load.
1479 if (MI.getParent() != LoadMI->getParent())
1480 return false;
1481 const unsigned MaxIter = 20;
1482 unsigned Iter = 0;
1483 for (const auto &Instr :
1484 instructionsWithoutDebug(LoadMI->getIterator(), MI.getIterator())) {
1485 if (Instr.isLoadFoldBarrier())
1486 return false;
1487 if (Iter++ == MaxIter)
1488 return false;
1489 }
1490
1491 // Check if the new load that we are going to create is legal
1492 // if we are in the post-legalization phase.
1493 MachineMemOperand MMO = LoadMI->getMMO();
1494 Align Alignment = MMO.getAlign();
1495 MachinePointerInfo PtrInfo;
1496 uint64_t Offset;
1497
1498 // Finding the appropriate PtrInfo if offset is a known constant.
1499 // This is required to create the memory operand for the narrowed load.
1500 // This machine memory operand object helps us infer about legality
1501 // before we proceed to combine the instruction.
1502 Register Idx = MI.getOperand(2).getReg();
1503 if (auto CVal = getIConstantVRegVal(Idx, MRI)) {
1504 int Elt = CVal->getZExtValue();
1505 Offset = VecEltTy.getSizeInBits() * Elt / 8;
1506 PtrInfo = MMO.getPointerInfo().getWithOffset(Offset);
1507 } else {
1508 // Discard the pointer info except the address space because the memory
1509 // operand can't represent this new access since the offset is variable.
1510 Offset = VecEltTy.getSizeInBits() / 8;
1512 }
1513
1514 Alignment = commonAlignment(Alignment, Offset);
1515
1516 Register VecPtr = LoadMI->getPointerReg();
1517 LLT PtrTy = MRI.getType(VecPtr);
1518
1519 MachineFunction &MF = *MI.getMF();
1520 auto *NewMMO = MF.getMachineMemOperand(&MMO, PtrInfo, VecEltTy);
1521
1522 LegalityQuery::MemDesc MMDesc(*NewMMO);
1523
1525 {TargetOpcode::G_LOAD, {VecEltTy, PtrTy}, {MMDesc}}))
1526 return false;
1527
1528 // Load must be allowed and fast on the target.
1530 auto &DL = MF.getDataLayout();
1531 unsigned Fast = 0;
1532 if (!getTargetLowering().allowsMemoryAccess(C, DL, VecEltTy, *NewMMO,
1533 &Fast) ||
1534 !Fast)
1535 return false;
1536
1537 Register Result = MI.getOperand(0).getReg();
1538 Register Index = MI.getOperand(2).getReg();
1539
1540 MatchInfo = [=](MachineIRBuilder &B) {
1541 GISelObserverWrapper DummyObserver;
1542 LegalizerHelper Helper(B.getMF(), DummyObserver, B);
1543 //// Get pointer to the vector element.
1544 Register finalPtr = Helper.getVectorElementPointer(
1545 LoadMI->getPointerReg(), MRI.getType(LoadMI->getOperand(0).getReg()),
1546 Index);
1547 // New G_LOAD instruction.
1548 B.buildLoad(Result, finalPtr, PtrInfo, Alignment);
1549 // Remove original GLOAD instruction.
1550 LoadMI->eraseFromParent();
1551 };
1552
1553 return true;
1554}
1555
1557 MachineInstr &MI, IndexedLoadStoreMatchInfo &MatchInfo) const {
1558 auto &LdSt = cast<GLoadStore>(MI);
1559
1560 if (LdSt.isAtomic())
1561 return false;
1562
1563 MatchInfo.IsPre = findPreIndexCandidate(LdSt, MatchInfo.Addr, MatchInfo.Base,
1564 MatchInfo.Offset);
1565 if (!MatchInfo.IsPre &&
1566 !findPostIndexCandidate(LdSt, MatchInfo.Addr, MatchInfo.Base,
1567 MatchInfo.Offset, MatchInfo.RematOffset))
1568 return false;
1569
1570 return true;
1571}
1572
1574 MachineInstr &MI, IndexedLoadStoreMatchInfo &MatchInfo) const {
1575 MachineInstr &AddrDef = *MRI.getVRegDef(MatchInfo.Addr);
1576 unsigned Opcode = MI.getOpcode();
1577 bool IsStore = Opcode == TargetOpcode::G_STORE;
1578 unsigned NewOpcode = getIndexedOpc(Opcode);
1579
1580 // If the offset constant didn't happen to dominate the load/store, we can
1581 // just clone it as needed.
1582 if (MatchInfo.RematOffset) {
1583 auto *OldCst = MRI.getVRegDef(MatchInfo.Offset);
1584 auto NewCst = Builder.buildConstant(MRI.getType(MatchInfo.Offset),
1585 *OldCst->getOperand(1).getCImm());
1586 MatchInfo.Offset = NewCst.getReg(0);
1587 }
1588
1589 auto MIB = Builder.buildInstr(NewOpcode);
1590 if (IsStore) {
1591 MIB.addDef(MatchInfo.Addr);
1592 MIB.addUse(MI.getOperand(0).getReg());
1593 } else {
1594 MIB.addDef(MI.getOperand(0).getReg());
1595 MIB.addDef(MatchInfo.Addr);
1596 }
1597
1598 MIB.addUse(MatchInfo.Base);
1599 MIB.addUse(MatchInfo.Offset);
1600 MIB.addImm(MatchInfo.IsPre);
1601 MIB->cloneMemRefs(*MI.getMF(), MI);
1602 MI.eraseFromParent();
1603 AddrDef.eraseFromParent();
1604
1605 LLVM_DEBUG(dbgs() << " Combinined to indexed operation");
1606}
1607
1609 MachineInstr *&OtherMI) const {
1610 unsigned Opcode = MI.getOpcode();
1611 bool IsDiv, IsSigned;
1612
1613 switch (Opcode) {
1614 default:
1615 llvm_unreachable("Unexpected opcode!");
1616 case TargetOpcode::G_SDIV:
1617 case TargetOpcode::G_UDIV: {
1618 IsDiv = true;
1619 IsSigned = Opcode == TargetOpcode::G_SDIV;
1620 break;
1621 }
1622 case TargetOpcode::G_SREM:
1623 case TargetOpcode::G_UREM: {
1624 IsDiv = false;
1625 IsSigned = Opcode == TargetOpcode::G_SREM;
1626 break;
1627 }
1628 }
1629
1630 Register Src1 = MI.getOperand(1).getReg();
1631 unsigned DivOpcode, RemOpcode, DivremOpcode;
1632 if (IsSigned) {
1633 DivOpcode = TargetOpcode::G_SDIV;
1634 RemOpcode = TargetOpcode::G_SREM;
1635 DivremOpcode = TargetOpcode::G_SDIVREM;
1636 } else {
1637 DivOpcode = TargetOpcode::G_UDIV;
1638 RemOpcode = TargetOpcode::G_UREM;
1639 DivremOpcode = TargetOpcode::G_UDIVREM;
1640 }
1641
1642 if (!isLegalOrBeforeLegalizer({DivremOpcode, {MRI.getType(Src1)}}))
1643 return false;
1644
1645 // Combine:
1646 // %div:_ = G_[SU]DIV %src1:_, %src2:_
1647 // %rem:_ = G_[SU]REM %src1:_, %src2:_
1648 // into:
1649 // %div:_, %rem:_ = G_[SU]DIVREM %src1:_, %src2:_
1650
1651 // Combine:
1652 // %rem:_ = G_[SU]REM %src1:_, %src2:_
1653 // %div:_ = G_[SU]DIV %src1:_, %src2:_
1654 // into:
1655 // %div:_, %rem:_ = G_[SU]DIVREM %src1:_, %src2:_
1656
1657 for (auto &UseMI : MRI.use_nodbg_instructions(Src1)) {
1658 if (MI.getParent() == UseMI.getParent() &&
1659 ((IsDiv && UseMI.getOpcode() == RemOpcode) ||
1660 (!IsDiv && UseMI.getOpcode() == DivOpcode)) &&
1661 matchEqualDefs(MI.getOperand(2), UseMI.getOperand(2)) &&
1662 matchEqualDefs(MI.getOperand(1), UseMI.getOperand(1))) {
1663 OtherMI = &UseMI;
1664 return true;
1665 }
1666 }
1667
1668 return false;
1669}
1670
1672 MachineInstr *&OtherMI) const {
1673 unsigned Opcode = MI.getOpcode();
1674 assert(OtherMI && "OtherMI shouldn't be empty.");
1675
1676 Register DestDivReg, DestRemReg;
1677 if (Opcode == TargetOpcode::G_SDIV || Opcode == TargetOpcode::G_UDIV) {
1678 DestDivReg = MI.getOperand(0).getReg();
1679 DestRemReg = OtherMI->getOperand(0).getReg();
1680 } else {
1681 DestDivReg = OtherMI->getOperand(0).getReg();
1682 DestRemReg = MI.getOperand(0).getReg();
1683 }
1684
1685 bool IsSigned =
1686 Opcode == TargetOpcode::G_SDIV || Opcode == TargetOpcode::G_SREM;
1687
1688 // Check which instruction is first in the block so we don't break def-use
1689 // deps by "moving" the instruction incorrectly. Also keep track of which
1690 // instruction is first so we pick it's operands, avoiding use-before-def
1691 // bugs.
1692 MachineInstr *FirstInst = dominates(MI, *OtherMI) ? &MI : OtherMI;
1693 Builder.setInstrAndDebugLoc(*FirstInst);
1694
1695 Builder.buildInstr(IsSigned ? TargetOpcode::G_SDIVREM
1696 : TargetOpcode::G_UDIVREM,
1697 {DestDivReg, DestRemReg},
1698 { FirstInst->getOperand(1), FirstInst->getOperand(2) });
1699 MI.eraseFromParent();
1700 OtherMI->eraseFromParent();
1701}
1702
1704 MachineInstr &MI, MachineInstr *&BrCond) const {
1705 assert(MI.getOpcode() == TargetOpcode::G_BR);
1706
1707 // Try to match the following:
1708 // bb1:
1709 // G_BRCOND %c1, %bb2
1710 // G_BR %bb3
1711 // bb2:
1712 // ...
1713 // bb3:
1714
1715 // The above pattern does not have a fall through to the successor bb2, always
1716 // resulting in a branch no matter which path is taken. Here we try to find
1717 // and replace that pattern with conditional branch to bb3 and otherwise
1718 // fallthrough to bb2. This is generally better for branch predictors.
1719
1720 MachineBasicBlock *MBB = MI.getParent();
1722 if (BrIt == MBB->begin())
1723 return false;
1724 assert(std::next(BrIt) == MBB->end() && "expected G_BR to be a terminator");
1725
1726 BrCond = &*std::prev(BrIt);
1727 if (BrCond->getOpcode() != TargetOpcode::G_BRCOND)
1728 return false;
1729
1730 // Check that the next block is the conditional branch target. Also make sure
1731 // that it isn't the same as the G_BR's target (otherwise, this will loop.)
1732 MachineBasicBlock *BrCondTarget = BrCond->getOperand(1).getMBB();
1733 return BrCondTarget != MI.getOperand(0).getMBB() &&
1734 MBB->isLayoutSuccessor(BrCondTarget);
1735}
1736
1738 MachineInstr &MI, MachineInstr *&BrCond) const {
1739 MachineBasicBlock *BrTarget = MI.getOperand(0).getMBB();
1740 Builder.setInstrAndDebugLoc(*BrCond);
1741 LLT Ty = MRI.getType(BrCond->getOperand(0).getReg());
1742 // FIXME: Does int/fp matter for this? If so, we might need to restrict
1743 // this to i1 only since we might not know for sure what kind of
1744 // compare generated the condition value.
1745 auto True = Builder.buildConstant(
1746 Ty, getICmpTrueVal(getTargetLowering(), false, false));
1747 auto Xor = Builder.buildXor(Ty, BrCond->getOperand(0), True);
1748
1749 auto *FallthroughBB = BrCond->getOperand(1).getMBB();
1750 Observer.changingInstr(MI);
1751 MI.getOperand(0).setMBB(FallthroughBB);
1752 Observer.changedInstr(MI);
1753
1754 // Change the conditional branch to use the inverted condition and
1755 // new target block.
1756 Observer.changingInstr(*BrCond);
1757 BrCond->getOperand(0).setReg(Xor.getReg(0));
1758 BrCond->getOperand(1).setMBB(BrTarget);
1759 Observer.changedInstr(*BrCond);
1760}
1761
1764 unsigned MaxLen) const {
1765 auto &[Dst, Src, KnownLen, Alignment, DstAlignCanChange, MemOps] = MatchInfo;
1766 return canLowerMemCpyFamily(MI, MRI, MaxLen, Dst, Src, KnownLen, Alignment,
1767 DstAlignCanChange, MemOps);
1768}
1769
1771 MachineInstr &MI, MemCpyFamilyLoweringInfo &MatchInfo) const {
1772 auto &[Dst, Src, KnownLen, Alignment, DstAlignCanChange, MemOps] = MatchInfo;
1773 MachineIRBuilder HelperBuilder(MI);
1774 GISelObserverWrapper DummyObserver;
1775 LegalizerHelper Helper(HelperBuilder.getMF(), DummyObserver, HelperBuilder);
1776 bool Changed = Helper.lowerMemCpyFamily(MI, Dst, Src, KnownLen, Alignment,
1777 DstAlignCanChange, MemOps) ==
1779 assert(Changed && "expected memcpy-family instruction to lower");
1780 (void)Changed;
1781}
1782
1784 unsigned MaxLen) const {
1785 MachineIRBuilder HelperBuilder(MI);
1786 GISelObserverWrapper DummyObserver;
1787 LegalizerHelper Helper(HelperBuilder.getMF(), DummyObserver, HelperBuilder);
1788 return Helper.lowerMemCpyFamily(MI, MaxLen) ==
1790}
1791
1793 PtrAddChain &MatchInfo) const {
1794 // We're trying to match the following pattern:
1795 // %t1 = G_PTR_ADD %base, G_CONSTANT imm1
1796 // %root = G_PTR_ADD %t1, G_CONSTANT imm2
1797 // -->
1798 // %root = G_PTR_ADD %base, G_CONSTANT (imm1 + imm2)
1799
1800 if (MI.getOpcode() != TargetOpcode::G_PTR_ADD)
1801 return false;
1802
1803 Register Add2 = MI.getOperand(1).getReg();
1804 Register Imm1 = MI.getOperand(2).getReg();
1805 auto MaybeImmVal = getIConstantVRegValWithLookThrough(Imm1, MRI);
1806 if (!MaybeImmVal)
1807 return false;
1808
1809 Register Base, Imm2;
1810 uint32_t LHSPtrAddFlags;
1811 if (!mi_match(Add2, MRI,
1812 m_GPtrAdd(m_Reg(Base), m_Reg(Imm2), m_MIFlags(LHSPtrAddFlags))))
1813 return false;
1814
1815 auto MaybeImm2Val = getIConstantVRegValWithLookThrough(Imm2, MRI);
1816 if (!MaybeImm2Val)
1817 return false;
1818
1819 // Check if the new combined immediate forms an illegal addressing mode.
1820 // Do not combine if it was legal before but would get illegal.
1821 // To do so, we need to find a load/store user of the pointer to get
1822 // the access type.
1823 Type *AccessTy = nullptr;
1824 auto &MF = *MI.getMF();
1825 for (auto &UseMI : MRI.use_nodbg_instructions(MI.getOperand(0).getReg())) {
1826 if (auto *LdSt = dyn_cast<GLoadStore>(&UseMI)) {
1827 AccessTy = getTypeForLLT(MRI.getType(LdSt->getReg(0)),
1828 MF.getFunction().getContext());
1829 break;
1830 }
1831 }
1833 APInt CombinedImm = MaybeImmVal->Value + MaybeImm2Val->Value;
1834 AMNew.BaseOffs = CombinedImm.getSExtValue();
1835 if (AccessTy) {
1836 AMNew.HasBaseReg = true;
1838 AMOld.BaseOffs = MaybeImmVal->Value.getSExtValue();
1839 AMOld.HasBaseReg = true;
1840 unsigned AS = MRI.getType(Add2).getAddressSpace();
1841 const auto &TLI = *MF.getSubtarget().getTargetLowering();
1842 if (TLI.isLegalAddressingMode(MF.getDataLayout(), AMOld, AccessTy, AS) &&
1843 !TLI.isLegalAddressingMode(MF.getDataLayout(), AMNew, AccessTy, AS))
1844 return false;
1845 }
1846
1847 // Reassociating nuw additions preserves nuw. If both original G_PTR_ADDs are
1848 // inbounds, reaching the same result in one G_PTR_ADD is also inbounds.
1849 // The nusw constraints are satisfied because imm1+imm2 cannot exceed the
1850 // largest signed integer that fits into the index type, which is the maximum
1851 // size of allocated objects according to the IR Language Reference.
1852 unsigned PtrAddFlags = MI.getFlags();
1853 bool IsNoUWrap = PtrAddFlags & LHSPtrAddFlags & MachineInstr::MIFlag::NoUWrap;
1854 bool IsInBounds =
1855 PtrAddFlags & LHSPtrAddFlags & MachineInstr::MIFlag::InBounds;
1856 unsigned Flags = 0;
1857 if (IsNoUWrap)
1859 if (IsInBounds) {
1862 }
1863
1864 // Pass the combined immediate to the apply function.
1865 MatchInfo.Imm = AMNew.BaseOffs;
1866 MatchInfo.Base = Base;
1867 MatchInfo.Bank = getRegBank(Imm2);
1868 MatchInfo.Flags = Flags;
1869 return true;
1870}
1871
1873 PtrAddChain &MatchInfo) const {
1874 assert(MI.getOpcode() == TargetOpcode::G_PTR_ADD && "Expected G_PTR_ADD");
1875 MachineIRBuilder MIB(MI);
1876 LLT OffsetTy = MRI.getType(MI.getOperand(2).getReg());
1877 auto NewOffset = MIB.buildConstant(OffsetTy, MatchInfo.Imm);
1878 setRegBank(NewOffset.getReg(0), MatchInfo.Bank);
1879 Observer.changingInstr(MI);
1880 MI.getOperand(1).setReg(MatchInfo.Base);
1881 MI.getOperand(2).setReg(NewOffset.getReg(0));
1882 MI.setFlags(MatchInfo.Flags);
1883 Observer.changedInstr(MI);
1884}
1885
1887 RegisterImmPair &MatchInfo) const {
1888 // We're trying to match the following pattern with any of
1889 // G_SHL/G_ASHR/G_LSHR/G_SSHLSAT/G_USHLSAT shift instructions:
1890 // %t1 = SHIFT %base, G_CONSTANT imm1
1891 // %root = SHIFT %t1, G_CONSTANT imm2
1892 // -->
1893 // %root = SHIFT %base, G_CONSTANT (imm1 + imm2)
1894
1895 unsigned Opcode = MI.getOpcode();
1896 assert((Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_ASHR ||
1897 Opcode == TargetOpcode::G_LSHR || Opcode == TargetOpcode::G_SSHLSAT ||
1898 Opcode == TargetOpcode::G_USHLSAT) &&
1899 "Expected G_SHL, G_ASHR, G_LSHR, G_SSHLSAT or G_USHLSAT");
1900
1901 Register Shl2 = MI.getOperand(1).getReg();
1902 Register Imm1 = MI.getOperand(2).getReg();
1903 auto MaybeImmVal = getIConstantVRegValWithLookThrough(Imm1, MRI);
1904 if (!MaybeImmVal)
1905 return false;
1906
1907 MachineInstr *Shl2Def;
1908 if (!mi_match(Shl2, MRI, m_MInstr(Shl2Def)) || Shl2Def->getOpcode() != Opcode)
1909 return false;
1910
1911 Register Base = Shl2Def->getOperand(1).getReg();
1912 Register Imm2 = Shl2Def->getOperand(2).getReg();
1913 auto MaybeImm2Val = getIConstantVRegValWithLookThrough(Imm2, MRI);
1914 if (!MaybeImm2Val)
1915 return false;
1916
1917 // Pass the combined immediate to the apply function.
1918 MatchInfo.Imm =
1919 (MaybeImmVal->Value.getZExtValue() + MaybeImm2Val->Value).getZExtValue();
1920 MatchInfo.Reg = Base;
1921
1922 // There is no simple replacement for a saturating unsigned left shift that
1923 // exceeds the scalar size.
1924 if (Opcode == TargetOpcode::G_USHLSAT &&
1925 MatchInfo.Imm >= MRI.getType(Shl2).getScalarSizeInBits())
1926 return false;
1927
1928 return true;
1929}
1930
1932 RegisterImmPair &MatchInfo) const {
1933 unsigned Opcode = MI.getOpcode();
1934 assert((Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_ASHR ||
1935 Opcode == TargetOpcode::G_LSHR || Opcode == TargetOpcode::G_SSHLSAT ||
1936 Opcode == TargetOpcode::G_USHLSAT) &&
1937 "Expected G_SHL, G_ASHR, G_LSHR, G_SSHLSAT or G_USHLSAT");
1938
1939 LLT Ty = MRI.getType(MI.getOperand(1).getReg());
1940 unsigned const ScalarSizeInBits = Ty.getScalarSizeInBits();
1941 auto Imm = MatchInfo.Imm;
1942
1943 if (Imm >= ScalarSizeInBits) {
1944 // Any logical shift that exceeds scalar size will produce zero.
1945 if (Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_LSHR) {
1946 Builder.buildConstant(MI.getOperand(0), 0);
1947 MI.eraseFromParent();
1948 return;
1949 }
1950 // Arithmetic shift and saturating signed left shift have no effect beyond
1951 // scalar size.
1952 Imm = ScalarSizeInBits - 1;
1953 }
1954
1955 LLT ImmTy = MRI.getType(MI.getOperand(2).getReg());
1956 Register NewImm = Builder.buildConstant(ImmTy, Imm).getReg(0);
1957 Observer.changingInstr(MI);
1958 MI.getOperand(1).setReg(MatchInfo.Reg);
1959 MI.getOperand(2).setReg(NewImm);
1960 Observer.changedInstr(MI);
1961}
1962
1964 MachineInstr &MI, ShiftOfShiftedLogic &MatchInfo) const {
1965 // We're trying to match the following pattern with any of
1966 // G_SHL/G_ASHR/G_LSHR shift instructions in combination
1967 // with any of G_AND/G_OR/G_XOR logic instructions.
1968 // %t1 = SHIFT %X, G_CONSTANT C0
1969 // %t2 = LOGIC %t1, %Y
1970 // %root = SHIFT %t2, G_CONSTANT C1
1971 // -->
1972 // %t3 = SHIFT %X, G_CONSTANT (C0+C1)
1973 // %t4 = SHIFT %Y, G_CONSTANT C1
1974 // %root = LOGIC %t3, %t4
1975 unsigned ShiftOpcode = MI.getOpcode();
1976 assert((ShiftOpcode == TargetOpcode::G_SHL ||
1977 ShiftOpcode == TargetOpcode::G_ASHR ||
1978 ShiftOpcode == TargetOpcode::G_LSHR) &&
1979 "Expected G_SHL, G_ASHR and G_LSHR");
1980
1981 // Match a one-use bitwise logic op.
1982 Register LogicDest = MI.getOperand(1).getReg();
1983 if (!MRI.hasOneNonDBGUse(LogicDest))
1984 return false;
1985
1986 MachineInstr *LogicMI;
1987 if (!mi_match(LogicDest, MRI, m_MInstr(LogicMI)))
1988 return false;
1989 unsigned LogicOpcode = LogicMI->getOpcode();
1990 if (LogicOpcode != TargetOpcode::G_AND && LogicOpcode != TargetOpcode::G_OR &&
1991 LogicOpcode != TargetOpcode::G_XOR)
1992 return false;
1993
1994 // Find a matching one-use shift by constant.
1995 const Register C1 = MI.getOperand(2).getReg();
1996 auto MaybeImmVal = getIConstantVRegValWithLookThrough(C1, MRI);
1997 if (!MaybeImmVal || MaybeImmVal->Value == 0)
1998 return false;
1999
2000 const uint64_t C1Val = MaybeImmVal->Value.getZExtValue();
2001
2002 auto matchFirstShift = [&](const MachineInstr *MI, uint64_t &ShiftVal) {
2003 // Shift should match previous one and should be a one-use.
2004 if (MI->getOpcode() != ShiftOpcode ||
2005 !MRI.hasOneNonDBGUse(MI->getOperand(0).getReg()))
2006 return false;
2007
2008 // Must be a constant.
2009 auto MaybeImmVal =
2010 getIConstantVRegValWithLookThrough(MI->getOperand(2).getReg(), MRI);
2011 if (!MaybeImmVal)
2012 return false;
2013
2014 ShiftVal = MaybeImmVal->Value.getSExtValue();
2015 return true;
2016 };
2017
2018 // Logic ops are commutative, so check each operand for a match.
2019 Register LogicMIReg1 = LogicMI->getOperand(1).getReg();
2020 MachineInstr *LogicMIOp1;
2021 Register LogicMIReg2 = LogicMI->getOperand(2).getReg();
2022 MachineInstr *LogicMIOp2;
2023 if (!mi_match(LogicMIReg1, MRI, m_MInstr(LogicMIOp1)) ||
2024 !mi_match(LogicMIReg2, MRI, m_MInstr(LogicMIOp2)))
2025 return false;
2026 uint64_t C0Val;
2027
2028 if (matchFirstShift(LogicMIOp1, C0Val)) {
2029 MatchInfo.LogicNonShiftReg = LogicMIReg2;
2030 MatchInfo.Shift2 = LogicMIOp1;
2031 } else if (matchFirstShift(LogicMIOp2, C0Val)) {
2032 MatchInfo.LogicNonShiftReg = LogicMIReg1;
2033 MatchInfo.Shift2 = LogicMIOp2;
2034 } else
2035 return false;
2036
2037 MatchInfo.ValSum = C0Val + C1Val;
2038
2039 // The fold is not valid if the sum of the shift values exceeds bitwidth.
2040 if (MatchInfo.ValSum >= MRI.getType(LogicDest).getScalarSizeInBits())
2041 return false;
2042
2043 MatchInfo.Logic = LogicMI;
2044 return true;
2045}
2046
2048 MachineInstr &MI, ShiftOfShiftedLogic &MatchInfo) const {
2049 unsigned Opcode = MI.getOpcode();
2050 assert((Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_ASHR ||
2051 Opcode == TargetOpcode::G_LSHR) &&
2052 "Expected G_SHL, G_ASHR and G_LSHR");
2053
2054 LLT ShlType = MRI.getType(MI.getOperand(2).getReg());
2055 LLT DestType = MRI.getType(MI.getOperand(0).getReg());
2056
2057 Register Const = Builder.buildConstant(ShlType, MatchInfo.ValSum).getReg(0);
2058
2059 Register Shift1Base = MatchInfo.Shift2->getOperand(1).getReg();
2060 Register Shift1 =
2061 Builder.buildInstr(Opcode, {DestType}, {Shift1Base, Const}).getReg(0);
2062
2063 // If LogicNonShiftReg is the same to Shift1Base, and shift1 const is the same
2064 // to MatchInfo.Shift2 const, CSEMIRBuilder will reuse the old shift1 when
2065 // build shift2. So, if we erase MatchInfo.Shift2 at the end, actually we
2066 // remove old shift1. And it will cause crash later. So erase it earlier to
2067 // avoid the crash.
2068 MatchInfo.Shift2->eraseFromParent();
2069
2070 Register Shift2Const = MI.getOperand(2).getReg();
2071 Register Shift2 = Builder
2072 .buildInstr(Opcode, {DestType},
2073 {MatchInfo.LogicNonShiftReg, Shift2Const})
2074 .getReg(0);
2075
2076 Register Dest = MI.getOperand(0).getReg();
2077 Builder.buildInstr(MatchInfo.Logic->getOpcode(), {Dest}, {Shift1, Shift2});
2078
2079 // This was one use so it's safe to remove it.
2080 MatchInfo.Logic->eraseFromParent();
2081
2082 MI.eraseFromParent();
2083}
2084
2090
2092 LshrOfTruncOfLshr &MatchInfo,
2093 MachineInstr &ShiftMI) const {
2094 assert(MI.getOpcode() == TargetOpcode::G_LSHR && "Expected a G_LSHR");
2095
2096 Register N0 = MI.getOperand(1).getReg();
2097 Register N1 = MI.getOperand(2).getReg();
2098 unsigned OpSizeInBits = MRI.getType(N0).getScalarSizeInBits();
2099
2100 APInt N1C, N001C;
2101 if (!mi_match(N1, MRI, m_ICstOrSplat(N1C)))
2102 return false;
2103 auto N001 = ShiftMI.getOperand(2).getReg();
2104 if (!mi_match(N001, MRI, m_ICstOrSplat(N001C)))
2105 return false;
2106
2107 if (N001C.getBitWidth() > N1C.getBitWidth())
2108 N1C = N1C.zext(N001C.getBitWidth());
2109 else
2110 N001C = N001C.zext(N1C.getBitWidth());
2111
2112 Register InnerShift = ShiftMI.getOperand(0).getReg();
2113 LLT InnerShiftTy = MRI.getType(InnerShift);
2114 uint64_t InnerShiftSize = InnerShiftTy.getScalarSizeInBits();
2115 if ((N1C + N001C).ult(InnerShiftSize)) {
2116 MatchInfo.Src = ShiftMI.getOperand(1).getReg();
2117 MatchInfo.ShiftAmt = N1C + N001C;
2118 MatchInfo.ShiftAmtTy = MRI.getType(N001);
2119 MatchInfo.InnerShiftTy = InnerShiftTy;
2120
2121 if ((N001C + OpSizeInBits) == InnerShiftSize)
2122 return true;
2123 if (MRI.hasOneNonDBGUse(N0) && MRI.hasOneNonDBGUse(InnerShift)) {
2124 MatchInfo.Mask = true;
2125 MatchInfo.MaskVal = APInt(N1C.getBitWidth(), OpSizeInBits) - N1C;
2126 return true;
2127 }
2128 }
2129 return false;
2130}
2131
2133 MachineInstr &MI, LshrOfTruncOfLshr &MatchInfo) const {
2134 assert(MI.getOpcode() == TargetOpcode::G_LSHR && "Expected a G_LSHR");
2135
2136 Register Dst = MI.getOperand(0).getReg();
2137 auto ShiftAmt =
2138 Builder.buildConstant(MatchInfo.ShiftAmtTy, MatchInfo.ShiftAmt);
2139 auto Shift =
2140 Builder.buildLShr(MatchInfo.InnerShiftTy, MatchInfo.Src, ShiftAmt);
2141 if (MatchInfo.Mask == true) {
2142 APInt MaskVal =
2144 MatchInfo.MaskVal.getZExtValue());
2145 auto Mask = Builder.buildConstant(MatchInfo.InnerShiftTy, MaskVal);
2146 auto And = Builder.buildAnd(MatchInfo.InnerShiftTy, Shift, Mask);
2147 Builder.buildTrunc(Dst, And);
2148 } else
2149 Builder.buildTrunc(Dst, Shift);
2150 MI.eraseFromParent();
2151}
2152
2154 unsigned &ShiftVal) const {
2155 assert(MI.getOpcode() == TargetOpcode::G_MUL && "Expected a G_MUL");
2156 auto MaybeImmVal =
2157 getIConstantVRegValWithLookThrough(MI.getOperand(2).getReg(), MRI);
2158 if (!MaybeImmVal)
2159 return false;
2160
2161 ShiftVal = MaybeImmVal->Value.exactLogBase2();
2162 return (static_cast<int32_t>(ShiftVal) != -1);
2163}
2164
2166 unsigned &ShiftVal) const {
2167 assert(MI.getOpcode() == TargetOpcode::G_MUL && "Expected a G_MUL");
2168 MachineIRBuilder MIB(MI);
2169 LLT ShiftTy = MRI.getType(MI.getOperand(0).getReg());
2170 auto ShiftCst = MIB.buildConstant(ShiftTy, ShiftVal);
2171 Observer.changingInstr(MI);
2172 MI.setDesc(MIB.getTII().get(TargetOpcode::G_SHL));
2173 MI.getOperand(2).setReg(ShiftCst.getReg(0));
2174 if (ShiftVal == ShiftTy.getScalarSizeInBits() - 1)
2176 Observer.changedInstr(MI);
2177}
2178
2180 BuildFnTy &MatchInfo) const {
2181 GSub &Sub = cast<GSub>(MI);
2182
2183 LLT Ty = MRI.getType(Sub.getReg(0));
2184
2185 if (!isLegalOrBeforeLegalizer({TargetOpcode::G_ADD, {Ty}}))
2186 return false;
2187
2189 return false;
2190
2191 APInt Imm = getIConstantFromReg(Sub.getRHSReg(), MRI);
2192
2193 MatchInfo = [=, &MI](MachineIRBuilder &B) {
2194 auto NegCst = B.buildConstant(Ty, -Imm);
2195 Observer.changingInstr(MI);
2196 MI.setDesc(B.getTII().get(TargetOpcode::G_ADD));
2197 MI.getOperand(2).setReg(NegCst.getReg(0));
2199 if (Imm.isMinSignedValue())
2201 Observer.changedInstr(MI);
2202 };
2203 return true;
2204}
2205
2206// shl ([sza]ext x), y => zext (shl x, y), if shift does not overflow source
2208 RegisterImmPair &MatchData) const {
2209 assert(MI.getOpcode() == TargetOpcode::G_SHL && VT);
2210 if (!getTargetLowering().isDesirableToPullExtFromShl(MI))
2211 return false;
2212
2213 Register LHS = MI.getOperand(1).getReg();
2214
2215 Register ExtSrc;
2216 if (!mi_match(LHS, MRI, m_GAnyExt(m_Reg(ExtSrc))) &&
2217 !mi_match(LHS, MRI, m_GZExt(m_Reg(ExtSrc))) &&
2218 !mi_match(LHS, MRI, m_GSExt(m_Reg(ExtSrc))))
2219 return false;
2220
2221 Register RHS = MI.getOperand(2).getReg();
2222 auto MaybeShiftAmtVal = isConstantOrConstantSplatVector(RHS, MRI);
2223 if (!MaybeShiftAmtVal)
2224 return false;
2225
2226 if (LI) {
2227 LLT SrcTy = MRI.getType(ExtSrc);
2228
2229 // We only really care about the legality with the shifted value. We can
2230 // pick any type the constant shift amount, so ask the target what to
2231 // use. Otherwise we would have to guess and hope it is reported as legal.
2232 LLT ShiftAmtTy = getTargetLowering().getPreferredShiftAmountTy(SrcTy);
2233 if (!isLegalOrBeforeLegalizer({TargetOpcode::G_SHL, {SrcTy, ShiftAmtTy}}))
2234 return false;
2235 }
2236
2237 int64_t ShiftAmt = MaybeShiftAmtVal->getSExtValue();
2238 MatchData.Reg = ExtSrc;
2239 MatchData.Imm = ShiftAmt;
2240
2241 unsigned MinLeadingZeros = VT->getKnownZeroes(ExtSrc).countl_one();
2242 unsigned SrcTySize = MRI.getType(ExtSrc).getScalarSizeInBits();
2243 return MinLeadingZeros >= ShiftAmt && ShiftAmt < SrcTySize;
2244}
2245
2247 MachineInstr &MI, const RegisterImmPair &MatchData) const {
2248 Register ExtSrcReg = MatchData.Reg;
2249 int64_t ShiftAmtVal = MatchData.Imm;
2250
2251 LLT ExtSrcTy = MRI.getType(ExtSrcReg);
2252 auto ShiftAmt = Builder.buildConstant(ExtSrcTy, ShiftAmtVal);
2253 auto NarrowShift =
2254 Builder.buildShl(ExtSrcTy, ExtSrcReg, ShiftAmt, MI.getFlags());
2255 Builder.buildZExt(MI.getOperand(0), NarrowShift);
2256 MI.eraseFromParent();
2257}
2258
2260 const MachineRegisterInfo &MRI) {
2261 while (mi_match(Reg, MRI, m_GBitcast(m_Reg(Reg))))
2262 ;
2263
2264 return Reg;
2265}
2266
2269 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2270 "Expected an unmerge");
2271 auto &Unmerge = cast<GUnmerge>(MI);
2272 Register SrcReg = peekThroughBitcast(Unmerge.getSourceReg(), MRI);
2273
2274 auto *SrcInstr = getOpcodeDef<GMergeLikeInstr>(SrcReg, MRI);
2275 if (!SrcInstr)
2276 return false;
2277
2278 // Check the source type of the merge.
2279 LLT SrcMergeTy = MRI.getType(SrcInstr->getSourceReg(0));
2280 LLT Dst0Ty = MRI.getType(Unmerge.getReg(0));
2281 bool SameSize = Dst0Ty.getSizeInBits() == SrcMergeTy.getSizeInBits();
2282 if (SrcMergeTy != Dst0Ty && !SameSize)
2283 return false;
2284 // They are the same now (modulo a bitcast).
2285 // We can collect all the src registers.
2286 for (unsigned Idx = 0; Idx < SrcInstr->getNumSources(); ++Idx)
2287 Operands.push_back(SrcInstr->getSourceReg(Idx));
2288 return true;
2289}
2290
2293 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2294 "Expected an unmerge");
2295 assert((MI.getNumOperands() - 1 == Operands.size()) &&
2296 "Not enough operands to replace all defs");
2297 unsigned NumElems = MI.getNumOperands() - 1;
2298
2299 LLT SrcTy = MRI.getType(Operands[0]);
2300 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
2301 bool CanReuseInputDirectly = DstTy == SrcTy;
2302 for (unsigned Idx = 0; Idx < NumElems; ++Idx) {
2303 Register DstReg = MI.getOperand(Idx).getReg();
2304 Register SrcReg = Operands[Idx];
2305
2306 // This combine may run after RegBankSelect, so we need to be aware of
2307 // register banks.
2308 const auto &DstCB = MRI.getRegClassOrRegBank(DstReg);
2309 if (!DstCB.isNull() && DstCB != MRI.getRegClassOrRegBank(SrcReg)) {
2310 SrcReg = Builder.buildCopy(MRI.getType(SrcReg), SrcReg).getReg(0);
2311 MRI.setRegClassOrRegBank(SrcReg, DstCB);
2312 }
2313
2314 if (CanReuseInputDirectly)
2315 replaceRegWith(MRI, DstReg, SrcReg);
2316 else
2317 Builder.buildCast(DstReg, SrcReg);
2318 }
2319 MI.eraseFromParent();
2320}
2321
2323 MachineInstr &MI, SmallVectorImpl<APInt> &Csts) const {
2324 unsigned SrcIdx = MI.getNumOperands() - 1;
2325 Register SrcReg = MI.getOperand(SrcIdx).getReg();
2326 // Break down the big constant in smaller ones.
2327 APInt Val;
2328 if (!mi_match(SrcReg, MRI, m_GConstantOrFConstantBits(Val)))
2329 return false;
2330
2331 LLT Dst0Ty = MRI.getType(MI.getOperand(0).getReg());
2332 unsigned ShiftAmt = Dst0Ty.getSizeInBits();
2333 // Unmerge a constant.
2334 for (unsigned Idx = 0; Idx != SrcIdx; ++Idx) {
2335 Csts.emplace_back(Val.trunc(ShiftAmt));
2336 Val = Val.lshr(ShiftAmt);
2337 }
2338
2339 return true;
2340}
2341
2343 MachineInstr &MI, SmallVectorImpl<APInt> &Csts) const {
2344 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2345 "Expected an unmerge");
2346 assert((MI.getNumOperands() - 1 == Csts.size()) &&
2347 "Not enough operands to replace all defs");
2348 unsigned NumElems = MI.getNumOperands() - 1;
2349 for (unsigned Idx = 0; Idx < NumElems; ++Idx) {
2350 Register DstReg = MI.getOperand(Idx).getReg();
2351 Builder.buildConstant(DstReg, Csts[Idx]);
2352 }
2353
2354 MI.eraseFromParent();
2355}
2356
2359 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
2360 unsigned SrcIdx = MI.getNumOperands() - 1;
2361 Register SrcReg = MI.getOperand(SrcIdx).getReg();
2362 MatchInfo = [&MI](MachineIRBuilder &B) {
2363 unsigned NumElems = MI.getNumOperands() - 1;
2364 for (unsigned Idx = 0; Idx < NumElems; ++Idx) {
2365 Register DstReg = MI.getOperand(Idx).getReg();
2366 B.buildUndef(DstReg);
2367 }
2368 };
2369 return mi_match(SrcReg, MRI, m_GImplicitDef());
2370}
2371
2373 MachineInstr &MI) const {
2374 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2375 "Expected an unmerge");
2376 if (!MRI.getType(MI.getOperand(0).getReg()).isScalar() ||
2377 !MRI.getType(MI.getOperand(MI.getNumDefs()).getReg()).isScalar())
2378 return false;
2379 // Check that all the lanes are dead except the first one.
2380 for (unsigned Idx = 1, EndIdx = MI.getNumDefs(); Idx != EndIdx; ++Idx) {
2381 if (!MRI.use_nodbg_empty(MI.getOperand(Idx).getReg()))
2382 return false;
2383 }
2384 return true;
2385}
2386
2388 MachineInstr &MI) const {
2389 Register SrcReg = MI.getOperand(MI.getNumDefs()).getReg();
2390 Register Dst0Reg = MI.getOperand(0).getReg();
2391 Builder.buildTrunc(Dst0Reg, SrcReg);
2392 MI.eraseFromParent();
2393}
2394
2396 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2397 "Expected an unmerge");
2398 Register Dst0Reg = MI.getOperand(0).getReg();
2399 LLT Dst0Ty = MRI.getType(Dst0Reg);
2400 // G_ZEXT on vector applies to each lane, so it will
2401 // affect all destinations. Therefore we won't be able
2402 // to simplify the unmerge to just the first definition.
2403 if (Dst0Ty.isVector())
2404 return false;
2405 Register SrcReg = MI.getOperand(MI.getNumDefs()).getReg();
2406 LLT SrcTy = MRI.getType(SrcReg);
2407 if (SrcTy.isVector())
2408 return false;
2409
2410 Register ZExtSrcReg;
2411 if (!mi_match(SrcReg, MRI, m_GZExt(m_Reg(ZExtSrcReg))))
2412 return false;
2413
2414 // Finally we can replace the first definition with
2415 // a zext of the source if the definition is big enough to hold
2416 // all of ZExtSrc bits.
2417 LLT ZExtSrcTy = MRI.getType(ZExtSrcReg);
2418 return ZExtSrcTy.getSizeInBits() <= Dst0Ty.getSizeInBits();
2419}
2420
2422 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2423 "Expected an unmerge");
2424
2425 Register Dst0Reg = MI.getOperand(0).getReg();
2426
2427 GZext *ZExtInstr =
2428 cast<GZext>(MRI.getVRegDef(MI.getOperand(MI.getNumDefs()).getReg()));
2429 Register ZExtSrcReg = ZExtInstr->getSrcReg();
2430 LLT Dst0Ty = MRI.getType(Dst0Reg);
2431 LLT ZExtSrcTy = MRI.getType(ZExtSrcReg);
2432
2433 if (Dst0Ty.getSizeInBits() > ZExtSrcTy.getSizeInBits()) {
2434 Builder.buildZExt(Dst0Reg, ZExtSrcReg);
2435 } else {
2436 assert(Dst0Ty.getSizeInBits() == ZExtSrcTy.getSizeInBits() &&
2437 "ZExt src doesn't fit in destination");
2438 replaceRegWith(MRI, Dst0Reg, ZExtSrcReg);
2439 }
2440
2441 Register ZeroReg;
2442 for (unsigned Idx = 1, EndIdx = MI.getNumDefs(); Idx != EndIdx; ++Idx) {
2443 if (!ZeroReg)
2444 ZeroReg = Builder.buildConstant(Dst0Ty, 0).getReg(0);
2445 replaceRegWith(MRI, MI.getOperand(Idx).getReg(), ZeroReg);
2446 }
2447 MI.eraseFromParent();
2448}
2449
2451 unsigned TargetShiftSize,
2452 unsigned &ShiftVal) const {
2453 assert((MI.getOpcode() == TargetOpcode::G_SHL ||
2454 MI.getOpcode() == TargetOpcode::G_LSHR ||
2455 MI.getOpcode() == TargetOpcode::G_ASHR) && "Expected a shift");
2456
2457 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
2458 if (Ty.isVector()) // TODO:
2459 return false;
2460
2461 // Don't narrow further than the requested size.
2462 unsigned Size = Ty.getSizeInBits();
2463 if (Size <= TargetShiftSize)
2464 return false;
2465
2466 auto MaybeImmVal =
2467 getIConstantVRegValWithLookThrough(MI.getOperand(2).getReg(), MRI);
2468 if (!MaybeImmVal)
2469 return false;
2470
2471 ShiftVal = MaybeImmVal->Value.getSExtValue();
2472 return ShiftVal >= Size / 2 && ShiftVal < Size;
2473}
2474
2476 MachineInstr &MI, const unsigned &ShiftVal) const {
2477 Register DstReg = MI.getOperand(0).getReg();
2478 Register SrcReg = MI.getOperand(1).getReg();
2479 LLT Ty = MRI.getType(SrcReg);
2480 unsigned Size = Ty.getSizeInBits();
2481 unsigned HalfSize = Size / 2;
2482 assert(ShiftVal >= HalfSize);
2483
2484 LLT HalfTy = Ty.changeElementSize(HalfSize);
2485
2486 auto Unmerge = Builder.buildUnmerge(HalfTy, SrcReg);
2487 unsigned NarrowShiftAmt = ShiftVal - HalfSize;
2488
2489 if (MI.getOpcode() == TargetOpcode::G_LSHR) {
2490 Register Narrowed = Unmerge.getReg(1);
2491
2492 // dst = G_LSHR s64:x, C for C >= 32
2493 // =>
2494 // lo, hi = G_UNMERGE_VALUES x
2495 // dst = G_MERGE_VALUES (G_LSHR hi, C - 32), 0
2496
2497 if (NarrowShiftAmt != 0) {
2498 Narrowed = Builder.buildLShr(HalfTy, Narrowed,
2499 Builder.buildConstant(HalfTy, NarrowShiftAmt)).getReg(0);
2500 }
2501
2502 auto Zero = Builder.buildConstant(HalfTy, 0);
2503 Builder.buildMergeLikeInstr(DstReg, {Narrowed, Zero});
2504 } else if (MI.getOpcode() == TargetOpcode::G_SHL) {
2505 Register Narrowed = Unmerge.getReg(0);
2506 // dst = G_SHL s64:x, C for C >= 32
2507 // =>
2508 // lo, hi = G_UNMERGE_VALUES x
2509 // dst = G_MERGE_VALUES 0, (G_SHL hi, C - 32)
2510 if (NarrowShiftAmt != 0) {
2511 Narrowed = Builder.buildShl(HalfTy, Narrowed,
2512 Builder.buildConstant(HalfTy, NarrowShiftAmt)).getReg(0);
2513 }
2514
2515 auto Zero = Builder.buildConstant(HalfTy, 0);
2516 Builder.buildMergeLikeInstr(DstReg, {Zero, Narrowed});
2517 } else {
2518 assert(MI.getOpcode() == TargetOpcode::G_ASHR);
2519 auto Hi = Builder.buildAShr(
2520 HalfTy, Unmerge.getReg(1),
2521 Builder.buildConstant(HalfTy, HalfSize - 1));
2522
2523 if (ShiftVal == HalfSize) {
2524 // (G_ASHR i64:x, 32) ->
2525 // G_MERGE_VALUES hi_32(x), (G_ASHR hi_32(x), 31)
2526 Builder.buildMergeLikeInstr(DstReg, {Unmerge.getReg(1), Hi});
2527 } else if (ShiftVal == Size - 1) {
2528 // Don't need a second shift.
2529 // (G_ASHR i64:x, 63) ->
2530 // %narrowed = (G_ASHR hi_32(x), 31)
2531 // G_MERGE_VALUES %narrowed, %narrowed
2532 Builder.buildMergeLikeInstr(DstReg, {Hi, Hi});
2533 } else {
2534 auto Lo = Builder.buildAShr(
2535 HalfTy, Unmerge.getReg(1),
2536 Builder.buildConstant(HalfTy, ShiftVal - HalfSize));
2537
2538 // (G_ASHR i64:x, C) ->, for C >= 32
2539 // G_MERGE_VALUES (G_ASHR hi_32(x), C - 32), (G_ASHR hi_32(x), 31)
2540 Builder.buildMergeLikeInstr(DstReg, {Lo, Hi});
2541 }
2542 }
2543
2544 MI.eraseFromParent();
2545}
2546
2548 MachineInstr &MI, unsigned TargetShiftAmount) const {
2549 unsigned ShiftAmt;
2550 if (matchCombineShiftToUnmerge(MI, TargetShiftAmount, ShiftAmt)) {
2551 applyCombineShiftToUnmerge(MI, ShiftAmt);
2552 return true;
2553 }
2554
2555 return false;
2556}
2557
2559 Register &Reg) const {
2560 assert(MI.getOpcode() == TargetOpcode::G_ANYEXT && "Expected a G_ANYEXT");
2561 Register DstReg = MI.getOperand(0).getReg();
2562 Register SrcReg = MI.getOperand(1).getReg();
2563 Register OriginalSrcReg = getSrcRegIgnoringCopies(SrcReg, MRI);
2564 if (OriginalSrcReg.isValid())
2565 SrcReg = OriginalSrcReg;
2566 LLT DstTy = MRI.getType(DstReg);
2567 return mi_match(SrcReg, MRI,
2568 m_GTrunc(m_all_of(m_Reg(Reg), m_SpecificType(DstTy)))) &&
2569 canReplaceReg(DstReg, Reg, MRI);
2570}
2571
2573 Register &Reg) const {
2574 assert(MI.getOpcode() == TargetOpcode::G_ZEXT && "Expected a G_ZEXT");
2575 Register DstReg = MI.getOperand(0).getReg();
2576 Register SrcReg = MI.getOperand(1).getReg();
2577 LLT DstTy = MRI.getType(DstReg);
2578 if (mi_match(SrcReg, MRI,
2579 m_GTrunc(m_all_of(m_Reg(Reg), m_SpecificType(DstTy)))) &&
2580 canReplaceReg(DstReg, Reg, MRI)) {
2581 unsigned DstSize = DstTy.getScalarSizeInBits();
2582 unsigned SrcSize = MRI.getType(SrcReg).getScalarSizeInBits();
2583 return VT->getKnownBits(Reg).countMinLeadingZeros() >= DstSize - SrcSize;
2584 }
2585 return false;
2586}
2587
2589 const unsigned ShiftSize = ShiftTy.getScalarSizeInBits();
2590 const unsigned TruncSize = TruncTy.getScalarSizeInBits();
2591
2592 // ShiftTy > 32 > TruncTy -> 32
2593 if (ShiftSize > 32 && TruncSize < 32)
2594 return ShiftTy.changeElementSize(32);
2595
2596 // TODO: We could also reduce to 16 bits, but that's more target-dependent.
2597 // Some targets like it, some don't, some only like it under certain
2598 // conditions/processor versions, etc.
2599 // A TL hook might be needed for this.
2600
2601 // Don't combine
2602 return ShiftTy;
2603}
2604
2606 MachineInstr &MI, std::pair<MachineInstr *, LLT> &MatchInfo) const {
2607 assert(MI.getOpcode() == TargetOpcode::G_TRUNC && "Expected a G_TRUNC");
2608 Register DstReg = MI.getOperand(0).getReg();
2609 Register SrcReg = MI.getOperand(1).getReg();
2610
2611 if (!MRI.hasOneNonDBGUse(SrcReg))
2612 return false;
2613
2614 LLT SrcTy = MRI.getType(SrcReg);
2615 LLT DstTy = MRI.getType(DstReg);
2616
2617 MachineInstr *SrcMI = getDefIgnoringCopies(SrcReg, MRI);
2618 const auto &TL = getTargetLowering();
2619
2620 LLT NewShiftTy;
2621 switch (SrcMI->getOpcode()) {
2622 default:
2623 return false;
2624 case TargetOpcode::G_SHL: {
2625 NewShiftTy = DstTy;
2626
2627 // Make sure new shift amount is legal.
2628 KnownBits Known = VT->getKnownBits(SrcMI->getOperand(2).getReg());
2629 if (Known.getMaxValue().uge(NewShiftTy.getScalarSizeInBits()))
2630 return false;
2631 break;
2632 }
2633 case TargetOpcode::G_LSHR:
2634 case TargetOpcode::G_ASHR: {
2635 // For right shifts, we conservatively do not do the transform if the TRUNC
2636 // has any STORE users. The reason is that if we change the type of the
2637 // shift, we may break the truncstore combine.
2638 //
2639 // TODO: Fix truncstore combine to handle (trunc(lshr (trunc x), k)).
2640 for (auto &User : MRI.use_instructions(DstReg))
2641 if (User.getOpcode() == TargetOpcode::G_STORE)
2642 return false;
2643
2644 NewShiftTy = getMidVTForTruncRightShiftCombine(SrcTy, DstTy);
2645 if (NewShiftTy == SrcTy)
2646 return false;
2647
2648 // Make sure we won't lose information by truncating the high bits.
2649 KnownBits Known = VT->getKnownBits(SrcMI->getOperand(2).getReg());
2650 if (Known.getMaxValue().ugt(NewShiftTy.getScalarSizeInBits() -
2651 DstTy.getScalarSizeInBits()))
2652 return false;
2653 break;
2654 }
2655 }
2656
2658 {SrcMI->getOpcode(),
2659 {NewShiftTy, TL.getPreferredShiftAmountTy(NewShiftTy)}}))
2660 return false;
2661
2662 MatchInfo = std::make_pair(SrcMI, NewShiftTy);
2663 return true;
2664}
2665
2667 MachineInstr &MI, std::pair<MachineInstr *, LLT> &MatchInfo) const {
2668 MachineInstr *ShiftMI = MatchInfo.first;
2669 LLT NewShiftTy = MatchInfo.second;
2670
2671 Register Dst = MI.getOperand(0).getReg();
2672 LLT DstTy = MRI.getType(Dst);
2673
2674 Register ShiftAmt = ShiftMI->getOperand(2).getReg();
2675 Register ShiftSrc = ShiftMI->getOperand(1).getReg();
2676 ShiftSrc = Builder.buildTrunc(NewShiftTy, ShiftSrc).getReg(0);
2677
2678 const auto &TL = getTargetLowering();
2679 LLT PrefShiftTy = TL.getPreferredShiftAmountTy(NewShiftTy);
2680 if (MRI.getType(ShiftAmt) != PrefShiftTy)
2681 ShiftAmt = Builder.buildZExtOrTrunc(PrefShiftTy, ShiftAmt).getReg(0);
2682
2683 Register NewShift =
2684 Builder
2685 .buildInstr(ShiftMI->getOpcode(), {NewShiftTy}, {ShiftSrc, ShiftAmt})
2686 .getReg(0);
2687
2688 if (NewShiftTy == DstTy)
2689 replaceRegWith(MRI, Dst, NewShift);
2690 else
2691 Builder.buildTrunc(Dst, NewShift);
2692
2693 eraseInst(MI);
2694}
2695
2697 return all_of(MI.explicit_uses(), [this](const MachineOperand &MO) {
2698 return !MO.isReg() ||
2699 getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, MO.getReg(), MRI);
2700 });
2701}
2702
2704 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
2705 ArrayRef<int> Mask = MI.getOperand(3).getShuffleMask();
2706 return all_of(Mask, [](int Elt) { return Elt < 0; });
2707}
2708
2710 MachineInstr &MI) const {
2711 assert((MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT ||
2712 MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT) &&
2713 "Expected an insert/extract element op");
2714 LLT VecTy = MRI.getType(MI.getOperand(1).getReg());
2715 if (VecTy.isScalableVector())
2716 return false;
2717
2718 unsigned IdxIdx =
2719 MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT ? 2 : 3;
2720 auto Idx = getIConstantVRegVal(MI.getOperand(IdxIdx).getReg(), MRI);
2721 if (!Idx)
2722 return false;
2723 return Idx->getZExtValue() >= VecTy.getNumElements();
2724}
2725
2726void CombinerHelper::eraseInst(MachineInstr &MI) const { MI.eraseFromParent(); }
2727
2729 const MachineOperand &MOP2) const {
2730 if (!MOP1.isReg() || !MOP2.isReg())
2731 return false;
2732 auto InstAndDef1 = getDefSrcRegIgnoringCopies(MOP1.getReg(), MRI);
2733 if (!InstAndDef1)
2734 return false;
2735 auto InstAndDef2 = getDefSrcRegIgnoringCopies(MOP2.getReg(), MRI);
2736 if (!InstAndDef2)
2737 return false;
2738 MachineInstr *I1 = InstAndDef1->MI;
2739 MachineInstr *I2 = InstAndDef2->MI;
2740
2741 // Handle a case like this:
2742 //
2743 // %0:_(s64), %1:_(s64) = G_UNMERGE_VALUES %2:_(<2 x s64>)
2744 //
2745 // Even though %0 and %1 are produced by the same instruction they are not
2746 // the same values.
2747 if (I1 == I2)
2748 return MOP1.getReg() == MOP2.getReg();
2749
2750 // If we have an instruction which loads or stores, we can't guarantee that
2751 // it is identical.
2752 //
2753 // For example, we may have
2754 //
2755 // %x1 = G_LOAD %addr (load N from @somewhere)
2756 // ...
2757 // call @foo
2758 // ...
2759 // %x2 = G_LOAD %addr (load N from @somewhere)
2760 // ...
2761 // %or = G_OR %x1, %x2
2762 //
2763 // It's possible that @foo will modify whatever lives at the address we're
2764 // loading from. To be safe, let's just assume that all loads and stores
2765 // are different (unless we have something which is guaranteed to not
2766 // change.)
2767 if (I1->mayLoadOrStore() && !I1->isDereferenceableInvariantLoad())
2768 return false;
2769
2770 // If both instructions are loads or stores, they are equal only if both
2771 // are dereferenceable invariant loads with the same number of bits.
2772 if (I1->mayLoadOrStore() && I2->mayLoadOrStore()) {
2775 if (!LS1 || !LS2)
2776 return false;
2777
2778 if (!I2->isDereferenceableInvariantLoad() ||
2779 (LS1->getMemSizeInBits() != LS2->getMemSizeInBits()))
2780 return false;
2781 }
2782
2783 // Check for physical registers on the instructions first to avoid cases
2784 // like this:
2785 //
2786 // %a = COPY $physreg
2787 // ...
2788 // SOMETHING implicit-def $physreg
2789 // ...
2790 // %b = COPY $physreg
2791 //
2792 // These copies are not equivalent.
2793 if (any_of(I1->uses(), [](const MachineOperand &MO) {
2794 return MO.isReg() && MO.getReg().isPhysical();
2795 })) {
2796 // Check if we have a case like this:
2797 //
2798 // %a = COPY $physreg
2799 // %b = COPY %a
2800 //
2801 // In this case, I1 and I2 will both be equal to %a = COPY $physreg.
2802 // From that, we know that they must have the same value, since they must
2803 // have come from the same COPY.
2804 return I1->isIdenticalTo(*I2);
2805 }
2806
2807 // We don't have any physical registers, so we don't necessarily need the
2808 // same vreg defs.
2809 //
2810 // On the off-chance that there's some target instruction feeding into the
2811 // instruction, let's use produceSameValue instead of isIdenticalTo.
2812 if (Builder.getTII().produceSameValue(*I1, *I2, &MRI)) {
2813 // Handle instructions with multiple defs that produce same values. Values
2814 // are same for operands with same index.
2815 // %0:_(s8), %1:_(s8), %2:_(s8), %3:_(s8) = G_UNMERGE_VALUES %4:_(<4 x s8>)
2816 // %5:_(s8), %6:_(s8), %7:_(s8), %8:_(s8) = G_UNMERGE_VALUES %4:_(<4 x s8>)
2817 // I1 and I2 are different instructions but produce same values,
2818 // %1 and %6 are same, %1 and %7 are not the same value.
2819 return I1->findRegisterDefOperandIdx(InstAndDef1->Reg, /*TRI=*/nullptr) ==
2820 I2->findRegisterDefOperandIdx(InstAndDef2->Reg, /*TRI=*/nullptr);
2821 }
2822 return false;
2823}
2824
2826 int64_t C) const {
2827 if (!MOP.isReg())
2828 return false;
2829 auto MaybeCst = isConstantOrConstantSplatVector(MOP.getReg(), MRI);
2830 // Compare as APInt so constants wider than 64 bits (e.g. i128 2) still match.
2831 return MaybeCst &&
2832 APInt::isSameValue(*MaybeCst, APInt(64, C, /*isSigned=*/true),
2833 /*SignedCompare=*/true);
2834}
2835
2837 double C) const {
2838 if (!MOP.isReg())
2839 return false;
2840 std::optional<FPValueAndVReg> MaybeCst;
2841 if (!mi_match(MOP.getReg(), MRI, m_GFCstOrSplat(MaybeCst)))
2842 return false;
2843
2844 return MaybeCst->Value.isExactlyValue(C);
2845}
2846
2848 unsigned OpIdx) const {
2849 assert(MI.getNumExplicitDefs() == 1 && "Expected one explicit def?");
2850 Register OldReg = MI.getOperand(0).getReg();
2851 Register Replacement = MI.getOperand(OpIdx).getReg();
2852 assert(canReplaceReg(OldReg, Replacement, MRI) && "Cannot replace register?");
2853 replaceRegWith(MRI, OldReg, Replacement);
2854 MI.eraseFromParent();
2855}
2856
2858 Register Replacement) const {
2859 assert(MI.getNumExplicitDefs() == 1 && "Expected one explicit def?");
2860 Register OldReg = MI.getOperand(0).getReg();
2861 assert(canReplaceReg(OldReg, Replacement, MRI) && "Cannot replace register?");
2862 replaceRegWith(MRI, OldReg, Replacement);
2863 MI.eraseFromParent();
2864}
2865
2867 unsigned ConstIdx) const {
2868 Register ConstReg = MI.getOperand(ConstIdx).getReg();
2869 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
2870
2871 // Get the shift amount
2872 auto VRegAndVal = getIConstantVRegValWithLookThrough(ConstReg, MRI);
2873 if (!VRegAndVal)
2874 return false;
2875
2876 // Return true of shift amount >= Bitwidth
2877 return (VRegAndVal->Value.uge(DstTy.getSizeInBits()));
2878}
2879
2881 assert((MI.getOpcode() == TargetOpcode::G_FSHL ||
2882 MI.getOpcode() == TargetOpcode::G_FSHR) &&
2883 "This is not a funnel shift operation");
2884
2885 Register ConstReg = MI.getOperand(3).getReg();
2886 LLT ConstTy = MRI.getType(ConstReg);
2887 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
2888
2889 auto VRegAndVal = getIConstantVRegValWithLookThrough(ConstReg, MRI);
2890 assert((VRegAndVal) && "Value is not a constant");
2891
2892 // Calculate the new Shift Amount = Old Shift Amount % BitWidth
2893 APInt NewConst = VRegAndVal->Value.urem(
2894 APInt(ConstTy.getSizeInBits(), DstTy.getScalarSizeInBits()));
2895
2896 auto NewConstInstr = Builder.buildConstant(ConstTy, NewConst.getZExtValue());
2897 Builder.buildInstr(
2898 MI.getOpcode(), {MI.getOperand(0)},
2899 {MI.getOperand(1), MI.getOperand(2), NewConstInstr.getReg(0)});
2900
2901 MI.eraseFromParent();
2902}
2903
2905 assert(MI.getOpcode() == TargetOpcode::G_SELECT);
2906 // Match (cond ? x : x)
2907 return matchEqualDefs(MI.getOperand(2), MI.getOperand(3)) &&
2908 canReplaceReg(MI.getOperand(0).getReg(), MI.getOperand(2).getReg(),
2909 MRI);
2910}
2911
2913 const MachineOperand &MO, bool OrNegative, bool OrZero) const {
2914 return isKnownToBeAPowerOfTwo(MO.getReg(), MRI, VT, OrNegative, OrZero);
2915}
2916
2918 double C) const {
2919 assert(MI.getNumDefs() == 1 && "Expected only one def?");
2920 Builder.buildFConstant(MI.getOperand(0), C);
2921 MI.eraseFromParent();
2922}
2923
2925 int64_t C) const {
2926 assert(MI.getNumDefs() == 1 && "Expected only one def?");
2927 Builder.buildConstant(MI.getOperand(0), C);
2928 MI.eraseFromParent();
2929}
2930
2932 assert(MI.getNumDefs() == 1 && "Expected only one def?");
2933 Builder.buildConstant(MI.getOperand(0), C);
2934 MI.eraseFromParent();
2935}
2936
2938 ConstantFP *CFP) const {
2939 assert(MI.getNumDefs() == 1 && "Expected only one def?");
2940 Builder.buildFConstant(MI.getOperand(0), CFP->getValueAPF());
2941 MI.eraseFromParent();
2942}
2943
2945 assert(MI.getNumDefs() == 1 && "Expected only one def?");
2946 Builder.buildUndef(MI.getOperand(0));
2947 MI.eraseFromParent();
2948}
2949
2951 MachineInstr &MI, SmallVectorImpl<Register> &MatchInfo) const {
2952 assert(MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT &&
2953 "Invalid opcode");
2954 Register DstReg = MI.getOperand(0).getReg();
2955 LLT DstTy = MRI.getType(DstReg);
2956 assert(DstTy.isVector() && "Invalid G_INSERT_VECTOR_ELT?");
2957
2958 if (DstTy.isScalableVector())
2959 return false;
2960
2961 unsigned NumElts = DstTy.getNumElements();
2962 // If this MI is part of a sequence of insert_vec_elts, then
2963 // don't do the combine in the middle of the sequence.
2964 if (MRI.hasOneNonDBGUse(DstReg) &&
2965 MRI.use_nodbg_instructions(DstReg).begin()->getOpcode() ==
2966 TargetOpcode::G_INSERT_VECTOR_ELT)
2967 return false;
2968 MachineInstr *CurrInst = &MI;
2969 MachineInstr *TmpInst;
2970 int64_t IntImm;
2971 Register TmpReg;
2972 MatchInfo.resize(NumElts);
2973 while (mi_match(
2974 *CurrInst, MRI,
2975 m_GInsertVecElt(m_MInstr(TmpInst), m_Reg(TmpReg), m_ICst(IntImm)))) {
2976 if (IntImm >= NumElts || IntImm < 0)
2977 return false;
2978 if (!MatchInfo[IntImm])
2979 MatchInfo[IntImm] = TmpReg;
2980 CurrInst = TmpInst;
2981 }
2982 // Variable index.
2983 if (CurrInst->getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT)
2984 return false;
2985 if (TmpInst->getOpcode() == TargetOpcode::G_BUILD_VECTOR) {
2986 for (unsigned I = 1; I < TmpInst->getNumOperands(); ++I) {
2987 if (!MatchInfo[I - 1].isValid())
2988 MatchInfo[I - 1] = TmpInst->getOperand(I).getReg();
2989 }
2990 return true;
2991 }
2992 // If we didn't end in a G_IMPLICIT_DEF and the source is not fully
2993 // overwritten, bail out.
2994 return TmpInst->getOpcode() == TargetOpcode::G_IMPLICIT_DEF ||
2995 all_of(MatchInfo, [](Register Reg) { return !!Reg; });
2996}
2997
2999 MachineInstr &MI, SmallVectorImpl<Register> &MatchInfo) const {
3000 Register UndefReg;
3001 auto GetUndef = [&]() {
3002 if (UndefReg)
3003 return UndefReg;
3004 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
3005 UndefReg = Builder.buildUndef(DstTy.getScalarType()).getReg(0);
3006 return UndefReg;
3007 };
3008 for (Register &Reg : MatchInfo) {
3009 if (!Reg)
3010 Reg = GetUndef();
3011 }
3012 Builder.buildBuildVector(MI.getOperand(0).getReg(), MatchInfo);
3013 MI.eraseFromParent();
3014}
3015
3016bool CombinerHelper::matchBinopWithNegInner(Register MInner, Register Other,
3017 unsigned RootOpc, Register Dst,
3018 LLT Ty,
3019 BuildFnTy &MatchInfo) const {
3020 /// Helper function for matchBinopWithNeg: tries to match one commuted form
3021 /// of `a bitwiseop (~b +/- c)` -> `a bitwiseop ~(b -/+ c)`.
3022 MachineInstr *InnerDef;
3023 if (!mi_match(MInner, MRI, m_MInstr(InnerDef)))
3024 return false;
3025
3026 unsigned InnerOpc = InnerDef->getOpcode();
3027 if (InnerOpc != TargetOpcode::G_ADD && InnerOpc != TargetOpcode::G_SUB)
3028 return false;
3029
3030 if (!MRI.hasOneNonDBGUse(MInner))
3031 return false;
3032
3033 Register InnerLHS = InnerDef->getOperand(1).getReg();
3034 Register InnerRHS = InnerDef->getOperand(2).getReg();
3035 Register NotSrc;
3036 Register B, C;
3037
3038 // Check if either operand is ~b
3039 auto TryMatch = [&](Register MaybeNot, Register Other) {
3040 if (mi_match(MaybeNot, MRI, m_Not(m_Reg(NotSrc)))) {
3041 if (!MRI.hasOneNonDBGUse(MaybeNot))
3042 return false;
3043 B = NotSrc;
3044 C = Other;
3045 return true;
3046 }
3047 return false;
3048 };
3049
3050 // For SUB, the not must be the LHS. For ADD, it can be either operand.
3051 if (!TryMatch(InnerLHS, InnerRHS) &&
3052 !(InnerOpc == TargetOpcode::G_ADD && TryMatch(InnerRHS, InnerLHS)))
3053 return false;
3054
3055 // Flip add/sub
3056 unsigned FlippedOpc = (InnerOpc == TargetOpcode::G_ADD) ? TargetOpcode::G_SUB
3057 : TargetOpcode::G_ADD;
3058
3059 Register A = Other;
3060 MatchInfo = [=](MachineIRBuilder &Builder) {
3061 auto NewInner = Builder.buildInstr(FlippedOpc, {Ty}, {B, C});
3062 auto NewNot = Builder.buildNot(Ty, NewInner);
3063 Builder.buildInstr(RootOpc, {Dst}, {A, NewNot});
3064 };
3065 return true;
3066}
3067
3069 BuildFnTy &MatchInfo) const {
3070 // Fold `a bitwiseop (~b +/- c)` -> `a bitwiseop ~(b -/+ c)`
3071 // Root MI is one of G_AND, G_OR, G_XOR.
3072 // We also look for commuted forms of operations. Pattern shouldn't apply
3073 // if there are multiple reasons of inner operations.
3074
3075 unsigned RootOpc = MI.getOpcode();
3076 Register Dst = MI.getOperand(0).getReg();
3077 LLT Ty = MRI.getType(Dst);
3078
3079 Register LHS = MI.getOperand(1).getReg();
3080 Register RHS = MI.getOperand(2).getReg();
3081 // Check the commuted and uncommuted forms of the operation.
3082 return matchBinopWithNegInner(LHS, RHS, RootOpc, Dst, Ty, MatchInfo) ||
3083 matchBinopWithNegInner(RHS, LHS, RootOpc, Dst, Ty, MatchInfo);
3084}
3085
3087 MachineInstr &MI, InstructionStepsMatchInfo &MatchInfo) const {
3088 // Matches: logic (hand x, ...), (hand y, ...) -> hand (logic x, y), ...
3089 //
3090 // Creates the new hand + logic instruction (but does not insert them.)
3091 //
3092 // On success, MatchInfo is populated with the new instructions. These are
3093 // inserted in applyHoistLogicOpWithSameOpcodeHands.
3094 unsigned LogicOpcode = MI.getOpcode();
3095 assert(LogicOpcode == TargetOpcode::G_AND ||
3096 LogicOpcode == TargetOpcode::G_OR ||
3097 LogicOpcode == TargetOpcode::G_XOR);
3098 Register Dst = MI.getOperand(0).getReg();
3099 Register LHSReg = MI.getOperand(1).getReg();
3100 Register RHSReg = MI.getOperand(2).getReg();
3101
3102 // Don't recompute anything.
3103 if (!MRI.hasOneNonDBGUse(LHSReg) || !MRI.hasOneNonDBGUse(RHSReg))
3104 return false;
3105
3106 // Make sure we have (hand x, ...), (hand y, ...)
3107 MachineInstr *LeftHandInst = getDefIgnoringCopies(LHSReg, MRI);
3108 MachineInstr *RightHandInst = getDefIgnoringCopies(RHSReg, MRI);
3109 if (!LeftHandInst || !RightHandInst)
3110 return false;
3111 unsigned HandOpcode = LeftHandInst->getOpcode();
3112 if (HandOpcode != RightHandInst->getOpcode())
3113 return false;
3114 if (LeftHandInst->getNumOperands() < 2 ||
3115 !LeftHandInst->getOperand(1).isReg() ||
3116 RightHandInst->getNumOperands() < 2 ||
3117 !RightHandInst->getOperand(1).isReg())
3118 return false;
3119
3120 // Make sure the types match up, and if we're doing this post-legalization,
3121 // we end up with legal types.
3122 Register X = LeftHandInst->getOperand(1).getReg();
3123 Register Y = RightHandInst->getOperand(1).getReg();
3124 LLT XTy = MRI.getType(X);
3125 LLT YTy = MRI.getType(Y);
3126 if (!XTy.isValid() || XTy != YTy)
3127 return false;
3128
3129 // Optional extra source register.
3130 Register ExtraHandOpSrcReg;
3131 switch (HandOpcode) {
3132 default:
3133 return false;
3134 case TargetOpcode::G_ANYEXT:
3135 case TargetOpcode::G_SEXT:
3136 case TargetOpcode::G_ZEXT: {
3137 // Match: logic (ext X), (ext Y) --> ext (logic X, Y)
3138 break;
3139 }
3140 case TargetOpcode::G_TRUNC: {
3141 // Match: logic (trunc X), (trunc Y) -> trunc (logic X, Y)
3142 const MachineFunction *MF = MI.getMF();
3143 LLVMContext &Ctx = MF->getFunction().getContext();
3144
3145 LLT DstTy = MRI.getType(Dst);
3147
3148 // Be extra careful sinking truncate. If it's free, there's no benefit in
3149 // widening a binop.
3150 if (TLI.isZExtFree(DstTy, XTy, Ctx) && TLI.isTruncateFree(XTy, DstTy, Ctx))
3151 return false;
3152 break;
3153 }
3154 case TargetOpcode::G_AND:
3155 case TargetOpcode::G_ASHR:
3156 case TargetOpcode::G_LSHR:
3157 case TargetOpcode::G_SHL: {
3158 // Match: logic (binop x, z), (binop y, z) -> binop (logic x, y), z
3159 MachineOperand &ZOp = LeftHandInst->getOperand(2);
3160 if (!matchEqualDefs(ZOp, RightHandInst->getOperand(2)))
3161 return false;
3162 ExtraHandOpSrcReg = ZOp.getReg();
3163 break;
3164 }
3165 }
3166
3167 if (!isLegalOrBeforeLegalizer({LogicOpcode, {XTy, YTy}}))
3168 return false;
3169
3170 // Record the steps to build the new instructions.
3171 //
3172 // Steps to build (logic x, y)
3173 auto NewLogicDst = MRI.createGenericVirtualRegister(XTy);
3174 OperandBuildSteps LogicBuildSteps = {
3175 [=](MachineInstrBuilder &MIB) { MIB.addDef(NewLogicDst); },
3176 [=](MachineInstrBuilder &MIB) { MIB.addReg(X); },
3177 [=](MachineInstrBuilder &MIB) { MIB.addReg(Y); }};
3178 InstructionBuildSteps LogicSteps(LogicOpcode, LogicBuildSteps);
3179
3180 // Steps to build hand (logic x, y), ...z
3181 OperandBuildSteps HandBuildSteps = {
3182 [=](MachineInstrBuilder &MIB) { MIB.addDef(Dst); },
3183 [=](MachineInstrBuilder &MIB) { MIB.addReg(NewLogicDst); }};
3184 if (ExtraHandOpSrcReg.isValid())
3185 HandBuildSteps.push_back(
3186 [=](MachineInstrBuilder &MIB) { MIB.addReg(ExtraHandOpSrcReg); });
3187 InstructionBuildSteps HandSteps(HandOpcode, HandBuildSteps);
3188
3189 MatchInfo = InstructionStepsMatchInfo({LogicSteps, HandSteps});
3190 return true;
3191}
3192
3194 MachineInstr &MI, InstructionStepsMatchInfo &MatchInfo) const {
3195 assert(MatchInfo.InstrsToBuild.size() &&
3196 "Expected at least one instr to build?");
3197 for (auto &InstrToBuild : MatchInfo.InstrsToBuild) {
3198 assert(InstrToBuild.Opcode && "Expected a valid opcode?");
3199 assert(InstrToBuild.OperandFns.size() && "Expected at least one operand?");
3200 MachineInstrBuilder Instr = Builder.buildInstr(InstrToBuild.Opcode);
3201 for (auto &OperandFn : InstrToBuild.OperandFns)
3202 OperandFn(Instr);
3203 }
3204 MI.eraseFromParent();
3205}
3206
3208 MachineInstr &MI, std::tuple<Register, int64_t> &MatchInfo) const {
3209 assert(MI.getOpcode() == TargetOpcode::G_ASHR);
3210 int64_t ShlCst, AshrCst;
3211 Register Src;
3212 if (!mi_match(MI.getOperand(0).getReg(), MRI,
3213 m_GAShr(m_GShl(m_Reg(Src), m_ICstOrSplat(ShlCst)),
3214 m_ICstOrSplat(AshrCst))))
3215 return false;
3216 if (ShlCst != AshrCst)
3217 return false;
3219 {TargetOpcode::G_SEXT_INREG,
3220 {MRI.getType(Src)},
3221 {},
3222 {MRI.getType(Src).getScalarSizeInBits() - ShlCst}}))
3223 return false;
3224 MatchInfo = std::make_tuple(Src, ShlCst);
3225 return true;
3226}
3227
3229 MachineInstr &MI, std::tuple<Register, int64_t> &MatchInfo) const {
3230 assert(MI.getOpcode() == TargetOpcode::G_ASHR);
3231 Register Src;
3232 int64_t ShiftAmt;
3233 std::tie(Src, ShiftAmt) = MatchInfo;
3234 unsigned Size = MRI.getType(Src).getScalarSizeInBits();
3235 Builder.buildSExtInReg(MI.getOperand(0).getReg(), Src, Size - ShiftAmt);
3236 MI.eraseFromParent();
3237}
3238
3239/// and(and(x, C1), C2) -> C1&C2 ? and(x, C1&C2) : 0
3242 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
3243 assert(MI.getOpcode() == TargetOpcode::G_AND);
3244
3245 Register Dst = MI.getOperand(0).getReg();
3246 LLT Ty = MRI.getType(Dst);
3247
3248 Register R;
3249 int64_t C1;
3250 int64_t C2;
3251 if (!mi_match(
3252 Dst, MRI,
3253 m_GAnd(m_GAnd(m_Reg(R), m_ICst(C1)), m_ICst(C2))))
3254 return false;
3255
3256 MatchInfo = [=](MachineIRBuilder &B) {
3257 if (C1 & C2) {
3258 B.buildAnd(Dst, R, B.buildConstant(Ty, C1 & C2));
3259 return;
3260 }
3261 auto Zero = B.buildConstant(Ty, 0);
3262 replaceRegWith(MRI, Dst, Zero->getOperand(0).getReg());
3263 };
3264 return true;
3265}
3266
3268 Register &Replacement) const {
3269 // Given
3270 //
3271 // %y:_(sN) = G_SOMETHING
3272 // %x:_(sN) = G_SOMETHING
3273 // %res:_(sN) = G_AND %x, %y
3274 //
3275 // Eliminate the G_AND when it is known that x & y == x or x & y == y.
3276 //
3277 // Patterns like this can appear as a result of legalization. E.g.
3278 //
3279 // %cmp:_(s32) = G_ICMP intpred(pred), %x(s32), %y
3280 // %one:_(s32) = G_CONSTANT i32 1
3281 // %and:_(s32) = G_AND %cmp, %one
3282 //
3283 // In this case, G_ICMP only produces a single bit, so x & 1 == x.
3284 assert(MI.getOpcode() == TargetOpcode::G_AND);
3285 if (!VT)
3286 return false;
3287
3288 Register AndDst = MI.getOperand(0).getReg();
3289 Register LHS = MI.getOperand(1).getReg();
3290 Register RHS = MI.getOperand(2).getReg();
3291
3292 // Check the RHS (maybe a constant) first, and if we have no KnownBits there,
3293 // we can't do anything. If we do, then it depends on whether we have
3294 // KnownBits on the LHS.
3295 KnownBits RHSBits = VT->getKnownBits(RHS);
3296 if (RHSBits.isUnknown())
3297 return false;
3298
3299 KnownBits LHSBits = VT->getKnownBits(LHS);
3300
3301 // Check that x & Mask == x.
3302 // x & 1 == x, always
3303 // x & 0 == x, only if x is also 0
3304 // Meaning Mask has no effect if every bit is either one in Mask or zero in x.
3305 //
3306 // Check if we can replace AndDst with the LHS of the G_AND
3307 if (canReplaceReg(AndDst, LHS, MRI) &&
3308 (LHSBits.Zero | RHSBits.One).isAllOnes()) {
3309 Replacement = LHS;
3310 return true;
3311 }
3312
3313 // Check if we can replace AndDst with the RHS of the G_AND
3314 if (canReplaceReg(AndDst, RHS, MRI) &&
3315 (LHSBits.One | RHSBits.Zero).isAllOnes()) {
3316 Replacement = RHS;
3317 return true;
3318 }
3319
3320 return false;
3321}
3322
3324 Register &Replacement) const {
3325 // Given
3326 //
3327 // %y:_(sN) = G_SOMETHING
3328 // %x:_(sN) = G_SOMETHING
3329 // %res:_(sN) = G_OR %x, %y
3330 //
3331 // Eliminate the G_OR when it is known that x | y == x or x | y == y.
3332 assert(MI.getOpcode() == TargetOpcode::G_OR);
3333 if (!VT)
3334 return false;
3335
3336 Register OrDst = MI.getOperand(0).getReg();
3337 Register LHS = MI.getOperand(1).getReg();
3338 Register RHS = MI.getOperand(2).getReg();
3339
3340 KnownBits LHSBits = VT->getKnownBits(LHS);
3341 KnownBits RHSBits = VT->getKnownBits(RHS);
3342
3343 // Check that x | Mask == x.
3344 // x | 0 == x, always
3345 // x | 1 == x, only if x is also 1
3346 // Meaning Mask has no effect if every bit is either zero in Mask or one in x.
3347 //
3348 // Check if we can replace OrDst with the LHS of the G_OR
3349 if (canReplaceReg(OrDst, LHS, MRI) &&
3350 (LHSBits.One | RHSBits.Zero).isAllOnes()) {
3351 Replacement = LHS;
3352 return true;
3353 }
3354
3355 // Check if we can replace OrDst with the RHS of the G_OR
3356 if (canReplaceReg(OrDst, RHS, MRI) &&
3357 (LHSBits.Zero | RHSBits.One).isAllOnes()) {
3358 Replacement = RHS;
3359 return true;
3360 }
3361
3362 return false;
3363}
3364
3366 // If the input is already sign extended, just drop the extension.
3367 Register Src = MI.getOperand(1).getReg();
3368 unsigned ExtBits = MI.getOperand(2).getImm();
3369 unsigned TypeSize = MRI.getType(Src).getScalarSizeInBits();
3370 return VT->computeNumSignBits(Src) >= (TypeSize - ExtBits + 1);
3371}
3372
3373static bool isConstValidTrue(const TargetLowering &TLI, unsigned ScalarSizeBits,
3374 int64_t Cst, bool IsVector, bool IsFP) {
3375 // For i1, Cst will always be -1 regardless of boolean contents.
3376 return (ScalarSizeBits == 1 && Cst == -1) ||
3377 isConstTrueVal(TLI, Cst, IsVector, IsFP);
3378}
3379
3380// This pattern aims to match the following shape to avoid extra mov
3381// instructions
3382// G_BUILD_VECTOR(
3383// G_UNMERGE_VALUES(src, 0)
3384// G_UNMERGE_VALUES(src, 1)
3385// G_IMPLICIT_DEF
3386// G_IMPLICIT_DEF
3387// )
3388// ->
3389// G_CONCAT_VECTORS(
3390// src,
3391// undef
3392// )
3395 Register &UnmergeSrc) const {
3396 auto &BV = cast<GBuildVector>(MI);
3397
3398 unsigned BuildUseCount = BV.getNumSources();
3399 if (BuildUseCount % 2 != 0)
3400 return false;
3401
3402 unsigned NumUnmerge = BuildUseCount / 2;
3403
3404 auto *Unmerge = getOpcodeDef<GUnmerge>(BV.getSourceReg(0), MRI);
3405
3406 // Check the first operand is an unmerge and has the correct number of
3407 // operands
3408 if (!Unmerge || Unmerge->getNumDefs() != NumUnmerge)
3409 return false;
3410
3411 UnmergeSrc = Unmerge->getSourceReg();
3412
3413 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
3414 LLT UnmergeSrcTy = MRI.getType(UnmergeSrc);
3415
3416 if (!UnmergeSrcTy.isVector())
3417 return false;
3418
3419 // Ensure we only generate legal instructions post-legalizer
3420 if (!IsPreLegalize &&
3421 !isLegal({TargetOpcode::G_CONCAT_VECTORS, {DstTy, UnmergeSrcTy}}))
3422 return false;
3423
3424 // Check that all of the operands before the midpoint come from the same
3425 // unmerge and are in the same order as they are used in the build_vector
3426 for (unsigned I = 0; I < NumUnmerge; ++I) {
3427 auto MaybeUnmergeReg = BV.getSourceReg(I);
3428 auto *LoopUnmerge = getOpcodeDef<GUnmerge>(MaybeUnmergeReg, MRI);
3429
3430 if (!LoopUnmerge || LoopUnmerge != Unmerge)
3431 return false;
3432
3433 if (LoopUnmerge->getOperand(I).getReg() != MaybeUnmergeReg)
3434 return false;
3435 }
3436
3437 // Check that all of the unmerged values are used
3438 if (Unmerge->getNumDefs() != NumUnmerge)
3439 return false;
3440
3441 // Check that all of the operands after the mid point are undefs.
3442 for (unsigned I = NumUnmerge; I < BuildUseCount; ++I) {
3443 auto *Undef = getDefIgnoringCopies(BV.getSourceReg(I), MRI);
3444
3445 if (Undef->getOpcode() != TargetOpcode::G_IMPLICIT_DEF)
3446 return false;
3447 }
3448
3449 return true;
3450}
3451
3455 Register &UnmergeSrc) const {
3456 assert(UnmergeSrc && "Expected there to be one matching G_UNMERGE_VALUES");
3457 B.setInstrAndDebugLoc(MI);
3458
3459 Register UndefVec = B.buildUndef(MRI.getType(UnmergeSrc)).getReg(0);
3460 B.buildConcatVectors(MI.getOperand(0), {UnmergeSrc, UndefVec});
3461
3462 MI.eraseFromParent();
3463}
3464
3465// This combine tries to reduce the number of scalarised G_TRUNC instructions by
3466// using vector truncates instead
3467//
3468// EXAMPLE:
3469// %a(i32), %b(i32) = G_UNMERGE_VALUES %src(<2 x i32>)
3470// %T_a(i16) = G_TRUNC %a(i32)
3471// %T_b(i16) = G_TRUNC %b(i32)
3472// %Undef(i16) = G_IMPLICIT_DEF(i16)
3473// %dst(v4i16) = G_BUILD_VECTORS %T_a(i16), %T_b(i16), %Undef(i16), %Undef(i16)
3474//
3475// ===>
3476// %Undef(<2 x i32>) = G_IMPLICIT_DEF(<2 x i32>)
3477// %Mid(<4 x s32>) = G_CONCAT_VECTORS %src(<2 x i32>), %Undef(<2 x i32>)
3478// %dst(<4 x s16>) = G_TRUNC %Mid(<4 x s32>)
3479//
3480// Only matches sources made up of G_TRUNCs followed by G_IMPLICIT_DEFs
3482 Register &MatchInfo) const {
3483 auto BuildMI = cast<GBuildVector>(&MI);
3484 unsigned NumOperands = BuildMI->getNumSources();
3485 LLT DstTy = MRI.getType(BuildMI->getReg(0));
3486
3487 // Check the G_BUILD_VECTOR sources
3488 unsigned I;
3489 GUnmerge *UnmergeMI = nullptr;
3490
3491 // Check all source TRUNCs come from the same UNMERGE instruction
3492 // and that the element order matches (BUILD_VECTOR position I
3493 // corresponds to UNMERGE result I)
3494 for (I = 0; I < NumOperands; ++I) {
3495 // Check if the G_TRUNC instructions all come from the same MI
3496 Register TruncSrcReg;
3497 if (!mi_match(BuildMI->getSourceReg(I), MRI, m_GTrunc(m_Reg(TruncSrcReg))))
3498 break;
3499
3500 if (!UnmergeMI) {
3501 if (!mi_match(TruncSrcReg, MRI, m_GUnmerge(UnmergeMI)))
3502 return false;
3503 } else {
3504 MachineInstr *UnmergeSrcMI;
3505 if (!mi_match(TruncSrcReg, MRI, m_MInstr(UnmergeSrcMI)) ||
3506 UnmergeMI != UnmergeSrcMI)
3507 return false;
3508 }
3509 // Element order must match: position I must use UNMERGE result I.
3510 if (UnmergeMI->getOperand(I).getReg() != TruncSrcReg)
3511 return false;
3512 }
3513 if (I < 2)
3514 return false;
3515
3516 // Check the remaining source elements are only G_IMPLICIT_DEF
3517 for (; I < NumOperands; ++I) {
3518 if (!mi_match(BuildMI->getSourceReg(I), MRI, m_GImplicitDef()))
3519 return false;
3520 }
3521
3522 // Check the size of unmerge source
3523 MatchInfo = UnmergeMI->getSourceReg();
3524 LLT UnmergeSrcTy = MRI.getType(MatchInfo);
3525 if (!UnmergeSrcTy.isVector())
3526 return false;
3527
3528 if (!DstTy.getElementCount().isKnownMultipleOf(UnmergeSrcTy.getNumElements()))
3529 return false;
3530
3531 // Check the unmerge source and destination element types match
3532 LLT UnmergeSrcEltTy = UnmergeSrcTy.getElementType();
3533 Register UnmergeDstReg = UnmergeMI->getOperand(0).getReg();
3534 LLT UnmergeDstEltTy = MRI.getType(UnmergeDstReg);
3535 if (UnmergeSrcEltTy != UnmergeDstEltTy)
3536 return false;
3537
3538 // Only generate legal instructions post-legalizer
3539 if (!IsPreLegalize) {
3540 LLT MidTy = DstTy.changeElementType(UnmergeSrcTy.getScalarType());
3541
3542 if (DstTy.getElementCount() != UnmergeSrcTy.getElementCount() &&
3543 !isLegal({TargetOpcode::G_CONCAT_VECTORS, {MidTy, UnmergeSrcTy}}))
3544 return false;
3545
3546 if (!isLegal({TargetOpcode::G_TRUNC, {DstTy, MidTy}}))
3547 return false;
3548 }
3549
3550 return true;
3551}
3552
3554 Register &MatchInfo) const {
3555 Register MidReg;
3556 auto BuildMI = cast<GBuildVector>(&MI);
3557 Register DstReg = BuildMI->getReg(0);
3558 LLT DstTy = MRI.getType(DstReg);
3559 LLT UnmergeSrcTy = MRI.getType(MatchInfo);
3560 unsigned DstTyNumElt = DstTy.getNumElements();
3561 unsigned UnmergeSrcTyNumElt = UnmergeSrcTy.getNumElements();
3562
3563 // No need to pad vector if only G_TRUNC is needed
3564 if (DstTyNumElt / UnmergeSrcTyNumElt == 1) {
3565 MidReg = MatchInfo;
3566 } else {
3567 Register UndefReg = Builder.buildUndef(UnmergeSrcTy).getReg(0);
3568 SmallVector<Register> ConcatRegs = {MatchInfo};
3569 for (unsigned I = 1; I < DstTyNumElt / UnmergeSrcTyNumElt; ++I)
3570 ConcatRegs.push_back(UndefReg);
3571
3572 auto MidTy = DstTy.changeElementType(UnmergeSrcTy.getScalarType());
3573 MidReg = Builder.buildConcatVectors(MidTy, ConcatRegs).getReg(0);
3574 }
3575
3576 Builder.buildTrunc(DstReg, MidReg);
3577 MI.eraseFromParent();
3578}
3579
3581 MachineInstr &MI, SmallVectorImpl<Register> &RegsToNegate) const {
3582 assert(MI.getOpcode() == TargetOpcode::G_XOR);
3583 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
3584 const auto &TLI = *Builder.getMF().getSubtarget().getTargetLowering();
3585 Register XorSrc;
3586 Register CstReg;
3587 // We match xor(src, true) here.
3588 if (!mi_match(MI.getOperand(0).getReg(), MRI,
3589 m_GXor(m_Reg(XorSrc), m_Reg(CstReg))))
3590 return false;
3591
3592 if (!MRI.hasOneNonDBGUse(XorSrc))
3593 return false;
3594
3595 // Check that XorSrc is the root of a tree of comparisons combined with ANDs
3596 // and ORs. The suffix of RegsToNegate starting from index I is used a work
3597 // list of tree nodes to visit.
3598 RegsToNegate.push_back(XorSrc);
3599 // Remember whether the comparisons are all integer or all floating point.
3600 bool IsInt = false;
3601 bool IsFP = false;
3602 for (unsigned I = 0; I < RegsToNegate.size(); ++I) {
3603 Register Reg = RegsToNegate[I];
3604 if (!MRI.hasOneNonDBGUse(Reg))
3605 return false;
3606 MachineInstr *Def;
3607 if (!mi_match(Reg, MRI, m_MInstr(Def)))
3608 return false;
3609 switch (Def->getOpcode()) {
3610 default:
3611 // Don't match if the tree contains anything other than ANDs, ORs and
3612 // comparisons.
3613 return false;
3614 case TargetOpcode::G_ICMP:
3615 if (IsFP)
3616 return false;
3617 IsInt = true;
3618 // When we apply the combine we will invert the predicate.
3619 break;
3620 case TargetOpcode::G_FCMP:
3621 if (IsInt)
3622 return false;
3623 IsFP = true;
3624 // When we apply the combine we will invert the predicate.
3625 break;
3626 case TargetOpcode::G_AND:
3627 case TargetOpcode::G_OR:
3628 // Implement De Morgan's laws:
3629 // ~(x & y) -> ~x | ~y
3630 // ~(x | y) -> ~x & ~y
3631 // When we apply the combine we will change the opcode and recursively
3632 // negate the operands.
3633 RegsToNegate.push_back(Def->getOperand(1).getReg());
3634 RegsToNegate.push_back(Def->getOperand(2).getReg());
3635 break;
3636 }
3637 }
3638
3639 // Now we know whether the comparisons are integer or floating point, check
3640 // the constant in the xor.
3641 int64_t Cst;
3642 if (Ty.isVector()) {
3643 int64_t SplatCst;
3644 if (!mi_match(CstReg, MRI, m_ICstOrSplat(SplatCst)))
3645 return false;
3646 if (!isConstValidTrue(TLI, Ty.getScalarSizeInBits(), SplatCst, true, IsFP))
3647 return false;
3648 } else {
3649 if (!mi_match(CstReg, MRI, m_ICst(Cst)))
3650 return false;
3651 if (!isConstValidTrue(TLI, Ty.getSizeInBits(), Cst, false, IsFP))
3652 return false;
3653 }
3654
3655 return true;
3656}
3657
3659 MachineInstr &MI, SmallVectorImpl<Register> &RegsToNegate) const {
3660 for (Register Reg : RegsToNegate) {
3661 MachineInstr *Def = MRI.getVRegDef(Reg);
3662 Observer.changingInstr(*Def);
3663 // For each comparison, invert the opcode. For each AND and OR, change the
3664 // opcode.
3665 switch (Def->getOpcode()) {
3666 default:
3667 llvm_unreachable("Unexpected opcode");
3668 case TargetOpcode::G_ICMP:
3669 case TargetOpcode::G_FCMP: {
3670 MachineOperand &PredOp = Def->getOperand(1);
3673 PredOp.setPredicate(NewP);
3674 break;
3675 }
3676 case TargetOpcode::G_AND:
3677 Def->setDesc(Builder.getTII().get(TargetOpcode::G_OR));
3678 break;
3679 case TargetOpcode::G_OR:
3680 Def->setDesc(Builder.getTII().get(TargetOpcode::G_AND));
3681 break;
3682 }
3683 Observer.changedInstr(*Def);
3684 }
3685
3686 replaceRegWith(MRI, MI.getOperand(0).getReg(), MI.getOperand(1).getReg());
3687 MI.eraseFromParent();
3688}
3689
3691 MachineInstr &MI, std::pair<Register, Register> &MatchInfo) const {
3692 // Match (xor (and x, y), y) (or any of its commuted cases)
3693 assert(MI.getOpcode() == TargetOpcode::G_XOR);
3694 Register &X = MatchInfo.first;
3695 Register &Y = MatchInfo.second;
3696 Register AndReg = MI.getOperand(1).getReg();
3697 Register SharedReg = MI.getOperand(2).getReg();
3698
3699 // Find a G_AND on either side of the G_XOR.
3700 // Look for one of
3701 //
3702 // (xor (and x, y), SharedReg)
3703 // (xor SharedReg, (and x, y))
3704 if (!mi_match(AndReg, MRI, m_GAnd(m_Reg(X), m_Reg(Y)))) {
3705 std::swap(AndReg, SharedReg);
3706 if (!mi_match(AndReg, MRI, m_GAnd(m_Reg(X), m_Reg(Y))))
3707 return false;
3708 }
3709
3710 // Only do this if we'll eliminate the G_AND.
3711 if (!MRI.hasOneNonDBGUse(AndReg))
3712 return false;
3713
3714 // We can combine if SharedReg is the same as either the LHS or RHS of the
3715 // G_AND.
3716 if (Y != SharedReg)
3717 std::swap(X, Y);
3718 return Y == SharedReg;
3719}
3720
3722 MachineInstr &MI, std::pair<Register, Register> &MatchInfo) const {
3723 // Fold (xor (and x, y), y) -> (and (not x), y)
3724 Register X, Y;
3725 std::tie(X, Y) = MatchInfo;
3726 auto Not = Builder.buildNot(MRI.getType(X), X);
3727 Observer.changingInstr(MI);
3728 MI.setDesc(Builder.getTII().get(TargetOpcode::G_AND));
3729 MI.getOperand(1).setReg(Not->getOperand(0).getReg());
3730 MI.getOperand(2).setReg(Y);
3731 Observer.changedInstr(MI);
3732}
3733
3735 auto &PtrAdd = cast<GPtrAdd>(MI);
3736 Register DstReg = PtrAdd.getReg(0);
3737 LLT Ty = MRI.getType(DstReg);
3738 const DataLayout &DL = Builder.getMF().getDataLayout();
3739
3740 if (DL.isNonIntegralAddressSpace(Ty.getScalarType().getAddressSpace()))
3741 return false;
3742
3743 if (!Ty.isVector())
3744 return false;
3745 const MachineInstr *VecMI;
3746 if (!mi_match(PtrAdd.getBaseReg(), MRI, m_MInstr(VecMI)))
3747 return false;
3748 return isBuildVectorAllZeros(*VecMI, MRI);
3749}
3750
3751/// The second source operand is known to be a power of 2.
3753 Register DstReg = MI.getOperand(0).getReg();
3754 Register Src0 = MI.getOperand(1).getReg();
3755 Register Pow2Src1 = MI.getOperand(2).getReg();
3756 LLT Ty = MRI.getType(DstReg);
3757
3758 // Fold (urem x, pow2) -> (and x, pow2-1)
3759 auto NegOne = Builder.buildConstant(Ty, -1);
3760 auto Add = Builder.buildAdd(Ty, Pow2Src1, NegOne);
3761 Builder.buildAnd(DstReg, Src0, Add);
3762 MI.eraseFromParent();
3763}
3764
3766 unsigned &SelectOpNo) const {
3767 Register LHS = MI.getOperand(1).getReg();
3768 Register RHS = MI.getOperand(2).getReg();
3769
3770 Register OtherOperandReg = RHS;
3771 SelectOpNo = 1;
3772 Register SelectTrue, SelectFalse;
3773
3774 // Don't do this unless the old select is going away. We want to eliminate the
3775 // binary operator, not replace a binop with a select.
3776 if (!mi_match(LHS, MRI,
3777 m_GISelect(m_Reg(), m_Reg(SelectTrue), m_Reg(SelectFalse))) ||
3778 !MRI.hasOneNonDBGUse(LHS)) {
3779 OtherOperandReg = LHS;
3780 SelectOpNo = 2;
3781 if (!mi_match(RHS, MRI,
3782 m_GISelect(m_Reg(), m_Reg(SelectTrue), m_Reg(SelectFalse))) ||
3783 !MRI.hasOneNonDBGUse(RHS))
3784 return false;
3785 }
3786
3787 MachineInstr *SelectLHS, *SelectRHS;
3788 if (!mi_match(SelectTrue, MRI, m_MInstr(SelectLHS)) ||
3789 !mi_match(SelectFalse, MRI, m_MInstr(SelectRHS)))
3790 return false;
3791
3792 if (!isConstantOrConstantVector(*SelectLHS, MRI,
3793 /*AllowFP*/ true,
3794 /*AllowOpaqueConstants*/ false))
3795 return false;
3796 if (!isConstantOrConstantVector(*SelectRHS, MRI,
3797 /*AllowFP*/ true,
3798 /*AllowOpaqueConstants*/ false))
3799 return false;
3800
3801 unsigned BinOpcode = MI.getOpcode();
3802
3803 // We know that one of the operands is a select of constants. Now verify that
3804 // the other binary operator operand is either a constant, or we can handle a
3805 // variable.
3806 bool CanFoldNonConst =
3807 (BinOpcode == TargetOpcode::G_AND || BinOpcode == TargetOpcode::G_OR) &&
3808 (isNullOrNullSplat(*SelectLHS, MRI) ||
3809 isAllOnesOrAllOnesSplat(*SelectLHS, MRI)) &&
3810 (isNullOrNullSplat(*SelectRHS, MRI) ||
3811 isAllOnesOrAllOnesSplat(*SelectRHS, MRI));
3812 if (CanFoldNonConst)
3813 return true;
3814
3815 MachineInstr *OtherOperandDef;
3816 if (!mi_match(OtherOperandReg, MRI, m_MInstr(OtherOperandDef)))
3817 return false;
3818 return isConstantOrConstantVector(*OtherOperandDef, MRI,
3819 /*AllowFP*/ true,
3820 /*AllowOpaqueConstants*/ false);
3821}
3822
3823/// \p SelectOperand is the operand in binary operator \p MI that is the select
3824/// to fold.
3826 MachineInstr &MI, const unsigned &SelectOperand) const {
3827 Register Dst = MI.getOperand(0).getReg();
3828 Register LHS = MI.getOperand(1).getReg();
3829 Register RHS = MI.getOperand(2).getReg();
3830 GSelect *Select =
3831 cast<GSelect>(MRI.getVRegDef(MI.getOperand(SelectOperand).getReg()));
3832
3833 Register SelectCond = Select->getCondReg();
3834 Register SelectTrue = Select->getTrueReg();
3835 Register SelectFalse = Select->getFalseReg();
3836
3837 LLT Ty = MRI.getType(Dst);
3838 unsigned BinOpcode = MI.getOpcode();
3839
3840 Register FoldTrue, FoldFalse;
3841
3842 // We have a select-of-constants followed by a binary operator with a
3843 // constant. Eliminate the binop by pulling the constant math into the select.
3844 // Example: add (select Cond, CT, CF), CBO --> select Cond, CT + CBO, CF + CBO
3845 if (SelectOperand == 1) {
3846 // TODO: SelectionDAG verifies this actually constant folds before
3847 // committing to the combine.
3848
3849 FoldTrue = Builder.buildInstr(BinOpcode, {Ty}, {SelectTrue, RHS}).getReg(0);
3850 FoldFalse =
3851 Builder.buildInstr(BinOpcode, {Ty}, {SelectFalse, RHS}).getReg(0);
3852 } else {
3853 FoldTrue = Builder.buildInstr(BinOpcode, {Ty}, {LHS, SelectTrue}).getReg(0);
3854 FoldFalse =
3855 Builder.buildInstr(BinOpcode, {Ty}, {LHS, SelectFalse}).getReg(0);
3856 }
3857
3858 Builder.buildSelect(Dst, SelectCond, FoldTrue, FoldFalse, MI.getFlags());
3859 MI.eraseFromParent();
3860}
3861
3862std::optional<SmallVector<Register, 8>>
3863CombinerHelper::findCandidatesForLoadOrCombine(const MachineInstr *Root) const {
3864 assert(Root->getOpcode() == TargetOpcode::G_OR && "Expected G_OR only!");
3865 // We want to detect if Root is part of a tree which represents a bunch
3866 // of loads being merged into a larger load. We'll try to recognize patterns
3867 // like, for example:
3868 //
3869 // Reg Reg
3870 // \ /
3871 // OR_1 Reg
3872 // \ /
3873 // OR_2
3874 // \ Reg
3875 // .. /
3876 // Root
3877 //
3878 // Reg Reg Reg Reg
3879 // \ / \ /
3880 // OR_1 OR_2
3881 // \ /
3882 // \ /
3883 // ...
3884 // Root
3885 //
3886 // Each "Reg" may have been produced by a load + some arithmetic. This
3887 // function will save each of them.
3888 SmallVector<Register, 8> RegsToVisit;
3890
3891 // In the "worst" case, we're dealing with a load for each byte. So, there
3892 // are at most #bytes - 1 ORs.
3893 const unsigned MaxIter =
3894 MRI.getType(Root->getOperand(0).getReg()).getSizeInBytes() - 1;
3895 for (unsigned Iter = 0; Iter < MaxIter; ++Iter) {
3896 if (Ors.empty())
3897 break;
3898 const MachineInstr *Curr = Ors.pop_back_val();
3899 Register OrLHS = Curr->getOperand(1).getReg();
3900 Register OrRHS = Curr->getOperand(2).getReg();
3901
3902 // In the combine, we want to elimate the entire tree.
3903 if (!MRI.hasOneNonDBGUse(OrLHS) || !MRI.hasOneNonDBGUse(OrRHS))
3904 return std::nullopt;
3905
3906 // If it's a G_OR, save it and continue to walk. If it's not, then it's
3907 // something that may be a load + arithmetic.
3908 if (const MachineInstr *Or = getOpcodeDef(TargetOpcode::G_OR, OrLHS, MRI))
3909 Ors.push_back(Or);
3910 else
3911 RegsToVisit.push_back(OrLHS);
3912 if (const MachineInstr *Or = getOpcodeDef(TargetOpcode::G_OR, OrRHS, MRI))
3913 Ors.push_back(Or);
3914 else
3915 RegsToVisit.push_back(OrRHS);
3916 }
3917
3918 // We're going to try and merge each register into a wider power-of-2 type,
3919 // so we ought to have an even number of registers.
3920 if (RegsToVisit.empty() || RegsToVisit.size() % 2 != 0)
3921 return std::nullopt;
3922 return RegsToVisit;
3923}
3924
3925/// Helper function for findLoadOffsetsForLoadOrCombine.
3926///
3927/// Check if \p Reg is the result of loading a \p MemSizeInBits wide value,
3928/// and then moving that value into a specific byte offset.
3929///
3930/// e.g. x[i] << 24
3931///
3932/// \returns The load instruction and the byte offset it is moved into.
3933static std::optional<std::pair<GZExtLoad *, int64_t>>
3934matchLoadAndBytePosition(Register Reg, unsigned MemSizeInBits,
3935 const MachineRegisterInfo &MRI) {
3936 assert(MRI.hasOneNonDBGUse(Reg) &&
3937 "Expected Reg to only have one non-debug use?");
3938 Register MaybeLoad;
3939 int64_t Shift;
3940 if (!mi_match(Reg, MRI, m_OneUse(m_GShl(m_Reg(MaybeLoad), m_ICst(Shift))))) {
3941 Shift = 0;
3942 MaybeLoad = Reg;
3943 }
3944
3945 if (Shift % MemSizeInBits != 0)
3946 return std::nullopt;
3947
3948 // TODO: Handle other types of loads.
3949 auto *Load = getOpcodeDef<GZExtLoad>(MaybeLoad, MRI);
3950 if (!Load)
3951 return std::nullopt;
3952
3953 if (!Load->isUnordered() || Load->getMemSizeInBits() != MemSizeInBits)
3954 return std::nullopt;
3955
3956 return std::make_pair(Load, Shift / MemSizeInBits);
3957}
3958
3959std::optional<std::tuple<GZExtLoad *, int64_t, GZExtLoad *>>
3960CombinerHelper::findLoadOffsetsForLoadOrCombine(
3962 const SmallVector<Register, 8> &RegsToVisit,
3963 const unsigned MemSizeInBits) const {
3964
3965 // Each load found for the pattern. There should be one for each RegsToVisit.
3966 SmallSetVector<const MachineInstr *, 8> Loads;
3967
3968 // The lowest index used in any load. (The lowest "i" for each x[i].)
3969 int64_t LowestIdx = INT64_MAX;
3970
3971 // The load which uses the lowest index.
3972 GZExtLoad *LowestIdxLoad = nullptr;
3973
3974 // Keeps track of the load indices we see. We shouldn't see any indices twice.
3975 SmallSet<int64_t, 8> SeenIdx;
3976
3977 // Ensure each load is in the same MBB.
3978 // TODO: Support multiple MachineBasicBlocks.
3979 MachineBasicBlock *MBB = nullptr;
3980 const MachineMemOperand *MMO = nullptr;
3981
3982 // Earliest instruction-order load in the pattern.
3983 GZExtLoad *EarliestLoad = nullptr;
3984
3985 // Latest instruction-order load in the pattern.
3986 GZExtLoad *LatestLoad = nullptr;
3987
3988 // Base pointer which every load should share.
3990
3991 // We want to find a load for each register. Each load should have some
3992 // appropriate bit twiddling arithmetic. During this loop, we will also keep
3993 // track of the load which uses the lowest index. Later, we will check if we
3994 // can use its pointer in the final, combined load.
3995 for (auto Reg : RegsToVisit) {
3996 // Find the load, and find the position that it will end up in (e.g. a
3997 // shifted) value.
3998 auto LoadAndPos = matchLoadAndBytePosition(Reg, MemSizeInBits, MRI);
3999 if (!LoadAndPos)
4000 return std::nullopt;
4001 GZExtLoad *Load;
4002 int64_t DstPos;
4003 std::tie(Load, DstPos) = *LoadAndPos;
4004
4005 // TODO: Handle multiple MachineBasicBlocks. Currently not handled because
4006 // it is difficult to check for stores/calls/etc between loads.
4007 MachineBasicBlock *LoadMBB = Load->getParent();
4008 if (!MBB)
4009 MBB = LoadMBB;
4010 if (LoadMBB != MBB)
4011 return std::nullopt;
4012
4013 // Make sure that the MachineMemOperands of every seen load are compatible.
4014 auto &LoadMMO = Load->getMMO();
4015 if (!MMO)
4016 MMO = &LoadMMO;
4017 if (MMO->getAddrSpace() != LoadMMO.getAddrSpace())
4018 return std::nullopt;
4019
4020 // Find out what the base pointer and index for the load is.
4021 Register LoadPtr;
4022 int64_t Idx;
4023 if (!mi_match(Load->getOperand(1).getReg(), MRI,
4024 m_GPtrAdd(m_Reg(LoadPtr), m_ICst(Idx)))) {
4025 LoadPtr = Load->getOperand(1).getReg();
4026 Idx = 0;
4027 }
4028
4029 // Don't combine things like a[i], a[i] -> a bigger load.
4030 if (!SeenIdx.insert(Idx).second)
4031 return std::nullopt;
4032
4033 // Every load must share the same base pointer; don't combine things like:
4034 //
4035 // a[i], b[i + 1] -> a bigger load.
4036 if (!BasePtr.isValid())
4037 BasePtr = LoadPtr;
4038 if (BasePtr != LoadPtr)
4039 return std::nullopt;
4040
4041 if (Idx < LowestIdx) {
4042 LowestIdx = Idx;
4043 LowestIdxLoad = Load;
4044 }
4045
4046 // Keep track of the byte offset that this load ends up at. If we have seen
4047 // the byte offset, then stop here. We do not want to combine:
4048 //
4049 // a[i] << 16, a[i + k] << 16 -> a bigger load.
4050 if (!MemOffset2Idx.try_emplace(DstPos, Idx).second)
4051 return std::nullopt;
4052 Loads.insert(Load);
4053
4054 // Keep track of the position of the earliest/latest loads in the pattern.
4055 // We will check that there are no load fold barriers between them later
4056 // on.
4057 //
4058 // FIXME: Is there a better way to check for load fold barriers?
4059 if (!EarliestLoad || dominates(*Load, *EarliestLoad))
4060 EarliestLoad = Load;
4061 if (!LatestLoad || dominates(*LatestLoad, *Load))
4062 LatestLoad = Load;
4063 }
4064
4065 // We found a load for each register. Let's check if each load satisfies the
4066 // pattern.
4067 assert(Loads.size() == RegsToVisit.size() &&
4068 "Expected to find a load for each register?");
4069 assert(EarliestLoad != LatestLoad && EarliestLoad &&
4070 LatestLoad && "Expected at least two loads?");
4071
4072 // Check if there are any stores, calls, etc. between any of the loads. If
4073 // there are, then we can't safely perform the combine.
4074 //
4075 // MaxIter is chosen based off the (worst case) number of iterations it
4076 // typically takes to succeed in the LLVM test suite plus some padding.
4077 //
4078 // FIXME: Is there a better way to check for load fold barriers?
4079 const unsigned MaxIter = 20;
4080 unsigned Iter = 0;
4081 for (const auto &MI : instructionsWithoutDebug(EarliestLoad->getIterator(),
4082 LatestLoad->getIterator())) {
4083 if (Loads.count(&MI))
4084 continue;
4085 if (MI.isLoadFoldBarrier())
4086 return std::nullopt;
4087 if (Iter++ == MaxIter)
4088 return std::nullopt;
4089 }
4090
4091 return std::make_tuple(LowestIdxLoad, LowestIdx, LatestLoad);
4092}
4093
4096 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
4097 assert(MI.getOpcode() == TargetOpcode::G_OR);
4098 MachineFunction &MF = *MI.getMF();
4099 // Assuming a little-endian target, transform:
4100 // s8 *a = ...
4101 // s32 val = a[0] | (a[1] << 8) | (a[2] << 16) | (a[3] << 24)
4102 // =>
4103 // s32 val = *((i32)a)
4104 //
4105 // s8 *a = ...
4106 // s32 val = (a[0] << 24) | (a[1] << 16) | (a[2] << 8) | a[3]
4107 // =>
4108 // s32 val = BSWAP(*((s32)a))
4109 Register Dst = MI.getOperand(0).getReg();
4110 LLT Ty = MRI.getType(Dst);
4111 if (Ty.isVector())
4112 return false;
4113
4114 // We need to combine at least two loads into this type. Since the smallest
4115 // possible load is into a byte, we need at least a 16-bit wide type.
4116 const unsigned WideMemSizeInBits = Ty.getSizeInBits();
4117 if (WideMemSizeInBits < 16 || WideMemSizeInBits % 8 != 0)
4118 return false;
4119
4120 // Match a collection of non-OR instructions in the pattern.
4121 auto RegsToVisit = findCandidatesForLoadOrCombine(&MI);
4122 if (!RegsToVisit)
4123 return false;
4124
4125 // We have a collection of non-OR instructions. Figure out how wide each of
4126 // the small loads should be based off of the number of potential loads we
4127 // found.
4128 const unsigned NarrowMemSizeInBits = WideMemSizeInBits / RegsToVisit->size();
4129 if (NarrowMemSizeInBits % 8 != 0)
4130 return false;
4131
4132 // Check if each register feeding into each OR is a load from the same
4133 // base pointer + some arithmetic.
4134 //
4135 // e.g. a[0], a[1] << 8, a[2] << 16, etc.
4136 //
4137 // Also verify that each of these ends up putting a[i] into the same memory
4138 // offset as a load into a wide type would.
4140 GZExtLoad *LowestIdxLoad, *LatestLoad;
4141 int64_t LowestIdx;
4142 auto MaybeLoadInfo = findLoadOffsetsForLoadOrCombine(
4143 MemOffset2Idx, *RegsToVisit, NarrowMemSizeInBits);
4144 if (!MaybeLoadInfo)
4145 return false;
4146 std::tie(LowestIdxLoad, LowestIdx, LatestLoad) = *MaybeLoadInfo;
4147
4148 // We have a bunch of loads being OR'd together. Using the addresses + offsets
4149 // we found before, check if this corresponds to a big or little endian byte
4150 // pattern. If it does, then we can represent it using a load + possibly a
4151 // BSWAP.
4152 bool IsBigEndianTarget = MF.getDataLayout().isBigEndian();
4153 std::optional<bool> IsBigEndian = isBigEndian(MemOffset2Idx, LowestIdx);
4154 if (!IsBigEndian)
4155 return false;
4156 bool NeedsBSwap = IsBigEndianTarget != *IsBigEndian;
4157 if (NeedsBSwap && !isLegalOrBeforeLegalizer({TargetOpcode::G_BSWAP, {Ty}}))
4158 return false;
4159
4160 // Make sure that the load from the lowest index produces offset 0 in the
4161 // final value.
4162 //
4163 // This ensures that we won't combine something like this:
4164 //
4165 // load x[i] -> byte 2
4166 // load x[i+1] -> byte 0 ---> wide_load x[i]
4167 // load x[i+2] -> byte 1
4168 const unsigned NumLoadsInTy = WideMemSizeInBits / NarrowMemSizeInBits;
4169 const unsigned ZeroByteOffset =
4170 *IsBigEndian
4171 ? bigEndianByteAt(NumLoadsInTy, 0)
4172 : littleEndianByteAt(NumLoadsInTy, 0);
4173 auto ZeroOffsetIdx = MemOffset2Idx.find(ZeroByteOffset);
4174 if (ZeroOffsetIdx == MemOffset2Idx.end() ||
4175 ZeroOffsetIdx->second != LowestIdx)
4176 return false;
4177
4178 // We wil reuse the pointer from the load which ends up at byte offset 0. It
4179 // may not use index 0.
4180 Register Ptr = LowestIdxLoad->getPointerReg();
4181 const MachineMemOperand &MMO = LowestIdxLoad->getMMO();
4182 LegalityQuery::MemDesc MMDesc(MMO);
4183 MMDesc.MemoryTy = Ty;
4185 {TargetOpcode::G_LOAD, {Ty, MRI.getType(Ptr)}, {MMDesc}}))
4186 return false;
4187 auto PtrInfo = MMO.getPointerInfo();
4188 auto *NewMMO = MF.getMachineMemOperand(&MMO, PtrInfo, WideMemSizeInBits / 8);
4189
4190 // Load must be allowed and fast on the target.
4192 auto &DL = MF.getDataLayout();
4193 unsigned Fast = 0;
4194 if (!getTargetLowering().allowsMemoryAccess(C, DL, Ty, *NewMMO, &Fast) ||
4195 !Fast)
4196 return false;
4197
4198 MatchInfo = [=](MachineIRBuilder &MIB) {
4199 MIB.setInstrAndDebugLoc(*LatestLoad);
4200 Register LoadDst = NeedsBSwap ? MRI.cloneVirtualRegister(Dst) : Dst;
4201 MIB.buildLoad(LoadDst, Ptr, *NewMMO);
4202 if (NeedsBSwap)
4203 MIB.buildBSwap(Dst, LoadDst);
4204 };
4205 return true;
4206}
4207
4209 MachineInstr *&ExtMI) const {
4210 auto &PHI = cast<GPhi>(MI);
4211 Register DstReg = PHI.getReg(0);
4212
4213 // TODO: Extending a vector may be expensive, don't do this until heuristics
4214 // are better.
4215 if (MRI.getType(DstReg).isVector())
4216 return false;
4217
4218 // Try to match a phi, whose only use is an extend.
4219 if (!MRI.hasOneNonDBGUse(DstReg))
4220 return false;
4221 ExtMI = &*MRI.use_instr_nodbg_begin(DstReg);
4222 switch (ExtMI->getOpcode()) {
4223 case TargetOpcode::G_ANYEXT:
4224 return true; // G_ANYEXT is usually free.
4225 case TargetOpcode::G_ZEXT:
4226 case TargetOpcode::G_SEXT:
4227 break;
4228 default:
4229 return false;
4230 }
4231
4232 // If the target is likely to fold this extend away, don't propagate.
4233 if (Builder.getTII().isExtendLikelyToBeFolded(*ExtMI, MRI))
4234 return false;
4235
4236 // We don't want to propagate the extends unless there's a good chance that
4237 // they'll be optimized in some way.
4238 // Collect the unique incoming values.
4240 for (unsigned I = 0; I < PHI.getNumIncomingValues(); ++I) {
4241 auto *DefMI = getDefIgnoringCopies(PHI.getIncomingValue(I), MRI);
4242 switch (DefMI->getOpcode()) {
4243 case TargetOpcode::G_LOAD:
4244 case TargetOpcode::G_TRUNC:
4245 case TargetOpcode::G_SEXT:
4246 case TargetOpcode::G_ZEXT:
4247 case TargetOpcode::G_ANYEXT:
4248 case TargetOpcode::G_CONSTANT:
4249 InSrcs.insert(DefMI);
4250 // Don't try to propagate if there are too many places to create new
4251 // extends, chances are it'll increase code size.
4252 if (InSrcs.size() > 2)
4253 return false;
4254 break;
4255 default:
4256 return false;
4257 }
4258 }
4259 return true;
4260}
4261
4263 MachineInstr *&ExtMI) const {
4264 auto &PHI = cast<GPhi>(MI);
4265 Register DstReg = ExtMI->getOperand(0).getReg();
4266 LLT ExtTy = MRI.getType(DstReg);
4267
4268 // Propagate the extension into the block of each incoming reg's block.
4269 // Use a SetVector here because PHIs can have duplicate edges, and we want
4270 // deterministic iteration order.
4273 for (unsigned I = 0; I < PHI.getNumIncomingValues(); ++I) {
4274 auto SrcReg = PHI.getIncomingValue(I);
4275 MachineInstr *SrcMI;
4276 if (!mi_match(SrcReg, MRI, m_MInstr(SrcMI)))
4277 continue;
4278 if (!SrcMIs.insert(SrcMI))
4279 continue;
4280
4281 // Build an extend after each src inst.
4282 auto *MBB = SrcMI->getParent();
4283 MachineBasicBlock::iterator InsertPt = ++SrcMI->getIterator();
4284 if (InsertPt != MBB->end() && InsertPt->isPHI())
4285 InsertPt = MBB->getFirstNonPHI();
4286
4287 Builder.setInsertPt(*SrcMI->getParent(), InsertPt);
4288 Builder.setDebugLoc(MI.getDebugLoc());
4289 auto NewExt = Builder.buildExtOrTrunc(ExtMI->getOpcode(), ExtTy, SrcReg);
4290 OldToNewSrcMap[SrcMI] = NewExt;
4291 }
4292
4293 // Create a new phi with the extended inputs.
4294 Builder.setInstrAndDebugLoc(MI);
4295 auto NewPhi = Builder.buildInstrNoInsert(TargetOpcode::G_PHI);
4296 NewPhi.addDef(DstReg);
4297 for (const MachineOperand &MO : llvm::drop_begin(MI.operands())) {
4298 if (!MO.isReg()) {
4299 NewPhi.addMBB(MO.getMBB());
4300 continue;
4301 }
4302 auto *NewSrc = OldToNewSrcMap[MRI.getVRegDef(MO.getReg())];
4303 NewPhi.addUse(NewSrc->getOperand(0).getReg());
4304 }
4305 Builder.insertInstr(NewPhi);
4306 ExtMI->eraseFromParent();
4307}
4308
4310 Register &Reg) const {
4311 assert(MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT);
4312 // If we have a constant index, look for a G_BUILD_VECTOR source
4313 // and find the source register that the index maps to.
4314 Register SrcVec = MI.getOperand(1).getReg();
4315 LLT SrcTy = MRI.getType(SrcVec);
4316 if (SrcTy.isScalableVector())
4317 return false;
4318
4319 auto Cst = getIConstantVRegValWithLookThrough(MI.getOperand(2).getReg(), MRI);
4320 if (!Cst || Cst->Value.getZExtValue() >= SrcTy.getNumElements())
4321 return false;
4322
4323 unsigned VecIdx = Cst->Value.getZExtValue();
4324
4325 // Check if we have a build_vector or build_vector_trunc with an optional
4326 // trunc in front.
4327 MachineInstr *SrcVecMI;
4328 Register TruncSrc;
4329 if (mi_match(SrcVec, MRI, m_GTrunc(m_Reg(TruncSrc)))) {
4330 if (!mi_match(TruncSrc, MRI, m_MInstr(SrcVecMI)))
4331 return false;
4332 } else if (!mi_match(SrcVec, MRI, m_MInstr(SrcVecMI)))
4333 return false;
4334
4335 if (SrcVecMI->getOpcode() != TargetOpcode::G_BUILD_VECTOR &&
4336 SrcVecMI->getOpcode() != TargetOpcode::G_BUILD_VECTOR_TRUNC)
4337 return false;
4338
4339 EVT Ty(getMVTForLLT(SrcTy));
4340 if (!MRI.hasOneNonDBGUse(SrcVec) &&
4341 !getTargetLowering().aggressivelyPreferBuildVectorSources(Ty))
4342 return false;
4343
4344 Reg = SrcVecMI->getOperand(VecIdx + 1).getReg();
4345 return true;
4346}
4347
4349 Register &Reg) const {
4350 // Check the type of the register, since it may have come from a
4351 // G_BUILD_VECTOR_TRUNC.
4352 LLT ScalarTy = MRI.getType(Reg);
4353 Register DstReg = MI.getOperand(0).getReg();
4354 LLT DstTy = MRI.getType(DstReg);
4355
4356 if (ScalarTy != DstTy) {
4357 assert(ScalarTy.getSizeInBits() > DstTy.getSizeInBits());
4358 Builder.buildTrunc(DstReg, Reg);
4359 MI.eraseFromParent();
4360 return;
4361 }
4363}
4364
4367 SmallVectorImpl<std::pair<Register, MachineInstr *>> &SrcDstPairs) const {
4368 assert(MI.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
4369 // This combine tries to find build_vector's which have every source element
4370 // extracted using G_EXTRACT_VECTOR_ELT. This can happen when transforms like
4371 // the masked load scalarization is run late in the pipeline. There's already
4372 // a combine for a similar pattern starting from the extract, but that
4373 // doesn't attempt to do it if there are multiple uses of the build_vector,
4374 // which in this case is true. Starting the combine from the build_vector
4375 // feels more natural than trying to find sibling nodes of extracts.
4376 // E.g.
4377 // %vec(<4 x s32>) = G_BUILD_VECTOR %s1(s32), %s2, %s3, %s4
4378 // %ext1 = G_EXTRACT_VECTOR_ELT %vec, 0
4379 // %ext2 = G_EXTRACT_VECTOR_ELT %vec, 1
4380 // %ext3 = G_EXTRACT_VECTOR_ELT %vec, 2
4381 // %ext4 = G_EXTRACT_VECTOR_ELT %vec, 3
4382 // ==>
4383 // replace ext{1,2,3,4} with %s{1,2,3,4}
4384
4385 Register DstReg = MI.getOperand(0).getReg();
4386 LLT DstTy = MRI.getType(DstReg);
4387 unsigned NumElts = DstTy.getNumElements();
4388
4389 SmallBitVector ExtractedElts(NumElts);
4390 for (MachineInstr &II : MRI.use_nodbg_instructions(DstReg)) {
4391 if (II.getOpcode() != TargetOpcode::G_EXTRACT_VECTOR_ELT)
4392 return false;
4393 auto Cst = getIConstantVRegVal(II.getOperand(2).getReg(), MRI);
4394 if (!Cst)
4395 return false;
4396 unsigned Idx = Cst->getZExtValue();
4397 if (Idx >= NumElts)
4398 return false; // Out of range.
4399 ExtractedElts.set(Idx);
4400 SrcDstPairs.emplace_back(
4401 std::make_pair(MI.getOperand(Idx + 1).getReg(), &II));
4402 }
4403 // Match if every element was extracted.
4404 return ExtractedElts.all();
4405}
4406
4409 SmallVectorImpl<std::pair<Register, MachineInstr *>> &SrcDstPairs) const {
4410 assert(MI.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
4411 for (auto &Pair : SrcDstPairs) {
4412 auto *ExtMI = Pair.second;
4413 replaceRegWith(MRI, ExtMI->getOperand(0).getReg(), Pair.first);
4414 ExtMI->eraseFromParent();
4415 }
4416 MI.eraseFromParent();
4417}
4418
4421 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
4422 applyBuildFnNoErase(MI, MatchInfo);
4423 MI.eraseFromParent();
4424}
4425
4428 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
4429 MatchInfo(Builder);
4430}
4431
4433 bool AllowScalarConstants,
4434 BuildFnTy &MatchInfo) const {
4435 assert(MI.getOpcode() == TargetOpcode::G_OR);
4436
4437 Register Dst = MI.getOperand(0).getReg();
4438 LLT Ty = MRI.getType(Dst);
4439 unsigned BitWidth = Ty.getScalarSizeInBits();
4440
4441 Register ShlSrc, ShlAmt, LShrSrc, LShrAmt, Amt;
4442 unsigned FshOpc = 0;
4443
4444 // Match (or (shl ...), (lshr ...)).
4445 if (!mi_match(Dst, MRI,
4446 // m_GOr() handles the commuted version as well.
4447 m_GOr(m_GShl(m_Reg(ShlSrc), m_Reg(ShlAmt)),
4448 m_GLShr(m_Reg(LShrSrc), m_Reg(LShrAmt)))))
4449 return false;
4450
4451 // Given constants C0 and C1 such that C0 + C1 is bit-width:
4452 // (or (shl x, C0), (lshr y, C1)) -> (fshl x, y, C0) or (fshr x, y, C1)
4453 int64_t CstShlAmt = 0, CstLShrAmt;
4454 if (mi_match(ShlAmt, MRI, m_ICstOrSplat(CstShlAmt)) &&
4455 mi_match(LShrAmt, MRI, m_ICstOrSplat(CstLShrAmt)) &&
4456 CstShlAmt + CstLShrAmt == BitWidth) {
4457 FshOpc = TargetOpcode::G_FSHR;
4458 Amt = LShrAmt;
4459 } else if (mi_match(LShrAmt, MRI,
4461 ShlAmt == Amt) {
4462 // (or (shl x, amt), (lshr y, (sub bw, amt))) -> (fshl x, y, amt)
4463 FshOpc = TargetOpcode::G_FSHL;
4464 } else if (mi_match(ShlAmt, MRI,
4466 LShrAmt == Amt) {
4467 // (or (shl x, (sub bw, amt)), (lshr y, amt)) -> (fshr x, y, amt)
4468 FshOpc = TargetOpcode::G_FSHR;
4469 } else {
4470 return false;
4471 }
4472
4473 LLT AmtTy = MRI.getType(Amt);
4474 if (!isLegalOrBeforeLegalizer({FshOpc, {Ty, AmtTy}}) &&
4475 (!AllowScalarConstants || CstShlAmt == 0 || !Ty.isScalar()))
4476 return false;
4477
4478 MatchInfo = [=](MachineIRBuilder &B) {
4479 B.buildInstr(FshOpc, {Dst}, {ShlSrc, LShrSrc, Amt});
4480 };
4481 return true;
4482}
4483
4484// Fold (rot x, c) -> (rot x, c % BitSize)
4486 assert(MI.getOpcode() == TargetOpcode::G_ROTL ||
4487 MI.getOpcode() == TargetOpcode::G_ROTR);
4488 unsigned Bitsize =
4489 MRI.getType(MI.getOperand(0).getReg()).getScalarSizeInBits();
4490 Register AmtReg = MI.getOperand(2).getReg();
4491 bool OutOfRange = false;
4492 auto MatchOutOfRange = [Bitsize, &OutOfRange](const Constant *C) {
4493 if (auto *CI = dyn_cast<ConstantInt>(C))
4494 OutOfRange |= CI->getValue().uge(Bitsize);
4495 return true;
4496 };
4497 return matchUnaryPredicate(MRI, AmtReg, MatchOutOfRange) && OutOfRange;
4498}
4499
4501 assert(MI.getOpcode() == TargetOpcode::G_ROTL ||
4502 MI.getOpcode() == TargetOpcode::G_ROTR);
4503 unsigned Bitsize =
4504 MRI.getType(MI.getOperand(0).getReg()).getScalarSizeInBits();
4505 Register Amt = MI.getOperand(2).getReg();
4506 LLT AmtTy = MRI.getType(Amt);
4507 auto Bits = Builder.buildConstant(AmtTy, Bitsize);
4508 Amt = Builder.buildURem(AmtTy, MI.getOperand(2).getReg(), Bits).getReg(0);
4509 Observer.changingInstr(MI);
4510 MI.getOperand(2).setReg(Amt);
4511 Observer.changedInstr(MI);
4512}
4513
4515 int64_t &MatchInfo) const {
4516 assert(MI.getOpcode() == TargetOpcode::G_ICMP);
4517 auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
4518
4519 // We want to avoid calling KnownBits on the LHS if possible, as this combine
4520 // has no filter and runs on every G_ICMP instruction. We can avoid calling
4521 // KnownBits on the LHS in two cases:
4522 //
4523 // - The RHS is unknown: Constants are always on RHS. If the RHS is unknown
4524 // we cannot do any transforms so we can safely bail out early.
4525 // - The RHS is zero: we don't need to know the LHS to do unsigned <0 and
4526 // >=0.
4527 auto KnownRHS = VT->getKnownBits(MI.getOperand(3).getReg());
4528 if (KnownRHS.isUnknown())
4529 return false;
4530
4531 std::optional<bool> KnownVal;
4532 if (KnownRHS.isZero()) {
4533 // ? uge 0 -> always true
4534 // ? ult 0 -> always false
4535 if (Pred == CmpInst::ICMP_UGE)
4536 KnownVal = true;
4537 else if (Pred == CmpInst::ICMP_ULT)
4538 KnownVal = false;
4539 }
4540
4541 if (!KnownVal) {
4542 auto KnownLHS = VT->getKnownBits(MI.getOperand(2).getReg());
4543 KnownVal = ICmpInst::compare(KnownLHS, KnownRHS, Pred);
4544 }
4545
4546 if (!KnownVal)
4547 return false;
4548 MatchInfo =
4549 *KnownVal
4551 /*IsVector = */
4552 MRI.getType(MI.getOperand(0).getReg()).isVector(),
4553 /* IsFP = */ false)
4554 : 0;
4555 return true;
4556}
4557
4560 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
4561 assert(MI.getOpcode() == TargetOpcode::G_ICMP);
4562 // Given:
4563 //
4564 // %x = G_WHATEVER (... x is known to be 0 or 1 ...)
4565 // %cmp = G_ICMP ne %x, 0
4566 //
4567 // Or:
4568 //
4569 // %x = G_WHATEVER (... x is known to be 0 or 1 ...)
4570 // %cmp = G_ICMP eq %x, 1
4571 //
4572 // We can replace %cmp with %x assuming true is 1 on the target.
4573 auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
4574 if (!CmpInst::isEquality(Pred))
4575 return false;
4576 Register Dst = MI.getOperand(0).getReg();
4577 LLT DstTy = MRI.getType(Dst);
4579 /* IsFP = */ false) != 1)
4580 return false;
4581 int64_t OneOrZero = Pred == CmpInst::ICMP_EQ;
4582 if (!mi_match(MI.getOperand(3).getReg(), MRI, m_SpecificICst(OneOrZero)))
4583 return false;
4584 Register LHS = MI.getOperand(2).getReg();
4585 auto KnownLHS = VT->getKnownBits(LHS);
4586 if (KnownLHS.getMinValue() != 0 || KnownLHS.getMaxValue() != 1)
4587 return false;
4588 // Make sure replacing Dst with the LHS is a legal operation.
4589 LLT LHSTy = MRI.getType(LHS);
4590 unsigned LHSSize = LHSTy.getSizeInBits();
4591 unsigned DstSize = DstTy.getSizeInBits();
4592 unsigned Op = TargetOpcode::COPY;
4593 if (DstSize != LHSSize)
4594 Op = DstSize < LHSSize ? TargetOpcode::G_TRUNC : TargetOpcode::G_ZEXT;
4595 if (!isLegalOrBeforeLegalizer({Op, {DstTy, LHSTy}}))
4596 return false;
4597 MatchInfo = [=](MachineIRBuilder &B) { B.buildInstr(Op, {Dst}, {LHS}); };
4598 return true;
4599}
4600
4601// Replace (and (or x, c1), c2) with (and x, c2) iff c1 & c2 == 0
4604 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
4605 assert(MI.getOpcode() == TargetOpcode::G_AND);
4606
4607 // Ignore vector types to simplify matching the two constants.
4608 // TODO: do this for vectors and scalars via a demanded bits analysis.
4609 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
4610 if (Ty.isVector())
4611 return false;
4612
4613 Register Src;
4614 Register AndMaskReg;
4615 int64_t AndMaskBits;
4616 int64_t OrMaskBits;
4617 if (!mi_match(MI, MRI,
4618 m_GAnd(m_GOr(m_Reg(Src), m_ICst(OrMaskBits)),
4619 m_all_of(m_ICst(AndMaskBits), m_Reg(AndMaskReg)))))
4620 return false;
4621
4622 // Check if OrMask could turn on any bits in Src.
4623 if (AndMaskBits & OrMaskBits)
4624 return false;
4625
4626 MatchInfo = [=, &MI](MachineIRBuilder &B) {
4627 Observer.changingInstr(MI);
4628 // Canonicalize the result to have the constant on the RHS.
4629 if (MI.getOperand(1).getReg() == AndMaskReg)
4630 MI.getOperand(2).setReg(AndMaskReg);
4631 MI.getOperand(1).setReg(Src);
4632 Observer.changedInstr(MI);
4633 };
4634 return true;
4635}
4636
4637/// Form a G_SBFX from a G_SEXT_INREG fed by a right shift.
4640 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
4641 assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
4642 Register Dst = MI.getOperand(0).getReg();
4643 Register Src = MI.getOperand(1).getReg();
4644 LLT Ty = MRI.getType(Src);
4646 if (!LI || !LI->isLegalOrCustom({TargetOpcode::G_SBFX, {Ty, ExtractTy}}))
4647 return false;
4648 int64_t Width = MI.getOperand(2).getImm();
4649 Register ShiftSrc;
4650 int64_t ShiftImm;
4651 if (!mi_match(Src, MRI,
4652 m_OneUse(m_any_of(m_GAShr(m_Reg(ShiftSrc), m_ICst(ShiftImm)),
4653 m_GLShr(m_Reg(ShiftSrc), m_ICst(ShiftImm))))))
4654 return false;
4655 if (ShiftImm < 0 || ShiftImm + Width > Ty.getScalarSizeInBits())
4656 return false;
4657
4658 MatchInfo = [=](MachineIRBuilder &B) {
4659 auto Cst1 = B.buildConstant(ExtractTy, ShiftImm);
4660 auto Cst2 = B.buildConstant(ExtractTy, Width);
4661 B.buildSbfx(Dst, ShiftSrc, Cst1, Cst2);
4662 };
4663 return true;
4664}
4665
4666/// Form a G_UBFX from "(a srl b) & mask", where b and mask are constants.
4668 BuildFnTy &MatchInfo) const {
4669 GAnd *And = cast<GAnd>(&MI);
4670 Register Dst = And->getReg(0);
4671 LLT Ty = MRI.getType(Dst);
4673 // Note that isLegalOrBeforeLegalizer is stricter and does not take custom
4674 // into account.
4675 if (LI && !LI->isLegalOrCustom({TargetOpcode::G_UBFX, {Ty, ExtractTy}}))
4676 return false;
4677
4678 int64_t AndImm, LSBImm;
4679 Register ShiftSrc;
4680 const unsigned Size = Ty.getScalarSizeInBits();
4681 if (!mi_match(And->getReg(0), MRI,
4682 m_GAnd(m_OneUse(m_GLShr(m_Reg(ShiftSrc), m_ICst(LSBImm))),
4683 m_ICst(AndImm))))
4684 return false;
4685
4686 // AndImm is sign-extended to 64 bits by m_ICst; restrict it to the operand
4687 // width so an all-ones mask (a redundant AND) is not misread as a wider mask.
4688 uint64_t MaybeMask = static_cast<uint64_t>(AndImm);
4689 if (Size < 64)
4690 MaybeMask &= maskTrailingOnes<uint64_t>(Size);
4691
4692 // The mask is a mask of the low bits iff imm & (imm+1) == 0.
4693 if (MaybeMask & (MaybeMask + 1))
4694 return false;
4695
4696 // LSB must fit within the register.
4697 if (static_cast<uint64_t>(LSBImm) >= Size)
4698 return false;
4699
4700 uint64_t Width = APInt(Size, MaybeMask).countr_one();
4701 // The extracted field [LSB, LSB+Width) must fit within the register.
4702 // Otherwise this is a redundant AND (e.g. an all-ones mask combined with a
4703 // non-zero shift) that is better handled by other combines, and would form
4704 // an out-of-range bitfield extract.
4705 if (static_cast<uint64_t>(LSBImm) + Width > Size)
4706 return false;
4707
4708 MatchInfo = [=](MachineIRBuilder &B) {
4709 auto WidthCst = B.buildConstant(ExtractTy, Width);
4710 auto LSBCst = B.buildConstant(ExtractTy, LSBImm);
4711 B.buildInstr(TargetOpcode::G_UBFX, {Dst}, {ShiftSrc, LSBCst, WidthCst});
4712 };
4713 return true;
4714}
4715
4718 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
4719 const unsigned Opcode = MI.getOpcode();
4720 assert(Opcode == TargetOpcode::G_ASHR || Opcode == TargetOpcode::G_LSHR);
4721
4722 const Register Dst = MI.getOperand(0).getReg();
4723
4724 const unsigned ExtrOpcode = Opcode == TargetOpcode::G_ASHR
4725 ? TargetOpcode::G_SBFX
4726 : TargetOpcode::G_UBFX;
4727
4728 // Check if the type we would use for the extract is legal
4729 LLT Ty = MRI.getType(Dst);
4731 if (!LI || !LI->isLegalOrCustom({ExtrOpcode, {Ty, ExtractTy}}))
4732 return false;
4733
4734 Register ShlSrc;
4735 int64_t ShrAmt;
4736 int64_t ShlAmt;
4737 const unsigned Size = Ty.getScalarSizeInBits();
4738
4739 // Try to match shr (shl x, c1), c2
4740 if (!mi_match(Dst, MRI,
4741 m_BinOp(Opcode, m_OneUse(m_GShl(m_Reg(ShlSrc), m_ICst(ShlAmt))),
4742 m_ICst(ShrAmt))))
4743 return false;
4744
4745 // Make sure that the shift sizes can fit a bitfield extract
4746 if (ShlAmt < 0 || ShlAmt > ShrAmt || ShrAmt >= Size)
4747 return false;
4748
4749 // Skip this combine if the G_SEXT_INREG combine could handle it
4750 if (Opcode == TargetOpcode::G_ASHR && ShlAmt == ShrAmt)
4751 return false;
4752
4753 // Calculate start position and width of the extract
4754 const int64_t Pos = ShrAmt - ShlAmt;
4755 const int64_t Width = Size - ShrAmt;
4756
4757 MatchInfo = [=](MachineIRBuilder &B) {
4758 auto WidthCst = B.buildConstant(ExtractTy, Width);
4759 auto PosCst = B.buildConstant(ExtractTy, Pos);
4760 B.buildInstr(ExtrOpcode, {Dst}, {ShlSrc, PosCst, WidthCst});
4761 };
4762 return true;
4763}
4764
4767 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
4768 const unsigned Opcode = MI.getOpcode();
4769 assert(Opcode == TargetOpcode::G_LSHR || Opcode == TargetOpcode::G_ASHR);
4770
4771 const Register Dst = MI.getOperand(0).getReg();
4772 LLT Ty = MRI.getType(Dst);
4774 if (LI && !LI->isLegalOrCustom({TargetOpcode::G_UBFX, {Ty, ExtractTy}}))
4775 return false;
4776
4777 // Try to match shr (and x, c1), c2
4778 Register AndSrc;
4779 int64_t ShrAmt;
4780 int64_t SMask;
4781 if (!mi_match(Dst, MRI,
4782 m_BinOp(Opcode, m_OneUse(m_GAnd(m_Reg(AndSrc), m_ICst(SMask))),
4783 m_ICst(ShrAmt))))
4784 return false;
4785
4786 const unsigned Size = Ty.getScalarSizeInBits();
4787 if (ShrAmt < 0 || ShrAmt >= Size)
4788 return false;
4789
4790 // If the shift subsumes the mask, emit the 0 directly.
4791 if (0 == (SMask >> ShrAmt)) {
4792 MatchInfo = [=](MachineIRBuilder &B) {
4793 B.buildConstant(Dst, 0);
4794 };
4795 return true;
4796 }
4797
4798 // Check that ubfx can do the extraction, with no holes in the mask.
4799 uint64_t UMask = SMask;
4800 UMask |= maskTrailingOnes<uint64_t>(ShrAmt);
4802 if (!isMask_64(UMask))
4803 return false;
4804
4805 // Calculate start position and width of the extract.
4806 const int64_t Pos = ShrAmt;
4807 const int64_t Width = llvm::countr_one(UMask) - ShrAmt;
4808
4809 // It's preferable to keep the shift, rather than form G_SBFX.
4810 // TODO: remove the G_AND via demanded bits analysis.
4811 if (Opcode == TargetOpcode::G_ASHR && Width + ShrAmt == Size)
4812 return false;
4813
4814 MatchInfo = [=](MachineIRBuilder &B) {
4815 auto WidthCst = B.buildConstant(ExtractTy, Width);
4816 auto PosCst = B.buildConstant(ExtractTy, Pos);
4817 B.buildInstr(TargetOpcode::G_UBFX, {Dst}, {AndSrc, PosCst, WidthCst});
4818 };
4819 return true;
4820}
4821
4822bool CombinerHelper::reassociationCanBreakAddressingModePattern(
4823 MachineInstr &MI) const {
4824 auto &PtrAdd = cast<GPtrAdd>(MI);
4825
4826 Register Src1Reg = PtrAdd.getBaseReg();
4827 auto *Src1Def = getOpcodeDef<GPtrAdd>(Src1Reg, MRI);
4828 if (!Src1Def)
4829 return false;
4830
4831 Register Src2Reg = PtrAdd.getOffsetReg();
4832
4833 if (MRI.hasOneNonDBGUse(Src1Reg))
4834 return false;
4835
4836 auto C1 = getIConstantVRegVal(Src1Def->getOffsetReg(), MRI);
4837 if (!C1)
4838 return false;
4839 auto C2 = getIConstantVRegVal(Src2Reg, MRI);
4840 if (!C2)
4841 return false;
4842
4843 const APInt &C1APIntVal = *C1;
4844 const APInt &C2APIntVal = *C2;
4845 const int64_t CombinedValue = (C1APIntVal + C2APIntVal).getSExtValue();
4846
4847 for (auto &UseMI : MRI.use_nodbg_instructions(PtrAdd.getReg(0))) {
4848 // This combine may end up running before ptrtoint/inttoptr combines
4849 // manage to eliminate redundant conversions, so try to look through them.
4850 MachineInstr *ConvUseMI = &UseMI;
4851 unsigned ConvUseOpc = ConvUseMI->getOpcode();
4852 while (ConvUseOpc == TargetOpcode::G_INTTOPTR ||
4853 ConvUseOpc == TargetOpcode::G_PTRTOINT) {
4854 Register DefReg = ConvUseMI->getOperand(0).getReg();
4855 if (!MRI.hasOneNonDBGUse(DefReg))
4856 break;
4857 ConvUseMI = &*MRI.use_instr_nodbg_begin(DefReg);
4858 ConvUseOpc = ConvUseMI->getOpcode();
4859 }
4860 auto *LdStMI = dyn_cast<GLoadStore>(ConvUseMI);
4861 if (!LdStMI)
4862 continue;
4863 // Is x[offset2] already not a legal addressing mode? If so then
4864 // reassociating the constants breaks nothing (we test offset2 because
4865 // that's the one we hope to fold into the load or store).
4866 TargetLoweringBase::AddrMode AM;
4867 AM.HasBaseReg = true;
4868 AM.BaseOffs = C2APIntVal.getSExtValue();
4869 unsigned AS = MRI.getType(LdStMI->getPointerReg()).getAddressSpace();
4870 Type *AccessTy = getTypeForLLT(LdStMI->getMMO().getMemoryType(),
4871 PtrAdd.getMF()->getFunction().getContext());
4872 const auto &TLI = *PtrAdd.getMF()->getSubtarget().getTargetLowering();
4873 if (!TLI.isLegalAddressingMode(PtrAdd.getMF()->getDataLayout(), AM,
4874 AccessTy, AS))
4875 continue;
4876
4877 // Would x[offset1+offset2] still be a legal addressing mode?
4878 AM.BaseOffs = CombinedValue;
4879 if (!TLI.isLegalAddressingMode(PtrAdd.getMF()->getDataLayout(), AM,
4880 AccessTy, AS))
4881 return true;
4882 }
4883
4884 return false;
4885}
4886
4888 MachineInstr *RHS,
4889 BuildFnTy &MatchInfo) const {
4890 // G_PTR_ADD(BASE, G_ADD(X, C)) -> G_PTR_ADD(G_PTR_ADD(BASE, X), C)
4891 Register Src1Reg = MI.getOperand(1).getReg();
4892 if (RHS->getOpcode() != TargetOpcode::G_ADD)
4893 return false;
4894 auto C2 = getIConstantVRegVal(RHS->getOperand(2).getReg(), MRI);
4895 if (!C2)
4896 return false;
4897
4898 // If both additions are nuw, the reassociated additions are also nuw.
4899 // If the original G_PTR_ADD is additionally nusw, X and C are both not
4900 // negative, so BASE+X is between BASE and BASE+(X+C). The new G_PTR_ADDs are
4901 // therefore also nusw.
4902 // If the original G_PTR_ADD is additionally inbounds (which implies nusw),
4903 // the new G_PTR_ADDs are then also inbounds.
4904 unsigned PtrAddFlags = MI.getFlags();
4905 unsigned AddFlags = RHS->getFlags();
4906 bool IsNoUWrap = PtrAddFlags & AddFlags & MachineInstr::MIFlag::NoUWrap;
4907 bool IsNoUSWrap = IsNoUWrap && (PtrAddFlags & MachineInstr::MIFlag::NoUSWrap);
4908 bool IsInBounds = IsNoUWrap && (PtrAddFlags & MachineInstr::MIFlag::InBounds);
4909 unsigned Flags = 0;
4910 if (IsNoUWrap)
4912 if (IsNoUSWrap)
4914 if (IsInBounds)
4916
4917 MatchInfo = [=, &MI](MachineIRBuilder &B) {
4918 LLT PtrTy = MRI.getType(MI.getOperand(0).getReg());
4919
4920 auto NewBase =
4921 Builder.buildPtrAdd(PtrTy, Src1Reg, RHS->getOperand(1).getReg(), Flags);
4922 Observer.changingInstr(MI);
4923 MI.getOperand(1).setReg(NewBase.getReg(0));
4924 MI.getOperand(2).setReg(RHS->getOperand(2).getReg());
4925 MI.setFlags(Flags);
4926 Observer.changedInstr(MI);
4927 };
4928 return !reassociationCanBreakAddressingModePattern(MI);
4929}
4930
4932 MachineInstr *LHS,
4933 MachineInstr *RHS,
4934 BuildFnTy &MatchInfo) const {
4935 // G_PTR_ADD (G_PTR_ADD X, C), Y) -> (G_PTR_ADD (G_PTR_ADD(X, Y), C)
4936 // if and only if (G_PTR_ADD X, C) has one use.
4937 Register LHSBase;
4938 std::optional<ValueAndVReg> LHSCstOff;
4939 if (!mi_match(MI.getBaseReg(), MRI,
4940 m_OneUse(m_GPtrAdd(m_Reg(LHSBase), m_GCst(LHSCstOff)))))
4941 return false;
4942
4943 auto *LHSPtrAdd = cast<GPtrAdd>(LHS);
4944
4945 // Reassociating nuw additions preserves nuw. If both original G_PTR_ADDs are
4946 // nuw and inbounds (which implies nusw), the offsets are both non-negative,
4947 // so the new G_PTR_ADDs are also inbounds.
4948 unsigned PtrAddFlags = MI.getFlags();
4949 unsigned LHSPtrAddFlags = LHSPtrAdd->getFlags();
4950 bool IsNoUWrap = PtrAddFlags & LHSPtrAddFlags & MachineInstr::MIFlag::NoUWrap;
4951 bool IsNoUSWrap = IsNoUWrap && (PtrAddFlags & LHSPtrAddFlags &
4953 bool IsInBounds = IsNoUWrap && (PtrAddFlags & LHSPtrAddFlags &
4955 unsigned Flags = 0;
4956 if (IsNoUWrap)
4958 if (IsNoUSWrap)
4960 if (IsInBounds)
4962
4963 MatchInfo = [=, &MI](MachineIRBuilder &B) {
4964 // When we change LHSPtrAdd's offset register we might cause it to use a reg
4965 // before its def. Sink the instruction so the outer PTR_ADD to ensure this
4966 // doesn't happen.
4967 LHSPtrAdd->moveBefore(&MI);
4968 Register RHSReg = MI.getOffsetReg();
4969 // set VReg will cause type mismatch if it comes from extend/trunc
4970 auto NewCst = B.buildConstant(MRI.getType(RHSReg), LHSCstOff->Value);
4971 Observer.changingInstr(MI);
4972 MI.getOperand(2).setReg(NewCst.getReg(0));
4973 MI.setFlags(Flags);
4974 Observer.changedInstr(MI);
4975 Observer.changingInstr(*LHSPtrAdd);
4976 LHSPtrAdd->getOperand(2).setReg(RHSReg);
4977 LHSPtrAdd->setFlags(Flags);
4978 Observer.changedInstr(*LHSPtrAdd);
4979 };
4980 return !reassociationCanBreakAddressingModePattern(MI);
4981}
4982
4984 GPtrAdd &MI, MachineInstr *LHS, MachineInstr *RHS,
4985 BuildFnTy &MatchInfo) const {
4986 // G_PTR_ADD(G_PTR_ADD(BASE, C1), C2) -> G_PTR_ADD(BASE, C1+C2)
4987 auto *LHSPtrAdd = dyn_cast<GPtrAdd>(LHS);
4988 if (!LHSPtrAdd)
4989 return false;
4990
4991 Register Src2Reg = MI.getOperand(2).getReg();
4992 Register LHSSrc1 = LHSPtrAdd->getBaseReg();
4993 Register LHSSrc2 = LHSPtrAdd->getOffsetReg();
4994 auto C1 = getIConstantVRegVal(LHSSrc2, MRI);
4995 if (!C1)
4996 return false;
4997 auto C2 = getIConstantVRegVal(Src2Reg, MRI);
4998 if (!C2)
4999 return false;
5000
5001 // Reassociating nuw additions preserves nuw. If both original G_PTR_ADDs are
5002 // inbounds, reaching the same result in one G_PTR_ADD is also inbounds.
5003 // The nusw constraints are satisfied because imm1+imm2 cannot exceed the
5004 // largest signed integer that fits into the index type, which is the maximum
5005 // size of allocated objects according to the IR Language Reference.
5006 unsigned PtrAddFlags = MI.getFlags();
5007 unsigned LHSPtrAddFlags = LHSPtrAdd->getFlags();
5008 bool IsNoUWrap = PtrAddFlags & LHSPtrAddFlags & MachineInstr::MIFlag::NoUWrap;
5009 bool IsInBounds =
5010 PtrAddFlags & LHSPtrAddFlags & MachineInstr::MIFlag::InBounds;
5011 unsigned Flags = 0;
5012 if (IsNoUWrap)
5014 if (IsInBounds) {
5017 }
5018
5019 MatchInfo = [=, &MI](MachineIRBuilder &B) {
5020 auto NewCst = B.buildConstant(MRI.getType(Src2Reg), *C1 + *C2);
5021 Observer.changingInstr(MI);
5022 MI.getOperand(1).setReg(LHSSrc1);
5023 MI.getOperand(2).setReg(NewCst.getReg(0));
5024 MI.setFlags(Flags);
5025 Observer.changedInstr(MI);
5026 };
5027 return !reassociationCanBreakAddressingModePattern(MI);
5028}
5029
5031 BuildFnTy &MatchInfo) const {
5032 auto &PtrAdd = cast<GPtrAdd>(MI);
5033 // We're trying to match a few pointer computation patterns here for
5034 // re-association opportunities.
5035 // 1) Isolating a constant operand to be on the RHS, e.g.:
5036 // G_PTR_ADD(BASE, G_ADD(X, C)) -> G_PTR_ADD(G_PTR_ADD(BASE, X), C)
5037 //
5038 // 2) Folding two constants in each sub-tree as long as such folding
5039 // doesn't break a legal addressing mode.
5040 // G_PTR_ADD(G_PTR_ADD(BASE, C1), C2) -> G_PTR_ADD(BASE, C1+C2)
5041 //
5042 // 3) Move a constant from the LHS of an inner op to the RHS of the outer.
5043 // G_PTR_ADD (G_PTR_ADD X, C), Y) -> G_PTR_ADD (G_PTR_ADD(X, Y), C)
5044 // iif (G_PTR_ADD X, C) has one use.
5045 MachineInstr *LHS, *RHS;
5046 if (!mi_match(PtrAdd.getBaseReg(), MRI, m_MInstr(LHS)) ||
5047 !mi_match(PtrAdd.getOffsetReg(), MRI, m_MInstr(RHS)))
5048 return false;
5049
5050 // Try to match example 2.
5051 if (matchReassocFoldConstantsInSubTree(PtrAdd, LHS, RHS, MatchInfo))
5052 return true;
5053
5054 // Try to match example 3.
5055 if (matchReassocConstantInnerLHS(PtrAdd, LHS, RHS, MatchInfo))
5056 return true;
5057
5058 // Try to match example 1.
5059 if (matchReassocConstantInnerRHS(PtrAdd, RHS, MatchInfo))
5060 return true;
5061
5062 return false;
5063}
5065 Register OpLHS, Register OpRHS,
5066 BuildFnTy &MatchInfo) const {
5067 LLT OpRHSTy = MRI.getType(OpRHS);
5068 MachineInstr *OpLHSDef;
5069 if (!mi_match(OpLHS, MRI, m_MInstr(OpLHSDef)) || OpLHSDef->getOpcode() != Opc)
5070 return false;
5071
5072 Register OpLHSLHS = OpLHSDef->getOperand(1).getReg();
5073 Register OpLHSRHS = OpLHSDef->getOperand(2).getReg();
5074
5075 // If the inner op is (X op C), pull the constant out so it can be folded with
5076 // other constants in the expression tree. Folding is not guaranteed so we
5077 // might have (C1 op C2). In that case do not pull a constant out because it
5078 // won't help and can lead to infinite loops.
5079 if (isConstantOrConstantSplatVector(OpLHSRHS, MRI) &&
5082 // (Opc (Opc X, C1), C2) -> (Opc X, (Opc C1, C2))
5083 MatchInfo = [=](MachineIRBuilder &B) {
5084 auto NewCst = B.buildInstr(Opc, {OpRHSTy}, {OpLHSRHS, OpRHS});
5085 B.buildInstr(Opc, {DstReg}, {OpLHSLHS, NewCst});
5086 };
5087 return true;
5088 }
5089 if (getTargetLowering().isReassocProfitable(MRI, OpLHS, OpRHS)) {
5090 // Reassociate: (op (op x, c1), y) -> (op (op x, y), c1)
5091 // iff (op x, c1) has one use
5092 MatchInfo = [=](MachineIRBuilder &B) {
5093 auto NewLHSLHS = B.buildInstr(Opc, {OpRHSTy}, {OpLHSLHS, OpRHS});
5094 B.buildInstr(Opc, {DstReg}, {NewLHSLHS, OpLHSRHS});
5095 };
5096 return true;
5097 }
5098 }
5099
5100 return false;
5101}
5102
5104 BuildFnTy &MatchInfo) const {
5105 // We don't check if the reassociation will break a legal addressing mode
5106 // here since pointer arithmetic is handled by G_PTR_ADD.
5107 unsigned Opc = MI.getOpcode();
5108 Register DstReg = MI.getOperand(0).getReg();
5109 Register LHSReg = MI.getOperand(1).getReg();
5110 Register RHSReg = MI.getOperand(2).getReg();
5111
5112 if (tryReassocBinOp(Opc, DstReg, LHSReg, RHSReg, MatchInfo))
5113 return true;
5114 if (tryReassocBinOp(Opc, DstReg, RHSReg, LHSReg, MatchInfo))
5115 return true;
5116 return false;
5117}
5118
5120 BuildFnTy &MatchInfo) const {
5121 Register Dst = MI.getOperand(0).getReg();
5122 Register SrcOp = MI.getOperand(1).getReg();
5123 LLT DstTy = MRI.getType(Dst);
5124 unsigned Opc = MI.getOpcode();
5125
5127 return false;
5128
5129 if (DstTy.isScalableVector())
5130 return false;
5131
5132 if (DstTy.isFixedVector()) {
5134 if (!BV)
5135 return false;
5136 SmallVector<APInt> Csts;
5137 for (unsigned I = 0, E = BV->getNumSources(); I != E; ++I) {
5138 auto MaybeCst = ConstantFoldCastOp(Opc, DstTy, BV->getSourceReg(I), MRI);
5139 if (!MaybeCst)
5140 return false;
5141 Csts.push_back(*MaybeCst);
5142 }
5143 MatchInfo = [Dst, Csts = std::move(Csts)](MachineIRBuilder &B) {
5144 B.buildBuildVectorConstant(Dst, Csts);
5145 };
5146 return true;
5147 }
5148
5149 if (auto MaybeCst = ConstantFoldCastOp(Opc, DstTy, SrcOp, MRI)) {
5150 MatchInfo = [Dst, Cst = *MaybeCst](MachineIRBuilder &B) {
5151 B.buildConstant(Dst, Cst);
5152 };
5153 return true;
5154 }
5155
5156 return false;
5157}
5158
5160 BuildFnTy &MatchInfo) const {
5161 Register Dst = MI.getOperand(0).getReg();
5162 LLT DstTy = MRI.getType(Dst);
5163
5165 MI.getOpcode(), DstTy, MI.getOperand(1).getReg(), MRI);
5166 if (Csts.empty())
5167 return false;
5168
5169 MatchInfo = [Dst, Csts = std::move(Csts)](MachineIRBuilder &B) {
5170 if (Csts.size() == 1)
5171 B.buildConstant(Dst, Csts[0]);
5172 else
5173 B.buildBuildVectorConstant(Dst, Csts);
5174 };
5175 return true;
5176}
5177
5179 BuildFnTy &MatchInfo) const {
5180 Register Dst = MI.getOperand(0).getReg();
5181 Register Op1 = MI.getOperand(1).getReg();
5182 Register Op2 = MI.getOperand(2).getReg();
5183 LLT DstTy = MRI.getType(Dst);
5184
5185 // Scalable and pointer-element vectors have no G_BUILD_VECTOR of
5186 // G_CONSTANT to fold to.
5187 if (DstTy.isScalableVector() ||
5188 (DstTy.isVector() && DstTy.getElementType().isPointer()))
5189 return false;
5190
5191 if (DstTy.isFixedVector()) {
5192 SmallVector<APInt> Csts =
5193 ConstantFoldVectorBinop(MI.getOpcode(), Op1, Op2, MRI);
5194 if (Csts.empty())
5195 return false;
5196 MatchInfo = [Dst, Csts = std::move(Csts)](MachineIRBuilder &B) {
5197 B.buildBuildVectorConstant(Dst, Csts);
5198 };
5199 return true;
5200 }
5201
5202 auto MaybeCst = ConstantFoldBinOp(MI.getOpcode(), Op1, Op2, MRI);
5203 if (!MaybeCst)
5204 return false;
5205 MatchInfo = [Dst, Cst = *MaybeCst](MachineIRBuilder &B) {
5206 B.buildConstant(Dst, Cst);
5207 };
5208 return true;
5209}
5210
5211// Vector FP constant folding assumes IEEE denormal arithmetic. Flush-to-zero
5212// and dynamic modes can produce a different value, so skip those folds.
5213// Targets such as RISC-V represent FP values as untyped scalars
5214// (s16/s32/s64/s128). getFltSemanticForLLT maps those onto the IEEE formats.
5215//
5216// TODO: Honor non-IEEE and dynamic denormal modes instead of skipping the
5217// vector folds. Scalar folds have the same limitation and stay unchanged here.
5219 LLT ScalarTy = Ty.getScalarType();
5220 if (!ScalarTy.isFloat() && !ScalarTy.isAnyScalar())
5221 return true;
5222 return MF.getDenormalMode(getFltSemanticForLLT(ScalarTy)) ==
5224}
5225
5227 BuildFnTy &MatchInfo) const {
5228 Register Dst = MI.getOperand(0).getReg();
5229 Register Op1 = MI.getOperand(1).getReg();
5230 Register Op2 = MI.getOperand(2).getReg();
5231 LLT DstTy = MRI.getType(Dst);
5232
5233 if (DstTy.isScalableVector())
5234 return false;
5235
5236 if (DstTy.isFixedVector()) {
5238 return false;
5240 ConstantFoldVectorFPBinop(MI.getOpcode(), Op1, Op2, MRI);
5241 if (Csts.empty())
5242 return false;
5243 MatchInfo = [Dst, Csts = std::move(Csts)](MachineIRBuilder &B) {
5244 B.buildBuildVectorFConstant(Dst, Csts);
5245 };
5246 return true;
5247 }
5248
5249 auto MaybeCst = ConstantFoldFPBinOp(MI.getOpcode(), Op1, Op2, MRI);
5250 if (!MaybeCst)
5251 return false;
5252 MatchInfo = [Dst, Cst = *MaybeCst](MachineIRBuilder &B) {
5253 B.buildFConstant(Dst, Cst);
5254 };
5255 return true;
5256}
5257
5259 BuildFnTy &MatchInfo) const {
5260 Register Dst = MI.getOperand(0).getReg();
5261 Register Src = MI.getOperand(1).getReg();
5262 LLT DstTy = MRI.getType(Dst);
5263 LLT SrcTy = MRI.getType(Src);
5264
5266 return false;
5267
5268 // Source and destination element types can differ (G_FPEXT/G_FPTRUNC) and
5269 // can have different denormal modes.
5270 if (SrcTy.isVector() &&
5273 return false;
5274
5276 ConstantFoldUnaryFPOp(MI.getOpcode(), DstTy, Src, MRI);
5277 if (Csts.empty())
5278 return false;
5279 MatchInfo = [Dst, Csts = std::move(Csts)](MachineIRBuilder &B) {
5280 if (Csts.size() == 1)
5281 B.buildFConstant(Dst, Csts[0]);
5282 else
5283 B.buildBuildVectorFConstant(Dst, Csts);
5284 };
5285 return true;
5286}
5287
5289 BuildFnTy &MatchInfo) const {
5290 assert(MI.getOpcode() == TargetOpcode::G_FMA ||
5291 MI.getOpcode() == TargetOpcode::G_FMAD);
5292 Register Dst = MI.getOperand(0).getReg();
5293 auto [_, Op1, Op2, Op3] = MI.getFirst4Regs();
5294 LLT DstTy = MRI.getType(Dst);
5295
5296 auto FoldFMA = [](const APFloat &A, const APFloat &B,
5297 const APFloat &C) -> APFloat {
5298 APFloat Res(A);
5300 return Res;
5301 };
5302
5303 if (DstTy.isScalableVector())
5304 return false;
5305
5306 if (DstTy.isFixedVector()) {
5308 return false;
5309 auto *BV1 = getOpcodeDef<GBuildVector>(Op1, MRI);
5310 auto *BV2 = getOpcodeDef<GBuildVector>(Op2, MRI);
5311 auto *BV3 = getOpcodeDef<GBuildVector>(Op3, MRI);
5312 if (!BV1 || !BV2 || !BV3)
5313 return false;
5314 unsigned NumElts = BV1->getNumSources();
5315 if (BV2->getNumSources() != NumElts || BV3->getNumSources() != NumElts)
5316 return false;
5318 for (unsigned I = 0; I != NumElts; ++I) {
5319 const ConstantFP *C1 = getConstantFPVRegVal(BV1->getSourceReg(I), MRI);
5320 const ConstantFP *C2 = getConstantFPVRegVal(BV2->getSourceReg(I), MRI);
5321 const ConstantFP *C3 = getConstantFPVRegVal(BV3->getSourceReg(I), MRI);
5322 if (!C1 || !C2 || !C3)
5323 return false;
5324 Csts.push_back(
5325 FoldFMA(C1->getValueAPF(), C2->getValueAPF(), C3->getValueAPF()));
5326 }
5327 MatchInfo = [Dst, Csts = std::move(Csts)](MachineIRBuilder &B) {
5328 B.buildBuildVectorFConstant(Dst, Csts);
5329 };
5330 return true;
5331 }
5332
5333 const ConstantFP *Op3Cst = getConstantFPVRegVal(Op3, MRI);
5334 const ConstantFP *Op2Cst = getConstantFPVRegVal(Op2, MRI);
5335 const ConstantFP *Op1Cst = getConstantFPVRegVal(Op1, MRI);
5336 if (!Op1Cst || !Op2Cst || !Op3Cst)
5337 return false;
5338
5339 APFloat Res = FoldFMA(Op1Cst->getValueAPF(), Op2Cst->getValueAPF(),
5340 Op3Cst->getValueAPF());
5341 MatchInfo = [Dst, Res](MachineIRBuilder &B) { B.buildFConstant(Dst, Res); };
5342 return true;
5343}
5344
5347 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
5348 // Look for a binop feeding into an AND with a mask:
5349 //
5350 // %add = G_ADD %lhs, %rhs
5351 // %and = G_AND %add, 000...11111111
5352 //
5353 // Check if it's possible to perform the binop at a narrower width and zext
5354 // back to the original width like so:
5355 //
5356 // %narrow_lhs = G_TRUNC %lhs
5357 // %narrow_rhs = G_TRUNC %rhs
5358 // %narrow_add = G_ADD %narrow_lhs, %narrow_rhs
5359 // %new_add = G_ZEXT %narrow_add
5360 // %and = G_AND %new_add, 000...11111111
5361 //
5362 // This can allow later combines to eliminate the G_AND if it turns out
5363 // that the mask is irrelevant.
5364 assert(MI.getOpcode() == TargetOpcode::G_AND);
5365 Register Dst = MI.getOperand(0).getReg();
5366 Register AndLHS = MI.getOperand(1).getReg();
5367 Register AndRHS = MI.getOperand(2).getReg();
5368 LLT WideTy = MRI.getType(Dst);
5369
5370 // If the potential binop has more than one use, then it's possible that one
5371 // of those uses will need its full width.
5372 if (!WideTy.isScalar() || !MRI.hasOneNonDBGUse(AndLHS))
5373 return false;
5374
5375 // Check if the LHS feeding the AND is impacted by the high bits that we're
5376 // masking out.
5377 //
5378 // e.g. for 64-bit x, y:
5379 //
5380 // add_64(x, y) & 65535 == zext(add_16(trunc(x), trunc(y))) & 65535
5381 MachineInstr *LHSInst = getDefIgnoringCopies(AndLHS, MRI);
5382 if (!LHSInst)
5383 return false;
5384 unsigned LHSOpc = LHSInst->getOpcode();
5385 switch (LHSOpc) {
5386 default:
5387 return false;
5388 case TargetOpcode::G_ADD:
5389 case TargetOpcode::G_SUB:
5390 case TargetOpcode::G_MUL:
5391 case TargetOpcode::G_AND:
5392 case TargetOpcode::G_OR:
5393 case TargetOpcode::G_XOR:
5394 break;
5395 }
5396
5397 // Find the mask on the RHS.
5398 auto Cst = getIConstantVRegValWithLookThrough(AndRHS, MRI);
5399 if (!Cst)
5400 return false;
5401 auto Mask = Cst->Value;
5402 if (!Mask.isMask())
5403 return false;
5404
5405 // No point in combining if there's nothing to truncate.
5406 unsigned NarrowWidth = Mask.countr_one();
5407 if (NarrowWidth == WideTy.getSizeInBits())
5408 return false;
5409 LLT NarrowTy = LLT::integer(NarrowWidth);
5410
5411 // Check if adding the zext + truncates could be harmful.
5412 auto &MF = *MI.getMF();
5413 const auto &TLI = getTargetLowering();
5414 LLVMContext &Ctx = MF.getFunction().getContext();
5415 if (!TLI.isTruncateFree(WideTy, NarrowTy, Ctx) ||
5416 !TLI.isZExtFree(NarrowTy, WideTy, Ctx))
5417 return false;
5418 if (!isLegalOrBeforeLegalizer({TargetOpcode::G_TRUNC, {NarrowTy, WideTy}}) ||
5419 !isLegalOrBeforeLegalizer({TargetOpcode::G_ZEXT, {WideTy, NarrowTy}}))
5420 return false;
5421 Register BinOpLHS = LHSInst->getOperand(1).getReg();
5422 Register BinOpRHS = LHSInst->getOperand(2).getReg();
5423 MatchInfo = [=, &MI](MachineIRBuilder &B) {
5424 auto NarrowLHS = Builder.buildTrunc(NarrowTy, BinOpLHS);
5425 auto NarrowRHS = Builder.buildTrunc(NarrowTy, BinOpRHS);
5426 auto NarrowBinOp =
5427 Builder.buildInstr(LHSOpc, {NarrowTy}, {NarrowLHS, NarrowRHS});
5428 auto Ext = Builder.buildZExt(WideTy, NarrowBinOp);
5429 Observer.changingInstr(MI);
5430 MI.getOperand(1).setReg(Ext.getReg(0));
5431 Observer.changedInstr(MI);
5432 };
5433 return true;
5434}
5435
5437 BuildFnTy &MatchInfo) const {
5438 // (G_*MULO x, 0) -> 0 + no carry out
5439 assert(MI.getOpcode() == TargetOpcode::G_UMULO ||
5440 MI.getOpcode() == TargetOpcode::G_SMULO);
5441 if (!mi_match(MI.getOperand(3).getReg(), MRI, m_SpecificICstOrSplat(0)))
5442 return false;
5443 Register Dst = MI.getOperand(0).getReg();
5444 Register Carry = MI.getOperand(1).getReg();
5445 if (!isConstantLegalOrBeforeLegalizer(MRI.getType(Dst)) ||
5446 !isConstantLegalOrBeforeLegalizer(MRI.getType(Carry)))
5447 return false;
5448 MatchInfo = [=](MachineIRBuilder &B) {
5449 B.buildConstant(Dst, 0);
5450 B.buildConstant(Carry, 0);
5451 };
5452 return true;
5453}
5454
5458 if (SrcA.isValid() && SrcA == SrcB)
5459 return true;
5460 int64_t Cst;
5461 return mi_match(A, MRI, m_ICstOrSplat(Cst)) &&
5463}
5464
5466 unsigned Opcode = MI.getOpcode();
5467 assert(Opcode == TargetOpcode::G_UDIV || Opcode == TargetOpcode::G_UREM);
5468 auto &UDivorRem = cast<GenericMachineInstr>(MI);
5469 Register Dst = UDivorRem.getReg(0);
5470 Register LHS = UDivorRem.getReg(1);
5471 Register RHS = UDivorRem.getReg(2);
5472 LLT Ty = MRI.getType(Dst);
5473 LLT ScalarTy = Ty.getScalarType();
5474 const unsigned EltBits = ScalarTy.getScalarSizeInBits();
5476 LLT ScalarShiftAmtTy = ShiftAmtTy.getScalarType();
5477
5478 auto &MIB = Builder;
5479
5480 bool UseSRL = false;
5481 SmallVector<Register, 16> Shifts, Factors;
5482 auto *RHSDefInstr = cast<GenericMachineInstr>(getDefIgnoringCopies(RHS, MRI));
5483 bool IsSplat = getIConstantSplatVal(*RHSDefInstr, MRI).has_value();
5484
5485 auto BuildExactUDIVPattern = [&](const Constant *C) {
5486 // Don't recompute inverses for each splat element.
5487 if (IsSplat && !Factors.empty()) {
5488 Shifts.push_back(Shifts[0]);
5489 Factors.push_back(Factors[0]);
5490 return true;
5491 }
5492
5493 auto *CI = cast<ConstantInt>(C);
5494 APInt Divisor = CI->getValue();
5495 unsigned Shift = Divisor.countr_zero();
5496 if (Shift) {
5497 Divisor.lshrInPlace(Shift);
5498 UseSRL = true;
5499 }
5500
5501 // Calculate the multiplicative inverse modulo BW.
5502 APInt Factor = Divisor.multiplicativeInverse();
5503 Shifts.push_back(MIB.buildConstant(ScalarShiftAmtTy, Shift).getReg(0));
5504 Factors.push_back(MIB.buildConstant(ScalarTy, Factor).getReg(0));
5505 return true;
5506 };
5507
5508 if (MI.getFlag(MachineInstr::MIFlag::IsExact)) {
5509 // Collect all magic values from the build vector.
5510 if (!matchUnaryPredicate(MRI, RHS, BuildExactUDIVPattern))
5511 llvm_unreachable("Expected unary predicate match to succeed");
5512
5513 Register Shift, Factor;
5514 if (Ty.isVector()) {
5515 Shift = MIB.buildBuildVector(ShiftAmtTy, Shifts).getReg(0);
5516 Factor = MIB.buildBuildVector(Ty, Factors).getReg(0);
5517 } else {
5518 Shift = Shifts[0];
5519 Factor = Factors[0];
5520 }
5521
5522 Register Res = LHS;
5523
5524 if (UseSRL)
5525 Res = MIB.buildLShr(Ty, Res, Shift, MachineInstr::IsExact).getReg(0);
5526
5527 return MIB.buildMul(Ty, Res, Factor);
5528 }
5529
5530 unsigned KnownLeadingZeros =
5531 VT ? VT->getKnownBits(LHS).countMinLeadingZeros() : 0;
5532
5533 bool UseNPQ = false;
5534 SmallVector<Register, 16> PreShifts, PostShifts, MagicFactors, NPQFactors;
5535 auto BuildUDIVPattern = [&](const Constant *C) {
5536 auto *CI = cast<ConstantInt>(C);
5537 const APInt &Divisor = CI->getValue();
5538
5539 bool SelNPQ = false;
5540 APInt Magic(Divisor.getBitWidth(), 0);
5541 unsigned PreShift = 0, PostShift = 0;
5542
5543 // Magic algorithm doesn't work for division by 1. We need to emit a select
5544 // at the end.
5545 // TODO: Use undef values for divisor of 1.
5546 if (!Divisor.isOne()) {
5547
5548 // UnsignedDivisionByConstantInfo doesn't work correctly if leading zeros
5549 // in the dividend exceeds the leading zeros for the divisor.
5552 Divisor, std::min(KnownLeadingZeros, Divisor.countl_zero()));
5553
5554 Magic = std::move(magics.Magic);
5555
5556 assert(magics.PreShift < Divisor.getBitWidth() &&
5557 "We shouldn't generate an undefined shift!");
5558 assert(magics.PostShift < Divisor.getBitWidth() &&
5559 "We shouldn't generate an undefined shift!");
5560 assert((!magics.IsAdd || magics.PreShift == 0) && "Unexpected pre-shift");
5561 PreShift = magics.PreShift;
5562 PostShift = magics.PostShift;
5563 SelNPQ = magics.IsAdd;
5564 }
5565
5566 PreShifts.push_back(
5567 MIB.buildConstant(ScalarShiftAmtTy, PreShift).getReg(0));
5568 MagicFactors.push_back(MIB.buildConstant(ScalarTy, Magic).getReg(0));
5569 NPQFactors.push_back(
5570 MIB.buildConstant(ScalarTy,
5571 SelNPQ ? APInt::getOneBitSet(EltBits, EltBits - 1)
5572 : APInt::getZero(EltBits))
5573 .getReg(0));
5574 PostShifts.push_back(
5575 MIB.buildConstant(ScalarShiftAmtTy, PostShift).getReg(0));
5576 UseNPQ |= SelNPQ;
5577 return true;
5578 };
5579
5580 // Collect the shifts/magic values from each element.
5581 bool Matched = matchUnaryPredicate(MRI, RHS, BuildUDIVPattern);
5582 (void)Matched;
5583 assert(Matched && "Expected unary predicate match to succeed");
5584
5585 Register PreShift, PostShift, MagicFactor, NPQFactor;
5586 auto *RHSDef = getOpcodeDef<GBuildVector>(RHS, MRI);
5587 if (RHSDef) {
5588 PreShift = MIB.buildBuildVector(ShiftAmtTy, PreShifts).getReg(0);
5589 MagicFactor = MIB.buildBuildVector(Ty, MagicFactors).getReg(0);
5590 NPQFactor = MIB.buildBuildVector(Ty, NPQFactors).getReg(0);
5591 PostShift = MIB.buildBuildVector(ShiftAmtTy, PostShifts).getReg(0);
5592 } else {
5593 assert(MRI.getType(RHS).isScalar() &&
5594 "Non-build_vector operation should have been a scalar");
5595 PreShift = PreShifts[0];
5596 MagicFactor = MagicFactors[0];
5597 PostShift = PostShifts[0];
5598 }
5599
5600 Register Q = LHS;
5601 Q = MIB.buildLShr(Ty, Q, PreShift).getReg(0);
5602
5603 // Multiply the numerator (operand 0) by the magic value.
5604 Q = MIB.buildUMulH(Ty, Q, MagicFactor).getReg(0);
5605
5606 if (UseNPQ) {
5607 Register NPQ = MIB.buildSub(Ty, LHS, Q).getReg(0);
5608
5609 // For vectors we might have a mix of non-NPQ/NPQ paths, so use
5610 // G_UMULH to act as a SRL-by-1 for NPQ, else multiply by zero.
5611 if (Ty.isVector())
5612 NPQ = MIB.buildUMulH(Ty, NPQ, NPQFactor).getReg(0);
5613 else
5614 NPQ = MIB.buildLShr(Ty, NPQ, MIB.buildConstant(ShiftAmtTy, 1)).getReg(0);
5615
5616 Q = MIB.buildAdd(Ty, NPQ, Q).getReg(0);
5617 }
5618
5619 Q = MIB.buildLShr(Ty, Q, PostShift).getReg(0);
5620 auto One = MIB.buildConstant(Ty, 1);
5621 auto IsOne = MIB.buildICmp(
5623 Ty.isScalar() ? LLT::integer(1) : Ty.changeElementType(LLT::integer(1)),
5624 RHS, One);
5625 auto ret = MIB.buildSelect(Ty, IsOne, LHS, Q);
5626
5627 if (Opcode == TargetOpcode::G_UREM) {
5628 auto Prod = MIB.buildMul(Ty, ret, RHS);
5629 return MIB.buildSub(Ty, LHS, Prod);
5630 }
5631 return ret;
5632}
5633
5635 unsigned Opcode = MI.getOpcode();
5636 assert(Opcode == TargetOpcode::G_UDIV || Opcode == TargetOpcode::G_UREM);
5637 Register Dst = MI.getOperand(0).getReg();
5638 Register RHS = MI.getOperand(2).getReg();
5639 LLT DstTy = MRI.getType(Dst);
5640
5641 auto &MF = *MI.getMF();
5642 AttributeList Attr = MF.getFunction().getAttributes();
5643 const auto &TLI = getTargetLowering();
5644 LLVMContext &Ctx = MF.getFunction().getContext();
5645 if (DstTy.getScalarSizeInBits() == 1 ||
5646 TLI.isIntDivCheap(getApproximateEVTForLLT(DstTy, Ctx), Attr))
5647 return false;
5648
5649 // Don't do this for minsize because the instruction sequence is usually
5650 // larger.
5651 if (MF.getFunction().hasMinSize())
5652 return false;
5653
5654 if (Opcode == TargetOpcode::G_UDIV &&
5656 return matchUnaryPredicate(
5657 MRI, RHS, [](const Constant *C) { return C && !C->isNullValue(); });
5658 }
5659
5660 MachineInstr *RHSDef;
5661 if (!mi_match(RHS, MRI, m_MInstr(RHSDef)) ||
5663 return false;
5664
5665 // Don't do this if the types are not going to be legal.
5666 if (LI) {
5667 if (!isLegalOrBeforeLegalizer({TargetOpcode::G_MUL, {DstTy, DstTy}}))
5668 return false;
5669 if (!isLegalOrBeforeLegalizer({TargetOpcode::G_UMULH, {DstTy}}))
5670 return false;
5672 {TargetOpcode::G_ICMP,
5673 {DstTy.isVector() ? DstTy.changeElementSize(1) : LLT::scalar(1),
5674 DstTy}}))
5675 return false;
5676 if (Opcode == TargetOpcode::G_UREM &&
5677 !isLegalOrBeforeLegalizer({TargetOpcode::G_SUB, {DstTy, DstTy}}))
5678 return false;
5679 }
5680
5681 return matchUnaryPredicate(
5682 MRI, RHS, [](const Constant *C) { return C && !C->isNullValue(); });
5683}
5684
5686 auto *NewMI = buildUDivOrURemUsingMul(MI);
5687 replaceSingleDefInstWithReg(MI, NewMI->getOperand(0).getReg());
5688}
5689
5691 unsigned Opcode = MI.getOpcode();
5692 assert(Opcode == TargetOpcode::G_SDIV || Opcode == TargetOpcode::G_SREM);
5693 Register Dst = MI.getOperand(0).getReg();
5694 Register RHS = MI.getOperand(2).getReg();
5695 LLT DstTy = MRI.getType(Dst);
5696 auto SizeInBits = DstTy.getScalarSizeInBits();
5697 LLT WideTy = DstTy.changeElementSize(SizeInBits * 2);
5698
5699 auto &MF = *MI.getMF();
5700 AttributeList Attr = MF.getFunction().getAttributes();
5701 const auto &TLI = getTargetLowering();
5702 LLVMContext &Ctx = MF.getFunction().getContext();
5703 if (DstTy.getScalarSizeInBits() < 3 ||
5704 TLI.isIntDivCheap(getApproximateEVTForLLT(DstTy, Ctx), Attr))
5705 return false;
5706
5707 // Don't do this for minsize because the instruction sequence is usually
5708 // larger.
5709 if (MF.getFunction().hasMinSize())
5710 return false;
5711
5712 // If the sdiv has an 'exact' flag we can use a simpler lowering.
5713 if (Opcode == TargetOpcode::G_SDIV &&
5715 return matchUnaryPredicate(
5716 MRI, RHS, [](const Constant *C) { return C && !C->isNullValue(); });
5717 }
5718
5719 MachineInstr *RHSDef;
5720 if (!mi_match(RHS, MRI, m_MInstr(RHSDef)) ||
5722 return false;
5723
5724 // Don't do this if the types are not going to be legal.
5725 if (LI) {
5726 if (!isLegalOrBeforeLegalizer({TargetOpcode::G_MUL, {DstTy, DstTy}}))
5727 return false;
5728 if (isPreLegalize()) {
5729 // The legalizer can still split a vector G_SMULH or lower it through a
5730 // wider G_MUL.
5731 if (!isLegalOrHasFewerElements({TargetOpcode::G_SMULH, {DstTy}}) &&
5732 !isLegalOrHasWidenScalar({TargetOpcode::G_MUL, {WideTy, WideTy}}))
5733 return false;
5734 } else if (!isLegal({TargetOpcode::G_SMULH, {DstTy}})) {
5735 // After legalization nothing will fix up an illegal G_SMULH.
5736 return false;
5737 }
5738 if (Opcode == TargetOpcode::G_SREM &&
5739 !isLegalOrBeforeLegalizer({TargetOpcode::G_SUB, {DstTy, DstTy}}))
5740 return false;
5741 }
5742
5743 return matchUnaryPredicate(
5744 MRI, RHS, [](const Constant *C) { return C && !C->isNullValue(); });
5745}
5746
5748 auto *NewMI = buildSDivOrSRemUsingMul(MI);
5749 replaceSingleDefInstWithReg(MI, NewMI->getOperand(0).getReg());
5750}
5751
5753 unsigned Opcode = MI.getOpcode();
5754 assert(MI.getOpcode() == TargetOpcode::G_SDIV ||
5755 Opcode == TargetOpcode::G_SREM);
5756 auto &SDivorRem = cast<GenericMachineInstr>(MI);
5757 Register Dst = SDivorRem.getReg(0);
5758 Register LHS = SDivorRem.getReg(1);
5759 Register RHS = SDivorRem.getReg(2);
5760 LLT Ty = MRI.getType(Dst);
5761 LLT ScalarTy = Ty.getScalarType();
5762 const unsigned EltBits = ScalarTy.getScalarSizeInBits();
5764 LLT ScalarShiftAmtTy = ShiftAmtTy.getScalarType();
5765 auto &MIB = Builder;
5766
5767 bool UseSRA = false;
5768 SmallVector<Register, 16> ExactShifts, ExactFactors;
5769
5770 auto *RHSDefInstr = cast<GenericMachineInstr>(getDefIgnoringCopies(RHS, MRI));
5771 bool IsSplat = getIConstantSplatVal(*RHSDefInstr, MRI).has_value();
5772
5773 auto BuildExactSDIVPattern = [&](const Constant *C) {
5774 // Don't recompute inverses for each splat element.
5775 if (IsSplat && !ExactFactors.empty()) {
5776 ExactShifts.push_back(ExactShifts[0]);
5777 ExactFactors.push_back(ExactFactors[0]);
5778 return true;
5779 }
5780
5781 auto *CI = cast<ConstantInt>(C);
5782 APInt Divisor = CI->getValue();
5783 unsigned Shift = Divisor.countr_zero();
5784 if (Shift) {
5785 Divisor.ashrInPlace(Shift);
5786 UseSRA = true;
5787 }
5788
5789 // Calculate the multiplicative inverse modulo BW.
5790 // 2^W requires W + 1 bits, so we have to extend and then truncate.
5791 APInt Factor = Divisor.multiplicativeInverse();
5792 ExactShifts.push_back(MIB.buildConstant(ScalarShiftAmtTy, Shift).getReg(0));
5793 ExactFactors.push_back(MIB.buildConstant(ScalarTy, Factor).getReg(0));
5794 return true;
5795 };
5796
5797 if (MI.getFlag(MachineInstr::MIFlag::IsExact)) {
5798 // Collect all magic values from the build vector.
5799 bool Matched = matchUnaryPredicate(MRI, RHS, BuildExactSDIVPattern);
5800 (void)Matched;
5801 assert(Matched && "Expected unary predicate match to succeed");
5802
5803 Register Shift, Factor;
5804 if (Ty.isVector()) {
5805 Shift = MIB.buildBuildVector(ShiftAmtTy, ExactShifts).getReg(0);
5806 Factor = MIB.buildBuildVector(Ty, ExactFactors).getReg(0);
5807 } else {
5808 Shift = ExactShifts[0];
5809 Factor = ExactFactors[0];
5810 }
5811
5812 Register Res = LHS;
5813
5814 if (UseSRA)
5815 Res = MIB.buildAShr(Ty, Res, Shift, MachineInstr::IsExact).getReg(0);
5816
5817 return MIB.buildMul(Ty, Res, Factor);
5818 }
5819
5820 SmallVector<Register, 16> MagicFactors, Factors, Shifts, ShiftMasks;
5821
5822 auto BuildSDIVPattern = [&](const Constant *C) {
5823 auto *CI = cast<ConstantInt>(C);
5824 const APInt &Divisor = CI->getValue();
5825
5828 int NumeratorFactor = 0;
5829 int ShiftMask = -1;
5830
5831 if (Divisor.isOne() || Divisor.isAllOnes()) {
5832 // If d is +1/-1, we just multiply the numerator by +1/-1.
5833 NumeratorFactor = Divisor.getSExtValue();
5834 Magics.Magic = 0;
5835 Magics.ShiftAmount = 0;
5836 ShiftMask = 0;
5837 } else if (Divisor.isStrictlyPositive() && Magics.Magic.isNegative()) {
5838 // If d > 0 and m < 0, add the numerator.
5839 NumeratorFactor = 1;
5840 } else if (Divisor.isNegative() && Magics.Magic.isStrictlyPositive()) {
5841 // If d < 0 and m > 0, subtract the numerator.
5842 NumeratorFactor = -1;
5843 }
5844
5845 MagicFactors.push_back(MIB.buildConstant(ScalarTy, Magics.Magic).getReg(0));
5846 Factors.push_back(MIB.buildConstant(ScalarTy, NumeratorFactor).getReg(0));
5847 Shifts.push_back(
5848 MIB.buildConstant(ScalarShiftAmtTy, Magics.ShiftAmount).getReg(0));
5849 ShiftMasks.push_back(MIB.buildConstant(ScalarTy, ShiftMask).getReg(0));
5850
5851 return true;
5852 };
5853
5854 // Collect the shifts/magic values from each element.
5855 bool Matched = matchUnaryPredicate(MRI, RHS, BuildSDIVPattern);
5856 (void)Matched;
5857 assert(Matched && "Expected unary predicate match to succeed");
5858
5859 Register MagicFactor, Factor, Shift, ShiftMask;
5860 auto *RHSDef = getOpcodeDef<GBuildVector>(RHS, MRI);
5861 if (RHSDef) {
5862 MagicFactor = MIB.buildBuildVector(Ty, MagicFactors).getReg(0);
5863 Factor = MIB.buildBuildVector(Ty, Factors).getReg(0);
5864 Shift = MIB.buildBuildVector(ShiftAmtTy, Shifts).getReg(0);
5865 ShiftMask = MIB.buildBuildVector(Ty, ShiftMasks).getReg(0);
5866 } else {
5867 assert(MRI.getType(RHS).isScalar() &&
5868 "Non-build_vector operation should have been a scalar");
5869 MagicFactor = MagicFactors[0];
5870 Factor = Factors[0];
5871 Shift = Shifts[0];
5872 ShiftMask = ShiftMasks[0];
5873 }
5874
5875 Register Q = LHS;
5876 Q = MIB.buildSMulH(Ty, LHS, MagicFactor).getReg(0);
5877
5878 // (Optionally) Add/subtract the numerator using Factor.
5879 Factor = MIB.buildMul(Ty, LHS, Factor).getReg(0);
5880 Q = MIB.buildAdd(Ty, Q, Factor).getReg(0);
5881
5882 // Shift right algebraic by shift value.
5883 Q = MIB.buildAShr(Ty, Q, Shift).getReg(0);
5884
5885 // Extract the sign bit, mask it and add it to the quotient.
5886 auto SignShift = MIB.buildConstant(ShiftAmtTy, EltBits - 1);
5887 auto T = MIB.buildLShr(Ty, Q, SignShift);
5888 T = MIB.buildAnd(Ty, T, ShiftMask);
5889 auto ret = MIB.buildAdd(Ty, Q, T);
5890
5891 if (Opcode == TargetOpcode::G_SREM) {
5892 auto Prod = MIB.buildMul(Ty, ret, RHS);
5893 return MIB.buildSub(Ty, LHS, Prod);
5894 }
5895 return ret;
5896}
5897
5899 assert((MI.getOpcode() == TargetOpcode::G_SDIV ||
5900 MI.getOpcode() == TargetOpcode::G_UDIV) &&
5901 "Expected SDIV or UDIV");
5902 auto &Div = cast<GenericMachineInstr>(MI);
5903 Register RHS = Div.getReg(2);
5904 auto MatchPow2 = [&](const Constant *C) {
5905 auto *CI = dyn_cast<ConstantInt>(C);
5906 return CI && (CI->getValue().isPowerOf2() ||
5907 (IsSigned && CI->getValue().isNegatedPowerOf2()));
5908 };
5909 return matchUnaryPredicate(MRI, RHS, MatchPow2, /*AllowUndefs=*/false);
5910}
5911
5913 assert(MI.getOpcode() == TargetOpcode::G_SDIV && "Expected SDIV");
5914 auto &SDiv = cast<GenericMachineInstr>(MI);
5915 Register Dst = SDiv.getReg(0);
5916 Register LHS = SDiv.getReg(1);
5917 Register RHS = SDiv.getReg(2);
5918 LLT Ty = MRI.getType(Dst);
5920 LLT CCVT = Ty.isVector() ? LLT::vector(Ty.getElementCount(), LLT::integer(1))
5921 : LLT::integer(1);
5922
5923 // Effectively we want to lower G_SDIV %lhs, %rhs, where %rhs is a power of 2,
5924 // to the following version:
5925 //
5926 // %c1 = G_CTTZ %rhs
5927 // %inexact = G_SUB $bitwidth, %c1
5928 // %sign = %G_ASHR %lhs, $(bitwidth - 1)
5929 // %lshr = G_LSHR %sign, %inexact
5930 // %add = G_ADD %lhs, %lshr
5931 // %ashr = G_ASHR %add, %c1
5932 // %ashr = G_SELECT, %isoneorallones, %lhs, %ashr
5933 // %zero = G_CONSTANT $0
5934 // %neg = G_NEG %ashr
5935 // %isneg = G_ICMP SLT %rhs, %zero
5936 // %res = G_SELECT %isneg, %neg, %ashr
5937
5938 unsigned BitWidth = Ty.getScalarSizeInBits();
5939 auto Zero = Builder.buildConstant(Ty, 0);
5940
5941 auto Bits = Builder.buildConstant(ShiftAmtTy, BitWidth);
5942 auto C1 = Builder.buildCTTZ(ShiftAmtTy, RHS);
5943 auto Inexact = Builder.buildSub(ShiftAmtTy, Bits, C1);
5944 // Splat the sign bit into the register
5945 auto Sign = Builder.buildAShr(
5946 Ty, LHS, Builder.buildConstant(ShiftAmtTy, BitWidth - 1));
5947
5948 // Add (LHS < 0) ? abs2 - 1 : 0;
5949 auto LSrl = Builder.buildLShr(Ty, Sign, Inexact);
5950 auto Add = Builder.buildAdd(Ty, LHS, LSrl);
5951 auto AShr = Builder.buildAShr(Ty, Add, C1);
5952
5953 // Special case: (sdiv X, 1) -> X
5954 // Special Case: (sdiv X, -1) -> 0-X
5955 auto One = Builder.buildConstant(Ty, 1);
5956 auto MinusOne = Builder.buildConstant(Ty, -1);
5957 auto IsOne = Builder.buildICmp(CmpInst::Predicate::ICMP_EQ, CCVT, RHS, One);
5958 auto IsMinusOne =
5959 Builder.buildICmp(CmpInst::Predicate::ICMP_EQ, CCVT, RHS, MinusOne);
5960 auto IsOneOrMinusOne = Builder.buildOr(CCVT, IsOne, IsMinusOne);
5961 AShr = Builder.buildSelect(Ty, IsOneOrMinusOne, LHS, AShr);
5962
5963 // If divided by a positive value, we're done. Otherwise, the result must be
5964 // negated.
5965 auto Neg = Builder.buildNeg(Ty, AShr);
5966 auto IsNeg = Builder.buildICmp(CmpInst::Predicate::ICMP_SLT, CCVT, RHS, Zero);
5967 Builder.buildSelect(MI.getOperand(0).getReg(), IsNeg, Neg, AShr);
5968 MI.eraseFromParent();
5969}
5970
5972 assert(MI.getOpcode() == TargetOpcode::G_UDIV && "Expected UDIV");
5973 auto &UDiv = cast<GenericMachineInstr>(MI);
5974 Register Dst = UDiv.getReg(0);
5975 Register LHS = UDiv.getReg(1);
5976 Register RHS = UDiv.getReg(2);
5977 LLT Ty = MRI.getType(Dst);
5979
5980 auto C1 = Builder.buildCTTZ(ShiftAmtTy, RHS);
5981 Builder.buildLShr(MI.getOperand(0).getReg(), LHS, C1);
5982 MI.eraseFromParent();
5983}
5984
5986 BuildFnTy &MatchInfo) const {
5987 Register Dst = MI.getOperand(0).getReg();
5988 Register LHS = MI.getOperand(1).getReg();
5989 Register RHS = MI.getOperand(2).getReg();
5990 Register Base, Shift;
5991 if (!mi_match(RHS, MRI, m_GShl(m_Reg(Base), m_Reg(Shift))))
5992 return false;
5993
5994 if (!matchUnaryPredicate(MRI, Base, [](const Constant *C) {
5996 return CI && CI->getValue().isPowerOf2();
5997 }))
5998 return false;
5999
6000 LLT Ty = MRI.getType(Dst);
6001 LLT ShiftTy = MRI.getType(Shift);
6002 // The shift type must be able to represent every valid shift amount.
6003 if (ShiftTy.getScalarSizeInBits() < Log2_64_Ceil(Ty.getScalarSizeInBits()))
6004 return false;
6005 if (!isLegalOrBeforeLegalizer({TargetOpcode::G_CTTZ, {ShiftTy, Ty}}) ||
6006 !isLegalOrBeforeLegalizer({TargetOpcode::G_ADD, {ShiftTy, ShiftTy}}) ||
6007 !isLegalOrBeforeLegalizer({TargetOpcode::G_LSHR, {Ty, ShiftTy}}))
6008 return false;
6009
6010 unsigned Flags =
6012 MatchInfo = [=](MachineIRBuilder &B) {
6013 auto Log2 = B.buildCTTZ(ShiftTy, Base);
6014 auto Amount = B.buildAdd(ShiftTy, Shift, Log2);
6015 B.buildLShr(Dst, LHS, Amount, Flags);
6016 };
6017 return true;
6018}
6019
6021 MachineInstr &MI, unsigned UnsignedOpcode) const {
6022 Register Dst = MI.getOperand(0).getReg();
6023 Register LHS = MI.getOperand(1).getReg();
6024 Register RHS = MI.getOperand(2).getReg();
6025 if (!VT || !VT->signBitIsZero(LHS) || !VT->signBitIsZero(RHS))
6026 return false;
6027
6028 LLT Ty = MRI.getType(Dst);
6029 return isLegalOrBeforeLegalizer({UnsignedOpcode, {Ty}});
6030}
6031
6033 assert(MI.getOpcode() == TargetOpcode::G_SREM && "Expected SREM");
6034 auto &SRem = cast<GBinOp>(MI);
6035 Register Dst = SRem.getReg(0);
6036 Register LHS = SRem.getLHSReg();
6037 Register RHS = SRem.getRHSReg();
6038 LLT Ty = MRI.getType(Dst);
6040
6041 // Effectively we want to lower G_SREM %lhs, %rhs, where %rhs is +/- a power
6042 // of 2, to the following branch-free bias-and-mask version:
6043 //
6044 // %abs = G_ABS %rhs
6045 // %mask = G_SUB %abs, 1
6046 // %sign = G_ASHR %lhs, $(bitwidth - 1)
6047 // %bias = G_AND %sign, %mask
6048 // %biased = G_ADD %lhs, %bias
6049 // %masked = G_AND %biased, %mask
6050 // %res = G_SUB %masked, %bias
6051 //
6052 // The bias adds (|%rhs| - 1) for negative %lhs, correcting rounding towards
6053 // zero (instead of towards -inf that a plain mask would give). Constant
6054 // divisors collapse %mask to a single G_CONSTANT via the CSEMIRBuilder folds
6055 // for G_ABS and G_SUB.
6056
6057 unsigned BitWidth = Ty.getScalarSizeInBits();
6058 auto AbsRHS = Builder.buildAbs(Ty, RHS);
6059 auto Mask = Builder.buildSub(Ty, AbsRHS, Builder.buildConstant(Ty, 1));
6060 auto BWMinusOne = Builder.buildConstant(ShiftAmtTy, BitWidth - 1);
6061 auto Sign = Builder.buildAShr(Ty, LHS, BWMinusOne);
6062 auto Bias = Builder.buildAnd(Ty, Sign, Mask);
6063 auto Biased = Builder.buildAdd(Ty, LHS, Bias);
6064 auto Masked = Builder.buildAnd(Ty, Biased, Mask);
6065 Builder.buildSub(Dst, Masked, Bias);
6066 MI.eraseFromParent();
6067}
6068
6070 assert(MI.getOpcode() == TargetOpcode::G_UMULH);
6071 Register RHS = MI.getOperand(2).getReg();
6072 Register Dst = MI.getOperand(0).getReg();
6073 LLT Ty = MRI.getType(Dst);
6074 LLT RHSTy = MRI.getType(RHS);
6076 auto MatchPow2ExceptOne = [&](const Constant *C) {
6077 if (auto *CI = dyn_cast<ConstantInt>(C))
6078 return CI->getValue().isPowerOf2() && !CI->getValue().isOne();
6079 return false;
6080 };
6081 if (!matchUnaryPredicate(MRI, RHS, MatchPow2ExceptOne, false))
6082 return false;
6083 // We need to check both G_LSHR and G_CTLZ because the combine uses G_CTLZ to
6084 // get log base 2, and it is not always legal for on a target.
6085 return isLegalOrBeforeLegalizer({TargetOpcode::G_LSHR, {Ty, ShiftAmtTy}}) &&
6086 isLegalOrBeforeLegalizer({TargetOpcode::G_CTLZ, {RHSTy, RHSTy}});
6087}
6088
6090 Register LHS = MI.getOperand(1).getReg();
6091 Register RHS = MI.getOperand(2).getReg();
6092 Register Dst = MI.getOperand(0).getReg();
6093 LLT Ty = MRI.getType(Dst);
6095 unsigned NumEltBits = Ty.getScalarSizeInBits();
6096
6097 auto LogBase2 = buildLogBase2(RHS, Builder);
6098 auto ShiftAmt =
6099 Builder.buildSub(Ty, Builder.buildConstant(Ty, NumEltBits), LogBase2);
6100 auto Trunc = Builder.buildZExtOrTrunc(ShiftAmtTy, ShiftAmt);
6101 Builder.buildLShr(Dst, LHS, Trunc);
6102 MI.eraseFromParent();
6103}
6104
6106 Register &MatchInfo) const {
6107 Register Dst = MI.getOperand(0).getReg();
6108 Register Src = MI.getOperand(1).getReg();
6109 LLT DstTy = MRI.getType(Dst);
6110 LLT SrcTy = MRI.getType(Src);
6111 unsigned NumDstBits = DstTy.getScalarSizeInBits();
6112 unsigned NumSrcBits = SrcTy.getScalarSizeInBits();
6113 assert(NumSrcBits > NumDstBits && "Unexpected types for truncate operation");
6114
6116 {TargetOpcode::G_TRUNC_SSAT_S, {DstTy, SrcTy}}))
6117 return false;
6118
6119 APInt SignedMax = APInt::getSignedMaxValue(NumDstBits).sext(NumSrcBits);
6120 APInt SignedMin = APInt::getSignedMinValue(NumDstBits).sext(NumSrcBits);
6121 if (mi_match(
6122 Src, MRI,
6123 m_GSMin(m_GSMax(m_Reg(MatchInfo), m_SpecificICstOrSplat(SignedMin)),
6124 m_SpecificICstOrSplat(SignedMax))))
6125 return true;
6126 if (mi_match(
6127 Src, MRI,
6128 m_GSMax(m_GSMin(m_Reg(MatchInfo), m_SpecificICstOrSplat(SignedMax)),
6129 m_SpecificICstOrSplat(SignedMin))))
6130 return true;
6131
6132 // CVP in the midend will often transform trunc(smin(smax(..)) into
6133 // trunc nsw(smin(..)) as the smax against INT_MIN never saturates.
6134 if (MI.getFlag(MachineInstr::MIFlag::NoSWrap) &&
6135 mi_match(Src, MRI,
6136 m_GSMin(m_Reg(MatchInfo), m_SpecificICstOrSplat(SignedMax))))
6137 return true;
6138
6139 return false;
6140}
6141
6143 Register &MatchInfo) const {
6144 Register Dst = MI.getOperand(0).getReg();
6145 Builder.buildTruncSSatS(Dst, MatchInfo);
6146 MI.eraseFromParent();
6147}
6148
6150 Register &MatchInfo) const {
6151 Register Dst = MI.getOperand(0).getReg();
6152 Register Src = MI.getOperand(1).getReg();
6153 LLT DstTy = MRI.getType(Dst);
6154 LLT SrcTy = MRI.getType(Src);
6155 unsigned NumDstBits = DstTy.getScalarSizeInBits();
6156 unsigned NumSrcBits = SrcTy.getScalarSizeInBits();
6157 assert(NumSrcBits > NumDstBits && "Unexpected types for truncate operation");
6158
6160 {TargetOpcode::G_TRUNC_SSAT_U, {DstTy, SrcTy}}))
6161 return false;
6162 APInt UnsignedMax = APInt::getMaxValue(NumDstBits).zext(NumSrcBits);
6163 return mi_match(Src, MRI,
6165 m_SpecificICstOrSplat(UnsignedMax))) ||
6166 mi_match(Src, MRI,
6167 m_GSMax(m_GSMin(m_Reg(MatchInfo),
6168 m_SpecificICstOrSplat(UnsignedMax)),
6169 m_SpecificICstOrSplat(0))) ||
6170 mi_match(Src, MRI,
6172 m_SpecificICstOrSplat(UnsignedMax)));
6173}
6174
6176 Register &MatchInfo) const {
6177 Register Dst = MI.getOperand(0).getReg();
6178 Builder.buildTruncSSatU(Dst, MatchInfo);
6179 MI.eraseFromParent();
6180}
6181
6183 MachineInstr &MinMI) const {
6184 Register Min = MinMI.getOperand(2).getReg();
6185 Register Val = MinMI.getOperand(1).getReg();
6186 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
6187 LLT SrcTy = MRI.getType(Val);
6188 unsigned NumDstBits = DstTy.getScalarSizeInBits();
6189 unsigned NumSrcBits = SrcTy.getScalarSizeInBits();
6190 assert(NumSrcBits > NumDstBits && "Unexpected types for truncate operation");
6191
6193 {TargetOpcode::G_TRUNC_SSAT_U, {DstTy, SrcTy}}))
6194 return false;
6195 APInt UnsignedMax = APInt::getMaxValue(NumDstBits).zext(NumSrcBits);
6196 return mi_match(Min, MRI, m_SpecificICstOrSplat(UnsignedMax)) &&
6197 !mi_match(Val, MRI, m_GSMax(m_Reg(), m_Reg()));
6198}
6199
6201 MachineInstr &SrcMI) const {
6202 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
6203 LLT SrcTy = MRI.getType(SrcMI.getOperand(1).getReg());
6204
6205 return LI &&
6206 isLegalOrBeforeLegalizer({TargetOpcode::G_FPTOUI_SAT, {DstTy, SrcTy}});
6207}
6208
6210 assert(MI.getOpcode() == TargetOpcode::G_FSUB);
6211
6212 Register LHS = MI.getOperand(1).getReg();
6213 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
6214
6215 const auto LHSCst = Ty.isVector()
6216 ? getFConstantSplat(LHS, MRI, /* allowUndef */ true)
6218 if (!LHSCst)
6219 return false;
6220
6221 // Skip fsub_to_fneg when both LHS and RHS are scalar constants (vector FP
6222 // binop constant folding is not supported yet), so constant_fold_fp_binop
6223 // folds these cases.
6224 if (!Ty.isVector() && getConstantFPVRegVal(LHS, MRI) &&
6225 getConstantFPVRegVal(MI.getOperand(2).getReg(), MRI))
6226 return false;
6227
6228 // -0.0 is always allowed
6229 if (LHSCst->Value.isNegZero())
6230 return true;
6231
6232 // +0.0 is only allowed if nsz is set.
6233 if (LHSCst->Value.isPosZero())
6234 return MI.getFlag(MachineInstr::FmNsz);
6235
6236 return false;
6237}
6238
6239/// Checks if \p MI is TargetOpcode::G_FMUL and contractable either
6240/// due to global flags or MachineInstr flags.
6241static bool isContractableFMul(MachineInstr &MI, bool AllowFusionGlobally) {
6242 if (MI.getOpcode() != TargetOpcode::G_FMUL)
6243 return false;
6244 return AllowFusionGlobally || MI.getFlag(MachineInstr::MIFlag::FmContract);
6245}
6246
6247static bool hasMoreUses(const MachineInstr &MI0, const MachineInstr &MI1,
6248 const MachineRegisterInfo &MRI) {
6249 return std::distance(MRI.use_instr_nodbg_begin(MI0.getOperand(0).getReg()),
6250 MRI.use_instr_nodbg_end()) >
6251 std::distance(MRI.use_instr_nodbg_begin(MI1.getOperand(0).getReg()),
6252 MRI.use_instr_nodbg_end());
6253}
6254
6256 bool &AllowFusionGlobally,
6257 bool &HasFMAD, bool &Aggressive,
6258 bool CanReassociate) const {
6259
6260 auto *MF = MI.getMF();
6261 const auto &TLI = *MF->getSubtarget().getTargetLowering();
6262 LLT DstType = MRI.getType(MI.getOperand(0).getReg());
6263
6264 if (CanReassociate && !MI.getFlag(MachineInstr::MIFlag::FmReassoc))
6265 return false;
6266
6267 // Floating-point multiply-add with intermediate rounding.
6268 HasFMAD = (!isPreLegalize() && TLI.isFMADLegal(MI, DstType));
6269 // Floating-point multiply-add without intermediate rounding.
6270 bool HasFMA = TLI.isFMAFasterThanFMulAndFAdd(*MF, DstType) &&
6271 isLegalOrBeforeLegalizer({TargetOpcode::G_FMA, {DstType}});
6272 // No valid opcode, do not combine.
6273 if (!HasFMAD && !HasFMA)
6274 return false;
6275
6276 // FMAD (with intermediate rounding) is always safe to form; FMA requires the
6277 // contract fast-math flag.
6278 AllowFusionGlobally = HasFMAD;
6279 // If the addition is not contractable, do not combine.
6280 if (!AllowFusionGlobally && !MI.getFlag(MachineInstr::MIFlag::FmContract))
6281 return false;
6282
6283 Aggressive = TLI.enableAggressiveFMAFusion(DstType);
6284 return true;
6285}
6286
6289 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
6290 assert(MI.getOpcode() == TargetOpcode::G_FADD);
6291
6292 bool AllowFusionGlobally, HasFMAD, Aggressive;
6293 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive))
6294 return false;
6295
6296 Register Op1 = MI.getOperand(1).getReg();
6297 Register Op2 = MI.getOperand(2).getReg();
6298 MachineInstr *Op1Def, *Op2Def;
6299 if (!mi_match(Op1, MRI, m_MInstr(Op1Def)) ||
6300 !mi_match(Op2, MRI, m_MInstr(Op2Def)))
6301 return false;
6302 DefinitionAndSourceRegister LHS = {Op1Def, Op1};
6303 DefinitionAndSourceRegister RHS = {Op2Def, Op2};
6304 unsigned PreferredFusedOpcode =
6305 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6306
6307 // If we have two choices trying to fold (fadd (fmul u, v), (fmul x, y)),
6308 // prefer to fold the multiply with fewer uses.
6309 if (Aggressive && isContractableFMul(*LHS.MI, AllowFusionGlobally) &&
6310 isContractableFMul(*RHS.MI, AllowFusionGlobally)) {
6311 if (hasMoreUses(*LHS.MI, *RHS.MI, MRI))
6312 std::swap(LHS, RHS);
6313 }
6314
6315 // fold (fadd (fmul x, y), z) -> (fma x, y, z)
6316 if (isContractableFMul(*LHS.MI, AllowFusionGlobally) &&
6317 (Aggressive || MRI.hasOneNonDBGUse(LHS.Reg))) {
6318 unsigned Flags = MI.getFlags() & LHS.MI->getFlags();
6319 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6320 B.buildInstr(PreferredFusedOpcode, {MI.getOperand(0).getReg()},
6321 {LHS.MI->getOperand(1).getReg(),
6322 LHS.MI->getOperand(2).getReg(), RHS.Reg},
6323 Flags);
6324 };
6325 return true;
6326 }
6327
6328 // fold (fadd x, (fmul y, z)) -> (fma y, z, x)
6329 if (isContractableFMul(*RHS.MI, AllowFusionGlobally) &&
6330 (Aggressive || MRI.hasOneNonDBGUse(RHS.Reg))) {
6331 unsigned Flags = MI.getFlags() & RHS.MI->getFlags();
6332 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6333 B.buildInstr(PreferredFusedOpcode, {MI.getOperand(0).getReg()},
6334 {RHS.MI->getOperand(1).getReg(),
6335 RHS.MI->getOperand(2).getReg(), LHS.Reg},
6336 Flags);
6337 };
6338 return true;
6339 }
6340
6341 return false;
6342}
6343
6346 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
6347 assert(MI.getOpcode() == TargetOpcode::G_FADD);
6348
6349 bool AllowFusionGlobally, HasFMAD, Aggressive;
6350 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive))
6351 return false;
6352
6353 const auto &TLI = *MI.getMF()->getSubtarget().getTargetLowering();
6354 Register Op1 = MI.getOperand(1).getReg();
6355 Register Op2 = MI.getOperand(2).getReg();
6356 MachineInstr *Op1Def, *Op2Def;
6357 if (!mi_match(Op1, MRI, m_MInstr(Op1Def)) ||
6358 !mi_match(Op2, MRI, m_MInstr(Op2Def)))
6359 return false;
6360 DefinitionAndSourceRegister LHS = {Op1Def, Op1};
6361 DefinitionAndSourceRegister RHS = {Op2Def, Op2};
6362 LLT DstType = MRI.getType(MI.getOperand(0).getReg());
6363
6364 unsigned PreferredFusedOpcode =
6365 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6366
6367 MachineInstr *LHSFpExtSrc;
6368 bool LHSContractable =
6369 mi_match(LHS.Reg, MRI, m_GFPExt(m_MInstr(LHSFpExtSrc))) &&
6370 isContractableFMul(*LHSFpExtSrc, AllowFusionGlobally) &&
6371 TLI.isFPExtFoldable(MI, PreferredFusedOpcode, DstType,
6372 MRI.getType(LHSFpExtSrc->getOperand(1).getReg()));
6373 MachineInstr *RHSFpExtSrc;
6374 bool RHSContractable =
6375 mi_match(RHS.Reg, MRI, m_GFPExt(m_MInstr(RHSFpExtSrc))) &&
6376 isContractableFMul(*RHSFpExtSrc, AllowFusionGlobally) &&
6377 TLI.isFPExtFoldable(MI, PreferredFusedOpcode, DstType,
6378 MRI.getType(RHSFpExtSrc->getOperand(1).getReg()));
6379
6380 // fold (fadd (fpext (fmul x, y)), z) -> (fma (fpext x), (fpext y), z)
6381 if (LHSContractable || RHSContractable) {
6382 // Ensure that the contractable fmul with the fewest uses (if both are
6383 // contractable) is the LHS operand.
6384 if (!LHSContractable ||
6385 (RHSContractable && hasMoreUses(*LHSFpExtSrc, *RHSFpExtSrc, MRI))) {
6386 std::swap(LHS, RHS);
6387 LHSFpExtSrc = RHSFpExtSrc;
6388 }
6389
6390 unsigned Flags = MI.getFlags() & LHSFpExtSrc->getFlags();
6391 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6392 auto FpExtX = B.buildFPExt(DstType, LHSFpExtSrc->getOperand(1).getReg());
6393 auto FpExtY = B.buildFPExt(DstType, LHSFpExtSrc->getOperand(2).getReg());
6394 B.buildInstr(PreferredFusedOpcode, {MI.getOperand(0).getReg()},
6395 {FpExtX.getReg(0), FpExtY.getReg(0), RHS.Reg}, Flags);
6396 };
6397 return true;
6398 }
6399
6400 return false;
6401}
6402
6405 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
6406 assert(MI.getOpcode() == TargetOpcode::G_FADD);
6407
6408 bool AllowFusionGlobally, HasFMAD, Aggressive;
6409 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive, true))
6410 return false;
6411
6412 Register Op1 = MI.getOperand(1).getReg();
6413 Register Op2 = MI.getOperand(2).getReg();
6414 MachineInstr *Op1Def, *Op2Def;
6415 if (!mi_match(Op1, MRI, m_MInstr(Op1Def)) ||
6416 !mi_match(Op2, MRI, m_MInstr(Op2Def)))
6417 return false;
6418 DefinitionAndSourceRegister LHS = {Op1Def, Op1};
6419 DefinitionAndSourceRegister RHS = {Op2Def, Op2};
6420 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
6421
6422 unsigned PreferredFusedOpcode =
6423 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6424
6425 MachineInstr *FMA = nullptr;
6426 Register Z;
6427 // fold (fadd (fma x, y, (fmul u, v)), z) -> (fma x, y, (fma u, v, z))
6428 if (LHS.MI->getOpcode() == PreferredFusedOpcode &&
6429 mi_match(LHS.MI->getOperand(3).getReg(), MRI,
6430 m_GFMul(m_Reg(), m_Reg())) &&
6431 MRI.hasOneNonDBGUse(LHS.MI->getOperand(0).getReg()) &&
6432 MRI.hasOneNonDBGUse(LHS.MI->getOperand(3).getReg())) {
6433 FMA = LHS.MI;
6434 Z = RHS.Reg;
6435 }
6436 // fold (fadd z, (fma x, y, (fmul u, v))) -> (fma x, y, (fma u, v, z))
6437 else if (RHS.MI->getOpcode() == PreferredFusedOpcode &&
6438 mi_match(RHS.MI->getOperand(3).getReg(), MRI,
6439 m_GFMul(m_Reg(), m_Reg())) &&
6440 MRI.hasOneNonDBGUse(RHS.MI->getOperand(0).getReg()) &&
6441 MRI.hasOneNonDBGUse(RHS.MI->getOperand(3).getReg())) {
6442 Z = LHS.Reg;
6443 FMA = RHS.MI;
6444 }
6445
6446 if (FMA) {
6447 MachineInstr *FMulMI;
6448 if (!mi_match(FMA->getOperand(3).getReg(), MRI, m_MInstr(FMulMI)))
6449 return false;
6450 Register X = FMA->getOperand(1).getReg();
6451 Register Y = FMA->getOperand(2).getReg();
6452 Register U = FMulMI->getOperand(1).getReg();
6453 Register V = FMulMI->getOperand(2).getReg();
6454 unsigned InnerFlags = MI.getFlags() & FMulMI->getFlags();
6455 unsigned OuterFlags = MI.getFlags() & FMA->getFlags();
6456
6457 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6458 Register InnerFMA = MRI.createGenericVirtualRegister(DstTy);
6459 B.buildInstr(PreferredFusedOpcode, {InnerFMA}, {U, V, Z}, InnerFlags);
6460 B.buildInstr(PreferredFusedOpcode, {MI.getOperand(0).getReg()},
6461 {X, Y, InnerFMA}, OuterFlags);
6462 };
6463 return true;
6464 }
6465
6466 return false;
6467}
6468
6471 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
6472 assert(MI.getOpcode() == TargetOpcode::G_FADD);
6473
6474 bool AllowFusionGlobally, HasFMAD, Aggressive;
6475 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive))
6476 return false;
6477
6478 if (!Aggressive)
6479 return false;
6480
6481 const auto &TLI = *MI.getMF()->getSubtarget().getTargetLowering();
6482 LLT DstType = MRI.getType(MI.getOperand(0).getReg());
6483 Register Op1 = MI.getOperand(1).getReg();
6484 Register Op2 = MI.getOperand(2).getReg();
6485 MachineInstr *Op1Def, *Op2Def;
6486 if (!mi_match(Op1, MRI, m_MInstr(Op1Def)) ||
6487 !mi_match(Op2, MRI, m_MInstr(Op2Def)))
6488 return false;
6489 DefinitionAndSourceRegister LHS = {Op1Def, Op1};
6490 DefinitionAndSourceRegister RHS = {Op2Def, Op2};
6491
6492 unsigned PreferredFusedOpcode =
6493 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6494
6495 // If we have two choices trying to fold (fadd (fmul u, v), (fmul x, y)),
6496 // prefer to fold the multiply with fewer uses.
6497 if (Aggressive && isContractableFMul(*LHS.MI, AllowFusionGlobally) &&
6498 isContractableFMul(*RHS.MI, AllowFusionGlobally)) {
6499 if (hasMoreUses(*LHS.MI, *RHS.MI, MRI))
6500 std::swap(LHS, RHS);
6501 }
6502
6503 // Builds: (fma x, y, (fma (fpext u), (fpext v), z))
6504 auto buildMatchInfo = [=, &MI](Register U, Register V, Register Z, Register X,
6505 Register Y, unsigned InnerFlags,
6506 unsigned OuterFlags, MachineIRBuilder &B) {
6507 Register FpExtU = B.buildFPExt(DstType, U).getReg(0);
6508 Register FpExtV = B.buildFPExt(DstType, V).getReg(0);
6509 Register InnerFMA = B.buildInstr(PreferredFusedOpcode, {DstType},
6510 {FpExtU, FpExtV, Z}, InnerFlags)
6511 .getReg(0);
6512 B.buildInstr(PreferredFusedOpcode, {MI.getOperand(0).getReg()},
6513 {X, Y, InnerFMA}, OuterFlags);
6514 };
6515
6516 MachineInstr *FMulMI, *FMAMI;
6517 // fold (fadd (fma x, y, (fpext (fmul u, v))), z)
6518 // -> (fma x, y, (fma (fpext u), (fpext v), z))
6519 if (LHS.MI->getOpcode() == PreferredFusedOpcode &&
6520 mi_match(LHS.MI->getOperand(3).getReg(), MRI,
6521 m_GFPExt(m_MInstr(FMulMI))) &&
6522 isContractableFMul(*FMulMI, AllowFusionGlobally) &&
6523 TLI.isFPExtFoldable(MI, PreferredFusedOpcode, DstType,
6524 MRI.getType(FMulMI->getOperand(0).getReg()))) {
6525 unsigned InnerFlags = MI.getFlags() & FMulMI->getFlags();
6526 unsigned OuterFlags = MI.getFlags() & LHS.MI->getFlags();
6527 MatchInfo = [=](MachineIRBuilder &B) {
6528 buildMatchInfo(FMulMI->getOperand(1).getReg(),
6529 FMulMI->getOperand(2).getReg(), RHS.Reg,
6530 LHS.MI->getOperand(1).getReg(),
6531 LHS.MI->getOperand(2).getReg(), InnerFlags, OuterFlags, B);
6532 };
6533 return true;
6534 }
6535
6536 // fold (fadd (fpext (fma x, y, (fmul u, v))), z)
6537 // -> (fma (fpext x), (fpext y), (fma (fpext u), (fpext v), z))
6538 // FIXME: This turns two single-precision and one double-precision
6539 // operation into two double-precision operations, which might not be
6540 // interesting for all targets, especially GPUs.
6541 if (mi_match(LHS.Reg, MRI, m_GFPExt(m_MInstr(FMAMI))) &&
6542 FMAMI->getOpcode() == PreferredFusedOpcode) {
6543 MachineInstr *FMulMI;
6544 if (!mi_match(FMAMI->getOperand(3).getReg(), MRI, m_MInstr(FMulMI)))
6545 return false;
6546 if (isContractableFMul(*FMulMI, AllowFusionGlobally) &&
6547 TLI.isFPExtFoldable(MI, PreferredFusedOpcode, DstType,
6548 MRI.getType(FMAMI->getOperand(0).getReg()))) {
6549 unsigned InnerFlags = MI.getFlags() & FMulMI->getFlags();
6550 unsigned OuterFlags = MI.getFlags() & FMAMI->getFlags();
6551 MatchInfo = [=](MachineIRBuilder &B) {
6552 Register X = FMAMI->getOperand(1).getReg();
6553 Register Y = FMAMI->getOperand(2).getReg();
6554 X = B.buildFPExt(DstType, X).getReg(0);
6555 Y = B.buildFPExt(DstType, Y).getReg(0);
6556 buildMatchInfo(FMulMI->getOperand(1).getReg(),
6557 FMulMI->getOperand(2).getReg(), RHS.Reg, X, Y,
6558 InnerFlags, OuterFlags, B);
6559 };
6560
6561 return true;
6562 }
6563 }
6564
6565 // fold (fadd z, (fma x, y, (fpext (fmul u, v)))
6566 // -> (fma x, y, (fma (fpext u), (fpext v), z))
6567 if (RHS.MI->getOpcode() == PreferredFusedOpcode &&
6568 mi_match(RHS.MI->getOperand(3).getReg(), MRI,
6569 m_GFPExt(m_MInstr(FMulMI))) &&
6570 isContractableFMul(*FMulMI, AllowFusionGlobally) &&
6571 TLI.isFPExtFoldable(MI, PreferredFusedOpcode, DstType,
6572 MRI.getType(FMulMI->getOperand(0).getReg()))) {
6573 unsigned InnerFlags = MI.getFlags() & FMulMI->getFlags();
6574 unsigned OuterFlags = MI.getFlags() & RHS.MI->getFlags();
6575 MatchInfo = [=](MachineIRBuilder &B) {
6576 buildMatchInfo(FMulMI->getOperand(1).getReg(),
6577 FMulMI->getOperand(2).getReg(), LHS.Reg,
6578 RHS.MI->getOperand(1).getReg(),
6579 RHS.MI->getOperand(2).getReg(), InnerFlags, OuterFlags, B);
6580 };
6581 return true;
6582 }
6583
6584 // fold (fadd z, (fpext (fma x, y, (fmul u, v)))
6585 // -> (fma (fpext x), (fpext y), (fma (fpext u), (fpext v), z))
6586 // FIXME: This turns two single-precision and one double-precision
6587 // operation into two double-precision operations, which might not be
6588 // interesting for all targets, especially GPUs.
6589 if (mi_match(RHS.Reg, MRI, m_GFPExt(m_MInstr(FMAMI))) &&
6590 FMAMI->getOpcode() == PreferredFusedOpcode) {
6591 MachineInstr *FMulMI;
6592 if (!mi_match(FMAMI->getOperand(3).getReg(), MRI, m_MInstr(FMulMI)))
6593 return false;
6594 if (isContractableFMul(*FMulMI, AllowFusionGlobally) &&
6595 TLI.isFPExtFoldable(MI, PreferredFusedOpcode, DstType,
6596 MRI.getType(FMAMI->getOperand(0).getReg()))) {
6597 unsigned InnerFlags = MI.getFlags() & FMulMI->getFlags();
6598 unsigned OuterFlags = MI.getFlags() & FMAMI->getFlags();
6599 MatchInfo = [=](MachineIRBuilder &B) {
6600 Register X = FMAMI->getOperand(1).getReg();
6601 Register Y = FMAMI->getOperand(2).getReg();
6602 X = B.buildFPExt(DstType, X).getReg(0);
6603 Y = B.buildFPExt(DstType, Y).getReg(0);
6604 buildMatchInfo(FMulMI->getOperand(1).getReg(),
6605 FMulMI->getOperand(2).getReg(), LHS.Reg, X, Y,
6606 InnerFlags, OuterFlags, B);
6607 };
6608 return true;
6609 }
6610 }
6611
6612 return false;
6613}
6614
6617 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
6618 assert(MI.getOpcode() == TargetOpcode::G_FSUB);
6619
6620 bool AllowFusionGlobally, HasFMAD, Aggressive;
6621 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive))
6622 return false;
6623
6624 Register Op1 = MI.getOperand(1).getReg();
6625 Register Op2 = MI.getOperand(2).getReg();
6626 MachineInstr *Op1Def, *Op2Def;
6627 if (!mi_match(Op1, MRI, m_MInstr(Op1Def)) ||
6628 !mi_match(Op2, MRI, m_MInstr(Op2Def)))
6629 return false;
6630 DefinitionAndSourceRegister LHS = {Op1Def, Op1};
6631 DefinitionAndSourceRegister RHS = {Op2Def, Op2};
6632 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
6633
6634 // If we have two choices trying to fold (fsub (fmul u, v), (fmul x, y)),
6635 // prefer to fold the multiply with fewer uses.
6636 int FirstMulHasFewerUses = true;
6637 if (isContractableFMul(*LHS.MI, AllowFusionGlobally) &&
6638 isContractableFMul(*RHS.MI, AllowFusionGlobally) &&
6639 hasMoreUses(*LHS.MI, *RHS.MI, MRI))
6640 FirstMulHasFewerUses = false;
6641
6642 unsigned PreferredFusedOpcode =
6643 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6644
6645 // fold (fsub (fmul x, y), z) -> (fma x, y, -z)
6646 if (FirstMulHasFewerUses &&
6647 (isContractableFMul(*LHS.MI, AllowFusionGlobally) &&
6648 (Aggressive || MRI.hasOneNonDBGUse(LHS.Reg)))) {
6649 unsigned Flags = MI.getFlags() & LHS.MI->getFlags();
6650 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6651 Register NegZ = B.buildFNeg(DstTy, RHS.Reg).getReg(0);
6652 B.buildInstr(PreferredFusedOpcode, {MI.getOperand(0).getReg()},
6653 {LHS.MI->getOperand(1).getReg(),
6654 LHS.MI->getOperand(2).getReg(), NegZ},
6655 Flags);
6656 };
6657 return true;
6658 }
6659 // fold (fsub x, (fmul y, z)) -> (fma -y, z, x)
6660 else if ((isContractableFMul(*RHS.MI, AllowFusionGlobally) &&
6661 (Aggressive || MRI.hasOneNonDBGUse(RHS.Reg)))) {
6662 unsigned Flags = MI.getFlags() & RHS.MI->getFlags();
6663 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6664 Register NegY =
6665 B.buildFNeg(DstTy, RHS.MI->getOperand(1).getReg()).getReg(0);
6666 B.buildInstr(PreferredFusedOpcode, {MI.getOperand(0).getReg()},
6667 {NegY, RHS.MI->getOperand(2).getReg(), LHS.Reg}, Flags);
6668 };
6669 return true;
6670 }
6671
6672 return false;
6673}
6674
6677 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
6678 assert(MI.getOpcode() == TargetOpcode::G_FSUB);
6679
6680 bool AllowFusionGlobally, HasFMAD, Aggressive;
6681 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive))
6682 return false;
6683
6684 Register LHSReg = MI.getOperand(1).getReg();
6685 Register RHSReg = MI.getOperand(2).getReg();
6686 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
6687
6688 unsigned PreferredFusedOpcode =
6689 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6690
6691 MachineInstr *FMulMI;
6692 // fold (fsub (fneg (fmul x, y)), z) -> (fma (fneg x), y, (fneg z))
6693 if (mi_match(LHSReg, MRI, m_GFNeg(m_MInstr(FMulMI))) &&
6694 (Aggressive || (MRI.hasOneNonDBGUse(LHSReg) &&
6695 MRI.hasOneNonDBGUse(FMulMI->getOperand(0).getReg()))) &&
6696 isContractableFMul(*FMulMI, AllowFusionGlobally)) {
6697 unsigned Flags = MI.getFlags() & FMulMI->getFlags();
6698 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6699 Register NegX =
6700 B.buildFNeg(DstTy, FMulMI->getOperand(1).getReg()).getReg(0);
6701 Register NegZ = B.buildFNeg(DstTy, RHSReg).getReg(0);
6702 B.buildInstr(PreferredFusedOpcode, {MI.getOperand(0).getReg()},
6703 {NegX, FMulMI->getOperand(2).getReg(), NegZ}, Flags);
6704 };
6705 return true;
6706 }
6707
6708 // fold (fsub x, (fneg (fmul, y, z))) -> (fma y, z, x)
6709 if (mi_match(RHSReg, MRI, m_GFNeg(m_MInstr(FMulMI))) &&
6710 (Aggressive || (MRI.hasOneNonDBGUse(RHSReg) &&
6711 MRI.hasOneNonDBGUse(FMulMI->getOperand(0).getReg()))) &&
6712 isContractableFMul(*FMulMI, AllowFusionGlobally)) {
6713 unsigned Flags = MI.getFlags() & FMulMI->getFlags();
6714 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6715 B.buildInstr(PreferredFusedOpcode, {MI.getOperand(0).getReg()},
6716 {FMulMI->getOperand(1).getReg(),
6717 FMulMI->getOperand(2).getReg(), LHSReg},
6718 Flags);
6719 };
6720 return true;
6721 }
6722
6723 return false;
6724}
6725
6728 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
6729 assert(MI.getOpcode() == TargetOpcode::G_FSUB);
6730
6731 bool AllowFusionGlobally, HasFMAD, Aggressive;
6732 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive))
6733 return false;
6734
6735 Register LHSReg = MI.getOperand(1).getReg();
6736 Register RHSReg = MI.getOperand(2).getReg();
6737 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
6738
6739 unsigned PreferredFusedOpcode =
6740 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6741
6742 MachineInstr *FMulMI;
6743 // fold (fsub (fpext (fmul x, y)), z) -> (fma (fpext x), (fpext y), (fneg z))
6744 if (mi_match(LHSReg, MRI, m_GFPExt(m_MInstr(FMulMI))) &&
6745 isContractableFMul(*FMulMI, AllowFusionGlobally) &&
6746 (Aggressive || MRI.hasOneNonDBGUse(LHSReg))) {
6747 unsigned Flags = MI.getFlags() & FMulMI->getFlags();
6748 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6749 Register FpExtX =
6750 B.buildFPExt(DstTy, FMulMI->getOperand(1).getReg()).getReg(0);
6751 Register FpExtY =
6752 B.buildFPExt(DstTy, FMulMI->getOperand(2).getReg()).getReg(0);
6753 Register NegZ = B.buildFNeg(DstTy, RHSReg).getReg(0);
6754 B.buildInstr(PreferredFusedOpcode, {MI.getOperand(0).getReg()},
6755 {FpExtX, FpExtY, NegZ}, Flags);
6756 };
6757 return true;
6758 }
6759
6760 // fold (fsub x, (fpext (fmul y, z))) -> (fma (fneg (fpext y)), (fpext z), x)
6761 if (mi_match(RHSReg, MRI, m_GFPExt(m_MInstr(FMulMI))) &&
6762 isContractableFMul(*FMulMI, AllowFusionGlobally) &&
6763 (Aggressive || MRI.hasOneNonDBGUse(RHSReg))) {
6764 unsigned Flags = MI.getFlags() & FMulMI->getFlags();
6765 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6766 Register FpExtY =
6767 B.buildFPExt(DstTy, FMulMI->getOperand(1).getReg()).getReg(0);
6768 Register NegY = B.buildFNeg(DstTy, FpExtY).getReg(0);
6769 Register FpExtZ =
6770 B.buildFPExt(DstTy, FMulMI->getOperand(2).getReg()).getReg(0);
6771 B.buildInstr(PreferredFusedOpcode, {MI.getOperand(0).getReg()},
6772 {NegY, FpExtZ, LHSReg}, Flags);
6773 };
6774 return true;
6775 }
6776
6777 return false;
6778}
6779
6782 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
6783 assert(MI.getOpcode() == TargetOpcode::G_FSUB);
6784
6785 bool AllowFusionGlobally, HasFMAD, Aggressive;
6786 if (!canCombineFMadOrFMA(MI, AllowFusionGlobally, HasFMAD, Aggressive))
6787 return false;
6788
6789 const auto &TLI = *MI.getMF()->getSubtarget().getTargetLowering();
6790 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
6791 Register LHSReg = MI.getOperand(1).getReg();
6792 Register RHSReg = MI.getOperand(2).getReg();
6793
6794 unsigned PreferredFusedOpcode =
6795 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6796
6797 auto buildMatchInfo = [=](Register Dst, Register X, Register Y, Register Z,
6798 unsigned Flags, MachineIRBuilder &B) {
6799 Register FpExtX = B.buildFPExt(DstTy, X).getReg(0);
6800 Register FpExtY = B.buildFPExt(DstTy, Y).getReg(0);
6801 B.buildInstr(PreferredFusedOpcode, {Dst}, {FpExtX, FpExtY, Z}, Flags);
6802 };
6803
6804 MachineInstr *FMulMI;
6805 // fold (fsub (fpext (fneg (fmul x, y))), z) ->
6806 // (fneg (fma (fpext x), (fpext y), z))
6807 // fold (fsub (fneg (fpext (fmul x, y))), z) ->
6808 // (fneg (fma (fpext x), (fpext y), z))
6809 if ((mi_match(LHSReg, MRI, m_GFPExt(m_GFNeg(m_MInstr(FMulMI)))) ||
6810 mi_match(LHSReg, MRI, m_GFNeg(m_GFPExt(m_MInstr(FMulMI))))) &&
6811 isContractableFMul(*FMulMI, AllowFusionGlobally) &&
6812 TLI.isFPExtFoldable(MI, PreferredFusedOpcode, DstTy,
6813 MRI.getType(FMulMI->getOperand(0).getReg()))) {
6814 unsigned Flags = MI.getFlags() & FMulMI->getFlags();
6815 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6816 Register FMAReg = MRI.createGenericVirtualRegister(DstTy);
6817 buildMatchInfo(FMAReg, FMulMI->getOperand(1).getReg(),
6818 FMulMI->getOperand(2).getReg(), RHSReg, Flags, B);
6819 B.buildFNeg(MI.getOperand(0).getReg(), FMAReg);
6820 };
6821 return true;
6822 }
6823
6824 // fold (fsub x, (fpext (fneg (fmul y, z)))) -> (fma (fpext y), (fpext z), x)
6825 // fold (fsub x, (fneg (fpext (fmul y, z)))) -> (fma (fpext y), (fpext z), x)
6826 if ((mi_match(RHSReg, MRI, m_GFPExt(m_GFNeg(m_MInstr(FMulMI)))) ||
6827 mi_match(RHSReg, MRI, m_GFNeg(m_GFPExt(m_MInstr(FMulMI))))) &&
6828 isContractableFMul(*FMulMI, AllowFusionGlobally) &&
6829 TLI.isFPExtFoldable(MI, PreferredFusedOpcode, DstTy,
6830 MRI.getType(FMulMI->getOperand(0).getReg()))) {
6831 unsigned Flags = MI.getFlags() & FMulMI->getFlags();
6832 MatchInfo = [=, &MI](MachineIRBuilder &B) {
6833 buildMatchInfo(MI.getOperand(0).getReg(), FMulMI->getOperand(1).getReg(),
6834 FMulMI->getOperand(2).getReg(), LHSReg, Flags, B);
6835 };
6836 return true;
6837 }
6838
6839 return false;
6840}
6841
6842// Combine multiple FDIVs with the same divisor into multiple FMULs by the
6843// reciprocal.
6844// E.g., (a / Y; b / Y;) -> (recip = 1.0 / Y; a * recip; b * recip)
6846 MachineInstr &MI, SmallVector<MachineInstr *> &MatchInfo) const {
6847 assert(MI.getOpcode() == TargetOpcode::G_FDIV);
6848
6849 Register X = MI.getOperand(1).getReg();
6850 Register Y = MI.getOperand(2).getReg();
6851
6852 if (!MI.getFlag(MachineInstr::MIFlag::FmArcp))
6853 return false;
6854
6855 auto IsOne = [this](Register X) {
6857 return N0CFP && (N0CFP->isOne() || N0CFP->isMinusOne());
6858 };
6859
6860 // Skip if current node is a reciprocal/fneg-reciprocal.
6861 if (IsOne(X))
6862 return false;
6863
6864 // Exit early if the target does not want this transform or if there can't
6865 // possibly be enough uses of the divisor to make the transform worthwhile.
6866 unsigned MinUses = getTargetLowering().combineRepeatedFPDivisors();
6867 if (!MinUses)
6868 return false;
6869
6870 // Find all FDIV users of the same divisor. For the moment we limit all
6871 // instructions to a single BB and use the first Instr in MatchInfo as the
6872 // dominating position.
6873 MatchInfo.push_back(&MI);
6874 for (auto &U : MRI.use_nodbg_instructions(Y)) {
6875 if (&U == &MI || U.getParent() != MI.getParent())
6876 continue;
6877 if (U.getOpcode() == TargetOpcode::G_FDIV &&
6878 U.getOperand(2).getReg() == Y && U.getOperand(1).getReg() != Y &&
6879 !IsOne(U.getOperand(1).getReg())) {
6880 // This division is eligible for optimization only if global unsafe math
6881 // is enabled or if this division allows reciprocal formation.
6882 if (U.getFlag(MachineInstr::MIFlag::FmArcp)) {
6883 MatchInfo.push_back(&U);
6884 if (dominates(U, *MatchInfo[0]))
6885 std::swap(MatchInfo[0], MatchInfo.back());
6886 }
6887 }
6888 }
6889
6890 // Now that we have the actual number of divisor uses, make sure it meets
6891 // the minimum threshold specified by the target.
6892 return MatchInfo.size() >= MinUses;
6893}
6894
6896 SmallVector<MachineInstr *> &MatchInfo) const {
6897 // Generate the new div at the position of the first instruction, that we have
6898 // ensured will dominate all other instructions.
6899 Builder.setInsertPt(*MatchInfo[0]->getParent(), MatchInfo[0]);
6900 LLT Ty = MRI.getType(MatchInfo[0]->getOperand(0).getReg());
6901 auto Div = Builder.buildFDiv(Ty, Builder.buildFConstant(Ty, 1.0),
6902 MatchInfo[0]->getOperand(2).getReg(),
6903 MatchInfo[0]->getFlags());
6904
6905 // Replace all found div's with fmul instructions.
6906 for (MachineInstr *MI : MatchInfo) {
6907 Builder.setInsertPt(*MI->getParent(), MI);
6908 Builder.buildFMul(MI->getOperand(0).getReg(), MI->getOperand(1).getReg(),
6909 Div->getOperand(0).getReg(), MI->getFlags());
6910 MI->eraseFromParent();
6911 }
6912}
6913
6915 Register &MatchInfo) const {
6916 // This combine folds the following patterns:
6917 //
6918 // G_BUILD_VECTOR_TRUNC (G_BITCAST(x), G_LSHR(G_BITCAST(x), k))
6919 // G_BUILD_VECTOR(G_TRUNC(G_BITCAST(x)), G_TRUNC(G_LSHR(G_BITCAST(x), k)))
6920 // into
6921 // x
6922 // if
6923 // k == sizeof(VecEltTy)/2
6924 // type(x) == type(dst)
6925 //
6926 // G_BUILD_VECTOR(G_TRUNC(G_BITCAST(x)), undef)
6927 // into
6928 // x
6929 // if
6930 // type(x) == type(dst)
6931
6932 LLT DstVecTy = MRI.getType(MI.getOperand(0).getReg());
6933 LLT DstEltTy = DstVecTy.getElementType();
6934
6935 Register Lo, Hi;
6936
6937 if (mi_match(
6938 MI, MRI,
6940 MatchInfo = Lo;
6941 return MRI.getType(MatchInfo) == DstVecTy;
6942 }
6943
6944 std::optional<ValueAndVReg> ShiftAmount;
6945 const auto LoPattern = m_GBitcast(m_Reg(Lo));
6946 const auto HiPattern = m_GLShr(m_GBitcast(m_Reg(Hi)), m_GCst(ShiftAmount));
6947 if (mi_match(
6948 MI, MRI,
6949 m_any_of(m_GBuildVectorTrunc(LoPattern, HiPattern),
6950 m_GBuildVector(m_GTrunc(LoPattern), m_GTrunc(HiPattern))))) {
6951 if (Lo == Hi && ShiftAmount->Value == DstEltTy.getSizeInBits()) {
6952 MatchInfo = Lo;
6953 return MRI.getType(MatchInfo) == DstVecTy;
6954 }
6955 }
6956
6957 return false;
6958}
6959
6961 Register &MatchInfo) const {
6962 // Replace (G_TRUNC (G_BITCAST (G_BUILD_VECTOR x, y)) with just x
6963 // if type(x) == type(G_TRUNC)
6964 if (!mi_match(MI.getOperand(1).getReg(), MRI,
6965 m_GBitcast(m_GBuildVector(m_Reg(MatchInfo), m_Reg()))))
6966 return false;
6967
6968 return MRI.getType(MatchInfo) == MRI.getType(MI.getOperand(0).getReg());
6969}
6970
6972 Register &MatchInfo) const {
6973 // Replace (G_TRUNC (G_LSHR (G_BITCAST (G_BUILD_VECTOR x, y)), K)) with
6974 // y if K == size of vector element type
6975 std::optional<ValueAndVReg> ShiftAmt;
6976 if (!mi_match(MI.getOperand(1).getReg(), MRI,
6978 m_GCst(ShiftAmt))))
6979 return false;
6980
6981 LLT MatchTy = MRI.getType(MatchInfo);
6982 return ShiftAmt->Value.getZExtValue() == MatchTy.getSizeInBits() &&
6983 MatchTy == MRI.getType(MI.getOperand(0).getReg());
6984}
6985
6986unsigned CombinerHelper::getFPMinMaxOpcForSelect(
6987 CmpInst::Predicate Pred, LLT DstTy,
6988 SelectPatternNaNBehaviour VsNaNRetVal) const {
6989 assert(VsNaNRetVal != SelectPatternNaNBehaviour::NOT_APPLICABLE &&
6990 "Expected a NaN behaviour?");
6991 // Choose an opcode based off of legality or the behaviour when one of the
6992 // LHS/RHS may be NaN.
6993 switch (Pred) {
6994 default:
6995 return 0;
6996 case CmpInst::FCMP_UGT:
6997 case CmpInst::FCMP_UGE:
6998 case CmpInst::FCMP_OGT:
6999 case CmpInst::FCMP_OGE:
7000 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_OTHER)
7001 return TargetOpcode::G_FMAXNUM;
7002 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_NAN)
7003 return TargetOpcode::G_FMAXIMUM;
7004 if (isLegal({TargetOpcode::G_FMAXNUM, {DstTy}}))
7005 return TargetOpcode::G_FMAXNUM;
7006 if (isLegal({TargetOpcode::G_FMAXIMUM, {DstTy}}))
7007 return TargetOpcode::G_FMAXIMUM;
7008 return 0;
7009 case CmpInst::FCMP_ULT:
7010 case CmpInst::FCMP_ULE:
7011 case CmpInst::FCMP_OLT:
7012 case CmpInst::FCMP_OLE:
7013 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_OTHER)
7014 return TargetOpcode::G_FMINNUM;
7015 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_NAN)
7016 return TargetOpcode::G_FMINIMUM;
7017 if (isLegal({TargetOpcode::G_FMINNUM, {DstTy}}))
7018 return TargetOpcode::G_FMINNUM;
7019 if (!isLegal({TargetOpcode::G_FMINIMUM, {DstTy}}))
7020 return 0;
7021 return TargetOpcode::G_FMINIMUM;
7022 }
7023}
7024
7025CombinerHelper::SelectPatternNaNBehaviour
7026CombinerHelper::computeRetValAgainstNaN(Register LHS, Register RHS,
7027 bool IsOrderedComparison) const {
7028 bool LHSSafe = VT->isKnownNeverNaN(LHS);
7029 bool RHSSafe = VT->isKnownNeverNaN(RHS);
7030 // Completely unsafe.
7031 if (!LHSSafe && !RHSSafe)
7032 return SelectPatternNaNBehaviour::NOT_APPLICABLE;
7033 if (LHSSafe && RHSSafe)
7034 return SelectPatternNaNBehaviour::RETURNS_ANY;
7035 // An ordered comparison will return false when given a NaN, so it
7036 // returns the RHS.
7037 if (IsOrderedComparison)
7038 return LHSSafe ? SelectPatternNaNBehaviour::RETURNS_NAN
7039 : SelectPatternNaNBehaviour::RETURNS_OTHER;
7040 // An unordered comparison will return true when given a NaN, so it
7041 // returns the LHS.
7042 return LHSSafe ? SelectPatternNaNBehaviour::RETURNS_OTHER
7043 : SelectPatternNaNBehaviour::RETURNS_NAN;
7044}
7045
7046bool CombinerHelper::matchFPSelectToMinMax(Register Dst, Register Cond,
7047 Register TrueVal, Register FalseVal,
7048 BuildFnTy &MatchInfo) const {
7049 // Match: select (fcmp cond x, y) x, y
7050 // select (fcmp cond x, y) y, x
7051 // And turn it into fminnum/fmaxnum or fmin/fmax based off of the condition.
7052 LLT DstTy = MRI.getType(Dst);
7053 // Bail out early on pointers, since we'll never want to fold to a min/max.
7054 if (DstTy.isPointer())
7055 return false;
7056 // Match a floating point compare with a less-than/greater-than predicate.
7057 // TODO: Allow multiple users of the compare if they are all selects.
7059 Register CmpLHS, CmpRHS;
7060 if (!mi_match(
7061 Cond, MRI,
7062 m_OneUse(m_GFCmp(m_Pred(Pred), m_Reg(CmpLHS), m_Reg(CmpRHS)))) ||
7064 return false;
7065 SelectPatternNaNBehaviour ResWithKnownNaNInfo =
7066 computeRetValAgainstNaN(CmpLHS, CmpRHS, CmpInst::isOrdered(Pred));
7067 if (ResWithKnownNaNInfo == SelectPatternNaNBehaviour::NOT_APPLICABLE)
7068 return false;
7069 if (TrueVal == CmpRHS && FalseVal == CmpLHS) {
7070 std::swap(CmpLHS, CmpRHS);
7072 if (ResWithKnownNaNInfo == SelectPatternNaNBehaviour::RETURNS_NAN)
7073 ResWithKnownNaNInfo = SelectPatternNaNBehaviour::RETURNS_OTHER;
7074 else if (ResWithKnownNaNInfo == SelectPatternNaNBehaviour::RETURNS_OTHER)
7075 ResWithKnownNaNInfo = SelectPatternNaNBehaviour::RETURNS_NAN;
7076 }
7077 if (TrueVal != CmpLHS || FalseVal != CmpRHS)
7078 return false;
7079 // Decide what type of max/min this should be based off of the predicate.
7080 unsigned Opc = getFPMinMaxOpcForSelect(Pred, DstTy, ResWithKnownNaNInfo);
7081 if (!Opc || !isLegal({Opc, {DstTy}}))
7082 return false;
7083 // Comparisons between signed zero and zero may have different results...
7084 // unless we have fmaximum/fminimum. In that case, we know -0 < 0.
7085 if (Opc != TargetOpcode::G_FMAXIMUM && Opc != TargetOpcode::G_FMINIMUM) {
7086 // We don't know if a comparison between two 0s will give us a consistent
7087 // result. Be conservative and only proceed if at least one side is
7088 // non-zero.
7089 auto KnownNonZeroSide = getFConstantVRegValWithLookThrough(CmpLHS, MRI);
7090 if (!KnownNonZeroSide || !KnownNonZeroSide->Value.isNonZero()) {
7091 KnownNonZeroSide = getFConstantVRegValWithLookThrough(CmpRHS, MRI);
7092 if (!KnownNonZeroSide || !KnownNonZeroSide->Value.isNonZero())
7093 return false;
7094 }
7095 }
7096 MatchInfo = [=](MachineIRBuilder &B) {
7097 B.buildInstr(Opc, {Dst}, {CmpLHS, CmpRHS});
7098 };
7099 return true;
7100}
7101
7103 BuildFnTy &MatchInfo) const {
7104 // TODO: Handle integer cases.
7105 assert(MI.getOpcode() == TargetOpcode::G_SELECT);
7106 // Condition may be fed by a truncated compare.
7107 Register Cond = MI.getOperand(1).getReg();
7108 Register MaybeTrunc;
7109 if (mi_match(Cond, MRI, m_OneUse(m_GTrunc(m_Reg(MaybeTrunc)))))
7110 Cond = MaybeTrunc;
7111 Register Dst = MI.getOperand(0).getReg();
7112 Register TrueVal = MI.getOperand(2).getReg();
7113 Register FalseVal = MI.getOperand(3).getReg();
7114 return matchFPSelectToMinMax(Dst, Cond, TrueVal, FalseVal, MatchInfo);
7115}
7116
7118 BuildFnTy &MatchInfo) const {
7119 assert(MI.getOpcode() == TargetOpcode::G_ICMP);
7120 // (X + Y) == X --> Y == 0
7121 // (X + Y) != X --> Y != 0
7122 // (X - Y) == X --> Y == 0
7123 // (X - Y) != X --> Y != 0
7124 // (X ^ Y) == X --> Y == 0
7125 // (X ^ Y) != X --> Y != 0
7126 Register Dst = MI.getOperand(0).getReg();
7127 CmpInst::Predicate Pred;
7128 Register X, Y, OpLHS, OpRHS;
7129 bool MatchedSub = mi_match(
7130 Dst, MRI,
7131 m_c_GICmp(m_Pred(Pred), m_Reg(X), m_GSub(m_Reg(OpLHS), m_Reg(Y))));
7132 if (MatchedSub && X != OpLHS)
7133 return false;
7134 if (!MatchedSub) {
7135 if (!mi_match(Dst, MRI,
7136 m_c_GICmp(m_Pred(Pred), m_Reg(X),
7137 m_any_of(m_GAdd(m_Reg(OpLHS), m_Reg(OpRHS)),
7138 m_GXor(m_Reg(OpLHS), m_Reg(OpRHS))))))
7139 return false;
7140 Y = X == OpLHS ? OpRHS : X == OpRHS ? OpLHS : Register();
7141 }
7142 MatchInfo = [=](MachineIRBuilder &B) {
7143 auto Zero = B.buildConstant(MRI.getType(Y), 0);
7144 B.buildICmp(Pred, Dst, Y, Zero);
7145 };
7146 return CmpInst::isEquality(Pred) && Y.isValid();
7147}
7148
7149/// Return the minimum useless shift amount that results in complete loss of the
7150/// source value. Return std::nullopt when it cannot determine a value.
7151static std::optional<unsigned>
7152getMinUselessShift(KnownBits ValueKB, unsigned Opcode,
7153 std::optional<int64_t> &Result) {
7154 assert((Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_LSHR ||
7155 Opcode == TargetOpcode::G_ASHR) &&
7156 "Expect G_SHL, G_LSHR or G_ASHR.");
7157 auto SignificantBits = 0;
7158 switch (Opcode) {
7159 case TargetOpcode::G_SHL:
7160 SignificantBits = ValueKB.countMinTrailingZeros();
7161 Result = 0;
7162 break;
7163 case TargetOpcode::G_LSHR:
7164 Result = 0;
7165 SignificantBits = ValueKB.countMinLeadingZeros();
7166 break;
7167 case TargetOpcode::G_ASHR:
7168 if (ValueKB.isNonNegative()) {
7169 SignificantBits = ValueKB.countMinLeadingZeros();
7170 Result = 0;
7171 } else if (ValueKB.isNegative()) {
7172 SignificantBits = ValueKB.countMinLeadingOnes();
7173 Result = -1;
7174 } else {
7175 // Cannot determine shift result.
7176 Result = std::nullopt;
7177 }
7178 break;
7179 default:
7180 break;
7181 }
7182 return ValueKB.getBitWidth() - SignificantBits;
7183}
7184
7186 BuildFnTy &MatchInfo) const {
7187 Register Dst = MI.getOperand(0).getReg();
7188 Register ShiftVal = MI.getOperand(1).getReg();
7189 Register ShiftReg = MI.getOperand(2).getReg();
7190 LLT ResTy = MRI.getType(Dst);
7191 std::optional<int64_t> Result;
7192 auto IsShiftTooBig = [&](const Constant *C) {
7193 auto *CI = dyn_cast<ConstantInt>(C);
7194 if (!CI)
7195 return false;
7196 if (CI->uge(ResTy.getScalarSizeInBits())) {
7197 Result = std::nullopt;
7198 return true;
7199 }
7200 auto OptMaxUsefulShift =
7201 getMinUselessShift(VT->getKnownBits(ShiftVal), MI.getOpcode(), Result);
7202 return OptMaxUsefulShift && CI->uge(*OptMaxUsefulShift);
7203 };
7204 if (!matchUnaryPredicate(MRI, ShiftReg, IsShiftTooBig))
7205 return false;
7206
7207 if (!Result) {
7208 MatchInfo = [Dst](MachineIRBuilder &B) { B.buildUndef(Dst); };
7209 return true;
7210 }
7211
7212 int64_t Val = *Result;
7213 MatchInfo = [Dst, Val](MachineIRBuilder &B) { B.buildConstant(Dst, Val); };
7214 return true;
7215}
7216
7218 unsigned LHSOpndIdx = 1;
7219 unsigned RHSOpndIdx = 2;
7220 switch (MI.getOpcode()) {
7221 case TargetOpcode::G_UADDO:
7222 case TargetOpcode::G_SADDO:
7223 case TargetOpcode::G_UMULO:
7224 case TargetOpcode::G_SMULO:
7225 LHSOpndIdx = 2;
7226 RHSOpndIdx = 3;
7227 break;
7228 default:
7229 break;
7230 }
7231 Register LHS = MI.getOperand(LHSOpndIdx).getReg();
7232 Register RHS = MI.getOperand(RHSOpndIdx).getReg();
7233 MachineInstr *LHSDef, *RHSDef;
7234 if (!mi_match(LHS, MRI, m_MInstr(LHSDef)) ||
7235 !mi_match(RHS, MRI, m_MInstr(RHSDef)))
7236 return false;
7237
7238 if (!isConstantOrConstantVector(*LHSDef, MRI, /*AllowFP=*/false,
7239 /*AllowOpaqueConstants=*/false)) {
7240 // Skip commuting if LHS is not a constant or a vector of constants. But,
7241 // LHS may be a G_CONSTANT_FOLD_BARRIER. If so we commute as long as we
7242 // don't already have a constant on the RHS.
7243 if (LHSDef->getOpcode() != TargetOpcode::G_CONSTANT_FOLD_BARRIER)
7244 return false;
7245 }
7246 // Commute as long as RHS is not a constant, a vector of constants or
7247 // G_CONSTANT_FOLD_BARRIER.
7248 return RHSDef->getOpcode() != TargetOpcode::G_CONSTANT_FOLD_BARRIER &&
7249 !isConstantOrConstantVector(*RHSDef, MRI, /*AllowFP=*/false,
7250 /*AllowOpaqueConstants=*/false);
7251}
7252
7254 Register LHS = MI.getOperand(1).getReg();
7255 Register RHS = MI.getOperand(2).getReg();
7256 std::optional<FPValueAndVReg> ValAndVReg;
7257 if (!mi_match(LHS, MRI, m_GFCstOrSplat(ValAndVReg)))
7258 return false;
7259 return !mi_match(RHS, MRI, m_GFCstOrSplat(ValAndVReg));
7260}
7261
7262bool CombinerHelper::isOneOrOneSplat(Register Src, bool AllowUndefs) const {
7263 LLT SrcTy = MRI.getType(Src);
7264 if (SrcTy.isFixedVector())
7265 return isConstantSplatVector(Src, 1, AllowUndefs);
7266 if (SrcTy.isScalar()) {
7267 if (AllowUndefs && getOpcodeDef<GImplicitDef>(Src, MRI) != nullptr)
7268 return true;
7269 auto IConstant = getIConstantVRegValWithLookThrough(Src, MRI);
7270 return IConstant && IConstant->Value == 1;
7271 }
7272 return false; // scalable vector
7273}
7274
7275bool CombinerHelper::isZeroOrZeroSplat(Register Src, bool AllowUndefs) const {
7276 LLT SrcTy = MRI.getType(Src);
7277 if (SrcTy.isFixedVector())
7278 return isConstantSplatVector(Src, 0, AllowUndefs);
7279 if (SrcTy.isScalar()) {
7280 if (AllowUndefs && getOpcodeDef<GImplicitDef>(Src, MRI) != nullptr)
7281 return true;
7282 auto IConstant = getIConstantVRegValWithLookThrough(Src, MRI);
7283 return IConstant && IConstant->Value == 0;
7284 }
7285 return false; // scalable vector
7286}
7287
7288// Ignores COPYs during conformance checks.
7289// FIXME scalable vectors.
7290bool CombinerHelper::isConstantSplatVector(Register Src, int64_t SplatValue,
7291 bool AllowUndefs) const {
7292 GBuildVector *BuildVector = getOpcodeDef<GBuildVector>(Src, MRI);
7293 if (!BuildVector)
7294 return false;
7295 unsigned NumSources = BuildVector->getNumSources();
7296
7297 for (unsigned I = 0; I < NumSources; ++I) {
7298 GImplicitDef *ImplicitDef =
7300 if (ImplicitDef && AllowUndefs)
7301 continue;
7302 if (ImplicitDef && !AllowUndefs)
7303 return false;
7304 std::optional<ValueAndVReg> IConstant =
7306 if (IConstant && IConstant->Value == SplatValue)
7307 continue;
7308 return false;
7309 }
7310 return true;
7311}
7312
7313// Ignores COPYs during lookups.
7314// FIXME scalable vectors
7315std::optional<APInt>
7316CombinerHelper::getConstantOrConstantSplatVector(Register Src) const {
7317 auto IConstant = getIConstantVRegValWithLookThrough(Src, MRI);
7318 if (IConstant)
7319 return IConstant->Value;
7320
7321 GBuildVector *BuildVector = getOpcodeDef<GBuildVector>(Src, MRI);
7322 if (!BuildVector)
7323 return std::nullopt;
7324 unsigned NumSources = BuildVector->getNumSources();
7325
7326 std::optional<APInt> Value = std::nullopt;
7327 for (unsigned I = 0; I < NumSources; ++I) {
7328 std::optional<ValueAndVReg> IConstant =
7330 if (!IConstant)
7331 return std::nullopt;
7332 if (!Value)
7333 Value = IConstant->Value;
7334 else if (*Value != IConstant->Value)
7335 return std::nullopt;
7336 }
7337 return Value;
7338}
7339
7340// FIXME G_SPLAT_VECTOR
7341bool CombinerHelper::isConstantOrConstantVectorI(Register Src) const {
7342 auto IConstant = getIConstantVRegValWithLookThrough(Src, MRI);
7343 if (IConstant)
7344 return true;
7345
7346 GBuildVector *BuildVector = getOpcodeDef<GBuildVector>(Src, MRI);
7347 if (!BuildVector)
7348 return false;
7349
7350 unsigned NumSources = BuildVector->getNumSources();
7351 for (unsigned I = 0; I < NumSources; ++I) {
7352 std::optional<ValueAndVReg> IConstant =
7354 if (!IConstant)
7355 return false;
7356 }
7357 return true;
7358}
7359
7360// TODO: use knownbits to determine zeros
7361bool CombinerHelper::tryFoldSelectOfConstants(GSelect *Select,
7362 BuildFnTy &MatchInfo) const {
7363 uint32_t Flags = Select->getFlags();
7364 Register Dest = Select->getReg(0);
7365 Register Cond = Select->getCondReg();
7366 Register True = Select->getTrueReg();
7367 Register False = Select->getFalseReg();
7368 LLT CondTy = MRI.getType(Select->getCondReg());
7369 LLT TrueTy = MRI.getType(Select->getTrueReg());
7370
7371 // We only do this combine for scalar boolean conditions.
7372 if (CondTy != LLT::scalar(1))
7373 return false;
7374
7375 if (TrueTy.isPointer())
7376 return false;
7377
7378 // Both are scalars.
7379 std::optional<ValueAndVReg> TrueOpt =
7381 std::optional<ValueAndVReg> FalseOpt =
7383
7384 if (!TrueOpt || !FalseOpt)
7385 return false;
7386
7387 APInt TrueValue = TrueOpt->Value;
7388 APInt FalseValue = FalseOpt->Value;
7389
7390 // select Cond, 1, 0 --> zext (Cond)
7391 if (TrueValue.isOne() && FalseValue.isZero()) {
7392 MatchInfo = [=](MachineIRBuilder &B) {
7393 B.setInstrAndDebugLoc(*Select);
7394 B.buildZExtOrTrunc(Dest, Cond);
7395 };
7396 return true;
7397 }
7398
7399 // select Cond, -1, 0 --> sext (Cond)
7400 if (TrueValue.isAllOnes() && FalseValue.isZero()) {
7401 MatchInfo = [=](MachineIRBuilder &B) {
7402 B.setInstrAndDebugLoc(*Select);
7403 B.buildSExtOrTrunc(Dest, Cond);
7404 };
7405 return true;
7406 }
7407
7408 // select Cond, 0, 1 --> zext (!Cond)
7409 if (TrueValue.isZero() && FalseValue.isOne()) {
7410 MatchInfo = [=](MachineIRBuilder &B) {
7411 B.setInstrAndDebugLoc(*Select);
7412 Register Inner = MRI.createGenericVirtualRegister(CondTy);
7413 B.buildNot(Inner, Cond);
7414 B.buildZExtOrTrunc(Dest, Inner);
7415 };
7416 return true;
7417 }
7418
7419 // select Cond, 0, -1 --> sext (!Cond)
7420 if (TrueValue.isZero() && FalseValue.isAllOnes()) {
7421 MatchInfo = [=](MachineIRBuilder &B) {
7422 B.setInstrAndDebugLoc(*Select);
7423 Register Inner = MRI.createGenericVirtualRegister(CondTy);
7424 B.buildNot(Inner, Cond);
7425 B.buildSExtOrTrunc(Dest, Inner);
7426 };
7427 return true;
7428 }
7429
7430 // select Cond, C1, C1-1 --> add (zext Cond), C1-1
7431 if (TrueValue - 1 == FalseValue) {
7432 MatchInfo = [=](MachineIRBuilder &B) {
7433 B.setInstrAndDebugLoc(*Select);
7434 Register Inner = MRI.createGenericVirtualRegister(TrueTy);
7435 B.buildZExtOrTrunc(Inner, Cond);
7436 B.buildAdd(Dest, Inner, False);
7437 };
7438 return true;
7439 }
7440
7441 // select Cond, C1, C1+1 --> add (sext Cond), C1+1
7442 if (TrueValue + 1 == FalseValue) {
7443 MatchInfo = [=](MachineIRBuilder &B) {
7444 B.setInstrAndDebugLoc(*Select);
7445 Register Inner = MRI.createGenericVirtualRegister(TrueTy);
7446 B.buildSExtOrTrunc(Inner, Cond);
7447 B.buildAdd(Dest, Inner, False);
7448 };
7449 return true;
7450 }
7451
7452 // select Cond, Pow2, 0 --> (zext Cond) << log2(Pow2)
7453 if (TrueValue.isPowerOf2() && FalseValue.isZero()) {
7454 MatchInfo = [=](MachineIRBuilder &B) {
7455 B.setInstrAndDebugLoc(*Select);
7456 Register Inner = MRI.createGenericVirtualRegister(TrueTy);
7457 B.buildZExtOrTrunc(Inner, Cond);
7458 // The shift amount must be scalar.
7459 LLT ShiftTy = TrueTy.isVector() ? TrueTy.getElementType() : TrueTy;
7460 auto ShAmtC = B.buildConstant(ShiftTy, TrueValue.exactLogBase2());
7461 B.buildShl(Dest, Inner, ShAmtC, Flags);
7462 };
7463 return true;
7464 }
7465
7466 // select Cond, 0, Pow2 --> (zext (!Cond)) << log2(Pow2)
7467 if (FalseValue.isPowerOf2() && TrueValue.isZero()) {
7468 MatchInfo = [=](MachineIRBuilder &B) {
7469 B.setInstrAndDebugLoc(*Select);
7470 Register Not = MRI.createGenericVirtualRegister(CondTy);
7471 B.buildNot(Not, Cond);
7472 Register Inner = MRI.createGenericVirtualRegister(TrueTy);
7473 B.buildZExtOrTrunc(Inner, Not);
7474 // The shift amount must be scalar.
7475 LLT ShiftTy = TrueTy.isVector() ? TrueTy.getElementType() : TrueTy;
7476 auto ShAmtC = B.buildConstant(ShiftTy, FalseValue.exactLogBase2());
7477 B.buildShl(Dest, Inner, ShAmtC, Flags);
7478 };
7479 return true;
7480 }
7481
7482 // select Cond, -1, C --> or (sext Cond), C
7483 if (TrueValue.isAllOnes()) {
7484 MatchInfo = [=](MachineIRBuilder &B) {
7485 B.setInstrAndDebugLoc(*Select);
7486 Register Inner = MRI.createGenericVirtualRegister(TrueTy);
7487 B.buildSExtOrTrunc(Inner, Cond);
7488 B.buildOr(Dest, Inner, False, Flags);
7489 };
7490 return true;
7491 }
7492
7493 // select Cond, C, -1 --> or (sext (not Cond)), C
7494 if (FalseValue.isAllOnes()) {
7495 MatchInfo = [=](MachineIRBuilder &B) {
7496 B.setInstrAndDebugLoc(*Select);
7497 Register Not = MRI.createGenericVirtualRegister(CondTy);
7498 B.buildNot(Not, Cond);
7499 Register Inner = MRI.createGenericVirtualRegister(TrueTy);
7500 B.buildSExtOrTrunc(Inner, Not);
7501 B.buildOr(Dest, Inner, True, Flags);
7502 };
7503 return true;
7504 }
7505
7506 return false;
7507}
7508
7509// TODO: use knownbits to determine zeros
7510bool CombinerHelper::tryFoldBoolSelectToLogic(GSelect *Select,
7511 BuildFnTy &MatchInfo) const {
7512 uint32_t Flags = Select->getFlags();
7513 Register DstReg = Select->getReg(0);
7514 Register Cond = Select->getCondReg();
7515 Register True = Select->getTrueReg();
7516 Register False = Select->getFalseReg();
7517 LLT CondTy = MRI.getType(Select->getCondReg());
7518 LLT TrueTy = MRI.getType(Select->getTrueReg());
7519
7520 // Boolean or fixed vector of booleans.
7521 if (CondTy.isScalableVector() ||
7522 (CondTy.isFixedVector() &&
7523 CondTy.getElementType().getScalarSizeInBits() != 1) ||
7524 CondTy.getScalarSizeInBits() != 1)
7525 return false;
7526
7527 if (CondTy != TrueTy)
7528 return false;
7529
7530 // select Cond, Cond, F --> or Cond, F
7531 // select Cond, 1, F --> or Cond, F
7532 if ((Cond == True) || isOneOrOneSplat(True, /* AllowUndefs */ true)) {
7533 MatchInfo = [=](MachineIRBuilder &B) {
7534 B.setInstrAndDebugLoc(*Select);
7535 Register Ext = MRI.createGenericVirtualRegister(TrueTy);
7536 B.buildZExtOrTrunc(Ext, Cond);
7537 auto FreezeFalse = B.buildFreeze(TrueTy, False);
7538 B.buildOr(DstReg, Ext, FreezeFalse, Flags);
7539 };
7540 return true;
7541 }
7542
7543 // select Cond, T, Cond --> and Cond, T
7544 // select Cond, T, 0 --> and Cond, T
7545 if ((Cond == False) || isZeroOrZeroSplat(False, /* AllowUndefs */ true)) {
7546 MatchInfo = [=](MachineIRBuilder &B) {
7547 B.setInstrAndDebugLoc(*Select);
7548 Register Ext = MRI.createGenericVirtualRegister(TrueTy);
7549 B.buildZExtOrTrunc(Ext, Cond);
7550 auto FreezeTrue = B.buildFreeze(TrueTy, True);
7551 B.buildAnd(DstReg, Ext, FreezeTrue);
7552 };
7553 return true;
7554 }
7555
7556 // select Cond, T, 1 --> or (not Cond), T
7557 if (isOneOrOneSplat(False, /* AllowUndefs */ true)) {
7558 MatchInfo = [=](MachineIRBuilder &B) {
7559 B.setInstrAndDebugLoc(*Select);
7560 // First the not.
7561 Register Inner = MRI.createGenericVirtualRegister(CondTy);
7562 B.buildNot(Inner, Cond);
7563 // Then an ext to match the destination register.
7564 Register Ext = MRI.createGenericVirtualRegister(TrueTy);
7565 B.buildZExtOrTrunc(Ext, Inner);
7566 auto FreezeTrue = B.buildFreeze(TrueTy, True);
7567 B.buildOr(DstReg, Ext, FreezeTrue, Flags);
7568 };
7569 return true;
7570 }
7571
7572 // select Cond, 0, F --> and (not Cond), F
7573 if (isZeroOrZeroSplat(True, /* AllowUndefs */ true)) {
7574 MatchInfo = [=](MachineIRBuilder &B) {
7575 B.setInstrAndDebugLoc(*Select);
7576 // First the not.
7577 Register Inner = MRI.createGenericVirtualRegister(CondTy);
7578 B.buildNot(Inner, Cond);
7579 // Then an ext to match the destination register.
7580 Register Ext = MRI.createGenericVirtualRegister(TrueTy);
7581 B.buildZExtOrTrunc(Ext, Inner);
7582 auto FreezeFalse = B.buildFreeze(TrueTy, False);
7583 B.buildAnd(DstReg, Ext, FreezeFalse);
7584 };
7585 return true;
7586 }
7587
7588 return false;
7589}
7590
7592 BuildFnTy &MatchInfo) const {
7593 Register DstReg = MO.getReg();
7594 Register CondReg, True, False;
7595 if (!mi_match(DstReg, MRI,
7596 m_GISelect(m_Reg(CondReg), m_Reg(True), m_Reg(False))))
7597 return false;
7598
7599 CmpInst::Predicate Pred;
7600 Register CmpLHS, CmpRHS;
7601 if (!mi_match(CondReg, MRI,
7602 m_GICmp(m_Pred(Pred), m_Reg(CmpLHS), m_Reg(CmpRHS))))
7603 return false;
7604
7605 LLT DstTy = MRI.getType(DstReg);
7606 if (DstTy.isPointerOrPointerVector())
7607 return false;
7608
7609 // We want to fold the icmp and replace the select.
7610 if (!MRI.hasOneNonDBGUse(CondReg))
7611 return false;
7612
7613 // We need a larger or smaller predicate for
7614 // canonicalization.
7615 if (CmpInst::isEquality(Pred))
7616 return false;
7617
7618 // We can swap CmpLHS and CmpRHS for higher hitrate.
7619 if (True == CmpRHS && False == CmpLHS) {
7620 std::swap(CmpLHS, CmpRHS);
7621 Pred = CmpInst::getSwappedPredicate(Pred);
7622 }
7623
7624 // (icmp X, Y) ? X : Y -> integer minmax.
7625 // see matchSelectPattern in ValueTracking.
7626 // Legality between G_SELECT and integer minmax can differ.
7627 if (True != CmpLHS || False != CmpRHS)
7628 return false;
7629
7630 switch (Pred) {
7631 case ICmpInst::ICMP_UGT:
7632 case ICmpInst::ICMP_UGE: {
7633 if (!isLegalOrBeforeLegalizer({TargetOpcode::G_UMAX, DstTy}))
7634 return false;
7635 MatchInfo = [=](MachineIRBuilder &B) { B.buildUMax(DstReg, True, False); };
7636 return true;
7637 }
7638 case ICmpInst::ICMP_SGT:
7639 case ICmpInst::ICMP_SGE: {
7640 if (!isLegalOrBeforeLegalizer({TargetOpcode::G_SMAX, DstTy}))
7641 return false;
7642 MatchInfo = [=](MachineIRBuilder &B) { B.buildSMax(DstReg, True, False); };
7643 return true;
7644 }
7645 case ICmpInst::ICMP_ULT:
7646 case ICmpInst::ICMP_ULE: {
7647 if (!isLegalOrBeforeLegalizer({TargetOpcode::G_UMIN, DstTy}))
7648 return false;
7649 MatchInfo = [=](MachineIRBuilder &B) { B.buildUMin(DstReg, True, False); };
7650 return true;
7651 }
7652 case ICmpInst::ICMP_SLT:
7653 case ICmpInst::ICMP_SLE: {
7654 if (!isLegalOrBeforeLegalizer({TargetOpcode::G_SMIN, DstTy}))
7655 return false;
7656 MatchInfo = [=](MachineIRBuilder &B) { B.buildSMin(DstReg, True, False); };
7657 return true;
7658 }
7659 default:
7660 return false;
7661 }
7662}
7663
7664// (neg (min/max x, (neg x))) --> (max/min x, (neg x))
7666 BuildFnTy &MatchInfo) const {
7667 assert(MI.getOpcode() == TargetOpcode::G_SUB);
7668 Register DestReg = MI.getOperand(0).getReg();
7669 LLT DestTy = MRI.getType(DestReg);
7670
7671 Register X;
7672 Register Sub0;
7673 auto NegPattern = m_all_of(m_Neg(m_DeferredReg(X)), m_Reg(Sub0));
7674 if (mi_match(DestReg, MRI,
7675 m_Neg(m_OneUse(m_any_of(m_GSMin(m_Reg(X), NegPattern),
7676 m_GSMax(m_Reg(X), NegPattern),
7677 m_GUMin(m_Reg(X), NegPattern),
7678 m_GUMax(m_Reg(X), NegPattern)))))) {
7679 MachineInstr *MinMaxMI;
7680 if (!mi_match(MI.getOperand(2).getReg(), MRI, m_MInstr(MinMaxMI)))
7681 return false;
7682 unsigned NewOpc = getInverseGMinMaxOpcode(MinMaxMI->getOpcode());
7683 if (isLegal({NewOpc, {DestTy}})) {
7684 MatchInfo = [=](MachineIRBuilder &B) {
7685 B.buildInstr(NewOpc, {DestReg}, {X, Sub0});
7686 };
7687 return true;
7688 }
7689 }
7690
7691 return false;
7692}
7693
7696
7697 if (tryFoldSelectOfConstants(Select, MatchInfo))
7698 return true;
7699
7700 if (tryFoldBoolSelectToLogic(Select, MatchInfo))
7701 return true;
7702
7703 return false;
7704}
7705
7706/// Fold (icmp Pred1 V1, C1) && (icmp Pred2 V2, C2)
7707/// or (icmp Pred1 V1, C1) || (icmp Pred2 V2, C2)
7708/// into a single comparison using range-based reasoning.
7709/// see InstCombinerImpl::foldAndOrOfICmpsUsingRanges.
7710bool CombinerHelper::tryFoldAndOrOrICmpsUsingRanges(
7711 GLogicalBinOp *Logic, BuildFnTy &MatchInfo) const {
7712 assert(Logic->getOpcode() != TargetOpcode::G_XOR && "unexpected xor");
7713 bool IsAnd = Logic->getOpcode() == TargetOpcode::G_AND;
7714 Register DstReg = Logic->getReg(0);
7715 Register LHS = Logic->getLHSReg();
7716 Register RHS = Logic->getRHSReg();
7717 unsigned Flags = Logic->getFlags();
7718
7719 // We need an G_ICMP on the LHS register.
7720 GICmp *Cmp1 = getOpcodeDef<GICmp>(LHS, MRI);
7721 if (!Cmp1)
7722 return false;
7723
7724 // We need an G_ICMP on the RHS register.
7725 GICmp *Cmp2 = getOpcodeDef<GICmp>(RHS, MRI);
7726 if (!Cmp2)
7727 return false;
7728
7729 // We want to fold the icmps.
7730 if (!MRI.hasOneNonDBGUse(Cmp1->getReg(0)) ||
7731 !MRI.hasOneNonDBGUse(Cmp2->getReg(0)))
7732 return false;
7733
7734 APInt C1;
7735 APInt C2;
7736 std::optional<ValueAndVReg> MaybeC1 =
7738 if (!MaybeC1)
7739 return false;
7740 C1 = MaybeC1->Value;
7741
7742 std::optional<ValueAndVReg> MaybeC2 =
7744 if (!MaybeC2)
7745 return false;
7746 C2 = MaybeC2->Value;
7747
7748 Register R1 = Cmp1->getLHSReg();
7749 Register R2 = Cmp2->getLHSReg();
7750 CmpInst::Predicate Pred1 = Cmp1->getCond();
7751 CmpInst::Predicate Pred2 = Cmp2->getCond();
7752 LLT CmpTy = MRI.getType(Cmp1->getReg(0));
7753 LLT CmpOperandTy = MRI.getType(R1);
7754
7755 if (CmpOperandTy.isPointer())
7756 return false;
7757
7758 // We build ands, adds, and constants of type CmpOperandTy.
7759 // They must be legal to build.
7760 if (!isLegalOrBeforeLegalizer({TargetOpcode::G_AND, CmpOperandTy}) ||
7761 !isLegalOrBeforeLegalizer({TargetOpcode::G_ADD, CmpOperandTy}) ||
7762 !isConstantLegalOrBeforeLegalizer(CmpOperandTy))
7763 return false;
7764
7765 // Look through add of a constant offset on R1, R2, or both operands. This
7766 // allows us to interpret the R + C' < C'' range idiom into a proper range.
7767 std::optional<APInt> Offset1;
7768 std::optional<APInt> Offset2;
7769 if (R1 != R2) {
7770 if (GAdd *Add = getOpcodeDef<GAdd>(R1, MRI)) {
7771 std::optional<ValueAndVReg> MaybeOffset1 =
7773 if (MaybeOffset1) {
7774 R1 = Add->getLHSReg();
7775 Offset1 = MaybeOffset1->Value;
7776 }
7777 }
7778 if (GAdd *Add = getOpcodeDef<GAdd>(R2, MRI)) {
7779 std::optional<ValueAndVReg> MaybeOffset2 =
7781 if (MaybeOffset2) {
7782 R2 = Add->getLHSReg();
7783 Offset2 = MaybeOffset2->Value;
7784 }
7785 }
7786 }
7787
7788 if (R1 != R2)
7789 return false;
7790
7791 // We calculate the icmp ranges including maybe offsets.
7792 ConstantRange CR1 = ConstantRange::makeExactICmpRegion(
7793 IsAnd ? ICmpInst::getInversePredicate(Pred1) : Pred1, C1);
7794 if (Offset1)
7795 CR1 = CR1.subtract(*Offset1);
7796
7797 ConstantRange CR2 = ConstantRange::makeExactICmpRegion(
7798 IsAnd ? ICmpInst::getInversePredicate(Pred2) : Pred2, C2);
7799 if (Offset2)
7800 CR2 = CR2.subtract(*Offset2);
7801
7802 bool CreateMask = false;
7803 APInt LowerDiff;
7804 std::optional<ConstantRange> CR = CR1.exactUnionWith(CR2);
7805 if (!CR) {
7806 // We need non-wrapping ranges.
7807 if (CR1.isWrappedSet() || CR2.isWrappedSet())
7808 return false;
7809
7810 // Check whether we have equal-size ranges that only differ by one bit.
7811 // In that case we can apply a mask to map one range onto the other.
7812 LowerDiff = CR1.getLower() ^ CR2.getLower();
7813 APInt UpperDiff = (CR1.getUpper() - 1) ^ (CR2.getUpper() - 1);
7814 APInt CR1Size = CR1.getUpper() - CR1.getLower();
7815 if (!LowerDiff.isPowerOf2() || LowerDiff != UpperDiff ||
7816 CR1Size != CR2.getUpper() - CR2.getLower())
7817 return false;
7818
7819 CR = CR1.getLower().ult(CR2.getLower()) ? CR1 : CR2;
7820 CreateMask = true;
7821 }
7822
7823 if (IsAnd)
7824 CR = CR->inverse();
7825
7826 CmpInst::Predicate NewPred;
7827 APInt NewC, Offset;
7828 CR->getEquivalentICmp(NewPred, NewC, Offset);
7829
7830 // We take the result type of one of the original icmps, CmpTy, for
7831 // the to be build icmp. The operand type, CmpOperandTy, is used for
7832 // the other instructions and constants to be build. The types of
7833 // the parameters and output are the same for add and and. CmpTy
7834 // and the type of DstReg might differ. That is why we zext or trunc
7835 // the icmp into the destination register.
7836
7837 MatchInfo = [=](MachineIRBuilder &B) {
7838 if (CreateMask && Offset != 0) {
7839 auto TildeLowerDiff = B.buildConstant(CmpOperandTy, ~LowerDiff);
7840 auto And = B.buildAnd(CmpOperandTy, R1, TildeLowerDiff); // the mask.
7841 auto OffsetC = B.buildConstant(CmpOperandTy, Offset);
7842 auto Add = B.buildAdd(CmpOperandTy, And, OffsetC, Flags);
7843 auto NewCon = B.buildConstant(CmpOperandTy, NewC);
7844 auto ICmp = B.buildICmp(NewPred, CmpTy, Add, NewCon);
7845 B.buildZExtOrTrunc(DstReg, ICmp);
7846 } else if (CreateMask && Offset == 0) {
7847 auto TildeLowerDiff = B.buildConstant(CmpOperandTy, ~LowerDiff);
7848 auto And = B.buildAnd(CmpOperandTy, R1, TildeLowerDiff); // the mask.
7849 auto NewCon = B.buildConstant(CmpOperandTy, NewC);
7850 auto ICmp = B.buildICmp(NewPred, CmpTy, And, NewCon);
7851 B.buildZExtOrTrunc(DstReg, ICmp);
7852 } else if (!CreateMask && Offset != 0) {
7853 auto OffsetC = B.buildConstant(CmpOperandTy, Offset);
7854 auto Add = B.buildAdd(CmpOperandTy, R1, OffsetC, Flags);
7855 auto NewCon = B.buildConstant(CmpOperandTy, NewC);
7856 auto ICmp = B.buildICmp(NewPred, CmpTy, Add, NewCon);
7857 B.buildZExtOrTrunc(DstReg, ICmp);
7858 } else if (!CreateMask && Offset == 0) {
7859 auto NewCon = B.buildConstant(CmpOperandTy, NewC);
7860 auto ICmp = B.buildICmp(NewPred, CmpTy, R1, NewCon);
7861 B.buildZExtOrTrunc(DstReg, ICmp);
7862 } else {
7863 llvm_unreachable("unexpected configuration of CreateMask and Offset");
7864 }
7865 };
7866 return true;
7867}
7868
7869bool CombinerHelper::tryFoldLogicOfFCmps(GLogicalBinOp *Logic,
7870 BuildFnTy &MatchInfo) const {
7871 assert(Logic->getOpcode() != TargetOpcode::G_XOR && "unexpecte xor");
7872 Register DestReg = Logic->getReg(0);
7873 Register LHS = Logic->getLHSReg();
7874 Register RHS = Logic->getRHSReg();
7875 bool IsAnd = Logic->getOpcode() == TargetOpcode::G_AND;
7876
7877 // We need a compare on the LHS register.
7878 GFCmp *Cmp1 = getOpcodeDef<GFCmp>(LHS, MRI);
7879 if (!Cmp1)
7880 return false;
7881
7882 // We need a compare on the RHS register.
7883 GFCmp *Cmp2 = getOpcodeDef<GFCmp>(RHS, MRI);
7884 if (!Cmp2)
7885 return false;
7886
7887 LLT CmpTy = MRI.getType(Cmp1->getReg(0));
7888 LLT CmpOperandTy = MRI.getType(Cmp1->getLHSReg());
7889
7890 // We build one fcmp, want to fold the fcmps, replace the logic op,
7891 // and the fcmps must have the same shape.
7893 {TargetOpcode::G_FCMP, {CmpTy, CmpOperandTy}}) ||
7894 !MRI.hasOneNonDBGUse(Logic->getReg(0)) ||
7895 !MRI.hasOneNonDBGUse(Cmp1->getReg(0)) ||
7896 !MRI.hasOneNonDBGUse(Cmp2->getReg(0)) ||
7897 MRI.getType(Cmp1->getLHSReg()) != MRI.getType(Cmp2->getLHSReg()))
7898 return false;
7899
7900 CmpInst::Predicate PredL = Cmp1->getCond();
7901 CmpInst::Predicate PredR = Cmp2->getCond();
7902 Register LHS0 = Cmp1->getLHSReg();
7903 Register LHS1 = Cmp1->getRHSReg();
7904 Register RHS0 = Cmp2->getLHSReg();
7905 Register RHS1 = Cmp2->getRHSReg();
7906
7907 if (LHS0 == RHS1 && LHS1 == RHS0) {
7908 // Swap RHS operands to match LHS.
7909 PredR = CmpInst::getSwappedPredicate(PredR);
7910 std::swap(RHS0, RHS1);
7911 }
7912
7913 if (LHS0 == RHS0 && LHS1 == RHS1) {
7914 // We determine the new predicate.
7915 unsigned CmpCodeL = getFCmpCode(PredL);
7916 unsigned CmpCodeR = getFCmpCode(PredR);
7917 unsigned NewPred = IsAnd ? CmpCodeL & CmpCodeR : CmpCodeL | CmpCodeR;
7918 unsigned Flags = Cmp1->getFlags() | Cmp2->getFlags();
7919 MatchInfo = [=](MachineIRBuilder &B) {
7920 // The fcmp predicates fill the lower part of the enum.
7921 FCmpInst::Predicate Pred = static_cast<FCmpInst::Predicate>(NewPred);
7922 if (Pred == FCmpInst::FCMP_FALSE &&
7924 auto False = B.buildConstant(CmpTy, 0);
7925 B.buildZExtOrTrunc(DestReg, False);
7926 } else if (Pred == FCmpInst::FCMP_TRUE &&
7928 auto True =
7929 B.buildConstant(CmpTy, getICmpTrueVal(getTargetLowering(),
7930 CmpTy.isVector() /*isVector*/,
7931 true /*isFP*/));
7932 B.buildZExtOrTrunc(DestReg, True);
7933 } else { // We take the predicate without predicate optimizations.
7934 auto Cmp = B.buildFCmp(Pred, CmpTy, LHS0, LHS1, Flags);
7935 B.buildZExtOrTrunc(DestReg, Cmp);
7936 }
7937 };
7938 return true;
7939 }
7940
7941 return false;
7942}
7943
7945 GAnd *And = cast<GAnd>(&MI);
7946
7947 if (tryFoldAndOrOrICmpsUsingRanges(And, MatchInfo))
7948 return true;
7949
7950 if (tryFoldLogicOfFCmps(And, MatchInfo))
7951 return true;
7952
7953 return false;
7954}
7955
7957 GOr *Or = cast<GOr>(&MI);
7958
7959 if (tryFoldAndOrOrICmpsUsingRanges(Or, MatchInfo))
7960 return true;
7961
7962 if (tryFoldLogicOfFCmps(Or, MatchInfo))
7963 return true;
7964
7965 return false;
7966}
7967
7969 BuildFnTy &MatchInfo) const {
7971
7972 // Addo has no flags
7973 Register Dst = Add->getReg(0);
7974 Register Carry = Add->getReg(1);
7975 Register LHS = Add->getLHSReg();
7976 Register RHS = Add->getRHSReg();
7977 bool IsSigned = Add->isSigned();
7978 LLT DstTy = MRI.getType(Dst);
7979 LLT CarryTy = MRI.getType(Carry);
7980
7981 // Canonicalize constant to RHS.
7982 if (isConstantOrConstantVectorI(LHS) && !isConstantOrConstantVectorI(RHS)) {
7983 if (IsSigned) {
7984 MatchInfo = [=](MachineIRBuilder &B) {
7985 B.buildSAddo(Dst, Carry, RHS, LHS);
7986 };
7987 return true;
7988 }
7989 // !IsSigned
7990 MatchInfo = [=](MachineIRBuilder &B) {
7991 B.buildUAddo(Dst, Carry, RHS, LHS);
7992 };
7993 return true;
7994 }
7995
7996 std::optional<APInt> MaybeLHS = getConstantOrConstantSplatVector(LHS);
7997 std::optional<APInt> MaybeRHS = getConstantOrConstantSplatVector(RHS);
7998
7999 // Fold addo(c1, c2) -> c3, carry.
8000 if (MaybeLHS && MaybeRHS && isConstantLegalOrBeforeLegalizer(DstTy) &&
8002 bool Overflow;
8003 APInt Result = IsSigned ? MaybeLHS->sadd_ov(*MaybeRHS, Overflow)
8004 : MaybeLHS->uadd_ov(*MaybeRHS, Overflow);
8005 MatchInfo = [=](MachineIRBuilder &B) {
8006 B.buildConstant(Dst, Result);
8007 B.buildConstant(Carry, Overflow);
8008 };
8009 return true;
8010 }
8011
8012 // Fold (addo x, 0) -> x, no carry
8013 if (MaybeRHS && *MaybeRHS == 0 && isConstantLegalOrBeforeLegalizer(CarryTy)) {
8014 MatchInfo = [=](MachineIRBuilder &B) {
8015 B.buildCopy(Dst, LHS);
8016 B.buildConstant(Carry, 0);
8017 };
8018 return true;
8019 }
8020
8021 // Given 2 constant operands whose sum does not overflow:
8022 // uaddo (X +nuw C0), C1 -> uaddo X, C0 + C1
8023 // saddo (X +nsw C0), C1 -> saddo X, C0 + C1
8024 GAdd *AddLHS = getOpcodeDef<GAdd>(LHS, MRI);
8025 if (MaybeRHS && AddLHS && MRI.hasOneNonDBGUse(Add->getReg(0)) &&
8026 ((IsSigned && AddLHS->getFlag(MachineInstr::MIFlag::NoSWrap)) ||
8027 (!IsSigned && AddLHS->getFlag(MachineInstr::MIFlag::NoUWrap)))) {
8028 std::optional<APInt> MaybeAddRHS =
8029 getConstantOrConstantSplatVector(AddLHS->getRHSReg());
8030 if (MaybeAddRHS) {
8031 bool Overflow;
8032 APInt NewC = IsSigned ? MaybeAddRHS->sadd_ov(*MaybeRHS, Overflow)
8033 : MaybeAddRHS->uadd_ov(*MaybeRHS, Overflow);
8034 if (!Overflow && isConstantLegalOrBeforeLegalizer(DstTy)) {
8035 if (IsSigned) {
8036 MatchInfo = [=](MachineIRBuilder &B) {
8037 auto ConstRHS = B.buildConstant(DstTy, NewC);
8038 B.buildSAddo(Dst, Carry, AddLHS->getLHSReg(), ConstRHS);
8039 };
8040 return true;
8041 }
8042 // !IsSigned
8043 MatchInfo = [=](MachineIRBuilder &B) {
8044 auto ConstRHS = B.buildConstant(DstTy, NewC);
8045 B.buildUAddo(Dst, Carry, AddLHS->getLHSReg(), ConstRHS);
8046 };
8047 return true;
8048 }
8049 }
8050 };
8051
8052 // We try to combine addo to non-overflowing add.
8053 if (!isLegalOrBeforeLegalizer({TargetOpcode::G_ADD, {DstTy}}) ||
8055 return false;
8056
8057 // We try to combine uaddo to non-overflowing add.
8058 if (!IsSigned) {
8059 ConstantRange CRLHS =
8060 ConstantRange::fromKnownBits(VT->getKnownBits(LHS), /*IsSigned=*/false);
8061 ConstantRange CRRHS =
8062 ConstantRange::fromKnownBits(VT->getKnownBits(RHS), /*IsSigned=*/false);
8063
8064 switch (CRLHS.unsignedAddMayOverflow(CRRHS)) {
8066 return false;
8068 MatchInfo = [=](MachineIRBuilder &B) {
8069 B.buildAdd(Dst, LHS, RHS, MachineInstr::MIFlag::NoUWrap);
8070 B.buildConstant(Carry, 0);
8071 };
8072 return true;
8073 }
8076 MatchInfo = [=](MachineIRBuilder &B) {
8077 B.buildAdd(Dst, LHS, RHS);
8078 B.buildConstant(Carry, 1);
8079 };
8080 return true;
8081 }
8082 }
8083 return false;
8084 }
8085
8086 // We try to combine saddo to non-overflowing add.
8087
8088 // If LHS and RHS each have at least two sign bits, then there is no signed
8089 // overflow.
8090 if (VT->computeNumSignBits(RHS) > 1 && VT->computeNumSignBits(LHS) > 1) {
8091 MatchInfo = [=](MachineIRBuilder &B) {
8092 B.buildAdd(Dst, LHS, RHS, MachineInstr::MIFlag::NoSWrap);
8093 B.buildConstant(Carry, 0);
8094 };
8095 return true;
8096 }
8097
8098 ConstantRange CRLHS =
8099 ConstantRange::fromKnownBits(VT->getKnownBits(LHS), /*IsSigned=*/true);
8100 ConstantRange CRRHS =
8101 ConstantRange::fromKnownBits(VT->getKnownBits(RHS), /*IsSigned=*/true);
8102
8103 switch (CRLHS.signedAddMayOverflow(CRRHS)) {
8105 return false;
8107 MatchInfo = [=](MachineIRBuilder &B) {
8108 B.buildAdd(Dst, LHS, RHS, MachineInstr::MIFlag::NoSWrap);
8109 B.buildConstant(Carry, 0);
8110 };
8111 return true;
8112 }
8115 MatchInfo = [=](MachineIRBuilder &B) {
8116 B.buildAdd(Dst, LHS, RHS);
8117 B.buildConstant(Carry, 1);
8118 };
8119 return true;
8120 }
8121 }
8122
8123 return false;
8124}
8125
8127 BuildFnTy &MatchInfo) const {
8129 MatchInfo(Builder);
8130 Root->eraseFromParent();
8131}
8132
8134 int64_t Exponent) const {
8135 bool OptForSize = MI.getMF()->getFunction().hasOptSize();
8137}
8138
8140 int64_t Exponent) const {
8141 auto [Dst, Base] = MI.getFirst2Regs();
8142 LLT Ty = MRI.getType(Dst);
8143 int64_t ExpVal = Exponent;
8144
8145 if (ExpVal == 0) {
8146 Builder.buildFConstant(Dst, 1.0);
8147 MI.removeFromParent();
8148 return;
8149 }
8150
8151 if (ExpVal < 0)
8152 ExpVal = -ExpVal;
8153
8154 // We use the simple binary decomposition method from SelectionDAG ExpandPowI
8155 // to generate the multiply sequence. There are more optimal ways to do this
8156 // (for example, powi(x,15) generates one more multiply than it should), but
8157 // this has the benefit of being both really simple and much better than a
8158 // libcall.
8159 std::optional<SrcOp> Res;
8160 SrcOp CurSquare = Base;
8161 while (ExpVal > 0) {
8162 if (ExpVal & 1) {
8163 if (!Res)
8164 Res = CurSquare;
8165 else
8166 Res = Builder.buildFMul(Ty, *Res, CurSquare);
8167 }
8168
8169 CurSquare = Builder.buildFMul(Ty, CurSquare, CurSquare);
8170 ExpVal >>= 1;
8171 }
8172
8173 // If the original exponent was negative, invert the result, producing
8174 // 1/(x*x*x).
8175 if (Exponent < 0)
8176 Res = Builder.buildFDiv(Ty, Builder.buildFConstant(Ty, 1.0), *Res,
8177 MI.getFlags());
8178
8179 Builder.buildCopy(Dst, *Res);
8180 MI.eraseFromParent();
8181}
8182
8184 BuildFnTy &MatchInfo) const {
8185 // fold (A+C1)-C2 -> A+(C1-C2)
8186 const GSub *Sub = cast<GSub>(&MI);
8187 Register A, C1Reg;
8188 if (!mi_match(Sub->getLHSReg(), MRI, m_GAdd(m_Reg(A), m_Reg(C1Reg))))
8189 return false;
8190
8191 if (!MRI.hasOneNonDBGUse(Sub->getLHSReg()))
8192 return false;
8193
8194 APInt C2 = getIConstantFromReg(Sub->getRHSReg(), MRI);
8195 APInt C1 = getIConstantFromReg(C1Reg, MRI);
8196
8197 Register Dst = Sub->getReg(0);
8198 LLT DstTy = MRI.getType(Dst);
8199
8200 MatchInfo = [=](MachineIRBuilder &B) {
8201 auto Const = B.buildConstant(DstTy, C1 - C2);
8202 B.buildAdd(Dst, A, Const);
8203 };
8204
8205 return true;
8206}
8207
8209 BuildFnTy &MatchInfo) const {
8210 // fold C2-(A+C1) -> (C2-C1)-A
8211 const GSub *Sub = cast<GSub>(&MI);
8212 Register A, C1Reg;
8213 if (!mi_match(Sub->getRHSReg(), MRI, m_GAdd(m_Reg(A), m_Reg(C1Reg))))
8214 return false;
8215
8216 if (!MRI.hasOneNonDBGUse(Sub->getRHSReg()))
8217 return false;
8218
8219 APInt C2 = getIConstantFromReg(Sub->getLHSReg(), MRI);
8220 APInt C1 = getIConstantFromReg(C1Reg, MRI);
8221
8222 Register Dst = Sub->getReg(0);
8223 LLT DstTy = MRI.getType(Dst);
8224
8225 MatchInfo = [=](MachineIRBuilder &B) {
8226 auto Const = B.buildConstant(DstTy, C2 - C1);
8227 B.buildSub(Dst, Const, A);
8228 };
8229
8230 return true;
8231}
8232
8234 BuildFnTy &MatchInfo) const {
8235 // fold (A-C1)-C2 -> A-(C1+C2)
8236 const GSub *Sub1 = cast<GSub>(&MI);
8237 Register A, C1Reg;
8238 if (!mi_match(Sub1->getLHSReg(), MRI, m_GSub(m_Reg(A), m_Reg(C1Reg))))
8239 return false;
8240
8241 if (!MRI.hasOneNonDBGUse(Sub1->getLHSReg()))
8242 return false;
8243
8244 APInt C2 = getIConstantFromReg(Sub1->getRHSReg(), MRI);
8245 APInt C1 = getIConstantFromReg(C1Reg, MRI);
8246
8247 Register Dst = Sub1->getReg(0);
8248 LLT DstTy = MRI.getType(Dst);
8249
8250 MatchInfo = [=](MachineIRBuilder &B) {
8251 auto Const = B.buildConstant(DstTy, C1 + C2);
8252 B.buildSub(Dst, A, Const);
8253 };
8254
8255 return true;
8256}
8257
8259 BuildFnTy &MatchInfo) const {
8260 // fold (C1-A)-C2 -> (C1-C2)-A
8261 const GSub *Sub1 = cast<GSub>(&MI);
8262 Register C1Reg, A;
8263 if (!mi_match(Sub1->getLHSReg(), MRI, m_GSub(m_Reg(C1Reg), m_Reg(A))))
8264 return false;
8265
8266 if (!MRI.hasOneNonDBGUse(Sub1->getLHSReg()))
8267 return false;
8268
8269 APInt C2 = getIConstantFromReg(Sub1->getRHSReg(), MRI);
8270 APInt C1 = getIConstantFromReg(C1Reg, MRI);
8271
8272 Register Dst = Sub1->getReg(0);
8273 LLT DstTy = MRI.getType(Dst);
8274
8275 MatchInfo = [=](MachineIRBuilder &B) {
8276 auto Const = B.buildConstant(DstTy, C1 - C2);
8277 B.buildSub(Dst, Const, A);
8278 };
8279
8280 return true;
8281}
8282
8284 BuildFnTy &MatchInfo) const {
8285 // fold ((A-C1)+C2) -> (A+(C2-C1))
8286 const GAdd *Add = cast<GAdd>(&MI);
8287 Register A, C1Reg;
8288 if (!mi_match(Add->getLHSReg(), MRI, m_GSub(m_Reg(A), m_Reg(C1Reg))))
8289 return false;
8290
8291 if (!MRI.hasOneNonDBGUse(Add->getLHSReg()))
8292 return false;
8293
8294 APInt C2 = getIConstantFromReg(Add->getRHSReg(), MRI);
8295 APInt C1 = getIConstantFromReg(C1Reg, MRI);
8296
8297 Register Dst = Add->getReg(0);
8298 LLT DstTy = MRI.getType(Dst);
8299
8300 MatchInfo = [=](MachineIRBuilder &B) {
8301 auto Const = B.buildConstant(DstTy, C2 - C1);
8302 B.buildAdd(Dst, A, Const);
8303 };
8304
8305 return true;
8306}
8307
8309 const MachineInstr &MI, BuildFnTy &MatchInfo) const {
8310 const GUnmerge *Unmerge = cast<GUnmerge>(&MI);
8311
8312 if (!MRI.hasOneNonDBGUse(Unmerge->getSourceReg()))
8313 return false;
8314
8315 LLT DstTy = MRI.getType(Unmerge->getReg(0));
8316
8317 // $bv:_(<8 x s8>) = G_BUILD_VECTOR ....
8318 // $any:_(<8 x s16>) = G_ANYEXT $bv
8319 // $uv:_(<4 x s16>), $uv1:_(<4 x s16>) = G_UNMERGE_VALUES $any
8320 //
8321 // ->
8322 //
8323 // $any:_(s16) = G_ANYEXT $bv[0]
8324 // $any1:_(s16) = G_ANYEXT $bv[1]
8325 // $any2:_(s16) = G_ANYEXT $bv[2]
8326 // $any3:_(s16) = G_ANYEXT $bv[3]
8327 // $any4:_(s16) = G_ANYEXT $bv[4]
8328 // $any5:_(s16) = G_ANYEXT $bv[5]
8329 // $any6:_(s16) = G_ANYEXT $bv[6]
8330 // $any7:_(s16) = G_ANYEXT $bv[7]
8331 // $uv:_(<4 x s16>) = G_BUILD_VECTOR $any, $any1, $any2, $any3
8332 // $uv1:_(<4 x s16>) = G_BUILD_VECTOR $any4, $any5, $any6, $any7
8333
8334 // We want to unmerge into vectors.
8335 if (!DstTy.isFixedVector())
8336 return false;
8337
8338 Register AnySrcReg;
8339 if (!mi_match(Unmerge->getSourceReg(), MRI, m_GAnyExt(m_Reg(AnySrcReg))))
8340 return false;
8341
8342 GBuildVector *BV;
8343 if (mi_match(AnySrcReg, MRI, m_GBuildVector(BV))) {
8344 // G_UNMERGE_VALUES G_ANYEXT G_BUILD_VECTOR
8345
8346 if (!MRI.hasOneNonDBGUse(BV->getReg(0)))
8347 return false;
8348
8349 // FIXME: check element types?
8350 if (BV->getNumSources() % Unmerge->getNumDefs() != 0)
8351 return false;
8352
8353 LLT BigBvTy = MRI.getType(BV->getReg(0));
8354 LLT SmallBvTy = DstTy;
8355 LLT SmallBvElemenTy = SmallBvTy.getElementType();
8356
8358 {TargetOpcode::G_BUILD_VECTOR, {SmallBvTy, SmallBvElemenTy}}))
8359 return false;
8360
8361 // We check the legality of scalar anyext.
8363 {TargetOpcode::G_ANYEXT,
8364 {SmallBvElemenTy, BigBvTy.getElementType()}}))
8365 return false;
8366
8367 MatchInfo = [=](MachineIRBuilder &B) {
8368 // Build into each G_UNMERGE_VALUES def
8369 // a small build vector with anyext from the source build vector.
8370 for (unsigned I = 0; I < Unmerge->getNumDefs(); ++I) {
8372 for (unsigned J = 0; J < SmallBvTy.getNumElements(); ++J) {
8373 Register SourceArray =
8374 BV->getSourceReg(I * SmallBvTy.getNumElements() + J);
8375 auto AnyExt = B.buildAnyExt(SmallBvElemenTy, SourceArray);
8376 Ops.push_back(AnyExt.getReg(0));
8377 }
8378 B.buildBuildVector(Unmerge->getOperand(I).getReg(), Ops);
8379 };
8380 };
8381 return true;
8382 };
8383
8384 return false;
8385}
8386
8388 BuildFnTy &MatchInfo) const {
8389
8390 bool Changed = false;
8391 auto &Shuffle = cast<GShuffleVector>(MI);
8392 ArrayRef<int> OrigMask = Shuffle.getMask();
8393 SmallVector<int, 16> NewMask;
8394 const LLT SrcTy = MRI.getType(Shuffle.getSrc1Reg());
8395 const unsigned NumSrcElems = SrcTy.isVector() ? SrcTy.getNumElements() : 1;
8396 const unsigned NumDstElts = OrigMask.size();
8397 for (unsigned i = 0; i != NumDstElts; ++i) {
8398 int Idx = OrigMask[i];
8399 if (Idx >= (int)NumSrcElems) {
8400 Idx = -1;
8401 Changed = true;
8402 }
8403 NewMask.push_back(Idx);
8404 }
8405
8406 if (!Changed)
8407 return false;
8408
8409 MatchInfo = [&, NewMask = std::move(NewMask)](MachineIRBuilder &B) {
8410 B.buildShuffleVector(MI.getOperand(0), MI.getOperand(1), MI.getOperand(2),
8411 std::move(NewMask));
8412 };
8413
8414 return true;
8415}
8416
8417static void commuteMask(MutableArrayRef<int> Mask, const unsigned NumElems) {
8418 const unsigned MaskSize = Mask.size();
8419 for (unsigned I = 0; I < MaskSize; ++I) {
8420 int Idx = Mask[I];
8421 if (Idx < 0)
8422 continue;
8423
8424 if (Idx < (int)NumElems)
8425 Mask[I] = Idx + NumElems;
8426 else
8427 Mask[I] = Idx - NumElems;
8428 }
8429}
8430
8432 if (!MRI.getType(Reg).isFixedVector())
8433 return false;
8434 return matchUnaryPredicate(MRI, Reg, [](const Constant *C) {
8435 auto *CI = dyn_cast_or_null<ConstantInt>(C);
8436 return CI && (CI->isZero() || CI->isOne());
8437 });
8438}
8439
8441 BuildFnTy &MatchInfo) const {
8442
8443 auto &Shuffle = cast<GShuffleVector>(MI);
8444 // If any of the two inputs is already undef, don't check the mask again to
8445 // prevent infinite loop
8446 if (getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, Shuffle.getSrc1Reg(), MRI))
8447 return false;
8448
8449 if (getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, Shuffle.getSrc2Reg(), MRI))
8450 return false;
8451
8452 const LLT DstTy = MRI.getType(Shuffle.getReg(0));
8453 const LLT Src1Ty = MRI.getType(Shuffle.getSrc1Reg());
8455 {TargetOpcode::G_SHUFFLE_VECTOR, {DstTy, Src1Ty}}))
8456 return false;
8457
8458 ArrayRef<int> Mask = Shuffle.getMask();
8459 const unsigned NumSrcElems = Src1Ty.getNumElements();
8460
8461 bool TouchesSrc1 = false;
8462 bool TouchesSrc2 = false;
8463 const unsigned NumElems = Mask.size();
8464 for (unsigned Idx = 0; Idx < NumElems; ++Idx) {
8465 if (Mask[Idx] < 0)
8466 continue;
8467
8468 if (Mask[Idx] < (int)NumSrcElems)
8469 TouchesSrc1 = true;
8470 else
8471 TouchesSrc2 = true;
8472 }
8473
8474 if (TouchesSrc1 == TouchesSrc2)
8475 return false;
8476
8477 Register NewSrc1 = Shuffle.getSrc1Reg();
8478 SmallVector<int, 16> NewMask(Mask);
8479 if (TouchesSrc2) {
8480 NewSrc1 = Shuffle.getSrc2Reg();
8481 commuteMask(NewMask, NumSrcElems);
8482 }
8483
8484 MatchInfo = [=, &Shuffle](MachineIRBuilder &B) {
8485 auto Undef = B.buildUndef(Src1Ty);
8486 B.buildShuffleVector(Shuffle.getReg(0), NewSrc1, Undef, NewMask);
8487 };
8488
8489 return true;
8490}
8491
8493 BuildFnTy &MatchInfo) const {
8494 const GSubCarryOut *Subo = cast<GSubCarryOut>(&MI);
8495
8496 Register Dst = Subo->getReg(0);
8497 Register LHS = Subo->getLHSReg();
8498 Register RHS = Subo->getRHSReg();
8499 Register Carry = Subo->getCarryOutReg();
8500 LLT DstTy = MRI.getType(Dst);
8501 LLT CarryTy = MRI.getType(Carry);
8502
8503 // Check legality before known bits.
8504 if (!isLegalOrBeforeLegalizer({TargetOpcode::G_SUB, {DstTy}}) ||
8506 return false;
8507
8508 ConstantRange KBLHS =
8509 ConstantRange::fromKnownBits(VT->getKnownBits(LHS),
8510 /* IsSigned=*/Subo->isSigned());
8511 ConstantRange KBRHS =
8512 ConstantRange::fromKnownBits(VT->getKnownBits(RHS),
8513 /* IsSigned=*/Subo->isSigned());
8514
8515 if (Subo->isSigned()) {
8516 // G_SSUBO
8517 switch (KBLHS.signedSubMayOverflow(KBRHS)) {
8519 return false;
8521 MatchInfo = [=](MachineIRBuilder &B) {
8522 B.buildSub(Dst, LHS, RHS, MachineInstr::MIFlag::NoSWrap);
8523 B.buildConstant(Carry, 0);
8524 };
8525 return true;
8526 }
8529 MatchInfo = [=](MachineIRBuilder &B) {
8530 B.buildSub(Dst, LHS, RHS);
8531 B.buildConstant(Carry, getICmpTrueVal(getTargetLowering(),
8532 /*isVector=*/CarryTy.isVector(),
8533 /*isFP=*/false));
8534 };
8535 return true;
8536 }
8537 }
8538 return false;
8539 }
8540
8541 // G_USUBO
8542 switch (KBLHS.unsignedSubMayOverflow(KBRHS)) {
8544 return false;
8546 MatchInfo = [=](MachineIRBuilder &B) {
8547 B.buildSub(Dst, LHS, RHS, MachineInstr::MIFlag::NoUWrap);
8548 B.buildConstant(Carry, 0);
8549 };
8550 return true;
8551 }
8554 MatchInfo = [=](MachineIRBuilder &B) {
8555 B.buildSub(Dst, LHS, RHS);
8556 B.buildConstant(Carry, getICmpTrueVal(getTargetLowering(),
8557 /*isVector=*/CarryTy.isVector(),
8558 /*isFP=*/false));
8559 };
8560 return true;
8561 }
8562 }
8563
8564 return false;
8565}
8566
8567// Fold (ctlz (xor x, (sra x, bitwidth-1))) -> (add (ctls x), 1).
8568// Fold (ctlz (or (shl (xor x, (sra x, bitwidth-1)), 1), 1) -> (ctls x)
8570 BuildFnTy &MatchInfo) const {
8571 assert((CtlzMI.getOpcode() == TargetOpcode::G_CTLZ ||
8572 CtlzMI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_POISON) &&
8573 "Expected G_CTLZ variant");
8574
8575 const Register Dst = CtlzMI.getOperand(0).getReg();
8576 Register Src = CtlzMI.getOperand(1).getReg();
8577
8578 LLT Ty = MRI.getType(Dst);
8579 LLT SrcTy = MRI.getType(Src);
8580
8581 if (!(Ty.isValid() && Ty.isScalar()))
8582 return false;
8583
8584 if (!LI)
8585 return false;
8586
8587 SmallVector<LLT, 2> QueryTypes = {Ty, SrcTy};
8588 LegalityQuery Query(TargetOpcode::G_CTLS, QueryTypes);
8589
8590 switch (LI->getAction(Query).Action) {
8591 default:
8592 return false;
8596 break;
8597 }
8598
8599 // Src = or(shl(V, 1), 1) -> Src=V; NeedAdd = False
8600 Register V;
8601 bool NeedAdd = true;
8602 if (mi_match(Src, MRI,
8604 m_SpecificICst(1))))) {
8605 NeedAdd = false;
8606 Src = V;
8607 }
8608
8609 unsigned BitWidth = Ty.getScalarSizeInBits();
8610
8611 Register X;
8612 if (!mi_match(Src, MRI,
8615 m_SpecificICst(BitWidth - 1)))))))
8616 return false;
8617
8618 MatchInfo = [=](MachineIRBuilder &B) {
8619 if (!NeedAdd) {
8620 B.buildCTLS(Dst, X);
8621 return;
8622 }
8623
8624 auto Ctls = B.buildCTLS(Ty, X);
8625 auto One = B.buildConstant(Ty, 1);
8626
8627 B.buildAdd(Dst, Ctls, One);
8628 };
8629
8630 return true;
8631}
8632
8634 const ConstantFP *C = getConstantFPVRegVal(Reg, MRI);
8635 return C && C->getValueAPF().isNaN();
8636}
8637
8639 unsigned ZeroPoisonOpc) const {
8640 const bool IsCtlzPair = Opc == TargetOpcode::G_CTLZ &&
8641 ZeroPoisonOpc == TargetOpcode::G_CTLZ_ZERO_POISON;
8642 const bool IsCttzPair = Opc == TargetOpcode::G_CTTZ &&
8643 ZeroPoisonOpc == TargetOpcode::G_CTTZ_ZERO_POISON;
8644 if (MI.getOpcode() != Opc || (!IsCtlzPair && !IsCttzPair))
8645 return false;
8646 if (!VT)
8647 return false;
8648
8649 Register Src = MI.getOperand(1).getReg();
8650 if (!VT->isKnownNeverZero(Src))
8651 return false;
8652
8653 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
8654 LLT SrcTy = MRI.getType(Src);
8655 return isLegalOrBeforeLegalizer({ZeroPoisonOpc, {DstTy, SrcTy}});
8656}
8657
8658static APInt getDemandedBitsForLogicalOp(unsigned Opcode, const APInt &Demanded,
8659 const KnownBits &OtherKnown) {
8660 assert((Opcode == TargetOpcode::G_AND || Opcode == TargetOpcode::G_OR) &&
8661 "Expected G_AND or G_OR");
8662 return Demanded &
8663 ~(Opcode == TargetOpcode::G_AND ? OtherKnown.Zero : OtherKnown.One);
8664}
8665
8666static std::optional<unsigned>
8667getValidConstShiftAmt(const std::optional<APInt> &Amt, unsigned BW) {
8668 if (!Amt || Amt->uge(BW))
8669 return std::nullopt;
8670 return Amt->getZExtValue();
8671}
8672
8674 const APInt &DemandedBits,
8675 unsigned ShAmt) {
8676 assert(ShAmt < DemandedBits.getBitWidth() &&
8677 "shift amount must be less than the bit width");
8678 switch (Opcode) {
8679 case TargetOpcode::G_SHL:
8680 return DemandedBits.lshr(ShAmt);
8681 case TargetOpcode::G_LSHR:
8682 return DemandedBits.shl(ShAmt);
8683 case TargetOpcode::G_ASHR: {
8684 APInt Src = DemandedBits.shl(ShAmt);
8685 if (DemandedBits.countLeadingZeros() < ShAmt)
8686 Src.setSignBit();
8687 return Src;
8688 }
8689 default:
8690 llvm_unreachable("not a shift opcode");
8691 }
8692}
8693
8694Register CombinerHelper::simplifyMultipleUseDemandedBits(
8695 Register R, const APInt &DemandedBits, unsigned Depth) const {
8696 assert(R.isVirtual() && "Expected a virtual register");
8697 assert(!DemandedBits.isZero() && "Zero demand is handled by the caller");
8699 return Register();
8700
8701 [[maybe_unused]] LLT Ty = MRI.getType(R);
8702 assert(Ty.isValid() && "Expected a typed register");
8703 assert(DemandedBits.getBitWidth() == Ty.getScalarSizeInBits() &&
8704 "DemandedBits width must match the register scalar type");
8705
8706 MachineInstr *DefMI = MRI.getVRegDef(R);
8707 assert(DefMI && "Expected a definition in generic SSA MIR");
8708 if (!isa<GAnd>(DefMI) && !isa<GOr>(DefMI))
8709 return Register();
8710
8711 unsigned Opcode = DefMI->getOpcode();
8712 auto *Logic = cast<GLogicalBinOp>(DefMI);
8713 Register LHS = Logic->getLHSReg();
8714 Register RHS = Logic->getRHSReg();
8715 assert(MRI.getType(LHS) == Ty && MRI.getType(RHS) == Ty &&
8716 "Logical operands must match the result type");
8717
8718 auto LookThrough = [&](Register X, Register CReg) -> Register {
8719 std::optional<APInt> C = getConstantOrConstantSplatVector(CReg);
8720 if (!C || C->getBitWidth() != DemandedBits.getBitWidth())
8721 return Register();
8722 bool BypassToX = Opcode == TargetOpcode::G_AND
8723 ? DemandedBits.isSubsetOf(*C)
8724 : DemandedBits.isSubsetOf(~*C);
8725 if (BypassToX) {
8726 if (Register Deeper =
8727 simplifyMultipleUseDemandedBits(X, DemandedBits, Depth + 1))
8728 return Deeper;
8729 return X;
8730 }
8731 bool BypassToC = Opcode == TargetOpcode::G_AND
8732 ? DemandedBits.isSubsetOf(~*C)
8733 : DemandedBits.isSubsetOf(*C);
8734 if (BypassToC)
8735 return CReg;
8736 return Register();
8737 };
8738
8739 if (Register Repl = LookThrough(LHS, RHS))
8740 return Repl;
8741 return LookThrough(RHS, LHS);
8742}
8743
8744bool CombinerHelper::simplifyDemandedBitsImpl(MachineInstr &MI, unsigned OpNo,
8745 const APInt &DemandedBits,
8746 KnownBits &Known, unsigned Depth,
8747 bool DoRewrite) const {
8748 Known = KnownBits(DemandedBits.getBitWidth());
8749 assert(MI.getOperand(OpNo).isReg() && !MI.getOperand(OpNo).isDef() &&
8750 "Expected a register use");
8751 Register OpReg = MI.getOperand(OpNo).getReg();
8752 assert(OpReg.isVirtual() && "Expected a virtual register");
8753 LLT OpTy = MRI.getType(OpReg);
8754 assert(OpTy.isValid() && "Expected a typed register");
8755 assert(DemandedBits.getBitWidth() == OpTy.getScalarSizeInBits() &&
8756 "DemandedBits width must match the operand scalar type");
8757
8758 MachineInstr *DefMI = MRI.getVRegDef(OpReg);
8759 assert(DefMI && "Expected a definition in generic SSA MIR");
8760 // Undemanded bits can still affect poison-generating flags in the user.
8761 auto DropPoisonFlags = [&]() {
8762 if (!DoRewrite ||
8764 return;
8765 Observer.changingInstr(MI);
8767 Observer.changedInstr(MI);
8768 };
8769 // Debug users can observe bits that are not demanded by other users.
8770 auto InvalidateDebugUses = [&](const APInt &Demanded) {
8771 if (DoRewrite && !Demanded.isAllOnes())
8772 MRI.markUsesInDebugValueAsUndef(OpReg);
8773 };
8774 auto Rewrite = [&](Register Repl) {
8775 if (!DoRewrite || Repl == OpReg)
8776 return Repl != OpReg;
8777 DropPoisonFlags();
8778 if (DefMI->getNumExplicitDefs() == 1 && MRI.hasOneNonDBGUse(OpReg)) {
8779 InvalidateDebugUses(DemandedBits);
8780 replaceRegWith(MRI, OpReg, Repl);
8781 eraseInst(*DefMI);
8782 } else {
8783 replaceRegOpWith(MRI, MI.getOperand(OpNo), Repl);
8784 }
8785 return true;
8786 };
8787
8788 if (DemandedBits.isZero()) {
8789 if (DefMI->getOpcode() == TargetOpcode::G_IMPLICIT_DEF)
8790 return false;
8791 if (!DoRewrite)
8792 return true;
8793 MachineBasicBlock &SaveMBB = Builder.getMBB();
8794 MachineBasicBlock::iterator SavePt = Builder.getInsertPt();
8795 DebugLoc SaveDL = Builder.getDL();
8796 Builder.setInstrAndDebugLoc(MI);
8797 Register Undef = Builder.buildUndef(OpTy).getReg(0);
8798 Builder.setInsertPt(SaveMBB, SavePt);
8799 Builder.setDebugLoc(SaveDL);
8800 // The producer may have side effects; replace only this use.
8801 DropPoisonFlags();
8802 replaceRegOpWith(MRI, MI.getOperand(OpNo), Undef);
8803 return true;
8804 }
8805
8806 if (!VT)
8807 return false;
8808
8809 unsigned BW = DemandedBits.getBitWidth();
8810 auto GiveUp = [&]() {
8811 APInt DemandedElts = OpTy.isFixedVector()
8813 : APInt(1, 1);
8814 Known = VT->getKnownBits(OpReg, DemandedElts, Depth);
8815 return false;
8816 };
8817
8819 return GiveUp();
8820
8821 // Shared defs require full demand unless this use can be rerouted to an
8822 // existing value that agrees on its demanded bits.
8823 APInt Demanded = DemandedBits;
8824 if (!MRI.hasOneNonDBGUse(OpReg) && !Demanded.isAllOnes()) {
8825 if (Register Repl =
8826 simplifyMultipleUseDemandedBits(OpReg, Demanded, Depth)) {
8827 Known = VT->getKnownBits(Repl);
8828 if (Rewrite(Repl))
8829 return true;
8830 }
8831 Demanded = APInt::getAllOnes(BW);
8832 }
8833
8834 unsigned Opcode = DefMI->getOpcode();
8835 switch (Opcode) {
8836 case TargetOpcode::G_AND:
8837 case TargetOpcode::G_OR: {
8838 auto *Logic = cast<GLogicalBinOp>(DefMI);
8839 Register LHS = Logic->getLHSReg();
8840 Register RHS = Logic->getRHSReg();
8841 assert(MRI.getType(LHS) == OpTy && MRI.getType(RHS) == OpTy &&
8842 "Logical operands must match the result type");
8843 auto SimplifyWithConst = [&](Register X,
8844 Register CReg) -> std::optional<Register> {
8845 std::optional<APInt> C = getConstantOrConstantSplatVector(CReg);
8846 if (!C)
8847 return std::nullopt;
8848 assert(C->getBitWidth() == BW && "Constant width must match its type");
8849 // Check redundancy before rewriting X can change its known bits.
8850 if (Opcode == TargetOpcode::G_AND) {
8851 if (Demanded.isSubsetOf(*C | VT->getKnownBits(X).Zero))
8852 return X;
8853 if (Demanded.isSubsetOf(~*C))
8854 return CReg;
8855 return std::nullopt;
8856 }
8857 if (Demanded.isSubsetOf(~*C | VT->getKnownBits(X).One))
8858 return X;
8859 if (Demanded.isSubsetOf(*C))
8860 return CReg;
8861 return std::nullopt;
8862 };
8863
8864 // Compute known bits before a rewrite can erase the def.
8865 if (std::optional<Register> Repl = SimplifyWithConst(LHS, RHS)) {
8866 Known = VT->getKnownBits(*Repl);
8867 if (Rewrite(*Repl))
8868 return true;
8869 }
8870 if (std::optional<Register> Repl = SimplifyWithConst(RHS, LHS)) {
8871 Known = VT->getKnownBits(*Repl);
8872 if (Rewrite(*Repl))
8873 return true;
8874 }
8875
8876 KnownBits RHSKnown(BW);
8877 bool Changed = simplifyDemandedBitsImpl(*DefMI, /*OpNo=*/2, Demanded,
8878 RHSKnown, Depth + 1, DoRewrite);
8879 APInt LHSDemand = getDemandedBitsForLogicalOp(Opcode, Demanded, RHSKnown);
8880 KnownBits LHSKnown(BW);
8881 Changed |= simplifyDemandedBitsImpl(*DefMI, /*OpNo=*/1, LHSDemand, LHSKnown,
8882 Depth + 1, DoRewrite);
8883 if (!Changed && MRI.hasOneNonDBGUse(RHS) &&
8884 !getConstantOrConstantSplatVector(RHS)) {
8885 // Reduced-demand LHS facts cannot justify reducing the RHS demand.
8886 APInt RHSDemand =
8887 LHSDemand == Demanded
8888 ? getDemandedBitsForLogicalOp(Opcode, Demanded, LHSKnown)
8890 Opcode, Demanded,
8891 VT->getKnownBits(
8892 LHS,
8893 OpTy.isFixedVector()
8894 ? APInt::getAllOnes(OpTy.getNumElements())
8895 : APInt(1, 1),
8896 Depth + 1));
8897 if (RHSDemand != Demanded) {
8898 // Keep full-demand RHS facts if the reduced-demand retry fails.
8899 KnownBits RetryKnown(BW);
8900 Changed = simplifyDemandedBitsImpl(*DefMI, /*OpNo=*/2, RHSDemand,
8901 RetryKnown, Depth + 1, DoRewrite);
8902 }
8903 }
8904 if (Changed) {
8905 DropPoisonFlags();
8906 InvalidateDebugUses(Demanded);
8907 }
8908 Known = Opcode == TargetOpcode::G_AND ? LHSKnown & RHSKnown
8909 : LHSKnown | RHSKnown;
8910 return Changed;
8911 }
8912 case TargetOpcode::G_SHL:
8913 case TargetOpcode::G_LSHR:
8914 case TargetOpcode::G_ASHR: {
8915 std::optional<unsigned> ShAmt = getValidConstShiftAmt(
8916 getConstantOrConstantSplatVector(DefMI->getOperand(2).getReg()), BW);
8917 if (!ShAmt)
8918 return GiveUp();
8919
8920 APInt SrcDemand = getDemandedSrcBitsForShiftConst(Opcode, Demanded, *ShAmt);
8921 KnownBits SrcKnown(BW);
8922 bool Changed = simplifyDemandedBitsImpl(*DefMI, /*OpNo=*/1, SrcDemand,
8923 SrcKnown, Depth + 1, DoRewrite);
8924 if (Changed) {
8925 DropPoisonFlags();
8926 InvalidateDebugUses(Demanded);
8927 }
8928 KnownBits AmtKnown = KnownBits::makeConstant(APInt(BW, *ShAmt));
8929 switch (Opcode) {
8930 case TargetOpcode::G_SHL:
8931 Known = KnownBits::shl(SrcKnown, AmtKnown);
8932 break;
8933 case TargetOpcode::G_LSHR:
8934 Known = KnownBits::lshr(SrcKnown, AmtKnown);
8935 break;
8936 case TargetOpcode::G_ASHR: {
8937 // ASHR and LSHR agree outside sign fill or for nonnegative inputs.
8938 LLT AmtTy = MRI.getType(DefMI->getOperand(2).getReg());
8939 bool LShrLegal = isPreLegalize() || !LI ||
8940 isLegal({TargetOpcode::G_LSHR, {OpTy, AmtTy}});
8941 if ((Demanded.countLeadingZeros() >= *ShAmt ||
8942 SrcKnown.isNonNegative()) &&
8943 LShrLegal) {
8944 if (DoRewrite) {
8945 MachineBasicBlock &SaveMBB = Builder.getMBB();
8946 MachineBasicBlock::iterator SavePt = Builder.getInsertPt();
8947 DebugLoc SaveDL = Builder.getDL();
8948 Builder.setInstrAndDebugLoc(*DefMI);
8949 auto Lshr = Builder.buildLShr(OpTy, DefMI->getOperand(1).getReg(),
8950 DefMI->getOperand(2).getReg());
8951 Builder.setInsertPt(SaveMBB, SavePt);
8952 Builder.setDebugLoc(SaveDL);
8953 Rewrite(Lshr.getReg(0));
8954 Known = KnownBits::lshr(SrcKnown, AmtKnown);
8955 }
8956 return true;
8957 }
8958 Known = KnownBits::ashr(SrcKnown, AmtKnown);
8959 break;
8960 }
8961 }
8962 return Changed;
8963 }
8964 default:
8965 return GiveUp();
8966 }
8967}
8968
8970 const APInt &DemandedBits,
8972 unsigned Depth) const {
8973 return simplifyDemandedBitsImpl(MI, OpNo, DemandedBits, Known, Depth,
8974 /*DoRewrite=*/true);
8975}
8976
8978 BuildFnTy &MatchInfo) const {
8979 Register Dst = MI.getOperand(0).getReg();
8980 LLT Ty = MRI.getType(Dst);
8981 assert(Ty.isValid() && "Expected a typed generic result");
8982 if (!VT || Ty.isVector())
8983 return false;
8984
8985 APInt RootDemand = APInt::getAllOnes(Ty.getScalarSizeInBits());
8986 auto Probe = [&](unsigned OpNo, const APInt &OpDemand, KnownBits &Known) {
8987 if (!simplifyDemandedBitsImpl(MI, OpNo, OpDemand, Known, /*Depth=*/0,
8988 /*DoRewrite=*/false))
8989 return false;
8990
8991 MatchInfo = [this, &MI, OpNo, OpDemand](MachineIRBuilder &) {
8992 KnownBits K(OpDemand.getBitWidth());
8993 simplifyDemandedBits(MI, OpNo, OpDemand, K);
8994 };
8995 return true;
8996 };
8997
8998 unsigned Opcode = MI.getOpcode();
8999 switch (Opcode) {
9000 case TargetOpcode::G_SHL:
9001 case TargetOpcode::G_LSHR:
9002 case TargetOpcode::G_ASHR: {
9003 std::optional<unsigned> ShAmt = getValidConstShiftAmt(
9004 getConstantOrConstantSplatVector(MI.getOperand(2).getReg()),
9005 RootDemand.getBitWidth());
9006 if (!ShAmt)
9007 return false;
9008 APInt SrcDemand =
9009 getDemandedSrcBitsForShiftConst(Opcode, RootDemand, *ShAmt);
9010 KnownBits SrcKnown(RootDemand.getBitWidth());
9011 return Probe(/*OpNo=*/1, SrcDemand, SrcKnown);
9012 }
9013 default:
9014 return false;
9015 case TargetOpcode::G_AND:
9016 case TargetOpcode::G_OR: {
9017 // Let redundant_and/redundant_or fold the root before operand rewrites can
9018 // discard the known bits that make it redundant.
9019 KnownBits L = VT->getKnownBits(MI.getOperand(1).getReg());
9020 KnownBits R = VT->getKnownBits(MI.getOperand(2).getReg());
9021 bool RootRedundant = Opcode == TargetOpcode::G_AND
9022 ? RootDemand.isSubsetOf(L.Zero | R.One) ||
9023 RootDemand.isSubsetOf(R.Zero | L.One)
9024 : RootDemand.isSubsetOf(L.One | R.Zero) ||
9025 RootDemand.isSubsetOf(R.One | L.Zero);
9026 if (RootRedundant)
9027 return false;
9028
9029 KnownBits RHSKnown(RootDemand.getBitWidth());
9030 if (Probe(/*OpNo=*/2, RootDemand, RHSKnown))
9031 return true;
9032
9033 APInt LHSDemand = getDemandedBitsForLogicalOp(Opcode, RootDemand, RHSKnown);
9034
9035 KnownBits LHSKnown(RootDemand.getBitWidth());
9036 if (Probe(/*OpNo=*/1, LHSDemand, LHSKnown))
9037 return true;
9038
9039 Register RHS = MI.getOperand(2).getReg();
9040 if (!MRI.hasOneNonDBGUse(RHS) || getConstantOrConstantSplatVector(RHS))
9041 return false;
9042
9043 APInt RHSDemand =
9044 LHSDemand == RootDemand
9045 ? getDemandedBitsForLogicalOp(Opcode, RootDemand, LHSKnown)
9047 Opcode, RootDemand,
9048 VT->getKnownBits(MI.getOperand(1).getReg()));
9049 KnownBits RetryKnown(RootDemand.getBitWidth());
9050 return RHSDemand != RootDemand && Probe(/*OpNo=*/2, RHSDemand, RetryKnown);
9051 }
9052 }
9053}
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
const TargetLowering & TLI
SDValue Pred
unsigned RegSize
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
uint64_t uint64_t
uint64_t Imm
AMDGPU Register Bank Select
Rewrite undef for PHI
This file declares a class to represent arbitrary precision floating point values and provide a varie...
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static const Function * getParent(const Value *V)
#define X(NUM, ENUM, NAME)
Definition ELF.h:665
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static APInt getDemandedBitsForLogicalOp(unsigned Opcode, const APInt &Demanded, const KnownBits &OtherKnown)
static bool hasMoreUses(const MachineInstr &MI0, const MachineInstr &MI1, const MachineRegisterInfo &MRI)
static bool isContractableFMul(MachineInstr &MI, bool AllowFusionGlobally)
Checks if MI is TargetOpcode::G_FMUL and contractable either due to global flags or MachineInstr flag...
static unsigned getIndexedOpc(unsigned LdStOpc)
static std::optional< unsigned > getValidConstShiftAmt(const std::optional< APInt > &Amt, unsigned BW)
static APInt getDemandedSrcBitsForShiftConst(unsigned Opcode, const APInt &DemandedBits, unsigned ShAmt)
static std::optional< std::pair< GZExtLoad *, int64_t > > matchLoadAndBytePosition(Register Reg, unsigned MemSizeInBits, const MachineRegisterInfo &MRI)
Helper function for findLoadOffsetsForLoadOrCombine.
static std::optional< unsigned > getMinUselessShift(KnownBits ValueKB, unsigned Opcode, std::optional< int64_t > &Result)
Return the minimum useless shift amount that results in complete loss of the source value.
static Register peekThroughBitcast(Register Reg, const MachineRegisterInfo &MRI)
static unsigned bigEndianByteAt(const unsigned ByteWidth, const unsigned I)
static cl::opt< bool > ForceLegalIndexing("force-legal-indexing", cl::Hidden, cl::init(false), cl::desc("Force all indexed operations to be " "legal for the GlobalISel combiner"))
static void commuteMask(MutableArrayRef< int > Mask, const unsigned NumElems)
static cl::opt< unsigned > PostIndexUseThreshold("post-index-use-threshold", cl::Hidden, cl::init(32), cl::desc("Number of uses of a base pointer to check before it is no longer " "considered for post-indexing."))
static std::optional< bool > isBigEndian(const SmallDenseMap< int64_t, int64_t, 8 > &MemOffset2Idx, int64_t LowestIdx)
Given a map from byte offsets in memory to indices in a load/store, determine if that map corresponds...
static bool vectorFPFoldUsesIEEEDenormals(const MachineFunction &MF, LLT Ty)
static unsigned getExtLoadOpcForExtend(unsigned ExtOpc)
static bool isConstValidTrue(const TargetLowering &TLI, unsigned ScalarSizeBits, int64_t Cst, bool IsVector, bool IsFP)
static LLT getMidVTForTruncRightShiftCombine(LLT ShiftTy, LLT TruncTy)
static bool canFoldInAddressingMode(GLoadStore *MI, const TargetLowering &TLI, MachineRegisterInfo &MRI)
Return true if 'MI' is a load or a store that may be fold it's address operand into the load / store ...
static unsigned littleEndianByteAt(const unsigned ByteWidth, const unsigned I)
static Register buildLogBase2(Register V, MachineIRBuilder &MIB)
Determines the LogBase2 value for a non-null input value using the transform: LogBase2(V) = (EltBits ...
This contains common combine transformations that may be used in a combine pass,or by the target else...
Utilities for dealing with flags related to floating point properties and mode controls.
This contains common code to allow clients to notify changes to machine instr.
Provides analysis for querying information about KnownBits during GISel passes.
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
#define _
IRTranslator LLVM IR MI
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Interface for Targets to specify which operations they can successfully select and how the others sho...
static bool isConstantSplatVector(SDValue N, APInt &SplatValue, unsigned MinSizeInBits)
Implement a low-level type suitable for MachineInstr level instruction selection.
#define I(x, y, z)
Definition MD5.cpp:57
Contains matchers for matching SSA Machine Instructions.
This file declares the MachineIRBuilder class.
Register Reg
#define R2(n)
Promote Memory to Register
Definition Mem2Reg.cpp:110
#define T
static MCRegister getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
uint64_t IntrinsicInst * II
SmallVector< MachineOperand, 4 > Cond
int64_t C1
MachineRegisterInfo & MRI
Remove Loads Into Fake Uses
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
SI Fold Operands
This file contains some templates that are useful if you are working with the STL at all.
This file implements a set that has insertion order iteration characteristics.
This file implements the SmallBitVector class.
#define LLVM_DEBUG(...)
Definition Debug.h:119
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
This file describes how to lower LLVM code to machine code.
Value * RHS
Value * LHS
static constexpr roundingMode rmNearestTiesToEven
Definition APFloat.h:361
opStatus fusedMultiplyAdd(const APFloat &Multiplicand, const APFloat &Addend, roundingMode RM)
Definition APFloat.h:1339
Class for arbitrary precision integers.
Definition APInt.h:78
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition APInt.h:230
LLVM_ABI APInt zext(unsigned width) const
Zero extend to a new width.
Definition APInt.cpp:1057
uint64_t getZExtValue() const
Get zero extended value.
Definition APInt.h:1560
LLVM_ABI APInt trunc(unsigned width) const
Truncate to new width.
Definition APInt.cpp:970
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
Definition APInt.h:202
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
Definition APInt.h:367
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
Definition APInt.h:376
LLVM_ABI APInt urem(const APInt &RHS) const
Unsigned remainder operation.
Definition APInt.cpp:1695
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition APInt.h:1508
bool ult(const APInt &RHS) const
Unsigned less than comparison.
Definition APInt.h:1115
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
Definition APInt.h:205
bool isNegative() const
Determine sign of this APInt.
Definition APInt.h:325
int32_t exactLogBase2() const
Definition APInt.h:1803
void ashrInPlace(unsigned ShiftAmt)
Arithmetic right-shift this APInt by ShiftAmt in place.
Definition APInt.h:836
unsigned countr_zero() const
Count the number of trailing zero bits.
Definition APInt.h:1659
unsigned countl_zero() const
The APInt version of std::countl_zero.
Definition APInt.h:1618
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
Definition APInt.h:215
unsigned countLeadingZeros() const
Definition APInt.h:1626
bool isStrictlyPositive() const
Determine if this APInt Value is positive.
Definition APInt.h:352
static bool isSameValue(const APInt &I1, const APInt &I2, bool SignedCompare=false)
Determine if two APInts have the same value, after zero-extending or sign-extending (if SignedCompare...
Definition APInt.h:550
LLVM_ABI APInt multiplicativeInverse() const
Definition APInt.cpp:1303
bool isMask(unsigned numBits) const
Definition APInt.h:484
LLVM_ABI APInt sext(unsigned width) const
Sign extend to a new width.
Definition APInt.cpp:1030
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Definition APInt.h:1261
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
Definition APInt.h:436
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
Definition APInt.h:302
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
Definition APInt.h:196
bool isOne() const
Determine if this is a value of 1.
Definition APInt.h:385
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
Definition APInt.h:235
int64_t getSExtValue() const
Get sign extended value.
Definition APInt.h:1582
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
Definition APInt.h:860
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
Definition APInt.h:853
unsigned countr_one() const
Count the number of trailing one bits.
Definition APInt.h:1676
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
size_t size() const
Get the array size.
Definition ArrayRef.h:141
bool isEquality() const
Determine if this is an equals/not equals predicate.
Definition InstrTypes.h:978
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:740
@ FCMP_TRUE
1 1 1 1 Always true (always folded)
Definition InstrTypes.h:757
@ ICMP_SLT
signed less than
Definition InstrTypes.h:769
@ ICMP_SLE
signed less or equal
Definition InstrTypes.h:770
@ FCMP_OLT
0 1 0 0 True if ordered and less than
Definition InstrTypes.h:746
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
Definition InstrTypes.h:755
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
Definition InstrTypes.h:744
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
Definition InstrTypes.h:745
@ ICMP_UGE
unsigned greater or equal
Definition InstrTypes.h:764
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:763
@ ICMP_SGT
signed greater than
Definition InstrTypes.h:767
@ FCMP_ULT
1 1 0 0 True if unordered or less than
Definition InstrTypes.h:754
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:765
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
Definition InstrTypes.h:752
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
Definition InstrTypes.h:747
@ ICMP_SGE
signed greater or equal
Definition InstrTypes.h:768
@ ICMP_ULE
unsigned less or equal
Definition InstrTypes.h:766
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
Definition InstrTypes.h:753
@ FCMP_FALSE
0 0 0 0 Always false (always folded)
Definition InstrTypes.h:742
static LLVM_ABI bool isEquality(Predicate pred)
Determine if this is an equals/not equals predicate.
Predicate getSwappedPredicate() const
For example, EQ->EQ, SLE->SGE, ULT->UGT, OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
Definition InstrTypes.h:890
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE,...
Definition InstrTypes.h:852
static LLVM_ABI bool isOrdered(Predicate predicate)
Determine if the predicate is an ordered operation.
LLVM_ABI void applyCombineBuildVectorOfBitcast(MachineInstr &MI, SmallVector< Register > &Ops) const
LLVM_ABI void applyCombineExtendingLoads(MachineInstr &MI, PreferredTuple &MatchInfo) const
LLVM_ABI bool matchRepeatedFPDivisor(MachineInstr &MI, SmallVector< MachineInstr * > &MatchInfo) const
LLVM_ABI bool isFPConstantLegalOrBeforeLegalizer(const LLT Ty) const
LLVM_ABI bool matchFoldC2MinusAPlusC1(const MachineInstr &MI, BuildFnTy &MatchInfo) const
LLVM_ABI bool matchLoadOrCombine(MachineInstr &MI, BuildFnTy &MatchInfo) const
Match expression trees of the form.
LLVM_ABI const RegisterBank * getRegBank(Register Reg) const
Get the register bank of Reg.
LLVM_ABI bool matchEqualDefs(const MachineOperand &MOP1, const MachineOperand &MOP2) const
Return true if MOP1 and MOP2 are register operands are defined by equivalent instructions.
LLVM_ABI void applyUDivOrURemByConst(MachineInstr &MI) const
LLVM_ABI void applyCombineUnmergeWithDeadLanesToTrunc(MachineInstr &MI) const
LLVM_ABI bool matchUnmergeValuesAnyExtBuildVector(const MachineInstr &MI, BuildFnTy &MatchInfo) const
LLVM_ABI bool matchFsubToFneg(MachineInstr &MI) const
LLVM_ABI bool matchCtls(MachineInstr &CtlzMI, BuildFnTy &MatchInfo) const
LLVM_ABI bool matchSelectSameVal(MachineInstr &MI) const
Optimize (cond ? x : x) -> x.
LLVM_ABI bool matchReassocConstantInnerRHS(GPtrAdd &MI, MachineInstr *RHS, BuildFnTy &MatchInfo) const
LLVM_ABI bool matchBitfieldExtractFromShr(MachineInstr &MI, BuildFnTy &MatchInfo) const
Match: shr (shl x, n), k -> sbfx/ubfx x, pos, width.
LLVM_ABI bool matchFoldAMinusC1PlusC2(const MachineInstr &MI, BuildFnTy &MatchInfo) const
LLVM_ABI bool matchTruncSSatU(MachineInstr &MI, Register &MatchInfo) const
LLVM_ABI void applySimplifyURemByPow2(MachineInstr &MI) const
Combine G_UREM x, (known power of 2) to an add and bitmasking.
LLVM_ABI bool matchCombineUnmergeZExtToZExt(MachineInstr &MI) const
Transform X, Y = G_UNMERGE(G_ZEXT(Z)) -> X = G_ZEXT(Z); Y = G_CONSTANT 0.
LLVM_ABI bool matchSimplifyDemandedBits(MachineInstr &MI, BuildFnTy &MatchInfo) const
Match demanded-bits simplification of a root's register use.
const TargetInstrInfo * TII
LLVM_ABI void applyCombineConcatVectors(MachineInstr &MI, SmallVector< Register > &Ops) const
Replace MI with a flattened build_vector with Ops or an implicit_def if Ops is empty.
LLVM_ABI void applyXorOfAndWithSameReg(MachineInstr &MI, std::pair< Register, Register > &MatchInfo) const
LLVM_ABI bool canCombineFMadOrFMA(MachineInstr &MI, bool &AllowFusionGlobally, bool &HasFMAD, bool &Aggressive, bool CanReassociate=false) const
LLVM_ABI bool matchFoldAPlusC1MinusC2(const MachineInstr &MI, BuildFnTy &MatchInfo) const
LLVM_ABI bool matchExtractVecEltBuildVec(MachineInstr &MI, Register &Reg) const
LLVM_ABI void applyCombineUnmergeConstant(MachineInstr &MI, SmallVectorImpl< APInt > &Csts) const
LLVM_ABI bool matchCombineFAddFpExtFMulToFMadOrFMA(MachineInstr &MI, BuildFnTy &MatchInfo) const
Transform (fadd (fpext (fmul x, y)), z) -> (fma (fpext x), (fpext y), z) (fadd (fpext (fmul x,...
LLVM_ABI bool matchCombineIndexedLoadStore(MachineInstr &MI, IndexedLoadStoreMatchInfo &MatchInfo) const
LLVM_ABI void applyCombineShuffleConcat(MachineInstr &MI, SmallVector< Register > &Ops) const
Replace MI with a flattened build_vector with Ops or an implicit_def if Ops is empty.
LLVM_ABI void replaceSingleDefInstWithReg(MachineInstr &MI, Register Replacement) const
Delete MI and replace all of its uses with Replacement.
LLVM_ABI void applyCombineShuffleToBuildVector(MachineInstr &MI) const
Replace shuffle MI with a build_vector.
LLVM_ABI bool matchCombineExtractedVectorLoad(MachineInstr &MI, BuildFnTy &MatchInfo) const
Combine a G_EXTRACT_VECTOR_ELT of a load into a narrowed load.
LLVM_ABI void replaceRegWith(MachineRegisterInfo &MRI, Register FromReg, Register ToReg) const
MachineRegisterInfo::replaceRegWith() and inform the observer of the changes.
LLVM_ABI void replaceRegOpWith(MachineRegisterInfo &MRI, MachineOperand &FromRegOp, Register ToReg) const
Replace a single register operand with a new register and inform the observer of the changes.
LLVM_ABI void applyCombineMemCpyFamily(MachineInstr &MI, MemCpyFamilyLoweringInfo &MatchInfo) const
LLVM_ABI bool matchReassocCommBinOp(MachineInstr &MI, BuildFnTy &MatchInfo) const
Reassociate commutative binary operations like G_ADD.
LLVM_ABI void applyBuildFnMO(const MachineOperand &MO, BuildFnTy &MatchInfo) const
Use a function which takes in a MachineIRBuilder to perform a combine.
LLVM_ABI bool matchCommuteConstantToRHS(MachineInstr &MI) const
Match constant LHS ops that should be commuted.
LLVM_ABI bool matchSimplifyNegMinMax(MachineInstr &MI, BuildFnTy &MatchInfo) const
Tranform (neg (min/max x, (neg x))) into (max/min x, (neg x)).
LLVM_ABI bool matchCombineDivRem(MachineInstr &MI, MachineInstr *&OtherMI) const
Try to combine G_[SU]DIV and G_[SU]REM into a single G_[SU]DIVREM when their source operands are iden...
LLVM_ABI void applyUMulHToLShr(MachineInstr &MI) const
LLVM_ABI void applyNotCmp(MachineInstr &MI, SmallVectorImpl< Register > &RegsToNegate) const
LLVM_ABI bool isLegalOrHasFewerElements(const LegalityQuery &Query) const
LLVM_ABI bool matchShiftImmedChain(MachineInstr &MI, RegisterImmPair &MatchInfo) const
Fold (shift (shift base, x), y) -> (shift base (x+y))
LLVM_ABI bool matchTruncLshrBuildVectorFold(MachineInstr &MI, Register &MatchInfo) const
LLVM_ABI bool matchAllExplicitUsesAreUndef(MachineInstr &MI) const
Return true if all register explicit use operands on MI are defined by a G_IMPLICIT_DEF.
LLVM_ABI bool isPredecessor(const MachineInstr &DefMI, const MachineInstr &UseMI) const
Returns true if DefMI precedes UseMI or they are the same instruction.
LLVM_ABI bool matchPtrAddImmedChain(MachineInstr &MI, PtrAddChain &MatchInfo) const
LLVM_ABI bool matchTruncSSatS(MachineInstr &MI, Register &MatchInfo) const
LLVM_ABI const TargetLowering & getTargetLowering() const
LLVM_ABI bool matchShuffleUndefRHS(MachineInstr &MI, BuildFnTy &MatchInfo) const
Remove references to rhs if it is undef.
LLVM_ABI void applyBuildInstructionSteps(MachineInstr &MI, InstructionStepsMatchInfo &MatchInfo) const
Replace MI with a series of instructions described in MatchInfo.
LLVM_ABI void applySDivByPow2(MachineInstr &MI) const
LLVM_ABI bool matchConstantFoldFPBinOp(MachineInstr &MI, BuildFnTy &MatchInfo) const
Do constant FP folding when opportunities are exposed after MIR building.
LLVM_ABI void applyUDivByPow2(MachineInstr &MI) const
Given an G_UDIV MI expressing an unsigned divided by a pow2 constant, return expressions that impleme...
LLVM_ABI bool matchOr(MachineInstr &MI, BuildFnTy &MatchInfo) const
Combine ors.
LLVM_ABI bool matchLshrOfTruncOfLshr(MachineInstr &MI, LshrOfTruncOfLshr &MatchInfo, MachineInstr &ShiftMI) const
Fold (lshr (trunc (lshr x, C1)), C2) -> trunc (shift x, (C1 + C2))
LLVM_ABI void replaceInstWithConstant(MachineInstr &MI, int64_t C) const
Replace an instruction with a G_CONSTANT with value C.
LLVM_ABI bool matchCombineFSubFpExtFMulToFMadOrFMA(MachineInstr &MI, BuildFnTy &MatchInfo) const
Transform (fsub (fpext (fmul x, y)), z) -> (fma (fpext x), (fpext y), (fneg z)) (fsub (fpext (fmul x,...
LLVM_ABI bool matchConstantLargerBitWidth(MachineInstr &MI, unsigned ConstIdx) const
Checks if constant at ConstIdx is larger than MI 's bitwidth.
LLVM_ABI void applyCombineCopy(MachineInstr &MI) const
LLVM_ABI bool matchCombineShlOfExtend(MachineInstr &MI, RegisterImmPair &MatchData) const
LLVM_ABI bool matchCombineFSubFMulToFMadOrFMA(MachineInstr &MI, BuildFnTy &MatchInfo) const
Transform (fsub (fmul x, y), z) -> (fma x, y, -z) (fsub (fmul x, y), z) -> (fmad x,...
LLVM_ABI bool matchCombineFAddFMAFMulToFMadOrFMA(MachineInstr &MI, BuildFnTy &MatchInfo) const
Transform (fadd (fma x, y, (fmul u, v)), z) -> (fma x, y, (fma u, v, z)) (fadd (fmad x,...
LLVM_ABI bool matchSextTruncSextLoad(MachineInstr &MI) const
LLVM_ABI bool matchCombineInsertVecElts(MachineInstr &MI, SmallVectorImpl< Register > &MatchInfo) const
LLVM_ABI bool matchCombineBuildUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI, Register &UnmergeSrc) const
LLVM_ABI bool matchDivByPow2(MachineInstr &MI, bool IsSigned) const
Given an G_SDIV MI expressing a signed divided by a pow2 constant, return expressions that implements...
LLVM_ABI bool matchNarrowBinopFeedingAnd(MachineInstr &MI, BuildFnTy &MatchInfo) const
LLVM_ABI bool matchCombineLoadWithAndMask(MachineInstr &MI, BuildFnTy &MatchInfo) const
Match (and (load x), mask) -> zextload x.
LLVM_ABI bool matchCombineFAddFMulToFMadOrFMA(MachineInstr &MI, BuildFnTy &MatchInfo) const
Transform (fadd (fmul x, y), z) -> (fma x, y, z) (fadd (fmul x, y), z) -> (fmad x,...
LLVM_ABI bool matchCombineCopy(MachineInstr &MI) const
LLVM_ABI bool matchExtendThroughPhis(MachineInstr &MI, MachineInstr *&ExtMI) const
LLVM_ABI void applyShiftImmedChain(MachineInstr &MI, RegisterImmPair &MatchInfo) const
LLVM_ABI bool matchXorOfAndWithSameReg(MachineInstr &MI, std::pair< Register, Register > &MatchInfo) const
Fold (xor (and x, y), y) -> (and (not x), y) {.
LLVM_ABI bool matchCombineShuffleVector(MachineInstr &MI, SmallVectorImpl< Register > &Ops) const
Check if the G_SHUFFLE_VECTOR MI can be replaced by a concat_vectors.
LLVM_ABI void replaceInstWithFConstant(MachineInstr &MI, double C) const
Replace an instruction with a G_FCONSTANT with value C.
LLVM_ABI bool matchOrShiftToFunnelShift(MachineInstr &MI, bool AllowScalarConstants, BuildFnTy &MatchInfo) const
LLVM_ABI bool matchRedundantSExtInReg(MachineInstr &MI) const
LLVM_ABI void replaceOpcodeWith(MachineInstr &FromMI, unsigned ToOpcode) const
Replace the opcode in instruction with a new opcode and inform the observer of the changes.
LLVM_ABI void applyFunnelShiftConstantModulo(MachineInstr &MI) const
Replaces the shift amount in MI with ShiftAmt % BW.
LLVM_ABI bool matchFoldC1Minus2MinusC2(const MachineInstr &MI, BuildFnTy &MatchInfo) const
LLVM_ABI void applyCombineShlOfExtend(MachineInstr &MI, const RegisterImmPair &MatchData) const
LLVM_ABI void applyUseVectorTruncate(MachineInstr &MI, Register &MatchInfo) const
LLVM_ABI CombinerHelper(GISelChangeObserver &Observer, MachineIRBuilder &B, bool IsPreLegalize, GISelValueTracking *VT=nullptr, MachineDominatorTree *MDT=nullptr, const LegalizerInfo *LI=nullptr)
LLVM_ABI bool matchShuffleDisjointMask(MachineInstr &MI, BuildFnTy &MatchInfo) const
Turn shuffle a, b, mask -> shuffle undef, b, mask iff mask does not reference a.
LLVM_ABI bool matchCombineMulToShl(MachineInstr &MI, unsigned &ShiftVal) const
Transform a multiply by a power-of-2 value to a left shift.
LLVM_ABI void applyCombineShuffleVector(MachineInstr &MI, ArrayRef< Register > Ops) const
Replace MI with a concat_vectors with Ops.
LLVM_ABI bool matchCombineUnmergeUndef(MachineInstr &MI, std::function< void(MachineIRBuilder &)> &MatchInfo) const
Transform G_UNMERGE G_IMPLICIT_DEF -> G_IMPLICIT_DEF, G_IMPLICIT_DEF, ...
LLVM_ABI void applyFoldBinOpIntoSelect(MachineInstr &MI, const unsigned &SelectOpNo) const
SelectOperand is the operand in binary operator MI that is the select to fold.
LLVM_ABI bool matchFoldAMinusC1MinusC2(const MachineInstr &MI, BuildFnTy &MatchInfo) const
LLVM_ABI void applyCombineIndexedLoadStore(MachineInstr &MI, IndexedLoadStoreMatchInfo &MatchInfo) const
LLVM_ABI bool matchCombineShuffleConcat(MachineInstr &MI, SmallVector< Register > &Ops) const
LLVM_ABI void applySextInRegOfLoad(MachineInstr &MI, std::tuple< Register, unsigned > &MatchInfo) const
LLVM_ABI bool matchTruncUSatU(MachineInstr &MI, MachineInstr &MinMI) const
LLVM_ABI bool matchICmpToLHSKnownBits(MachineInstr &MI, BuildFnTy &MatchInfo) const
LLVM_ABI bool matchReassocPtrAdd(MachineInstr &MI, BuildFnTy &MatchInfo) const
Reassociate pointer calculations with G_ADD involved, to allow better addressing mode usage.
LLVM_ABI bool isPreLegalize() const
LLVM_ABI bool matchUndefShuffleVectorMask(MachineInstr &MI) const
Return true if a G_SHUFFLE_VECTOR instruction MI has an undef mask.
LLVM_ABI bool matchCombineSubToAdd(MachineInstr &MI, BuildFnTy &MatchInfo) const
LLVM_ABI bool matchShiftOfShiftedLogic(MachineInstr &MI, ShiftOfShiftedLogic &MatchInfo) const
If we have a shift-by-constant of a bitwise logic op that itself has a shift-by-constant operand with...
LLVM_ABI bool matchCombineConcatVectors(MachineInstr &MI, SmallVector< Register > &Ops) const
If MI is G_CONCAT_VECTORS, try to combine it.
LLVM_ABI bool matchInsertExtractVecEltOutOfBounds(MachineInstr &MI) const
Return true if a G_{EXTRACT,INSERT}_VECTOR_ELT has an out of range index.
LLVM_ABI bool matchOperandIsKnownToBeAPowerOfTwo(const MachineOperand &MO, bool OrNegative=false, bool OrZero=false) const
Check if operand MO is known to be a power of 2.
LLVM_ABI bool matchExtractAllEltsFromBuildVector(MachineInstr &MI, SmallVectorImpl< std::pair< Register, MachineInstr * > > &MatchInfo) const
LLVM_ABI LLVMContext & getContext() const
LLVM_ABI void applyPtrAddImmedChain(MachineInstr &MI, PtrAddChain &MatchInfo) const
LLVM_ABI bool isConstantLegalOrBeforeLegalizer(const LLT Ty) const
LLVM_ABI bool matchNotCmp(MachineInstr &MI, SmallVectorImpl< Register > &RegsToNegate) const
Combine inverting a result of a compare into the opposite cond code.
LLVM_ABI bool matchSextInRegOfLoad(MachineInstr &MI, std::tuple< Register, unsigned > &MatchInfo) const
Match sext_inreg(load p), imm -> sextload p.
LLVM_ABI bool matchSelectIMinMax(const MachineOperand &MO, BuildFnTy &MatchInfo) const
Combine select to integer min/max.
LLVM_ABI bool simplifyDemandedBits(MachineInstr &MI, unsigned OpNo, const APInt &DemandedBits, KnownBits &Known, unsigned Depth=0) const
Simplify a register use under DemandedBits and compute Known in the same walk.
LLVM_ABI bool matchConstantFoldUnaryIntOp(MachineInstr &MI, BuildFnTy &MatchInfo) const
Constant fold a unary integer op (G_CTLZ, G_CTTZ, G_CTPOP and their _ZERO_POISON variants,...
LLVM_ABI bool isLegal(const LegalityQuery &Query) const
LLVM_ABI bool matchShiftsTooBig(MachineInstr &MI, BuildFnTy &MatchInfo) const
Match shifts greater or equal to the range (the bitwidth of the result datatype, or the effective bit...
LLVM_ABI void applyCombineShuffleOfBuildVector(MachineInstr &MI) const
Replace shuffle of build_vector with a build_vector.
LLVM_ABI bool matchICmpToTrueFalseKnownBits(MachineInstr &MI, int64_t &MatchInfo) const
LLVM_ABI bool tryReassocBinOp(unsigned Opc, Register DstReg, Register Op0, Register Op1, BuildFnTy &MatchInfo) const
Try to reassociate to reassociate operands of a commutative binop.
LLVM_ABI void eraseInst(MachineInstr &MI) const
Erase MI.
LLVM_ABI void applyBuildFnNoErase(MachineInstr &MI, BuildFnTy &MatchInfo) const
Use a function which takes in a MachineIRBuilder to perform a combine.
LLVM_ABI bool matchUseVectorTruncate(MachineInstr &MI, Register &MatchInfo) const
LLVM_ABI bool matchConstantFoldBinOp(MachineInstr &MI, BuildFnTy &MatchInfo) const
Do constant folding when opportunities are exposed after MIR building.
MachineRegisterInfo & MRI
LLVM_ABI void applyExtendThroughPhis(MachineInstr &MI, MachineInstr *&ExtMI) const
LLVM_ABI bool matchConstantFPOp(const MachineOperand &MOP, double C) const
Return true if MOP is defined by a G_FCONSTANT or splat with a value exactly equal to C.
LLVM_ABI MachineInstr * buildUDivOrURemUsingMul(MachineInstr &MI) const
Given an G_UDIV MI or G_UREM MI expressing a divide by constant, return an expression that implements...
LLVM_ABI void applyExtractVecEltBuildVec(MachineInstr &MI, Register &Reg) const
LLVM_ABI bool matchFoldBinOpIntoSelect(MachineInstr &MI, unsigned &SelectOpNo) const
Push a binary operator through a select on constants.
LLVM_ABI bool tryCombineShiftToUnmerge(MachineInstr &MI, unsigned TargetShiftAmount) const
LLVM_ABI bool isConstantNaN(Register Reg) const
LLVM_ABI bool matchPtrAddZeroVector(MachineInstr &MI) const
}
LLVM_ABI bool tryCombineExtendingLoads(MachineInstr &MI) const
If MI is extend that consumes the result of a load, try to combine it.
LLVM_ABI bool isLegalOrBeforeLegalizer(const LegalityQuery &Query) const
LLVM_ABI bool matchBuildVectorIdentityFold(MachineInstr &MI, Register &MatchInfo) const
LLVM_ABI bool matchBitfieldExtractFromShrAnd(MachineInstr &MI, BuildFnTy &MatchInfo) const
Match: shr (and x, n), k -> ubfx x, pos, width.
LLVM_ABI void applyTruncSSatS(MachineInstr &MI, Register &MatchInfo) const
LLVM_ABI void applyRotateOutOfRange(MachineInstr &MI) const
LLVM_ABI bool matchReassocFoldConstantsInSubTree(GPtrAdd &MI, MachineInstr *LHS, MachineInstr *RHS, BuildFnTy &MatchInfo) const
LLVM_ABI bool matchHoistLogicOpWithSameOpcodeHands(MachineInstr &MI, InstructionStepsMatchInfo &MatchInfo) const
Match (logic_op (op x...), (op y...)) -> (op (logic_op x, y))
LLVM_ABI bool matchBitfieldExtractFromAnd(MachineInstr &MI, BuildFnTy &MatchInfo) const
Match: and (lshr x, cst), mask -> ubfx x, cst, width.
LLVM_ABI bool matchBitfieldExtractFromSExtInReg(MachineInstr &MI, BuildFnTy &MatchInfo) const
Form a G_SBFX from a G_SEXT_INREG fed by a right shift.
LLVM_ABI bool matchAndOrDisjointMask(MachineInstr &MI, BuildFnTy &MatchInfo) const
LLVM_ABI void replaceInstWithUndef(MachineInstr &MI) const
Replace an instruction with a G_IMPLICIT_DEF.
LLVM_ABI bool isDesirableToCommuteWithShift(const MachineInstr &MI) const
LLVM_ABI bool matchRedundantBinOpInEquality(MachineInstr &MI, BuildFnTy &MatchInfo) const
Transform: (X + Y) == X -> Y == 0 (X - Y) == X -> Y == 0 (X ^ Y) == X -> Y == 0 (X + Y) !...
LLVM_ABI bool matchConstantFoldFPUnary(MachineInstr &MI, BuildFnTy &MatchInfo) const
Constant fold a unary FP operation when the source is constant.
LLVM_ABI bool matchOptBrCondByInvertingCond(MachineInstr &MI, MachineInstr *&BrCond) const
If a brcond's true block is not the fallthrough, make it so by inverting the condition and swapping o...
LLVM_ABI bool matchAddOverflow(MachineInstr &MI, BuildFnTy &MatchInfo) const
Combine addos.
LLVM_ABI void applyAshShlToSextInreg(MachineInstr &MI, std::tuple< Register, int64_t > &MatchInfo) const
LLVM_ABI bool matchSelect(MachineInstr &MI, BuildFnTy &MatchInfo) const
Combine selects.
LLVM_ABI bool matchCombineExtendingLoads(MachineInstr &MI, PreferredTuple &MatchInfo) const
LLVM_ABI bool matchConstantFoldCastOp(MachineInstr &MI, BuildFnTy &MatchInfo) const
Do constant folding when opportunities are exposed after MIR building.
LLVM_ABI bool matchCombineUnmergeWithDeadLanesToTrunc(MachineInstr &MI) const
Transform X, Y<dead> = G_UNMERGE Z -> X = G_TRUNC Z.
LLVM_ABI bool matchRotateOutOfRange(MachineInstr &MI) const
LLVM_ABI void applyExpandFPowI(MachineInstr &MI, int64_t Exponent) const
Expands FPOWI into a series of multiplications and a division if the exponent is negative.
LLVM_ABI void setRegBank(Register Reg, const RegisterBank *RegBank) const
Set the register bank of Reg.
LLVM_ABI bool matchCommuteFPConstantToRHS(MachineInstr &MI) const
Match constant LHS FP ops that should be commuted.
LLVM_ABI void applyCombineDivRem(MachineInstr &MI, MachineInstr *&OtherMI) const
LLVM_ABI bool matchRedundantOr(MachineInstr &MI, Register &Replacement) const
LLVM_ABI void applyTruncSSatU(MachineInstr &MI, Register &MatchInfo) const
LLVM_ABI void applySimplifySRemByPow2(MachineInstr &MI) const
Combine G_SREM x, (+/-2^k) to a bias-and-mask sequence.
LLVM_ABI bool matchCombineFSubFpExtFNegFMulToFMadOrFMA(MachineInstr &MI, BuildFnTy &MatchInfo) const
Transform (fsub (fpext (fneg (fmul x, y))), z) -> (fneg (fma (fpext x), (fpext y),...
LLVM_ABI bool matchTruncBuildVectorFold(MachineInstr &MI, Register &MatchInfo) const
LLVM_ABI void applyCombineTruncOfShift(MachineInstr &MI, std::pair< MachineInstr *, LLT > &MatchInfo) const
LLVM_ABI bool matchConstantOp(const MachineOperand &MOP, int64_t C) const
Return true if MOP is defined by a G_CONSTANT or splat with a value equal to C.
const LegalizerInfo * LI
LLVM_ABI void applyCombineMulToShl(MachineInstr &MI, unsigned &ShiftVal) const
LLVM_ABI void applyCombineBuildUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B, Register &UnmergeSrc) const
LLVM_ABI bool matchUMulHToLShr(MachineInstr &MI) const
MachineDominatorTree * MDT
LLVM_ABI bool matchSimplifySelectToMinMax(MachineInstr &MI, BuildFnTy &MatchInfo) const
LLVM_ABI void applyRepeatedFPDivisor(SmallVector< MachineInstr * > &MatchInfo) const
LLVM_ABI bool matchTruncUSatUToFPTOUISat(MachineInstr &MI, MachineInstr &SrcMI) const
const RegisterBankInfo * RBI
LLVM_ABI bool matchMulOBy0(MachineInstr &MI, BuildFnTy &MatchInfo) const
Match: (G_*MULO x, 0) -> 0 + no carry out.
GISelValueTracking * VT
LLVM_ABI bool matchBinopWithNeg(MachineInstr &MI, BuildFnTy &MatchInfo) const
Fold a bitwiseop (~b +/- c) -> a bitwiseop ~(b -/+ c)
LLVM_ABI bool matchCombineUnmergeConstant(MachineInstr &MI, SmallVectorImpl< APInt > &Csts) const
Transform G_UNMERGE Constant -> Constant1, Constant2, ...
LLVM_ABI bool matchCountZeroToZeroPoison(MachineInstr &MI, unsigned Opc, unsigned ZeroPoisonOpc) const
LLVM_ABI void applyShiftOfShiftedLogic(MachineInstr &MI, ShiftOfShiftedLogic &MatchInfo) const
const TargetRegisterInfo * TRI
LLVM_ABI bool matchRedundantAnd(MachineInstr &MI, Register &Replacement) const
LLVM_ABI bool dominates(const MachineInstr &DefMI, const MachineInstr &UseMI) const
Returns true if DefMI dominates UseMI.
GISelChangeObserver & Observer
LLVM_ABI void applyBuildFn(MachineInstr &MI, BuildFnTy &MatchInfo) const
Use a function which takes in a MachineIRBuilder to perform a combine.
LLVM_ABI bool matchCombineTruncOfShift(MachineInstr &MI, std::pair< MachineInstr *, LLT > &MatchInfo) const
Transform trunc (shl x, K) to shl (trunc x), K if K < VT.getScalarSizeInBits().
LLVM_ABI bool matchCombineShiftToUnmerge(MachineInstr &MI, unsigned TargetShiftSize, unsigned &ShiftVal) const
Reduce a shift by a constant to an unmerge and a shift on a half sized type.
LLVM_ABI bool matchUDivOrURemByConst(MachineInstr &MI) const
Combine G_UDIV or G_UREM by constant into a multiply by magic constant.
LLVM_ABI bool matchAnd(MachineInstr &MI, BuildFnTy &MatchInfo) const
Combine ands.
LLVM_ABI bool matchSuboCarryOut(const MachineInstr &MI, BuildFnTy &MatchInfo) const
LLVM_ABI bool matchCombineFSubFNegFMulToFMadOrFMA(MachineInstr &MI, BuildFnTy &MatchInfo) const
Transform (fsub (fneg (fmul, x, y)), z) -> (fma (fneg x), y, (fneg z)) (fsub (fneg (fmul,...
LLVM_ABI bool matchCombineZextTrunc(MachineInstr &MI, Register &Reg) const
Transform zext(trunc(x)) to x.
LLVM_ABI bool matchUDivByShiftedPow2(MachineInstr &MI, BuildFnTy &MatchInfo) const
Combine unsigned division by a shifted power of two into a right shift.
LLVM_ABI bool matchSameRegOrICstOrSplat(Register A, Register B) const
True when A and B are the same register, including through copies, or different registers holding the...
LLVM_ABI void applyLshrOfTruncOfLshr(MachineInstr &MI, LshrOfTruncOfLshr &MatchInfo) const
LLVM_ABI bool tryCombineMemCpyFamily(MachineInstr &MI, unsigned MaxLen=0) const
Optimize memcpy intrinsics et al, e.g.
LLVM_ABI bool matchFreezeOfSingleMaybePoisonOperand(MachineInstr &MI, BuildFnTy &MatchInfo) const
LLVM_ABI void applySDivOrSRemByConst(MachineInstr &MI) const
LLVM_ABI bool matchCombineMemCpyFamily(MachineInstr &MI, MemCpyFamilyLoweringInfo &MatchInfo, unsigned MaxLen=0) const
LLVM_ABI MachineInstr * buildSDivOrSRemUsingMul(MachineInstr &MI) const
Given an G_SDIV MI or G_SREM MI expressing a signed divide by constant, return an expression that imp...
LLVM_ABI bool isLegalOrHasWidenScalar(const LegalityQuery &Query) const
LLVM_ABI bool matchSignedDivRemToUnsigned(MachineInstr &MI, unsigned UnsignedOpcode) const
Convert signed division or remainder of nonnegative operands to unsigned.
LLVM_ABI bool matchReassocConstantInnerLHS(GPtrAdd &MI, MachineInstr *LHS, MachineInstr *RHS, BuildFnTy &MatchInfo) const
LLVM_ABI bool matchOverlappingAnd(MachineInstr &MI, BuildFnTy &MatchInfo) const
Fold and(and(x, C1), C2) -> C1&C2 ? and(x, C1&C2) : 0.
LLVM_ABI bool matchCombineAnyExtTrunc(MachineInstr &MI, Register &Reg) const
Transform anyext(trunc(x)) to x.
LLVM_ABI void applyExtractAllEltsFromBuildVector(MachineInstr &MI, SmallVectorImpl< std::pair< Register, MachineInstr * > > &MatchInfo) const
MachineIRBuilder & Builder
LLVM_ABI bool isZeroOrOneVector(Register Reg) const
LLVM_ABI void replaceSingleDefInstWithOperand(MachineInstr &MI, unsigned OpIdx) const
Delete MI and replace all of its uses with its OpIdx-th operand.
LLVM_ABI const MachineFunction & getMachineFunction() const
LLVM_ABI bool matchCombineBuildVectorOfBitcast(MachineInstr &MI, SmallVector< Register > &Ops) const
Combine G_BUILD_VECTOR(G_UNMERGE(G_BITCAST), Undef) to G_BITCAST(G_BUILD_VECTOR(.....
LLVM_ABI bool matchCombineFAddFpExtFMulToFMadOrFMAAggressive(MachineInstr &MI, BuildFnTy &MatchInfo) const
LLVM_ABI bool matchConstantFoldFMA(MachineInstr &MI, BuildFnTy &MatchInfo) const
Constant fold G_FMA/G_FMAD.
LLVM_ABI bool matchSDivOrSRemByConst(MachineInstr &MI) const
Combine G_SDIV or G_SREM by constant into a multiply by magic constant.
LLVM_ABI void applyOptBrCondByInvertingCond(MachineInstr &MI, MachineInstr *&BrCond) const
LLVM_ABI void applyCombineShiftToUnmerge(MachineInstr &MI, const unsigned &ShiftVal) const
LLVM_ABI bool matchFPowIExpansion(MachineInstr &MI, int64_t Exponent) const
Match FPOWI if it's safe to extend it into a series of multiplications.
LLVM_ABI void applyCombineInsertVecElts(MachineInstr &MI, SmallVectorImpl< Register > &MatchInfo) const
LLVM_ABI bool matchCombineUnmergeMergeToPlainValues(MachineInstr &MI, SmallVectorImpl< Register > &Operands) const
Transform <ty,...> G_UNMERGE(G_MERGE ty X, Y, Z) -> ty X, Y, Z.
LLVM_ABI void applyCombineUnmergeMergeToPlainValues(MachineInstr &MI, SmallVectorImpl< Register > &Operands) const
LLVM_ABI bool matchAshrShlToSextInreg(MachineInstr &MI, std::tuple< Register, int64_t > &MatchInfo) const
Match ashr (shl x, C), C -> sext_inreg (C)
LLVM_ABI void applyCombineUnmergeZExtToZExt(MachineInstr &MI) const
ConstantFP - Floating Point Values [float, double].
Definition Constants.h:420
const APFloat & getValueAPF() const
Definition Constants.h:463
This class represents a range of values.
LLVM_ABI std::optional< ConstantRange > exactUnionWith(const ConstantRange &CR) const
Union the two ranges and return the result if it can be represented exactly, otherwise return std::nu...
LLVM_ABI ConstantRange subtract(const APInt &CI) const
Subtract the specified constant from the endpoints of this constant range.
static LLVM_ABI ConstantRange fromKnownBits(const KnownBits &Known, bool IsSigned)
Initialize a range based on a known bits constraint.
const APInt & getLower() const
Return the lower value for this range.
LLVM_ABI OverflowResult unsignedSubMayOverflow(const ConstantRange &Other) const
Return whether unsigned sub of the two ranges always/never overflows.
LLVM_ABI OverflowResult unsignedAddMayOverflow(const ConstantRange &Other) const
Return whether unsigned add of the two ranges always/never overflows.
LLVM_ABI bool isWrappedSet() const
Return true if this set wraps around the unsigned domain.
const APInt & getUpper() const
Return the upper value for this range.
static LLVM_ABI ConstantRange makeExactICmpRegion(CmpInst::Predicate Pred, const APInt &Other)
Produce the exact range such that all values in the returned range satisfy the given predicate with a...
LLVM_ABI OverflowResult signedAddMayOverflow(const ConstantRange &Other) const
Return whether signed add of the two ranges always/never overflows.
@ AlwaysOverflowsHigh
Always overflows in the direction of signed/unsigned max value.
@ AlwaysOverflowsLow
Always overflows in the direction of signed/unsigned min value.
@ MayOverflow
May or may not overflow.
LLVM_ABI OverflowResult signedSubMayOverflow(const ConstantRange &Other) const
Return whether signed sub of the two ranges always/never overflows.
This is an important base class in LLVM.
Definition Constant.h:44
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
bool isBigEndian() const
Definition DataLayout.h:221
iterator find(const_arg_type_t< KeyT > Val)
Definition DenseMap.h:766
iterator end()
Definition DenseMap.h:686
unsigned size() const
Definition DenseMap.h:717
ValueT lookup(const_arg_type_t< KeyT > Val) const
Return the entry for the specified key, or a default constructed value if no such entry exists.
Definition DenseMap.h:793
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&...Args)
Definition DenseMap.h:856
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition Function.cpp:352
Represents overflowing add operations.
Represents an integer addition.
Represents a logical and.
CmpInst::Predicate getCond() const
Register getLHSReg() const
Register getRHSReg() const
Represents any generic load, including sign/zero extending variants.
Register getDstReg() const
Get the definition register of the loaded value.
Register getCarryOutReg() const
Register getLHSReg() const
Register getRHSReg() const
Represents a G_BUILD_VECTOR.
Register getSrcReg() const
Represents a G_CONCAT_VECTORS.
Represent a G_ICMP.
Abstract class that contains various methods for clients to notify about changes.
Simple wrapper observer that takes several observers, and calls each one for each event.
Represents any type of generic load or store.
Register getPointerReg() const
Get the source register of the pointer value.
Represents a G_LOAD.
Represents a logical binary operation.
MachineMemOperand & getMMO() const
Get the MachineMemOperand on this instruction.
bool isAtomic() const
Returns true if the attached MachineMemOperand has the atomic flag set.
LocationSize getMemSizeInBits() const
Returns the size in bits of the memory access.
Register getSourceReg(unsigned I) const
Returns the I'th source register.
unsigned getNumSources() const
Returns the number of source registers.
Represents a logical or.
Represents a G_PTR_ADD.
Represents a G_SELECT.
Represents overflowing sub operations.
Represents an integer subtraction.
Represents a G_UNMERGE_VALUES.
unsigned getNumDefs() const
Returns the number of def registers.
Register getSourceReg() const
Get the unmerge source register.
Represents a G_ZEXTLOAD.
Represents a zext.
Register getReg(unsigned Idx) const
Access the Idx'th operand as a register and return it.
static LLVM_ABI bool compare(const APInt &LHS, const APInt &RHS, ICmpInst::Predicate Pred)
Return result of LHS Pred RHS comparison.
constexpr bool isScalableVector() const
Returns true if the LLT is a scalable vector.
constexpr unsigned getScalarSizeInBits() const
constexpr bool isScalar() const
constexpr LLT changeElementType(LLT NewEltTy) const
If this type is a vector, return a vector with the same number of elements but the new element type.
static constexpr LLT vector(ElementCount EC, unsigned ScalarSizeInBits)
Get a low-level vector of some number of elements and element width.
LLT getScalarType() const
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
constexpr bool isValid() const
constexpr uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
constexpr bool isFloat() const
constexpr bool isVector() const
constexpr bool isByteSized() const
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
constexpr bool isPointer() const
constexpr bool isAnyScalar() const
constexpr ElementCount getElementCount() const
static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
constexpr bool isPointerOrPointerVector() const
constexpr bool isFixedVector() const
Returns true if the LLT is a fixed vector.
static LLT integer(unsigned SizeInBits)
LLT getElementType() const
Returns the vector's element type. Only valid for vector types.
LLT changeElementSize(unsigned NewEltSize) const
If this type is a vector, return a vector with the same number of elements but the new element size.
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:76
LLVM_ABI LegalizeResult lowerMemCpyFamily(MachineInstr &MI, Register Dst, Register Src, uint64_t KnownLen, Align Alignment, bool DstAlignCanChange, ArrayRef< LLT > MemOps)
@ Legalized
Instruction has been legalized and the MachineFunction changed.
LLVM_ABI Register getVectorElementPointer(Register VecPtr, LLT VecTy, Register Index)
Get a pointer to vector element Index located in memory for a vector of type VecTy starting at a base...
TypeSize getValue() const
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode.
Definition MCInstrInfo.h:89
LLVM_ABI iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
MachineInstrBundleIterator< MachineInstr > iterator
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
DenormalMode getDenormalMode(const fltSemantics &FPType) const
Returns the denormal handling type for the default rounding mode of the function.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy, Align BaseAlignment, const MMOMetadata &Metadata=MMOMetadata(), SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
Helper class to build MachineInstr.
const TargetInstrInfo & getTII()
MachineInstrBuilder buildSub(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_SUB Op0, Op1.
MachineInstrBuilder buildCTLZ(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_CTLZ Op0, Src0.
MachineFunction & getMF()
Getter for the function we currently build.
MachineRegisterInfo * getMRI()
Getter for MRI.
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
bool mayLoadOrStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read or modify memory.
const MachineBasicBlock * getParent() const
LLVM_ABI bool isDereferenceableInvariantLoad() const
Return true if this load instruction never traps and points to a memory location whose value doesn't ...
bool getFlag(MIFlag Flag) const
Return whether an MI flag is set.
unsigned getNumOperands() const
Retuns the total number of operands.
static constexpr uint32_t getPoisonGeneratingFlags()
LLVM_ABI void setDesc(const MCInstrDesc &TID)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one.
LLVM_ABI unsigned getNumExplicitDefs() const
Returns the number of non-implicit definitions.
mop_range uses()
Returns all operands which may be register uses.
MachineOperand * findRegisterUseOperand(Register Reg, const TargetRegisterInfo *TRI, bool isKill=false)
Wrapper for findRegisterUseOperandIdx, it returns a pointer to the MachineOperand rather than an inde...
const MachineOperand & getOperand(unsigned i) const
uint32_t getFlags() const
Return the MI flags bitvector.
LLVM_ABI int findRegisterDefOperandIdx(Register Reg, const TargetRegisterInfo *TRI, bool isDead=false, bool Overlap=false) const
Returns the operand index that is a def of the specified register or -1 if it is not found.
LLVM_ABI MachineInstrBundleIterator< MachineInstr > eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
A description of a memory reference used in the backend.
LLT getMemoryType() const
Return the memory type of the memory reference.
unsigned getAddrSpace() const
bool isAtomic() const
Returns true if this operation has an atomic ordering requirement of unordered or higher,...
const MachinePointerInfo & getPointerInfo() const
LLVM_ABI Align getAlign() const
Return the minimum known alignment in bytes of the actual memory reference.
LocationSize getSizeInBits() const
Return the size in bits of the memory reference.
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
LLVM_ABI void setReg(Register Reg)
Change the register this operand corresponds to.
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
void setMBB(MachineBasicBlock *MBB)
void setPredicate(unsigned Predicate)
Register getReg() const
getReg - Returns the register number.
unsigned getPredicate() const
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Represent a mutable reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:294
This class implements the register bank concept.
Wrapper class representing virtual and physical registers.
Definition Register.h:20
constexpr bool isValid() const
Definition Register.h:112
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Definition Register.h:79
size_type size() const
Determine the number of elements in the SetVector.
Definition SetVector.h:103
size_type count(const_arg_type key) const
Count the number of elements of a given key in the SetVector.
Definition SetVector.h:268
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition SetVector.h:157
This is a 'bitvector' (really, a variable-sized bit array), optimized for the case when the array is ...
SmallBitVector & set()
bool all() const
Returns true if all bits are set.
size_type size() const
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
Definition SetVector.h:346
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition SmallSet.h:184
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void resize(size_type N)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
virtual LLVM_READONLY LLT getPreferredShiftAmountTy(LLT ShiftValueTy) const
Return the preferred type to use for a shift opcode, given the shifted amount type is ShiftValueTy.
bool isBeneficialToExpandPowI(int64_t Exponent, bool OptForSize) const
Return true if it is beneficial to expand an @llvm.powi.
virtual bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AddrSpace, Instruction *I=nullptr) const
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual bool isIndexingLegal(MachineInstr &MI, Register Base, Register Offset, bool IsPre, MachineRegisterInfo &MRI) const
Returns true if the specified base+offset is a legal indexed addressing mode for this target.
virtual bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const
Return true if it is profitable to move this shift by a constant amount through its operand,...
virtual unsigned combineRepeatedFPDivisors() const
Indicate whether this target prefers to combine FDIVs with the same divisor.
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
constexpr bool isKnownMultipleOf(ScalarTy RHS) const
This function tells the caller whether the element count is known at compile time to be a multiple of...
Definition TypeSize.h:181
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition TypeSize.h:166
self_iterator getIterator()
Definition ilist_node.h:123
#define INT64_MAX
Definition DataTypes.h:71
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ FewerElements
The (vector) operation should be implemented by splitting it into sub-vectors where the operation is ...
@ Legal
The operation is expected to be selectable directly by the target, and no transformation is necessary...
@ WidenScalar
The operation should be implemented in terms of a wider scalar base-type.
@ Custom
The target wants to do something special with this combination of operand and type.
operand_type_match m_Reg()
SpecificConstantMatch m_SpecificICst(const APInt &RequestedValue)
Matches a constant equal to RequestedValue.
GInstrBind< GBuildVector > m_GBuildVector(GBuildVector *&Inst)
GCstAndRegMatch m_GCst(std::optional< ValueAndVReg > &ValReg)
LoadOp_match< GLoad, PtrP > m_GLoad(const PtrP &Ptr)
MIFlagsRef m_MIFlags(uint32_t &Flags)
operand_type_match m_Pred()
BinaryOp_match< LHS, RHS, TargetOpcode::G_UMIN, true > m_GUMin(const LHS &L, const RHS &R)
UnaryOp_match< SrcTy, TargetOpcode::G_ZEXT > m_GZExt(const SrcTy &Src)
BinaryOp_match< LHS, RHS, TargetOpcode::G_XOR, true > m_GXor(const LHS &L, const RHS &R)
UnaryOp_match< SrcTy, TargetOpcode::G_SEXT > m_GSExt(const SrcTy &Src)
UnaryOp_match< SrcTy, TargetOpcode::G_FPEXT > m_GFPExt(const SrcTy &Src)
ConstantMatch< APInt > m_ICst(APInt &Cst)
BinaryOp_match< LHS, RHS, TargetOpcode::G_ADD, true > m_GAdd(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, TargetOpcode::G_OR, true > m_GOr(const LHS &L, const RHS &R)
BinaryOp_match< SpecificConstantMatch, SrcTy, TargetOpcode::G_SUB > m_Neg(const SrcTy &&Src)
Matches a register negated by a G_SUB.
ICstOrSplatMatch< APInt > m_ICstOrSplat(APInt &Cst)
ImplicitDefMatch m_GImplicitDef()
GInstrBind< GConcatVectors > m_GConcatVectors(GConcatVectors *&Inst)
GConstantBitsMatch m_GConstantOrFConstantBits(APInt &Bits)
CheckType m_SpecificType(LLT Ty)
deferred_ty< Register > m_DeferredReg(Register &R)
Similar to m_SpecificReg/Type, but the specific value to match originated from an earlier sub-pattern...
BinaryOp_match< LHS, RHS, TargetOpcode::G_UMAX, true > m_GUMax(const LHS &L, const RHS &R)
BinaryOp_match< SrcTy, SpecificConstantMatch, TargetOpcode::G_XOR, true > m_Not(const SrcTy &&Src)
Matches a register not-ed by a G_XOR.
CompareOp_match< Pred, LHS, RHS, TargetOpcode::G_ICMP > m_GICmp(const Pred &P, const LHS &L, const RHS &R)
GInstrBind< GUnmerge > m_GUnmerge(GUnmerge *&Inst)
Instruction binders for ops with no operand-form matcher (constant-immediate or variadic-source ops).
MMORef m_MMO(const MachineMemOperand *&MMO)
BinaryOp_match< LHS, RHS, TargetOpcode::G_SUB > m_GSub(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, TargetOpcode::G_ASHR, false > m_GAShr(const LHS &L, const RHS &R)
TernaryOp_match< Src0Ty, Src1Ty, Src2Ty, TargetOpcode::G_SELECT > m_GISelect(const Src0Ty &Src0, const Src1Ty &Src1, const Src2Ty &Src2)
bool mi_match(Reg R, const MachineRegisterInfo &MRI, Pattern &&P)
BinaryOp_match< LHS, RHS, TargetOpcode::G_PTR_ADD, false > m_GPtrAdd(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, TargetOpcode::G_SHL, false > m_GShl(const LHS &L, const RHS &R)
Or< Preds... > m_any_of(Preds &&... preds)
SpecificConstantOrSplatMatch m_SpecificICstOrSplat(const APInt &RequestedValue)
Matches a RequestedValue constant or a constant splat of RequestedValue.
BinaryOp_match< LHS, RHS, TargetOpcode::G_AND, true > m_GAnd(const LHS &L, const RHS &R)
UnaryOp_match< SrcTy, TargetOpcode::G_BITCAST > m_GBitcast(const SrcTy &Src)
BinaryOp_match< LHS, RHS, TargetOpcode::G_BUILD_VECTOR_TRUNC, false > m_GBuildVectorTrunc(const LHS &L, const RHS &R)
bind_ty< MachineInstr * > m_MInstr(MachineInstr *&MI)
UnaryOp_match< SrcTy, TargetOpcode::G_FNEG > m_GFNeg(const SrcTy &Src)
CompareOp_match< Pred, LHS, RHS, TargetOpcode::G_ICMP, true > m_c_GICmp(const Pred &P, const LHS &L, const RHS &R)
G_ICMP matcher that also matches commuted compares.
LoadOp_match< GAnyLoad, PtrP > m_GAnyLoad(const PtrP &Ptr)
TernaryOp_match< Src0Ty, Src1Ty, Src2Ty, TargetOpcode::G_INSERT_VECTOR_ELT > m_GInsertVecElt(const Src0Ty &Src0, const Src1Ty &Src1, const Src2Ty &Src2)
GFCstOrSplatGFCstMatch m_GFCstOrSplat(std::optional< FPValueAndVReg > &FPValReg)
And< Preds... > m_all_of(Preds &&... preds)
BinaryOp_match< LHS, RHS, TargetOpcode::G_SMIN, true > m_GSMin(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, TargetOpcode::G_LSHR, false > m_GLShr(const LHS &L, const RHS &R)
UnaryOp_match< SrcTy, TargetOpcode::G_ANYEXT > m_GAnyExt(const SrcTy &Src)
OneUse_match< SubPat > m_OneUse(const SubPat &SP)
BinaryOp_match< LHS, RHS, TargetOpcode::G_FMUL, true > m_GFMul(const LHS &L, const RHS &R)
UnaryOp_match< SrcTy, TargetOpcode::G_TRUNC > m_GTrunc(const SrcTy &Src)
BinaryOp_match< LHS, RHS, TargetOpcode::G_SMAX, true > m_GSMax(const LHS &L, const RHS &R)
CompareOp_match< Pred, LHS, RHS, TargetOpcode::G_FCMP > m_GFCmp(const Pred &P, const LHS &L, const RHS &R)
auto m_BinOp()
Match an arbitrary binary operation and ignore it.
Not(const Pred &P) -> Not< Pred >
initializer< Ty > init(const Ty &Val)
unsigned getNumElements(Type *Ty)
Definition SLPUtils.cpp:117
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition STLExtras.h:316
LLVM_ABI std::optional< APInt > isConstantOrConstantSplatVector(Register Def, const MachineRegisterInfo &MRI)
Determines if Def defines a constant integer or a splat vector of constant integers.
Definition Utils.cpp:1677
@ Offset
Definition DWP.cpp:577
LLVM_ABI bool isBuildVectorAllZeros(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndef=false)
Return true if the specified instruction is a G_BUILD_VECTOR or G_BUILD_VECTOR_TRUNC where all of the...
Definition Utils.cpp:1593
LLVM_ABI Type * getTypeForLLT(LLT Ty, LLVMContext &C)
Get the type back from LLT.
Definition Utils.cpp:2140
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
LLVM_ABI MachineInstr * getOpcodeDef(unsigned Opcode, Register Reg, const MachineRegisterInfo &MRI)
See if Reg is defined by an single def instruction that is Opcode.
Definition Utils.cpp:679
LLVM_ABI std::optional< APFloat > isConstantOrConstantSplatVectorFP(Register Def, const MachineRegisterInfo &MRI)
Determines if Def defines a float constant integer or a splat vector of float constant integers.
Definition Utils.cpp:1689
LLVM_ABI const ConstantFP * getConstantFPVRegVal(Register VReg, const MachineRegisterInfo &MRI)
Definition Utils.cpp:487
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
LLVM_ABI std::optional< APInt > getIConstantVRegVal(Register VReg, const MachineRegisterInfo &MRI)
If VReg is defined by a G_CONSTANT, return the corresponding value.
Definition Utils.cpp:301
LLVM_ABI std::optional< APInt > getIConstantSplatVal(const Register Reg, const MachineRegisterInfo &MRI)
Definition Utils.cpp:1553
LLVM_ABI bool isAllOnesOrAllOnesSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant -1 integer or a splatted vector of a constant -1 integer (with...
Definition Utils.cpp:1717
@ Known
Known to have no common set bits.
@ Undef
Value of the register doesn't matter.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
unsigned Log2_64_Ceil(uint64_t Value)
Return the ceil log base 2 of the specified value, 64 if the value is zero.
Definition MathExtras.h:345
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
Definition bit.h:315
std::function< void(MachineIRBuilder &)> BuildFnTy
LLVM_ABI const llvm::fltSemantics & getFltSemanticForLLT(LLT Ty)
Get the appropriate floating point arithmetic semantic based on the bit size of the given scalar LLT.
LLVM_ABI std::optional< APFloat > ConstantFoldFPBinOp(unsigned Opcode, const Register Op1, const Register Op2, const MachineRegisterInfo &MRI)
Definition Utils.cpp:754
@ Load
The value being inserted comes from a load (InsertElement only).
LLVM_ABI bool matchUnaryPredicate(const MachineRegisterInfo &MRI, Register Reg, llvm::function_ref< bool(const Constant *ConstVal)> Match, bool AllowUndefs=false)
Attempt to match a unary predicate against a scalar/splat constant or every element of a constant G_B...
Definition Utils.cpp:1732
auto dyn_cast_if_present(const Y &Val)
dyn_cast_if_present<X> - Functionally identical to dyn_cast, except that a null (or none in the case ...
Definition Casting.h:732
LLVM_ABI MVT getMVTForLLT(LLT Ty)
Get a rough equivalent of an MVT for a given LLT.
LLVM_ABI bool isNullOrNullSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
Definition Utils.cpp:1699
LLVM_ABI MachineInstr * getDefIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the def instruction for Reg, folding away any trivial copies.
Definition Utils.cpp:520
LLVM_ABI bool isConstTrueVal(const TargetLowering &TLI, int64_t Val, bool IsVector, bool IsFP)
Returns true if given the TargetLowering's boolean contents information, the value Val contains a tru...
Definition Utils.cpp:1765
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
Definition InstrProf.h:143
LLVM_ABI bool isKnownToBeAPowerOfTwo(const Value *V, const DataLayout &DL, bool OrZero=false, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true, unsigned Depth=0)
Return true if the given value is known to have exactly one bit set when defined.
LLVM_ABI std::optional< APInt > ConstantFoldBinOp(unsigned Opcode, const Register Op1, const Register Op2, const MachineRegisterInfo &MRI)
Definition Utils.cpp:685
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
constexpr bool has_single_bit(T Value) noexcept
Definition bit.h:149
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1762
LLVM_ABI const APInt & getIConstantFromReg(Register VReg, const MachineRegisterInfo &MRI)
VReg is defined by a G_CONSTANT, return the corresponding value.
Definition Utils.cpp:312
LLVM_ABI bool isConstantOrConstantVector(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowFP=true, bool AllowOpaqueConstants=true)
Return true if the specified instruction is known to be a constant, or a vector of constants.
Definition Utils.cpp:1657
constexpr unsigned MaxAnalysisRecursionDepth
SmallVector< std::function< void(MachineInstrBuilder &)>, 4 > OperandBuildSteps
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
LLVM_ABI bool canReplaceReg(Register DstReg, Register SrcReg, MachineRegisterInfo &MRI)
Check if DstReg can be replaced with SrcReg depending on the register constraints.
Definition Utils.cpp:207
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
std::tuple< Register, Register, uint64_t, Align, bool, std::vector< LLT > > MemCpyFamilyLoweringInfo
Definition Utils.h:212
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
Definition MathExtras.h:262
LLVM_ABI bool canCreateUndefOrPoison(const Operator *Op, bool ConsiderFlagsAndMetadata=true)
canCreateUndefOrPoison returns true if Op can create undef or poison from non-undef & non-poison oper...
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
LLVM_ABI SmallVector< APInt > ConstantFoldVectorBinop(unsigned Opcode, const Register Op1, const Register Op2, const MachineRegisterInfo &MRI)
Tries to constant fold a vector binop with sources Op1 and Op2.
Definition Utils.cpp:832
auto instructionsWithoutDebug(IterT It, IterT End, bool SkipPseudoOp=true)
Construct a range iterator which begins at It and moves forwards until End is reached,...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
LLVM_ABI std::optional< FPValueAndVReg > getFConstantSplat(Register VReg, const MachineRegisterInfo &MRI, bool AllowUndef=true)
Returns a floating point scalar constant of a build vector splat if it exists.
Definition Utils.cpp:1586
LLVM_ABI EVT getApproximateEVTForLLT(LLT Ty, LLVMContext &Ctx)
LLVM_ABI std::optional< APInt > ConstantFoldCastOp(unsigned Opcode, LLT DstTy, const Register Op0, const MachineRegisterInfo &MRI)
Definition Utils.cpp:946
@ Other
Any other memory.
Definition ModRef.h:68
LLVM_ABI bool canLowerMemCpyFamily(const MachineInstr &MI, const MachineRegisterInfo &MRI, unsigned MaxLen, Register &Dst, Register &Src, uint64_t &KnownLen, Align &Alignment, bool &DstAlignCanChange, std::vector< LLT > &MemOps)
Matcher for memcpy-like instructions.
Definition Utils.cpp:2307
LLVM_ABI SmallVector< APFloat > ConstantFoldUnaryFPOp(unsigned Opcode, LLT DstTy, Register Src, const MachineRegisterInfo &MRI)
Tries to constant fold a unary floating-point operation on Src.
Definition Utils.cpp:1031
LLVM_ABI unsigned getInverseGMinMaxOpcode(unsigned MinMaxOpc)
Returns the inverse opcode of MinMaxOpc, which is a generic min/max opcode like G_SMIN.
Definition Utils.cpp:286
LLVM_ABI SmallVector< APFloat > ConstantFoldVectorFPBinop(unsigned Opcode, const Register Op1, const Register Op2, const MachineRegisterInfo &MRI)
Tries to constant fold a vector floating-point binop with sources Op1 and Op2.
Definition Utils.cpp:855
@ Xor
Bitwise or logical XOR of integers.
@ And
Bitwise or logical AND of integers.
@ Sub
Subtraction of integers.
@ Add
Sum of integers.
@ Fast
Assign the register banks as fast as possible (default).
DWARFExpression::Operation Op
LLVM_ABI bool isGuaranteedNotToBeUndefOrPoison(const Value *V, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Return true if this function can prove that V does not have undef bits and is never poison.
LLVM_ABI std::optional< FPValueAndVReg > getFConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_FCONSTANT returns it...
Definition Utils.cpp:454
constexpr unsigned BitWidth
LLVM_ABI int64_t getICmpTrueVal(const TargetLowering &TLI, bool IsVector, bool IsFP)
Returns an integer representing true, as defined by the TargetBooleanContents.
Definition Utils.cpp:1790
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
LLVM_ABI std::optional< ValueAndVReg > getIConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_CONSTANT returns its...
Definition Utils.cpp:440
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1788
iterator_range< pointer_iterator< WrappedIteratorT > > make_pointer_range(RangeT &&Range)
Definition iterator.h:368
LLVM_ABI std::optional< DefinitionAndSourceRegister > getDefSrcRegIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the def instruction for Reg, and underlying value Register folding away any copies.
Definition Utils.cpp:495
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition Alignment.h:206
LLVM_ABI SmallVector< APInt > ConstantFoldUnaryIntOp(unsigned Opcode, LLT DstTy, Register Src, const MachineRegisterInfo &MRI)
Tries to constant fold a unary integer operation (G_CTLZ, G_CTTZ, G_CTPOP and their _ZERO_POISON vari...
Definition Utils.cpp:983
unsigned Log2(Align A)
Returns the log2 of the alignment.
Definition Alignment.h:202
LLVM_ABI Register getSrcRegIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the source register for Reg, folding away any trivial copies.
Definition Utils.cpp:527
constexpr T maskTrailingOnes(unsigned N)
Create a bitmask with the N right-most bits set to 1, and all other bits set to 0.
Definition MathExtras.h:78
unsigned getFCmpCode(CmpInst::Predicate CC)
Similar to getICmpCode but for FCmpInst.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:881
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
Simple struct used to hold a Register value and the instruction which defines it.
Definition Utils.h:249
static constexpr DenormalMode getIEEE()
Extended Value Type.
Definition ValueTypes.h:35
SmallVector< InstructionBuildSteps, 2 > InstrsToBuild
Describes instructions to be built during a combine.
static KnownBits makeConstant(const APInt &C)
Create known bits from a known constant.
Definition KnownBits.h:315
bool isNonNegative() const
Returns true if this value is known to be non-negative.
Definition KnownBits.h:106
unsigned countMinLeadingOnes() const
Returns the minimum number of leading one bits.
Definition KnownBits.h:265
unsigned countMinTrailingZeros() const
Returns the minimum number of trailing zero bits.
Definition KnownBits.h:256
static LLVM_ABI KnownBits ashr(const KnownBits &LHS, const KnownBits &RHS, bool ShAmtNonZero=false, bool Exact=false)
Compute known bits for ashr(LHS, RHS).
bool isUnknown() const
Returns true if we don't know any bits.
Definition KnownBits.h:64
unsigned getBitWidth() const
Get the bit width of this value.
Definition KnownBits.h:44
static LLVM_ABI KnownBits lshr(const KnownBits &LHS, const KnownBits &RHS, bool ShAmtNonZero=false, bool Exact=false)
Compute known bits for lshr(LHS, RHS).
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
Definition KnownBits.h:262
bool isNegative() const
Returns true if this value is known to be negative.
Definition KnownBits.h:103
static LLVM_ABI KnownBits shl(const KnownBits &LHS, const KnownBits &RHS, bool NUW=false, bool NSW=false, bool ShAmtNonZero=false)
Compute known bits for shl(LHS, RHS).
The LegalityQuery object bundles together all the information that's needed to decide whether a given...
Matching combinators.
This class contains a discriminated union of information about pointers in memory operands,...
LLVM_ABI unsigned getAddrSpace() const
Return the LLVM IR address space number that this pointer points into.
MachinePointerInfo getWithOffset(int64_t O) const
const RegisterBank * Bank
Magic data for optimising signed division by a constant.
static LLVM_ABI SignedDivisionByConstantInfo get(const APInt &D)
Calculate the magic numbers required to implement a signed integer division by a constant as a sequen...
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
Magic data for optimising unsigned division by a constant.
static LLVM_ABI UnsignedDivisionByConstantInfo get(const APInt &D, unsigned LeadingZeros=0, bool AllowEvenDivisorOptimization=true, bool AllowWidenOptimization=false)
Calculate the magic numbers required to implement an unsigned integer division by a constant as a seq...