LLVM  16.0.0git
AArch64PostLegalizerLowering.cpp
Go to the documentation of this file.
1 //=== AArch64PostLegalizerLowering.cpp --------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// \file
10 /// Post-legalization lowering for instructions.
11 ///
12 /// This is used to offload pattern matching from the selector.
13 ///
14 /// For example, this combiner will notice that a G_SHUFFLE_VECTOR is actually
15 /// a G_ZIP, G_UZP, etc.
16 ///
17 /// General optimization combines should be handled by either the
18 /// AArch64PostLegalizerCombiner or the AArch64PreLegalizerCombiner.
19 ///
20 //===----------------------------------------------------------------------===//
21 
22 #include "AArch64GlobalISelUtils.h"
23 #include "AArch64Subtarget.h"
24 #include "AArch64TargetMachine.h"
28 #include "Utils/AArch64BaseInfo.h"
43 #include "llvm/IR/InstrTypes.h"
44 #include "llvm/InitializePasses.h"
45 #include "llvm/Support/Debug.h"
47 #include <optional>
48 
49 #define DEBUG_TYPE "aarch64-postlegalizer-lowering"
50 
51 using namespace llvm;
52 using namespace MIPatternMatch;
53 using namespace AArch64GISelUtils;
54 
55 /// Represents a pseudo instruction which replaces a G_SHUFFLE_VECTOR.
56 ///
57 /// Used for matching target-supported shuffles before codegen.
59  unsigned Opc; ///< Opcode for the instruction. (E.g. G_ZIP1)
60  Register Dst; ///< Destination register.
61  SmallVector<SrcOp, 2> SrcOps; ///< Source registers.
62  ShuffleVectorPseudo(unsigned Opc, Register Dst,
63  std::initializer_list<SrcOp> SrcOps)
64  : Opc(Opc), Dst(Dst), SrcOps(SrcOps){};
65  ShuffleVectorPseudo() = default;
66 };
67 
68 /// Check if a vector shuffle corresponds to a REV instruction with the
69 /// specified blocksize.
70 static bool isREVMask(ArrayRef<int> M, unsigned EltSize, unsigned NumElts,
71  unsigned BlockSize) {
72  assert((BlockSize == 16 || BlockSize == 32 || BlockSize == 64) &&
73  "Only possible block sizes for REV are: 16, 32, 64");
74  assert(EltSize != 64 && "EltSize cannot be 64 for REV mask.");
75 
76  unsigned BlockElts = M[0] + 1;
77 
78  // If the first shuffle index is UNDEF, be optimistic.
79  if (M[0] < 0)
80  BlockElts = BlockSize / EltSize;
81 
82  if (BlockSize <= EltSize || BlockSize != BlockElts * EltSize)
83  return false;
84 
85  for (unsigned i = 0; i < NumElts; ++i) {
86  // Ignore undef indices.
87  if (M[i] < 0)
88  continue;
89  if (static_cast<unsigned>(M[i]) !=
90  (i - i % BlockElts) + (BlockElts - 1 - i % BlockElts))
91  return false;
92  }
93 
94  return true;
95 }
96 
97 /// Determines if \p M is a shuffle vector mask for a TRN of \p NumElts.
98 /// Whether or not G_TRN1 or G_TRN2 should be used is stored in \p WhichResult.
99 static bool isTRNMask(ArrayRef<int> M, unsigned NumElts,
100  unsigned &WhichResult) {
101  if (NumElts % 2 != 0)
102  return false;
103  WhichResult = (M[0] == 0 ? 0 : 1);
104  for (unsigned i = 0; i < NumElts; i += 2) {
105  if ((M[i] >= 0 && static_cast<unsigned>(M[i]) != i + WhichResult) ||
106  (M[i + 1] >= 0 &&
107  static_cast<unsigned>(M[i + 1]) != i + NumElts + WhichResult))
108  return false;
109  }
110  return true;
111 }
112 
113 /// Check if a G_EXT instruction can handle a shuffle mask \p M when the vector
114 /// sources of the shuffle are different.
115 static std::optional<std::pair<bool, uint64_t>> getExtMask(ArrayRef<int> M,
116  unsigned NumElts) {
117  // Look for the first non-undef element.
118  auto FirstRealElt = find_if(M, [](int Elt) { return Elt >= 0; });
119  if (FirstRealElt == M.end())
120  return std::nullopt;
121 
122  // Use APInt to handle overflow when calculating expected element.
123  unsigned MaskBits = APInt(32, NumElts * 2).logBase2();
124  APInt ExpectedElt = APInt(MaskBits, *FirstRealElt + 1);
125 
126  // The following shuffle indices must be the successive elements after the
127  // first real element.
128  if (any_of(
129  make_range(std::next(FirstRealElt), M.end()),
130  [&ExpectedElt](int Elt) { return Elt != ExpectedElt++ && Elt >= 0; }))
131  return std::nullopt;
132 
133  // The index of an EXT is the first element if it is not UNDEF.
134  // Watch out for the beginning UNDEFs. The EXT index should be the expected
135  // value of the first element. E.g.
136  // <-1, -1, 3, ...> is treated as <1, 2, 3, ...>.
137  // <-1, -1, 0, 1, ...> is treated as <2*NumElts-2, 2*NumElts-1, 0, 1, ...>.
138  // ExpectedElt is the last mask index plus 1.
139  uint64_t Imm = ExpectedElt.getZExtValue();
140  bool ReverseExt = false;
141 
142  // There are two difference cases requiring to reverse input vectors.
143  // For example, for vector <4 x i32> we have the following cases,
144  // Case 1: shufflevector(<4 x i32>,<4 x i32>,<-1, -1, -1, 0>)
145  // Case 2: shufflevector(<4 x i32>,<4 x i32>,<-1, -1, 7, 0>)
146  // For both cases, we finally use mask <5, 6, 7, 0>, which requires
147  // to reverse two input vectors.
148  if (Imm < NumElts)
149  ReverseExt = true;
150  else
151  Imm -= NumElts;
152  return std::make_pair(ReverseExt, Imm);
153 }
154 
155 /// Determines if \p M is a shuffle vector mask for a UZP of \p NumElts.
156 /// Whether or not G_UZP1 or G_UZP2 should be used is stored in \p WhichResult.
157 static bool isUZPMask(ArrayRef<int> M, unsigned NumElts,
158  unsigned &WhichResult) {
159  WhichResult = (M[0] == 0 ? 0 : 1);
160  for (unsigned i = 0; i != NumElts; ++i) {
161  // Skip undef indices.
162  if (M[i] < 0)
163  continue;
164  if (static_cast<unsigned>(M[i]) != 2 * i + WhichResult)
165  return false;
166  }
167  return true;
168 }
169 
170 /// \return true if \p M is a zip mask for a shuffle vector of \p NumElts.
171 /// Whether or not G_ZIP1 or G_ZIP2 should be used is stored in \p WhichResult.
172 static bool isZipMask(ArrayRef<int> M, unsigned NumElts,
173  unsigned &WhichResult) {
174  if (NumElts % 2 != 0)
175  return false;
176 
177  // 0 means use ZIP1, 1 means use ZIP2.
178  WhichResult = (M[0] == 0 ? 0 : 1);
179  unsigned Idx = WhichResult * NumElts / 2;
180  for (unsigned i = 0; i != NumElts; i += 2) {
181  if ((M[i] >= 0 && static_cast<unsigned>(M[i]) != Idx) ||
182  (M[i + 1] >= 0 && static_cast<unsigned>(M[i + 1]) != Idx + NumElts))
183  return false;
184  Idx += 1;
185  }
186  return true;
187 }
188 
189 /// Helper function for matchINS.
190 ///
191 /// \returns a value when \p M is an ins mask for \p NumInputElements.
192 ///
193 /// First element of the returned pair is true when the produced
194 /// G_INSERT_VECTOR_ELT destination should be the LHS of the G_SHUFFLE_VECTOR.
195 ///
196 /// Second element is the destination lane for the G_INSERT_VECTOR_ELT.
197 static std::optional<std::pair<bool, int>> isINSMask(ArrayRef<int> M,
198  int NumInputElements) {
199  if (M.size() != static_cast<size_t>(NumInputElements))
200  return std::nullopt;
201  int NumLHSMatch = 0, NumRHSMatch = 0;
202  int LastLHSMismatch = -1, LastRHSMismatch = -1;
203  for (int Idx = 0; Idx < NumInputElements; ++Idx) {
204  if (M[Idx] == -1) {
205  ++NumLHSMatch;
206  ++NumRHSMatch;
207  continue;
208  }
209  M[Idx] == Idx ? ++NumLHSMatch : LastLHSMismatch = Idx;
210  M[Idx] == Idx + NumInputElements ? ++NumRHSMatch : LastRHSMismatch = Idx;
211  }
212  const int NumNeededToMatch = NumInputElements - 1;
213  if (NumLHSMatch == NumNeededToMatch)
214  return std::make_pair(true, LastLHSMismatch);
215  if (NumRHSMatch == NumNeededToMatch)
216  return std::make_pair(false, LastRHSMismatch);
217  return std::nullopt;
218 }
219 
220 /// \return true if a G_SHUFFLE_VECTOR instruction \p MI can be replaced with a
221 /// G_REV instruction. Returns the appropriate G_REV opcode in \p Opc.
223  ShuffleVectorPseudo &MatchInfo) {
224  assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
225  ArrayRef<int> ShuffleMask = MI.getOperand(3).getShuffleMask();
226  Register Dst = MI.getOperand(0).getReg();
227  Register Src = MI.getOperand(1).getReg();
228  LLT Ty = MRI.getType(Dst);
229  unsigned EltSize = Ty.getScalarSizeInBits();
230 
231  // Element size for a rev cannot be 64.
232  if (EltSize == 64)
233  return false;
234 
235  unsigned NumElts = Ty.getNumElements();
236 
237  // Try to produce G_REV64
238  if (isREVMask(ShuffleMask, EltSize, NumElts, 64)) {
239  MatchInfo = ShuffleVectorPseudo(AArch64::G_REV64, Dst, {Src});
240  return true;
241  }
242 
243  // TODO: Produce G_REV32 and G_REV16 once we have proper legalization support.
244  // This should be identical to above, but with a constant 32 and constant
245  // 16.
246  return false;
247 }
248 
249 /// \return true if a G_SHUFFLE_VECTOR instruction \p MI can be replaced with
250 /// a G_TRN1 or G_TRN2 instruction.
252  ShuffleVectorPseudo &MatchInfo) {
253  assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
254  unsigned WhichResult;
255  ArrayRef<int> ShuffleMask = MI.getOperand(3).getShuffleMask();
256  Register Dst = MI.getOperand(0).getReg();
257  unsigned NumElts = MRI.getType(Dst).getNumElements();
258  if (!isTRNMask(ShuffleMask, NumElts, WhichResult))
259  return false;
260  unsigned Opc = (WhichResult == 0) ? AArch64::G_TRN1 : AArch64::G_TRN2;
261  Register V1 = MI.getOperand(1).getReg();
262  Register V2 = MI.getOperand(2).getReg();
263  MatchInfo = ShuffleVectorPseudo(Opc, Dst, {V1, V2});
264  return true;
265 }
266 
267 /// \return true if a G_SHUFFLE_VECTOR instruction \p MI can be replaced with
268 /// a G_UZP1 or G_UZP2 instruction.
269 ///
270 /// \param [in] MI - The shuffle vector instruction.
271 /// \param [out] MatchInfo - Either G_UZP1 or G_UZP2 on success.
273  ShuffleVectorPseudo &MatchInfo) {
274  assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
275  unsigned WhichResult;
276  ArrayRef<int> ShuffleMask = MI.getOperand(3).getShuffleMask();
277  Register Dst = MI.getOperand(0).getReg();
278  unsigned NumElts = MRI.getType(Dst).getNumElements();
279  if (!isUZPMask(ShuffleMask, NumElts, WhichResult))
280  return false;
281  unsigned Opc = (WhichResult == 0) ? AArch64::G_UZP1 : AArch64::G_UZP2;
282  Register V1 = MI.getOperand(1).getReg();
283  Register V2 = MI.getOperand(2).getReg();
284  MatchInfo = ShuffleVectorPseudo(Opc, Dst, {V1, V2});
285  return true;
286 }
287 
289  ShuffleVectorPseudo &MatchInfo) {
290  assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
291  unsigned WhichResult;
292  ArrayRef<int> ShuffleMask = MI.getOperand(3).getShuffleMask();
293  Register Dst = MI.getOperand(0).getReg();
294  unsigned NumElts = MRI.getType(Dst).getNumElements();
295  if (!isZipMask(ShuffleMask, NumElts, WhichResult))
296  return false;
297  unsigned Opc = (WhichResult == 0) ? AArch64::G_ZIP1 : AArch64::G_ZIP2;
298  Register V1 = MI.getOperand(1).getReg();
299  Register V2 = MI.getOperand(2).getReg();
300  MatchInfo = ShuffleVectorPseudo(Opc, Dst, {V1, V2});
301  return true;
302 }
303 
304 /// Helper function for matchDup.
307  ShuffleVectorPseudo &MatchInfo) {
308  if (Lane != 0)
309  return false;
310 
311  // Try to match a vector splat operation into a dup instruction.
312  // We're looking for this pattern:
313  //
314  // %scalar:gpr(s64) = COPY $x0
315  // %undef:fpr(<2 x s64>) = G_IMPLICIT_DEF
316  // %cst0:gpr(s32) = G_CONSTANT i32 0
317  // %zerovec:fpr(<2 x s32>) = G_BUILD_VECTOR %cst0(s32), %cst0(s32)
318  // %ins:fpr(<2 x s64>) = G_INSERT_VECTOR_ELT %undef, %scalar(s64), %cst0(s32)
319  // %splat:fpr(<2 x s64>) = G_SHUFFLE_VECTOR %ins(<2 x s64>), %undef, %zerovec(<2 x s32>)
320  //
321  // ...into:
322  // %splat = G_DUP %scalar
323 
324  // Begin matching the insert.
325  auto *InsMI = getOpcodeDef(TargetOpcode::G_INSERT_VECTOR_ELT,
326  MI.getOperand(1).getReg(), MRI);
327  if (!InsMI)
328  return false;
329  // Match the undef vector operand.
330  if (!getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, InsMI->getOperand(1).getReg(),
331  MRI))
332  return false;
333 
334  // Match the index constant 0.
335  if (!mi_match(InsMI->getOperand(3).getReg(), MRI, m_ZeroInt()))
336  return false;
337 
338  MatchInfo = ShuffleVectorPseudo(AArch64::G_DUP, MI.getOperand(0).getReg(),
339  {InsMI->getOperand(2).getReg()});
340  return true;
341 }
342 
343 /// Helper function for matchDup.
344 static bool matchDupFromBuildVector(int Lane, MachineInstr &MI,
346  ShuffleVectorPseudo &MatchInfo) {
347  assert(Lane >= 0 && "Expected positive lane?");
348  // Test if the LHS is a BUILD_VECTOR. If it is, then we can just reference the
349  // lane's definition directly.
350  auto *BuildVecMI = getOpcodeDef(TargetOpcode::G_BUILD_VECTOR,
351  MI.getOperand(1).getReg(), MRI);
352  if (!BuildVecMI)
353  return false;
354  Register Reg = BuildVecMI->getOperand(Lane + 1).getReg();
355  MatchInfo =
356  ShuffleVectorPseudo(AArch64::G_DUP, MI.getOperand(0).getReg(), {Reg});
357  return true;
358 }
359 
361  ShuffleVectorPseudo &MatchInfo) {
362  assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
363  auto MaybeLane = getSplatIndex(MI);
364  if (!MaybeLane)
365  return false;
366  int Lane = *MaybeLane;
367  // If this is undef splat, generate it via "just" vdup, if possible.
368  if (Lane < 0)
369  Lane = 0;
370  if (matchDupFromInsertVectorElt(Lane, MI, MRI, MatchInfo))
371  return true;
372  if (matchDupFromBuildVector(Lane, MI, MRI, MatchInfo))
373  return true;
374  return false;
375 }
376 
377 // Check if an EXT instruction can handle the shuffle mask when the vector
378 // sources of the shuffle are the same.
380  unsigned NumElts = Ty.getNumElements();
381 
382  // Assume that the first shuffle index is not UNDEF. Fail if it is.
383  if (M[0] < 0)
384  return false;
385 
386  // If this is a VEXT shuffle, the immediate value is the index of the first
387  // element. The other shuffle indices must be the successive elements after
388  // the first one.
389  unsigned ExpectedElt = M[0];
390  for (unsigned I = 1; I < NumElts; ++I) {
391  // Increment the expected index. If it wraps around, just follow it
392  // back to index zero and keep going.
393  ++ExpectedElt;
394  if (ExpectedElt == NumElts)
395  ExpectedElt = 0;
396 
397  if (M[I] < 0)
398  continue; // Ignore UNDEF indices.
399  if (ExpectedElt != static_cast<unsigned>(M[I]))
400  return false;
401  }
402 
403  return true;
404 }
405 
407  ShuffleVectorPseudo &MatchInfo) {
408  assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
409  Register Dst = MI.getOperand(0).getReg();
410  LLT DstTy = MRI.getType(Dst);
411  Register V1 = MI.getOperand(1).getReg();
412  Register V2 = MI.getOperand(2).getReg();
413  auto Mask = MI.getOperand(3).getShuffleMask();
414  uint64_t Imm;
415  auto ExtInfo = getExtMask(Mask, DstTy.getNumElements());
416  uint64_t ExtFactor = MRI.getType(V1).getScalarSizeInBits() / 8;
417 
418  if (!ExtInfo) {
419  if (!getOpcodeDef<GImplicitDef>(V2, MRI) ||
420  !isSingletonExtMask(Mask, DstTy))
421  return false;
422 
423  Imm = Mask[0] * ExtFactor;
424  MatchInfo = ShuffleVectorPseudo(AArch64::G_EXT, Dst, {V1, V1, Imm});
425  return true;
426  }
427  bool ReverseExt;
428  std::tie(ReverseExt, Imm) = *ExtInfo;
429  if (ReverseExt)
430  std::swap(V1, V2);
431  Imm *= ExtFactor;
432  MatchInfo = ShuffleVectorPseudo(AArch64::G_EXT, Dst, {V1, V2, Imm});
433  return true;
434 }
435 
436 /// Replace a G_SHUFFLE_VECTOR instruction with a pseudo.
437 /// \p Opc is the opcode to use. \p MI is the G_SHUFFLE_VECTOR.
439  ShuffleVectorPseudo &MatchInfo) {
440  MachineIRBuilder MIRBuilder(MI);
441  MIRBuilder.buildInstr(MatchInfo.Opc, {MatchInfo.Dst}, MatchInfo.SrcOps);
442  MI.eraseFromParent();
443  return true;
444 }
445 
446 /// Replace a G_SHUFFLE_VECTOR instruction with G_EXT.
447 /// Special-cased because the constant operand must be emitted as a G_CONSTANT
448 /// for the imported tablegen patterns to work.
449 static bool applyEXT(MachineInstr &MI, ShuffleVectorPseudo &MatchInfo) {
450  MachineIRBuilder MIRBuilder(MI);
451  // Tablegen patterns expect an i32 G_CONSTANT as the final op.
452  auto Cst =
453  MIRBuilder.buildConstant(LLT::scalar(32), MatchInfo.SrcOps[2].getImm());
454  MIRBuilder.buildInstr(MatchInfo.Opc, {MatchInfo.Dst},
455  {MatchInfo.SrcOps[0], MatchInfo.SrcOps[1], Cst});
456  MI.eraseFromParent();
457  return true;
458 }
459 
460 /// Match a G_SHUFFLE_VECTOR with a mask which corresponds to a
461 /// G_INSERT_VECTOR_ELT and G_EXTRACT_VECTOR_ELT pair.
462 ///
463 /// e.g.
464 /// %shuf = G_SHUFFLE_VECTOR %left, %right, shufflemask(0, 0)
465 ///
466 /// Can be represented as
467 ///
468 /// %extract = G_EXTRACT_VECTOR_ELT %left, 0
469 /// %ins = G_INSERT_VECTOR_ELT %left, %extract, 1
470 ///
472  std::tuple<Register, int, Register, int> &MatchInfo) {
473  assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
474  ArrayRef<int> ShuffleMask = MI.getOperand(3).getShuffleMask();
475  Register Dst = MI.getOperand(0).getReg();
476  int NumElts = MRI.getType(Dst).getNumElements();
477  auto DstIsLeftAndDstLane = isINSMask(ShuffleMask, NumElts);
478  if (!DstIsLeftAndDstLane)
479  return false;
480  bool DstIsLeft;
481  int DstLane;
482  std::tie(DstIsLeft, DstLane) = *DstIsLeftAndDstLane;
483  Register Left = MI.getOperand(1).getReg();
484  Register Right = MI.getOperand(2).getReg();
485  Register DstVec = DstIsLeft ? Left : Right;
486  Register SrcVec = Left;
487 
488  int SrcLane = ShuffleMask[DstLane];
489  if (SrcLane >= NumElts) {
490  SrcVec = Right;
491  SrcLane -= NumElts;
492  }
493 
494  MatchInfo = std::make_tuple(DstVec, DstLane, SrcVec, SrcLane);
495  return true;
496 }
497 
500  std::tuple<Register, int, Register, int> &MatchInfo) {
501  Builder.setInstrAndDebugLoc(MI);
502  Register Dst = MI.getOperand(0).getReg();
503  auto ScalarTy = MRI.getType(Dst).getElementType();
504  Register DstVec, SrcVec;
505  int DstLane, SrcLane;
506  std::tie(DstVec, DstLane, SrcVec, SrcLane) = MatchInfo;
507  auto SrcCst = Builder.buildConstant(LLT::scalar(64), SrcLane);
508  auto Extract = Builder.buildExtractVectorElement(ScalarTy, SrcVec, SrcCst);
509  auto DstCst = Builder.buildConstant(LLT::scalar(64), DstLane);
510  Builder.buildInsertVectorElement(Dst, DstVec, Extract, DstCst);
511  MI.eraseFromParent();
512  return true;
513 }
514 
515 /// isVShiftRImm - Check if this is a valid vector for the immediate
516 /// operand of a vector shift right operation. The value must be in the range:
517 /// 1 <= Value <= ElementBits for a right shift.
519  int64_t &Cnt) {
520  assert(Ty.isVector() && "vector shift count is not a vector type");
522  auto Cst = getAArch64VectorSplatScalar(*MI, MRI);
523  if (!Cst)
524  return false;
525  Cnt = *Cst;
526  int64_t ElementBits = Ty.getScalarSizeInBits();
527  return Cnt >= 1 && Cnt <= ElementBits;
528 }
529 
530 /// Match a vector G_ASHR or G_LSHR with a valid immediate shift.
532  int64_t &Imm) {
533  assert(MI.getOpcode() == TargetOpcode::G_ASHR ||
534  MI.getOpcode() == TargetOpcode::G_LSHR);
535  LLT Ty = MRI.getType(MI.getOperand(1).getReg());
536  if (!Ty.isVector())
537  return false;
538  return isVShiftRImm(MI.getOperand(2).getReg(), MRI, Ty, Imm);
539 }
540 
542  int64_t &Imm) {
543  unsigned Opc = MI.getOpcode();
544  assert(Opc == TargetOpcode::G_ASHR || Opc == TargetOpcode::G_LSHR);
545  unsigned NewOpc =
546  Opc == TargetOpcode::G_ASHR ? AArch64::G_VASHR : AArch64::G_VLSHR;
547  MachineIRBuilder MIB(MI);
548  auto ImmDef = MIB.buildConstant(LLT::scalar(32), Imm);
549  MIB.buildInstr(NewOpc, {MI.getOperand(0)}, {MI.getOperand(1), ImmDef});
550  MI.eraseFromParent();
551  return true;
552 }
553 
554 /// Determine if it is possible to modify the \p RHS and predicate \p P of a
555 /// G_ICMP instruction such that the right-hand side is an arithmetic immediate.
556 ///
557 /// \returns A pair containing the updated immediate and predicate which may
558 /// be used to optimize the instruction.
559 ///
560 /// \note This assumes that the comparison has been legalized.
561 std::optional<std::pair<uint64_t, CmpInst::Predicate>>
563  const MachineRegisterInfo &MRI) {
564  const auto &Ty = MRI.getType(RHS);
565  if (Ty.isVector())
566  return std::nullopt;
567  unsigned Size = Ty.getSizeInBits();
568  assert((Size == 32 || Size == 64) && "Expected 32 or 64 bit compare only?");
569 
570  // If the RHS is not a constant, or the RHS is already a valid arithmetic
571  // immediate, then there is nothing to change.
572  auto ValAndVReg = getIConstantVRegValWithLookThrough(RHS, MRI);
573  if (!ValAndVReg)
574  return std::nullopt;
575  uint64_t C = ValAndVReg->Value.getZExtValue();
576  if (isLegalArithImmed(C))
577  return std::nullopt;
578 
579  // We have a non-arithmetic immediate. Check if adjusting the immediate and
580  // adjusting the predicate will result in a legal arithmetic immediate.
581  switch (P) {
582  default:
583  return std::nullopt;
584  case CmpInst::ICMP_SLT:
585  case CmpInst::ICMP_SGE:
586  // Check for
587  //
588  // x slt c => x sle c - 1
589  // x sge c => x sgt c - 1
590  //
591  // When c is not the smallest possible negative number.
592  if ((Size == 64 && static_cast<int64_t>(C) == INT64_MIN) ||
593  (Size == 32 && static_cast<int32_t>(C) == INT32_MIN))
594  return std::nullopt;
596  C -= 1;
597  break;
598  case CmpInst::ICMP_ULT:
599  case CmpInst::ICMP_UGE:
600  // Check for
601  //
602  // x ult c => x ule c - 1
603  // x uge c => x ugt c - 1
604  //
605  // When c is not zero.
606  if (C == 0)
607  return std::nullopt;
609  C -= 1;
610  break;
611  case CmpInst::ICMP_SLE:
612  case CmpInst::ICMP_SGT:
613  // Check for
614  //
615  // x sle c => x slt c + 1
616  // x sgt c => s sge c + 1
617  //
618  // When c is not the largest possible signed integer.
619  if ((Size == 32 && static_cast<int32_t>(C) == INT32_MAX) ||
620  (Size == 64 && static_cast<int64_t>(C) == INT64_MAX))
621  return std::nullopt;
623  C += 1;
624  break;
625  case CmpInst::ICMP_ULE:
626  case CmpInst::ICMP_UGT:
627  // Check for
628  //
629  // x ule c => x ult c + 1
630  // x ugt c => s uge c + 1
631  //
632  // When c is not the largest possible unsigned integer.
633  if ((Size == 32 && static_cast<uint32_t>(C) == UINT32_MAX) ||
634  (Size == 64 && C == UINT64_MAX))
635  return std::nullopt;
637  C += 1;
638  break;
639  }
640 
641  // Check if the new constant is valid, and return the updated constant and
642  // predicate if it is.
643  if (Size == 32)
644  C = static_cast<uint32_t>(C);
645  if (!isLegalArithImmed(C))
646  return std::nullopt;
647  return {{C, P}};
648 }
649 
650 /// Determine whether or not it is possible to update the RHS and predicate of
651 /// a G_ICMP instruction such that the RHS will be selected as an arithmetic
652 /// immediate.
653 ///
654 /// \p MI - The G_ICMP instruction
655 /// \p MatchInfo - The new RHS immediate and predicate on success
656 ///
657 /// See tryAdjustICmpImmAndPred for valid transformations.
660  std::pair<uint64_t, CmpInst::Predicate> &MatchInfo) {
661  assert(MI.getOpcode() == TargetOpcode::G_ICMP);
662  Register RHS = MI.getOperand(3).getReg();
663  auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
664  if (auto MaybeNewImmAndPred = tryAdjustICmpImmAndPred(RHS, Pred, MRI)) {
665  MatchInfo = *MaybeNewImmAndPred;
666  return true;
667  }
668  return false;
669 }
670 
672  MachineInstr &MI, std::pair<uint64_t, CmpInst::Predicate> &MatchInfo,
673  MachineIRBuilder &MIB, GISelChangeObserver &Observer) {
674  MIB.setInstrAndDebugLoc(MI);
675  MachineOperand &RHS = MI.getOperand(3);
676  MachineRegisterInfo &MRI = *MIB.getMRI();
677  auto Cst = MIB.buildConstant(MRI.cloneVirtualRegister(RHS.getReg()),
678  MatchInfo.first);
679  Observer.changingInstr(MI);
680  RHS.setReg(Cst->getOperand(0).getReg());
681  MI.getOperand(1).setPredicate(MatchInfo.second);
682  Observer.changedInstr(MI);
683  return true;
684 }
685 
687  std::pair<unsigned, int> &MatchInfo) {
688  assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
689  Register Src1Reg = MI.getOperand(1).getReg();
690  const LLT SrcTy = MRI.getType(Src1Reg);
691  const LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
692 
693  auto LaneIdx = getSplatIndex(MI);
694  if (!LaneIdx)
695  return false;
696 
697  // The lane idx should be within the first source vector.
698  if (*LaneIdx >= SrcTy.getNumElements())
699  return false;
700 
701  if (DstTy != SrcTy)
702  return false;
703 
704  LLT ScalarTy = SrcTy.getElementType();
705  unsigned ScalarSize = ScalarTy.getSizeInBits();
706 
707  unsigned Opc = 0;
708  switch (SrcTy.getNumElements()) {
709  case 2:
710  if (ScalarSize == 64)
711  Opc = AArch64::G_DUPLANE64;
712  else if (ScalarSize == 32)
713  Opc = AArch64::G_DUPLANE32;
714  break;
715  case 4:
716  if (ScalarSize == 32)
717  Opc = AArch64::G_DUPLANE32;
718  break;
719  case 8:
720  if (ScalarSize == 16)
721  Opc = AArch64::G_DUPLANE16;
722  break;
723  case 16:
724  if (ScalarSize == 8)
725  Opc = AArch64::G_DUPLANE8;
726  break;
727  default:
728  break;
729  }
730  if (!Opc)
731  return false;
732 
733  MatchInfo.first = Opc;
734  MatchInfo.second = *LaneIdx;
735  return true;
736 }
737 
739  MachineIRBuilder &B, std::pair<unsigned, int> &MatchInfo) {
740  assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
741  Register Src1Reg = MI.getOperand(1).getReg();
742  const LLT SrcTy = MRI.getType(Src1Reg);
743 
744  B.setInstrAndDebugLoc(MI);
745  auto Lane = B.buildConstant(LLT::scalar(64), MatchInfo.second);
746 
747  Register DupSrc = MI.getOperand(1).getReg();
748  // For types like <2 x s32>, we can use G_DUPLANE32, with a <4 x s32> source.
749  // To do this, we can use a G_CONCAT_VECTORS to do the widening.
750  if (SrcTy == LLT::fixed_vector(2, LLT::scalar(32))) {
751  assert(MRI.getType(MI.getOperand(0).getReg()).getNumElements() == 2 &&
752  "Unexpected dest elements");
753  auto Undef = B.buildUndef(SrcTy);
754  DupSrc = B.buildConcatVectors(
756  {Src1Reg, Undef.getReg(0)})
757  .getReg(0);
758  }
759  B.buildInstr(MatchInfo.first, {MI.getOperand(0).getReg()}, {DupSrc, Lane});
760  MI.eraseFromParent();
761  return true;
762 }
763 
765  assert(MI.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
766  auto Splat = getAArch64VectorSplat(MI, MRI);
767  if (!Splat)
768  return false;
769  if (Splat->isReg())
770  return true;
771  // Later, during selection, we'll try to match imported patterns using
772  // immAllOnesV and immAllZerosV. These require G_BUILD_VECTOR. Don't lower
773  // G_BUILD_VECTORs which could match those patterns.
774  int64_t Cst = Splat->getCst();
775  return (Cst != 0 && Cst != -1);
776 }
777 
779  MachineIRBuilder &B) {
780  B.setInstrAndDebugLoc(MI);
781  B.buildInstr(AArch64::G_DUP, {MI.getOperand(0).getReg()},
782  {MI.getOperand(1).getReg()});
783  MI.eraseFromParent();
784  return true;
785 }
786 
787 /// \returns how many instructions would be saved by folding a G_ICMP's shift
788 /// and/or extension operations.
789 static unsigned getCmpOperandFoldingProfit(Register CmpOp,
790  const MachineRegisterInfo &MRI) {
791  // No instructions to save if there's more than one use or no uses.
792  if (!MRI.hasOneNonDBGUse(CmpOp))
793  return 0;
794 
795  // FIXME: This is duplicated with the selector. (See: selectShiftedRegister)
796  auto IsSupportedExtend = [&](const MachineInstr &MI) {
797  if (MI.getOpcode() == TargetOpcode::G_SEXT_INREG)
798  return true;
799  if (MI.getOpcode() != TargetOpcode::G_AND)
800  return false;
801  auto ValAndVReg =
802  getIConstantVRegValWithLookThrough(MI.getOperand(2).getReg(), MRI);
803  if (!ValAndVReg)
804  return false;
805  uint64_t Mask = ValAndVReg->Value.getZExtValue();
806  return (Mask == 0xFF || Mask == 0xFFFF || Mask == 0xFFFFFFFF);
807  };
808 
810  if (IsSupportedExtend(*Def))
811  return 1;
812 
813  unsigned Opc = Def->getOpcode();
814  if (Opc != TargetOpcode::G_SHL && Opc != TargetOpcode::G_ASHR &&
815  Opc != TargetOpcode::G_LSHR)
816  return 0;
817 
818  auto MaybeShiftAmt =
819  getIConstantVRegValWithLookThrough(Def->getOperand(2).getReg(), MRI);
820  if (!MaybeShiftAmt)
821  return 0;
822  uint64_t ShiftAmt = MaybeShiftAmt->Value.getZExtValue();
823  MachineInstr *ShiftLHS =
824  getDefIgnoringCopies(Def->getOperand(1).getReg(), MRI);
825 
826  // Check if we can fold an extend and a shift.
827  // FIXME: This is duplicated with the selector. (See:
828  // selectArithExtendedRegister)
829  if (IsSupportedExtend(*ShiftLHS))
830  return (ShiftAmt <= 4) ? 2 : 1;
831 
832  LLT Ty = MRI.getType(Def->getOperand(0).getReg());
833  if (Ty.isVector())
834  return 0;
835  unsigned ShiftSize = Ty.getSizeInBits();
836  if ((ShiftSize == 32 && ShiftAmt <= 31) ||
837  (ShiftSize == 64 && ShiftAmt <= 63))
838  return 1;
839  return 0;
840 }
841 
842 /// \returns true if it would be profitable to swap the LHS and RHS of a G_ICMP
843 /// instruction \p MI.
845  const MachineRegisterInfo &MRI) {
846  assert(MI.getOpcode() == TargetOpcode::G_ICMP);
847  // Swap the operands if it would introduce a profitable folding opportunity.
848  // (e.g. a shift + extend).
849  //
850  // For example:
851  // lsl w13, w11, #1
852  // cmp w13, w12
853  // can be turned into:
854  // cmp w12, w11, lsl #1
855 
856  // Don't swap if there's a constant on the RHS, because we know we can fold
857  // that.
858  Register RHS = MI.getOperand(3).getReg();
860  if (RHSCst && isLegalArithImmed(RHSCst->Value.getSExtValue()))
861  return false;
862 
863  Register LHS = MI.getOperand(2).getReg();
864  auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
865  auto GetRegForProfit = [&](Register Reg) {
867  return isCMN(Def, Pred, MRI) ? Def->getOperand(2).getReg() : Reg;
868  };
869 
870  // Don't have a constant on the RHS. If we swap the LHS and RHS of the
871  // compare, would we be able to fold more instructions?
872  Register TheLHS = GetRegForProfit(LHS);
873  Register TheRHS = GetRegForProfit(RHS);
874 
875  // If the LHS is more likely to give us a folding opportunity, then swap the
876  // LHS and RHS.
877  return (getCmpOperandFoldingProfit(TheLHS, MRI) >
879 }
880 
882  GISelChangeObserver &Observer) {
883  auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
884  Register LHS = MI.getOperand(2).getReg();
885  Register RHS = MI.getOperand(3).getReg();
886  Observer.changedInstr(MI);
887  MI.getOperand(1).setPredicate(CmpInst::getSwappedPredicate(Pred));
888  MI.getOperand(2).setReg(RHS);
889  MI.getOperand(3).setReg(LHS);
890  Observer.changedInstr(MI);
891  return true;
892 }
893 
894 /// \returns a function which builds a vector floating point compare instruction
895 /// for a condition code \p CC.
896 /// \param [in] IsZero - True if the comparison is against 0.
897 /// \param [in] NoNans - True if the target has NoNansFPMath.
900  bool NoNans, MachineRegisterInfo &MRI) {
901  LLT DstTy = MRI.getType(LHS);
902  assert(DstTy.isVector() && "Expected vector types only?");
903  assert(DstTy == MRI.getType(RHS) && "Src and Dst types must match!");
904  switch (CC) {
905  default:
906  llvm_unreachable("Unexpected condition code!");
907  case AArch64CC::NE:
908  return [LHS, RHS, IsZero, DstTy](MachineIRBuilder &MIB) {
909  auto FCmp = IsZero
910  ? MIB.buildInstr(AArch64::G_FCMEQZ, {DstTy}, {LHS})
911  : MIB.buildInstr(AArch64::G_FCMEQ, {DstTy}, {LHS, RHS});
912  return MIB.buildNot(DstTy, FCmp).getReg(0);
913  };
914  case AArch64CC::EQ:
915  return [LHS, RHS, IsZero, DstTy](MachineIRBuilder &MIB) {
916  return IsZero
917  ? MIB.buildInstr(AArch64::G_FCMEQZ, {DstTy}, {LHS}).getReg(0)
918  : MIB.buildInstr(AArch64::G_FCMEQ, {DstTy}, {LHS, RHS})
919  .getReg(0);
920  };
921  case AArch64CC::GE:
922  return [LHS, RHS, IsZero, DstTy](MachineIRBuilder &MIB) {
923  return IsZero
924  ? MIB.buildInstr(AArch64::G_FCMGEZ, {DstTy}, {LHS}).getReg(0)
925  : MIB.buildInstr(AArch64::G_FCMGE, {DstTy}, {LHS, RHS})
926  .getReg(0);
927  };
928  case AArch64CC::GT:
929  return [LHS, RHS, IsZero, DstTy](MachineIRBuilder &MIB) {
930  return IsZero
931  ? MIB.buildInstr(AArch64::G_FCMGTZ, {DstTy}, {LHS}).getReg(0)
932  : MIB.buildInstr(AArch64::G_FCMGT, {DstTy}, {LHS, RHS})
933  .getReg(0);
934  };
935  case AArch64CC::LS:
936  return [LHS, RHS, IsZero, DstTy](MachineIRBuilder &MIB) {
937  return IsZero
938  ? MIB.buildInstr(AArch64::G_FCMLEZ, {DstTy}, {LHS}).getReg(0)
939  : MIB.buildInstr(AArch64::G_FCMGE, {DstTy}, {RHS, LHS})
940  .getReg(0);
941  };
942  case AArch64CC::MI:
943  return [LHS, RHS, IsZero, DstTy](MachineIRBuilder &MIB) {
944  return IsZero
945  ? MIB.buildInstr(AArch64::G_FCMLTZ, {DstTy}, {LHS}).getReg(0)
946  : MIB.buildInstr(AArch64::G_FCMGT, {DstTy}, {RHS, LHS})
947  .getReg(0);
948  };
949  }
950 }
951 
952 /// Try to lower a vector G_FCMP \p MI into an AArch64-specific pseudo.
954  MachineIRBuilder &MIB) {
955  assert(MI.getOpcode() == TargetOpcode::G_FCMP);
956  const auto &ST = MI.getMF()->getSubtarget<AArch64Subtarget>();
957  Register Dst = MI.getOperand(0).getReg();
958  LLT DstTy = MRI.getType(Dst);
959  if (!DstTy.isVector() || !ST.hasNEON())
960  return false;
961  const auto Pred =
962  static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
963  Register LHS = MI.getOperand(2).getReg();
964  // TODO: Handle v4s16 case.
965  unsigned EltSize = MRI.getType(LHS).getScalarSizeInBits();
966  if (EltSize != 32 && EltSize != 64)
967  return false;
968  Register RHS = MI.getOperand(3).getReg();
969  auto Splat = getAArch64VectorSplat(*MRI.getVRegDef(RHS), MRI);
970 
971  // Compares against 0 have special target-specific pseudos.
972  bool IsZero = Splat && Splat->isCst() && Splat->getCst() == 0;
973  bool Invert;
974  AArch64CC::CondCode CC, CC2;
975  changeVectorFCMPPredToAArch64CC(Pred, CC, CC2, Invert);
976  bool NoNans = ST.getTargetLowering()->getTargetMachine().Options.NoNaNsFPMath;
977 
978  // Instead of having an apply function, just build here to simplify things.
979  MIB.setInstrAndDebugLoc(MI);
980  auto Cmp = getVectorFCMP(CC, LHS, RHS, IsZero, NoNans, MRI);
981  Register CmpRes;
982  if (CC2 == AArch64CC::AL)
983  CmpRes = Cmp(MIB);
984  else {
985  auto Cmp2 = getVectorFCMP(CC2, LHS, RHS, IsZero, NoNans, MRI);
986  auto Cmp2Dst = Cmp2(MIB);
987  auto Cmp1Dst = Cmp(MIB);
988  CmpRes = MIB.buildOr(DstTy, Cmp1Dst, Cmp2Dst).getReg(0);
989  }
990  if (Invert)
991  CmpRes = MIB.buildNot(DstTy, CmpRes).getReg(0);
992  MRI.replaceRegWith(Dst, CmpRes);
993  MI.eraseFromParent();
994  return false;
995 }
996 
998  Register &SrcReg) {
999  assert(MI.getOpcode() == TargetOpcode::G_STORE);
1000  Register DstReg = MI.getOperand(0).getReg();
1001  if (MRI.getType(DstReg).isVector())
1002  return false;
1003  // Match a store of a truncate.
1004  if (!mi_match(DstReg, MRI, m_GTrunc(m_Reg(SrcReg))))
1005  return false;
1006  // Only form truncstores for value types of max 64b.
1007  return MRI.getType(SrcReg).getSizeInBits() <= 64;
1008 }
1009 
1012  GISelChangeObserver &Observer,
1013  Register &SrcReg) {
1014  assert(MI.getOpcode() == TargetOpcode::G_STORE);
1015  Observer.changingInstr(MI);
1016  MI.getOperand(0).setReg(SrcReg);
1017  Observer.changedInstr(MI);
1018  return true;
1019 }
1020 
1021 // Lower vector G_SEXT_INREG back to shifts for selection. We allowed them to
1022 // form in the first place for combine opportunities, so any remaining ones
1023 // at this stage need be lowered back.
1025  assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
1026  Register DstReg = MI.getOperand(0).getReg();
1027  LLT DstTy = MRI.getType(DstReg);
1028  return DstTy.isVector();
1029 }
1030 
1033  GISelChangeObserver &Observer) {
1034  assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
1035  B.setInstrAndDebugLoc(MI);
1036  LegalizerHelper Helper(*MI.getMF(), Observer, B);
1037  Helper.lower(MI, 0, /* Unused hint type */ LLT());
1038 }
1039 
1040 #define AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_DEPS
1041 #include "AArch64GenPostLegalizeGILowering.inc"
1042 #undef AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_DEPS
1043 
1044 namespace {
1045 #define AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_H
1046 #include "AArch64GenPostLegalizeGILowering.inc"
1047 #undef AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_H
1048 
1049 class AArch64PostLegalizerLoweringInfo : public CombinerInfo {
1050 public:
1051  AArch64GenPostLegalizerLoweringHelperRuleConfig GeneratedRuleCfg;
1052 
1053  AArch64PostLegalizerLoweringInfo(bool OptSize, bool MinSize)
1054  : CombinerInfo(/*AllowIllegalOps*/ true, /*ShouldLegalizeIllegal*/ false,
1055  /*LegalizerInfo*/ nullptr, /*OptEnabled = */ true, OptSize,
1056  MinSize) {
1057  if (!GeneratedRuleCfg.parseCommandLineOption())
1058  report_fatal_error("Invalid rule identifier");
1059  }
1060 
1061  bool combine(GISelChangeObserver &Observer, MachineInstr &MI,
1062  MachineIRBuilder &B) const override;
1063 };
1064 
1066  MachineInstr &MI,
1067  MachineIRBuilder &B) const {
1068  CombinerHelper Helper(Observer, B, /* IsPreLegalize*/ false);
1069  AArch64GenPostLegalizerLoweringHelper Generated(GeneratedRuleCfg);
1070  return Generated.tryCombineAll(Observer, MI, B, Helper);
1071 }
1072 
1073 #define AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_CPP
1074 #include "AArch64GenPostLegalizeGILowering.inc"
1075 #undef AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_CPP
1076 
1077 class AArch64PostLegalizerLowering : public MachineFunctionPass {
1078 public:
1079  static char ID;
1080 
1081  AArch64PostLegalizerLowering();
1082 
1083  StringRef getPassName() const override {
1084  return "AArch64PostLegalizerLowering";
1085  }
1086 
1087  bool runOnMachineFunction(MachineFunction &MF) override;
1088  void getAnalysisUsage(AnalysisUsage &AU) const override;
1089 };
1090 } // end anonymous namespace
1091 
1092 void AArch64PostLegalizerLowering::getAnalysisUsage(AnalysisUsage &AU) const {
1094  AU.setPreservesCFG();
1097 }
1098 
1099 AArch64PostLegalizerLowering::AArch64PostLegalizerLowering()
1100  : MachineFunctionPass(ID) {
1101  initializeAArch64PostLegalizerLoweringPass(*PassRegistry::getPassRegistry());
1102 }
1103 
1104 bool AArch64PostLegalizerLowering::runOnMachineFunction(MachineFunction &MF) {
1105  if (MF.getProperties().hasProperty(
1106  MachineFunctionProperties::Property::FailedISel))
1107  return false;
1109  MachineFunctionProperties::Property::Legalized) &&
1110  "Expected a legalized function?");
1111  auto *TPC = &getAnalysis<TargetPassConfig>();
1112  const Function &F = MF.getFunction();
1113  AArch64PostLegalizerLoweringInfo PCInfo(F.hasOptSize(), F.hasMinSize());
1114  Combiner C(PCInfo, TPC);
1115  return C.combineMachineInstrs(MF, /*CSEInfo*/ nullptr);
1116 }
1117 
1119 INITIALIZE_PASS_BEGIN(AArch64PostLegalizerLowering, DEBUG_TYPE,
1120  "Lower AArch64 MachineInstrs after legalization", false,
1121  false)
1123 INITIALIZE_PASS_END(AArch64PostLegalizerLowering, DEBUG_TYPE,
1124  "Lower AArch64 MachineInstrs after legalization", false,
1125  false)
1126 
1127 namespace llvm {
1129  return new AArch64PostLegalizerLowering();
1130 }
1131 } // end namespace llvm
AArch64LegalizerInfo.h
i
i
Definition: README.txt:29
MIPatternMatch.h
CombinerInfo.h
applyShuffleVectorPseudo
static bool applyShuffleVectorPseudo(MachineInstr &MI, ShuffleVectorPseudo &MatchInfo)
Replace a G_SHUFFLE_VECTOR instruction with a pseudo.
Definition: AArch64PostLegalizerLowering.cpp:438
applyDupLane
bool applyDupLane(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B, std::pair< unsigned, int > &MatchInfo)
Definition: AArch64PostLegalizerLowering.cpp:738
llvm::MachineFunctionProperties::hasProperty
bool hasProperty(Property P) const
Definition: MachineFunction.h:192
llvm::AArch64GISelUtils::changeVectorFCMPPredToAArch64CC
void changeVectorFCMPPredToAArch64CC(const CmpInst::Predicate P, AArch64CC::CondCode &CondCode, AArch64CC::CondCode &CondCode2, bool &Invert)
Find the AArch64 condition codes necessary to represent P for a vector floating point comparison.
Definition: AArch64GlobalISelUtils.cpp:153
llvm::getDefIgnoringCopies
MachineInstr * getDefIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the def instruction for Reg, folding away any trivial copies.
Definition: Utils.cpp:462
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:109
llvm::CmpInst::getSwappedPredicate
Predicate getSwappedPredicate() const
For example, EQ->EQ, SLE->SGE, ULT->UGT, OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
Definition: InstrTypes.h:849
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
llvm::tgtok::Def
@ Def
Definition: TGLexer.h:50
BlockSize
static const int BlockSize
Definition: TarWriter.cpp:33
M
We currently emits eax Perhaps this is what we really should generate is Is imull three or four cycles eax eax The current instruction priority is based on pattern complexity The former is more complex because it folds a load so the latter will not be emitted Perhaps we should use AddedComplexity to give LEA32r a higher priority We should always try to match LEA first since the LEA matching code does some estimate to determine whether the match is profitable if we care more about code then imull is better It s two bytes shorter than movl leal On a Pentium M
Definition: README.txt:252
llvm::AArch64CC::AL
@ AL
Definition: AArch64BaseInfo.h:269
llvm::make_range
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
Definition: iterator_range.h:53
llvm::AArch64CC::NE
@ NE
Definition: AArch64BaseInfo.h:256
llvm::AArch64PACKey::ID
ID
Definition: AArch64BaseInfo.h:818
llvm::LLT::getScalarSizeInBits
unsigned getScalarSizeInBits() const
Definition: LowLevelTypeImpl.h:224
llvm::CmpInst::Predicate
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:719
llvm::MIPatternMatch::m_Reg
operand_type_match m_Reg()
Definition: MIPatternMatch.h:268
llvm::AArch64CC::MI
@ MI
Definition: AArch64BaseInfo.h:259
llvm::MachineRegisterInfo
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Definition: MachineRegisterInfo.h:50
matchDupFromInsertVectorElt
static bool matchDupFromInsertVectorElt(int Lane, MachineInstr &MI, MachineRegisterInfo &MRI, ShuffleVectorPseudo &MatchInfo)
Helper function for matchDup.
Definition: AArch64PostLegalizerLowering.cpp:305
llvm::Function
Definition: Function.h:60
llvm::AArch64GISelUtils::getAArch64VectorSplat
Optional< RegOrConstant > getAArch64VectorSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI)
Definition: AArch64GlobalISelUtils.cpp:21
P
This currently compiles esp xmm0 movsd esp eax eax esp ret We should use not the dag combiner This is because dagcombine2 needs to be able to see through the X86ISD::Wrapper which DAGCombine can t really do The code for turning x load into a single vector load is target independent and should be moved to the dag combiner The code for turning x load into a vector load can only handle a direct load from a global or a direct load from the stack It should be generalized to handle any load from P
Definition: README-SSE.txt:411
llvm::getOpcodeDef
MachineInstr * getOpcodeDef(unsigned Opcode, Register Reg, const MachineRegisterInfo &MRI)
See if Reg is defined by an single def instruction that is Opcode.
Definition: Utils.cpp:476
llvm::SmallVector< SrcOp, 2 >
llvm::MachineIRBuilder::getMRI
MachineRegisterInfo * getMRI()
Getter for MRI.
Definition: MachineIRBuilder.h:289
llvm::logicalview::LVAttributeKind::Generated
@ Generated
ErrorHandling.h
llvm::X86Disassembler::Reg
Reg
All possible values of the reg field in the ModR/M byte.
Definition: X86DisassemblerDecoder.h:462
AArch64BaseInfo.h
llvm::MachineFunctionPass
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
Definition: MachineFunctionPass.h:30
isUZPMask
static bool isUZPMask(ArrayRef< int > M, unsigned NumElts, unsigned &WhichResult)
Determines if M is a shuffle vector mask for a UZP of NumElts.
Definition: AArch64PostLegalizerLowering.cpp:157
isSingletonExtMask
static bool isSingletonExtMask(ArrayRef< int > M, LLT Ty)
Definition: AArch64PostLegalizerLowering.cpp:379
Right
Vector Shift Left Right
Definition: README_P9.txt:118
llvm::MIPatternMatch::m_GTrunc
UnaryOp_match< SrcTy, TargetOpcode::G_TRUNC > m_GTrunc(const SrcTy &Src)
Definition: MIPatternMatch.h:589
applyAdjustICmpImmAndPred
bool applyAdjustICmpImmAndPred(MachineInstr &MI, std::pair< uint64_t, CmpInst::Predicate > &MatchInfo, MachineIRBuilder &MIB, GISelChangeObserver &Observer)
Definition: AArch64PostLegalizerLowering.cpp:671
llvm::CmpInst::ICMP_SGT
@ ICMP_SGT
signed greater than
Definition: InstrTypes.h:746
matchDupLane
bool matchDupLane(MachineInstr &MI, MachineRegisterInfo &MRI, std::pair< unsigned, int > &MatchInfo)
Definition: AArch64PostLegalizerLowering.cpp:686
INITIALIZE_PASS_BEGIN
INITIALIZE_PASS_BEGIN(AArch64PostLegalizerLowering, DEBUG_TYPE, "Lower AArch64 MachineInstrs after legalization", false, false) INITIALIZE_PASS_END(AArch64PostLegalizerLowering
matchZip
static bool matchZip(MachineInstr &MI, MachineRegisterInfo &MRI, ShuffleVectorPseudo &MatchInfo)
Definition: AArch64PostLegalizerLowering.cpp:288
GenericMachineInstrs.h
isINSMask
static std::optional< std::pair< bool, int > > isINSMask(ArrayRef< int > M, int NumInputElements)
Helper function for matchINS.
Definition: AArch64PostLegalizerLowering.cpp:197
llvm::LLT::changeElementCount
LLT changeElementCount(ElementCount EC) const
Return a vector or scalar with the same element type and the new element count.
Definition: LowLevelTypeImpl.h:189
llvm::getSelectionDAGFallbackAnalysisUsage
void getSelectionDAGFallbackAnalysisUsage(AnalysisUsage &AU)
Modify analysis usage so it preserves passes required for the SelectionDAG fallback.
Definition: Utils.cpp:895
llvm::LegalizerHelper::lower
LegalizeResult lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
Legalize an instruction by splitting it into simpler parts, hopefully understood by the target.
Definition: LegalizerHelper.cpp:3245
llvm::CmpInst::ICMP_SLE
@ ICMP_SLE
signed less or equal
Definition: InstrTypes.h:749
matchREV
static bool matchREV(MachineInstr &MI, MachineRegisterInfo &MRI, ShuffleVectorPseudo &MatchInfo)
Definition: AArch64PostLegalizerLowering.cpp:222
MachineIRBuilder.h
RHS
Value * RHS
Definition: X86PartialReduction.cpp:76
llvm::CombinerInfo
Definition: CombinerInfo.h:26
llvm::LegalizerHelper
Definition: LegalizerHelper.h:46
getCmpOperandFoldingProfit
static unsigned getCmpOperandFoldingProfit(Register CmpOp, const MachineRegisterInfo &MRI)
Definition: AArch64PostLegalizerLowering.cpp:789
llvm::MachineFunctionPass::getAnalysisUsage
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
Definition: MachineFunctionPass.cpp:167
llvm::MachineIRBuilder::buildConstant
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
Definition: MachineIRBuilder.cpp:293
F
#define F(x, y, z)
Definition: MD5.cpp:55
MachineRegisterInfo.h
applyVectorSextInReg
static void applyVectorSextInReg(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B, GISelChangeObserver &Observer)
Definition: AArch64PostLegalizerLowering.cpp:1031
llvm::LLT::fixed_vector
static LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
Definition: LowLevelTypeImpl.h:74
llvm::AlignStyle::Left
@ Left
llvm::BitmaskEnumDetail::Mask
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:80
LHS
Value * LHS
Definition: X86PartialReduction.cpp:75
llvm::createAArch64PostLegalizerLowering
FunctionPass * createAArch64PostLegalizerLowering()
Definition: AArch64PostLegalizerLowering.cpp:1128
AArch64TargetMachine.h
getReg
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
Definition: MipsDisassembler.cpp:517
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
applyBuildVectorToDup
static bool applyBuildVectorToDup(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B)
Definition: AArch64PostLegalizerLowering.cpp:778
llvm::CmpInst::ICMP_ULE
@ ICMP_ULE
unsigned less or equal
Definition: InstrTypes.h:745
InstrTypes.h
llvm::LLT::getSizeInBits
TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
Definition: LowLevelTypeImpl.h:152
ShuffleVectorPseudo::ShuffleVectorPseudo
ShuffleVectorPseudo(unsigned Opc, Register Dst, std::initializer_list< SrcOp > SrcOps)
Definition: AArch64PostLegalizerLowering.cpp:62
llvm::AnalysisUsage
Represent the analysis usage information of a pass.
Definition: PassAnalysisSupport.h:47
Utils.h
llvm::MachineFunction::getProperties
const MachineFunctionProperties & getProperties() const
Get the function properties.
Definition: MachineFunction.h:748
UINT64_MAX
#define UINT64_MAX
Definition: DataTypes.h:77
tryAdjustICmpImmAndPred
std::optional< std::pair< uint64_t, CmpInst::Predicate > > tryAdjustICmpImmAndPred(Register RHS, CmpInst::Predicate P, const MachineRegisterInfo &MRI)
Determine if it is possible to modify the RHS and predicate P of a G_ICMP instruction such that the r...
Definition: AArch64PostLegalizerLowering.cpp:562
false
Definition: StackSlotColoring.cpp:141
TargetOpcodes.h
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
llvm::MachineOperand
MachineOperand class - Representation of each machine instruction operand.
Definition: MachineOperand.h:48
llvm::MIPatternMatch::m_ZeroInt
SpecificConstantMatch m_ZeroInt()
{ Convenience matchers for specific integer values.
Definition: MIPatternMatch.h:236
llvm::MachineIRBuilder::buildOr
MachineInstrBuilder buildOr(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, Optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_OR Op0, Op1.
Definition: MachineIRBuilder.h:1614
llvm::report_fatal_error
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:145
llvm::APInt::getZExtValue
uint64_t getZExtValue() const
Get zero extended value.
Definition: APInt.h:1486
INT64_MAX
#define INT64_MAX
Definition: DataTypes.h:71
llvm::GISelChangeObserver::changingInstr
virtual void changingInstr(MachineInstr &MI)=0
This instruction is about to be mutated in some way.
llvm::CombinerHelper
Definition: CombinerHelper.h:109
llvm::MachineRegisterInfo::getVRegDef
MachineInstr * getVRegDef(Register Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
Definition: MachineRegisterInfo.cpp:396
llvm::LinearPolySize< ElementCount >::getFixed
static ElementCount getFixed(ScalarTy MinVal)
Definition: TypeSize.h:283
llvm::CallingConv::ID
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
llvm::MachineInstrBuilder::getReg
Register getReg(unsigned Idx) const
Get the register for the operand index.
Definition: MachineInstrBuilder.h:94
INITIALIZE_PASS_END
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:58
isREVMask
static bool isREVMask(ArrayRef< int > M, unsigned EltSize, unsigned NumElts, unsigned BlockSize)
Check if a vector shuffle corresponds to a REV instruction with the specified blocksize.
Definition: AArch64PostLegalizerLowering.cpp:70
ShuffleVectorPseudo::SrcOps
SmallVector< SrcOp, 2 > SrcOps
Source registers.
Definition: AArch64PostLegalizerLowering.cpp:61
isTRNMask
static bool isTRNMask(ArrayRef< int > M, unsigned NumElts, unsigned &WhichResult)
Determines if M is a shuffle vector mask for a TRN of NumElts.
Definition: AArch64PostLegalizerLowering.cpp:99
lowerVectorFCMP
static bool lowerVectorFCMP(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIB)
Try to lower a vector G_FCMP MI into an AArch64-specific pseudo.
Definition: AArch64PostLegalizerLowering.cpp:953
llvm::TargetPassConfig
Target-Independent Code Generator Pass Configuration Options.
Definition: TargetPassConfig.h:84
llvm::GISelChangeObserver::changedInstr
virtual void changedInstr(MachineInstr &MI)=0
This instruction was mutated in some way.
AArch64GlobalISelUtils.h
llvm::LegacyLegalizeActions::Lower
@ Lower
The operation itself must be expressed in terms of simpler actions on this target.
Definition: LegacyLegalizerInfo.h:58
isVShiftRImm
static bool isVShiftRImm(Register Reg, MachineRegisterInfo &MRI, LLT Ty, int64_t &Cnt)
isVShiftRImm - Check if this is a valid vector for the immediate operand of a vector shift right oper...
Definition: AArch64PostLegalizerLowering.cpp:518
llvm::MachineIRBuilder
Helper class to build MachineInstr.
Definition: MachineIRBuilder.h:221
llvm::RegState::Undef
@ Undef
Value of the register doesn't matter.
Definition: MachineInstrBuilder.h:52
llvm::MachineInstr
Representation of each machine instruction.
Definition: MachineInstr.h:66
ShuffleVectorPseudo::Opc
unsigned Opc
Opcode for the instruction. (E.g. G_ZIP1)
Definition: AArch64PostLegalizerLowering.cpp:59
uint64_t
llvm::Combiner
Definition: Combiner.h:26
isZipMask
static bool isZipMask(ArrayRef< int > M, unsigned NumElts, unsigned &WhichResult)
Definition: AArch64PostLegalizerLowering.cpp:172
applySwapICmpOperands
static bool applySwapICmpOperands(MachineInstr &MI, GISelChangeObserver &Observer)
Definition: AArch64PostLegalizerLowering.cpp:881
llvm::ARM_MB::ST
@ ST
Definition: ARMBaseInfo.h:73
isCMN
static bool isCMN(SDValue Op, ISD::CondCode CC)
Definition: AArch64ISelLowering.cpp:3024
INITIALIZE_PASS_DEPENDENCY
INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
llvm::APInt::logBase2
unsigned logBase2() const
Definition: APInt.h:1672
llvm::initializeAArch64PostLegalizerLoweringPass
void initializeAArch64PostLegalizerLoweringPass(PassRegistry &)
matchDupFromBuildVector
static bool matchDupFromBuildVector(int Lane, MachineInstr &MI, MachineRegisterInfo &MRI, ShuffleVectorPseudo &MatchInfo)
Helper function for matchDup.
Definition: AArch64PostLegalizerLowering.cpp:344
llvm::MachineIRBuilder::setInstrAndDebugLoc
void setInstrAndDebugLoc(MachineInstr &MI)
Set the insertion point to before MI, and set the debug loc to MI's loc.
Definition: MachineIRBuilder.h:352
I
#define I(x, y, z)
Definition: MD5.cpp:58
ShuffleVectorPseudo::Dst
Register Dst
Destination register.
Definition: AArch64PostLegalizerLowering.cpp:60
llvm::LLT::isVector
bool isVector() const
Definition: LowLevelTypeImpl.h:122
llvm::LLT::getNumElements
uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
Definition: LowLevelTypeImpl.h:126
TargetPassConfig.h
MachineFunctionPass.h
ShuffleVectorPseudo
Represents a pseudo instruction which replaces a G_SHUFFLE_VECTOR.
Definition: AArch64PostLegalizerLowering.cpp:58
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:853
llvm::CmpInst::ICMP_UGE
@ ICMP_UGE
unsigned greater or equal
Definition: InstrTypes.h:743
matchUZP
static bool matchUZP(MachineInstr &MI, MachineRegisterInfo &MRI, ShuffleVectorPseudo &MatchInfo)
Definition: AArch64PostLegalizerLowering.cpp:272
matchBuildVectorToDup
static bool matchBuildVectorToDup(MachineInstr &MI, MachineRegisterInfo &MRI)
Definition: AArch64PostLegalizerLowering.cpp:764
function
print Print MemDeps of function
Definition: MemDepPrinter.cpp:82
applyFormTruncstore
static bool applyFormTruncstore(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B, GISelChangeObserver &Observer, Register &SrcReg)
Definition: AArch64PostLegalizerLowering.cpp:1010
matchAdjustICmpImmAndPred
bool matchAdjustICmpImmAndPred(MachineInstr &MI, const MachineRegisterInfo &MRI, std::pair< uint64_t, CmpInst::Predicate > &MatchInfo)
Determine whether or not it is possible to update the RHS and predicate of a G_ICMP instruction such ...
Definition: AArch64PostLegalizerLowering.cpp:658
DEBUG_TYPE
#define DEBUG_TYPE
Definition: AArch64PostLegalizerLowering.cpp:49
Builder
assume Assume Builder
Definition: AssumeBundleBuilder.cpp:651
llvm::APInt
Class for arbitrary precision integers.
Definition: APInt.h:75
llvm::MachineFunction
Definition: MachineFunction.h:257
llvm::CmpInst::ICMP_SLT
@ ICMP_SLT
signed less than
Definition: InstrTypes.h:748
llvm::MachineIRBuilder::buildInstr
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
Definition: MachineIRBuilder.h:383
CombinerHelper.h
matchDup
static bool matchDup(MachineInstr &MI, MachineRegisterInfo &MRI, ShuffleVectorPseudo &MatchInfo)
Definition: AArch64PostLegalizerLowering.cpp:360
matchEXT
static bool matchEXT(MachineInstr &MI, MachineRegisterInfo &MRI, ShuffleVectorPseudo &MatchInfo)
Definition: AArch64PostLegalizerLowering.cpp:406
llvm::AArch64CC::GE
@ GE
Definition: AArch64BaseInfo.h:265
llvm::ArrayRef< int >
legalization
Lower AArch64 MachineInstrs after legalization
Definition: AArch64PostLegalizerLowering.cpp:1124
llvm::any_of
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1741
llvm::getIConstantVRegValWithLookThrough
Optional< ValueAndVReg > getIConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_CONSTANT returns its...
Definition: Utils.cpp:410
llvm::AnalysisUsage::setPreservesCFG
void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition: Pass.cpp:265
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
llvm::MachineRegisterInfo::hasOneNonDBGUse
bool hasOneNonDBGUse(Register RegNo) const
hasOneNonDBGUse - Return true if there is exactly one non-Debug use of the specified register.
Definition: MachineRegisterInfo.cpp:415
llvm::CmpInst::ICMP_ULT
@ ICMP_ULT
unsigned less than
Definition: InstrTypes.h:744
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
Combiner.h
uint32_t
CC
auto CC
Definition: RISCVRedundantCopyElimination.cpp:79
llvm::GISelChangeObserver
Abstract class that contains various methods for clients to notify about changes.
Definition: GISelChangeObserver.h:29
llvm::AArch64CC::EQ
@ EQ
Definition: AArch64BaseInfo.h:255
llvm::MachineIRBuilder::buildNot
MachineInstrBuilder buildNot(const DstOp &Dst, const SrcOp &Src0)
Build and insert a bitwise not, NegOne = G_CONSTANT -1 Res = G_OR Op0, NegOne.
Definition: MachineIRBuilder.h:1629
MRI
unsigned const MachineRegisterInfo * MRI
Definition: AArch64AdvSIMDScalarPass.cpp:105
llvm::Register
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
applyVAshrLshrImm
static bool applyVAshrLshrImm(MachineInstr &MI, MachineRegisterInfo &MRI, int64_t &Imm)
Definition: AArch64PostLegalizerLowering.cpp:541
llvm::find_if
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1761
llvm::MachineRegisterInfo::replaceRegWith
void replaceRegWith(Register FromReg, Register ToReg)
replaceRegWith - Replace all instances of FromReg with ToReg in the machine function.
Definition: MachineRegisterInfo.cpp:378
llvm::NVPTX::PTXLdStInstCode::V2
@ V2
Definition: NVPTX.h:123
llvm::MachineFunction::getFunction
Function & getFunction()
Return the LLVM function that this machine code represents.
Definition: MachineFunction.h:623
llvm::AArch64CC::LS
@ LS
Definition: AArch64BaseInfo.h:264
applyINS
static bool applyINS(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &Builder, std::tuple< Register, int, Register, int > &MatchInfo)
Definition: AArch64PostLegalizerLowering.cpp:498
matchVAshrLshrImm
static bool matchVAshrLshrImm(MachineInstr &MI, MachineRegisterInfo &MRI, int64_t &Imm)
Match a vector G_ASHR or G_LSHR with a valid immediate shift.
Definition: AArch64PostLegalizerLowering.cpp:531
LegalizerHelper.h
matchINS
static bool matchINS(MachineInstr &MI, MachineRegisterInfo &MRI, std::tuple< Register, int, Register, int > &MatchInfo)
Match a G_SHUFFLE_VECTOR with a mask which corresponds to a G_INSERT_VECTOR_ELT and G_EXTRACT_VECTOR_...
Definition: AArch64PostLegalizerLowering.cpp:471
AArch64MCTargetDesc.h
trySwapICmpOperands
static bool trySwapICmpOperands(MachineInstr &MI, const MachineRegisterInfo &MRI)
Definition: AArch64PostLegalizerLowering.cpp:844
llvm::CmpInst::ICMP_SGE
@ ICMP_SGE
signed greater or equal
Definition: InstrTypes.h:747
llvm::AArch64CC::GT
@ GT
Definition: AArch64BaseInfo.h:267
applyEXT
static bool applyEXT(MachineInstr &MI, ShuffleVectorPseudo &MatchInfo)
Replace a G_SHUFFLE_VECTOR instruction with G_EXT.
Definition: AArch64PostLegalizerLowering.cpp:449
llvm::RISCVMatInt::Imm
@ Imm
Definition: RISCVMatInt.h:23
matchFormTruncstore
static bool matchFormTruncstore(MachineInstr &MI, MachineRegisterInfo &MRI, Register &SrcReg)
Definition: AArch64PostLegalizerLowering.cpp:997
AArch64Subtarget.h
llvm::getSplatIndex
int getSplatIndex(ArrayRef< int > Mask)
If all non-negative Mask elements are the same value, return that value.
Definition: VectorUtils.cpp:349
llvm::MachineRegisterInfo::getType
LLT getType(Register Reg) const
Get the low-level type of Reg or LLT{} if Reg is not a generic (target independent) virtual register.
Definition: MachineRegisterInfo.h:745
MachineInstrBuilder.h
llvm::MachineRegisterInfo::cloneVirtualRegister
Register cloneVirtualRegister(Register VReg, StringRef Name="")
Create and return a new virtual register in the function with the same attributes as the given regist...
Definition: MachineRegisterInfo.cpp:170
llvm::CmpInst::ICMP_UGT
@ ICMP_UGT
unsigned greater than
Definition: InstrTypes.h:742
llvm::AArch64CC::CondCode
CondCode
Definition: AArch64BaseInfo.h:254
llvm::FunctionPass
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:308
matchTRN
static bool matchTRN(MachineInstr &MI, MachineRegisterInfo &MRI, ShuffleVectorPseudo &MatchInfo)
Definition: AArch64PostLegalizerLowering.cpp:251
llvm::MIPatternMatch::mi_match
bool mi_match(Reg R, const MachineRegisterInfo &MRI, Pattern &&P)
Definition: MIPatternMatch.h:25
llvm::AnalysisUsage::addRequired
AnalysisUsage & addRequired()
Definition: PassAnalysisSupport.h:75
GISelChangeObserver.h
llvm::LLT::getElementType
LLT getElementType() const
Returns the vector's element type. Only valid for vector types.
Definition: LowLevelTypeImpl.h:248
llvm::AArch64Subtarget
Definition: AArch64Subtarget.h:38
combine
vector combine
Definition: VectorCombine.cpp:1843
llvm::LLT::scalar
static LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
Definition: LowLevelTypeImpl.h:42
llvm::AArch64GISelUtils::getAArch64VectorSplatScalar
Optional< int64_t > getAArch64VectorSplatScalar(const MachineInstr &MI, const MachineRegisterInfo &MRI)
Definition: AArch64GlobalISelUtils.cpp:35
getExtMask
static std::optional< std::pair< bool, uint64_t > > getExtMask(ArrayRef< int > M, unsigned NumElts)
Check if a G_EXT instruction can handle a shuffle mask M when the vector sources of the shuffle are d...
Definition: AArch64PostLegalizerLowering.cpp:115
InitializePasses.h
Debug.h
matchVectorSextInReg
static bool matchVectorSextInReg(MachineInstr &MI, MachineRegisterInfo &MRI)
Definition: AArch64PostLegalizerLowering.cpp:1024
getVectorFCMP
static std::function< Register(MachineIRBuilder &)> getVectorFCMP(AArch64CC::CondCode CC, Register LHS, Register RHS, bool IsZero, bool NoNans, MachineRegisterInfo &MRI)
Definition: AArch64PostLegalizerLowering.cpp:899
AArch64TargetInfo.h
isLegalArithImmed
static bool isLegalArithImmed(uint64_t C)
Definition: AArch64ISelLowering.cpp:3006
llvm::LLT
Definition: LowLevelTypeImpl.h:39
INT64_MIN
#define INT64_MIN
Definition: DataTypes.h:74