LLVM  16.0.0git
AArch64LegalizerInfo.cpp
Go to the documentation of this file.
1 //===- AArch64LegalizerInfo.cpp ----------------------------------*- C++ -*-==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 /// This file implements the targeting of the Machinelegalizer class for
10 /// AArch64.
11 /// \todo This should be generated by TableGen.
12 //===----------------------------------------------------------------------===//
13 
14 #include "AArch64LegalizerInfo.h"
16 #include "AArch64Subtarget.h"
26 #include "llvm/IR/DerivedTypes.h"
27 #include "llvm/IR/Intrinsics.h"
28 #include "llvm/IR/IntrinsicsAArch64.h"
29 #include "llvm/IR/Type.h"
31 #include <initializer_list>
32 
33 #define DEBUG_TYPE "aarch64-legalinfo"
34 
35 using namespace llvm;
36 using namespace LegalizeActions;
37 using namespace LegalizeMutations;
38 using namespace LegalityPredicates;
39 using namespace MIPatternMatch;
40 
42  : ST(&ST) {
43  using namespace TargetOpcode;
44  const LLT p0 = LLT::pointer(0, 64);
45  const LLT s8 = LLT::scalar(8);
46  const LLT s16 = LLT::scalar(16);
47  const LLT s32 = LLT::scalar(32);
48  const LLT s64 = LLT::scalar(64);
49  const LLT s128 = LLT::scalar(128);
50  const LLT v16s8 = LLT::fixed_vector(16, 8);
51  const LLT v8s8 = LLT::fixed_vector(8, 8);
52  const LLT v4s8 = LLT::fixed_vector(4, 8);
53  const LLT v8s16 = LLT::fixed_vector(8, 16);
54  const LLT v4s16 = LLT::fixed_vector(4, 16);
55  const LLT v2s16 = LLT::fixed_vector(2, 16);
56  const LLT v2s32 = LLT::fixed_vector(2, 32);
57  const LLT v4s32 = LLT::fixed_vector(4, 32);
58  const LLT v2s64 = LLT::fixed_vector(2, 64);
59  const LLT v2p0 = LLT::fixed_vector(2, p0);
60 
61  std::initializer_list<LLT> PackedVectorAllTypeList = {/* Begin 128bit types */
62  v16s8, v8s16, v4s32,
63  v2s64, v2p0,
64  /* End 128bit types */
65  /* Begin 64bit types */
66  v8s8, v4s16, v2s32};
67 
68  const TargetMachine &TM = ST.getTargetLowering()->getTargetMachine();
69 
70  // FIXME: support subtargets which have neon/fp-armv8 disabled.
71  if (!ST.hasNEON() || !ST.hasFPARMv8()) {
73  return;
74  }
75 
76  // Some instructions only support s16 if the subtarget has full 16-bit FP
77  // support.
78  const bool HasFP16 = ST.hasFullFP16();
79  const LLT &MinFPScalar = HasFP16 ? s16 : s32;
80 
81  getActionDefinitionsBuilder({G_IMPLICIT_DEF, G_FREEZE})
82  .legalFor({p0, s8, s16, s32, s64})
83  .legalFor(PackedVectorAllTypeList)
85  .clampScalar(0, s8, s64)
87  [=](const LegalityQuery &Query) {
88  return Query.Types[0].isVector() &&
89  (Query.Types[0].getElementType() != s64 ||
90  Query.Types[0].getNumElements() != 2);
91  },
92  [=](const LegalityQuery &Query) {
93  LLT EltTy = Query.Types[0].getElementType();
94  if (EltTy == s64)
95  return std::make_pair(0, LLT::fixed_vector(2, 64));
96  return std::make_pair(0, EltTy);
97  });
98 
100  .legalFor({p0, s16, s32, s64})
101  .legalFor(PackedVectorAllTypeList)
103  .clampScalar(0, s16, s64)
104  // Maximum: sN * k = 128
105  .clampMaxNumElements(0, s8, 16)
106  .clampMaxNumElements(0, s16, 8)
107  .clampMaxNumElements(0, s32, 4)
108  .clampMaxNumElements(0, s64, 2)
109  .clampMaxNumElements(0, p0, 2);
110 
112  .legalFor({s32, s64, v4s32, v2s32, v2s64})
113  .widenScalarToNextPow2(0)
114  .clampScalar(0, s32, s64);
115 
116  getActionDefinitionsBuilder({G_ADD, G_SUB, G_MUL, G_AND, G_OR, G_XOR})
117  .legalFor({s32, s64, v2s32, v4s32, v4s16, v8s16, v16s8, v8s8})
118  .scalarizeIf(
119  [=](const LegalityQuery &Query) {
120  return Query.Opcode == G_MUL && Query.Types[0] == v2s64;
121  },
122  0)
123  .legalFor({v2s64})
124  .widenScalarToNextPow2(0)
125  .clampScalar(0, s32, s64)
126  .clampNumElements(0, v2s32, v4s32)
127  .clampNumElements(0, v2s64, v2s64)
129 
130  getActionDefinitionsBuilder({G_SHL, G_ASHR, G_LSHR})
131  .customIf([=](const LegalityQuery &Query) {
132  const auto &SrcTy = Query.Types[0];
133  const auto &AmtTy = Query.Types[1];
134  return !SrcTy.isVector() && SrcTy.getSizeInBits() == 32 &&
135  AmtTy.getSizeInBits() == 32;
136  })
137  .legalFor({
138  {s32, s32},
139  {s32, s64},
140  {s64, s64},
141  {v8s8, v8s8},
142  {v16s8, v16s8},
143  {v4s16, v4s16},
144  {v8s16, v8s16},
145  {v2s32, v2s32},
146  {v4s32, v4s32},
147  {v2s64, v2s64},
148  })
149  .widenScalarToNextPow2(0)
150  .clampScalar(1, s32, s64)
151  .clampScalar(0, s32, s64)
152  .clampNumElements(0, v2s32, v4s32)
153  .clampNumElements(0, v2s64, v2s64)
155  .minScalarSameAs(1, 0);
156 
157  getActionDefinitionsBuilder(G_PTR_ADD)
158  .legalFor({{p0, s64}, {v2p0, v2s64}})
159  .clampScalar(1, s64, s64);
160 
161  getActionDefinitionsBuilder(G_PTRMASK).legalFor({{p0, s64}});
162 
163  getActionDefinitionsBuilder({G_SDIV, G_UDIV})
164  .legalFor({s32, s64})
165  .libcallFor({s128})
166  .clampScalar(0, s32, s64)
168  .scalarize(0);
169 
170  getActionDefinitionsBuilder({G_SREM, G_UREM, G_SDIVREM, G_UDIVREM})
171  .lowerFor({s8, s16, s32, s64, v2s64, v4s32, v2s32})
173  .clampScalarOrElt(0, s32, s64)
174  .clampNumElements(0, v2s32, v4s32)
175  .clampNumElements(0, v2s64, v2s64)
176  .moreElementsToNextPow2(0);
177 
178 
179  getActionDefinitionsBuilder({G_SMULO, G_UMULO})
180  .widenScalarToNextPow2(0, /*Min = */ 32)
181  .clampScalar(0, s32, s64)
182  .lower();
183 
184  getActionDefinitionsBuilder({G_SMULH, G_UMULH})
185  .legalFor({s64, v8s16, v16s8, v4s32})
186  .lower();
187 
188  getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX})
189  .legalFor({v8s8, v16s8, v4s16, v8s16, v2s32, v4s32})
190  .clampNumElements(0, v8s8, v16s8)
191  .clampNumElements(0, v4s16, v8s16)
192  .clampNumElements(0, v2s32, v4s32)
193  // FIXME: This sholdn't be needed as v2s64 types are going to
194  // be expanded anyway, but G_ICMP doesn't support splitting vectors yet
195  .clampNumElements(0, v2s64, v2s64)
196  .lower();
197 
199  {G_SADDE, G_SSUBE, G_UADDE, G_USUBE, G_SADDO, G_SSUBO, G_UADDO, G_USUBO})
200  .legalFor({{s32, s32}, {s64, s32}})
201  .clampScalar(0, s32, s64)
202  .clampScalar(1, s32, s64)
204 
205  getActionDefinitionsBuilder({G_FADD, G_FSUB, G_FMUL, G_FDIV, G_FNEG})
206  .legalFor({MinFPScalar, s32, s64, v2s64, v4s32, v2s32})
207  .clampScalar(0, MinFPScalar, s64)
208  .clampNumElements(0, v2s32, v4s32)
209  .clampNumElements(0, v2s64, v2s64);
210 
211  getActionDefinitionsBuilder(G_FREM).libcallFor({s32, s64});
212 
213  getActionDefinitionsBuilder({G_FCEIL, G_FABS, G_FSQRT, G_FFLOOR, G_FRINT,
214  G_FMA, G_INTRINSIC_TRUNC, G_INTRINSIC_ROUND,
215  G_FNEARBYINT, G_INTRINSIC_LRINT})
216  // If we don't have full FP16 support, then scalarize the elements of
217  // vectors containing fp16 types.
218  .fewerElementsIf(
219  [=, &ST](const LegalityQuery &Query) {
220  const auto &Ty = Query.Types[0];
221  return Ty.isVector() && Ty.getElementType() == s16 &&
222  !ST.hasFullFP16();
223  },
224  [=](const LegalityQuery &Query) { return std::make_pair(0, s16); })
225  // If we don't have full FP16 support, then widen s16 to s32 if we
226  // encounter it.
227  .widenScalarIf(
228  [=, &ST](const LegalityQuery &Query) {
229  return Query.Types[0] == s16 && !ST.hasFullFP16();
230  },
231  [=](const LegalityQuery &Query) { return std::make_pair(0, s32); })
232  .legalFor({s16, s32, s64, v2s32, v4s32, v2s64, v2s16, v4s16, v8s16});
233 
235  {G_FCOS, G_FSIN, G_FLOG10, G_FLOG, G_FLOG2, G_FEXP, G_FEXP2, G_FPOW})
236  // We need a call for these, so we always need to scalarize.
237  .scalarize(0)
238  // Regardless of FP16 support, widen 16-bit elements to 32-bits.
239  .minScalar(0, s32)
240  .libcallFor({s32, s64, v2s32, v4s32, v2s64});
241 
243  .legalIf(all(typeInSet(0, {s32, s64, p0}),
244  typeInSet(1, {s8, s16, s32}), smallerThan(1, 0)))
246  .clampScalar(0, s32, s64)
248  .minScalar(1, s8)
249  .maxScalarIf(typeInSet(0, {s32}), 1, s16)
250  .maxScalarIf(typeInSet(0, {s64, p0}), 1, s32);
251 
252  getActionDefinitionsBuilder(G_EXTRACT)
253  .legalIf(all(typeInSet(0, {s16, s32, s64, p0}),
254  typeInSet(1, {s32, s64, s128, p0}), smallerThan(0, 1)))
256  .clampScalar(1, s32, s128)
258  .minScalar(0, s16)
259  .maxScalarIf(typeInSet(1, {s32}), 0, s16)
260  .maxScalarIf(typeInSet(1, {s64, p0}), 0, s32)
261  .maxScalarIf(typeInSet(1, {s128}), 0, s64);
262 
263 
264  for (unsigned Op : {G_SEXTLOAD, G_ZEXTLOAD}) {
265  auto &Actions = getActionDefinitionsBuilder(Op);
266 
267  if (Op == G_SEXTLOAD)
269 
270  // Atomics have zero extending behavior.
271  Actions
272  .legalForTypesWithMemDesc({{s32, p0, s8, 8},
273  {s32, p0, s16, 8},
274  {s32, p0, s32, 8},
275  {s64, p0, s8, 2},
276  {s64, p0, s16, 2},
277  {s64, p0, s32, 4},
278  {s64, p0, s64, 8},
279  {p0, p0, s64, 8},
280  {v2s32, p0, s64, 8}})
281  .widenScalarToNextPow2(0)
282  .clampScalar(0, s32, s64)
283  // TODO: We could support sum-of-pow2's but the lowering code doesn't know
284  // how to do that yet.
285  .unsupportedIfMemSizeNotPow2()
286  // Lower anything left over into G_*EXT and G_LOAD
287  .lower();
288  }
289 
290  auto IsPtrVecPred = [=](const LegalityQuery &Query) {
291  const LLT &ValTy = Query.Types[0];
292  if (!ValTy.isVector())
293  return false;
294  const LLT EltTy = ValTy.getElementType();
295  return EltTy.isPointer() && EltTy.getAddressSpace() == 0;
296  };
297 
299  .customIf([=](const LegalityQuery &Query) {
300  return Query.Types[0] == s128 &&
301  Query.MMODescrs[0].Ordering != AtomicOrdering::NotAtomic;
302  })
303  .legalForTypesWithMemDesc({{s8, p0, s8, 8},
304  {s16, p0, s16, 8},
305  {s32, p0, s32, 8},
306  {s64, p0, s64, 8},
307  {p0, p0, s64, 8},
308  {s128, p0, s128, 8},
309  {v8s8, p0, s64, 8},
310  {v16s8, p0, s128, 8},
311  {v4s16, p0, s64, 8},
312  {v8s16, p0, s128, 8},
313  {v2s32, p0, s64, 8},
314  {v4s32, p0, s128, 8},
315  {v2s64, p0, s128, 8}})
316  // These extends are also legal
317  .legalForTypesWithMemDesc({{s32, p0, s8, 8}, {s32, p0, s16, 8}})
318  .widenScalarToNextPow2(0, /* MinSize = */8)
320  .clampScalar(0, s8, s64)
321  .narrowScalarIf([=](const LegalityQuery &Query) {
322  // Clamp extending load results to 32-bits.
323  return Query.Types[0].isScalar() &&
324  Query.Types[0] != Query.MMODescrs[0].MemoryTy &&
325  Query.Types[0].getSizeInBits() > 32;
326  },
327  changeTo(0, s32))
328  .clampMaxNumElements(0, s8, 16)
329  .clampMaxNumElements(0, s16, 8)
330  .clampMaxNumElements(0, s32, 4)
331  .clampMaxNumElements(0, s64, 2)
332  .clampMaxNumElements(0, p0, 2)
333  .customIf(IsPtrVecPred)
334  .scalarizeIf(typeIs(0, v2s16), 0);
335 
337  .customIf([=](const LegalityQuery &Query) {
338  return Query.Types[0] == s128 &&
339  Query.MMODescrs[0].Ordering != AtomicOrdering::NotAtomic;
340  })
341  .legalForTypesWithMemDesc({{s8, p0, s8, 8},
342  {s16, p0, s8, 8}, // truncstorei8 from s16
343  {s32, p0, s8, 8}, // truncstorei8 from s32
344  {s64, p0, s8, 8}, // truncstorei8 from s64
345  {s16, p0, s16, 8},
346  {s32, p0, s16, 8}, // truncstorei16 from s32
347  {s64, p0, s16, 8}, // truncstorei16 from s64
348  {s32, p0, s8, 8},
349  {s32, p0, s16, 8},
350  {s32, p0, s32, 8},
351  {s64, p0, s64, 8},
352  {s64, p0, s32, 8}, // truncstorei32 from s64
353  {p0, p0, s64, 8},
354  {s128, p0, s128, 8},
355  {v16s8, p0, s128, 8},
356  {v8s8, p0, s64, 8},
357  {v4s16, p0, s64, 8},
358  {v8s16, p0, s128, 8},
359  {v2s32, p0, s64, 8},
360  {v4s32, p0, s128, 8},
361  {v2s64, p0, s128, 8}})
362  .clampScalar(0, s8, s64)
363  .lowerIf([=](const LegalityQuery &Query) {
364  return Query.Types[0].isScalar() &&
365  Query.Types[0] != Query.MMODescrs[0].MemoryTy;
366  })
367  // Maximum: sN * k = 128
368  .clampMaxNumElements(0, s8, 16)
369  .clampMaxNumElements(0, s16, 8)
370  .clampMaxNumElements(0, s32, 4)
371  .clampMaxNumElements(0, s64, 2)
372  .clampMaxNumElements(0, p0, 2)
374  .customIf(IsPtrVecPred)
375  .scalarizeIf(typeIs(0, v2s16), 0);
376 
377  // Constants
378  getActionDefinitionsBuilder(G_CONSTANT)
379  .legalFor({p0, s8, s16, s32, s64})
380  .widenScalarToNextPow2(0)
381  .clampScalar(0, s8, s64);
382  getActionDefinitionsBuilder(G_FCONSTANT)
383  .legalIf([=](const LegalityQuery &Query) {
384  const auto &Ty = Query.Types[0];
385  if (HasFP16 && Ty == s16)
386  return true;
387  return Ty == s32 || Ty == s64 || Ty == s128;
388  })
389  .clampScalar(0, MinFPScalar, s128);
390 
391  getActionDefinitionsBuilder({G_ICMP, G_FCMP})
392  .legalFor({{s32, s32},
393  {s32, s64},
394  {s32, p0},
395  {v4s32, v4s32},
396  {v2s32, v2s32},
397  {v2s64, v2s64},
398  {v2s64, v2p0},
399  {v4s16, v4s16},
400  {v8s16, v8s16},
401  {v8s8, v8s8},
402  {v16s8, v16s8}})
404  .clampScalar(1, s32, s64)
405  .clampScalar(0, s32, s32)
406  .minScalarEltSameAsIf(
407  [=](const LegalityQuery &Query) {
408  const LLT &Ty = Query.Types[0];
409  const LLT &SrcTy = Query.Types[1];
410  return Ty.isVector() && !SrcTy.getElementType().isPointer() &&
411  Ty.getElementType() != SrcTy.getElementType();
412  },
413  0, 1)
414  .minScalarOrEltIf(
415  [=](const LegalityQuery &Query) { return Query.Types[1] == v2s16; },
416  1, s32)
417  .minScalarOrEltIf(
418  [=](const LegalityQuery &Query) { return Query.Types[1] == v2p0; }, 0,
419  s64)
420  .clampNumElements(0, v2s32, v4s32);
421 
422  // Extensions
423  auto ExtLegalFunc = [=](const LegalityQuery &Query) {
424  unsigned DstSize = Query.Types[0].getSizeInBits();
425 
426  if (DstSize == 128 && !Query.Types[0].isVector())
427  return false; // Extending to a scalar s128 needs narrowing.
428 
429  // Make sure that we have something that will fit in a register, and
430  // make sure it's a power of 2.
431  if (DstSize < 8 || DstSize > 128 || !isPowerOf2_32(DstSize))
432  return false;
433 
434  const LLT &SrcTy = Query.Types[1];
435 
436  // Make sure we fit in a register otherwise. Don't bother checking that
437  // the source type is below 128 bits. We shouldn't be allowing anything
438  // through which is wider than the destination in the first place.
439  unsigned SrcSize = SrcTy.getSizeInBits();
440  if (SrcSize < 8 || !isPowerOf2_32(SrcSize))
441  return false;
442 
443  return true;
444  };
445  getActionDefinitionsBuilder({G_ZEXT, G_SEXT, G_ANYEXT})
446  .legalIf(ExtLegalFunc)
447  .clampScalar(0, s64, s64); // Just for s128, others are handled above.
448 
451  [=](const LegalityQuery &Query) { return Query.Types[0].isVector(); },
452  0, s8)
453  .customIf([=](const LegalityQuery &Query) {
454  LLT DstTy = Query.Types[0];
455  LLT SrcTy = Query.Types[1];
456  return DstTy == v8s8 && SrcTy.getSizeInBits() > 128;
457  })
458  .alwaysLegal();
459 
460  getActionDefinitionsBuilder(G_SEXT_INREG)
461  .legalFor({s32, s64})
462  .legalFor(PackedVectorAllTypeList)
463  .lower();
464 
465  // FP conversions
466  getActionDefinitionsBuilder(G_FPTRUNC)
467  .legalFor(
468  {{s16, s32}, {s16, s64}, {s32, s64}, {v4s16, v4s32}, {v2s32, v2s64}})
469  .clampMaxNumElements(0, s32, 2);
471  .legalFor(
472  {{s32, s16}, {s64, s16}, {s64, s32}, {v4s32, v4s16}, {v2s64, v2s32}})
473  .clampMaxNumElements(0, s64, 2);
474 
475  // Conversions
476  getActionDefinitionsBuilder({G_FPTOSI, G_FPTOUI})
477  .legalForCartesianProduct({s32, s64, v2s64, v4s32, v2s32})
478  .widenScalarToNextPow2(0)
479  .clampScalar(0, s32, s64)
481  .clampScalar(1, s32, s64);
482 
483  getActionDefinitionsBuilder({G_SITOFP, G_UITOFP})
484  .legalForCartesianProduct({s32, s64, v2s64, v4s32, v2s32})
485  .clampScalar(1, s32, s64)
486  .minScalarSameAs(1, 0)
487  .clampScalar(0, s32, s64)
489 
490  // Control-flow
492  .legalFor({s32})
493  .clampScalar(0, s32, s32);
494  getActionDefinitionsBuilder(G_BRINDIRECT).legalFor({p0});
495 
497  .legalFor({{s32, s32}, {s64, s32}, {p0, s32}})
498  .widenScalarToNextPow2(0)
499  .clampScalar(0, s32, s64)
500  .clampScalar(1, s32, s32)
501  .minScalarEltSameAsIf(all(isVector(0), isVector(1)), 1, 0)
502  .lowerIf(isVector(0));
503 
504  // Pointer-handling
505  getActionDefinitionsBuilder(G_FRAME_INDEX).legalFor({p0});
506 
507  if (TM.getCodeModel() == CodeModel::Small)
508  getActionDefinitionsBuilder(G_GLOBAL_VALUE).custom();
509  else
510  getActionDefinitionsBuilder(G_GLOBAL_VALUE).legalFor({p0});
511 
512  getActionDefinitionsBuilder(G_PTRTOINT)
513  .legalFor({{s64, p0}, {v2s64, v2p0}})
514  .widenScalarToNextPow2(0, 64)
515  .clampScalar(0, s64, s64);
516 
517  getActionDefinitionsBuilder(G_INTTOPTR)
518  .unsupportedIf([&](const LegalityQuery &Query) {
519  return Query.Types[0].getSizeInBits() != Query.Types[1].getSizeInBits();
520  })
521  .legalFor({{p0, s64}, {v2p0, v2s64}});
522 
523  // Casts for 32 and 64-bit width type are just copies.
524  // Same for 128-bit width type, except they are on the FPR bank.
525  getActionDefinitionsBuilder(G_BITCAST)
526  // FIXME: This is wrong since G_BITCAST is not allowed to change the
527  // number of bits but it's what the previous code described and fixing
528  // it breaks tests.
529  .legalForCartesianProduct({s8, s16, s32, s64, s128, v16s8, v8s8, v4s8,
530  v8s16, v4s16, v2s16, v4s32, v2s32, v2s64,
531  v2p0});
532 
533  getActionDefinitionsBuilder(G_VASTART).legalFor({p0});
534 
535  // va_list must be a pointer, but most sized types are pretty easy to handle
536  // as the destination.
538  .customForCartesianProduct({s8, s16, s32, s64, p0}, {p0})
539  .clampScalar(0, s8, s64)
540  .widenScalarToNextPow2(0, /*Min*/ 8);
541 
542  getActionDefinitionsBuilder(G_ATOMIC_CMPXCHG_WITH_SUCCESS)
543  .lowerIf(
544  all(typeInSet(0, {s8, s16, s32, s64, s128}), typeIs(2, p0)));
545 
546  getActionDefinitionsBuilder(G_ATOMIC_CMPXCHG)
547  .customIf([](const LegalityQuery &Query) {
548  return Query.Types[0].getSizeInBits() == 128;
549  })
550  .clampScalar(0, s32, s64)
551  .legalIf(all(typeInSet(0, {s32, s64}), typeIs(1, p0)));
552 
554  {G_ATOMICRMW_XCHG, G_ATOMICRMW_ADD, G_ATOMICRMW_SUB, G_ATOMICRMW_AND,
555  G_ATOMICRMW_OR, G_ATOMICRMW_XOR, G_ATOMICRMW_MIN, G_ATOMICRMW_MAX,
556  G_ATOMICRMW_UMIN, G_ATOMICRMW_UMAX})
557  .clampScalar(0, s32, s64)
558  .legalIf(all(typeInSet(0, {s32, s64}), typeIs(1, p0)));
559 
560  getActionDefinitionsBuilder(G_BLOCK_ADDR).legalFor({p0});
561 
562  // Merge/Unmerge
563  for (unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) {
564  unsigned BigTyIdx = Op == G_MERGE_VALUES ? 0 : 1;
565  unsigned LitTyIdx = Op == G_MERGE_VALUES ? 1 : 0;
567  .widenScalarToNextPow2(LitTyIdx, 8)
568  .widenScalarToNextPow2(BigTyIdx, 32)
569  .clampScalar(LitTyIdx, s8, s64)
570  .clampScalar(BigTyIdx, s32, s128)
571  .legalIf([=](const LegalityQuery &Q) {
572  switch (Q.Types[BigTyIdx].getSizeInBits()) {
573  case 32:
574  case 64:
575  case 128:
576  break;
577  default:
578  return false;
579  }
580  switch (Q.Types[LitTyIdx].getSizeInBits()) {
581  case 8:
582  case 16:
583  case 32:
584  case 64:
585  return true;
586  default:
587  return false;
588  }
589  });
590  }
591 
592  getActionDefinitionsBuilder(G_EXTRACT_VECTOR_ELT)
593  .unsupportedIf([=](const LegalityQuery &Query) {
594  const LLT &EltTy = Query.Types[1].getElementType();
595  return Query.Types[0] != EltTy;
596  })
597  .minScalar(2, s64)
598  .legalIf([=](const LegalityQuery &Query) {
599  const LLT &VecTy = Query.Types[1];
600  return VecTy == v2s16 || VecTy == v4s16 || VecTy == v8s16 ||
601  VecTy == v4s32 || VecTy == v2s64 || VecTy == v2s32 ||
602  VecTy == v8s8 || VecTy == v16s8 || VecTy == v2s32 ||
603  VecTy == v2p0;
604  })
605  .minScalarOrEltIf(
606  [=](const LegalityQuery &Query) {
607  // We want to promote to <M x s1> to <M x s64> if that wouldn't
608  // cause the total vec size to be > 128b.
609  return Query.Types[1].getNumElements() <= 2;
610  },
611  0, s64)
612  .minScalarOrEltIf(
613  [=](const LegalityQuery &Query) {
614  return Query.Types[1].getNumElements() <= 4;
615  },
616  0, s32)
617  .minScalarOrEltIf(
618  [=](const LegalityQuery &Query) {
619  return Query.Types[1].getNumElements() <= 8;
620  },
621  0, s16)
622  .minScalarOrEltIf(
623  [=](const LegalityQuery &Query) {
624  return Query.Types[1].getNumElements() <= 16;
625  },
626  0, s8)
627  .minScalarOrElt(0, s8) // Worst case, we need at least s8.
628  .clampMaxNumElements(1, s64, 2)
629  .clampMaxNumElements(1, s32, 4)
630  .clampMaxNumElements(1, s16, 8)
631  .clampMaxNumElements(1, p0, 2);
632 
633  getActionDefinitionsBuilder(G_INSERT_VECTOR_ELT)
634  .legalIf(typeInSet(0, {v8s16, v2s32, v4s32, v2s64}));
635 
636  getActionDefinitionsBuilder(G_BUILD_VECTOR)
637  .legalFor({{v8s8, s8},
638  {v16s8, s8},
639  {v2s16, s16},
640  {v4s16, s16},
641  {v8s16, s16},
642  {v2s32, s32},
643  {v4s32, s32},
644  {v2p0, p0},
645  {v2s64, s64}})
646  .clampNumElements(0, v4s32, v4s32)
647  .clampNumElements(0, v2s64, v2s64)
648  .minScalarOrElt(0, s8)
649  .minScalarSameAs(1, 0);
650 
651  getActionDefinitionsBuilder(G_BUILD_VECTOR_TRUNC).lower();
652 
655  {s32, s64, v8s8, v16s8, v4s16, v8s16, v2s32, v4s32})
656  .scalarize(1);
657  getActionDefinitionsBuilder(G_CTLZ_ZERO_UNDEF).lower();
658 
659  // TODO: Custom lowering for v2s32, v4s32, v2s64.
660  getActionDefinitionsBuilder(G_BITREVERSE)
661  .legalFor({s32, s64, v8s8, v16s8})
662  .widenScalarToNextPow2(0, /*Min = */ 32)
663  .clampScalar(0, s32, s64);
664 
665  getActionDefinitionsBuilder(G_CTTZ_ZERO_UNDEF).lower();
666 
668  .lowerIf(isVector(0))
669  .clampScalar(0, s32, s64)
670  .scalarSameSizeAs(1, 0)
671  .customFor({s32, s64});
672 
673  getActionDefinitionsBuilder(G_SHUFFLE_VECTOR)
674  .legalIf([=](const LegalityQuery &Query) {
675  const LLT &DstTy = Query.Types[0];
676  const LLT &SrcTy = Query.Types[1];
677  // For now just support the TBL2 variant which needs the source vectors
678  // to be the same size as the dest.
679  if (DstTy != SrcTy)
680  return false;
681  return llvm::is_contained({v2s32, v4s32, v2s64, v2p0, v16s8, v8s16},
682  DstTy);
683  })
684  // G_SHUFFLE_VECTOR can have scalar sources (from 1 x s vectors), we
685  // just want those lowered into G_BUILD_VECTOR
686  .lowerIf([=](const LegalityQuery &Query) {
687  return !Query.Types[1].isVector();
688  })
690  .clampNumElements(0, v4s32, v4s32)
691  .clampNumElements(0, v2s64, v2s64);
692 
693  getActionDefinitionsBuilder(G_CONCAT_VECTORS)
694  .legalFor({{v4s32, v2s32}, {v8s16, v4s16}, {v16s8, v8s8}});
695 
696  getActionDefinitionsBuilder(G_JUMP_TABLE).legalFor({{p0}, {s64}});
697 
698  getActionDefinitionsBuilder(G_BRJT).legalIf([=](const LegalityQuery &Query) {
699  return Query.Types[0] == p0 && Query.Types[1] == s64;
700  });
701 
702  getActionDefinitionsBuilder(G_DYN_STACKALLOC).lower();
703 
704  if (ST.hasMOPS()) {
705  // G_BZERO is not supported. Currently it is only emitted by
706  // PreLegalizerCombiner for G_MEMSET with zero constant.
708 
710  .legalForCartesianProduct({p0}, {s64}, {s64})
711  .customForCartesianProduct({p0}, {s8}, {s64})
712  .immIdx(0); // Inform verifier imm idx 0 is handled.
713 
714  getActionDefinitionsBuilder({G_MEMCPY, G_MEMMOVE})
715  .legalForCartesianProduct({p0}, {p0}, {s64})
716  .immIdx(0); // Inform verifier imm idx 0 is handled.
717 
718  // G_MEMCPY_INLINE does not have a tailcall immediate
719  getActionDefinitionsBuilder(G_MEMCPY_INLINE)
720  .legalForCartesianProduct({p0}, {p0}, {s64});
721 
722  } else {
723  getActionDefinitionsBuilder({G_BZERO, G_MEMCPY, G_MEMMOVE, G_MEMSET})
724  .libcall();
725  }
726 
727  // FIXME: Legal types are only legal with NEON.
729  .lowerIf(isScalar(0))
730  .legalFor(PackedVectorAllTypeList);
731 
732  getActionDefinitionsBuilder(G_VECREDUCE_FADD)
733  // We only have FADDP to do reduction-like operations. Lower the rest.
734  .legalFor({{s32, v2s32}, {s64, v2s64}})
735  .clampMaxNumElements(1, s64, 2)
736  .clampMaxNumElements(1, s32, 2)
737  .lower();
738 
739  getActionDefinitionsBuilder(G_VECREDUCE_ADD)
740  .legalFor(
741  {{s8, v16s8}, {s16, v8s16}, {s32, v4s32}, {s32, v2s32}, {s64, v2s64}})
742  .clampMaxNumElements(1, s64, 2)
743  .clampMaxNumElements(1, s32, 4)
744  .lower();
745 
747  {G_VECREDUCE_OR, G_VECREDUCE_AND, G_VECREDUCE_XOR})
748  // Try to break down into smaller vectors as long as they're at least 64
749  // bits. This lets us use vector operations for some parts of the
750  // reduction.
751  .fewerElementsIf(
752  [=](const LegalityQuery &Q) {
753  LLT SrcTy = Q.Types[1];
754  if (SrcTy.isScalar())
755  return false;
756  if (!isPowerOf2_32(SrcTy.getNumElements()))
757  return false;
758  // We can usually perform 64b vector operations.
759  return SrcTy.getSizeInBits() > 64;
760  },
761  [=](const LegalityQuery &Q) {
762  LLT SrcTy = Q.Types[1];
763  return std::make_pair(1, SrcTy.divide(2));
764  })
765  .scalarize(1)
766  .lower();
767 
768  getActionDefinitionsBuilder({G_UADDSAT, G_USUBSAT})
769  .lowerIf([=](const LegalityQuery &Q) { return Q.Types[0].isScalar(); });
770 
771  getActionDefinitionsBuilder({G_FSHL, G_FSHR}).lower();
772 
774  .legalFor({{s32, s64}, {s64, s64}})
775  .customIf([=](const LegalityQuery &Q) {
776  return Q.Types[0].isScalar() && Q.Types[1].getScalarSizeInBits() < 64;
777  })
778  .lower();
780 
781  getActionDefinitionsBuilder({G_SBFX, G_UBFX})
782  .customFor({{s32, s32}, {s64, s64}});
783 
784  auto always = [=](const LegalityQuery &Q) { return true; };
786  .legalFor({{v8s8, v8s8}, {v16s8, v16s8}})
787  .clampScalar(0, s32, s128)
791  .customFor({{s32, s32},
792  {s64, s64},
793  {s128, s128},
794  {v2s64, v2s64},
795  {v2s32, v2s32},
796  {v4s32, v4s32},
797  {v4s16, v4s16},
798  {v8s16, v8s16}});
799 
800  // TODO: Vector types.
801  getActionDefinitionsBuilder({G_SADDSAT, G_SSUBSAT}).lowerIf(isScalar(0));
802 
803  // TODO: Vector types.
804  getActionDefinitionsBuilder({G_FMAXNUM, G_FMINNUM})
805  .legalFor({MinFPScalar, s32, s64})
806  .libcallFor({s128})
807  .minScalar(0, MinFPScalar);
808 
809  getActionDefinitionsBuilder({G_FMAXIMUM, G_FMINIMUM})
810  .legalFor({MinFPScalar, s32, s64, v2s32, v4s32, v2s64})
811  .legalIf([=](const LegalityQuery &Query) {
812  const auto &Ty = Query.Types[0];
813  return (Ty == v8s16 || Ty == v4s16) && HasFP16;
814  })
815  .minScalar(0, MinFPScalar)
816  .clampNumElements(0, v4s16, v8s16)
817  .clampNumElements(0, v2s32, v4s32)
818  .clampNumElements(0, v2s64, v2s64);
819 
820  // TODO: Libcall support for s128.
821  // TODO: s16 should be legal with full FP16 support.
822  getActionDefinitionsBuilder({G_LROUND, G_LLROUND})
823  .legalFor({{s64, s32}, {s64, s64}});
824 
825  // TODO: Custom legalization for vector types.
826  // TODO: Custom legalization for mismatched types.
827  // TODO: s16 support.
828  getActionDefinitionsBuilder(G_FCOPYSIGN).customFor({{s32, s32}, {s64, s64}});
829 
831 
833  verify(*ST.getInstrInfo());
834 }
835 
837  MachineInstr &MI) const {
838  MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
839  MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
840  GISelChangeObserver &Observer = Helper.Observer;
841  switch (MI.getOpcode()) {
842  default:
843  // No idea what to do.
844  return false;
845  case TargetOpcode::G_VAARG:
846  return legalizeVaArg(MI, MRI, MIRBuilder);
847  case TargetOpcode::G_LOAD:
848  case TargetOpcode::G_STORE:
849  return legalizeLoadStore(MI, MRI, MIRBuilder, Observer);
850  case TargetOpcode::G_SHL:
851  case TargetOpcode::G_ASHR:
852  case TargetOpcode::G_LSHR:
853  return legalizeShlAshrLshr(MI, MRI, MIRBuilder, Observer);
854  case TargetOpcode::G_GLOBAL_VALUE:
855  return legalizeSmallCMGlobalValue(MI, MRI, MIRBuilder, Observer);
856  case TargetOpcode::G_TRUNC:
857  return legalizeVectorTrunc(MI, Helper);
858  case TargetOpcode::G_SBFX:
859  case TargetOpcode::G_UBFX:
860  return legalizeBitfieldExtract(MI, MRI, Helper);
861  case TargetOpcode::G_ROTR:
862  return legalizeRotate(MI, MRI, Helper);
863  case TargetOpcode::G_CTPOP:
864  return legalizeCTPOP(MI, MRI, Helper);
865  case TargetOpcode::G_ATOMIC_CMPXCHG:
866  return legalizeAtomicCmpxchg128(MI, MRI, Helper);
867  case TargetOpcode::G_CTTZ:
868  return legalizeCTTZ(MI, Helper);
869  case TargetOpcode::G_BZERO:
870  case TargetOpcode::G_MEMCPY:
871  case TargetOpcode::G_MEMMOVE:
872  case TargetOpcode::G_MEMSET:
873  return legalizeMemOps(MI, Helper);
874  case TargetOpcode::G_FCOPYSIGN:
875  return legalizeFCopySign(MI, Helper);
876  }
877 
878  llvm_unreachable("expected switch to return");
879 }
880 
881 bool AArch64LegalizerInfo::legalizeRotate(MachineInstr &MI,
883  LegalizerHelper &Helper) const {
884  // To allow for imported patterns to match, we ensure that the rotate amount
885  // is 64b with an extension.
886  Register AmtReg = MI.getOperand(2).getReg();
887  LLT AmtTy = MRI.getType(AmtReg);
888  (void)AmtTy;
889  assert(AmtTy.isScalar() && "Expected a scalar rotate");
890  assert(AmtTy.getSizeInBits() < 64 && "Expected this rotate to be legal");
891  auto NewAmt = Helper.MIRBuilder.buildZExt(LLT::scalar(64), AmtReg);
892  Helper.Observer.changingInstr(MI);
893  MI.getOperand(2).setReg(NewAmt.getReg(0));
894  Helper.Observer.changedInstr(MI);
895  return true;
896 }
897 
899  MachineIRBuilder &MIRBuilder, LLT Ty, int NumParts,
900  SmallVectorImpl<Register> &VRegs) {
901  for (int I = 0; I < NumParts; ++I)
902  VRegs.push_back(MRI.createGenericVirtualRegister(Ty));
903  MIRBuilder.buildUnmerge(VRegs, Reg);
904 }
905 
906 bool AArch64LegalizerInfo::legalizeVectorTrunc(
907  MachineInstr &MI, LegalizerHelper &Helper) const {
908  MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
909  MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
910  // Similar to how operand splitting is done in SelectiondDAG, we can handle
911  // %res(v8s8) = G_TRUNC %in(v8s32) by generating:
912  // %inlo(<4x s32>), %inhi(<4 x s32>) = G_UNMERGE %in(<8 x s32>)
913  // %lo16(<4 x s16>) = G_TRUNC %inlo
914  // %hi16(<4 x s16>) = G_TRUNC %inhi
915  // %in16(<8 x s16>) = G_CONCAT_VECTORS %lo16, %hi16
916  // %res(<8 x s8>) = G_TRUNC %in16
917 
918  Register DstReg = MI.getOperand(0).getReg();
919  Register SrcReg = MI.getOperand(1).getReg();
920  LLT DstTy = MRI.getType(DstReg);
921  LLT SrcTy = MRI.getType(SrcReg);
923  isPowerOf2_32(SrcTy.getSizeInBits()));
924 
925  // Split input type.
926  LLT SplitSrcTy =
928  // First, split the source into two smaller vectors.
929  SmallVector<Register, 2> SplitSrcs;
930  extractParts(SrcReg, MRI, MIRBuilder, SplitSrcTy, 2, SplitSrcs);
931 
932  // Truncate the splits into intermediate narrower elements.
933  LLT InterTy = SplitSrcTy.changeElementSize(DstTy.getScalarSizeInBits() * 2);
934  for (unsigned I = 0; I < SplitSrcs.size(); ++I)
935  SplitSrcs[I] = MIRBuilder.buildTrunc(InterTy, SplitSrcs[I]).getReg(0);
936 
937  auto Concat = MIRBuilder.buildConcatVectors(
938  DstTy.changeElementSize(DstTy.getScalarSizeInBits() * 2), SplitSrcs);
939 
940  Helper.Observer.changingInstr(MI);
941  MI.getOperand(1).setReg(Concat.getReg(0));
942  Helper.Observer.changedInstr(MI);
943  return true;
944 }
945 
946 bool AArch64LegalizerInfo::legalizeSmallCMGlobalValue(
948  GISelChangeObserver &Observer) const {
949  assert(MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE);
950  // We do this custom legalization to convert G_GLOBAL_VALUE into target ADRP +
951  // G_ADD_LOW instructions.
952  // By splitting this here, we can optimize accesses in the small code model by
953  // folding in the G_ADD_LOW into the load/store offset.
954  auto &GlobalOp = MI.getOperand(1);
955  const auto* GV = GlobalOp.getGlobal();
956  if (GV->isThreadLocal())
957  return true; // Don't want to modify TLS vars.
958 
959  auto &TM = ST->getTargetLowering()->getTargetMachine();
960  unsigned OpFlags = ST->ClassifyGlobalReference(GV, TM);
961 
962  if (OpFlags & AArch64II::MO_GOT)
963  return true;
964 
965  auto Offset = GlobalOp.getOffset();
966  Register DstReg = MI.getOperand(0).getReg();
967  auto ADRP = MIRBuilder.buildInstr(AArch64::ADRP, {LLT::pointer(0, 64)}, {})
968  .addGlobalAddress(GV, Offset, OpFlags | AArch64II::MO_PAGE);
969  // Set the regclass on the dest reg too.
970  MRI.setRegClass(ADRP.getReg(0), &AArch64::GPR64RegClass);
971 
972  // MO_TAGGED on the page indicates a tagged address. Set the tag now. We do so
973  // by creating a MOVK that sets bits 48-63 of the register to (global address
974  // + 0x100000000 - PC) >> 48. The additional 0x100000000 offset here is to
975  // prevent an incorrect tag being generated during relocation when the the
976  // global appears before the code section. Without the offset, a global at
977  // `0x0f00'0000'0000'1000` (i.e. at `0x1000` with tag `0xf`) that's referenced
978  // by code at `0x2000` would result in `0x0f00'0000'0000'1000 - 0x2000 =
979  // 0x0eff'ffff'ffff'f000`, meaning the tag would be incorrectly set to `0xe`
980  // instead of `0xf`.
981  // This assumes that we're in the small code model so we can assume a binary
982  // size of <= 4GB, which makes the untagged PC relative offset positive. The
983  // binary must also be loaded into address range [0, 2^48). Both of these
984  // properties need to be ensured at runtime when using tagged addresses.
985  if (OpFlags & AArch64II::MO_TAGGED) {
986  assert(!Offset &&
987  "Should not have folded in an offset for a tagged global!");
988  ADRP = MIRBuilder.buildInstr(AArch64::MOVKXi, {LLT::pointer(0, 64)}, {ADRP})
989  .addGlobalAddress(GV, 0x100000000,
991  .addImm(48);
992  MRI.setRegClass(ADRP.getReg(0), &AArch64::GPR64RegClass);
993  }
994 
995  MIRBuilder.buildInstr(AArch64::G_ADD_LOW, {DstReg}, {ADRP})
996  .addGlobalAddress(GV, Offset,
998  MI.eraseFromParent();
999  return true;
1000 }
1001 
1003  MachineInstr &MI) const {
1004  switch (MI.getIntrinsicID()) {
1005  case Intrinsic::vacopy: {
1006  unsigned PtrSize = ST->isTargetILP32() ? 4 : 8;
1007  unsigned VaListSize =
1008  (ST->isTargetDarwin() || ST->isTargetWindows())
1009  ? PtrSize
1010  : ST->isTargetILP32() ? 20 : 32;
1011 
1012  MachineFunction &MF = *MI.getMF();
1013  auto Val = MF.getRegInfo().createGenericVirtualRegister(
1014  LLT::scalar(VaListSize * 8));
1015  MachineIRBuilder MIB(MI);
1016  MIB.buildLoad(Val, MI.getOperand(2),
1019  VaListSize, Align(PtrSize)));
1020  MIB.buildStore(Val, MI.getOperand(1),
1023  VaListSize, Align(PtrSize)));
1024  MI.eraseFromParent();
1025  return true;
1026  }
1027  case Intrinsic::get_dynamic_area_offset: {
1028  MachineIRBuilder &MIB = Helper.MIRBuilder;
1029  MIB.buildConstant(MI.getOperand(0).getReg(), 0);
1030  MI.eraseFromParent();
1031  return true;
1032  }
1033  case Intrinsic::aarch64_mops_memset_tag: {
1034  assert(MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS);
1035  // Zext the value to 64 bit
1036  MachineIRBuilder MIB(MI);
1037  auto &Value = MI.getOperand(3);
1038  Register ZExtValueReg = MIB.buildAnyExt(LLT::scalar(64), Value).getReg(0);
1039  Value.setReg(ZExtValueReg);
1040  return true;
1041  }
1042  case Intrinsic::prefetch: {
1043  MachineIRBuilder MIB(MI);
1044  auto &AddrVal = MI.getOperand(1);
1045 
1046  int64_t IsWrite = MI.getOperand(2).getImm();
1047  int64_t Locality = MI.getOperand(3).getImm();
1048  int64_t IsData = MI.getOperand(4).getImm();
1049 
1050  bool IsStream = Locality == 0;
1051  if (Locality != 0) {
1052  assert(Locality <= 3 && "Prefetch locality out-of-range");
1053  // The locality degree is the opposite of the cache speed.
1054  // Put the number the other way around.
1055  // The encoding starts at 0 for level 1
1056  Locality = 3 - Locality;
1057  }
1058 
1059  unsigned PrfOp =
1060  (IsWrite << 4) | (!IsData << 3) | (Locality << 1) | IsStream;
1061 
1062  MIB.buildInstr(AArch64::G_PREFETCH).addImm(PrfOp).add(AddrVal);
1063  MI.eraseFromParent();
1064  return true;
1065  }
1066  }
1067 
1068  return true;
1069 }
1070 
1071 bool AArch64LegalizerInfo::legalizeShlAshrLshr(
1073  GISelChangeObserver &Observer) const {
1074  assert(MI.getOpcode() == TargetOpcode::G_ASHR ||
1075  MI.getOpcode() == TargetOpcode::G_LSHR ||
1076  MI.getOpcode() == TargetOpcode::G_SHL);
1077  // If the shift amount is a G_CONSTANT, promote it to a 64 bit type so the
1078  // imported patterns can select it later. Either way, it will be legal.
1079  Register AmtReg = MI.getOperand(2).getReg();
1080  auto VRegAndVal = getIConstantVRegValWithLookThrough(AmtReg, MRI);
1081  if (!VRegAndVal)
1082  return true;
1083  // Check the shift amount is in range for an immediate form.
1084  int64_t Amount = VRegAndVal->Value.getSExtValue();
1085  if (Amount > 31)
1086  return true; // This will have to remain a register variant.
1087  auto ExtCst = MIRBuilder.buildConstant(LLT::scalar(64), Amount);
1088  Observer.changingInstr(MI);
1089  MI.getOperand(2).setReg(ExtCst.getReg(0));
1090  Observer.changedInstr(MI);
1091  return true;
1092 }
1093 
1094 static void matchLDPSTPAddrMode(Register Root, Register &Base, int &Offset,
1096  Base = Root;
1097  Offset = 0;
1098 
1099  Register NewBase;
1100  int64_t NewOffset;
1101  if (mi_match(Root, MRI, m_GPtrAdd(m_Reg(NewBase), m_ICst(NewOffset))) &&
1102  isShiftedInt<7, 3>(NewOffset)) {
1103  Base = NewBase;
1104  Offset = NewOffset;
1105  }
1106 }
1107 
1108 // FIXME: This should be removed and replaced with the generic bitcast legalize
1109 // action.
1110 bool AArch64LegalizerInfo::legalizeLoadStore(
1112  GISelChangeObserver &Observer) const {
1113  assert(MI.getOpcode() == TargetOpcode::G_STORE ||
1114  MI.getOpcode() == TargetOpcode::G_LOAD);
1115  // Here we just try to handle vector loads/stores where our value type might
1116  // have pointer elements, which the SelectionDAG importer can't handle. To
1117  // allow the existing patterns for s64 to fire for p0, we just try to bitcast
1118  // the value to use s64 types.
1119 
1120  // Custom legalization requires the instruction, if not deleted, must be fully
1121  // legalized. In order to allow further legalization of the inst, we create
1122  // a new instruction and erase the existing one.
1123 
1124  Register ValReg = MI.getOperand(0).getReg();
1125  const LLT ValTy = MRI.getType(ValReg);
1126 
1127  if (ValTy == LLT::scalar(128)) {
1128  assert((*MI.memoperands_begin())->getSuccessOrdering() ==
1130  (*MI.memoperands_begin())->getSuccessOrdering() ==
1132  assert(ST->hasLSE2() && "ldp/stp not single copy atomic without +lse2");
1133  LLT s64 = LLT::scalar(64);
1134  MachineInstrBuilder NewI;
1135  if (MI.getOpcode() == TargetOpcode::G_LOAD) {
1136  NewI = MIRBuilder.buildInstr(AArch64::LDPXi, {s64, s64}, {});
1137  MIRBuilder.buildMerge(ValReg, {NewI->getOperand(0), NewI->getOperand(1)});
1138  } else {
1139  auto Split = MIRBuilder.buildUnmerge(s64, MI.getOperand(0));
1140  NewI = MIRBuilder.buildInstr(
1141  AArch64::STPXi, {}, {Split->getOperand(0), Split->getOperand(1)});
1142  }
1143  Register Base;
1144  int Offset;
1145  matchLDPSTPAddrMode(MI.getOperand(1).getReg(), Base, Offset, MRI);
1146  NewI.addUse(Base);
1147  NewI.addImm(Offset / 8);
1148 
1149  NewI.cloneMemRefs(MI);
1152  *ST->getRegBankInfo());
1153  MI.eraseFromParent();
1154  return true;
1155  }
1156 
1157  if (!ValTy.isVector() || !ValTy.getElementType().isPointer() ||
1158  ValTy.getElementType().getAddressSpace() != 0) {
1159  LLVM_DEBUG(dbgs() << "Tried to do custom legalization on wrong load/store");
1160  return false;
1161  }
1162 
1163  unsigned PtrSize = ValTy.getElementType().getSizeInBits();
1164  const LLT NewTy = LLT::vector(ValTy.getElementCount(), PtrSize);
1165  auto &MMO = **MI.memoperands_begin();
1166  MMO.setType(NewTy);
1167 
1168  if (MI.getOpcode() == TargetOpcode::G_STORE) {
1169  auto Bitcast = MIRBuilder.buildBitcast(NewTy, ValReg);
1170  MIRBuilder.buildStore(Bitcast.getReg(0), MI.getOperand(1), MMO);
1171  } else {
1172  auto NewLoad = MIRBuilder.buildLoad(NewTy, MI.getOperand(1), MMO);
1173  MIRBuilder.buildBitcast(ValReg, NewLoad);
1174  }
1175  MI.eraseFromParent();
1176  return true;
1177 }
1178 
1179 bool AArch64LegalizerInfo::legalizeVaArg(MachineInstr &MI,
1181  MachineIRBuilder &MIRBuilder) const {
1182  MachineFunction &MF = MIRBuilder.getMF();
1183  Align Alignment(MI.getOperand(2).getImm());
1184  Register Dst = MI.getOperand(0).getReg();
1185  Register ListPtr = MI.getOperand(1).getReg();
1186 
1187  LLT PtrTy = MRI.getType(ListPtr);
1188  LLT IntPtrTy = LLT::scalar(PtrTy.getSizeInBits());
1189 
1190  const unsigned PtrSize = PtrTy.getSizeInBits() / 8;
1191  const Align PtrAlign = Align(PtrSize);
1192  auto List = MIRBuilder.buildLoad(
1193  PtrTy, ListPtr,
1195  PtrTy, PtrAlign));
1196 
1197  MachineInstrBuilder DstPtr;
1198  if (Alignment > PtrAlign) {
1199  // Realign the list to the actual required alignment.
1200  auto AlignMinus1 =
1201  MIRBuilder.buildConstant(IntPtrTy, Alignment.value() - 1);
1202  auto ListTmp = MIRBuilder.buildPtrAdd(PtrTy, List, AlignMinus1.getReg(0));
1203  DstPtr = MIRBuilder.buildMaskLowPtrBits(PtrTy, ListTmp, Log2(Alignment));
1204  } else
1205  DstPtr = List;
1206 
1207  LLT ValTy = MRI.getType(Dst);
1208  uint64_t ValSize = ValTy.getSizeInBits() / 8;
1209  MIRBuilder.buildLoad(
1210  Dst, DstPtr,
1212  ValTy, std::max(Alignment, PtrAlign)));
1213 
1214  auto Size = MIRBuilder.buildConstant(IntPtrTy, alignTo(ValSize, PtrAlign));
1215 
1216  auto NewList = MIRBuilder.buildPtrAdd(PtrTy, DstPtr, Size.getReg(0));
1217 
1218  MIRBuilder.buildStore(NewList, ListPtr,
1221  PtrTy, PtrAlign));
1222 
1223  MI.eraseFromParent();
1224  return true;
1225 }
1226 
1227 bool AArch64LegalizerInfo::legalizeBitfieldExtract(
1229  // Only legal if we can select immediate forms.
1230  // TODO: Lower this otherwise.
1231  return getIConstantVRegValWithLookThrough(MI.getOperand(2).getReg(), MRI) &&
1232  getIConstantVRegValWithLookThrough(MI.getOperand(3).getReg(), MRI);
1233 }
1234 
1235 bool AArch64LegalizerInfo::legalizeCTPOP(MachineInstr &MI,
1237  LegalizerHelper &Helper) const {
1238  // While there is no integer popcount instruction, it can
1239  // be more efficiently lowered to the following sequence that uses
1240  // AdvSIMD registers/instructions as long as the copies to/from
1241  // the AdvSIMD registers are cheap.
1242  // FMOV D0, X0 // copy 64-bit int to vector, high bits zero'd
1243  // CNT V0.8B, V0.8B // 8xbyte pop-counts
1244  // ADDV B0, V0.8B // sum 8xbyte pop-counts
1245  // UMOV X0, V0.B[0] // copy byte result back to integer reg
1246  //
1247  // For 128 bit vector popcounts, we lower to the following sequence:
1248  // cnt.16b v0, v0 // v8s16, v4s32, v2s64
1249  // uaddlp.8h v0, v0 // v8s16, v4s32, v2s64
1250  // uaddlp.4s v0, v0 // v4s32, v2s64
1251  // uaddlp.2d v0, v0 // v2s64
1252  //
1253  // For 64 bit vector popcounts, we lower to the following sequence:
1254  // cnt.8b v0, v0 // v4s16, v2s32
1255  // uaddlp.4h v0, v0 // v4s16, v2s32
1256  // uaddlp.2s v0, v0 // v2s32
1257 
1258  MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
1259  Register Dst = MI.getOperand(0).getReg();
1260  Register Val = MI.getOperand(1).getReg();
1261  LLT Ty = MRI.getType(Val);
1262 
1263  assert(Ty == MRI.getType(Dst) &&
1264  "Expected src and dst to have the same type!");
1265  unsigned Size = Ty.getSizeInBits();
1266 
1267  if (!ST->hasNEON() ||
1268  MI.getMF()->getFunction().hasFnAttribute(Attribute::NoImplicitFloat)) {
1269  // Use generic lowering when custom lowering is not possible.
1270  return Ty.isScalar() && (Size == 32 || Size == 64) &&
1271  Helper.lowerBitCount(MI) ==
1272  LegalizerHelper::LegalizeResult::Legalized;
1273  }
1274 
1275  // Pre-conditioning: widen Val up to the nearest vector type.
1276  // s32,s64,v4s16,v2s32 -> v8i8
1277  // v8s16,v4s32,v2s64 -> v16i8
1278  LLT VTy = Size == 128 ? LLT::fixed_vector(16, 8) : LLT::fixed_vector(8, 8);
1279  if (Ty.isScalar()) {
1280  assert((Size == 32 || Size == 64 || Size == 128) && "Expected only 32, 64, or 128 bit scalars!");
1281  if (Size == 32) {
1282  Val = MIRBuilder.buildZExt(LLT::scalar(64), Val).getReg(0);
1283  }
1284  }
1285  Val = MIRBuilder.buildBitcast(VTy, Val).getReg(0);
1286 
1287  // Count bits in each byte-sized lane.
1288  auto CTPOP = MIRBuilder.buildCTPOP(VTy, Val);
1289 
1290  // Sum across lanes.
1291  Register HSum = CTPOP.getReg(0);
1292  unsigned Opc;
1293  SmallVector<LLT> HAddTys;
1294  if (Ty.isScalar()) {
1295  Opc = Intrinsic::aarch64_neon_uaddlv;
1296  HAddTys.push_back(LLT::scalar(32));
1297  } else if (Ty == LLT::fixed_vector(8, 16)) {
1298  Opc = Intrinsic::aarch64_neon_uaddlp;
1299  HAddTys.push_back(LLT::fixed_vector(8, 16));
1300  } else if (Ty == LLT::fixed_vector(4, 32)) {
1301  Opc = Intrinsic::aarch64_neon_uaddlp;
1302  HAddTys.push_back(LLT::fixed_vector(8, 16));
1303  HAddTys.push_back(LLT::fixed_vector(4, 32));
1304  } else if (Ty == LLT::fixed_vector(2, 64)) {
1305  Opc = Intrinsic::aarch64_neon_uaddlp;
1306  HAddTys.push_back(LLT::fixed_vector(8, 16));
1307  HAddTys.push_back(LLT::fixed_vector(4, 32));
1308  HAddTys.push_back(LLT::fixed_vector(2, 64));
1309  } else if (Ty == LLT::fixed_vector(4, 16)) {
1310  Opc = Intrinsic::aarch64_neon_uaddlp;
1311  HAddTys.push_back(LLT::fixed_vector(4, 16));
1312  } else if (Ty == LLT::fixed_vector(2, 32)) {
1313  Opc = Intrinsic::aarch64_neon_uaddlp;
1314  HAddTys.push_back(LLT::fixed_vector(4, 16));
1315  HAddTys.push_back(LLT::fixed_vector(2, 32));
1316  } else
1317  llvm_unreachable("unexpected vector shape");
1318  MachineInstrBuilder UADD;
1319  for (LLT HTy : HAddTys) {
1320  UADD = MIRBuilder.buildIntrinsic(Opc, {HTy}, /*HasSideEffects =*/false)
1321  .addUse(HSum);
1322  HSum = UADD.getReg(0);
1323  }
1324 
1325  // Post-conditioning.
1326  if (Ty.isScalar() && (Size == 64 || Size == 128))
1327  MIRBuilder.buildZExt(Dst, UADD);
1328  else
1329  UADD->getOperand(0).setReg(Dst);
1330  MI.eraseFromParent();
1331  return true;
1332 }
1333 
1334 bool AArch64LegalizerInfo::legalizeAtomicCmpxchg128(
1336  MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
1337  LLT s64 = LLT::scalar(64);
1338  auto Addr = MI.getOperand(1).getReg();
1339  auto DesiredI = MIRBuilder.buildUnmerge({s64, s64}, MI.getOperand(2));
1340  auto NewI = MIRBuilder.buildUnmerge({s64, s64}, MI.getOperand(3));
1341  auto DstLo = MRI.createGenericVirtualRegister(s64);
1342  auto DstHi = MRI.createGenericVirtualRegister(s64);
1343 
1344  MachineInstrBuilder CAS;
1345  if (ST->hasLSE()) {
1346  // We have 128-bit CASP instructions taking XSeqPair registers, which are
1347  // s128. We need the merge/unmerge to bracket the expansion and pair up with
1348  // the rest of the MIR so we must reassemble the extracted registers into a
1349  // 128-bit known-regclass one with code like this:
1350  //
1351  // %in1 = REG_SEQUENCE Lo, Hi ; One for each input
1352  // %out = CASP %in1, ...
1353  // %OldLo = G_EXTRACT %out, 0
1354  // %OldHi = G_EXTRACT %out, 64
1355  auto Ordering = (*MI.memoperands_begin())->getMergedOrdering();
1356  unsigned Opcode;
1357  switch (Ordering) {
1359  Opcode = AArch64::CASPAX;
1360  break;
1362  Opcode = AArch64::CASPLX;
1363  break;
1366  Opcode = AArch64::CASPALX;
1367  break;
1368  default:
1369  Opcode = AArch64::CASPX;
1370  break;
1371  }
1372 
1373  LLT s128 = LLT::scalar(128);
1374  auto CASDst = MRI.createGenericVirtualRegister(s128);
1375  auto CASDesired = MRI.createGenericVirtualRegister(s128);
1376  auto CASNew = MRI.createGenericVirtualRegister(s128);
1377  MIRBuilder.buildInstr(TargetOpcode::REG_SEQUENCE, {CASDesired}, {})
1378  .addUse(DesiredI->getOperand(0).getReg())
1379  .addImm(AArch64::sube64)
1380  .addUse(DesiredI->getOperand(1).getReg())
1381  .addImm(AArch64::subo64);
1382  MIRBuilder.buildInstr(TargetOpcode::REG_SEQUENCE, {CASNew}, {})
1383  .addUse(NewI->getOperand(0).getReg())
1384  .addImm(AArch64::sube64)
1385  .addUse(NewI->getOperand(1).getReg())
1386  .addImm(AArch64::subo64);
1387 
1388  CAS = MIRBuilder.buildInstr(Opcode, {CASDst}, {CASDesired, CASNew, Addr});
1389 
1390  MIRBuilder.buildExtract({DstLo}, {CASDst}, 0);
1391  MIRBuilder.buildExtract({DstHi}, {CASDst}, 64);
1392  } else {
1393  // The -O0 CMP_SWAP_128 is friendlier to generate code for because LDXP/STXP
1394  // can take arbitrary registers so it just has the normal GPR64 operands the
1395  // rest of AArch64 is expecting.
1396  auto Ordering = (*MI.memoperands_begin())->getMergedOrdering();
1397  unsigned Opcode;
1398  switch (Ordering) {
1400  Opcode = AArch64::CMP_SWAP_128_ACQUIRE;
1401  break;
1403  Opcode = AArch64::CMP_SWAP_128_RELEASE;
1404  break;
1407  Opcode = AArch64::CMP_SWAP_128;
1408  break;
1409  default:
1410  Opcode = AArch64::CMP_SWAP_128_MONOTONIC;
1411  break;
1412  }
1413 
1414  auto Scratch = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
1415  CAS = MIRBuilder.buildInstr(Opcode, {DstLo, DstHi, Scratch},
1416  {Addr, DesiredI->getOperand(0),
1417  DesiredI->getOperand(1), NewI->getOperand(0),
1418  NewI->getOperand(1)});
1419  }
1420 
1421  CAS.cloneMemRefs(MI);
1424  *ST->getRegBankInfo());
1425 
1426  MIRBuilder.buildMerge(MI.getOperand(0), {DstLo, DstHi});
1427  MI.eraseFromParent();
1428  return true;
1429 }
1430 
1431 bool AArch64LegalizerInfo::legalizeCTTZ(MachineInstr &MI,
1432  LegalizerHelper &Helper) const {
1433  MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
1434  MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
1435  LLT Ty = MRI.getType(MI.getOperand(1).getReg());
1436  auto BitReverse = MIRBuilder.buildBitReverse(Ty, MI.getOperand(1));
1437  MIRBuilder.buildCTLZ(MI.getOperand(0).getReg(), BitReverse);
1438  MI.eraseFromParent();
1439  return true;
1440 }
1441 
1442 bool AArch64LegalizerInfo::legalizeMemOps(MachineInstr &MI,
1443  LegalizerHelper &Helper) const {
1444  MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
1445 
1446  // Tagged version MOPSMemorySetTagged is legalised in legalizeIntrinsic
1447  if (MI.getOpcode() == TargetOpcode::G_MEMSET) {
1448  // Zext the value operand to 64 bit
1449  auto &Value = MI.getOperand(1);
1450  Register ZExtValueReg =
1451  MIRBuilder.buildAnyExt(LLT::scalar(64), Value).getReg(0);
1452  Value.setReg(ZExtValueReg);
1453  return true;
1454  }
1455 
1456  return false;
1457 }
1458 
1459 bool AArch64LegalizerInfo::legalizeFCopySign(MachineInstr &MI,
1460  LegalizerHelper &Helper) const {
1461  MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
1462  MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
1463  Register Dst = MI.getOperand(0).getReg();
1464  LLT DstTy = MRI.getType(Dst);
1465  assert(DstTy.isScalar() && "Only expected scalars right now!");
1466  const unsigned DstSize = DstTy.getSizeInBits();
1467  assert((DstSize == 32 || DstSize == 64) && "Unexpected dst type!");
1468  assert(MRI.getType(MI.getOperand(2).getReg()) == DstTy &&
1469  "Expected homogeneous types!");
1470 
1471  // We want to materialize a mask with the high bit set.
1472  uint64_t EltMask;
1473  LLT VecTy;
1474 
1475  // TODO: s16 support.
1476  switch (DstSize) {
1477  default:
1478  llvm_unreachable("Unexpected type for G_FCOPYSIGN!");
1479  case 64: {
1480  // AdvSIMD immediate moves cannot materialize out mask in a single
1481  // instruction for 64-bit elements. Instead, materialize zero and then
1482  // negate it.
1483  EltMask = 0;
1484  VecTy = LLT::fixed_vector(2, DstTy);
1485  break;
1486  }
1487  case 32:
1488  EltMask = 0x80000000ULL;
1489  VecTy = LLT::fixed_vector(4, DstTy);
1490  break;
1491  }
1492 
1493  // Widen In1 and In2 to 128 bits. We want these to eventually become
1494  // INSERT_SUBREGs.
1495  auto Undef = MIRBuilder.buildUndef(VecTy);
1496  auto Zero = MIRBuilder.buildConstant(DstTy, 0);
1497  auto Ins1 = MIRBuilder.buildInsertVectorElement(
1498  VecTy, Undef, MI.getOperand(1).getReg(), Zero);
1499  auto Ins2 = MIRBuilder.buildInsertVectorElement(
1500  VecTy, Undef, MI.getOperand(2).getReg(), Zero);
1501 
1502  // Construct the mask.
1503  auto Mask = MIRBuilder.buildConstant(VecTy, EltMask);
1504  if (DstSize == 64)
1505  Mask = MIRBuilder.buildFNeg(VecTy, Mask);
1506 
1507  auto Sel = MIRBuilder.buildInstr(AArch64::G_BIT, {VecTy}, {Ins1, Ins2, Mask});
1508 
1509  // Build an unmerge whose 0th elt is the original G_FCOPYSIGN destination. We
1510  // want this to eventually become an EXTRACT_SUBREG.
1511  SmallVector<Register, 2> DstRegs(1, Dst);
1512  for (unsigned I = 1, E = VecTy.getNumElements(); I < E; ++I)
1513  DstRegs.push_back(MRI.createGenericVirtualRegister(DstTy));
1514  MIRBuilder.buildUnmerge(DstRegs, Sel);
1515  MI.eraseFromParent();
1516  return true;
1517 }
AArch64LegalizerInfo.h
llvm::Check::Size
@ Size
Definition: FileCheck.h:77
MIPatternMatch.h
llvm::alignTo
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition: Alignment.h:156
llvm::LegalizeRuleSet::unsupported
LegalizeRuleSet & unsupported()
The instruction is unsupported.
Definition: LegalizerInfo.h:801
llvm::AArch64II::MO_G3
@ MO_G3
MO_G3 - A symbol operand with this flag (granule 3) represents the high 16-bits of a 64-bit address,...
Definition: AArch64BaseInfo.h:743
ValueTypes.h
llvm::MachineIRBuilder::buildInsertVectorElement
MachineInstrBuilder buildInsertVectorElement(const DstOp &Res, const SrcOp &Val, const SrcOp &Elt, const SrcOp &Idx)
Build and insert Res = G_INSERT_VECTOR_ELT Val, Elt, Idx.
Definition: MachineIRBuilder.cpp:797
llvm::AArch64Subtarget::isTargetWindows
bool isTargetWindows() const
Definition: AArch64Subtarget.h:264
llvm::LegalizeRuleSet::widenScalarToNextPow2
LegalizeRuleSet & widenScalarToNextPow2(unsigned TypeIdx, unsigned MinSize=0)
Widen the scalar to the next power of two that is at least MinSize.
Definition: LegalizerInfo.h:877
llvm::AtomicOrdering::AcquireRelease
@ AcquireRelease
matchLDPSTPAddrMode
static void matchLDPSTPAddrMode(Register Root, Register &Base, int &Offset, MachineRegisterInfo &MRI)
Definition: AArch64LegalizerInfo.cpp:1094
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:108
MachineInstr.h
MathExtras.h
llvm::MachineInstrBuilder::addImm
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Definition: MachineInstrBuilder.h:131
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
llvm::LLT::getScalarSizeInBits
unsigned getScalarSizeInBits() const
Definition: LowLevelTypeImpl.h:224
llvm::MachineRegisterInfo::createVirtualRegister
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
Definition: MachineRegisterInfo.cpp:156
AArch64RegisterBankInfo.h
llvm::MIPatternMatch::m_Reg
operand_type_match m_Reg()
Definition: MIPatternMatch.h:268
llvm::LegalizerInfo::getActionDefinitionsBuilder
LegalizeRuleSet & getActionDefinitionsBuilder(unsigned Opcode)
Get the action definition builder for the given opcode.
Definition: LegalizerInfo.cpp:288
llvm::MachineRegisterInfo
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Definition: MachineRegisterInfo.h:50
llvm::MachineInstrBuilder::add
const MachineInstrBuilder & add(const MachineOperand &MO) const
Definition: MachineInstrBuilder.h:224
llvm::LegalizeRuleSet::minScalarEltSameAsIf
LegalizeRuleSet & minScalarEltSameAsIf(LegalityPredicate Predicate, unsigned TypeIdx, unsigned LargeTypeIdx)
Conditionally widen the scalar or elt to match the size of another.
Definition: LegalizerInfo.h:1032
llvm::LegalizeRuleSet::maxScalarIf
LegalizeRuleSet & maxScalarIf(LegalityPredicate Predicate, unsigned TypeIdx, const LLT Ty)
Conditionally limit the maximum size of the scalar.
Definition: LegalizerInfo.h:974
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1199
llvm::MachineIRBuilder::getMRI
MachineRegisterInfo * getMRI()
Getter for MRI.
Definition: MachineIRBuilder.h:289
llvm::LegalizeRuleSet::customFor
LegalizeRuleSet & customFor(std::initializer_list< LLT > Types)
Definition: LegalizerInfo.h:840
llvm::MachineFunction::getMachineMemOperand
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
Definition: MachineFunction.cpp:454
llvm::LegacyLegalizeActions::Bitcast
@ Bitcast
Perform the operation on a different, but equivalently sized type.
Definition: LegacyLegalizerInfo.h:54
llvm::X86Disassembler::Reg
Reg
All possible values of the reg field in the ModR/M byte.
Definition: X86DisassemblerDecoder.h:462
llvm::LegalizeRuleSet::clampNumElements
LegalizeRuleSet & clampNumElements(unsigned TypeIdx, const LLT MinTy, const LLT MaxTy)
Limit the number of elements for the given vectors to at least MinTy's number of elements and at most...
Definition: LegalizerInfo.h:1139
llvm::MachineRegisterInfo::getTargetRegisterInfo
const TargetRegisterInfo * getTargetRegisterInfo() const
Definition: MachineRegisterInfo.h:151
llvm::LegacyLegalizerInfo::computeTables
void computeTables()
Compute any ancillary tables needed to quickly decide how an operation should be handled.
Definition: LegacyLegalizerInfo.cpp:102
llvm::AArch64Subtarget::isTargetDarwin
bool isTargetDarwin() const
Definition: AArch64Subtarget.h:261
llvm::AtomicOrdering::SequentiallyConsistent
@ SequentiallyConsistent
llvm::LegalizeRuleSet::minScalarOrEltIf
LegalizeRuleSet & minScalarOrEltIf(LegalityPredicate Predicate, unsigned TypeIdx, const LLT Ty)
Ensure the scalar or element is at least as wide as Ty.
Definition: LegalizerInfo.h:934
llvm::AArch64II::MO_PREL
@ MO_PREL
MO_PREL - Indicates that the bits of the symbol operand represented by MO_G0 etc are PC relative.
Definition: AArch64BaseInfo.h:794
llvm::AArch64Subtarget::getInstrInfo
const AArch64InstrInfo * getInstrInfo() const override
Definition: AArch64Subtarget.h:181
llvm::LLT::changeElementCount
LLT changeElementCount(ElementCount EC) const
Return a vector or scalar with the same element type and the new element count.
Definition: LowLevelTypeImpl.h:189
llvm::LLT::vector
static LLT vector(ElementCount EC, unsigned ScalarSizeInBits)
Get a low-level vector of some number of elements and element width.
Definition: LowLevelTypeImpl.h:56
llvm::max
Expected< ExpressionValue > max(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
Definition: FileCheck.cpp:337
llvm::LegalizeRuleSet::scalarizeIf
LegalizeRuleSet & scalarizeIf(LegalityPredicate Predicate, unsigned TypeIdx)
Definition: LegalizerInfo.h:917
MachineIRBuilder.h
llvm::LegalizeRuleSet::minScalarOrElt
LegalizeRuleSet & minScalarOrElt(unsigned TypeIdx, const LLT Ty)
Ensure the scalar or element is at least as wide as Ty.
Definition: LegalizerInfo.h:925
llvm::LegalizeRuleSet::scalarize
LegalizeRuleSet & scalarize(unsigned TypeIdx)
Definition: LegalizerInfo.h:911
llvm::isPowerOf2_32
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:458
llvm::LegalizeRuleSet::lower
LegalizeRuleSet & lower()
The instruction is lowered.
Definition: LegalizerInfo.h:659
llvm::LegalizerHelper
Definition: LegalizerHelper.h:46
llvm::LegalizeMutations::changeTo
LegalizeMutation changeTo(unsigned TypeIdx, LLT Ty)
Select this specific type for the given type index.
Definition: LegalizeMutations.cpp:17
LegalizerInfo.h
llvm::MIPatternMatch::m_GPtrAdd
BinaryOp_match< LHS, RHS, TargetOpcode::G_PTR_ADD, false > m_GPtrAdd(const LHS &L, const RHS &R)
Definition: MIPatternMatch.h:468
llvm::LegalityPredicates::atomicOrderingAtLeastOrStrongerThan
LegalityPredicate atomicOrderingAtLeastOrStrongerThan(unsigned MMOIdx, AtomicOrdering Ordering)
True iff the specified MMO index has at an atomic ordering of at Ordering or stronger.
Definition: LegalityPredicates.cpp:205
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
llvm::AArch64Subtarget::getTargetLowering
const AArch64TargetLowering * getTargetLowering() const override
Definition: AArch64Subtarget.h:178
llvm::constrainSelectedInstRegOperands
bool constrainSelectedInstRegOperands(MachineInstr &I, const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
Mutate the newly-selected instruction I to constrain its (possibly generic) virtual register operands...
Definition: Utils.cpp:153
llvm::MachineIRBuilder::buildConstant
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
Definition: MachineIRBuilder.cpp:293
llvm::LegalizeRuleSet::legalIf
LegalizeRuleSet & legalIf(LegalityPredicate Predicate)
The instruction is legal if predicate is true.
Definition: LegalizerInfo.h:586
MachineRegisterInfo.h
llvm::AtomicOrdering::Monotonic
@ Monotonic
llvm::logicalview::LVOutputKind::Split
@ Split
llvm::LegalizeRuleSet::minScalar
LegalizeRuleSet & minScalar(unsigned TypeIdx, const LLT Ty)
Ensure the scalar is at least as wide as Ty.
Definition: LegalizerInfo.h:945
always
bar al al movzbl eax ret Missed when stored in a memory are stored as single byte objects the value of which is always(false) or 1(true). We are not using this fact
Definition: README.txt:1412
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
llvm::LLT::fixed_vector
static LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
Definition: LowLevelTypeImpl.h:74
llvm::MachineIRBuilder::buildBitReverse
MachineInstrBuilder buildBitReverse(const DstOp &Dst, const SrcOp &Src)
Build and insert Dst = G_BITREVERSE Src.
Definition: MachineIRBuilder.h:1974
llvm::BitmaskEnumDetail::Mask
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:80
llvm::MachineFunction::getRegInfo
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Definition: MachineFunction.h:667
llvm::LegalizeRuleSet::customIf
LegalizeRuleSet & customIf(LegalityPredicate Predicate)
Definition: LegalizerInfo.h:834
llvm::LegalityPredicates::typeIs
LegalityPredicate typeIs(unsigned TypeIdx, LLT TypesInit)
True iff the given type index is the specified type.
Definition: LegalityPredicates.cpp:28
llvm::MachineIRBuilder::buildUnmerge
MachineInstrBuilder buildUnmerge(ArrayRef< LLT > Res, const SrcOp &Op)
Build and insert Res0, ...
Definition: MachineIRBuilder.cpp:617
llvm::MachineIRBuilder::buildZExt
MachineInstrBuilder buildZExt(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ZEXT Op.
Definition: MachineIRBuilder.cpp:462
llvm::MachineIRBuilder::buildLoad
MachineInstrBuilder buildLoad(const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert Res = G_LOAD Addr, MMO.
Definition: MachineIRBuilder.h:904
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::LegalityPredicates::smallerThan
LegalityPredicate smallerThan(unsigned TypeIdx0, unsigned TypeIdx1)
True iff the first type index has a smaller total bit size than second type index.
Definition: LegalityPredicates.cpp:117
llvm::AArch64Subtarget::isTargetILP32
bool isTargetILP32() const
Definition: AArch64Subtarget.h:273
Intrinsics.h
llvm::MachineInstr::getOperand
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:526
llvm::Log2
unsigned Log2(Align A)
Returns the log2 of the alignment.
Definition: Alignment.h:209
llvm::LLT::getSizeInBits
TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
Definition: LowLevelTypeImpl.h:152
llvm::AArch64LegalizerInfo::legalizeIntrinsic
bool legalizeIntrinsic(LegalizerHelper &Helper, MachineInstr &MI) const override
Definition: AArch64LegalizerInfo.cpp:1002
llvm::LegalizeRuleSet::lowerIfMemSizeNotPow2
LegalizeRuleSet & lowerIfMemSizeNotPow2()
Lower a memory operation if the memory size, rounded to bytes, is not a power of 2.
Definition: LegalizerInfo.h:821
Utils.h
llvm::MachineIRBuilder::buildCTLZ
MachineInstrBuilder buildCTLZ(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_CTLZ Op0, Src0.
Definition: MachineIRBuilder.h:1649
llvm::LegalizeRuleSet::fewerElementsIf
LegalizeRuleSet & fewerElementsIf(LegalityPredicate Predicate, LegalizeMutation Mutation)
Remove elements to reach the type selected by the mutation if the predicate is true.
Definition: LegalizerInfo.h:792
TargetOpcodes.h
llvm::MachineIRBuilder::buildConcatVectors
MachineInstrBuilder buildConcatVectors(const DstOp &Res, ArrayRef< Register > Ops)
Build and insert Res = G_CONCAT_VECTORS Op0, ...
Definition: MachineIRBuilder.cpp:712
llvm::CodeModel::Small
@ Small
Definition: CodeGen.h:28
Concat
static constexpr int Concat[]
Definition: X86InterleavedAccess.cpp:239
llvm::MachineIRBuilder::getMF
MachineFunction & getMF()
Getter for the function we currently build.
Definition: MachineIRBuilder.h:271
llvm::AtomicOrdering::Acquire
@ Acquire
llvm::AArch64LegalizerInfo::AArch64LegalizerInfo
AArch64LegalizerInfo(const AArch64Subtarget &ST)
Definition: AArch64LegalizerInfo.cpp:41
llvm::GISelChangeObserver::changingInstr
virtual void changingInstr(MachineInstr &MI)=0
This instruction is about to be mutated in some way.
llvm::LegalizeRuleSet::clampMaxNumElements
LegalizeRuleSet & clampMaxNumElements(unsigned TypeIdx, const LLT EltTy, unsigned MaxElements)
Limit the number of elements in EltTy vectors to at most MaxElements.
Definition: LegalizerInfo.h:1115
Align
uint64_t Align
Definition: ELFObjHandler.cpp:82
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
llvm::LegalizeRuleSet::maxScalarEltSameAsIf
LegalizeRuleSet & maxScalarEltSameAsIf(LegalityPredicate Predicate, unsigned TypeIdx, unsigned SmallTypeIdx)
Conditionally narrow the scalar or elt to match the size of another.
Definition: LegalizerInfo.h:1050
Type.h
llvm::MachineInstrBuilder::getReg
Register getReg(unsigned Idx) const
Get the register for the operand index.
Definition: MachineInstrBuilder.h:94
llvm::MachineInstrBuilder::cloneMemRefs
const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const
Definition: MachineInstrBuilder.h:213
llvm::AArch64LegalizerInfo::legalizeCustom
bool legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI) const override
Called for instructions with the Custom LegalizationAction.
Definition: AArch64LegalizerInfo.cpp:836
llvm::MachineIRBuilder::buildFNeg
MachineInstrBuilder buildFNeg(const DstOp &Dst, const SrcOp &Src0, Optional< unsigned > Flags=None)
Build and insert Res = G_FNEG Op0.
Definition: MachineIRBuilder.h:1716
llvm::MachineIRBuilder::buildCTPOP
MachineInstrBuilder buildCTPOP(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_CTPOP Op0, Src0.
Definition: MachineIRBuilder.h:1644
llvm::LLT::pointer
static LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
Definition: LowLevelTypeImpl.h:49
llvm::LLT::getAddressSpace
unsigned getAddressSpace() const
Definition: LowLevelTypeImpl.h:238
llvm::GISelChangeObserver::changedInstr
virtual void changedInstr(MachineInstr &MI)=0
This instruction was mutated in some way.
llvm::MachineIRBuilder::buildMaskLowPtrBits
MachineInstrBuilder buildMaskLowPtrBits(const DstOp &Res, const SrcOp &Op0, uint32_t NumBits)
Build and insert Res = G_PTRMASK Op0, G_CONSTANT (1 << NumBits) - 1.
Definition: MachineIRBuilder.cpp:216
llvm::LegalizeMutations::scalarize
LegalizeMutation scalarize(unsigned TypeIdx)
Break up the vector type for the given type index into the element type.
Definition: LegalizeMutations.cpp:108
llvm::LegalizeRuleSet::lowerIf
LegalizeRuleSet & lowerIf(LegalityPredicate Predicate)
The instruction is lowered if predicate is true.
Definition: LegalizerInfo.h:668
llvm::LLT::divide
LLT divide(int Factor) const
Return a type that is Factor times smaller.
Definition: LowLevelTypeImpl.h:196
llvm::MachineIRBuilder
Helper class to build MachineInstr.
Definition: MachineIRBuilder.h:221
llvm::LegalizeRuleSet::legalFor
LegalizeRuleSet & legalFor(std::initializer_list< LLT > Types)
The instruction is legal when type index 0 is any type in the given list.
Definition: LegalizerInfo.h:593
llvm::RegState::Undef
@ Undef
Value of the register doesn't matter.
Definition: MachineInstrBuilder.h:52
llvm::AMDGPU::Hwreg::Offset
Offset
Definition: SIDefines.h:416
llvm::MachineInstr
Representation of each machine instruction.
Definition: MachineInstr.h:66
llvm::MachineInstrBuilder
Definition: MachineInstrBuilder.h:69
uint64_t
llvm::ARM_MB::ST
@ ST
Definition: ARMBaseInfo.h:73
Addr
uint64_t Addr
Definition: ELFObjHandler.cpp:79
llvm::MachineIRBuilder::buildBitcast
MachineInstrBuilder buildBitcast(const DstOp &Dst, const SrcOp &Src)
Build and insert Dst = G_BITCAST Src.
Definition: MachineIRBuilder.h:677
llvm::LegalityPredicates::all
Predicate all(Predicate P0, Predicate P1)
True iff P0 and P1 are true.
Definition: LegalizerInfo.h:228
llvm::AArch64II::MO_NC
@ MO_NC
MO_NC - Indicates whether the linker is expected to check the symbol reference for overflow.
Definition: AArch64BaseInfo.h:775
llvm::MachinePointerInfo
This class contains a discriminated union of information about pointers in memory operands,...
Definition: MachineMemOperand.h:39
llvm::AArch64II::MO_PAGEOFF
@ MO_PAGEOFF
MO_PAGEOFF - A symbol operand with this flag represents the offset of that symbol within a 4K page.
Definition: AArch64BaseInfo.h:739
llvm::MachineIRBuilder::buildPtrAdd
MachineInstrBuilder buildPtrAdd(const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1)
Build and insert Res = G_PTR_ADD Op0, Op1.
Definition: MachineIRBuilder.cpp:190
llvm::AtomicOrdering::Unordered
@ Unordered
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::LLT::isVector
bool isVector() const
Definition: LowLevelTypeImpl.h:122
llvm::LegalizeRuleSet::clampScalar
LegalizeRuleSet & clampScalar(unsigned TypeIdx, const LLT MinTy, const LLT MaxTy)
Limit the range of scalar sizes to MinTy and MaxTy.
Definition: LegalizerInfo.h:990
llvm::LLT::getNumElements
uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
Definition: LowLevelTypeImpl.h:126
llvm::is_contained
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.
Definition: STLExtras.h:1868
llvm::LegalizeRuleSet::legalForCartesianProduct
LegalizeRuleSet & legalForCartesianProduct(std::initializer_list< LLT > Types)
The instruction is legal when type indexes 0 and 1 are both in the given list.
Definition: LegalizerInfo.h:625
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::TargetMachine
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:76
llvm::LLT::isPointer
bool isPointer() const
Definition: LowLevelTypeImpl.h:120
llvm::LegalityPredicates::typeInSet
LegalityPredicate typeInSet(unsigned TypeIdx, std::initializer_list< LLT > TypesInit)
True iff the given type index is one of the specified types.
Definition: LegalityPredicates.cpp:34
llvm::LegalizeRuleSet::lowerIfMemSizeNotByteSizePow2
LegalizeRuleSet & lowerIfMemSizeNotByteSizePow2()
Lower a memory operation if the memory access size is not a round power of 2 byte size.
Definition: LegalizerInfo.h:829
llvm::MachineRegisterInfo::createGenericVirtualRegister
Register createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
Definition: MachineRegisterInfo.cpp:186
llvm::MachineInstrBuilder::addUse
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
Definition: MachineInstrBuilder.h:123
llvm::MachineOperand::getReg
Register getReg() const
getReg - Returns the register number.
Definition: MachineOperand.h:359
llvm::LLT::isScalar
bool isScalar() const
Definition: LowLevelTypeImpl.h:118
llvm::MachineFunction
Definition: MachineFunction.h:257
llvm::LegalityQuery::Opcode
unsigned Opcode
Definition: LegalizerInfo.h:109
prefetch
loop data prefetch
Definition: LoopDataPrefetch.cpp:149
llvm::LegalizerHelper::lowerBitCount
LegalizeResult lowerBitCount(MachineInstr &MI)
Definition: LegalizerHelper.cpp:5560
llvm::MachineIRBuilder::buildInstr
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
Definition: MachineIRBuilder.h:383
llvm::LegalityQuery
The LegalityQuery object bundles together all the information that's needed to decide whether a given...
Definition: LegalizerInfo.h:108
llvm::LegalizeRuleSet::customForCartesianProduct
LegalizeRuleSet & customForCartesianProduct(std::initializer_list< LLT > Types)
Definition: LegalizerInfo.h:850
llvm::getIConstantVRegValWithLookThrough
Optional< ValueAndVReg > getIConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_CONSTANT returns its...
Definition: Utils.cpp:410
llvm::logicalview::LVAttributeKind::Zero
@ Zero
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
if
if(llvm_vc STREQUAL "") set(fake_version_inc "$
Definition: CMakeLists.txt:14
llvm::LegalizerHelper::Observer
GISelChangeObserver & Observer
To keep track of changes made by the LegalizerHelper.
Definition: LegalizerHelper.h:53
llvm::MachineIRBuilder::buildExtract
MachineInstrBuilder buildExtract(const DstOp &Res, const SrcOp &Src, uint64_t Index)
Build and insert Res0, ...
Definition: MachineIRBuilder.cpp:571
llvm::GISelChangeObserver
Abstract class that contains various methods for clients to notify about changes.
Definition: GISelChangeObserver.h:29
llvm::LegalizeMutations::widenScalarOrEltToNextPow2
LegalizeMutation widenScalarOrEltToNextPow2(unsigned TypeIdx, unsigned Min=0)
Widen the scalar type or vector element type for the given type index to the next power of 2.
Definition: LegalizeMutations.cpp:77
llvm::MachineMemOperand::MOLoad
@ MOLoad
The memory access reads data.
Definition: MachineMemOperand.h:134
MRI
unsigned const MachineRegisterInfo * MRI
Definition: AArch64AdvSIMDScalarPass.cpp:105
llvm::MachineIRBuilder::buildAnyExt
MachineInstrBuilder buildAnyExt(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ANYEXT Op0.
Definition: MachineIRBuilder.cpp:452
llvm::Register
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
llvm::LegalityPredicates::isScalar
LegalityPredicate isScalar(unsigned TypeIdx)
True iff the specified type index is a scalar.
Definition: LegalityPredicates.cpp:67
llvm::AtomicOrdering::Release
@ Release
llvm::LegalizerInfo::getLegacyLegalizerInfo
const LegacyLegalizerInfo & getLegacyLegalizerInfo() const
Definition: LegalizerInfo.h:1186
llvm::TargetLoweringBase::getTargetMachine
const TargetMachine & getTargetMachine() const
Definition: TargetLowering.h:349
llvm::LegalizeRuleSet::narrowScalarIf
LegalizeRuleSet & narrowScalarIf(LegalityPredicate Predicate, LegalizeMutation Mutation)
Narrow the scalar to the one selected by the mutation if the predicate is true.
Definition: LegalizerInfo.h:766
llvm::MachineIRBuilder::buildTrunc
MachineInstrBuilder buildTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_TRUNC Op.
Definition: MachineIRBuilder.cpp:760
llvm::AArch64Subtarget::getRegBankInfo
const RegisterBankInfo * getRegBankInfo() const override
Definition: AArch64Subtarget.cpp:343
llvm::MIPatternMatch::m_ICst
ConstantMatch< APInt > m_ICst(APInt &Cst)
Definition: MIPatternMatch.h:92
llvm::AArch64Subtarget::ClassifyGlobalReference
unsigned ClassifyGlobalReference(const GlobalValue *GV, const TargetMachine &TM) const
ClassifyGlobalReference - Find the target operand flags that describe how a global value should be re...
Definition: AArch64Subtarget.cpp:350
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:348
llvm::MachineIRBuilder::buildUndef
MachineInstrBuilder buildUndef(const DstOp &Res)
Build and insert Res = IMPLICIT_DEF.
Definition: MachineIRBuilder.cpp:596
llvm::LegalityQuery::MMODescrs
ArrayRef< MemDesc > MMODescrs
Operations which require memory can use this to place requirements on the memory type for each MMO.
Definition: LegalizerInfo.h:128
llvm::LegalizeRuleSet::unsupportedIf
LegalizeRuleSet & unsupportedIf(LegalityPredicate Predicate)
Definition: LegalizerInfo.h:805
LegalizerHelper.h
llvm::AArch64ISD::ADRP
@ ADRP
Definition: AArch64ISelLowering.h:75
llvm::LegalizeRuleSet::minScalarSameAs
LegalizeRuleSet & minScalarSameAs(unsigned TypeIdx, unsigned LargeTypeIdx)
Widen the scalar to match the size of another.
Definition: LegalizerInfo.h:1003
llvm::MachineMemOperand::MOStore
@ MOStore
The memory access writes data.
Definition: MachineMemOperand.h:136
llvm::LegalizerInfo::verify
void verify(const MCInstrInfo &MII) const
Perform simple self-diagnostic and assert if there is anything obviously wrong with the actions set u...
Definition: LegalizerInfo.cpp:376
llvm::LegalityPredicates::isVector
LegalityPredicate isVector(unsigned TypeIdx)
True iff the specified type index is a vector.
Definition: LegalityPredicates.cpp:73
AArch64Subtarget.h
llvm::MachineIRBuilder::buildIntrinsic
MachineInstrBuilder buildIntrinsic(Intrinsic::ID ID, ArrayRef< Register > Res, bool HasSideEffects)
Build and insert either a G_INTRINSIC (if HasSideEffects is false) or G_INTRINSIC_W_SIDE_EFFECTS inst...
Definition: MachineIRBuilder.cpp:736
llvm::MachineRegisterInfo::getType
LLT getType(Register Reg) const
Get the low-level type of Reg or LLT{} if Reg is not a generic (target independent) virtual register.
Definition: MachineRegisterInfo.h:745
llvm::LegalityQuery::Types
ArrayRef< LLT > Types
Definition: LegalizerInfo.h:110
llvm::LegalizeRuleSet::moreElementsToNextPow2
LegalizeRuleSet & moreElementsToNextPow2(unsigned TypeIdx)
Add more elements to the vector to reach the next power of two.
Definition: LegalizerInfo.h:1069
List
const NodeList & List
Definition: RDFGraph.cpp:199
llvm::LLT::getElementCount
ElementCount getElementCount() const
Definition: LowLevelTypeImpl.h:143
llvm::MachineOperand::setReg
void setReg(Register Reg)
Change the register this operand corresponds to.
Definition: MachineOperand.cpp:56
llvm::LegalizerHelper::MIRBuilder
MachineIRBuilder & MIRBuilder
Expose MIRBuilder so clients can set their own RecordInsertInstruction functions.
Definition: LegalizerHelper.h:50
llvm::MachineIRBuilder::buildMerge
MachineInstrBuilder buildMerge(const DstOp &Res, ArrayRef< Register > Ops)
Build and insert Res = G_MERGE_VALUES Op0, ...
Definition: MachineIRBuilder.cpp:600
llvm::LegalizeRuleSet::custom
LegalizeRuleSet & custom()
Unconditionally custom lower.
Definition: LegalizerInfo.h:871
llvm::SmallVectorImpl
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:42
llvm::LegalizeMutations::moreElementsToNextPow2
LegalizeMutation moreElementsToNextPow2(unsigned TypeIdx, unsigned Min=0)
Add more elements to the type for the given type index to the next power of.
Definition: LegalizeMutations.cpp:97
extractParts
static void extractParts(Register Reg, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder, LLT Ty, int NumParts, SmallVectorImpl< Register > &VRegs)
Definition: AArch64LegalizerInfo.cpp:898
DerivedTypes.h
TM
const char LLVMTargetMachineRef TM
Definition: PassBuilderBindings.cpp:47
llvm::LegalizeRuleSet::scalarSameSizeAs
LegalizeRuleSet & scalarSameSizeAs(unsigned TypeIdx, unsigned SameSizeIdx)
Change the type TypeIdx to have the same scalar size as type SameSizeIdx.
Definition: LegalizerInfo.h:1026
llvm::MIPatternMatch::mi_match
bool mi_match(Reg R, const MachineRegisterInfo &MRI, Pattern &&P)
Definition: MIPatternMatch.h:25
llvm::AArch64II::MO_TAGGED
@ MO_TAGGED
MO_TAGGED - With MO_PAGE, indicates that the page includes a memory tag in bits 56-63.
Definition: AArch64BaseInfo.h:802
llvm::MachineIRBuilder::buildStore
MachineInstrBuilder buildStore(const SrcOp &Val, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert G_STORE Val, Addr, MMO.
Definition: MachineIRBuilder.cpp:425
llvm::LLT::getElementType
LLT getElementType() const
Returns the vector's element type. Only valid for vector types.
Definition: LowLevelTypeImpl.h:248
llvm::AArch64Subtarget
Definition: AArch64Subtarget.h:38
llvm::LegalizeRuleSet::libcallFor
LegalizeRuleSet & libcallFor(std::initializer_list< LLT > Types)
Definition: LegalizerInfo.h:738
llvm::LinearPolySize::divideCoefficientBy
LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
Definition: TypeSize.h:360
llvm::LLT::scalar
static LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
Definition: LowLevelTypeImpl.h:42
llvm::LLT::changeElementSize
LLT changeElementSize(unsigned NewEltSize) const
If this type is a vector, return a vector with the same number of elements but the new element size.
Definition: LowLevelTypeImpl.h:180
llvm::AArch64II::MO_GOT
@ MO_GOT
MO_GOT - This flag indicates that a symbol operand represents the address of the GOT entry for the sy...
Definition: AArch64BaseInfo.h:770
llvm::Value
LLVM Value Representation.
Definition: Value.h:74
libcall
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM ID Predecessors according to mbb< bb27, 0x8b0a7c0 > Note ADDri is not a two address instruction its result reg1037 is an operand of the PHI node in bb76 and its operand reg1039 is the result of the PHI node We should treat it as a two address code and make sure the ADDri is scheduled after any node that reads reg1039 Use info(i.e. register scavenger) to assign it a free register to allow reuse the collector could move the objects and invalidate the derived pointer This is bad enough in the first but safe points can crop up unpredictably **array_addr i32 n y store obj obj **nth_el If the i64 division is lowered to a libcall
Definition: README.txt:127
llvm::AtomicOrdering::NotAtomic
@ NotAtomic
llvm::ISD::CTPOP
@ CTPOP
Definition: ISDOpcodes.h:703
llvm::MachineRegisterInfo::setRegClass
void setRegClass(Register Reg, const TargetRegisterClass *RC)
setRegClass - Set the register class of the specified virtual register.
Definition: MachineRegisterInfo.cpp:56
llvm::sampleprof::Base
@ Base
Definition: Discriminator.h:58
llvm::AArch64II::MO_PAGE
@ MO_PAGE
MO_PAGE - A symbol operand with this flag represents the pc-relative offset of the 4K page containing...
Definition: AArch64BaseInfo.h:734
llvm::LLT
Definition: LowLevelTypeImpl.h:39