LLVM 22.0.0git
AArch64RegisterBankInfo.cpp
Go to the documentation of this file.
1//===- AArch64RegisterBankInfo.cpp ----------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the RegisterBankInfo class for
10/// AArch64.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
15#include "AArch64RegisterInfo.h"
16#include "AArch64Subtarget.h"
18#include "llvm/ADT/STLExtras.h"
33#include "llvm/IR/IntrinsicsAArch64.h"
36#include <cassert>
37
38#define GET_TARGET_REGBANK_IMPL
39#include "AArch64GenRegisterBank.inc"
40
41// This file will be TableGen'ed at some point.
42#include "AArch64GenRegisterBankInfo.def"
43
44using namespace llvm;
45static const unsigned CustomMappingID = 1;
46
48 const TargetRegisterInfo &TRI) {
49 static llvm::once_flag InitializeRegisterBankFlag;
50
51 static auto InitializeRegisterBankOnce = [&]() {
52 // We have only one set of register banks, whatever the subtarget
53 // is. Therefore, the initialization of the RegBanks table should be
54 // done only once. Indeed the table of all register banks
55 // (AArch64::RegBanks) is unique in the compiler. At some point, it
56 // will get tablegen'ed and the whole constructor becomes empty.
57
58 const RegisterBank &RBGPR = getRegBank(AArch64::GPRRegBankID);
59 (void)RBGPR;
60 assert(&AArch64::GPRRegBank == &RBGPR &&
61 "The order in RegBanks is messed up");
62
63 const RegisterBank &RBFPR = getRegBank(AArch64::FPRRegBankID);
64 (void)RBFPR;
65 assert(&AArch64::FPRRegBank == &RBFPR &&
66 "The order in RegBanks is messed up");
67
68 const RegisterBank &RBCCR = getRegBank(AArch64::CCRegBankID);
69 (void)RBCCR;
70 assert(&AArch64::CCRegBank == &RBCCR &&
71 "The order in RegBanks is messed up");
72
73 // The GPR register bank is fully defined by all the registers in
74 // GR64all + its subclasses.
75 assert(RBGPR.covers(*TRI.getRegClass(AArch64::GPR32RegClassID)) &&
76 "Subclass not added?");
77 assert(getMaximumSize(RBGPR.getID()) == 128 &&
78 "GPRs should hold up to 128-bit");
79
80 // The FPR register bank is fully defined by all the registers in
81 // GR64all + its subclasses.
82 assert(RBFPR.covers(*TRI.getRegClass(AArch64::QQRegClassID)) &&
83 "Subclass not added?");
84 assert(RBFPR.covers(*TRI.getRegClass(AArch64::FPR64RegClassID)) &&
85 "Subclass not added?");
86 assert(getMaximumSize(RBFPR.getID()) == 512 &&
87 "FPRs should hold up to 512-bit via QQQQ sequence");
88
89 assert(RBCCR.covers(*TRI.getRegClass(AArch64::CCRRegClassID)) &&
90 "Class not added?");
91 assert(getMaximumSize(RBCCR.getID()) == 32 &&
92 "CCR should hold up to 32-bit");
93
94 // Check that the TableGen'ed like file is in sync we our expectations.
95 // First, the Idx.
98 "PartialMappingIdx's are incorrectly ordered");
102 "PartialMappingIdx's are incorrectly ordered");
103// Now, the content.
104// Check partial mapping.
105#define CHECK_PARTIALMAP(Idx, ValStartIdx, ValLength, RB) \
106 do { \
107 assert( \
108 checkPartialMap(PartialMappingIdx::Idx, ValStartIdx, ValLength, RB) && \
109 #Idx " is incorrectly initialized"); \
110 } while (false)
111
112 CHECK_PARTIALMAP(PMI_GPR32, 0, 32, RBGPR);
113 CHECK_PARTIALMAP(PMI_GPR64, 0, 64, RBGPR);
114 CHECK_PARTIALMAP(PMI_GPR128, 0, 128, RBGPR);
115 CHECK_PARTIALMAP(PMI_FPR16, 0, 16, RBFPR);
116 CHECK_PARTIALMAP(PMI_FPR32, 0, 32, RBFPR);
117 CHECK_PARTIALMAP(PMI_FPR64, 0, 64, RBFPR);
118 CHECK_PARTIALMAP(PMI_FPR128, 0, 128, RBFPR);
119 CHECK_PARTIALMAP(PMI_FPR256, 0, 256, RBFPR);
120 CHECK_PARTIALMAP(PMI_FPR512, 0, 512, RBFPR);
121
122// Check value mapping.
123#define CHECK_VALUEMAP_IMPL(RBName, Size, Offset) \
124 do { \
125 assert(checkValueMapImpl(PartialMappingIdx::PMI_##RBName##Size, \
126 PartialMappingIdx::PMI_First##RBName, Size, \
127 Offset) && \
128 #RBName #Size " " #Offset " is incorrectly initialized"); \
129 } while (false)
130
131#define CHECK_VALUEMAP(RBName, Size) CHECK_VALUEMAP_IMPL(RBName, Size, 0)
132
133 CHECK_VALUEMAP(GPR, 32);
134 CHECK_VALUEMAP(GPR, 64);
135 CHECK_VALUEMAP(GPR, 128);
136 CHECK_VALUEMAP(FPR, 16);
137 CHECK_VALUEMAP(FPR, 32);
138 CHECK_VALUEMAP(FPR, 64);
139 CHECK_VALUEMAP(FPR, 128);
140 CHECK_VALUEMAP(FPR, 256);
141 CHECK_VALUEMAP(FPR, 512);
142
143// Check the value mapping for 3-operands instructions where all the operands
144// map to the same value mapping.
145#define CHECK_VALUEMAP_3OPS(RBName, Size) \
146 do { \
147 CHECK_VALUEMAP_IMPL(RBName, Size, 0); \
148 CHECK_VALUEMAP_IMPL(RBName, Size, 1); \
149 CHECK_VALUEMAP_IMPL(RBName, Size, 2); \
150 } while (false)
151
152 CHECK_VALUEMAP_3OPS(GPR, 32);
153 CHECK_VALUEMAP_3OPS(GPR, 64);
154 CHECK_VALUEMAP_3OPS(GPR, 128);
160
161#define CHECK_VALUEMAP_CROSSREGCPY(RBNameDst, RBNameSrc, Size) \
162 do { \
163 unsigned PartialMapDstIdx = PMI_##RBNameDst##Size - PMI_Min; \
164 unsigned PartialMapSrcIdx = PMI_##RBNameSrc##Size - PMI_Min; \
165 (void)PartialMapDstIdx; \
166 (void)PartialMapSrcIdx; \
167 const ValueMapping *Map = getCopyMapping(AArch64::RBNameDst##RegBankID, \
168 AArch64::RBNameSrc##RegBankID, \
169 TypeSize::getFixed(Size)); \
170 (void)Map; \
171 assert(Map[0].BreakDown == \
172 &AArch64GenRegisterBankInfo::PartMappings[PartialMapDstIdx] && \
173 Map[0].NumBreakDowns == 1 && \
174 #RBNameDst #Size " Dst is incorrectly initialized"); \
175 assert(Map[1].BreakDown == \
176 &AArch64GenRegisterBankInfo::PartMappings[PartialMapSrcIdx] && \
177 Map[1].NumBreakDowns == 1 && \
178 #RBNameSrc #Size " Src is incorrectly initialized"); \
179 \
180 } while (false)
181
182 CHECK_VALUEMAP_CROSSREGCPY(GPR, GPR, 32);
184 CHECK_VALUEMAP_CROSSREGCPY(GPR, GPR, 64);
190
191#define CHECK_VALUEMAP_FPEXT(DstSize, SrcSize) \
192 do { \
193 unsigned PartialMapDstIdx = PMI_FPR##DstSize - PMI_Min; \
194 unsigned PartialMapSrcIdx = PMI_FPR##SrcSize - PMI_Min; \
195 (void)PartialMapDstIdx; \
196 (void)PartialMapSrcIdx; \
197 const ValueMapping *Map = getFPExtMapping(DstSize, SrcSize); \
198 (void)Map; \
199 assert(Map[0].BreakDown == \
200 &AArch64GenRegisterBankInfo::PartMappings[PartialMapDstIdx] && \
201 Map[0].NumBreakDowns == 1 && "FPR" #DstSize \
202 " Dst is incorrectly initialized"); \
203 assert(Map[1].BreakDown == \
204 &AArch64GenRegisterBankInfo::PartMappings[PartialMapSrcIdx] && \
205 Map[1].NumBreakDowns == 1 && "FPR" #SrcSize \
206 " Src is incorrectly initialized"); \
207 \
208 } while (false)
209
210 CHECK_VALUEMAP_FPEXT(32, 16);
211 CHECK_VALUEMAP_FPEXT(64, 16);
212 CHECK_VALUEMAP_FPEXT(64, 32);
213 CHECK_VALUEMAP_FPEXT(128, 64);
214
215 assert(verify(TRI) && "Invalid register bank information");
216 };
217
218 llvm::call_once(InitializeRegisterBankFlag, InitializeRegisterBankOnce);
219}
220
222 const RegisterBank &B,
223 const TypeSize Size) const {
224 // What do we do with different size?
225 // copy are same size.
226 // Will introduce other hooks for different size:
227 // * extract cost.
228 // * build_sequence cost.
229
230 // Copy from (resp. to) GPR to (resp. from) FPR involves FMOV.
231 // FIXME: This should be deduced from the scheduling model.
232 if (&A == &AArch64::GPRRegBank && &B == &AArch64::FPRRegBank)
233 // FMOVXDr or FMOVWSr.
234 return 5;
235 if (&A == &AArch64::FPRRegBank && &B == &AArch64::GPRRegBank)
236 // FMOVDXr or FMOVSWr.
237 return 4;
238
240}
241
242const RegisterBank &
244 LLT Ty) const {
245 switch (RC.getID()) {
246 case AArch64::GPR64sponlyRegClassID:
247 return getRegBank(AArch64::GPRRegBankID);
248 default:
250 }
251}
252
255 const MachineInstr &MI) const {
256 const MachineFunction &MF = *MI.getParent()->getParent();
257 const TargetSubtargetInfo &STI = MF.getSubtarget();
258 const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
259 const MachineRegisterInfo &MRI = MF.getRegInfo();
260
261 switch (MI.getOpcode()) {
262 case TargetOpcode::G_OR: {
263 // 32 and 64-bit or can be mapped on either FPR or
264 // GPR for the same cost.
265 TypeSize Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, TRI);
266 if (Size != 32 && Size != 64)
267 break;
268
269 // If the instruction has any implicit-defs or uses,
270 // do not mess with it.
271 if (MI.getNumOperands() != 3)
272 break;
273 InstructionMappings AltMappings;
274 const InstructionMapping &GPRMapping = getInstructionMapping(
275 /*ID*/ 1, /*Cost*/ 1, getValueMapping(PMI_FirstGPR, Size),
276 /*NumOperands*/ 3);
277 const InstructionMapping &FPRMapping = getInstructionMapping(
278 /*ID*/ 2, /*Cost*/ 1, getValueMapping(PMI_FirstFPR, Size),
279 /*NumOperands*/ 3);
280
281 AltMappings.push_back(&GPRMapping);
282 AltMappings.push_back(&FPRMapping);
283 return AltMappings;
284 }
285 case TargetOpcode::G_BITCAST: {
286 TypeSize Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, TRI);
287 if (Size != 32 && Size != 64)
288 break;
289
290 // If the instruction has any implicit-defs or uses,
291 // do not mess with it.
292 if (MI.getNumOperands() != 2)
293 break;
294
295 InstructionMappings AltMappings;
296 const InstructionMapping &GPRMapping = getInstructionMapping(
297 /*ID*/ 1, /*Cost*/ 1,
298 getCopyMapping(AArch64::GPRRegBankID, AArch64::GPRRegBankID, Size),
299 /*NumOperands*/ 2);
300 const InstructionMapping &FPRMapping = getInstructionMapping(
301 /*ID*/ 2, /*Cost*/ 1,
302 getCopyMapping(AArch64::FPRRegBankID, AArch64::FPRRegBankID, Size),
303 /*NumOperands*/ 2);
304 const InstructionMapping &GPRToFPRMapping = getInstructionMapping(
305 /*ID*/ 3,
306 /*Cost*/
307 copyCost(AArch64::GPRRegBank, AArch64::FPRRegBank,
309 getCopyMapping(AArch64::FPRRegBankID, AArch64::GPRRegBankID, Size),
310 /*NumOperands*/ 2);
311 const InstructionMapping &FPRToGPRMapping = getInstructionMapping(
312 /*ID*/ 3,
313 /*Cost*/
314 copyCost(AArch64::GPRRegBank, AArch64::FPRRegBank,
316 getCopyMapping(AArch64::GPRRegBankID, AArch64::FPRRegBankID, Size),
317 /*NumOperands*/ 2);
318
319 AltMappings.push_back(&GPRMapping);
320 AltMappings.push_back(&FPRMapping);
321 AltMappings.push_back(&GPRToFPRMapping);
322 AltMappings.push_back(&FPRToGPRMapping);
323 return AltMappings;
324 }
325 case TargetOpcode::G_LOAD: {
326 TypeSize Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, TRI);
327 if (Size != 64)
328 break;
329
330 // If the instruction has any implicit-defs or uses,
331 // do not mess with it.
332 if (MI.getNumOperands() != 2)
333 break;
334
335 InstructionMappings AltMappings;
336 const InstructionMapping &GPRMapping = getInstructionMapping(
337 /*ID*/ 1, /*Cost*/ 1,
340 // Addresses are GPR 64-bit.
342 /*NumOperands*/ 2);
343 const InstructionMapping &FPRMapping = getInstructionMapping(
344 /*ID*/ 2, /*Cost*/ 1,
347 // Addresses are GPR 64-bit.
349 /*NumOperands*/ 2);
350
351 AltMappings.push_back(&GPRMapping);
352 AltMappings.push_back(&FPRMapping);
353 return AltMappings;
354 }
355 default:
356 break;
357 }
359}
360
361void AArch64RegisterBankInfo::applyMappingImpl(
362 MachineIRBuilder &Builder, const OperandsMapper &OpdMapper) const {
363 MachineInstr &MI = OpdMapper.getMI();
364 MachineRegisterInfo &MRI = OpdMapper.getMRI();
365
366 switch (MI.getOpcode()) {
367 case TargetOpcode::G_OR:
368 case TargetOpcode::G_BITCAST:
369 case TargetOpcode::G_LOAD:
370 // Those ID must match getInstrAlternativeMappings.
371 assert((OpdMapper.getInstrMapping().getID() >= 1 &&
372 OpdMapper.getInstrMapping().getID() <= 4) &&
373 "Don't know how to handle that ID");
374 return applyDefaultMapping(OpdMapper);
375 case TargetOpcode::G_INSERT_VECTOR_ELT: {
376 // Extend smaller gpr operands to 32 bit.
377 Builder.setInsertPt(*MI.getParent(), MI.getIterator());
378 auto Ext = Builder.buildAnyExt(LLT::scalar(32), MI.getOperand(2).getReg());
379 MRI.setRegBank(Ext.getReg(0), getRegBank(AArch64::GPRRegBankID));
380 MI.getOperand(2).setReg(Ext.getReg(0));
381 return applyDefaultMapping(OpdMapper);
382 }
383 case AArch64::G_DUP: {
384 // Extend smaller gpr to 32-bits
385 assert(MRI.getType(MI.getOperand(1).getReg()).getSizeInBits() < 32 &&
386 "Expected sources smaller than 32-bits");
387 Builder.setInsertPt(*MI.getParent(), MI.getIterator());
388
389 Register ConstReg;
390 auto ConstMI = MRI.getVRegDef(MI.getOperand(1).getReg());
391 if (ConstMI->getOpcode() == TargetOpcode::G_CONSTANT) {
392 auto CstVal = ConstMI->getOperand(1).getCImm()->getValue();
393 ConstReg =
394 Builder.buildConstant(LLT::scalar(32), CstVal.sext(32)).getReg(0);
395 } else {
396 ConstReg = Builder.buildAnyExt(LLT::scalar(32), MI.getOperand(1).getReg())
397 .getReg(0);
398 }
399 MRI.setRegBank(ConstReg, getRegBank(AArch64::GPRRegBankID));
400 MI.getOperand(1).setReg(ConstReg);
401 return applyDefaultMapping(OpdMapper);
402 }
403 default:
404 llvm_unreachable("Don't know how to handle that operation");
405 }
406}
407
409AArch64RegisterBankInfo::getSameKindOfOperandsMapping(
410 const MachineInstr &MI) const {
411 const unsigned Opc = MI.getOpcode();
412 const MachineFunction &MF = *MI.getParent()->getParent();
413 const MachineRegisterInfo &MRI = MF.getRegInfo();
414
415 unsigned NumOperands = MI.getNumOperands();
416 assert(NumOperands <= 3 &&
417 "This code is for instructions with 3 or less operands");
418
419 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
420 TypeSize Size = Ty.getSizeInBits();
422
424
425#ifndef NDEBUG
426 // Make sure all the operands are using similar size and type.
427 // Should probably be checked by the machine verifier.
428 // This code won't catch cases where the number of lanes is
429 // different between the operands.
430 // If we want to go to that level of details, it is probably
431 // best to check that the types are the same, period.
432 // Currently, we just check that the register banks are the same
433 // for each types.
434 for (unsigned Idx = 1; Idx != NumOperands; ++Idx) {
435 LLT OpTy = MRI.getType(MI.getOperand(Idx).getReg());
436 assert(
438 RBIdx, OpTy.getSizeInBits()) ==
440 "Operand has incompatible size");
441 bool OpIsFPR = OpTy.isVector() || isPreISelGenericFloatingPointOpcode(Opc);
442 (void)OpIsFPR;
443 assert(IsFPR == OpIsFPR && "Operand has incompatible type");
444 }
445#endif // End NDEBUG.
446
448 getValueMapping(RBIdx, Size), NumOperands);
449}
450
451/// \returns true if a given intrinsic only uses and defines FPRs.
453 const MachineInstr &MI) {
454 // TODO: Add more intrinsics.
456 default:
457 return false;
458 case Intrinsic::aarch64_neon_uaddlv:
459 case Intrinsic::aarch64_neon_uaddv:
460 case Intrinsic::aarch64_neon_saddv:
461 case Intrinsic::aarch64_neon_umaxv:
462 case Intrinsic::aarch64_neon_smaxv:
463 case Intrinsic::aarch64_neon_uminv:
464 case Intrinsic::aarch64_neon_sminv:
465 case Intrinsic::aarch64_neon_faddv:
466 case Intrinsic::aarch64_neon_fmaxv:
467 case Intrinsic::aarch64_neon_fminv:
468 case Intrinsic::aarch64_neon_fmaxnmv:
469 case Intrinsic::aarch64_neon_fminnmv:
470 case Intrinsic::aarch64_neon_fmulx:
471 case Intrinsic::aarch64_neon_frecpe:
472 case Intrinsic::aarch64_neon_frecps:
473 case Intrinsic::aarch64_neon_frecpx:
474 case Intrinsic::aarch64_neon_frsqrte:
475 case Intrinsic::aarch64_neon_frsqrts:
476 case Intrinsic::aarch64_neon_facge:
477 case Intrinsic::aarch64_neon_facgt:
478 case Intrinsic::aarch64_neon_fabd:
479 case Intrinsic::aarch64_sisd_fabd:
480 case Intrinsic::aarch64_neon_sqrdmlah:
481 case Intrinsic::aarch64_neon_sqrdmlsh:
482 case Intrinsic::aarch64_neon_sqrdmulh:
483 case Intrinsic::aarch64_neon_sqadd:
484 case Intrinsic::aarch64_neon_sqsub:
485 case Intrinsic::aarch64_crypto_sha1h:
486 case Intrinsic::aarch64_crypto_sha1c:
487 case Intrinsic::aarch64_crypto_sha1p:
488 case Intrinsic::aarch64_crypto_sha1m:
489 case Intrinsic::aarch64_sisd_fcvtxn:
490 return true;
491 case Intrinsic::aarch64_neon_saddlv: {
492 const LLT SrcTy = MRI.getType(MI.getOperand(2).getReg());
493 return SrcTy.getElementType().getSizeInBits() >= 16 &&
494 SrcTy.getElementCount().getFixedValue() >= 4;
495 }
496 }
497}
498
499bool AArch64RegisterBankInfo::isPHIWithFPConstraints(
500 const MachineInstr &MI, const MachineRegisterInfo &MRI,
501 const AArch64RegisterInfo &TRI, const unsigned Depth) const {
502 if (!MI.isPHI() || Depth > MaxFPRSearchDepth)
503 return false;
504
505 return any_of(MRI.use_nodbg_instructions(MI.getOperand(0).getReg()),
506 [&](const MachineInstr &UseMI) {
507 if (onlyUsesFP(UseMI, MRI, TRI, Depth + 1))
508 return true;
509 return isPHIWithFPConstraints(UseMI, MRI, TRI, Depth + 1);
510 });
511}
512
513bool AArch64RegisterBankInfo::hasFPConstraints(const MachineInstr &MI,
516 unsigned Depth) const {
517 unsigned Op = MI.getOpcode();
518 if (Op == TargetOpcode::G_INTRINSIC && isFPIntrinsic(MRI, MI))
519 return true;
520
521 // Do we have an explicit floating point instruction?
523 return true;
524
525 // No. Check if we have a copy-like instruction. If we do, then we could
526 // still be fed by floating point instructions.
527 if (Op != TargetOpcode::COPY && !MI.isPHI() &&
529 return false;
530
531 // Check if we already know the register bank.
532 auto *RB = getRegBank(MI.getOperand(0).getReg(), MRI, TRI);
533 if (RB == &AArch64::FPRRegBank)
534 return true;
535 if (RB == &AArch64::GPRRegBank)
536 return false;
537
538 // We don't know anything.
539 //
540 // If we have a phi, we may be able to infer that it will be assigned a FPR
541 // based off of its inputs.
542 if (!MI.isPHI() || Depth > MaxFPRSearchDepth)
543 return false;
544
545 return any_of(MI.explicit_uses(), [&](const MachineOperand &Op) {
546 return Op.isReg() &&
547 onlyDefinesFP(*MRI.getVRegDef(Op.getReg()), MRI, TRI, Depth + 1);
548 });
549}
550
551bool AArch64RegisterBankInfo::onlyUsesFP(const MachineInstr &MI,
554 unsigned Depth) const {
555 switch (MI.getOpcode()) {
556 case TargetOpcode::G_FPTOSI:
557 case TargetOpcode::G_FPTOUI:
558 case TargetOpcode::G_FPTOSI_SAT:
559 case TargetOpcode::G_FPTOUI_SAT:
560 case TargetOpcode::G_FCMP:
561 case TargetOpcode::G_LROUND:
562 case TargetOpcode::G_LLROUND:
563 return true;
564 case TargetOpcode::G_INTRINSIC:
566 case Intrinsic::aarch64_neon_fcvtas:
567 case Intrinsic::aarch64_neon_fcvtau:
568 case Intrinsic::aarch64_neon_fcvtzs:
569 case Intrinsic::aarch64_neon_fcvtzu:
570 case Intrinsic::aarch64_neon_fcvtms:
571 case Intrinsic::aarch64_neon_fcvtmu:
572 case Intrinsic::aarch64_neon_fcvtns:
573 case Intrinsic::aarch64_neon_fcvtnu:
574 case Intrinsic::aarch64_neon_fcvtps:
575 case Intrinsic::aarch64_neon_fcvtpu:
576 return true;
577 default:
578 break;
579 }
580 break;
581 default:
582 break;
583 }
584 return hasFPConstraints(MI, MRI, TRI, Depth);
585}
586
587bool AArch64RegisterBankInfo::onlyDefinesFP(const MachineInstr &MI,
590 unsigned Depth) const {
591 switch (MI.getOpcode()) {
592 case AArch64::G_DUP:
593 case AArch64::G_SADDLP:
594 case AArch64::G_UADDLP:
595 case TargetOpcode::G_SITOFP:
596 case TargetOpcode::G_UITOFP:
597 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
598 case TargetOpcode::G_INSERT_VECTOR_ELT:
599 case TargetOpcode::G_BUILD_VECTOR:
600 case TargetOpcode::G_BUILD_VECTOR_TRUNC:
601 return true;
602 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
604 case Intrinsic::aarch64_neon_ld1x2:
605 case Intrinsic::aarch64_neon_ld1x3:
606 case Intrinsic::aarch64_neon_ld1x4:
607 case Intrinsic::aarch64_neon_ld2:
608 case Intrinsic::aarch64_neon_ld2lane:
609 case Intrinsic::aarch64_neon_ld2r:
610 case Intrinsic::aarch64_neon_ld3:
611 case Intrinsic::aarch64_neon_ld3lane:
612 case Intrinsic::aarch64_neon_ld3r:
613 case Intrinsic::aarch64_neon_ld4:
614 case Intrinsic::aarch64_neon_ld4lane:
615 case Intrinsic::aarch64_neon_ld4r:
616 return true;
617 default:
618 break;
619 }
620 break;
621 default:
622 break;
623 }
624 return hasFPConstraints(MI, MRI, TRI, Depth);
625}
626
627bool AArch64RegisterBankInfo::prefersFPUse(const MachineInstr &MI,
630 unsigned Depth) const {
631 switch (MI.getOpcode()) {
632 case TargetOpcode::G_SITOFP:
633 case TargetOpcode::G_UITOFP:
634 return MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() ==
635 MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
636 }
637 return onlyDefinesFP(MI, MRI, TRI, Depth);
638}
639
640bool AArch64RegisterBankInfo::isLoadFromFPType(const MachineInstr &MI) const {
641 // GMemOperation because we also want to match indexed loads.
642 auto *MemOp = cast<GMemOperation>(&MI);
643 const Value *LdVal = MemOp->getMMO().getValue();
644 if (!LdVal)
645 return false;
646
647 Type *EltTy = nullptr;
648 if (const GlobalValue *GV = dyn_cast<GlobalValue>(LdVal)) {
649 EltTy = GV->getValueType();
650 // Look at the first element of the struct to determine the type we are
651 // loading
652 while (StructType *StructEltTy = dyn_cast<StructType>(EltTy)) {
653 if (StructEltTy->getNumElements() == 0)
654 break;
655 EltTy = StructEltTy->getTypeAtIndex(0U);
656 }
657 // Look at the first element of the array to determine its type
658 if (isa<ArrayType>(EltTy))
659 EltTy = EltTy->getArrayElementType();
660 } else if (!isa<Constant>(LdVal)) {
661 // FIXME: grubbing around uses is pretty ugly, but with no more
662 // `getPointerElementType` there's not much else we can do.
663 for (const auto *LdUser : LdVal->users()) {
664 if (isa<LoadInst>(LdUser)) {
665 EltTy = LdUser->getType();
666 break;
667 }
668 if (isa<StoreInst>(LdUser) && LdUser->getOperand(1) == LdVal) {
669 EltTy = LdUser->getOperand(0)->getType();
670 break;
671 }
672 }
673 }
674 return EltTy && EltTy->isFPOrFPVectorTy();
675}
676
679 const unsigned Opc = MI.getOpcode();
680
681 // Try the default logic for non-generic instructions that are either copies
682 // or already have some operands assigned to banks.
683 if ((Opc != TargetOpcode::COPY && !isPreISelGenericOpcode(Opc)) ||
684 Opc == TargetOpcode::G_PHI) {
687 if (Mapping.isValid())
688 return Mapping;
689 }
690
691 const MachineFunction &MF = *MI.getParent()->getParent();
692 const MachineRegisterInfo &MRI = MF.getRegInfo();
695
696 switch (Opc) {
697 // G_{F|S|U}REM are not listed because they are not legal.
698 // Arithmetic ops.
699 case TargetOpcode::G_ADD:
700 case TargetOpcode::G_SUB:
701 case TargetOpcode::G_PTR_ADD:
702 case TargetOpcode::G_MUL:
703 case TargetOpcode::G_SDIV:
704 case TargetOpcode::G_UDIV:
705 // Bitwise ops.
706 case TargetOpcode::G_AND:
707 case TargetOpcode::G_OR:
708 case TargetOpcode::G_XOR:
709 // Floating point ops.
710 case TargetOpcode::G_FADD:
711 case TargetOpcode::G_FSUB:
712 case TargetOpcode::G_FMUL:
713 case TargetOpcode::G_FDIV:
714 case TargetOpcode::G_FMAXIMUM:
715 case TargetOpcode::G_FMINIMUM:
716 return getSameKindOfOperandsMapping(MI);
717 case TargetOpcode::G_FPEXT: {
718 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
719 LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());
721 DefaultMappingID, /*Cost*/ 1,
722 getFPExtMapping(DstTy.getSizeInBits(), SrcTy.getSizeInBits()),
723 /*NumOperands*/ 2);
724 }
725 // Shifts.
726 case TargetOpcode::G_SHL:
727 case TargetOpcode::G_LSHR:
728 case TargetOpcode::G_ASHR: {
729 LLT ShiftAmtTy = MRI.getType(MI.getOperand(2).getReg());
730 LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());
731 if (ShiftAmtTy.getSizeInBits() == 64 && SrcTy.getSizeInBits() == 32)
734 return getSameKindOfOperandsMapping(MI);
735 }
736 case TargetOpcode::COPY: {
737 Register DstReg = MI.getOperand(0).getReg();
738 Register SrcReg = MI.getOperand(1).getReg();
739 // Check if one of the register is not a generic register.
740 if ((DstReg.isPhysical() || !MRI.getType(DstReg).isValid()) ||
741 (SrcReg.isPhysical() || !MRI.getType(SrcReg).isValid())) {
742 const RegisterBank *DstRB = getRegBank(DstReg, MRI, TRI);
743 const RegisterBank *SrcRB = getRegBank(SrcReg, MRI, TRI);
744 if (!DstRB)
745 DstRB = SrcRB;
746 else if (!SrcRB)
747 SrcRB = DstRB;
748 // If both RB are null that means both registers are generic.
749 // We shouldn't be here.
750 assert(DstRB && SrcRB && "Both RegBank were nullptr");
751 TypeSize Size = getSizeInBits(DstReg, MRI, TRI);
753 DefaultMappingID, copyCost(*DstRB, *SrcRB, Size),
754 getCopyMapping(DstRB->getID(), SrcRB->getID(), Size),
755 // We only care about the mapping of the destination.
756 /*NumOperands*/ 1);
757 }
758 // Both registers are generic, use G_BITCAST.
759 [[fallthrough]];
760 }
761 case TargetOpcode::G_BITCAST: {
762 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
763 LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());
764 TypeSize Size = DstTy.getSizeInBits();
765 bool DstIsGPR = !DstTy.isVector() && DstTy.getSizeInBits() <= 64;
766 bool SrcIsGPR = !SrcTy.isVector() && SrcTy.getSizeInBits() <= 64;
767 const RegisterBank &DstRB =
768 DstIsGPR ? AArch64::GPRRegBank : AArch64::FPRRegBank;
769 const RegisterBank &SrcRB =
770 SrcIsGPR ? AArch64::GPRRegBank : AArch64::FPRRegBank;
772 DefaultMappingID, copyCost(DstRB, SrcRB, Size),
773 getCopyMapping(DstRB.getID(), SrcRB.getID(), Size),
774 // We only care about the mapping of the destination for COPY.
775 /*NumOperands*/ Opc == TargetOpcode::G_BITCAST ? 2 : 1);
776 }
777 default:
778 break;
779 }
780
781 unsigned NumOperands = MI.getNumOperands();
782 unsigned MappingID = DefaultMappingID;
783
784 // Track the size and bank of each register. We don't do partial mappings.
785 SmallVector<unsigned, 4> OpSize(NumOperands);
786 SmallVector<PartialMappingIdx, 4> OpRegBankIdx(NumOperands);
787 for (unsigned Idx = 0; Idx < NumOperands; ++Idx) {
788 auto &MO = MI.getOperand(Idx);
789 if (!MO.isReg() || !MO.getReg())
790 continue;
791
792 LLT Ty = MRI.getType(MO.getReg());
793 if (!Ty.isValid())
794 continue;
795 OpSize[Idx] = Ty.getSizeInBits().getKnownMinValue();
796
797 // As a top-level guess, vectors including both scalable and non-scalable
798 // ones go in FPRs, scalars and pointers in GPRs.
799 // For floating-point instructions, scalars go in FPRs.
800 if (Ty.isVector())
801 OpRegBankIdx[Idx] = PMI_FirstFPR;
803 (MO.isDef() && onlyDefinesFP(MI, MRI, TRI)) ||
804 (MO.isUse() && onlyUsesFP(MI, MRI, TRI)) ||
805 Ty.getSizeInBits() > 64)
806 OpRegBankIdx[Idx] = PMI_FirstFPR;
807 else
808 OpRegBankIdx[Idx] = PMI_FirstGPR;
809 }
810
811 unsigned Cost = 1;
812 // Some of the floating-point instructions have mixed GPR and FPR operands:
813 // fine-tune the computed mapping.
814 switch (Opc) {
815 case AArch64::G_DUP: {
816 Register ScalarReg = MI.getOperand(1).getReg();
817 LLT ScalarTy = MRI.getType(ScalarReg);
818 auto ScalarDef = MRI.getVRegDef(ScalarReg);
819 // We want to select dup(load) into LD1R.
820 if (ScalarDef->getOpcode() == TargetOpcode::G_LOAD)
821 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};
822 // s8 is an exception for G_DUP, which we always want on gpr.
823 else if (ScalarTy.getSizeInBits() != 8 &&
824 (getRegBank(ScalarReg, MRI, TRI) == &AArch64::FPRRegBank ||
825 onlyDefinesFP(*ScalarDef, MRI, TRI)))
826 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};
827 else {
828 if (ScalarTy.getSizeInBits() < 32 &&
829 getRegBank(ScalarReg, MRI, TRI) == &AArch64::GPRRegBank) {
830 // Calls applyMappingImpl()
831 MappingID = CustomMappingID;
832 }
833 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstGPR};
834 }
835 break;
836 }
837 case TargetOpcode::G_TRUNC: {
838 LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());
839 if (!SrcTy.isVector() && SrcTy.getSizeInBits() == 128)
840 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};
841 break;
842 }
843 case TargetOpcode::G_SITOFP:
844 case TargetOpcode::G_UITOFP: {
845 if (MRI.getType(MI.getOperand(0).getReg()).isVector())
846 break;
847 // Integer to FP conversions don't necessarily happen between GPR -> FPR
848 // regbanks. They can also be done within an FPR register.
849 Register SrcReg = MI.getOperand(1).getReg();
850 if (getRegBank(SrcReg, MRI, TRI) == &AArch64::FPRRegBank &&
851 MRI.getType(SrcReg).getSizeInBits() ==
852 MRI.getType(MI.getOperand(0).getReg()).getSizeInBits())
853 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};
854 else
855 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstGPR};
856 break;
857 }
858 case TargetOpcode::G_FPTOSI_SAT:
859 case TargetOpcode::G_FPTOUI_SAT: {
860 LLT DstType = MRI.getType(MI.getOperand(0).getReg());
861 if (DstType.isVector())
862 break;
863 if (DstType == LLT::scalar(16)) {
864 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};
865 break;
866 }
867 OpRegBankIdx = {PMI_FirstGPR, PMI_FirstFPR};
868 break;
869 }
870 case TargetOpcode::G_FPTOSI:
871 case TargetOpcode::G_FPTOUI:
872 case TargetOpcode::G_INTRINSIC_LRINT:
873 case TargetOpcode::G_INTRINSIC_LLRINT:
874 if (MRI.getType(MI.getOperand(0).getReg()).isVector())
875 break;
876 OpRegBankIdx = {PMI_FirstGPR, PMI_FirstFPR};
877 break;
878 case TargetOpcode::G_FCMP: {
879 // If the result is a vector, it must use a FPR.
881 MRI.getType(MI.getOperand(0).getReg()).isVector() ? PMI_FirstFPR
882 : PMI_FirstGPR;
883 OpRegBankIdx = {Idx0,
884 /* Predicate */ PMI_None, PMI_FirstFPR, PMI_FirstFPR};
885 break;
886 }
887 case TargetOpcode::G_BITCAST:
888 // This is going to be a cross register bank copy and this is expensive.
889 if (OpRegBankIdx[0] != OpRegBankIdx[1])
890 Cost = copyCost(
891 *AArch64GenRegisterBankInfo::PartMappings[OpRegBankIdx[0]].RegBank,
892 *AArch64GenRegisterBankInfo::PartMappings[OpRegBankIdx[1]].RegBank,
893 TypeSize::getFixed(OpSize[0]));
894 break;
895 case TargetOpcode::G_LOAD: {
896 // Loading in vector unit is slightly more expensive.
897 // This is actually only true for the LD1R and co instructions,
898 // but anyway for the fast mode this number does not matter and
899 // for the greedy mode the cost of the cross bank copy will
900 // offset this number.
901 // FIXME: Should be derived from the scheduling model.
902 if (OpRegBankIdx[0] != PMI_FirstGPR) {
903 Cost = 2;
904 break;
905 }
906
907 if (cast<GLoad>(MI).isAtomic()) {
908 // Atomics always use GPR destinations. Don't refine any further.
909 OpRegBankIdx[0] = PMI_FirstGPR;
910 break;
911 }
912
913 // Try to guess the type of the load from the MMO.
914 if (isLoadFromFPType(MI)) {
915 OpRegBankIdx[0] = PMI_FirstFPR;
916 break;
917 }
918
919 // Check if that load feeds fp instructions.
920 // In that case, we want the default mapping to be on FPR
921 // instead of blind map every scalar to GPR.
922 if (any_of(MRI.use_nodbg_instructions(MI.getOperand(0).getReg()),
923 [&](const MachineInstr &UseMI) {
924 // If we have at least one direct or indirect use
925 // in a FP instruction,
926 // assume this was a floating point load in the IR. If it was
927 // not, we would have had a bitcast before reaching that
928 // instruction.
929 //
930 // Int->FP conversion operations are also captured in
931 // prefersFPUse().
932
933 if (isPHIWithFPConstraints(UseMI, MRI, TRI))
934 return true;
935
936 return onlyUsesFP(UseMI, MRI, TRI) ||
937 prefersFPUse(UseMI, MRI, TRI);
938 }))
939 OpRegBankIdx[0] = PMI_FirstFPR;
940 break;
941 }
942 case TargetOpcode::G_STORE:
943 // Check if that store is fed by fp instructions.
944 if (OpRegBankIdx[0] == PMI_FirstGPR) {
945 Register VReg = MI.getOperand(0).getReg();
946 if (!VReg)
947 break;
948 MachineInstr *DefMI = MRI.getVRegDef(VReg);
949 if (onlyDefinesFP(*DefMI, MRI, TRI))
950 OpRegBankIdx[0] = PMI_FirstFPR;
951 break;
952 }
953 break;
954 case TargetOpcode::G_INDEXED_STORE:
955 if (OpRegBankIdx[1] == PMI_FirstGPR) {
956 Register VReg = MI.getOperand(1).getReg();
957 if (!VReg)
958 break;
959 MachineInstr *DefMI = MRI.getVRegDef(VReg);
960 if (onlyDefinesFP(*DefMI, MRI, TRI))
961 OpRegBankIdx[1] = PMI_FirstFPR;
962 break;
963 }
964 break;
965 case TargetOpcode::G_INDEXED_SEXTLOAD:
966 case TargetOpcode::G_INDEXED_ZEXTLOAD:
967 // These should always be GPR.
968 OpRegBankIdx[0] = PMI_FirstGPR;
969 break;
970 case TargetOpcode::G_INDEXED_LOAD: {
971 if (isLoadFromFPType(MI))
972 OpRegBankIdx[0] = PMI_FirstFPR;
973 break;
974 }
975 case TargetOpcode::G_SELECT: {
976 // If the destination is FPR, preserve that.
977 if (OpRegBankIdx[0] != PMI_FirstGPR)
978 break;
979
980 // If we're taking in vectors, we have no choice but to put everything on
981 // FPRs, except for the condition. The condition must always be on a GPR.
982 LLT SrcTy = MRI.getType(MI.getOperand(2).getReg());
983 if (SrcTy.isVector()) {
985 break;
986 }
987
988 // Try to minimize the number of copies. If we have more floating point
989 // constrained values than not, then we'll put everything on FPR. Otherwise,
990 // everything has to be on GPR.
991 unsigned NumFP = 0;
992
993 // Check if the uses of the result always produce floating point values.
994 //
995 // For example:
996 //
997 // %z = G_SELECT %cond %x %y
998 // fpr = G_FOO %z ...
999 if (any_of(MRI.use_nodbg_instructions(MI.getOperand(0).getReg()),
1000 [&](MachineInstr &MI) { return onlyUsesFP(MI, MRI, TRI); }))
1001 ++NumFP;
1002
1003 // Check if the defs of the source values always produce floating point
1004 // values.
1005 //
1006 // For example:
1007 //
1008 // %x = G_SOMETHING_ALWAYS_FLOAT %a ...
1009 // %z = G_SELECT %cond %x %y
1010 //
1011 // Also check whether or not the sources have already been decided to be
1012 // FPR. Keep track of this.
1013 //
1014 // This doesn't check the condition, since it's just whatever is in NZCV.
1015 // This isn't passed explicitly in a register to fcsel/csel.
1016 for (unsigned Idx = 2; Idx < 4; ++Idx) {
1017 Register VReg = MI.getOperand(Idx).getReg();
1018 MachineInstr *DefMI = MRI.getVRegDef(VReg);
1019 if (getRegBank(VReg, MRI, TRI) == &AArch64::FPRRegBank ||
1020 onlyDefinesFP(*DefMI, MRI, TRI))
1021 ++NumFP;
1022 }
1023
1024 // If we have more FP constraints than not, then move everything over to
1025 // FPR.
1026 if (NumFP >= 2)
1028
1029 break;
1030 }
1031 case TargetOpcode::G_UNMERGE_VALUES: {
1032 // If the first operand belongs to a FPR register bank, then make sure that
1033 // we preserve that.
1034 if (OpRegBankIdx[0] != PMI_FirstGPR)
1035 break;
1036
1037 LLT SrcTy = MRI.getType(MI.getOperand(MI.getNumOperands()-1).getReg());
1038 // UNMERGE into scalars from a vector should always use FPR.
1039 // Likewise if any of the uses are FP instructions.
1040 if (SrcTy.isVector() || SrcTy == LLT::scalar(128) ||
1041 any_of(MRI.use_nodbg_instructions(MI.getOperand(0).getReg()),
1042 [&](MachineInstr &MI) { return onlyUsesFP(MI, MRI, TRI); })) {
1043 // Set the register bank of every operand to FPR.
1044 for (unsigned Idx = 0, NumOperands = MI.getNumOperands();
1045 Idx < NumOperands; ++Idx)
1046 OpRegBankIdx[Idx] = PMI_FirstFPR;
1047 }
1048 break;
1049 }
1050 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
1051 // Destination and source need to be FPRs.
1052 OpRegBankIdx[0] = PMI_FirstFPR;
1053 OpRegBankIdx[1] = PMI_FirstFPR;
1054
1055 // Index needs to be a GPR.
1056 OpRegBankIdx[2] = PMI_FirstGPR;
1057 break;
1058 case TargetOpcode::G_INSERT_VECTOR_ELT:
1059 OpRegBankIdx[0] = PMI_FirstFPR;
1060 OpRegBankIdx[1] = PMI_FirstFPR;
1061
1062 // The element may be either a GPR or FPR. Preserve that behaviour.
1063 if (getRegBank(MI.getOperand(2).getReg(), MRI, TRI) == &AArch64::FPRRegBank)
1064 OpRegBankIdx[2] = PMI_FirstFPR;
1065 else {
1066 // If the type is i8/i16, and the regank will be GPR, then we change the
1067 // type to i32 in applyMappingImpl.
1068 LLT Ty = MRI.getType(MI.getOperand(2).getReg());
1069 if (Ty.getSizeInBits() == 8 || Ty.getSizeInBits() == 16) {
1070 // Calls applyMappingImpl()
1071 MappingID = CustomMappingID;
1072 }
1073 OpRegBankIdx[2] = PMI_FirstGPR;
1074 }
1075
1076 // Index needs to be a GPR.
1077 OpRegBankIdx[3] = PMI_FirstGPR;
1078 break;
1079 case TargetOpcode::G_EXTRACT: {
1080 // For s128 sources we have to use fpr unless we know otherwise.
1081 auto Src = MI.getOperand(1).getReg();
1082 LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());
1083 if (SrcTy.getSizeInBits() != 128)
1084 break;
1085 auto Idx = MRI.getRegClassOrNull(Src) == &AArch64::XSeqPairsClassRegClass
1086 ? PMI_FirstGPR
1087 : PMI_FirstFPR;
1088 OpRegBankIdx[0] = Idx;
1089 OpRegBankIdx[1] = Idx;
1090 break;
1091 }
1092 case TargetOpcode::G_BUILD_VECTOR: {
1093 // If the first source operand belongs to a FPR register bank, then make
1094 // sure that we preserve that.
1095 if (OpRegBankIdx[1] != PMI_FirstGPR)
1096 break;
1097 Register VReg = MI.getOperand(1).getReg();
1098 if (!VReg)
1099 break;
1100
1101 // Get the instruction that defined the source operand reg, and check if
1102 // it's a floating point operation. Or, if it's a type like s16 which
1103 // doesn't have a exact size gpr register class. The exception is if the
1104 // build_vector has all constant operands, which may be better to leave as
1105 // gpr without copies, so it can be matched in imported patterns.
1106 MachineInstr *DefMI = MRI.getVRegDef(VReg);
1107 unsigned DefOpc = DefMI->getOpcode();
1108 const LLT SrcTy = MRI.getType(VReg);
1109 if (all_of(MI.operands(), [&](const MachineOperand &Op) {
1110 return Op.isDef() || MRI.getVRegDef(Op.getReg())->getOpcode() ==
1111 TargetOpcode::G_CONSTANT;
1112 }))
1113 break;
1115 SrcTy.getSizeInBits() < 32 ||
1116 getRegBank(VReg, MRI, TRI) == &AArch64::FPRRegBank) {
1117 // Have a floating point op.
1118 // Make sure every operand gets mapped to a FPR register class.
1119 unsigned NumOperands = MI.getNumOperands();
1120 for (unsigned Idx = 0; Idx < NumOperands; ++Idx)
1121 OpRegBankIdx[Idx] = PMI_FirstFPR;
1122 }
1123 break;
1124 }
1125 case TargetOpcode::G_VECREDUCE_FADD:
1126 case TargetOpcode::G_VECREDUCE_FMUL:
1127 case TargetOpcode::G_VECREDUCE_FMAX:
1128 case TargetOpcode::G_VECREDUCE_FMIN:
1129 case TargetOpcode::G_VECREDUCE_FMAXIMUM:
1130 case TargetOpcode::G_VECREDUCE_FMINIMUM:
1131 case TargetOpcode::G_VECREDUCE_ADD:
1132 case TargetOpcode::G_VECREDUCE_MUL:
1133 case TargetOpcode::G_VECREDUCE_AND:
1134 case TargetOpcode::G_VECREDUCE_OR:
1135 case TargetOpcode::G_VECREDUCE_XOR:
1136 case TargetOpcode::G_VECREDUCE_SMAX:
1137 case TargetOpcode::G_VECREDUCE_SMIN:
1138 case TargetOpcode::G_VECREDUCE_UMAX:
1139 case TargetOpcode::G_VECREDUCE_UMIN:
1140 // Reductions produce a scalar value from a vector, the scalar should be on
1141 // FPR bank.
1142 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};
1143 break;
1144 case TargetOpcode::G_VECREDUCE_SEQ_FADD:
1145 case TargetOpcode::G_VECREDUCE_SEQ_FMUL:
1146 // These reductions also take a scalar accumulator input.
1147 // Assign them FPR for now.
1148 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR, PMI_FirstFPR};
1149 break;
1150 case TargetOpcode::G_INTRINSIC:
1151 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: {
1152 switch (cast<GIntrinsic>(MI).getIntrinsicID()) {
1153 case Intrinsic::aarch64_neon_fcvtas:
1154 case Intrinsic::aarch64_neon_fcvtau:
1155 case Intrinsic::aarch64_neon_fcvtzs:
1156 case Intrinsic::aarch64_neon_fcvtzu:
1157 case Intrinsic::aarch64_neon_fcvtms:
1158 case Intrinsic::aarch64_neon_fcvtmu:
1159 case Intrinsic::aarch64_neon_fcvtns:
1160 case Intrinsic::aarch64_neon_fcvtnu:
1161 case Intrinsic::aarch64_neon_fcvtps:
1162 case Intrinsic::aarch64_neon_fcvtpu: {
1163 OpRegBankIdx[2] = PMI_FirstFPR;
1164 if (MRI.getType(MI.getOperand(0).getReg()).isVector()) {
1165 OpRegBankIdx[0] = PMI_FirstFPR;
1166 break;
1167 }
1168 TypeSize DstSize = getSizeInBits(MI.getOperand(0).getReg(), MRI, TRI);
1169 TypeSize SrcSize = getSizeInBits(MI.getOperand(2).getReg(), MRI, TRI);
1170 if (((DstSize == SrcSize) || STI.hasFeature(AArch64::FeatureFPRCVT)) &&
1171 all_of(MRI.use_nodbg_instructions(MI.getOperand(0).getReg()),
1172 [&](const MachineInstr &UseMI) {
1173 return onlyUsesFP(UseMI, MRI, TRI) ||
1174 prefersFPUse(UseMI, MRI, TRI);
1175 }))
1176 OpRegBankIdx[0] = PMI_FirstFPR;
1177 else
1178 OpRegBankIdx[0] = PMI_FirstGPR;
1179 break;
1180 }
1181 case Intrinsic::aarch64_neon_vcvtfxs2fp:
1182 case Intrinsic::aarch64_neon_vcvtfxu2fp:
1183 case Intrinsic::aarch64_neon_vcvtfp2fxs:
1184 case Intrinsic::aarch64_neon_vcvtfp2fxu:
1185 // Override these intrinsics, because they would have a partial
1186 // mapping. This is needed for 'half' types, which otherwise don't
1187 // get legalised correctly.
1188 OpRegBankIdx[0] = PMI_FirstFPR;
1189 OpRegBankIdx[2] = PMI_FirstFPR;
1190 // OpRegBankIdx[1] is the intrinsic ID.
1191 // OpRegBankIdx[3] is an integer immediate.
1192 break;
1193 default: {
1194 // Check if we know that the intrinsic has any constraints on its register
1195 // banks. If it does, then update the mapping accordingly.
1196 unsigned Idx = 0;
1197 if (onlyDefinesFP(MI, MRI, TRI))
1198 for (const auto &Op : MI.defs()) {
1199 if (Op.isReg())
1200 OpRegBankIdx[Idx] = PMI_FirstFPR;
1201 ++Idx;
1202 }
1203 else
1204 Idx += MI.getNumExplicitDefs();
1205
1206 if (onlyUsesFP(MI, MRI, TRI))
1207 for (const auto &Op : MI.explicit_uses()) {
1208 if (Op.isReg())
1209 OpRegBankIdx[Idx] = PMI_FirstFPR;
1210 ++Idx;
1211 }
1212 break;
1213 }
1214 }
1215 break;
1216 }
1217 case TargetOpcode::G_LROUND:
1218 case TargetOpcode::G_LLROUND: {
1219 // Source is always floating point and destination is always integer.
1220 OpRegBankIdx = {PMI_FirstGPR, PMI_FirstFPR};
1221 break;
1222 }
1223 }
1224
1225 // Finally construct the computed mapping.
1226 SmallVector<const ValueMapping *, 8> OpdsMapping(NumOperands);
1227 for (unsigned Idx = 0; Idx < NumOperands; ++Idx) {
1228 if (MI.getOperand(Idx).isReg() && MI.getOperand(Idx).getReg()) {
1229 LLT Ty = MRI.getType(MI.getOperand(Idx).getReg());
1230 if (!Ty.isValid())
1231 continue;
1232 auto Mapping =
1233 getValueMapping(OpRegBankIdx[Idx], TypeSize::getFixed(OpSize[Idx]));
1234 if (!Mapping->isValid())
1236
1237 OpdsMapping[Idx] = Mapping;
1238 }
1239 }
1240
1241 return getInstructionMapping(MappingID, Cost, getOperandsMapping(OpdsMapping),
1242 NumOperands);
1243}
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
static unsigned getIntrinsicID(const SDNode *N)
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
#define CHECK_VALUEMAP(RBName, Size)
static bool isFPIntrinsic(const MachineRegisterInfo &MRI, const MachineInstr &MI)
#define CHECK_VALUEMAP_3OPS(RBName, Size)
static const unsigned CustomMappingID
#define CHECK_PARTIALMAP(Idx, ValStartIdx, ValLength, RB)
#define CHECK_VALUEMAP_CROSSREGCPY(RBNameDst, RBNameSrc, Size)
#define CHECK_VALUEMAP_FPEXT(DstSize, SrcSize)
This file declares the targeting of the RegisterBankInfo class for AArch64.
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
IRTranslator LLVM IR MI
Implement a low-level type suitable for MachineInstr level instruction selection.
This file declares the MachineIRBuilder class.
Register const TargetRegisterInfo * TRI
ppc ctr loops verify
static const MCPhysReg FPR[]
FPR - The set of FP registers that should be allocated for arguments on Darwin and AIX.
This file contains some templates that are useful if you are working with the STL at all.
This file defines the SmallVector class.
static unsigned getRegBankBaseIdxOffset(unsigned RBIdx, TypeSize Size)
static const RegisterBankInfo::ValueMapping * getCopyMapping(unsigned DstBankID, unsigned SrcBankID, TypeSize Size)
Get the pointer to the ValueMapping of the operands of a copy instruction from the SrcBankID register...
static bool checkPartialMappingIdx(PartialMappingIdx FirstAlias, PartialMappingIdx LastAlias, ArrayRef< PartialMappingIdx > Order)
static const RegisterBankInfo::PartialMapping PartMappings[]
static const RegisterBankInfo::ValueMapping * getFPExtMapping(unsigned DstSize, unsigned SrcSize)
Get the instruction mapping for G_FPEXT.
static const RegisterBankInfo::ValueMapping * getValueMapping(PartialMappingIdx RBIdx, TypeSize Size)
Get the pointer to the ValueMapping representing the RegisterBank at RBIdx with a size of Size.
static const RegisterBankInfo::ValueMapping ValMappings[]
InstructionMappings getInstrAlternativeMappings(const MachineInstr &MI) const override
Get the alternative mappings for MI.
unsigned copyCost(const RegisterBank &A, const RegisterBank &B, TypeSize Size) const override
Get the cost of a copy from B to A, or put differently, get the cost of A = COPY B.
const RegisterBank & getRegBankFromRegClass(const TargetRegisterClass &RC, LLT Ty) const override
Get a register bank that covers RC.
AArch64RegisterBankInfo(const TargetRegisterInfo &TRI)
const InstructionMapping & getInstrMapping(const MachineInstr &MI) const override
Get the mapping of the different operands of MI on the register bank.
const AArch64RegisterInfo * getRegisterInfo() const override
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
constexpr bool isVector() const
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Helper class to build MachineInstr.
MachineInstrBuilder buildAnyExt(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ANYEXT Op0.
Register getReg(unsigned Idx) const
Get the register for the operand index.
Representation of each machine instruction.
MachineOperand class - Representation of each machine instruction operand.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Helper class that represents how the value of an instruction may be mapped and what is the related co...
bool isValid() const
Check whether this object is valid.
virtual InstructionMappings getInstrAlternativeMappings(const MachineInstr &MI) const
Get the alternative mappings for MI.
const InstructionMapping & getInstructionMapping(unsigned ID, unsigned Cost, const ValueMapping *OperandsMapping, unsigned NumOperands) const
Method to get a uniquely generated InstructionMapping.
static void applyDefaultMapping(const OperandsMapper &OpdMapper)
Helper method to apply something that is like the default mapping.
const InstructionMapping & getInvalidInstructionMapping() const
Method to get a uniquely generated invalid InstructionMapping.
const RegisterBank & getRegBank(unsigned ID)
Get the register bank identified by ID.
unsigned getMaximumSize(unsigned RegBankID) const
Get the maximum size in bits that fits in the given register bank.
TypeSize getSizeInBits(Register Reg, const MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI) const
Get the size in bits of Reg.
virtual const RegisterBank & getRegBankFromRegClass(const TargetRegisterClass &RC, LLT Ty) const
Get a register bank that covers RC.
const ValueMapping * getOperandsMapping(Iterator Begin, Iterator End) const
Get the uniquely generated array of ValueMapping for the elements of between Begin and End.
static const unsigned DefaultMappingID
Identifier used when the related instruction mapping instance is generated by target independent code...
SmallVector< const InstructionMapping *, 4 > InstructionMappings
Convenient type to represent the alternatives for mapping an instruction.
virtual unsigned copyCost(const RegisterBank &A, const RegisterBank &B, TypeSize Size) const
Get the cost of a copy from B to A, or put differently, get the cost of A = COPY B.
const InstructionMapping & getInstrMappingImpl(const MachineInstr &MI) const
Try to get the mapping of MI.
This class implements the register bank concept.
LLVM_ABI bool covers(const TargetRegisterClass &RC) const
Check whether this register bank covers RC.
unsigned getID() const
Get the identifier of this register bank.
Wrapper class representing virtual and physical registers.
Definition Register.h:19
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Definition Register.h:78
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
unsigned getID() const
Return the register class ID number.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
TargetSubtargetInfo - Generic base class for all target subtargets.
virtual const TargetRegisterInfo * getRegisterInfo() const =0
Return the target's register information.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition TypeSize.h:344
Type * getArrayElementType() const
Definition Type.h:408
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
Definition Type.h:225
iterator_range< user_iterator > users()
Definition Value.h:426
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
This is an optimization pass for GlobalISel generic memory operations.
FunctionAddr VTableAddr Value
Definition InstrProf.h:137
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1725
InstructionCost Cost
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
bool isPreISelGenericOpcode(unsigned Opcode)
Check whether the given Opcode is a generic opcode that is not supposed to appear after ISel.
bool isPreISelGenericOptimizationHint(unsigned Opcode)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1732
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
DWARFExpression::Operation Op
void call_once(once_flag &flag, Function &&F, Args &&... ArgList)
Execute the function specified as a parameter once.
Definition Threading.h:86
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
LLVM_ABI bool isPreISelGenericFloatingPointOpcode(unsigned Opc)
Returns whether opcode Opc is a pre-isel generic floating-point opcode, having only floating-point op...
Definition Utils.cpp:1743
The llvm::once_flag structure.
Definition Threading.h:67