57#include "llvm/IR/IntrinsicsAArch64.h"
82class AArch64FastISel final :
public FastISel {
85 using BaseKind =
enum {
91 BaseKind
Kind = RegBase;
97 unsigned OffsetReg = 0;
105 void setKind(BaseKind K) {
Kind =
K; }
106 BaseKind getKind()
const {
return Kind; }
109 bool isRegBase()
const {
return Kind == RegBase; }
110 bool isFIBase()
const {
return Kind == FrameIndexBase; }
112 void setReg(
unsigned Reg) {
113 assert(isRegBase() &&
"Invalid base register access!");
118 assert(isRegBase() &&
"Invalid base register access!");
122 void setOffsetReg(
unsigned Reg) {
126 unsigned getOffsetReg()
const {
130 void setFI(
unsigned FI) {
131 assert(isFIBase() &&
"Invalid base frame index access!");
135 unsigned getFI()
const {
136 assert(isFIBase() &&
"Invalid base frame index access!");
140 void setOffset(int64_t O) {
Offset =
O; }
142 void setShift(
unsigned S) { Shift = S; }
143 unsigned getShift() {
return Shift; }
172 bool selectRem(
const Instruction *
I,
unsigned ISDOpcode);
185 bool isTypeLegal(
Type *Ty,
MVT &VT);
186 bool isTypeSupported(
Type *Ty,
MVT &VT,
bool IsVectorAllowed =
false);
187 bool isValueAvailable(
const Value *V)
const;
188 bool computeAddress(
const Value *Obj, Address &
Addr,
Type *Ty =
nullptr);
189 bool computeCallAddress(
const Value *V, Address &
Addr);
190 bool simplifyAddress(Address &
Addr,
MVT VT);
195 bool tryEmitSmallMemCpy(Address Dest, Address Src,
uint64_t Len,
204 unsigned emitAddSub(
bool UseAdd,
MVT RetVT,
const Value *LHS,
205 const Value *RHS,
bool SetFlags =
false,
206 bool WantResult =
true,
bool IsZExt =
false);
207 unsigned emitAddSub_rr(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
208 unsigned RHSReg,
bool SetFlags =
false,
209 bool WantResult =
true);
210 unsigned emitAddSub_ri(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
211 uint64_t Imm,
bool SetFlags =
false,
212 bool WantResult =
true);
213 unsigned emitAddSub_rs(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
215 uint64_t ShiftImm,
bool SetFlags =
false,
216 bool WantResult =
true);
217 unsigned emitAddSub_rx(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
219 uint64_t ShiftImm,
bool SetFlags =
false,
220 bool WantResult =
true);
223 bool emitCompareAndBranch(
const BranchInst *BI);
225 bool emitICmp(
MVT RetVT,
const Value *LHS,
const Value *RHS,
bool IsZExt);
226 bool emitICmp_ri(
MVT RetVT,
unsigned LHSReg,
uint64_t Imm);
227 bool emitFCmp(
MVT RetVT,
const Value *LHS,
const Value *RHS);
232 bool emitStoreRelease(
MVT VT,
unsigned SrcReg,
unsigned AddrReg,
234 unsigned emitIntExt(
MVT SrcVT,
unsigned SrcReg,
MVT DestVT,
bool isZExt);
235 unsigned emiti1Ext(
unsigned SrcReg,
MVT DestVT,
bool isZExt);
236 unsigned emitAdd(
MVT RetVT,
const Value *LHS,
const Value *RHS,
237 bool SetFlags =
false,
bool WantResult =
true,
238 bool IsZExt =
false);
239 unsigned emitAdd_ri_(
MVT VT,
unsigned Op0, int64_t Imm);
240 unsigned emitSub(
MVT RetVT,
const Value *LHS,
const Value *RHS,
241 bool SetFlags =
false,
bool WantResult =
true,
242 bool IsZExt =
false);
243 unsigned emitSubs_rr(
MVT RetVT,
unsigned LHSReg,
unsigned RHSReg,
244 bool WantResult =
true);
245 unsigned emitSubs_rs(
MVT RetVT,
unsigned LHSReg,
unsigned RHSReg,
247 bool WantResult =
true);
248 unsigned emitLogicalOp(
unsigned ISDOpc,
MVT RetVT,
const Value *LHS,
250 unsigned emitLogicalOp_ri(
unsigned ISDOpc,
MVT RetVT,
unsigned LHSReg,
252 unsigned emitLogicalOp_rs(
unsigned ISDOpc,
MVT RetVT,
unsigned LHSReg,
253 unsigned RHSReg,
uint64_t ShiftImm);
254 unsigned emitAnd_ri(
MVT RetVT,
unsigned LHSReg,
uint64_t Imm);
255 unsigned emitMul_rr(
MVT RetVT,
unsigned Op0,
unsigned Op1);
256 unsigned emitSMULL_rr(
MVT RetVT,
unsigned Op0,
unsigned Op1);
257 unsigned emitUMULL_rr(
MVT RetVT,
unsigned Op0,
unsigned Op1);
258 unsigned emitLSL_rr(
MVT RetVT,
unsigned Op0Reg,
unsigned Op1Reg);
259 unsigned emitLSL_ri(
MVT RetVT,
MVT SrcVT,
unsigned Op0Reg,
uint64_t Imm,
261 unsigned emitLSR_rr(
MVT RetVT,
unsigned Op0Reg,
unsigned Op1Reg);
262 unsigned emitLSR_ri(
MVT RetVT,
MVT SrcVT,
unsigned Op0Reg,
uint64_t Imm,
264 unsigned emitASR_rr(
MVT RetVT,
unsigned Op0Reg,
unsigned Op1Reg);
265 unsigned emitASR_ri(
MVT RetVT,
MVT SrcVT,
unsigned Op0Reg,
uint64_t Imm,
266 bool IsZExt =
false);
277 bool finishCall(CallLoweringInfo &CLI,
unsigned NumBytes);
294#include "AArch64GenFastISel.inc"
301 assert((isa<ZExtInst>(
I) || isa<SExtInst>(
I)) &&
302 "Unexpected integer extend instruction.");
303 assert(!
I->getType()->isVectorTy() &&
I->getType()->isIntegerTy() &&
304 "Unexpected value type.");
305 bool IsZExt = isa<ZExtInst>(
I);
307 if (
const auto *LI = dyn_cast<LoadInst>(
I->getOperand(0)))
311 if (
const auto *Arg = dyn_cast<Argument>(
I->getOperand(0)))
312 if ((IsZExt && Arg->hasZExtAttr()) || (!IsZExt && Arg->hasSExtAttr()))
343 if (Subtarget->isTargetDarwin())
345 if (Subtarget->isTargetWindows())
350unsigned AArch64FastISel::fastMaterializeAlloca(
const AllocaInst *AI) {
352 "Alloca should always return a pointer.");
355 if (!FuncInfo.StaticAllocaMap.count(AI))
359 FuncInfo.StaticAllocaMap.find(AI);
361 if (SI != FuncInfo.StaticAllocaMap.end()) {
362 Register ResultReg = createResultReg(&AArch64::GPR64spRegClass);
363 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::ADDXri),
374unsigned AArch64FastISel::materializeInt(
const ConstantInt *CI,
MVT VT) {
383 : &AArch64::GPR32RegClass;
384 unsigned ZeroReg = (VT == MVT::i64) ? AArch64::XZR : AArch64::WZR;
385 Register ResultReg = createResultReg(RC);
386 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(TargetOpcode::COPY),
391unsigned AArch64FastISel::materializeFP(
const ConstantFP *CFP,
MVT VT) {
395 return fastMaterializeFloatZero(CFP);
397 if (VT != MVT::f32 && VT != MVT::f64)
401 bool Is64Bit = (VT == MVT::f64);
407 unsigned Opc = Is64Bit ? AArch64::FMOVDi : AArch64::FMOVSi;
408 return fastEmitInst_i(Opc, TLI.getRegClassFor(VT), Imm);
413 unsigned Opc1 = Is64Bit ? AArch64::MOVi64imm : AArch64::MOVi32imm;
415 &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
417 Register TmpReg = createResultReg(RC);
418 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(Opc1), TmpReg)
421 Register ResultReg = createResultReg(TLI.getRegClassFor(VT));
422 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
423 TII.get(TargetOpcode::COPY), ResultReg)
433 unsigned CPI = MCP.getConstantPoolIndex(cast<Constant>(CFP), Alignment);
434 Register ADRPReg = createResultReg(&AArch64::GPR64commonRegClass);
435 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::ADRP),
438 unsigned Opc = Is64Bit ? AArch64::LDRDui : AArch64::LDRSui;
439 Register ResultReg = createResultReg(TLI.getRegClassFor(VT));
440 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(Opc), ResultReg)
446unsigned AArch64FastISel::materializeGV(
const GlobalValue *GV) {
453 if (!Subtarget->useSmallAddressing() && !Subtarget->isTargetMachO())
456 unsigned OpFlags = Subtarget->ClassifyGlobalReference(GV,
TM);
458 EVT DestEVT = TLI.getValueType(
DL, GV->
getType(),
true);
462 Register ADRPReg = createResultReg(&AArch64::GPR64commonRegClass);
467 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::ADRP),
472 if (Subtarget->isTargetILP32()) {
473 ResultReg = createResultReg(&AArch64::GPR32RegClass);
474 LdrOpc = AArch64::LDRWui;
476 ResultReg = createResultReg(&AArch64::GPR64RegClass);
477 LdrOpc = AArch64::LDRXui;
479 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(LdrOpc),
484 if (!Subtarget->isTargetILP32())
489 Register Result64 = createResultReg(&AArch64::GPR64RegClass);
490 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
491 TII.get(TargetOpcode::SUBREG_TO_REG))
499 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::ADRP),
517 unsigned DstReg = createResultReg(&AArch64::GPR64commonRegClass);
518 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::MOVKXi),
527 ResultReg = createResultReg(&AArch64::GPR64spRegClass);
528 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::ADDXri),
538unsigned AArch64FastISel::fastMaterializeConstant(
const Constant *
C) {
539 EVT CEVT = TLI.getValueType(
DL,
C->getType(),
true);
547 if (isa<ConstantPointerNull>(
C)) {
548 assert(VT == MVT::i64 &&
"Expected 64-bit pointers");
552 if (
const auto *CI = dyn_cast<ConstantInt>(
C))
553 return materializeInt(CI, VT);
554 else if (
const ConstantFP *CFP = dyn_cast<ConstantFP>(
C))
555 return materializeFP(CFP, VT);
556 else if (
const GlobalValue *GV = dyn_cast<GlobalValue>(
C))
557 return materializeGV(GV);
562unsigned AArch64FastISel::fastMaterializeFloatZero(
const ConstantFP* CFP) {
564 "Floating-point constant is not a positive zero.");
566 if (!isTypeLegal(CFP->
getType(), VT))
569 if (VT != MVT::f32 && VT != MVT::f64)
572 bool Is64Bit = (VT == MVT::f64);
573 unsigned ZReg = Is64Bit ? AArch64::XZR : AArch64::WZR;
574 unsigned Opc = Is64Bit ? AArch64::FMOVXDr : AArch64::FMOVWSr;
575 return fastEmitInst_r(Opc, TLI.getRegClassFor(VT), ZReg);
580 if (
const auto *
MI = dyn_cast<MulOperator>(
I)) {
581 if (
const auto *
C = dyn_cast<ConstantInt>(
MI->getOperand(0)))
582 if (
C->getValue().isPowerOf2())
584 if (
const auto *
C = dyn_cast<ConstantInt>(
MI->getOperand(1)))
585 if (
C->getValue().isPowerOf2())
592bool AArch64FastISel::computeAddress(
const Value *Obj, Address &
Addr,
Type *Ty)
594 const User *
U =
nullptr;
595 unsigned Opcode = Instruction::UserOp1;
596 if (
const Instruction *
I = dyn_cast<Instruction>(Obj)) {
599 if (FuncInfo.StaticAllocaMap.count(
static_cast<const AllocaInst *
>(Obj)) ||
600 FuncInfo.MBBMap[
I->getParent()] == FuncInfo.MBB) {
601 Opcode =
I->getOpcode();
604 }
else if (
const ConstantExpr *
C = dyn_cast<ConstantExpr>(Obj)) {
605 Opcode =
C->getOpcode();
609 if (
auto *Ty = dyn_cast<PointerType>(Obj->
getType()))
610 if (Ty->getAddressSpace() > 255)
618 case Instruction::BitCast:
620 return computeAddress(
U->getOperand(0),
Addr, Ty);
622 case Instruction::IntToPtr:
624 if (TLI.getValueType(
DL,
U->getOperand(0)->getType()) ==
625 TLI.getPointerTy(
DL))
626 return computeAddress(
U->getOperand(0),
Addr, Ty);
629 case Instruction::PtrToInt:
631 if (TLI.getValueType(
DL,
U->getType()) == TLI.getPointerTy(
DL))
632 return computeAddress(
U->getOperand(0),
Addr, Ty);
635 case Instruction::GetElementPtr: {
643 const Value *
Op = GTI.getOperand();
644 if (
StructType *STy = GTI.getStructTypeOrNull()) {
646 unsigned Idx = cast<ConstantInt>(
Op)->getZExtValue();
649 uint64_t S = GTI.getSequentialElementStride(
DL);
656 if (canFoldAddIntoGEP(U,
Op)) {
659 cast<ConstantInt>(cast<AddOperator>(
Op)->getOperand(1));
662 Op = cast<AddOperator>(
Op)->getOperand(0);
666 goto unsupported_gep;
672 Addr.setOffset(TmpOffset);
673 if (computeAddress(
U->getOperand(0),
Addr, Ty))
682 case Instruction::Alloca: {
685 FuncInfo.StaticAllocaMap.find(AI);
686 if (SI != FuncInfo.StaticAllocaMap.end()) {
687 Addr.setKind(Address::FrameIndexBase);
693 case Instruction::Add: {
698 if (isa<ConstantInt>(LHS))
701 if (
const ConstantInt *CI = dyn_cast<ConstantInt>(RHS)) {
703 return computeAddress(LHS,
Addr, Ty);
707 if (computeAddress(LHS,
Addr, Ty) && computeAddress(RHS,
Addr, Ty))
713 case Instruction::Sub: {
718 if (
const ConstantInt *CI = dyn_cast<ConstantInt>(RHS)) {
720 return computeAddress(LHS,
Addr, Ty);
724 case Instruction::Shl: {
725 if (
Addr.getOffsetReg())
728 const auto *CI = dyn_cast<ConstantInt>(
U->getOperand(1));
733 if (Val < 1 || Val > 3)
739 NumBytes = NumBits / 8;
744 if (NumBytes != (1ULL << Val))
750 const Value *Src =
U->getOperand(0);
751 if (
const auto *
I = dyn_cast<Instruction>(Src)) {
752 if (FuncInfo.MBBMap[
I->getParent()] == FuncInfo.MBB) {
754 if (
const auto *ZE = dyn_cast<ZExtInst>(
I)) {
756 ZE->getOperand(0)->getType()->isIntegerTy(32)) {
758 Src = ZE->getOperand(0);
760 }
else if (
const auto *SE = dyn_cast<SExtInst>(
I)) {
762 SE->getOperand(0)->getType()->isIntegerTy(32)) {
764 Src = SE->getOperand(0);
770 if (
const auto *AI = dyn_cast<BinaryOperator>(Src))
771 if (AI->
getOpcode() == Instruction::And) {
775 if (
const auto *
C = dyn_cast<ConstantInt>(LHS))
776 if (
C->getValue() == 0xffffffff)
779 if (
const auto *
C = dyn_cast<ConstantInt>(RHS))
780 if (
C->getValue() == 0xffffffff) {
785 Reg = fastEmitInst_extractsubreg(MVT::i32, Reg, AArch64::sub_32);
786 Addr.setOffsetReg(Reg);
794 Addr.setOffsetReg(Reg);
797 case Instruction::Mul: {
798 if (
Addr.getOffsetReg())
808 if (
const auto *
C = dyn_cast<ConstantInt>(LHS))
809 if (
C->getValue().isPowerOf2())
812 assert(isa<ConstantInt>(RHS) &&
"Expected an ConstantInt.");
813 const auto *
C = cast<ConstantInt>(RHS);
814 unsigned Val =
C->getValue().logBase2();
815 if (Val < 1 || Val > 3)
821 NumBytes = NumBits / 8;
826 if (NumBytes != (1ULL << Val))
833 if (
const auto *
I = dyn_cast<Instruction>(Src)) {
834 if (FuncInfo.MBBMap[
I->getParent()] == FuncInfo.MBB) {
836 if (
const auto *ZE = dyn_cast<ZExtInst>(
I)) {
838 ZE->getOperand(0)->getType()->isIntegerTy(32)) {
840 Src = ZE->getOperand(0);
842 }
else if (
const auto *SE = dyn_cast<SExtInst>(
I)) {
844 SE->getOperand(0)->getType()->isIntegerTy(32)) {
846 Src = SE->getOperand(0);
855 Addr.setOffsetReg(Reg);
858 case Instruction::And: {
859 if (
Addr.getOffsetReg())
862 if (!Ty ||
DL.getTypeSizeInBits(Ty) != 8)
868 if (
const auto *
C = dyn_cast<ConstantInt>(LHS))
869 if (
C->getValue() == 0xffffffff)
872 if (
const auto *
C = dyn_cast<ConstantInt>(RHS))
873 if (
C->getValue() == 0xffffffff) {
881 Reg = fastEmitInst_extractsubreg(MVT::i32, Reg, AArch64::sub_32);
882 Addr.setOffsetReg(Reg);
887 case Instruction::SExt:
888 case Instruction::ZExt: {
889 if (!
Addr.getReg() ||
Addr.getOffsetReg())
892 const Value *Src =
nullptr;
894 if (
const auto *ZE = dyn_cast<ZExtInst>(U)) {
895 if (!
isIntExtFree(ZE) && ZE->getOperand(0)->getType()->isIntegerTy(32)) {
897 Src = ZE->getOperand(0);
899 }
else if (
const auto *SE = dyn_cast<SExtInst>(U)) {
900 if (!
isIntExtFree(SE) && SE->getOperand(0)->getType()->isIntegerTy(32)) {
902 Src = SE->getOperand(0);
913 Addr.setOffsetReg(Reg);
918 if (
Addr.isRegBase() && !
Addr.getReg()) {
926 if (!
Addr.getOffsetReg()) {
930 Addr.setOffsetReg(Reg);
937bool AArch64FastISel::computeCallAddress(
const Value *V, Address &
Addr) {
938 const User *
U =
nullptr;
939 unsigned Opcode = Instruction::UserOp1;
942 if (
const auto *
I = dyn_cast<Instruction>(V)) {
943 Opcode =
I->getOpcode();
945 InMBB =
I->getParent() == FuncInfo.MBB->getBasicBlock();
946 }
else if (
const auto *
C = dyn_cast<ConstantExpr>(V)) {
947 Opcode =
C->getOpcode();
953 case Instruction::BitCast:
956 return computeCallAddress(
U->getOperand(0),
Addr);
958 case Instruction::IntToPtr:
961 TLI.getValueType(
DL,
U->getOperand(0)->getType()) ==
962 TLI.getPointerTy(
DL))
963 return computeCallAddress(
U->getOperand(0),
Addr);
965 case Instruction::PtrToInt:
967 if (InMBB && TLI.getValueType(
DL,
U->getType()) == TLI.getPointerTy(
DL))
968 return computeCallAddress(
U->getOperand(0),
Addr);
972 if (
const GlobalValue *GV = dyn_cast<GlobalValue>(V)) {
973 Addr.setGlobalValue(GV);
978 if (!
Addr.getGlobalValue()) {
979 Addr.setReg(getRegForValue(V));
980 return Addr.getReg() != 0;
986bool AArch64FastISel::isTypeLegal(
Type *Ty,
MVT &VT) {
987 EVT evt = TLI.getValueType(
DL, Ty,
true);
989 if (Subtarget->isTargetILP32() && Ty->
isPointerTy())
993 if (evt == MVT::Other || !evt.
isSimple())
1003 return TLI.isTypeLegal(VT);
1010bool AArch64FastISel::isTypeSupported(
Type *Ty,
MVT &VT,
bool IsVectorAllowed) {
1014 if (isTypeLegal(Ty, VT))
1019 if (VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16)
1025bool AArch64FastISel::isValueAvailable(
const Value *V)
const {
1026 if (!isa<Instruction>(V))
1029 const auto *
I = cast<Instruction>(V);
1030 return FuncInfo.MBBMap[
I->getParent()] == FuncInfo.MBB;
1033bool AArch64FastISel::simplifyAddress(Address &
Addr,
MVT VT) {
1034 if (Subtarget->isTargetILP32())
1041 bool ImmediateOffsetNeedsLowering =
false;
1042 bool RegisterOffsetNeedsLowering =
false;
1045 ImmediateOffsetNeedsLowering =
true;
1046 else if (
Offset > 0 && !(
Offset & (ScaleFactor - 1)) &&
1047 !isUInt<12>(
Offset / ScaleFactor))
1048 ImmediateOffsetNeedsLowering =
true;
1053 if (!ImmediateOffsetNeedsLowering &&
Addr.getOffset() &&
Addr.getOffsetReg())
1054 RegisterOffsetNeedsLowering =
true;
1057 if (
Addr.isRegBase() &&
Addr.getOffsetReg() && !
Addr.getReg())
1058 RegisterOffsetNeedsLowering =
true;
1063 if ((ImmediateOffsetNeedsLowering ||
Addr.getOffsetReg()) &&
Addr.isFIBase())
1065 Register ResultReg = createResultReg(&AArch64::GPR64spRegClass);
1066 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::ADDXri),
1071 Addr.setKind(Address::RegBase);
1072 Addr.setReg(ResultReg);
1075 if (RegisterOffsetNeedsLowering) {
1076 unsigned ResultReg = 0;
1077 if (
Addr.getReg()) {
1080 ResultReg = emitAddSub_rx(
true, MVT::i64,
Addr.getReg(),
1081 Addr.getOffsetReg(),
Addr.getExtendType(),
1084 ResultReg = emitAddSub_rs(
true, MVT::i64,
Addr.getReg(),
1089 ResultReg = emitLSL_ri(MVT::i64, MVT::i32,
Addr.getOffsetReg(),
1090 Addr.getShift(),
true);
1092 ResultReg = emitLSL_ri(MVT::i64, MVT::i32,
Addr.getOffsetReg(),
1093 Addr.getShift(),
false);
1095 ResultReg = emitLSL_ri(MVT::i64, MVT::i64,
Addr.getOffsetReg(),
1101 Addr.setReg(ResultReg);
1102 Addr.setOffsetReg(0);
1109 if (ImmediateOffsetNeedsLowering) {
1113 ResultReg = emitAdd_ri_(MVT::i64,
Addr.getReg(),
Offset);
1119 Addr.setReg(ResultReg);
1125void AArch64FastISel::addLoadStoreOperands(Address &
Addr,
1128 unsigned ScaleFactor,
1130 int64_t
Offset =
Addr.getOffset() / ScaleFactor;
1132 if (
Addr.isFIBase()) {
1133 int FI =
Addr.getFI();
1136 MMO = FuncInfo.MF->getMachineMemOperand(
1138 MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
1142 assert(
Addr.isRegBase() &&
"Unexpected address kind.");
1149 if (
Addr.getOffsetReg()) {
1150 assert(
Addr.getOffset() == 0 &&
"Unexpected offset");
1165unsigned AArch64FastISel::emitAddSub(
bool UseAdd,
MVT RetVT,
const Value *LHS,
1166 const Value *RHS,
bool SetFlags,
1167 bool WantResult,
bool IsZExt) {
1169 bool NeedExtend =
false;
1192 if (UseAdd && isa<Constant>(LHS) && !isa<Constant>(RHS))
1196 if (UseAdd &&
LHS->
hasOneUse() && isValueAvailable(LHS))
1201 if (UseAdd &&
LHS->
hasOneUse() && isValueAvailable(LHS))
1202 if (
const auto *SI = dyn_cast<BinaryOperator>(LHS))
1203 if (isa<ConstantInt>(
SI->getOperand(1)))
1204 if (
SI->getOpcode() == Instruction::Shl ||
1205 SI->getOpcode() == Instruction::LShr ||
1206 SI->getOpcode() == Instruction::AShr )
1209 Register LHSReg = getRegForValue(LHS);
1214 LHSReg = emitIntExt(SrcVT, LHSReg, RetVT, IsZExt);
1216 unsigned ResultReg = 0;
1217 if (
const auto *
C = dyn_cast<ConstantInt>(RHS)) {
1218 uint64_t Imm = IsZExt ?
C->getZExtValue() :
C->getSExtValue();
1219 if (
C->isNegative())
1220 ResultReg = emitAddSub_ri(!UseAdd, RetVT, LHSReg, -Imm, SetFlags,
1223 ResultReg = emitAddSub_ri(UseAdd, RetVT, LHSReg, Imm, SetFlags,
1225 }
else if (
const auto *
C = dyn_cast<Constant>(RHS))
1226 if (
C->isNullValue())
1227 ResultReg = emitAddSub_ri(UseAdd, RetVT, LHSReg, 0, SetFlags, WantResult);
1234 isValueAvailable(RHS)) {
1235 Register RHSReg = getRegForValue(RHS);
1238 return emitAddSub_rx(UseAdd, RetVT, LHSReg, RHSReg, ExtendType, 0,
1239 SetFlags, WantResult);
1245 const Value *MulLHS = cast<MulOperator>(RHS)->getOperand(0);
1246 const Value *MulRHS = cast<MulOperator>(RHS)->getOperand(1);
1248 if (
const auto *
C = dyn_cast<ConstantInt>(MulLHS))
1249 if (
C->getValue().isPowerOf2())
1252 assert(isa<ConstantInt>(MulRHS) &&
"Expected a ConstantInt.");
1253 uint64_t ShiftVal = cast<ConstantInt>(MulRHS)->getValue().logBase2();
1254 Register RHSReg = getRegForValue(MulLHS);
1257 ResultReg = emitAddSub_rs(UseAdd, RetVT, LHSReg, RHSReg,
AArch64_AM::LSL,
1258 ShiftVal, SetFlags, WantResult);
1266 if (
const auto *SI = dyn_cast<BinaryOperator>(RHS)) {
1267 if (
const auto *
C = dyn_cast<ConstantInt>(
SI->getOperand(1))) {
1269 switch (
SI->getOpcode()) {
1277 Register RHSReg = getRegForValue(
SI->getOperand(0));
1280 ResultReg = emitAddSub_rs(UseAdd, RetVT, LHSReg, RHSReg, ShiftType,
1281 ShiftVal, SetFlags, WantResult);
1289 Register RHSReg = getRegForValue(RHS);
1294 RHSReg = emitIntExt(SrcVT, RHSReg, RetVT, IsZExt);
1296 return emitAddSub_rr(UseAdd, RetVT, LHSReg, RHSReg, SetFlags, WantResult);
1299unsigned AArch64FastISel::emitAddSub_rr(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
1300 unsigned RHSReg,
bool SetFlags,
1302 assert(LHSReg && RHSReg &&
"Invalid register number.");
1304 if (LHSReg == AArch64::SP || LHSReg == AArch64::WSP ||
1305 RHSReg == AArch64::SP || RHSReg == AArch64::WSP)
1308 if (RetVT != MVT::i32 && RetVT != MVT::i64)
1311 static const unsigned OpcTable[2][2][2] = {
1312 { { AArch64::SUBWrr, AArch64::SUBXrr },
1313 { AArch64::ADDWrr, AArch64::ADDXrr } },
1314 { { AArch64::SUBSWrr, AArch64::SUBSXrr },
1315 { AArch64::ADDSWrr, AArch64::ADDSXrr } }
1317 bool Is64Bit = RetVT == MVT::i64;
1318 unsigned Opc = OpcTable[SetFlags][UseAdd][Is64Bit];
1320 Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
1323 ResultReg = createResultReg(RC);
1325 ResultReg = Is64Bit ? AArch64::XZR : AArch64::WZR;
1330 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
II, ResultReg)
1336unsigned AArch64FastISel::emitAddSub_ri(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
1339 assert(LHSReg &&
"Invalid register number.");
1341 if (RetVT != MVT::i32 && RetVT != MVT::i64)
1345 if (isUInt<12>(Imm))
1347 else if ((Imm & 0xfff000) == Imm) {
1353 static const unsigned OpcTable[2][2][2] = {
1354 { { AArch64::SUBWri, AArch64::SUBXri },
1355 { AArch64::ADDWri, AArch64::ADDXri } },
1356 { { AArch64::SUBSWri, AArch64::SUBSXri },
1357 { AArch64::ADDSWri, AArch64::ADDSXri } }
1359 bool Is64Bit = RetVT == MVT::i64;
1360 unsigned Opc = OpcTable[SetFlags][UseAdd][Is64Bit];
1363 RC = Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
1365 RC = Is64Bit ? &AArch64::GPR64spRegClass : &AArch64::GPR32spRegClass;
1368 ResultReg = createResultReg(RC);
1370 ResultReg = Is64Bit ? AArch64::XZR : AArch64::WZR;
1374 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
II, ResultReg)
1381unsigned AArch64FastISel::emitAddSub_rs(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
1386 assert(LHSReg && RHSReg &&
"Invalid register number.");
1387 assert(LHSReg != AArch64::SP && LHSReg != AArch64::WSP &&
1388 RHSReg != AArch64::SP && RHSReg != AArch64::WSP);
1390 if (RetVT != MVT::i32 && RetVT != MVT::i64)
1397 static const unsigned OpcTable[2][2][2] = {
1398 { { AArch64::SUBWrs, AArch64::SUBXrs },
1399 { AArch64::ADDWrs, AArch64::ADDXrs } },
1400 { { AArch64::SUBSWrs, AArch64::SUBSXrs },
1401 { AArch64::ADDSWrs, AArch64::ADDSXrs } }
1403 bool Is64Bit = RetVT == MVT::i64;
1404 unsigned Opc = OpcTable[SetFlags][UseAdd][Is64Bit];
1406 Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
1409 ResultReg = createResultReg(RC);
1411 ResultReg = Is64Bit ? AArch64::XZR : AArch64::WZR;
1416 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
II, ResultReg)
1419 .
addImm(getShifterImm(ShiftType, ShiftImm));
1423unsigned AArch64FastISel::emitAddSub_rx(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
1428 assert(LHSReg && RHSReg &&
"Invalid register number.");
1429 assert(LHSReg != AArch64::XZR && LHSReg != AArch64::WZR &&
1430 RHSReg != AArch64::XZR && RHSReg != AArch64::WZR);
1432 if (RetVT != MVT::i32 && RetVT != MVT::i64)
1438 static const unsigned OpcTable[2][2][2] = {
1439 { { AArch64::SUBWrx, AArch64::SUBXrx },
1440 { AArch64::ADDWrx, AArch64::ADDXrx } },
1441 { { AArch64::SUBSWrx, AArch64::SUBSXrx },
1442 { AArch64::ADDSWrx, AArch64::ADDSXrx } }
1444 bool Is64Bit = RetVT == MVT::i64;
1445 unsigned Opc = OpcTable[SetFlags][UseAdd][Is64Bit];
1448 RC = Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
1450 RC = Is64Bit ? &AArch64::GPR64spRegClass : &AArch64::GPR32spRegClass;
1453 ResultReg = createResultReg(RC);
1455 ResultReg = Is64Bit ? AArch64::XZR : AArch64::WZR;
1460 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
II, ResultReg)
1463 .
addImm(getArithExtendImm(ExtType, ShiftImm));
1467bool AArch64FastISel::emitCmp(
const Value *LHS,
const Value *RHS,
bool IsZExt) {
1469 EVT EVT = TLI.getValueType(
DL, Ty,
true);
1482 return emitICmp(VT, LHS, RHS, IsZExt);
1485 return emitFCmp(VT, LHS, RHS);
1489bool AArch64FastISel::emitICmp(
MVT RetVT,
const Value *LHS,
const Value *RHS,
1491 return emitSub(RetVT, LHS, RHS,
true,
false,
1495bool AArch64FastISel::emitICmp_ri(
MVT RetVT,
unsigned LHSReg,
uint64_t Imm) {
1496 return emitAddSub_ri(
false, RetVT, LHSReg, Imm,
1500bool AArch64FastISel::emitFCmp(
MVT RetVT,
const Value *LHS,
const Value *RHS) {
1501 if (RetVT != MVT::f32 && RetVT != MVT::f64)
1506 bool UseImm =
false;
1507 if (
const auto *CFP = dyn_cast<ConstantFP>(RHS))
1511 Register LHSReg = getRegForValue(LHS);
1516 unsigned Opc = (RetVT == MVT::f64) ? AArch64::FCMPDri : AArch64::FCMPSri;
1517 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(Opc))
1522 Register RHSReg = getRegForValue(RHS);
1526 unsigned Opc = (RetVT == MVT::f64) ? AArch64::FCMPDrr : AArch64::FCMPSrr;
1527 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(Opc))
1533unsigned AArch64FastISel::emitAdd(
MVT RetVT,
const Value *LHS,
const Value *RHS,
1534 bool SetFlags,
bool WantResult,
bool IsZExt) {
1535 return emitAddSub(
true, RetVT, LHS, RHS, SetFlags, WantResult,
1544unsigned AArch64FastISel::emitAdd_ri_(
MVT VT,
unsigned Op0, int64_t Imm) {
1547 ResultReg = emitAddSub_ri(
false, VT, Op0, -Imm);
1549 ResultReg = emitAddSub_ri(
true, VT, Op0, Imm);
1558 ResultReg = emitAddSub_rr(
true, VT, Op0, CReg);
1562unsigned AArch64FastISel::emitSub(
MVT RetVT,
const Value *LHS,
const Value *RHS,
1563 bool SetFlags,
bool WantResult,
bool IsZExt) {
1564 return emitAddSub(
false, RetVT, LHS, RHS, SetFlags, WantResult,
1568unsigned AArch64FastISel::emitSubs_rr(
MVT RetVT,
unsigned LHSReg,
1569 unsigned RHSReg,
bool WantResult) {
1570 return emitAddSub_rr(
false, RetVT, LHSReg, RHSReg,
1574unsigned AArch64FastISel::emitSubs_rs(
MVT RetVT,
unsigned LHSReg,
1577 uint64_t ShiftImm,
bool WantResult) {
1578 return emitAddSub_rs(
false, RetVT, LHSReg, RHSReg, ShiftType,
1579 ShiftImm,
true, WantResult);
1582unsigned AArch64FastISel::emitLogicalOp(
unsigned ISDOpc,
MVT RetVT,
1585 if (isa<ConstantInt>(LHS) && !isa<ConstantInt>(RHS))
1595 if (
const auto *SI = dyn_cast<ShlOperator>(LHS))
1596 if (isa<ConstantInt>(
SI->getOperand(1)))
1599 Register LHSReg = getRegForValue(LHS);
1603 unsigned ResultReg = 0;
1604 if (
const auto *
C = dyn_cast<ConstantInt>(RHS)) {
1606 ResultReg = emitLogicalOp_ri(ISDOpc, RetVT, LHSReg, Imm);
1614 const Value *MulLHS = cast<MulOperator>(RHS)->getOperand(0);
1615 const Value *MulRHS = cast<MulOperator>(RHS)->getOperand(1);
1617 if (
const auto *
C = dyn_cast<ConstantInt>(MulLHS))
1618 if (
C->getValue().isPowerOf2())
1621 assert(isa<ConstantInt>(MulRHS) &&
"Expected a ConstantInt.");
1622 uint64_t ShiftVal = cast<ConstantInt>(MulRHS)->getValue().logBase2();
1624 Register RHSReg = getRegForValue(MulLHS);
1627 ResultReg = emitLogicalOp_rs(ISDOpc, RetVT, LHSReg, RHSReg, ShiftVal);
1635 if (
const auto *SI = dyn_cast<ShlOperator>(RHS))
1636 if (
const auto *
C = dyn_cast<ConstantInt>(
SI->getOperand(1))) {
1638 Register RHSReg = getRegForValue(
SI->getOperand(0));
1641 ResultReg = emitLogicalOp_rs(ISDOpc, RetVT, LHSReg, RHSReg, ShiftVal);
1647 Register RHSReg = getRegForValue(RHS);
1652 ResultReg = fastEmit_rr(VT, VT, ISDOpc, LHSReg, RHSReg);
1653 if (RetVT >= MVT::i8 && RetVT <= MVT::i16) {
1655 ResultReg = emitAnd_ri(MVT::i32, ResultReg, Mask);
1660unsigned AArch64FastISel::emitLogicalOp_ri(
unsigned ISDOpc,
MVT RetVT,
1663 "ISD nodes are not consecutive!");
1664 static const unsigned OpcTable[3][2] = {
1665 { AArch64::ANDWri, AArch64::ANDXri },
1666 { AArch64::ORRWri, AArch64::ORRXri },
1667 { AArch64::EORWri, AArch64::EORXri }
1680 Opc = OpcTable[
Idx][0];
1681 RC = &AArch64::GPR32spRegClass;
1686 Opc = OpcTable[ISDOpc -
ISD::AND][1];
1687 RC = &AArch64::GPR64spRegClass;
1696 fastEmitInst_ri(Opc, RC, LHSReg,
1698 if (RetVT >= MVT::i8 && RetVT <= MVT::i16 && ISDOpc !=
ISD::AND) {
1700 ResultReg = emitAnd_ri(MVT::i32, ResultReg, Mask);
1705unsigned AArch64FastISel::emitLogicalOp_rs(
unsigned ISDOpc,
MVT RetVT,
1706 unsigned LHSReg,
unsigned RHSReg,
1709 "ISD nodes are not consecutive!");
1710 static const unsigned OpcTable[3][2] = {
1711 { AArch64::ANDWrs, AArch64::ANDXrs },
1712 { AArch64::ORRWrs, AArch64::ORRXrs },
1713 { AArch64::EORWrs, AArch64::EORXrs }
1729 Opc = OpcTable[ISDOpc -
ISD::AND][0];
1730 RC = &AArch64::GPR32RegClass;
1733 Opc = OpcTable[ISDOpc -
ISD::AND][1];
1734 RC = &AArch64::GPR64RegClass;
1738 fastEmitInst_rri(Opc, RC, LHSReg, RHSReg,
1740 if (RetVT >= MVT::i8 && RetVT <= MVT::i16) {
1742 ResultReg = emitAnd_ri(MVT::i32, ResultReg, Mask);
1747unsigned AArch64FastISel::emitAnd_ri(
MVT RetVT,
unsigned LHSReg,
1749 return emitLogicalOp_ri(
ISD::AND, RetVT, LHSReg, Imm);
1752unsigned AArch64FastISel::emitLoad(
MVT VT,
MVT RetVT, Address
Addr,
1754 if (!TLI.allowsMisalignedMemoryAccesses(VT))
1758 if (!simplifyAddress(
Addr, VT))
1767 bool UseScaled =
true;
1768 if ((
Addr.getOffset() < 0) || (
Addr.getOffset() & (ScaleFactor - 1))) {
1773 static const unsigned GPOpcTable[2][8][4] = {
1775 { { AArch64::LDURSBWi, AArch64::LDURSHWi, AArch64::LDURWi,
1777 { AArch64::LDURSBXi, AArch64::LDURSHXi, AArch64::LDURSWi,
1779 { AArch64::LDRSBWui, AArch64::LDRSHWui, AArch64::LDRWui,
1781 { AArch64::LDRSBXui, AArch64::LDRSHXui, AArch64::LDRSWui,
1783 { AArch64::LDRSBWroX, AArch64::LDRSHWroX, AArch64::LDRWroX,
1785 { AArch64::LDRSBXroX, AArch64::LDRSHXroX, AArch64::LDRSWroX,
1787 { AArch64::LDRSBWroW, AArch64::LDRSHWroW, AArch64::LDRWroW,
1789 { AArch64::LDRSBXroW, AArch64::LDRSHXroW, AArch64::LDRSWroW,
1793 { { AArch64::LDURBBi, AArch64::LDURHHi, AArch64::LDURWi,
1795 { AArch64::LDURBBi, AArch64::LDURHHi, AArch64::LDURWi,
1797 { AArch64::LDRBBui, AArch64::LDRHHui, AArch64::LDRWui,
1799 { AArch64::LDRBBui, AArch64::LDRHHui, AArch64::LDRWui,
1801 { AArch64::LDRBBroX, AArch64::LDRHHroX, AArch64::LDRWroX,
1803 { AArch64::LDRBBroX, AArch64::LDRHHroX, AArch64::LDRWroX,
1805 { AArch64::LDRBBroW, AArch64::LDRHHroW, AArch64::LDRWroW,
1807 { AArch64::LDRBBroW, AArch64::LDRHHroW, AArch64::LDRWroW,
1812 static const unsigned FPOpcTable[4][2] = {
1813 { AArch64::LDURSi, AArch64::LDURDi },
1814 { AArch64::LDRSui, AArch64::LDRDui },
1815 { AArch64::LDRSroX, AArch64::LDRDroX },
1816 { AArch64::LDRSroW, AArch64::LDRDroW }
1821 bool UseRegOffset =
Addr.isRegBase() && !
Addr.getOffset() &&
Addr.getReg() &&
1822 Addr.getOffsetReg();
1823 unsigned Idx = UseRegOffset ? 2 : UseScaled ? 1 : 0;
1828 bool IsRet64Bit = RetVT == MVT::i64;
1834 Opc = GPOpcTable[WantZExt][2 *
Idx + IsRet64Bit][0];
1835 RC = (IsRet64Bit && !WantZExt) ?
1836 &AArch64::GPR64RegClass: &AArch64::GPR32RegClass;
1839 Opc = GPOpcTable[WantZExt][2 *
Idx + IsRet64Bit][1];
1840 RC = (IsRet64Bit && !WantZExt) ?
1841 &AArch64::GPR64RegClass: &AArch64::GPR32RegClass;
1844 Opc = GPOpcTable[WantZExt][2 *
Idx + IsRet64Bit][2];
1845 RC = (IsRet64Bit && !WantZExt) ?
1846 &AArch64::GPR64RegClass: &AArch64::GPR32RegClass;
1849 Opc = GPOpcTable[WantZExt][2 *
Idx + IsRet64Bit][3];
1850 RC = &AArch64::GPR64RegClass;
1853 Opc = FPOpcTable[
Idx][0];
1854 RC = &AArch64::FPR32RegClass;
1857 Opc = FPOpcTable[
Idx][1];
1858 RC = &AArch64::FPR64RegClass;
1863 Register ResultReg = createResultReg(RC);
1865 TII.get(Opc), ResultReg);
1869 if (VT == MVT::i1) {
1870 unsigned ANDReg = emitAnd_ri(MVT::i32, ResultReg, 1);
1871 assert(ANDReg &&
"Unexpected AND instruction emission failure.");
1877 if (WantZExt && RetVT == MVT::i64 && VT <= MVT::i32) {
1878 Register Reg64 = createResultReg(&AArch64::GPR64RegClass);
1879 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
1880 TII.get(AArch64::SUBREG_TO_REG), Reg64)
1883 .
addImm(AArch64::sub_32);
1889bool AArch64FastISel::selectAddSub(
const Instruction *
I) {
1891 if (!isTypeSupported(
I->getType(), VT,
true))
1895 return selectOperator(
I,
I->getOpcode());
1898 switch (
I->getOpcode()) {
1901 case Instruction::Add:
1902 ResultReg = emitAdd(VT,
I->getOperand(0),
I->getOperand(1));
1904 case Instruction::Sub:
1905 ResultReg = emitSub(VT,
I->getOperand(0),
I->getOperand(1));
1911 updateValueMap(
I, ResultReg);
1915bool AArch64FastISel::selectLogicalOp(
const Instruction *
I) {
1917 if (!isTypeSupported(
I->getType(), VT,
true))
1921 return selectOperator(
I,
I->getOpcode());
1924 switch (
I->getOpcode()) {
1927 case Instruction::And:
1928 ResultReg = emitLogicalOp(
ISD::AND, VT,
I->getOperand(0),
I->getOperand(1));
1930 case Instruction::Or:
1931 ResultReg = emitLogicalOp(
ISD::OR, VT,
I->getOperand(0),
I->getOperand(1));
1933 case Instruction::Xor:
1934 ResultReg = emitLogicalOp(
ISD::XOR, VT,
I->getOperand(0),
I->getOperand(1));
1940 updateValueMap(
I, ResultReg);
1944bool AArch64FastISel::selectLoad(
const Instruction *
I) {
1949 if (!isTypeSupported(
I->getType(), VT,
true) ||
1950 cast<LoadInst>(
I)->isAtomic())
1953 const Value *SV =
I->getOperand(0);
1954 if (TLI.supportSwiftError()) {
1957 if (
const Argument *Arg = dyn_cast<Argument>(SV)) {
1958 if (Arg->hasSwiftErrorAttr())
1962 if (
const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
1963 if (Alloca->isSwiftError())
1970 if (!computeAddress(
I->getOperand(0),
Addr,
I->getType()))
1974 bool WantZExt =
true;
1976 const Value *IntExtVal =
nullptr;
1977 if (
I->hasOneUse()) {
1978 if (
const auto *ZE = dyn_cast<ZExtInst>(
I->use_begin()->getUser())) {
1979 if (isTypeSupported(ZE->getType(), RetVT))
1983 }
else if (
const auto *SE = dyn_cast<SExtInst>(
I->use_begin()->getUser())) {
1984 if (isTypeSupported(SE->getType(), RetVT))
1992 unsigned ResultReg =
1993 emitLoad(VT, RetVT,
Addr, WantZExt, createMachineMemOperandFor(
I));
2014 auto *
MI =
MRI.getUniqueVRegDef(Reg);
2016 if (RetVT == MVT::i64 && VT <= MVT::i32) {
2020 ResultReg = std::prev(
I)->getOperand(0).getReg();
2021 removeDeadCode(
I, std::next(
I));
2023 ResultReg = fastEmitInst_extractsubreg(MVT::i32, ResultReg,
2026 updateValueMap(
I, ResultReg);
2035 for (
auto &Opnd :
MI->uses()) {
2037 Reg = Opnd.getReg();
2042 removeDeadCode(
I, std::next(
I));
2045 MI =
MRI.getUniqueVRegDef(Reg);
2047 updateValueMap(IntExtVal, ResultReg);
2051 updateValueMap(
I, ResultReg);
2055bool AArch64FastISel::emitStoreRelease(
MVT VT,
unsigned SrcReg,
2060 default:
return false;
2061 case MVT::i8: Opc = AArch64::STLRB;
break;
2062 case MVT::i16: Opc = AArch64::STLRH;
break;
2063 case MVT::i32: Opc = AArch64::STLRW;
break;
2064 case MVT::i64: Opc = AArch64::STLRX;
break;
2070 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
II)
2077bool AArch64FastISel::emitStore(
MVT VT,
unsigned SrcReg, Address
Addr,
2079 if (!TLI.allowsMisalignedMemoryAccesses(VT))
2083 if (!simplifyAddress(
Addr, VT))
2092 bool UseScaled =
true;
2093 if ((
Addr.getOffset() < 0) || (
Addr.getOffset() & (ScaleFactor - 1))) {
2098 static const unsigned OpcTable[4][6] = {
2099 { AArch64::STURBBi, AArch64::STURHHi, AArch64::STURWi, AArch64::STURXi,
2100 AArch64::STURSi, AArch64::STURDi },
2101 { AArch64::STRBBui, AArch64::STRHHui, AArch64::STRWui, AArch64::STRXui,
2102 AArch64::STRSui, AArch64::STRDui },
2103 { AArch64::STRBBroX, AArch64::STRHHroX, AArch64::STRWroX, AArch64::STRXroX,
2104 AArch64::STRSroX, AArch64::STRDroX },
2105 { AArch64::STRBBroW, AArch64::STRHHroW, AArch64::STRWroW, AArch64::STRXroW,
2106 AArch64::STRSroW, AArch64::STRDroW }
2110 bool VTIsi1 =
false;
2111 bool UseRegOffset =
Addr.isRegBase() && !
Addr.getOffset() &&
Addr.getReg() &&
2112 Addr.getOffsetReg();
2113 unsigned Idx = UseRegOffset ? 2 : UseScaled ? 1 : 0;
2120 case MVT::i1: VTIsi1 =
true; [[fallthrough]];
2121 case MVT::i8: Opc = OpcTable[
Idx][0];
break;
2122 case MVT::i16: Opc = OpcTable[
Idx][1];
break;
2123 case MVT::i32: Opc = OpcTable[
Idx][2];
break;
2124 case MVT::i64: Opc = OpcTable[
Idx][3];
break;
2125 case MVT::f32: Opc = OpcTable[
Idx][4];
break;
2126 case MVT::f64: Opc = OpcTable[
Idx][5];
break;
2130 if (VTIsi1 && SrcReg != AArch64::WZR) {
2131 unsigned ANDReg = emitAnd_ri(MVT::i32, SrcReg, 1);
2132 assert(ANDReg &&
"Unexpected AND instruction emission failure.");
2145bool AArch64FastISel::selectStore(
const Instruction *
I) {
2147 const Value *Op0 =
I->getOperand(0);
2151 if (!isTypeSupported(Op0->
getType(), VT,
true))
2154 const Value *PtrV =
I->getOperand(1);
2155 if (TLI.supportSwiftError()) {
2158 if (
const Argument *Arg = dyn_cast<Argument>(PtrV)) {
2159 if (Arg->hasSwiftErrorAttr())
2163 if (
const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
2164 if (Alloca->isSwiftError())
2171 unsigned SrcReg = 0;
2172 if (
const auto *CI = dyn_cast<ConstantInt>(Op0)) {
2174 SrcReg = (VT == MVT::i64) ? AArch64::XZR : AArch64::WZR;
2175 }
else if (
const auto *CF = dyn_cast<ConstantFP>(Op0)) {
2176 if (CF->isZero() && !CF->isNegative()) {
2178 SrcReg = (VT == MVT::i64) ? AArch64::XZR : AArch64::WZR;
2183 SrcReg = getRegForValue(Op0);
2188 auto *
SI = cast<StoreInst>(
I);
2191 if (
SI->isAtomic()) {
2196 Register AddrReg = getRegForValue(PtrV);
2197 return emitStoreRelease(VT, SrcReg, AddrReg,
2198 createMachineMemOperandFor(
I));
2259bool AArch64FastISel::emitCompareAndBranch(
const BranchInst *BI) {
2263 if (FuncInfo.MF->getFunction().hasFnAttribute(
2264 Attribute::SpeculativeLoadHardening))
2286 if (FuncInfo.MBB->isLayoutSuccessor(
TBB)) {
2293 switch (Predicate) {
2298 if (isa<Constant>(LHS) && cast<Constant>(LHS)->isNullValue())
2301 if (!isa<Constant>(RHS) || !cast<Constant>(RHS)->isNullValue())
2304 if (
const auto *AI = dyn_cast<BinaryOperator>(LHS))
2305 if (AI->
getOpcode() == Instruction::And && isValueAvailable(AI)) {
2309 if (
const auto *
C = dyn_cast<ConstantInt>(AndLHS))
2310 if (
C->getValue().isPowerOf2())
2313 if (
const auto *
C = dyn_cast<ConstantInt>(AndRHS))
2314 if (
C->getValue().isPowerOf2()) {
2315 TestBit =
C->getValue().logBase2();
2327 if (!isa<Constant>(RHS) || !cast<Constant>(RHS)->isNullValue())
2335 if (!isa<ConstantInt>(RHS))
2338 if (cast<ConstantInt>(RHS)->getValue() !=
APInt(BW, -1,
true))
2346 static const unsigned OpcTable[2][2][2] = {
2347 { {AArch64::CBZW, AArch64::CBZX },
2348 {AArch64::CBNZW, AArch64::CBNZX} },
2349 { {AArch64::TBZW, AArch64::TBZX },
2350 {AArch64::TBNZW, AArch64::TBNZX} }
2353 bool IsBitTest = TestBit != -1;
2354 bool Is64Bit = BW == 64;
2355 if (TestBit < 32 && TestBit >= 0)
2358 unsigned Opc = OpcTable[IsBitTest][IsCmpNE][Is64Bit];
2361 Register SrcReg = getRegForValue(LHS);
2365 if (BW == 64 && !Is64Bit)
2366 SrcReg = fastEmitInst_extractsubreg(MVT::i32, SrcReg, AArch64::sub_32);
2368 if ((BW < 32) && !IsBitTest)
2369 SrcReg = emitIntExt(VT, SrcReg, MVT::i32,
true);
2374 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(Opc))
2384bool AArch64FastISel::selectBranch(
const Instruction *
I) {
2396 if (CI->
hasOneUse() && isValueAvailable(CI)) {
2399 switch (Predicate) {
2403 fastEmitBranch(FBB, MIMD.getDL());
2406 fastEmitBranch(
TBB, MIMD.getDL());
2411 if (emitCompareAndBranch(BI))
2415 if (FuncInfo.MBB->isLayoutSuccessor(
TBB)) {
2428 switch (Predicate) {
2444 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::Bcc))
2450 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::Bcc))
2457 }
else if (
const auto *CI = dyn_cast<ConstantInt>(BI->
getCondition())) {
2460 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::B))
2469 FuncInfo.MBB->addSuccessorWithoutProb(
Target);
2481 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::Bcc))
2495 unsigned Opcode = AArch64::TBNZW;
2496 if (FuncInfo.MBB->isLayoutSuccessor(
TBB)) {
2498 Opcode = AArch64::TBZW;
2504 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
II)
2505 .
addReg(ConstrainedCondReg)
2513bool AArch64FastISel::selectIndirectBr(
const Instruction *
I) {
2526 FuncInfo.MBB->addSuccessor(FuncInfo.MBBMap[Succ]);
2532 const CmpInst *CI = cast<CmpInst>(
I);
2540 unsigned ResultReg = 0;
2541 switch (Predicate) {
2545 ResultReg = createResultReg(&AArch64::GPR32RegClass);
2546 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
2547 TII.get(TargetOpcode::COPY), ResultReg)
2551 ResultReg = fastEmit_i(MVT::i32, MVT::i32,
ISD::Constant, 1);
2556 updateValueMap(
I, ResultReg);
2564 ResultReg = createResultReg(&AArch64::GPR32RegClass);
2568 static unsigned CondCodeTable[2][2] = {
2573 switch (Predicate) {
2585 Register TmpReg1 = createResultReg(&AArch64::GPR32RegClass);
2586 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::CSINCWr),
2591 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::CSINCWr),
2597 updateValueMap(
I, ResultReg);
2605 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::CSINCWr),
2611 updateValueMap(
I, ResultReg);
2617bool AArch64FastISel::optimizeSelect(
const SelectInst *SI) {
2618 if (!
SI->getType()->isIntegerTy(1))
2621 const Value *Src1Val, *Src2Val;
2623 bool NeedExtraOp =
false;
2624 if (
auto *CI = dyn_cast<ConstantInt>(
SI->getTrueValue())) {
2626 Src1Val =
SI->getCondition();
2627 Src2Val =
SI->getFalseValue();
2628 Opc = AArch64::ORRWrr;
2631 Src1Val =
SI->getFalseValue();
2632 Src2Val =
SI->getCondition();
2633 Opc = AArch64::BICWrr;
2635 }
else if (
auto *CI = dyn_cast<ConstantInt>(
SI->getFalseValue())) {
2637 Src1Val =
SI->getCondition();
2638 Src2Val =
SI->getTrueValue();
2639 Opc = AArch64::ORRWrr;
2643 Src1Val =
SI->getCondition();
2644 Src2Val =
SI->getTrueValue();
2645 Opc = AArch64::ANDWrr;
2652 Register Src1Reg = getRegForValue(Src1Val);
2656 Register Src2Reg = getRegForValue(Src2Val);
2661 Src1Reg = emitLogicalOp_ri(
ISD::XOR, MVT::i32, Src1Reg, 1);
2663 Register ResultReg = fastEmitInst_rr(Opc, &AArch64::GPR32RegClass, Src1Reg,
2665 updateValueMap(SI, ResultReg);
2669bool AArch64FastISel::selectSelect(
const Instruction *
I) {
2670 assert(isa<SelectInst>(
I) &&
"Expected a select instruction.");
2672 if (!isTypeSupported(
I->getType(), VT))
2684 Opc = AArch64::CSELWr;
2685 RC = &AArch64::GPR32RegClass;
2688 Opc = AArch64::CSELXr;
2689 RC = &AArch64::GPR64RegClass;
2692 Opc = AArch64::FCSELSrrr;
2693 RC = &AArch64::FPR32RegClass;
2696 Opc = AArch64::FCSELDrrr;
2697 RC = &AArch64::FPR64RegClass;
2706 if (optimizeSelect(SI))
2710 if (foldXALUIntrinsic(
CC,
I,
Cond)) {
2715 }
else if (isa<CmpInst>(
Cond) && cast<CmpInst>(
Cond)->hasOneUse() &&
2716 isValueAvailable(
Cond)) {
2717 const auto *
Cmp = cast<CmpInst>(
Cond);
2720 const Value *FoldSelect =
nullptr;
2721 switch (Predicate) {
2725 FoldSelect =
SI->getFalseValue();
2728 FoldSelect =
SI->getTrueValue();
2733 Register SrcReg = getRegForValue(FoldSelect);
2737 updateValueMap(
I, SrcReg);
2747 switch (Predicate) {
2769 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
II,
2775 Register Src1Reg = getRegForValue(
SI->getTrueValue());
2776 Register Src2Reg = getRegForValue(
SI->getFalseValue());
2778 if (!Src1Reg || !Src2Reg)
2782 Src2Reg = fastEmitInst_rri(Opc, RC, Src1Reg, Src2Reg, ExtraCC);
2784 Register ResultReg = fastEmitInst_rri(Opc, RC, Src1Reg, Src2Reg,
CC);
2785 updateValueMap(
I, ResultReg);
2789bool AArch64FastISel::selectFPExt(
const Instruction *
I) {
2791 if (!
I->getType()->isDoubleTy() || !
V->getType()->isFloatTy())
2798 Register ResultReg = createResultReg(&AArch64::FPR64RegClass);
2799 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::FCVTDSr),
2801 updateValueMap(
I, ResultReg);
2805bool AArch64FastISel::selectFPTrunc(
const Instruction *
I) {
2807 if (!
I->getType()->isFloatTy() || !
V->getType()->isDoubleTy())
2814 Register ResultReg = createResultReg(&AArch64::FPR32RegClass);
2815 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::FCVTSDr),
2817 updateValueMap(
I, ResultReg);
2824 if (!isTypeLegal(
I->getType(), DestVT) || DestVT.
isVector())
2827 Register SrcReg = getRegForValue(
I->getOperand(0));
2831 EVT SrcVT = TLI.getValueType(
DL,
I->getOperand(0)->getType(),
true);
2832 if (SrcVT == MVT::f128 || SrcVT == MVT::f16 || SrcVT == MVT::bf16)
2836 if (SrcVT == MVT::f64) {
2838 Opc = (DestVT == MVT::i32) ? AArch64::FCVTZSUWDr : AArch64::FCVTZSUXDr;
2840 Opc = (DestVT == MVT::i32) ? AArch64::FCVTZUUWDr : AArch64::FCVTZUUXDr;
2843 Opc = (DestVT == MVT::i32) ? AArch64::FCVTZSUWSr : AArch64::FCVTZSUXSr;
2845 Opc = (DestVT == MVT::i32) ? AArch64::FCVTZUUWSr : AArch64::FCVTZUUXSr;
2847 Register ResultReg = createResultReg(
2848 DestVT == MVT::i32 ? &AArch64::GPR32RegClass : &AArch64::GPR64RegClass);
2849 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(Opc), ResultReg)
2851 updateValueMap(
I, ResultReg);
2857 if (!isTypeLegal(
I->getType(), DestVT) || DestVT.
isVector())
2860 if (DestVT == MVT::f16 || DestVT == MVT::bf16)
2863 assert((DestVT == MVT::f32 || DestVT == MVT::f64) &&
2864 "Unexpected value type.");
2866 Register SrcReg = getRegForValue(
I->getOperand(0));
2870 EVT SrcVT = TLI.getValueType(
DL,
I->getOperand(0)->getType(),
true);
2873 if (SrcVT == MVT::i16 || SrcVT == MVT::i8 || SrcVT == MVT::i1) {
2881 if (SrcVT == MVT::i64) {
2883 Opc = (DestVT == MVT::f32) ? AArch64::SCVTFUXSri : AArch64::SCVTFUXDri;
2885 Opc = (DestVT == MVT::f32) ? AArch64::UCVTFUXSri : AArch64::UCVTFUXDri;
2888 Opc = (DestVT == MVT::f32) ? AArch64::SCVTFUWSri : AArch64::SCVTFUWDri;
2890 Opc = (DestVT == MVT::f32) ? AArch64::UCVTFUWSri : AArch64::UCVTFUWDri;
2893 Register ResultReg = fastEmitInst_r(Opc, TLI.getRegClassFor(DestVT), SrcReg);
2894 updateValueMap(
I, ResultReg);
2898bool AArch64FastISel::fastLowerArguments() {
2899 if (!FuncInfo.CanLowerReturn)
2910 if (Subtarget->hasCustomCallingConv())
2914 unsigned GPRCnt = 0;
2915 unsigned FPRCnt = 0;
2916 for (
auto const &Arg :
F->args()) {
2917 if (Arg.hasAttribute(Attribute::ByVal) ||
2918 Arg.hasAttribute(Attribute::InReg) ||
2919 Arg.hasAttribute(Attribute::StructRet) ||
2920 Arg.hasAttribute(Attribute::SwiftSelf) ||
2921 Arg.hasAttribute(Attribute::SwiftAsync) ||
2922 Arg.hasAttribute(Attribute::SwiftError) ||
2923 Arg.hasAttribute(Attribute::Nest))
2926 Type *ArgTy = Arg.getType();
2930 EVT ArgVT = TLI.getValueType(
DL, ArgTy);
2939 (!Subtarget->hasNEON() || !Subtarget->isLittleEndian()))
2942 if (VT >= MVT::i1 && VT <= MVT::i64)
2944 else if ((VT >= MVT::f16 && VT <= MVT::f64) || VT.
is64BitVector() ||
2950 if (GPRCnt > 8 || FPRCnt > 8)
2955 { AArch64::W0, AArch64::W1, AArch64::W2, AArch64::W3, AArch64::W4,
2956 AArch64::W5, AArch64::W6, AArch64::W7 },
2957 { AArch64::X0, AArch64::X1, AArch64::X2, AArch64::X3, AArch64::X4,
2958 AArch64::X5, AArch64::X6, AArch64::X7 },
2959 { AArch64::H0, AArch64::H1, AArch64::H2, AArch64::H3, AArch64::H4,
2960 AArch64::H5, AArch64::H6, AArch64::H7 },
2961 { AArch64::S0, AArch64::S1, AArch64::S2, AArch64::S3, AArch64::S4,
2962 AArch64::S5, AArch64::S6, AArch64::S7 },
2963 { AArch64::D0, AArch64::D1, AArch64::D2, AArch64::D3, AArch64::D4,
2964 AArch64::D5, AArch64::D6, AArch64::D7 },
2965 { AArch64::Q0, AArch64::Q1, AArch64::Q2, AArch64::Q3, AArch64::Q4,
2966 AArch64::Q5, AArch64::Q6, AArch64::Q7 }
2970 unsigned FPRIdx = 0;
2971 for (
auto const &Arg :
F->args()) {
2972 MVT VT = TLI.getSimpleValueType(
DL, Arg.getType());
2975 if (VT >= MVT::i1 && VT <= MVT::i32) {
2977 RC = &AArch64::GPR32RegClass;
2979 }
else if (VT == MVT::i64) {
2981 RC = &AArch64::GPR64RegClass;
2982 }
else if (VT == MVT::f16 || VT == MVT::bf16) {
2984 RC = &AArch64::FPR16RegClass;
2985 }
else if (VT == MVT::f32) {
2987 RC = &AArch64::FPR32RegClass;
2990 RC = &AArch64::FPR64RegClass;
2993 RC = &AArch64::FPR128RegClass;
2997 Register DstReg = FuncInfo.MF->addLiveIn(SrcReg, RC);
3001 Register ResultReg = createResultReg(RC);
3002 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3003 TII.get(TargetOpcode::COPY), ResultReg)
3005 updateValueMap(&Arg, ResultReg);
3010bool AArch64FastISel::processCallArgs(CallLoweringInfo &CLI,
3012 unsigned &NumBytes) {
3015 CCState CCInfo(
CC,
false, *FuncInfo.MF, ArgLocs, *Context);
3016 CCInfo.AnalyzeCallOperands(OutVTs, CLI.OutFlags, CCAssignFnForCall(
CC));
3019 NumBytes = CCInfo.getStackSize();
3022 unsigned AdjStackDown =
TII.getCallFrameSetupOpcode();
3023 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AdjStackDown))
3028 const Value *ArgVal = CLI.OutVals[VA.getValNo()];
3029 MVT ArgVT = OutVTs[VA.getValNo()];
3031 Register ArgReg = getRegForValue(ArgVal);
3036 switch (VA.getLocInfo()) {
3040 MVT DestVT = VA.getLocVT();
3042 ArgReg = emitIntExt(SrcVT, ArgReg, DestVT,
false);
3050 MVT DestVT = VA.getLocVT();
3052 ArgReg = emitIntExt(SrcVT, ArgReg, DestVT,
true);
3062 if (VA.isRegLoc() && !VA.needsCustom()) {
3063 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3064 TII.get(TargetOpcode::COPY), VA.getLocReg()).
addReg(ArgReg);
3065 CLI.OutRegs.push_back(VA.getLocReg());
3066 }
else if (VA.needsCustom()) {
3070 assert(VA.isMemLoc() &&
"Assuming store on stack.");
3073 if (isa<UndefValue>(ArgVal))
3079 unsigned BEAlign = 0;
3080 if (ArgSize < 8 && !Subtarget->isLittleEndian())
3081 BEAlign = 8 - ArgSize;
3084 Addr.setKind(Address::RegBase);
3085 Addr.setReg(AArch64::SP);
3086 Addr.setOffset(VA.getLocMemOffset() + BEAlign);
3100bool AArch64FastISel::finishCall(CallLoweringInfo &CLI,
unsigned NumBytes) {
3104 unsigned AdjStackUp =
TII.getCallFrameDestroyOpcode();
3105 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AdjStackUp))
3110 CCState CCInfo(
CC,
false, *FuncInfo.MF, RVLocs, *Context);
3111 CCInfo.AnalyzeCallResult(CLI.Ins, CCAssignFnForCall(
CC));
3113 Register ResultReg = FuncInfo.CreateRegs(CLI.RetTy);
3114 for (
unsigned i = 0; i != RVLocs.
size(); ++i) {
3117 unsigned CopyReg = ResultReg + i;
3120 if (CopyVT.
isVector() && !Subtarget->isLittleEndian())
3124 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(TargetOpcode::COPY),
3130 CLI.ResultReg = ResultReg;
3131 CLI.NumResultRegs = RVLocs.
size();
3136bool AArch64FastISel::fastLowerCall(CallLoweringInfo &CLI) {
3138 bool IsTailCall = CLI.IsTailCall;
3139 bool IsVarArg = CLI.IsVarArg;
3143 if (!Callee && !Symbol)
3148 if (CLI.CB && CLI.CB->hasFnAttr(Attribute::ReturnsTwice) &&
3149 !Subtarget->noBTIAtReturnTwice() &&
3154 if (CLI.CB && CLI.CB->isIndirectCall() &&
3164 if (Subtarget->isTargetILP32())
3178 if (MF->getFunction().getParent()->getRtLibUseGOT())
3185 if (Subtarget->isWindowsArm64EC())
3188 for (
auto Flag : CLI.OutFlags)
3190 Flag.isSwiftSelf() ||
Flag.isSwiftAsync() ||
Flag.isSwiftError())
3195 OutVTs.
reserve(CLI.OutVals.size());
3197 for (
auto *Val : CLI.OutVals) {
3199 if (!isTypeLegal(Val->getType(), VT) &&
3200 !(VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16))
3211 if (Callee && !computeCallAddress(Callee,
Addr))
3217 if (Subtarget->isTargetWindows() &&
Addr.getGlobalValue() &&
3218 Addr.getGlobalValue()->hasExternalWeakLinkage())
3223 if (!processCallArgs(CLI, OutVTs, NumBytes))
3227 if (
RegInfo->isAnyArgRegReserved(*MF))
3228 RegInfo->emitReservedArgRegCallError(*MF);
3232 if (Subtarget->useSmallAddressing()) {
3235 MIB =
BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
II);
3238 else if (
Addr.getGlobalValue())
3240 else if (
Addr.getReg()) {
3246 unsigned CallReg = 0;
3248 Register ADRPReg = createResultReg(&AArch64::GPR64commonRegClass);
3249 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::ADRP),
3253 CallReg = createResultReg(&AArch64::GPR64RegClass);
3254 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3255 TII.get(AArch64::LDRXui), CallReg)
3259 }
else if (
Addr.getGlobalValue())
3260 CallReg = materializeGV(
Addr.getGlobalValue());
3261 else if (
Addr.getReg())
3262 CallReg =
Addr.getReg();
3269 MIB =
BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
II).
addReg(CallReg);
3273 for (
auto Reg : CLI.OutRegs)
3283 return finishCall(CLI, NumBytes);
3288 return Len / Alignment->value() <= 4;
3293bool AArch64FastISel::tryEmitSmallMemCpy(Address Dest, Address Src,
3296 if (!isMemCpySmall(Len, Alignment))
3299 int64_t UnscaledOffset = 0;
3305 if (!Alignment || *Alignment >= 8) {
3316 assert(Alignment &&
"Alignment is set in this branch");
3318 if (Len >= 4 && *Alignment == 4)
3320 else if (Len >= 2 && *Alignment == 2)
3327 unsigned ResultReg =
emitLoad(VT, VT, Src);
3336 UnscaledOffset +=
Size;
3339 Dest.setOffset(OrigDest.getOffset() + UnscaledOffset);
3340 Src.setOffset(OrigSrc.getOffset() + UnscaledOffset);
3351 if (!isa<ExtractValueInst>(
Cond))
3354 const auto *EV = cast<ExtractValueInst>(
Cond);
3355 if (!isa<IntrinsicInst>(EV->getAggregateOperand()))
3358 const auto *
II = cast<IntrinsicInst>(EV->getAggregateOperand());
3362 cast<StructType>(
Callee->getReturnType())->getTypeAtIndex(0U);
3363 if (!isTypeLegal(
RetTy, RetVT))
3366 if (RetVT != MVT::i32 && RetVT != MVT::i64)
3373 if (isa<ConstantInt>(LHS) && !isa<ConstantInt>(RHS) &&
II->isCommutative())
3381 case Intrinsic::smul_with_overflow:
3382 if (
const auto *
C = dyn_cast<ConstantInt>(RHS))
3383 if (
C->getValue() == 2)
3384 IID = Intrinsic::sadd_with_overflow;
3386 case Intrinsic::umul_with_overflow:
3387 if (
const auto *
C = dyn_cast<ConstantInt>(RHS))
3388 if (
C->getValue() == 2)
3389 IID = Intrinsic::uadd_with_overflow;
3397 case Intrinsic::sadd_with_overflow:
3398 case Intrinsic::ssub_with_overflow:
3401 case Intrinsic::uadd_with_overflow:
3404 case Intrinsic::usub_with_overflow:
3407 case Intrinsic::smul_with_overflow:
3408 case Intrinsic::umul_with_overflow:
3414 if (!isValueAvailable(
II))
3420 for (
auto Itr = std::prev(Start); Itr !=
End; --Itr) {
3423 if (!isa<ExtractValueInst>(Itr))
3427 const auto *EVI = cast<ExtractValueInst>(Itr);
3428 if (EVI->getAggregateOperand() !=
II)
3436bool AArch64FastISel::fastLowerIntrinsicCall(
const IntrinsicInst *
II) {
3438 switch (
II->getIntrinsicID()) {
3439 default:
return false;
3440 case Intrinsic::frameaddress: {
3446 Register SrcReg =
MRI.createVirtualRegister(&AArch64::GPR64RegClass);
3447 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3455 unsigned Depth = cast<ConstantInt>(
II->getOperand(0))->getZExtValue();
3457 DestReg = fastEmitInst_ri(AArch64::LDRXui, &AArch64::GPR64RegClass,
3459 assert(DestReg &&
"Unexpected LDR instruction emission failure.");
3463 updateValueMap(
II, SrcReg);
3466 case Intrinsic::sponentry: {
3471 Register ResultReg = createResultReg(&AArch64::GPR64spRegClass);
3472 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3473 TII.get(AArch64::ADDXri), ResultReg)
3478 updateValueMap(
II, ResultReg);
3481 case Intrinsic::memcpy:
3482 case Intrinsic::memmove: {
3483 const auto *MTI = cast<MemTransferInst>(
II);
3485 if (MTI->isVolatile())
3490 bool IsMemCpy = (
II->getIntrinsicID() == Intrinsic::memcpy);
3491 if (isa<ConstantInt>(MTI->getLength()) && IsMemCpy) {
3494 uint64_t Len = cast<ConstantInt>(MTI->getLength())->getZExtValue();
3496 if (MTI->getDestAlign() || MTI->getSourceAlign())
3497 Alignment = std::min(MTI->getDestAlign().valueOrOne(),
3498 MTI->getSourceAlign().valueOrOne());
3499 if (isMemCpySmall(Len, Alignment)) {
3501 if (!computeAddress(MTI->getRawDest(), Dest) ||
3502 !computeAddress(MTI->getRawSource(), Src))
3504 if (tryEmitSmallMemCpy(Dest, Src, Len, Alignment))
3509 if (!MTI->getLength()->getType()->isIntegerTy(64))
3512 if (MTI->getSourceAddressSpace() > 255 || MTI->getDestAddressSpace() > 255)
3517 const char *IntrMemName = isa<MemCpyInst>(
II) ?
"memcpy" :
"memmove";
3518 return lowerCallTo(
II, IntrMemName,
II->arg_size() - 1);
3520 case Intrinsic::memset: {
3534 return lowerCallTo(
II,
"memset",
II->arg_size() - 1);
3536 case Intrinsic::sin:
3537 case Intrinsic::cos:
3538 case Intrinsic::tan:
3539 case Intrinsic::pow: {
3541 if (!isTypeLegal(
II->getType(), RetVT))
3544 if (RetVT != MVT::f32 && RetVT != MVT::f64)
3548 {RTLIB::SIN_F32, RTLIB::SIN_F64},
3549 {RTLIB::COS_F32, RTLIB::COS_F64},
3550 {RTLIB::TAN_F32, RTLIB::TAN_F64},
3551 {RTLIB::POW_F32, RTLIB::POW_F64}};
3553 bool Is64Bit = RetVT == MVT::f64;
3554 switch (
II->getIntrinsicID()) {
3557 case Intrinsic::sin:
3558 LC = LibCallTable[0][Is64Bit];
3560 case Intrinsic::cos:
3561 LC = LibCallTable[1][Is64Bit];
3563 case Intrinsic::tan:
3564 LC = LibCallTable[2][Is64Bit];
3566 case Intrinsic::pow:
3567 LC = LibCallTable[3][Is64Bit];
3572 Args.reserve(
II->arg_size());
3575 for (
auto &Arg :
II->args()) {
3578 Entry.Ty = Arg->getType();
3579 Args.push_back(Entry);
3582 CallLoweringInfo CLI;
3584 CLI.setCallee(
DL, Ctx, TLI.getLibcallCallingConv(LC),
II->getType(),
3585 TLI.getLibcallName(LC), std::move(Args));
3586 if (!lowerCallTo(CLI))
3588 updateValueMap(
II, CLI.ResultReg);
3591 case Intrinsic::fabs: {
3593 if (!isTypeLegal(
II->getType(), VT))
3601 Opc = AArch64::FABSSr;
3604 Opc = AArch64::FABSDr;
3607 Register SrcReg = getRegForValue(
II->getOperand(0));
3610 Register ResultReg = createResultReg(TLI.getRegClassFor(VT));
3611 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(Opc), ResultReg)
3613 updateValueMap(
II, ResultReg);
3616 case Intrinsic::trap:
3617 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::BRK))
3620 case Intrinsic::debugtrap:
3621 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::BRK))
3625 case Intrinsic::sqrt: {
3626 Type *
RetTy =
II->getCalledFunction()->getReturnType();
3629 if (!isTypeLegal(
RetTy, VT))
3632 Register Op0Reg = getRegForValue(
II->getOperand(0));
3636 unsigned ResultReg = fastEmit_r(VT, VT,
ISD::FSQRT, Op0Reg);
3640 updateValueMap(
II, ResultReg);
3643 case Intrinsic::sadd_with_overflow:
3644 case Intrinsic::uadd_with_overflow:
3645 case Intrinsic::ssub_with_overflow:
3646 case Intrinsic::usub_with_overflow:
3647 case Intrinsic::smul_with_overflow:
3648 case Intrinsic::umul_with_overflow: {
3651 auto *Ty = cast<StructType>(
Callee->getReturnType());
3655 if (!isTypeLegal(
RetTy, VT))
3658 if (VT != MVT::i32 && VT != MVT::i64)
3664 if (isa<ConstantInt>(LHS) && !isa<ConstantInt>(RHS) &&
II->isCommutative())
3672 case Intrinsic::smul_with_overflow:
3673 if (
const auto *
C = dyn_cast<ConstantInt>(RHS))
3674 if (
C->getValue() == 2) {
3675 IID = Intrinsic::sadd_with_overflow;
3679 case Intrinsic::umul_with_overflow:
3680 if (
const auto *
C = dyn_cast<ConstantInt>(RHS))
3681 if (
C->getValue() == 2) {
3682 IID = Intrinsic::uadd_with_overflow;
3688 unsigned ResultReg1 = 0, ResultReg2 = 0, MulReg = 0;
3692 case Intrinsic::sadd_with_overflow:
3693 ResultReg1 = emitAdd(VT, LHS, RHS,
true);
3696 case Intrinsic::uadd_with_overflow:
3697 ResultReg1 = emitAdd(VT, LHS, RHS,
true);
3700 case Intrinsic::ssub_with_overflow:
3701 ResultReg1 = emitSub(VT, LHS, RHS,
true);
3704 case Intrinsic::usub_with_overflow:
3705 ResultReg1 = emitSub(VT, LHS, RHS,
true);
3708 case Intrinsic::smul_with_overflow: {
3710 Register LHSReg = getRegForValue(LHS);
3714 Register RHSReg = getRegForValue(RHS);
3718 if (VT == MVT::i32) {
3719 MulReg = emitSMULL_rr(MVT::i64, LHSReg, RHSReg);
3721 fastEmitInst_extractsubreg(VT, MulReg, AArch64::sub_32);
3723 emitAddSub_rx(
false, MVT::i64, MulReg, MulSubReg,
3728 assert(VT == MVT::i64 &&
"Unexpected value type.");
3731 MulReg = emitMul_rr(VT, LHSReg, RHSReg);
3732 unsigned SMULHReg = fastEmit_rr(VT, VT,
ISD::MULHS, LHSReg, RHSReg);
3738 case Intrinsic::umul_with_overflow: {
3740 Register LHSReg = getRegForValue(LHS);
3744 Register RHSReg = getRegForValue(RHS);
3748 if (VT == MVT::i32) {
3749 MulReg = emitUMULL_rr(MVT::i64, LHSReg, RHSReg);
3751 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3752 TII.get(AArch64::ANDSXri), AArch64::XZR)
3755 MulReg = fastEmitInst_extractsubreg(VT, MulReg, AArch64::sub_32);
3757 assert(VT == MVT::i64 &&
"Unexpected value type.");
3760 MulReg = emitMul_rr(VT, LHSReg, RHSReg);
3761 unsigned UMULHReg = fastEmit_rr(VT, VT,
ISD::MULHU, LHSReg, RHSReg);
3762 emitSubs_rr(VT, AArch64::XZR, UMULHReg,
false);
3769 ResultReg1 = createResultReg(TLI.getRegClassFor(VT));
3770 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3771 TII.get(TargetOpcode::COPY), ResultReg1).
addReg(MulReg);
3777 ResultReg2 = fastEmitInst_rri(AArch64::CSINCWr, &AArch64::GPR32RegClass,
3778 AArch64::WZR, AArch64::WZR,
3779 getInvertedCondCode(
CC));
3781 assert((ResultReg1 + 1) == ResultReg2 &&
3782 "Nonconsecutive result registers.");
3783 updateValueMap(
II, ResultReg1, 2);
3786 case Intrinsic::aarch64_crc32b:
3787 case Intrinsic::aarch64_crc32h:
3788 case Intrinsic::aarch64_crc32w:
3789 case Intrinsic::aarch64_crc32x:
3790 case Intrinsic::aarch64_crc32cb:
3791 case Intrinsic::aarch64_crc32ch:
3792 case Intrinsic::aarch64_crc32cw:
3793 case Intrinsic::aarch64_crc32cx: {
3794 if (!Subtarget->hasCRC())
3798 switch (
II->getIntrinsicID()) {
3801 case Intrinsic::aarch64_crc32b:
3802 Opc = AArch64::CRC32Brr;
3804 case Intrinsic::aarch64_crc32h:
3805 Opc = AArch64::CRC32Hrr;
3807 case Intrinsic::aarch64_crc32w:
3808 Opc = AArch64::CRC32Wrr;
3810 case Intrinsic::aarch64_crc32x:
3811 Opc = AArch64::CRC32Xrr;
3813 case Intrinsic::aarch64_crc32cb:
3814 Opc = AArch64::CRC32CBrr;
3816 case Intrinsic::aarch64_crc32ch:
3817 Opc = AArch64::CRC32CHrr;
3819 case Intrinsic::aarch64_crc32cw:
3820 Opc = AArch64::CRC32CWrr;
3822 case Intrinsic::aarch64_crc32cx:
3823 Opc = AArch64::CRC32CXrr;
3827 Register LHSReg = getRegForValue(
II->getArgOperand(0));
3828 Register RHSReg = getRegForValue(
II->getArgOperand(1));
3829 if (!LHSReg || !RHSReg)
3833 fastEmitInst_rr(Opc, &AArch64::GPR32RegClass, LHSReg, RHSReg);
3834 updateValueMap(
II, ResultReg);
3843 const Function &
F = *
I->getParent()->getParent();
3845 if (!FuncInfo.CanLowerReturn)
3851 if (TLI.supportSwiftError() &&
3852 F.getAttributes().hasAttrSomewhere(Attribute::SwiftError))
3855 if (TLI.supportSplitCSR(FuncInfo.MF))
3861 if (
Ret->getNumOperands() > 0) {
3868 CCState CCInfo(
CC,
F.isVarArg(), *FuncInfo.MF, ValLocs,
I->getContext());
3872 if (ValLocs.
size() != 1)
3876 const Value *RV =
Ret->getOperand(0);
3894 if (!
MRI.getRegClass(SrcReg)->contains(DestReg))
3903 !Subtarget->isLittleEndian())
3907 if (RVVT == MVT::f128)
3912 if (RVVT != DestVT) {
3913 if (RVVT != MVT::i1 && RVVT != MVT::i8 && RVVT != MVT::i16)
3916 if (!Outs[0].
Flags.isZExt() && !Outs[0].Flags.isSExt())
3919 bool IsZExt = Outs[0].Flags.isZExt();
3920 SrcReg = emitIntExt(RVVT, SrcReg, DestVT, IsZExt);
3928 SrcReg = emitAnd_ri(MVT::i64, SrcReg, 0xffffffff);
3931 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3932 TII.get(TargetOpcode::COPY), DestReg).
addReg(SrcReg);
3939 TII.get(AArch64::RET_ReallyLR));
3940 for (
unsigned RetReg : RetRegs)
3945bool AArch64FastISel::selectTrunc(
const Instruction *
I) {
3946 Type *DestTy =
I->getType();
3948 Type *SrcTy =
Op->getType();
3950 EVT SrcEVT = TLI.getValueType(
DL, SrcTy,
true);
3951 EVT DestEVT = TLI.getValueType(
DL, DestTy,
true);
3960 if (SrcVT != MVT::i64 && SrcVT != MVT::i32 && SrcVT != MVT::i16 &&
3963 if (DestVT != MVT::i32 && DestVT != MVT::i16 && DestVT != MVT::i8 &&
3977 if (SrcVT == MVT::i64) {
3994 Register Reg32 = fastEmitInst_extractsubreg(MVT::i32, SrcReg,
3997 ResultReg = emitAnd_ri(MVT::i32, Reg32, Mask);
3998 assert(ResultReg &&
"Unexpected AND instruction emission failure.");
4000 ResultReg = createResultReg(&AArch64::GPR32RegClass);
4001 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
4002 TII.get(TargetOpcode::COPY), ResultReg)
4006 updateValueMap(
I, ResultReg);
4010unsigned AArch64FastISel::emiti1Ext(
unsigned SrcReg,
MVT DestVT,
bool IsZExt) {
4011 assert((DestVT == MVT::i8 || DestVT == MVT::i16 || DestVT == MVT::i32 ||
4012 DestVT == MVT::i64) &&
4013 "Unexpected value type.");
4015 if (DestVT == MVT::i8 || DestVT == MVT::i16)
4019 unsigned ResultReg = emitAnd_ri(MVT::i32, SrcReg, 1);
4020 assert(ResultReg &&
"Unexpected AND instruction emission failure.");
4021 if (DestVT == MVT::i64) {
4024 Register Reg64 =
MRI.createVirtualRegister(&AArch64::GPR64RegClass);
4025 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
4026 TII.get(AArch64::SUBREG_TO_REG), Reg64)
4029 .
addImm(AArch64::sub_32);
4034 if (DestVT == MVT::i64) {
4038 return fastEmitInst_rii(AArch64::SBFMWri, &AArch64::GPR32RegClass, SrcReg,
4043unsigned AArch64FastISel::emitMul_rr(
MVT RetVT,
unsigned Op0,
unsigned Op1) {
4051 Opc = AArch64::MADDWrrr; ZReg = AArch64::WZR;
break;
4053 Opc = AArch64::MADDXrrr; ZReg = AArch64::XZR;
break;
4057 (RetVT == MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
4058 return fastEmitInst_rrr(Opc, RC, Op0, Op1, ZReg);
4061unsigned AArch64FastISel::emitSMULL_rr(
MVT RetVT,
unsigned Op0,
unsigned Op1) {
4062 if (RetVT != MVT::i64)
4065 return fastEmitInst_rrr(AArch64::SMADDLrrr, &AArch64::GPR64RegClass,
4066 Op0, Op1, AArch64::XZR);
4069unsigned AArch64FastISel::emitUMULL_rr(
MVT RetVT,
unsigned Op0,
unsigned Op1) {
4070 if (RetVT != MVT::i64)
4073 return fastEmitInst_rrr(AArch64::UMADDLrrr, &AArch64::GPR64RegClass,
4074 Op0, Op1, AArch64::XZR);
4077unsigned AArch64FastISel::emitLSL_rr(
MVT RetVT,
unsigned Op0Reg,
4080 bool NeedTrunc =
false;
4084 case MVT::i8: Opc = AArch64::LSLVWr; NeedTrunc =
true;
Mask = 0xff;
break;
4085 case MVT::i16: Opc = AArch64::LSLVWr; NeedTrunc =
true;
Mask = 0xffff;
break;
4086 case MVT::i32: Opc = AArch64::LSLVWr;
break;
4087 case MVT::i64: Opc = AArch64::LSLVXr;
break;
4091 (RetVT == MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
4093 Op1Reg = emitAnd_ri(MVT::i32, Op1Reg, Mask);
4095 Register ResultReg = fastEmitInst_rr(Opc, RC, Op0Reg, Op1Reg);
4097 ResultReg = emitAnd_ri(MVT::i32, ResultReg, Mask);
4101unsigned AArch64FastISel::emitLSL_ri(
MVT RetVT,
MVT SrcVT,
unsigned Op0,
4104 "Unexpected source/return type pair.");
4105 assert((SrcVT == MVT::i1 || SrcVT == MVT::i8 || SrcVT == MVT::i16 ||
4106 SrcVT == MVT::i32 || SrcVT == MVT::i64) &&
4107 "Unexpected source value type.");
4108 assert((RetVT == MVT::i8 || RetVT == MVT::i16 || RetVT == MVT::i32 ||
4109 RetVT == MVT::i64) &&
"Unexpected return value type.");
4111 bool Is64Bit = (RetVT == MVT::i64);
4112 unsigned RegSize = Is64Bit ? 64 : 32;
4116 Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
4120 if (RetVT == SrcVT) {
4121 Register ResultReg = createResultReg(RC);
4122 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
4123 TII.get(TargetOpcode::COPY), ResultReg)
4127 return emitIntExt(SrcVT, Op0, RetVT, IsZExt);
4131 if (Shift >= DstBits)
4159 unsigned ImmR =
RegSize - Shift;
4161 unsigned ImmS = std::min<unsigned>(SrcBits - 1, DstBits - 1 - Shift);
4162 static const unsigned OpcTable[2][2] = {
4163 {AArch64::SBFMWri, AArch64::SBFMXri},
4164 {AArch64::UBFMWri, AArch64::UBFMXri}
4166 unsigned Opc = OpcTable[IsZExt][Is64Bit];
4167 if (SrcVT.
SimpleTy <= MVT::i32 && RetVT == MVT::i64) {
4169 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
4170 TII.get(AArch64::SUBREG_TO_REG), TmpReg)
4173 .
addImm(AArch64::sub_32);
4176 return fastEmitInst_rii(Opc, RC, Op0, ImmR, ImmS);
4179unsigned AArch64FastISel::emitLSR_rr(
MVT RetVT,
unsigned Op0Reg,
4182 bool NeedTrunc =
false;
4186 case MVT::i8: Opc = AArch64::LSRVWr; NeedTrunc =
true;
Mask = 0xff;
break;
4187 case MVT::i16: Opc = AArch64::LSRVWr; NeedTrunc =
true;
Mask = 0xffff;
break;
4188 case MVT::i32: Opc = AArch64::LSRVWr;
break;
4189 case MVT::i64: Opc = AArch64::LSRVXr;
break;
4193 (RetVT == MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
4195 Op0Reg = emitAnd_ri(MVT::i32, Op0Reg, Mask);
4196 Op1Reg = emitAnd_ri(MVT::i32, Op1Reg, Mask);
4198 Register ResultReg = fastEmitInst_rr(Opc, RC, Op0Reg, Op1Reg);
4200 ResultReg = emitAnd_ri(MVT::i32, ResultReg, Mask);
4204unsigned AArch64FastISel::emitLSR_ri(
MVT RetVT,
MVT SrcVT,
unsigned Op0,
4207 "Unexpected source/return type pair.");
4208 assert((SrcVT == MVT::i1 || SrcVT == MVT::i8 || SrcVT == MVT::i16 ||
4209 SrcVT == MVT::i32 || SrcVT == MVT::i64) &&
4210 "Unexpected source value type.");
4211 assert((RetVT == MVT::i8 || RetVT == MVT::i16 || RetVT == MVT::i32 ||
4212 RetVT == MVT::i64) &&
"Unexpected return value type.");
4214 bool Is64Bit = (RetVT == MVT::i64);
4215 unsigned RegSize = Is64Bit ? 64 : 32;
4219 Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
4223 if (RetVT == SrcVT) {
4224 Register ResultReg = createResultReg(RC);
4225 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
4226 TII.get(TargetOpcode::COPY), ResultReg)
4230 return emitIntExt(SrcVT, Op0, RetVT, IsZExt);
4234 if (Shift >= DstBits)
4262 if (Shift >= SrcBits && IsZExt)
4263 return materializeInt(ConstantInt::get(*Context,
APInt(
RegSize, 0)), RetVT);
4268 Op0 = emitIntExt(SrcVT, Op0, RetVT, IsZExt);
4276 unsigned ImmR = std::min<unsigned>(SrcBits - 1, Shift);
4277 unsigned ImmS = SrcBits - 1;
4278 static const unsigned OpcTable[2][2] = {
4279 {AArch64::SBFMWri, AArch64::SBFMXri},
4280 {AArch64::UBFMWri, AArch64::UBFMXri}
4282 unsigned Opc = OpcTable[IsZExt][Is64Bit];
4283 if (SrcVT.
SimpleTy <= MVT::i32 && RetVT == MVT::i64) {
4285 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
4286 TII.get(AArch64::SUBREG_TO_REG), TmpReg)
4289 .
addImm(AArch64::sub_32);
4292 return fastEmitInst_rii(Opc, RC, Op0, ImmR, ImmS);
4295unsigned AArch64FastISel::emitASR_rr(
MVT RetVT,
unsigned Op0Reg,
4298 bool NeedTrunc =
false;
4302 case MVT::i8: Opc = AArch64::ASRVWr; NeedTrunc =
true;
Mask = 0xff;
break;
4303 case MVT::i16: Opc = AArch64::ASRVWr; NeedTrunc =
true;
Mask = 0xffff;
break;
4304 case MVT::i32: Opc = AArch64::ASRVWr;
break;
4305 case MVT::i64: Opc = AArch64::ASRVXr;
break;
4309 (RetVT == MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
4311 Op0Reg = emitIntExt(RetVT, Op0Reg, MVT::i32,
false);
4312 Op1Reg = emitAnd_ri(MVT::i32, Op1Reg, Mask);
4314 Register ResultReg = fastEmitInst_rr(Opc, RC, Op0Reg, Op1Reg);
4316 ResultReg = emitAnd_ri(MVT::i32, ResultReg, Mask);
4320unsigned AArch64FastISel::emitASR_ri(
MVT RetVT,
MVT SrcVT,
unsigned Op0,
4323 "Unexpected source/return type pair.");
4324 assert((SrcVT == MVT::i1 || SrcVT == MVT::i8 || SrcVT == MVT::i16 ||
4325 SrcVT == MVT::i32 || SrcVT == MVT::i64) &&
4326 "Unexpected source value type.");
4327 assert((RetVT == MVT::i8 || RetVT == MVT::i16 || RetVT == MVT::i32 ||
4328 RetVT == MVT::i64) &&
"Unexpected return value type.");
4330 bool Is64Bit = (RetVT == MVT::i64);
4331 unsigned RegSize = Is64Bit ? 64 : 32;
4335 Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
4339 if (RetVT == SrcVT) {
4340 Register ResultReg = createResultReg(RC);
4341 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
4342 TII.get(TargetOpcode::COPY), ResultReg)
4346 return emitIntExt(SrcVT, Op0, RetVT, IsZExt);
4350 if (Shift >= DstBits)
4378 if (Shift >= SrcBits && IsZExt)
4379 return materializeInt(ConstantInt::get(*Context,
APInt(
RegSize, 0)), RetVT);
4381 unsigned ImmR = std::min<unsigned>(SrcBits - 1, Shift);
4382 unsigned ImmS = SrcBits - 1;
4383 static const unsigned OpcTable[2][2] = {
4384 {AArch64::SBFMWri, AArch64::SBFMXri},
4385 {AArch64::UBFMWri, AArch64::UBFMXri}
4387 unsigned Opc = OpcTable[IsZExt][Is64Bit];
4388 if (SrcVT.
SimpleTy <= MVT::i32 && RetVT == MVT::i64) {
4390 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
4391 TII.get(AArch64::SUBREG_TO_REG), TmpReg)
4394 .
addImm(AArch64::sub_32);
4397 return fastEmitInst_rii(Opc, RC, Op0, ImmR, ImmS);
4400unsigned AArch64FastISel::emitIntExt(
MVT SrcVT,
unsigned SrcReg,
MVT DestVT,
4402 assert(DestVT != MVT::i1 &&
"ZeroExt/SignExt an i1?");
4408 if (((DestVT != MVT::i8) && (DestVT != MVT::i16) &&
4409 (DestVT != MVT::i32) && (DestVT != MVT::i64)) ||
4410 ((SrcVT != MVT::i1) && (SrcVT != MVT::i8) &&
4411 (SrcVT != MVT::i16) && (SrcVT != MVT::i32)))
4421 return emiti1Ext(SrcReg, DestVT, IsZExt);
4423 if (DestVT == MVT::i64)
4424 Opc = IsZExt ? AArch64::UBFMXri : AArch64::SBFMXri;
4426 Opc = IsZExt ? AArch64::UBFMWri : AArch64::SBFMWri;
4430 if (DestVT == MVT::i64)
4431 Opc = IsZExt ? AArch64::UBFMXri : AArch64::SBFMXri;
4433 Opc = IsZExt ? AArch64::UBFMWri : AArch64::SBFMWri;
4437 assert(DestVT == MVT::i64 &&
"IntExt i32 to i32?!?");
4438 Opc = IsZExt ? AArch64::UBFMXri : AArch64::SBFMXri;
4444 if (DestVT == MVT::i8 || DestVT == MVT::i16)
4446 else if (DestVT == MVT::i64) {
4447 Register Src64 =
MRI.createVirtualRegister(&AArch64::GPR64RegClass);
4448 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
4449 TII.get(AArch64::SUBREG_TO_REG), Src64)
4452 .
addImm(AArch64::sub_32);
4457 (DestVT == MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
4458 return fastEmitInst_rii(Opc, RC, SrcReg, 0, Imm);
4465 case AArch64::LDURBBi:
4466 case AArch64::LDURHHi:
4467 case AArch64::LDURWi:
4468 case AArch64::LDRBBui:
4469 case AArch64::LDRHHui:
4470 case AArch64::LDRWui:
4471 case AArch64::LDRBBroX:
4472 case AArch64::LDRHHroX:
4473 case AArch64::LDRWroX:
4474 case AArch64::LDRBBroW:
4475 case AArch64::LDRHHroW:
4476 case AArch64::LDRWroW:
4485 case AArch64::LDURSBWi:
4486 case AArch64::LDURSHWi:
4487 case AArch64::LDURSBXi:
4488 case AArch64::LDURSHXi:
4489 case AArch64::LDURSWi:
4490 case AArch64::LDRSBWui:
4491 case AArch64::LDRSHWui:
4492 case AArch64::LDRSBXui:
4493 case AArch64::LDRSHXui:
4494 case AArch64::LDRSWui:
4495 case AArch64::LDRSBWroX:
4496 case AArch64::LDRSHWroX:
4497 case AArch64::LDRSBXroX:
4498 case AArch64::LDRSHXroX:
4499 case AArch64::LDRSWroX:
4500 case AArch64::LDRSBWroW:
4501 case AArch64::LDRSHWroW:
4502 case AArch64::LDRSBXroW:
4503 case AArch64::LDRSHXroW:
4504 case AArch64::LDRSWroW:
4509bool AArch64FastISel::optimizeIntExtLoad(
const Instruction *
I,
MVT RetVT,
4511 const auto *LI = dyn_cast<LoadInst>(
I->getOperand(0));
4512 if (!LI || !LI->hasOneUse())
4526 bool IsZExt = isa<ZExtInst>(
I);
4527 const auto *LoadMI =
MI;
4528 if (LoadMI->getOpcode() == TargetOpcode::COPY &&
4529 LoadMI->getOperand(1).getSubReg() == AArch64::sub_32) {
4530 Register LoadReg =
MI->getOperand(1).getReg();
4531 LoadMI =
MRI.getUniqueVRegDef(LoadReg);
4532 assert(LoadMI &&
"Expected valid instruction");
4538 if (RetVT != MVT::i64 || SrcVT > MVT::i32) {
4539 updateValueMap(
I, Reg);
4544 Register Reg64 = createResultReg(&AArch64::GPR64RegClass);
4545 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
4546 TII.get(AArch64::SUBREG_TO_REG), Reg64)
4549 .
addImm(AArch64::sub_32);
4552 assert((
MI->getOpcode() == TargetOpcode::COPY &&
4553 MI->getOperand(1).getSubReg() == AArch64::sub_32) &&
4554 "Expected copy instruction");
4555 Reg =
MI->getOperand(1).getReg();
4557 removeDeadCode(
I, std::next(
I));
4559 updateValueMap(
I, Reg);
4563bool AArch64FastISel::selectIntExt(
const Instruction *
I) {
4564 assert((isa<ZExtInst>(
I) || isa<SExtInst>(
I)) &&
4565 "Unexpected integer extend instruction.");
4568 if (!isTypeSupported(
I->getType(), RetVT))
4571 if (!isTypeSupported(
I->getOperand(0)->getType(), SrcVT))
4575 if (optimizeIntExtLoad(
I, RetVT, SrcVT))
4578 Register SrcReg = getRegForValue(
I->getOperand(0));
4583 bool IsZExt = isa<ZExtInst>(
I);
4584 if (
const auto *Arg = dyn_cast<Argument>(
I->getOperand(0))) {
4585 if ((IsZExt && Arg->hasZExtAttr()) || (!IsZExt && Arg->hasSExtAttr())) {
4586 if (RetVT == MVT::i64 && SrcVT != MVT::i64) {
4587 Register ResultReg = createResultReg(&AArch64::GPR64RegClass);
4588 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
4589 TII.get(AArch64::SUBREG_TO_REG), ResultReg)
4592 .
addImm(AArch64::sub_32);
4596 updateValueMap(
I, SrcReg);
4601 unsigned ResultReg = emitIntExt(SrcVT, SrcReg, RetVT, IsZExt);
4605 updateValueMap(
I, ResultReg);
4609bool AArch64FastISel::selectRem(
const Instruction *
I,
unsigned ISDOpcode) {
4610 EVT DestEVT = TLI.getValueType(
DL,
I->getType(),
true);
4615 if (DestVT != MVT::i64 && DestVT != MVT::i32)
4619 bool Is64bit = (DestVT == MVT::i64);
4620 switch (ISDOpcode) {
4624 DivOpc = Is64bit ? AArch64::SDIVXr : AArch64::SDIVWr;
4627 DivOpc = Is64bit ? AArch64::UDIVXr : AArch64::UDIVWr;
4630 unsigned MSubOpc = Is64bit ? AArch64::MSUBXrrr : AArch64::MSUBWrrr;
4631 Register Src0Reg = getRegForValue(
I->getOperand(0));
4635 Register Src1Reg = getRegForValue(
I->getOperand(1));
4640 (DestVT == MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
4641 Register QuotReg = fastEmitInst_rr(DivOpc, RC, Src0Reg, Src1Reg);
4642 assert(QuotReg &&
"Unexpected DIV instruction emission failure.");
4645 Register ResultReg = fastEmitInst_rrr(MSubOpc, RC, QuotReg, Src1Reg, Src0Reg);
4646 updateValueMap(
I, ResultReg);
4652 if (!isTypeSupported(
I->getType(), VT,
true))
4658 const Value *Src0 =
I->getOperand(0);
4659 const Value *Src1 =
I->getOperand(1);
4660 if (
const auto *
C = dyn_cast<ConstantInt>(Src0))
4661 if (
C->getValue().isPowerOf2())
4665 if (
const auto *
C = dyn_cast<ConstantInt>(Src1))
4666 if (
C->getValue().isPowerOf2()) {
4667 uint64_t ShiftVal =
C->getValue().logBase2();
4670 if (
const auto *ZExt = dyn_cast<ZExtInst>(Src0)) {
4673 if (isValueAvailable(ZExt) && isTypeSupported(ZExt->getSrcTy(), VT)) {
4676 Src0 = ZExt->getOperand(0);
4679 }
else if (
const auto *SExt = dyn_cast<SExtInst>(Src0)) {
4682 if (isValueAvailable(SExt) && isTypeSupported(SExt->getSrcTy(), VT)) {
4685 Src0 = SExt->getOperand(0);
4690 Register Src0Reg = getRegForValue(Src0);
4694 unsigned ResultReg =
4695 emitLSL_ri(VT, SrcVT, Src0Reg, ShiftVal, IsZExt);
4698 updateValueMap(
I, ResultReg);
4703 Register Src0Reg = getRegForValue(
I->getOperand(0));
4707 Register Src1Reg = getRegForValue(
I->getOperand(1));
4711 unsigned ResultReg = emitMul_rr(VT, Src0Reg, Src1Reg);
4716 updateValueMap(
I, ResultReg);
4720bool AArch64FastISel::selectShift(
const Instruction *
I) {
4722 if (!isTypeSupported(
I->getType(), RetVT,
true))
4726 return selectOperator(
I,
I->getOpcode());
4728 if (
const auto *
C = dyn_cast<ConstantInt>(
I->getOperand(1))) {
4729 unsigned ResultReg = 0;
4732 bool IsZExt =
I->getOpcode() != Instruction::AShr;
4733 const Value *Op0 =
I->getOperand(0);
4734 if (
const auto *ZExt = dyn_cast<ZExtInst>(Op0)) {
4737 if (isValueAvailable(ZExt) && isTypeSupported(ZExt->getSrcTy(), TmpVT)) {
4740 Op0 = ZExt->getOperand(0);
4743 }
else if (
const auto *SExt = dyn_cast<SExtInst>(Op0)) {
4746 if (isValueAvailable(SExt) && isTypeSupported(SExt->getSrcTy(), TmpVT)) {
4749 Op0 = SExt->getOperand(0);
4754 Register Op0Reg = getRegForValue(Op0);
4758 switch (
I->getOpcode()) {
4760 case Instruction::Shl:
4761 ResultReg = emitLSL_ri(RetVT, SrcVT, Op0Reg, ShiftVal, IsZExt);
4763 case Instruction::AShr:
4764 ResultReg = emitASR_ri(RetVT, SrcVT, Op0Reg, ShiftVal, IsZExt);
4766 case Instruction::LShr:
4767 ResultReg = emitLSR_ri(RetVT, SrcVT, Op0Reg, ShiftVal, IsZExt);
4773 updateValueMap(
I, ResultReg);
4777 Register Op0Reg = getRegForValue(
I->getOperand(0));
4781 Register Op1Reg = getRegForValue(
I->getOperand(1));
4785 unsigned ResultReg = 0;
4786 switch (
I->getOpcode()) {
4788 case Instruction::Shl:
4789 ResultReg = emitLSL_rr(RetVT, Op0Reg, Op1Reg);
4791 case Instruction::AShr:
4792 ResultReg = emitASR_rr(RetVT, Op0Reg, Op1Reg);
4794 case Instruction::LShr:
4795 ResultReg = emitLSR_rr(RetVT, Op0Reg, Op1Reg);
4802 updateValueMap(
I, ResultReg);
4806bool AArch64FastISel::selectBitCast(
const Instruction *
I) {
4809 if (!isTypeLegal(
I->getOperand(0)->getType(), SrcVT))
4811 if (!isTypeLegal(
I->getType(), RetVT))
4815 if (RetVT == MVT::f32 && SrcVT == MVT::i32)
4816 Opc = AArch64::FMOVWSr;
4817 else if (RetVT == MVT::f64 && SrcVT == MVT::i64)
4818 Opc = AArch64::FMOVXDr;
4819 else if (RetVT == MVT::i32 && SrcVT == MVT::f32)
4820 Opc = AArch64::FMOVSWr;
4821 else if (RetVT == MVT::i64 && SrcVT == MVT::f64)
4822 Opc = AArch64::FMOVDXr;
4829 case MVT::i32: RC = &AArch64::GPR32RegClass;
break;
4830 case MVT::i64: RC = &AArch64::GPR64RegClass;
break;
4831 case MVT::f32: RC = &AArch64::FPR32RegClass;
break;
4832 case MVT::f64: RC = &AArch64::FPR64RegClass;
break;
4834 Register Op0Reg = getRegForValue(
I->getOperand(0));
4838 Register ResultReg = fastEmitInst_r(Opc, RC, Op0Reg);
4842 updateValueMap(
I, ResultReg);
4846bool AArch64FastISel::selectFRem(
const Instruction *
I) {
4848 if (!isTypeLegal(
I->getType(), RetVT))
4856 LC = RTLIB::REM_F32;
4859 LC = RTLIB::REM_F64;
4864 Args.reserve(
I->getNumOperands());
4867 for (
auto &Arg :
I->operands()) {
4870 Entry.Ty = Arg->getType();
4871 Args.push_back(Entry);
4874 CallLoweringInfo CLI;
4876 CLI.setCallee(
DL, Ctx, TLI.getLibcallCallingConv(LC),
I->getType(),
4877 TLI.getLibcallName(LC), std::move(Args));
4878 if (!lowerCallTo(CLI))
4880 updateValueMap(
I, CLI.ResultReg);
4884bool AArch64FastISel::selectSDiv(
const Instruction *
I) {
4886 if (!isTypeLegal(
I->getType(), VT))
4889 if (!isa<ConstantInt>(
I->getOperand(1)))
4892 const APInt &
C = cast<ConstantInt>(
I->getOperand(1))->getValue();
4893 if ((VT != MVT::i32 && VT != MVT::i64) || !
C ||
4894 !(
C.isPowerOf2() ||
C.isNegatedPowerOf2()))
4897 unsigned Lg2 =
C.countr_zero();
4898 Register Src0Reg = getRegForValue(
I->getOperand(0));
4902 if (cast<BinaryOperator>(
I)->isExact()) {
4903 unsigned ResultReg = emitASR_ri(VT, VT, Src0Reg, Lg2);
4906 updateValueMap(
I, ResultReg);
4910 int64_t Pow2MinusOne = (1ULL << Lg2) - 1;
4911 unsigned AddReg = emitAdd_ri_(VT, Src0Reg, Pow2MinusOne);
4916 if (!emitICmp_ri(VT, Src0Reg, 0))
4921 if (VT == MVT::i64) {
4922 SelectOpc = AArch64::CSELXr;
4923 RC = &AArch64::GPR64RegClass;
4925 SelectOpc = AArch64::CSELWr;
4926 RC = &AArch64::GPR32RegClass;
4928 Register SelectReg = fastEmitInst_rri(SelectOpc, RC, AddReg, Src0Reg,
4935 unsigned ZeroReg = (VT == MVT::i64) ? AArch64::XZR : AArch64::WZR;
4938 ResultReg = emitAddSub_rs(
false, VT, ZeroReg, SelectReg,
4941 ResultReg = emitASR_ri(VT, VT, SelectReg, Lg2);
4946 updateValueMap(
I, ResultReg);
4953unsigned AArch64FastISel::getRegForGEPIndex(
const Value *
Idx) {
4960 MVT PtrVT = TLI.getPointerTy(
DL);
4962 if (IdxVT.
bitsLT(PtrVT)) {
4963 IdxN = emitIntExt(IdxVT.
getSimpleVT(), IdxN, PtrVT,
false);
4964 }
else if (IdxVT.
bitsGT(PtrVT))
4965 llvm_unreachable(
"AArch64 FastISel doesn't support types larger than i64");
4973bool AArch64FastISel::selectGetElementPtr(
const Instruction *
I) {
4974 if (Subtarget->isTargetILP32())
4977 Register N = getRegForValue(
I->getOperand(0));
4984 MVT VT = TLI.getPointerTy(
DL);
4987 const Value *
Idx = GTI.getOperand();
4988 if (
auto *StTy = GTI.getStructTypeOrNull()) {
4989 unsigned Field = cast<ConstantInt>(
Idx)->getZExtValue();
4992 TotalOffs +=
DL.getStructLayout(StTy)->getElementOffset(
Field);
4995 if (
const auto *CI = dyn_cast<ConstantInt>(
Idx)) {
4999 TotalOffs += GTI.getSequentialElementStride(
DL) *
5000 cast<ConstantInt>(CI)->getSExtValue();
5004 N = emitAdd_ri_(VT,
N, TotalOffs);
5011 uint64_t ElementSize = GTI.getSequentialElementStride(
DL);
5012 unsigned IdxN = getRegForGEPIndex(
Idx);
5016 if (ElementSize != 1) {
5020 IdxN = emitMul_rr(VT, IdxN,
C);
5030 N = emitAdd_ri_(VT,
N, TotalOffs);
5034 updateValueMap(
I,
N);
5039 assert(
TM.getOptLevel() == CodeGenOptLevel::None &&
5040 "cmpxchg survived AtomicExpand at optlevel > -O0");
5042 auto *RetPairTy = cast<StructType>(
I->getType());
5043 Type *
RetTy = RetPairTy->getTypeAtIndex(0U);
5044 assert(RetPairTy->getTypeAtIndex(1U)->isIntegerTy(1) &&
5045 "cmpxchg has a non-i1 status result");
5048 if (!isTypeLegal(
RetTy, VT))
5052 unsigned Opc, CmpOpc;
5055 if (VT == MVT::i32) {
5056 Opc = AArch64::CMP_SWAP_32;
5057 CmpOpc = AArch64::SUBSWrs;
5058 ResRC = &AArch64::GPR32RegClass;
5059 }
else if (VT == MVT::i64) {
5060 Opc = AArch64::CMP_SWAP_64;
5061 CmpOpc = AArch64::SUBSXrs;
5062 ResRC = &AArch64::GPR64RegClass;
5070 II, getRegForValue(
I->getPointerOperand()),
II.getNumDefs());
5072 II, getRegForValue(
I->getCompareOperand()),
II.getNumDefs() + 1);
5074 II, getRegForValue(
I->getNewValOperand()),
II.getNumDefs() + 2);
5076 const Register ResultReg1 = createResultReg(ResRC);
5077 const Register ResultReg2 = createResultReg(&AArch64::GPR32RegClass);
5078 const Register ScratchReg = createResultReg(&AArch64::GPR32RegClass);
5081 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
II)
5088 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(CmpOpc))
5089 .
addDef(VT == MVT::i32 ? AArch64::WZR : AArch64::XZR)
5094 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
TII.get(AArch64::CSINCWr))
5100 assert((ResultReg1 + 1) == ResultReg2 &&
"Nonconsecutive result registers.");
5101 updateValueMap(
I, ResultReg1, 2);
5105bool AArch64FastISel::fastSelectInstruction(
const Instruction *
I) {
5106 if (TLI.fallBackToDAGISel(*
I))
5108 switch (
I->getOpcode()) {
5111 case Instruction::Add:
5112 case Instruction::Sub:
5113 return selectAddSub(
I);
5114 case Instruction::Mul:
5115 return selectMul(
I);
5116 case Instruction::SDiv:
5117 return selectSDiv(
I);
5118 case Instruction::SRem:
5122 case Instruction::URem:
5126 case Instruction::Shl:
5127 case Instruction::LShr:
5128 case Instruction::AShr:
5129 return selectShift(
I);
5130 case Instruction::And:
5131 case Instruction::Or:
5132 case Instruction::Xor:
5133 return selectLogicalOp(
I);
5134 case Instruction::Br:
5135 return selectBranch(
I);
5136 case Instruction::IndirectBr:
5137 return selectIndirectBr(
I);
5138 case Instruction::BitCast:
5140 return selectBitCast(
I);
5142 case Instruction::FPToSI:
5144 return selectFPToInt(
I,
true);
5146 case Instruction::FPToUI:
5147 return selectFPToInt(
I,
false);
5148 case Instruction::ZExt:
5149 case Instruction::SExt:
5150 return selectIntExt(
I);
5151 case Instruction::Trunc:
5153 return selectTrunc(
I);
5155 case Instruction::FPExt:
5156 return selectFPExt(
I);
5157 case Instruction::FPTrunc:
5158 return selectFPTrunc(
I);
5159 case Instruction::SIToFP:
5161 return selectIntToFP(
I,
true);
5163 case Instruction::UIToFP:
5164 return selectIntToFP(
I,
false);
5165 case Instruction::Load:
5166 return selectLoad(
I);
5167 case Instruction::Store:
5168 return selectStore(
I);
5169 case Instruction::FCmp:
5170 case Instruction::ICmp:
5171 return selectCmp(
I);
5172 case Instruction::Select:
5173 return selectSelect(
I);
5174 case Instruction::Ret:
5175 return selectRet(
I);
5176 case Instruction::FRem:
5177 return selectFRem(
I);
5178 case Instruction::GetElementPtr:
5179 return selectGetElementPtr(
I);
5180 case Instruction::AtomicCmpXchg:
5181 return selectAtomicCmpXchg(cast<AtomicCmpXchgInst>(
I));
5185 return selectOperator(
I,
I->getOpcode());
5196 return new AArch64FastISel(FuncInfo, LibInfo);
unsigned const MachineRegisterInfo * MRI
static bool isIntExtFree(const Instruction *I)
Check if the sign-/zero-extend will be a noop.
static bool isSExtLoad(const MachineInstr *LI)
static AArch64CC::CondCode getCompareCC(CmpInst::Predicate Pred)
static bool isMulPowOf2(const Value *I)
Check if the multiply is by a power-of-2 constant.
static unsigned getImplicitScaleFactor(MVT VT)
Determine the implicit scale factor that is applied by a memory operation for a given value type.
static bool isZExtLoad(const MachineInstr *LI)
static unsigned selectBinaryOp(unsigned GenericOpc, unsigned RegBankID, unsigned OpSize)
Select the AArch64 opcode for the basic binary operation GenericOpc (such as G_OR or G_SDIV),...
static void emitLoad(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator Pos, const TargetInstrInfo &TII, unsigned Reg1, unsigned Reg2, int Offset, bool IsPostDec)
Emit a load-pair instruction for frame-destroy.
static void emitStore(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator Pos, const TargetInstrInfo &TII, unsigned Reg1, unsigned Reg2, int Offset, bool IsPreDec)
Emit a store-pair instruction for frame-setup.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file defines the DenseMap class.
This file defines the FastISel class.
const HexagonInstrInfo * TII
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
unsigned const TargetRegisterInfo * TRI
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
Module.h This file contains the declarations for the Module class.
uint64_t IntrinsicInst * II
const char LLVMTargetMachineRef TM
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
SI Pre allocate WWM Registers
This file defines the SmallVector class.
static SDValue emitCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C)
static const unsigned FramePtr
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
bool branchTargetEnforcement() const
APInt bitcastToAPInt() const
Class for arbitrary precision integers.
uint64_t getZExtValue() const
Get zero extended value.
an instruction to allocate memory on the stack
PointerType * getType() const
Overload to return most specific pointer type.
This class represents an incoming formal argument to a Function.
An instruction that atomically checks whether a specified value is in a memory location,...
InstListType::const_iterator const_iterator
Conditional or Unconditional Branch instruction.
BasicBlock * getSuccessor(unsigned i) const
bool isUnconditional() const
Value * getCondition() const
CCState - This class holds information needed while lowering arguments and return values.
CCValAssign - Represent assignment of one arg/retval to a location.
Register getLocReg() const
LocInfo getLocInfo() const
unsigned getValNo() const
This class is the base class for the comparison instructions.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
@ FCMP_TRUE
1 1 1 1 Always true (always folded)
@ ICMP_SLT
signed less than
@ ICMP_SLE
signed less or equal
@ FCMP_OLT
0 1 0 0 True if ordered and less than
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
@ ICMP_UGE
unsigned greater or equal
@ ICMP_UGT
unsigned greater than
@ ICMP_SGT
signed greater than
@ FCMP_ULT
1 1 0 0 True if unordered or less than
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
@ ICMP_ULT
unsigned less than
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
@ ICMP_SGE
signed greater or equal
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
@ ICMP_ULE
unsigned less or equal
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
@ FCMP_FALSE
0 0 0 0 Always false (always folded)
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE,...
A constant value that is initialized with an expression using other constant values.
ConstantFP - Floating Point Values [float, double].
const APFloat & getValueAPF() const
bool isNegative() const
Return true if the sign bit is set.
bool isZero() const
Return true if the value is positive or negative zero.
This is the shared class of boolean and integer constants.
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
This is an important base class in LLVM.
bool isNullValue() const
Return true if this is the value that would be returned by getNullValue.
This class represents an Operation in the Expression.
constexpr bool isVector() const
One or more elements.
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
bool selectGetElementPtr(const User *I)
virtual unsigned fastMaterializeFloatZero(const ConstantFP *CF)
Emit the floating-point constant +0.0 in a register using target- specific logic.
virtual bool fastLowerIntrinsicCall(const IntrinsicInst *II)
This method is called by target-independent code to do target- specific intrinsic lowering.
virtual unsigned fastMaterializeConstant(const Constant *C)
Emit a constant in a register using target-specific logic, such as constant pool loads.
virtual bool fastLowerCall(CallLoweringInfo &CLI)
This method is called by target-independent code to do target- specific call lowering.
virtual bool fastLowerArguments()
This method is called by target-independent code to do target- specific argument lowering.
Register getRegForGEPIndex(const Value *Idx)
This is a wrapper around getRegForValue that also takes care of truncating or sign-extending the give...
virtual bool fastSelectInstruction(const Instruction *I)=0
This method is called by target-independent code when the normal FastISel process fails to select an ...
bool selectBitCast(const User *I)
virtual unsigned fastMaterializeAlloca(const AllocaInst *C)
Emit an alloca address in a register using target-specific logic.
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
bool isThreadLocal() const
If the value is "Thread Local", its value isn't shared by the threads.
PointerType * getType() const
Global values are always pointers.
Indirect Branch Instruction.
iterator_range< succ_op_iterator > successors()
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
A wrapper class for inspecting calls to intrinsic functions.
This is an important class for using LLVM in a threaded context.
Context object for machine code objects.
Describe properties that are true of each instruction in the target description file.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
bool is128BitVector() const
Return true if this is a 128-bit vector type.
bool isVector() const
Return true if this is a vector value type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
static MVT getIntegerVT(unsigned BitWidth)
bool is64BitVector() const
Return true if this is a 64-bit vector type.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
void setFrameAddressIsTaken(bool T)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addConstantPoolIndex(unsigned Idx, int Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addRegMask(const uint32_t *Mask) const
const MachineInstrBuilder & addGlobalAddress(const GlobalValue *GV, int64_t Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
Value * getLength() const
unsigned getDestAddressSpace() const
This class wraps the llvm.memset and llvm.memset.inline intrinsics.
Wrapper class representing virtual and physical registers.
Return a value (possibly void), from a function.
SMEAttrs is a utility class to parse the SME ACLE attributes on functions.
bool hasStreamingCompatibleInterface() const
bool hasStreamingInterfaceOrBody() const
This class represents the LLVM 'select' instruction.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void reserve(size_type N)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Used to lazily calculate structure layout information for a target machine, based on the DataLayout s...
TypeSize getElementOffset(unsigned Idx) const
Class to represent struct types.
Provides information about what library functions are available for the current target.
Target - Wrapper for Target specific information.
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
bool isArrayTy() const
True if this is an instance of ArrayType.
bool isPointerTy() const
True if this is an instance of PointerType.
bool isStructTy() const
True if this is an instance of StructType.
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
static IntegerType * getInt64Ty(LLVMContext &C)
bool isIntegerTy() const
True if this is an instance of IntegerType.
Value * getOperand(unsigned i) const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
const ParentTy * getParent() const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ MO_NC
MO_NC - Indicates whether the linker is expected to check the symbol reference for overflow.
@ MO_PAGEOFF
MO_PAGEOFF - A symbol operand with this flag represents the offset of that symbol within a 4K page.
@ MO_GOT
MO_GOT - This flag indicates that a symbol operand represents the address of the GOT entry for the sy...
@ MO_PREL
MO_PREL - Indicates that the bits of the symbol operand represented by MO_G0 etc are PC relative.
@ MO_PAGE
MO_PAGE - A symbol operand with this flag represents the pc-relative offset of the 4K page containing...
@ MO_TAGGED
MO_TAGGED - With MO_PAGE, indicates that the page includes a memory tag in bits 56-63.
@ MO_G3
MO_G3 - A symbol operand with this flag (granule 3) represents the high 16-bits of a 64-bit address,...
static bool isLogicalImmediate(uint64_t imm, unsigned regSize)
isLogicalImmediate - Return true if the immediate is valid for a logical immediate instruction of the...
static uint64_t encodeLogicalImmediate(uint64_t imm, unsigned regSize)
encodeLogicalImmediate - Return the encoded immediate value for a logical immediate instruction of th...
static int getFP64Imm(const APInt &Imm)
getFP64Imm - Return an 8-bit floating-point version of the 64-bit floating-point value.
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo)
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
int getFP32Imm(const APInt &Imm)
getFP32Imm - Return an 8-bit floating-point version of the 32-bit floating-point value.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ Swift
Calling convention for Swift.
@ CFGuard_Check
Special calling convention on Windows for calling the Control Guard Check ICall funtion.
@ GHC
Used by the Glasgow Haskell Compiler (GHC).
@ C
The default llvm calling convention, compatible with C.
@ ADD
Simple integer binary arithmetic operators.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Predicate
Predicate - These are "(BI << 5) | BO" for various predicates.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Kill
The last use of a register.
Reg
All possible values of the reg field in the ModR/M byte.
This is an optimization pass for GlobalISel generic memory operations.
bool RetCC_AArch64_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool CC_AArch64_GHC(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
Register constrainOperandRegClass(const MachineFunction &MF, const TargetRegisterInfo &TRI, MachineRegisterInfo &MRI, const TargetInstrInfo &TII, const RegisterBankInfo &RBI, MachineInstr &InsertPt, const TargetRegisterClass &RegClass, MachineOperand &RegMO)
Constrain the Register operand OpIdx, so that it is now constrained to the TargetRegisterClass passed...
void GetReturnInfo(CallingConv::ID CC, Type *ReturnType, AttributeList attr, SmallVectorImpl< ISD::OutputArg > &Outs, const TargetLowering &TLI, const DataLayout &DL)
Given an LLVM IR type and return type attributes, compute the return value EVTs and flags,...
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool CC_AArch64_Win64PCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool CC_AArch64_Win64_CFGuard_Check(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
unsigned getBLRCallOpcode(const MachineFunction &MF)
Return opcode to be used for indirect calls.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
gep_type_iterator gep_type_end(const User *GEP)
bool isReleaseOrStronger(AtomicOrdering AO)
static Error getOffset(const SymbolRef &Sym, SectionRef Sec, uint64_t &Result)
bool CC_AArch64_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
AtomicOrdering
Atomic ordering for LLVM's memory model.
bool CCAssignFn(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
CCAssignFn - This function assigns a location for Val, updating State to reflect the change.
unsigned getKillRegState(bool B)
bool CC_AArch64_DarwinPCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
gep_type_iterator gep_type_begin(const User *GEP)
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
ElementCount getVectorElementCount() const
static EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool isVector() const
Return true if this is a vector value type.
static MachinePointerInfo getStack(MachineFunction &MF, int64_t Offset, uint8_t ID=0)
Stack pointer relative access.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.