44 class AArch64FastISel final :
public FastISel {
66 OffsetReg(0), Shift(0),
Offset(0), GV(nullptr) { Base.Reg = 0; }
67 void setKind(BaseKind K) {
Kind = K; }
68 BaseKind getKind()
const {
return Kind; }
71 bool isRegBase()
const {
return Kind == RegBase; }
72 bool isFIBase()
const {
return Kind == FrameIndexBase; }
73 void setReg(
unsigned Reg) {
74 assert(isRegBase() &&
"Invalid base register access!");
78 assert(isRegBase() &&
"Invalid base register access!");
81 void setOffsetReg(
unsigned Reg) {
84 unsigned getOffsetReg()
const {
87 void setFI(
unsigned FI) {
88 assert(isFIBase() &&
"Invalid base frame index access!");
91 unsigned getFI()
const {
92 assert(isFIBase() &&
"Invalid base frame index access!");
95 void setOffset(int64_t O) {
Offset = O; }
97 void setShift(
unsigned S) { Shift = S; }
98 unsigned getShift() {
return Shift; }
109 bool fastLowerArguments()
override;
110 bool fastLowerCall(CallLoweringInfo &CLI)
override;
111 bool fastLowerIntrinsicCall(
const IntrinsicInst *II)
override;
127 bool selectRem(
const Instruction *
I,
unsigned ISDOpcode);
140 bool isTypeLegal(
Type *Ty,
MVT &VT);
141 bool isTypeSupported(
Type *Ty,
MVT &VT,
bool IsVectorAllowed =
false);
142 bool isValueAvailable(
const Value *V)
const;
149 bool isMemCpySmall(uint64_t Len,
unsigned Alignment);
156 std::pair<unsigned, bool> getRegForGEPIndex(
const Value *Idx);
159 unsigned emitAddSub(
bool UseAdd,
MVT RetVT,
const Value *LHS,
160 const Value *RHS,
bool SetFlags =
false,
161 bool WantResult =
true,
bool IsZExt =
false);
162 unsigned emitAddSub_rr(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
163 bool LHSIsKill,
unsigned RHSReg,
bool RHSIsKill,
164 bool SetFlags =
false,
bool WantResult =
true);
165 unsigned emitAddSub_ri(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
166 bool LHSIsKill, uint64_t Imm,
bool SetFlags =
false,
167 bool WantResult =
true);
168 unsigned emitAddSub_rs(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
169 bool LHSIsKill,
unsigned RHSReg,
bool RHSIsKill,
171 uint64_t ShiftImm,
bool SetFlags =
false,
172 bool WantResult =
true);
173 unsigned emitAddSub_rx(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
174 bool LHSIsKill,
unsigned RHSReg,
bool RHSIsKill,
176 uint64_t ShiftImm,
bool SetFlags =
false,
177 bool WantResult =
true);
180 bool emitCompareAndBranch(
const BranchInst *BI);
182 bool emitICmp(
MVT RetVT,
const Value *LHS,
const Value *RHS,
bool IsZExt);
183 bool emitICmp_ri(
MVT RetVT,
unsigned LHSReg,
bool LHSIsKill, uint64_t Imm);
184 bool emitFCmp(
MVT RetVT,
const Value *LHS,
const Value *RHS);
185 unsigned emitLoad(
MVT VT,
MVT ResultVT,
Address Addr,
bool WantZExt =
true,
187 bool emitStore(
MVT VT,
unsigned SrcReg,
Address Addr,
189 bool emitStoreRelease(
MVT VT,
unsigned SrcReg,
unsigned AddrReg,
191 unsigned emitIntExt(
MVT SrcVT,
unsigned SrcReg,
MVT DestVT,
bool isZExt);
192 unsigned emiti1Ext(
unsigned SrcReg,
MVT DestVT,
bool isZExt);
193 unsigned emitAdd(
MVT RetVT,
const Value *LHS,
const Value *RHS,
194 bool SetFlags =
false,
bool WantResult =
true,
195 bool IsZExt =
false);
196 unsigned emitAdd_ri_(
MVT VT,
unsigned Op0,
bool Op0IsKill, int64_t Imm);
197 unsigned emitSub(
MVT RetVT,
const Value *LHS,
const Value *RHS,
198 bool SetFlags =
false,
bool WantResult =
true,
199 bool IsZExt =
false);
200 unsigned emitSubs_rr(
MVT RetVT,
unsigned LHSReg,
bool LHSIsKill,
201 unsigned RHSReg,
bool RHSIsKill,
bool WantResult =
true);
202 unsigned emitSubs_rs(
MVT RetVT,
unsigned LHSReg,
bool LHSIsKill,
203 unsigned RHSReg,
bool RHSIsKill,
205 bool WantResult =
true);
206 unsigned emitLogicalOp(
unsigned ISDOpc,
MVT RetVT,
const Value *LHS,
208 unsigned emitLogicalOp_ri(
unsigned ISDOpc,
MVT RetVT,
unsigned LHSReg,
209 bool LHSIsKill, uint64_t Imm);
210 unsigned emitLogicalOp_rs(
unsigned ISDOpc,
MVT RetVT,
unsigned LHSReg,
211 bool LHSIsKill,
unsigned RHSReg,
bool RHSIsKill,
213 unsigned emitAnd_ri(
MVT RetVT,
unsigned LHSReg,
bool LHSIsKill, uint64_t Imm);
214 unsigned emitMul_rr(
MVT RetVT,
unsigned Op0,
bool Op0IsKill,
215 unsigned Op1,
bool Op1IsKill);
216 unsigned emitSMULL_rr(
MVT RetVT,
unsigned Op0,
bool Op0IsKill,
217 unsigned Op1,
bool Op1IsKill);
218 unsigned emitUMULL_rr(
MVT RetVT,
unsigned Op0,
bool Op0IsKill,
219 unsigned Op1,
bool Op1IsKill);
220 unsigned emitLSL_rr(
MVT RetVT,
unsigned Op0Reg,
bool Op0IsKill,
221 unsigned Op1Reg,
bool Op1IsKill);
222 unsigned emitLSL_ri(
MVT RetVT,
MVT SrcVT,
unsigned Op0Reg,
bool Op0IsKill,
223 uint64_t Imm,
bool IsZExt =
true);
224 unsigned emitLSR_rr(
MVT RetVT,
unsigned Op0Reg,
bool Op0IsKill,
225 unsigned Op1Reg,
bool Op1IsKill);
226 unsigned emitLSR_ri(
MVT RetVT,
MVT SrcVT,
unsigned Op0Reg,
bool Op0IsKill,
227 uint64_t Imm,
bool IsZExt =
true);
228 unsigned emitASR_rr(
MVT RetVT,
unsigned Op0Reg,
bool Op0IsKill,
229 unsigned Op1Reg,
bool Op1IsKill);
230 unsigned emitASR_ri(
MVT RetVT,
MVT SrcVT,
unsigned Op0Reg,
bool Op0IsKill,
231 uint64_t Imm,
bool IsZExt =
false);
242 bool finishCall(CallLoweringInfo &CLI,
MVT RetVT,
unsigned NumBytes);
246 unsigned fastMaterializeAlloca(
const AllocaInst *AI)
override;
247 unsigned fastMaterializeConstant(
const Constant *
C)
override;
248 unsigned fastMaterializeFloatZero(
const ConstantFP* CF)
override;
258 bool fastSelectInstruction(
const Instruction *
I)
override;
260 #include "AArch64GenFastISel.inc"
265 #include "AArch64GenCallingConv.inc"
269 assert((isa<ZExtInst>(I) || isa<SExtInst>(I)) &&
270 "Unexpected integer extend instruction.");
272 "Unexpected value type.");
273 bool IsZExt = isa<ZExtInst>(
I);
275 if (
const auto *LI = dyn_cast<LoadInst>(I->
getOperand(0)))
279 if (
const auto *Arg = dyn_cast<Argument>(I->
getOperand(0)))
280 if ((IsZExt && Arg->hasZExtAttr()) || (!IsZExt && Arg->hasSExtAttr()))
308 return CC_AArch64_WebKit_JS;
310 return CC_AArch64_GHC;
311 return Subtarget->isTargetDarwin() ? CC_AArch64_DarwinPCS : CC_AArch64_AAPCS;
314 unsigned AArch64FastISel::fastMaterializeAlloca(
const AllocaInst *AI) {
316 "Alloca should always return a pointer.");
319 if (!FuncInfo.StaticAllocaMap.count(AI))
323 FuncInfo.StaticAllocaMap.find(AI);
325 if (SI != FuncInfo.StaticAllocaMap.
end()) {
326 unsigned ResultReg = createResultReg(&AArch64::GPR64spRegClass);
327 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(AArch64::ADDXri),
329 .addFrameIndex(SI->second)
338 unsigned AArch64FastISel::materializeInt(
const ConstantInt *CI,
MVT VT) {
347 : &AArch64::GPR32RegClass;
348 unsigned ZeroReg = (VT ==
MVT::i64) ? AArch64::XZR : AArch64::WZR;
349 unsigned ResultReg = createResultReg(RC);
350 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(TargetOpcode::COPY),
355 unsigned AArch64FastISel::materializeFP(
const ConstantFP *CFP,
MVT VT) {
359 return fastMaterializeFloatZero(CFP);
368 if (TLI.isFPImmLegal(Val, VT)) {
371 assert((Imm != -1) &&
"Cannot encode floating-point constant.");
372 unsigned Opc = Is64Bit ? AArch64::FMOVDi : AArch64::FMOVSi;
373 return fastEmitInst_i(Opc, TLI.getRegClassFor(VT), Imm);
378 unsigned Opc1 = Is64Bit ? AArch64::MOVi64imm : AArch64::MOVi32imm;
380 &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
382 unsigned TmpReg = createResultReg(RC);
383 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(Opc1), TmpReg)
386 unsigned ResultReg = createResultReg(TLI.getRegClassFor(VT));
387 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
388 TII.get(TargetOpcode::COPY), ResultReg)
396 unsigned Align =
DL.getPrefTypeAlignment(CFP->
getType());
398 Align =
DL.getTypeAllocSize(CFP->
getType());
400 unsigned CPI = MCP.getConstantPoolIndex(cast<Constant>(CFP), Align);
401 unsigned ADRPReg = createResultReg(&AArch64::GPR64commonRegClass);
405 unsigned Opc = Is64Bit ? AArch64::LDRDui : AArch64::LDRSui;
406 unsigned ResultReg = createResultReg(TLI.getRegClassFor(VT));
407 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(Opc), ResultReg)
413 unsigned AArch64FastISel::materializeGV(
const GlobalValue *GV) {
423 unsigned char OpFlags = Subtarget->ClassifyGlobalReference(GV,
TM);
425 EVT DestEVT = TLI.getValueType(DL, GV->
getType(),
true);
429 unsigned ADRPReg = createResultReg(&AArch64::GPR64commonRegClass);
438 ResultReg = createResultReg(&AArch64::GPR64RegClass);
439 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(AArch64::LDRXui),
450 ResultReg = createResultReg(&AArch64::GPR64spRegClass);
451 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(AArch64::ADDXri),
460 unsigned AArch64FastISel::fastMaterializeConstant(
const Constant *
C) {
461 EVT CEVT = TLI.getValueType(DL, C->
getType(),
true);
468 if (
const auto *CI = dyn_cast<ConstantInt>(C))
469 return materializeInt(CI, VT);
470 else if (
const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
471 return materializeFP(CFP, VT);
472 else if (
const GlobalValue *GV = dyn_cast<GlobalValue>(C))
473 return materializeGV(GV);
478 unsigned AArch64FastISel::fastMaterializeFloatZero(
const ConstantFP* CFP) {
480 "Floating-point constant is not a positive zero.");
482 if (!isTypeLegal(CFP->
getType(), VT))
489 unsigned ZReg = Is64Bit ? AArch64::XZR : AArch64::WZR;
490 unsigned Opc = Is64Bit ? AArch64::FMOVXDr : AArch64::FMOVWSr;
491 return fastEmitInst_r(Opc, TLI.getRegClassFor(VT), ZReg,
true);
496 if (
const auto *
MI = dyn_cast<MulOperator>(I)) {
497 if (
const auto *C = dyn_cast<ConstantInt>(
MI->getOperand(0)))
498 if (C->getValue().isPowerOf2())
500 if (
const auto *C = dyn_cast<ConstantInt>(
MI->getOperand(1)))
501 if (C->getValue().isPowerOf2())
508 bool AArch64FastISel::computeAddress(
const Value *Obj,
Address &Addr,
Type *Ty)
510 const User *U =
nullptr;
511 unsigned Opcode = Instruction::UserOp1;
512 if (
const Instruction *
I = dyn_cast<Instruction>(Obj)) {
515 if (FuncInfo.StaticAllocaMap.count(static_cast<const AllocaInst *>(Obj)) ||
516 FuncInfo.MBBMap[
I->getParent()] == FuncInfo.MBB) {
517 Opcode =
I->getOpcode();
520 }
else if (
const ConstantExpr *C = dyn_cast<ConstantExpr>(Obj)) {
521 Opcode = C->getOpcode();
525 if (
auto *Ty = dyn_cast<PointerType>(Obj->
getType()))
526 if (Ty->getAddressSpace() > 255)
534 case Instruction::BitCast: {
536 return computeAddress(U->
getOperand(0), Addr, Ty);
538 case Instruction::IntToPtr: {
541 TLI.getPointerTy(DL))
542 return computeAddress(U->
getOperand(0), Addr, Ty);
545 case Instruction::PtrToInt: {
547 if (TLI.getValueType(DL, U->
getType()) == TLI.getPointerTy(DL))
548 return computeAddress(U->
getOperand(0), Addr, Ty);
551 case Instruction::GetElementPtr: {
553 uint64_t TmpOffset = Addr.getOffset();
559 const Value *
Op = GTI.getOperand();
560 if (
StructType *STy = GTI.getStructTypeOrNull()) {
562 unsigned Idx = cast<ConstantInt>(
Op)->getZExtValue();
565 uint64_t S =
DL.getTypeAllocSize(GTI.getIndexedType());
567 if (
const ConstantInt *CI = dyn_cast<ConstantInt>(Op)) {
572 if (canFoldAddIntoGEP(U, Op)) {
575 cast<ConstantInt>(cast<AddOperator>(
Op)->getOperand(1));
578 Op = cast<AddOperator>(
Op)->getOperand(0);
582 goto unsupported_gep;
588 Addr.setOffset(TmpOffset);
589 if (computeAddress(U->
getOperand(0), Addr, Ty))
598 case Instruction::Alloca: {
601 FuncInfo.StaticAllocaMap.
find(AI);
602 if (SI != FuncInfo.StaticAllocaMap.end()) {
603 Addr.setKind(Address::FrameIndexBase);
604 Addr.setFI(SI->second);
614 if (isa<ConstantInt>(LHS))
617 if (
const ConstantInt *CI = dyn_cast<ConstantInt>(RHS)) {
619 return computeAddress(LHS, Addr, Ty);
623 if (computeAddress(LHS, Addr, Ty) && computeAddress(RHS, Addr, Ty))
629 case Instruction::Sub: {
634 if (
const ConstantInt *CI = dyn_cast<ConstantInt>(RHS)) {
636 return computeAddress(LHS, Addr, Ty);
640 case Instruction::Shl: {
641 if (Addr.getOffsetReg())
649 if (Val < 1 || Val > 3)
652 uint64_t NumBytes = 0;
654 uint64_t NumBits =
DL.getTypeSizeInBits(Ty);
655 NumBytes = NumBits / 8;
660 if (NumBytes != (1ULL << Val))
667 if (
const auto *
I = dyn_cast<Instruction>(Src)) {
668 if (FuncInfo.MBBMap[
I->getParent()] == FuncInfo.MBB) {
670 if (
const auto *ZE = dyn_cast<ZExtInst>(
I)) {
672 ZE->getOperand(0)->getType()->isIntegerTy(32)) {
674 Src = ZE->getOperand(0);
676 }
else if (
const auto *SE = dyn_cast<SExtInst>(
I)) {
678 SE->getOperand(0)->getType()->isIntegerTy(32)) {
680 Src = SE->getOperand(0);
686 if (
const auto *AI = dyn_cast<BinaryOperator>(Src))
691 if (
const auto *C = dyn_cast<ConstantInt>(LHS))
692 if (C->getValue() == 0xffffffff)
695 if (
const auto *C = dyn_cast<ConstantInt>(RHS))
696 if (C->getValue() == 0xffffffff) {
698 unsigned Reg = getRegForValue(LHS);
701 bool RegIsKill = hasTrivialKill(LHS);
702 Reg = fastEmitInst_extractsubreg(
MVT::i32, Reg, RegIsKill,
704 Addr.setOffsetReg(Reg);
709 unsigned Reg = getRegForValue(Src);
712 Addr.setOffsetReg(Reg);
715 case Instruction::Mul: {
716 if (Addr.getOffsetReg())
726 if (
const auto *C = dyn_cast<ConstantInt>(LHS))
727 if (C->getValue().isPowerOf2())
730 assert(isa<ConstantInt>(RHS) &&
"Expected an ConstantInt.");
731 const auto *C = cast<ConstantInt>(RHS);
732 unsigned Val = C->getValue().logBase2();
733 if (Val < 1 || Val > 3)
736 uint64_t NumBytes = 0;
738 uint64_t NumBits =
DL.getTypeSizeInBits(Ty);
739 NumBytes = NumBits / 8;
744 if (NumBytes != (1ULL << Val))
750 const Value *Src = LHS;
751 if (
const auto *
I = dyn_cast<Instruction>(Src)) {
752 if (FuncInfo.MBBMap[
I->getParent()] == FuncInfo.MBB) {
754 if (
const auto *ZE = dyn_cast<ZExtInst>(
I)) {
756 ZE->getOperand(0)->getType()->isIntegerTy(32)) {
758 Src = ZE->getOperand(0);
760 }
else if (
const auto *SE = dyn_cast<SExtInst>(
I)) {
762 SE->getOperand(0)->getType()->isIntegerTy(32)) {
764 Src = SE->getOperand(0);
770 unsigned Reg = getRegForValue(Src);
773 Addr.setOffsetReg(Reg);
777 if (Addr.getOffsetReg())
780 if (!Ty ||
DL.getTypeSizeInBits(Ty) != 8)
786 if (
const auto *C = dyn_cast<ConstantInt>(LHS))
787 if (C->getValue() == 0xffffffff)
790 if (
const auto *C = dyn_cast<ConstantInt>(RHS))
791 if (C->getValue() == 0xffffffff) {
796 unsigned Reg = getRegForValue(LHS);
799 bool RegIsKill = hasTrivialKill(LHS);
800 Reg = fastEmitInst_extractsubreg(
MVT::i32, Reg, RegIsKill,
802 Addr.setOffsetReg(Reg);
807 case Instruction::SExt:
808 case Instruction::ZExt: {
809 if (!Addr.getReg() || Addr.getOffsetReg())
812 const Value *Src =
nullptr;
814 if (
const auto *ZE = dyn_cast<ZExtInst>(U)) {
815 if (!
isIntExtFree(ZE) && ZE->getOperand(0)->getType()->isIntegerTy(32)) {
817 Src = ZE->getOperand(0);
819 }
else if (
const auto *SE = dyn_cast<SExtInst>(U)) {
820 if (!
isIntExtFree(SE) && SE->getOperand(0)->getType()->isIntegerTy(32)) {
822 Src = SE->getOperand(0);
830 unsigned Reg = getRegForValue(Src);
833 Addr.setOffsetReg(Reg);
838 if (Addr.isRegBase() && !Addr.getReg()) {
839 unsigned Reg = getRegForValue(Obj);
846 if (!Addr.getOffsetReg()) {
847 unsigned Reg = getRegForValue(Obj);
850 Addr.setOffsetReg(Reg);
857 bool AArch64FastISel::computeCallAddress(
const Value *V,
Address &Addr) {
858 const User *U =
nullptr;
859 unsigned Opcode = Instruction::UserOp1;
862 if (
const auto *
I = dyn_cast<Instruction>(V)) {
863 Opcode =
I->getOpcode();
865 InMBB =
I->getParent() == FuncInfo.MBB->getBasicBlock();
866 }
else if (
const auto *C = dyn_cast<ConstantExpr>(V)) {
867 Opcode = C->getOpcode();
873 case Instruction::BitCast:
876 return computeCallAddress(U->
getOperand(0), Addr);
878 case Instruction::IntToPtr:
882 TLI.getPointerTy(DL))
883 return computeCallAddress(U->
getOperand(0), Addr);
885 case Instruction::PtrToInt:
887 if (InMBB && TLI.getValueType(DL, U->
getType()) == TLI.getPointerTy(DL))
888 return computeCallAddress(U->
getOperand(0), Addr);
892 if (
const GlobalValue *GV = dyn_cast<GlobalValue>(V)) {
893 Addr.setGlobalValue(GV);
898 if (!Addr.getGlobalValue()) {
899 Addr.setReg(getRegForValue(V));
900 return Addr.getReg() != 0;
907 bool AArch64FastISel::isTypeLegal(
Type *Ty,
MVT &VT) {
908 EVT evt = TLI.getValueType(DL, Ty,
true);
921 return TLI.isTypeLegal(VT);
928 bool AArch64FastISel::isTypeSupported(
Type *Ty,
MVT &VT,
bool IsVectorAllowed) {
932 if (isTypeLegal(Ty, VT))
943 bool AArch64FastISel::isValueAvailable(
const Value *V)
const {
944 if (!isa<Instruction>(V))
947 const auto *
I = cast<Instruction>(V);
948 return FuncInfo.MBBMap[
I->getParent()] == FuncInfo.MBB;
951 bool AArch64FastISel::simplifyAddress(
Address &Addr,
MVT VT) {
956 bool ImmediateOffsetNeedsLowering =
false;
957 bool RegisterOffsetNeedsLowering =
false;
958 int64_t
Offset = Addr.getOffset();
959 if (((Offset < 0) || (Offset & (ScaleFactor - 1))) && !isInt<9>(Offset))
960 ImmediateOffsetNeedsLowering =
true;
961 else if (Offset > 0 && !(Offset & (ScaleFactor - 1)) &&
962 !isUInt<12>(Offset / ScaleFactor))
963 ImmediateOffsetNeedsLowering =
true;
968 if (!ImmediateOffsetNeedsLowering && Addr.getOffset() && Addr.getOffsetReg())
969 RegisterOffsetNeedsLowering =
true;
972 if (Addr.isRegBase() && Addr.getOffsetReg() && !Addr.getReg())
973 RegisterOffsetNeedsLowering =
true;
978 if ((ImmediateOffsetNeedsLowering || Addr.getOffsetReg()) && Addr.isFIBase())
980 unsigned ResultReg = createResultReg(&AArch64::GPR64spRegClass);
981 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(AArch64::ADDXri),
983 .addFrameIndex(Addr.getFI())
986 Addr.setKind(Address::RegBase);
987 Addr.setReg(ResultReg);
990 if (RegisterOffsetNeedsLowering) {
991 unsigned ResultReg = 0;
995 ResultReg = emitAddSub_rx(
true,
MVT::i64, Addr.getReg(),
996 false, Addr.getOffsetReg(),
997 false, Addr.getExtendType(),
1000 ResultReg = emitAddSub_rs(
true,
MVT::i64, Addr.getReg(),
1001 false, Addr.getOffsetReg(),
1007 false, Addr.getShift(),
1011 false, Addr.getShift(),
1015 false, Addr.getShift());
1020 Addr.setReg(ResultReg);
1021 Addr.setOffsetReg(0);
1028 if (ImmediateOffsetNeedsLowering) {
1038 Addr.setReg(ResultReg);
1044 void AArch64FastISel::addLoadStoreOperands(
Address &Addr,
1047 unsigned ScaleFactor,
1049 int64_t Offset = Addr.getOffset() / ScaleFactor;
1051 if (Addr.isFIBase()) {
1052 int FI = Addr.getFI();
1055 MMO = FuncInfo.MF->getMachineMemOperand(
1057 MFI.getObjectSize(FI), MFI.getObjectAlignment(FI));
1061 assert(Addr.isRegBase() &&
"Unexpected address kind.");
1068 if (Addr.getOffsetReg()) {
1069 assert(Addr.getOffset() == 0 &&
"Unexpected offset");
1072 MIB.
addReg(Addr.getReg());
1073 MIB.
addReg(Addr.getOffsetReg());
1075 MIB.
addImm(Addr.getShift() != 0);
1077 MIB.
addReg(Addr.getReg()).addImm(Offset);
1084 unsigned AArch64FastISel::emitAddSub(
bool UseAdd,
MVT RetVT,
const Value *LHS,
1085 const Value *RHS,
bool SetFlags,
1086 bool WantResult,
bool IsZExt) {
1088 bool NeedExtend =
false;
1111 if (UseAdd && isa<Constant>(LHS) && !isa<Constant>(RHS))
1115 if (UseAdd && LHS->
hasOneUse() && isValueAvailable(LHS))
1120 if (UseAdd && LHS->
hasOneUse() && isValueAvailable(LHS))
1121 if (
const auto *SI = dyn_cast<BinaryOperator>(LHS))
1122 if (isa<ConstantInt>(SI->getOperand(1)))
1123 if (SI->getOpcode() == Instruction::Shl ||
1124 SI->getOpcode() == Instruction::LShr ||
1125 SI->getOpcode() == Instruction::AShr )
1128 unsigned LHSReg = getRegForValue(LHS);
1131 bool LHSIsKill = hasTrivialKill(LHS);
1134 LHSReg = emitIntExt(SrcVT, LHSReg, RetVT, IsZExt);
1136 unsigned ResultReg = 0;
1137 if (
const auto *C = dyn_cast<ConstantInt>(RHS)) {
1138 uint64_t Imm = IsZExt ? C->getZExtValue() : C->getSExtValue();
1139 if (C->isNegative())
1140 ResultReg = emitAddSub_ri(!UseAdd, RetVT, LHSReg, LHSIsKill, -Imm,
1141 SetFlags, WantResult);
1143 ResultReg = emitAddSub_ri(UseAdd, RetVT, LHSReg, LHSIsKill, Imm, SetFlags,
1145 }
else if (
const auto *C = dyn_cast<Constant>(RHS))
1147 ResultReg = emitAddSub_ri(UseAdd, RetVT, LHSReg, LHSIsKill, 0, SetFlags,
1155 isValueAvailable(RHS)) {
1156 if (
const auto *SI = dyn_cast<BinaryOperator>(RHS))
1157 if (
const auto *C = dyn_cast<ConstantInt>(SI->getOperand(1)))
1158 if ((SI->getOpcode() == Instruction::Shl) && (C->getZExtValue() < 4)) {
1159 unsigned RHSReg = getRegForValue(SI->getOperand(0));
1162 bool RHSIsKill = hasTrivialKill(SI->getOperand(0));
1163 return emitAddSub_rx(UseAdd, RetVT, LHSReg, LHSIsKill, RHSReg,
1164 RHSIsKill, ExtendType, C->getZExtValue(),
1165 SetFlags, WantResult);
1167 unsigned RHSReg = getRegForValue(RHS);
1170 bool RHSIsKill = hasTrivialKill(RHS);
1171 return emitAddSub_rx(UseAdd, RetVT, LHSReg, LHSIsKill, RHSReg, RHSIsKill,
1172 ExtendType, 0, SetFlags, WantResult);
1176 if (RHS->
hasOneUse() && isValueAvailable(RHS)) {
1178 const Value *MulLHS = cast<MulOperator>(RHS)->getOperand(0);
1179 const Value *MulRHS = cast<MulOperator>(RHS)->getOperand(1);
1181 if (
const auto *C = dyn_cast<ConstantInt>(MulLHS))
1182 if (C->getValue().isPowerOf2())
1185 assert(isa<ConstantInt>(MulRHS) &&
"Expected a ConstantInt.");
1186 uint64_t ShiftVal = cast<ConstantInt>(MulRHS)->getValue().logBase2();
1187 unsigned RHSReg = getRegForValue(MulLHS);
1190 bool RHSIsKill = hasTrivialKill(MulLHS);
1191 ResultReg = emitAddSub_rs(UseAdd, RetVT, LHSReg, LHSIsKill, RHSReg,
1200 if (RHS->
hasOneUse() && isValueAvailable(RHS)) {
1201 if (
const auto *SI = dyn_cast<BinaryOperator>(RHS)) {
1202 if (
const auto *C = dyn_cast<ConstantInt>(SI->getOperand(1))) {
1204 switch (SI->getOpcode()) {
1210 uint64_t ShiftVal = C->getZExtValue();
1212 unsigned RHSReg = getRegForValue(SI->getOperand(0));
1215 bool RHSIsKill = hasTrivialKill(SI->getOperand(0));
1216 ResultReg = emitAddSub_rs(UseAdd, RetVT, LHSReg, LHSIsKill, RHSReg,
1217 RHSIsKill, ShiftType, ShiftVal, SetFlags,
1226 unsigned RHSReg = getRegForValue(RHS);
1229 bool RHSIsKill = hasTrivialKill(RHS);
1232 RHSReg = emitIntExt(SrcVT, RHSReg, RetVT, IsZExt);
1234 return emitAddSub_rr(UseAdd, RetVT, LHSReg, LHSIsKill, RHSReg, RHSIsKill,
1235 SetFlags, WantResult);
1238 unsigned AArch64FastISel::emitAddSub_rr(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
1239 bool LHSIsKill,
unsigned RHSReg,
1240 bool RHSIsKill,
bool SetFlags,
1242 assert(LHSReg && RHSReg &&
"Invalid register number.");
1247 static const unsigned OpcTable[2][2][2] = {
1248 { { AArch64::SUBWrr, AArch64::SUBXrr },
1249 { AArch64::ADDWrr, AArch64::ADDXrr } },
1250 { { AArch64::SUBSWrr, AArch64::SUBSXrr },
1251 { AArch64::ADDSWrr, AArch64::ADDSXrr } }
1254 unsigned Opc = OpcTable[SetFlags][UseAdd][Is64Bit];
1256 Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
1259 ResultReg = createResultReg(RC);
1261 ResultReg = Is64Bit ? AArch64::XZR : AArch64::WZR;
1266 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
1272 unsigned AArch64FastISel::emitAddSub_ri(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
1273 bool LHSIsKill, uint64_t Imm,
1274 bool SetFlags,
bool WantResult) {
1275 assert(LHSReg &&
"Invalid register number.");
1281 if (isUInt<12>(Imm))
1283 else if ((Imm & 0xfff000) == Imm) {
1289 static const unsigned OpcTable[2][2][2] = {
1290 { { AArch64::SUBWri, AArch64::SUBXri },
1291 { AArch64::ADDWri, AArch64::ADDXri } },
1292 { { AArch64::SUBSWri, AArch64::SUBSXri },
1293 { AArch64::ADDSWri, AArch64::ADDSXri } }
1296 unsigned Opc = OpcTable[SetFlags][UseAdd][Is64Bit];
1299 RC = Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
1301 RC = Is64Bit ? &AArch64::GPR64spRegClass : &AArch64::GPR32spRegClass;
1304 ResultReg = createResultReg(RC);
1306 ResultReg = Is64Bit ? AArch64::XZR : AArch64::WZR;
1310 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
1317 unsigned AArch64FastISel::emitAddSub_rs(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
1318 bool LHSIsKill,
unsigned RHSReg,
1321 uint64_t ShiftImm,
bool SetFlags,
1323 assert(LHSReg && RHSReg &&
"Invalid register number.");
1332 static const unsigned OpcTable[2][2][2] = {
1333 { { AArch64::SUBWrs, AArch64::SUBXrs },
1334 { AArch64::ADDWrs, AArch64::ADDXrs } },
1335 { { AArch64::SUBSWrs, AArch64::SUBSXrs },
1336 { AArch64::ADDSWrs, AArch64::ADDSXrs } }
1339 unsigned Opc = OpcTable[SetFlags][UseAdd][Is64Bit];
1341 Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
1344 ResultReg = createResultReg(RC);
1346 ResultReg = Is64Bit ? AArch64::XZR : AArch64::WZR;
1351 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
1358 unsigned AArch64FastISel::emitAddSub_rx(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
1359 bool LHSIsKill,
unsigned RHSReg,
1362 uint64_t ShiftImm,
bool SetFlags,
1364 assert(LHSReg && RHSReg &&
"Invalid register number.");
1372 static const unsigned OpcTable[2][2][2] = {
1373 { { AArch64::SUBWrx, AArch64::SUBXrx },
1374 { AArch64::ADDWrx, AArch64::ADDXrx } },
1375 { { AArch64::SUBSWrx, AArch64::SUBSXrx },
1376 { AArch64::ADDSWrx, AArch64::ADDSXrx } }
1379 unsigned Opc = OpcTable[SetFlags][UseAdd][Is64Bit];
1382 RC = Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
1384 RC = Is64Bit ? &AArch64::GPR64spRegClass : &AArch64::GPR32spRegClass;
1387 ResultReg = createResultReg(RC);
1389 ResultReg = Is64Bit ? AArch64::XZR : AArch64::WZR;
1394 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
1403 EVT EVT = TLI.getValueType(DL, Ty,
true);
1416 return emitICmp(VT, LHS, RHS, IsZExt);
1419 return emitFCmp(VT, LHS, RHS);
1423 bool AArch64FastISel::emitICmp(
MVT RetVT,
const Value *LHS,
const Value *RHS,
1425 return emitSub(RetVT, LHS, RHS,
true,
false,
1429 bool AArch64FastISel::emitICmp_ri(
MVT RetVT,
unsigned LHSReg,
bool LHSIsKill,
1431 return emitAddSub_ri(
false, RetVT, LHSReg, LHSIsKill, Imm,
1435 bool AArch64FastISel::emitFCmp(
MVT RetVT,
const Value *LHS,
const Value *RHS) {
1441 bool UseImm =
false;
1442 if (
const auto *CFP = dyn_cast<ConstantFP>(RHS))
1446 unsigned LHSReg = getRegForValue(LHS);
1449 bool LHSIsKill = hasTrivialKill(LHS);
1452 unsigned Opc = (RetVT ==
MVT::f64) ? AArch64::FCMPDri : AArch64::FCMPSri;
1453 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(Opc))
1458 unsigned RHSReg = getRegForValue(RHS);
1461 bool RHSIsKill = hasTrivialKill(RHS);
1463 unsigned Opc = (RetVT ==
MVT::f64) ? AArch64::FCMPDrr : AArch64::FCMPSrr;
1464 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(Opc))
1470 unsigned AArch64FastISel::emitAdd(
MVT RetVT,
const Value *LHS,
const Value *RHS,
1471 bool SetFlags,
bool WantResult,
bool IsZExt) {
1472 return emitAddSub(
true, RetVT, LHS, RHS, SetFlags, WantResult,
1481 unsigned AArch64FastISel::emitAdd_ri_(
MVT VT,
unsigned Op0,
bool Op0IsKill,
1485 ResultReg = emitAddSub_ri(
false, VT, Op0, Op0IsKill, -Imm);
1487 ResultReg = emitAddSub_ri(
true, VT, Op0, Op0IsKill, Imm);
1496 ResultReg = emitAddSub_rr(
true, VT, Op0, Op0IsKill, CReg,
true);
1500 unsigned AArch64FastISel::emitSub(
MVT RetVT,
const Value *LHS,
const Value *RHS,
1501 bool SetFlags,
bool WantResult,
bool IsZExt) {
1502 return emitAddSub(
false, RetVT, LHS, RHS, SetFlags, WantResult,
1506 unsigned AArch64FastISel::emitSubs_rr(
MVT RetVT,
unsigned LHSReg,
1507 bool LHSIsKill,
unsigned RHSReg,
1508 bool RHSIsKill,
bool WantResult) {
1509 return emitAddSub_rr(
false, RetVT, LHSReg, LHSIsKill, RHSReg,
1510 RHSIsKill,
true, WantResult);
1513 unsigned AArch64FastISel::emitSubs_rs(
MVT RetVT,
unsigned LHSReg,
1514 bool LHSIsKill,
unsigned RHSReg,
1517 uint64_t ShiftImm,
bool WantResult) {
1518 return emitAddSub_rs(
false, RetVT, LHSReg, LHSIsKill, RHSReg,
1519 RHSIsKill, ShiftType, ShiftImm,
true,
1523 unsigned AArch64FastISel::emitLogicalOp(
unsigned ISDOpc,
MVT RetVT,
1526 if (isa<ConstantInt>(LHS) && !isa<ConstantInt>(RHS))
1530 if (LHS->
hasOneUse() && isValueAvailable(LHS))
1535 if (LHS->
hasOneUse() && isValueAvailable(LHS))
1536 if (
const auto *SI = dyn_cast<ShlOperator>(LHS))
1537 if (isa<ConstantInt>(SI->getOperand(1)))
1540 unsigned LHSReg = getRegForValue(LHS);
1543 bool LHSIsKill = hasTrivialKill(LHS);
1545 unsigned ResultReg = 0;
1546 if (
const auto *C = dyn_cast<ConstantInt>(RHS)) {
1547 uint64_t Imm = C->getZExtValue();
1548 ResultReg = emitLogicalOp_ri(ISDOpc, RetVT, LHSReg, LHSIsKill, Imm);
1554 if (RHS->
hasOneUse() && isValueAvailable(RHS)) {
1556 const Value *MulLHS = cast<MulOperator>(RHS)->getOperand(0);
1557 const Value *MulRHS = cast<MulOperator>(RHS)->getOperand(1);
1559 if (
const auto *C = dyn_cast<ConstantInt>(MulLHS))
1560 if (C->getValue().isPowerOf2())
1563 assert(isa<ConstantInt>(MulRHS) &&
"Expected a ConstantInt.");
1564 uint64_t ShiftVal = cast<ConstantInt>(MulRHS)->getValue().logBase2();
1566 unsigned RHSReg = getRegForValue(MulLHS);
1569 bool RHSIsKill = hasTrivialKill(MulLHS);
1570 ResultReg = emitLogicalOp_rs(ISDOpc, RetVT, LHSReg, LHSIsKill, RHSReg,
1571 RHSIsKill, ShiftVal);
1578 if (RHS->
hasOneUse() && isValueAvailable(RHS)) {
1579 if (
const auto *SI = dyn_cast<ShlOperator>(RHS))
1580 if (
const auto *C = dyn_cast<ConstantInt>(SI->getOperand(1))) {
1581 uint64_t ShiftVal = C->getZExtValue();
1582 unsigned RHSReg = getRegForValue(SI->getOperand(0));
1585 bool RHSIsKill = hasTrivialKill(SI->getOperand(0));
1586 ResultReg = emitLogicalOp_rs(ISDOpc, RetVT, LHSReg, LHSIsKill, RHSReg,
1587 RHSIsKill, ShiftVal);
1593 unsigned RHSReg = getRegForValue(RHS);
1596 bool RHSIsKill = hasTrivialKill(RHS);
1599 ResultReg = fastEmit_rr(VT, VT, ISDOpc, LHSReg, LHSIsKill, RHSReg, RHSIsKill);
1601 uint64_t
Mask = (RetVT ==
MVT::i8) ? 0xff : 0xffff;
1602 ResultReg = emitAnd_ri(
MVT::i32, ResultReg,
true, Mask);
1607 unsigned AArch64FastISel::emitLogicalOp_ri(
unsigned ISDOpc,
MVT RetVT,
1608 unsigned LHSReg,
bool LHSIsKill,
1611 "ISD nodes are not consecutive!");
1612 static const unsigned OpcTable[3][2] = {
1613 { AArch64::ANDWri, AArch64::ANDXri },
1614 { AArch64::ORRWri, AArch64::ORRXri },
1615 { AArch64::EORWri, AArch64::EORXri }
1628 Opc = OpcTable[Idx][0];
1629 RC = &AArch64::GPR32spRegClass;
1634 Opc = OpcTable[ISDOpc -
ISD::AND][1];
1635 RC = &AArch64::GPR64spRegClass;
1643 unsigned ResultReg =
1644 fastEmitInst_ri(Opc, RC, LHSReg, LHSIsKill,
1647 uint64_t
Mask = (RetVT ==
MVT::i8) ? 0xff : 0xffff;
1648 ResultReg = emitAnd_ri(
MVT::i32, ResultReg,
true, Mask);
1653 unsigned AArch64FastISel::emitLogicalOp_rs(
unsigned ISDOpc,
MVT RetVT,
1654 unsigned LHSReg,
bool LHSIsKill,
1655 unsigned RHSReg,
bool RHSIsKill,
1656 uint64_t ShiftImm) {
1658 "ISD nodes are not consecutive!");
1659 static const unsigned OpcTable[3][2] = {
1660 { AArch64::ANDWrs, AArch64::ANDXrs },
1661 { AArch64::ORRWrs, AArch64::ORRXrs },
1662 { AArch64::EORWrs, AArch64::EORXrs }
1678 Opc = OpcTable[ISDOpc -
ISD::AND][0];
1679 RC = &AArch64::GPR32RegClass;
1682 Opc = OpcTable[ISDOpc -
ISD::AND][1];
1683 RC = &AArch64::GPR64RegClass;
1686 unsigned ResultReg =
1687 fastEmitInst_rri(Opc, RC, LHSReg, LHSIsKill, RHSReg, RHSIsKill,
1690 uint64_t
Mask = (RetVT ==
MVT::i8) ? 0xff : 0xffff;
1691 ResultReg = emitAnd_ri(
MVT::i32, ResultReg,
true, Mask);
1696 unsigned AArch64FastISel::emitAnd_ri(
MVT RetVT,
unsigned LHSReg,
bool LHSIsKill,
1698 return emitLogicalOp_ri(
ISD::AND, RetVT, LHSReg, LHSIsKill, Imm);
1701 unsigned AArch64FastISel::emitLoad(
MVT VT,
MVT RetVT,
Address Addr,
1703 if (!TLI.allowsMisalignedMemoryAccesses(VT))
1707 if (!simplifyAddress(Addr, VT))
1716 bool UseScaled =
true;
1717 if ((Addr.getOffset() < 0) || (Addr.getOffset() & (ScaleFactor - 1))) {
1722 static const unsigned GPOpcTable[2][8][4] = {
1724 { { AArch64::LDURSBWi, AArch64::LDURSHWi, AArch64::LDURWi,
1726 { AArch64::LDURSBXi, AArch64::LDURSHXi, AArch64::LDURSWi,
1728 { AArch64::LDRSBWui, AArch64::LDRSHWui, AArch64::LDRWui,
1730 { AArch64::LDRSBXui, AArch64::LDRSHXui, AArch64::LDRSWui,
1732 { AArch64::LDRSBWroX, AArch64::LDRSHWroX, AArch64::LDRWroX,
1734 { AArch64::LDRSBXroX, AArch64::LDRSHXroX, AArch64::LDRSWroX,
1736 { AArch64::LDRSBWroW, AArch64::LDRSHWroW, AArch64::LDRWroW,
1738 { AArch64::LDRSBXroW, AArch64::LDRSHXroW, AArch64::LDRSWroW,
1742 { { AArch64::LDURBBi, AArch64::LDURHHi, AArch64::LDURWi,
1744 { AArch64::LDURBBi, AArch64::LDURHHi, AArch64::LDURWi,
1746 { AArch64::LDRBBui, AArch64::LDRHHui, AArch64::LDRWui,
1748 { AArch64::LDRBBui, AArch64::LDRHHui, AArch64::LDRWui,
1750 { AArch64::LDRBBroX, AArch64::LDRHHroX, AArch64::LDRWroX,
1752 { AArch64::LDRBBroX, AArch64::LDRHHroX, AArch64::LDRWroX,
1754 { AArch64::LDRBBroW, AArch64::LDRHHroW, AArch64::LDRWroW,
1756 { AArch64::LDRBBroW, AArch64::LDRHHroW, AArch64::LDRWroW,
1761 static const unsigned FPOpcTable[4][2] = {
1762 { AArch64::LDURSi, AArch64::LDURDi },
1763 { AArch64::LDRSui, AArch64::LDRDui },
1764 { AArch64::LDRSroX, AArch64::LDRDroX },
1765 { AArch64::LDRSroW, AArch64::LDRDroW }
1770 bool UseRegOffset = Addr.isRegBase() && !Addr.getOffset() && Addr.getReg() &&
1771 Addr.getOffsetReg();
1772 unsigned Idx = UseRegOffset ? 2 : UseScaled ? 1 : 0;
1777 bool IsRet64Bit = RetVT ==
MVT::i64;
1783 Opc = GPOpcTable[WantZExt][2 * Idx + IsRet64Bit][0];
1784 RC = (IsRet64Bit && !WantZExt) ?
1785 &AArch64::GPR64RegClass: &AArch64::GPR32RegClass;
1788 Opc = GPOpcTable[WantZExt][2 * Idx + IsRet64Bit][1];
1789 RC = (IsRet64Bit && !WantZExt) ?
1790 &AArch64::GPR64RegClass: &AArch64::GPR32RegClass;
1793 Opc = GPOpcTable[WantZExt][2 * Idx + IsRet64Bit][2];
1794 RC = (IsRet64Bit && !WantZExt) ?
1795 &AArch64::GPR64RegClass: &AArch64::GPR32RegClass;
1798 Opc = GPOpcTable[WantZExt][2 * Idx + IsRet64Bit][3];
1799 RC = &AArch64::GPR64RegClass;
1802 Opc = FPOpcTable[Idx][0];
1803 RC = &AArch64::FPR32RegClass;
1806 Opc = FPOpcTable[Idx][1];
1807 RC = &AArch64::FPR64RegClass;
1812 unsigned ResultReg = createResultReg(RC);
1814 TII.get(Opc), ResultReg);
1819 unsigned ANDReg = emitAnd_ri(
MVT::i32, ResultReg,
true, 1);
1820 assert(ANDReg &&
"Unexpected AND instruction emission failure.");
1827 unsigned Reg64 = createResultReg(&AArch64::GPR64RegClass);
1828 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1829 TII.get(AArch64::SUBREG_TO_REG), Reg64)
1832 .
addImm(AArch64::sub_32);
1838 bool AArch64FastISel::selectAddSub(
const Instruction *
I) {
1840 if (!isTypeSupported(I->
getType(), VT,
true))
1844 return selectOperator(I, I->
getOpcode());
1853 case Instruction::Sub:
1860 updateValueMap(I, ResultReg);
1864 bool AArch64FastISel::selectLogicalOp(
const Instruction *I) {
1866 if (!isTypeSupported(I->
getType(), VT,
true))
1870 return selectOperator(I, I->
getOpcode());
1889 updateValueMap(I, ResultReg);
1893 bool AArch64FastISel::selectLoad(
const Instruction *I) {
1898 if (!isTypeSupported(I->
getType(), VT,
true) ||
1899 cast<LoadInst>(I)->isAtomic())
1903 if (TLI.supportSwiftError()) {
1906 if (
const Argument *Arg = dyn_cast<Argument>(SV)) {
1907 if (Arg->hasSwiftErrorAttr())
1911 if (
const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
1912 if (Alloca->isSwiftError())
1923 bool WantZExt =
true;
1925 const Value *IntExtVal =
nullptr;
1927 if (
const auto *ZE = dyn_cast<ZExtInst>(I->
use_begin()->getUser())) {
1928 if (isTypeSupported(ZE->getType(), RetVT))
1932 }
else if (
const auto *SE = dyn_cast<SExtInst>(I->
use_begin()->getUser())) {
1933 if (isTypeSupported(SE->getType(), RetVT))
1941 unsigned ResultReg =
1942 emitLoad(VT, RetVT, Addr, WantZExt, createMachineMemOperandFor(I));
1962 unsigned Reg = lookUpRegForValue(IntExtVal);
1963 auto *
MI =
MRI.getUniqueVRegDef(Reg);
1968 std::prev(FuncInfo.InsertPt)->eraseFromParent();
1969 ResultReg = std::prev(FuncInfo.InsertPt)->getOperand(0).getReg();
1971 ResultReg = fastEmitInst_extractsubreg(
MVT::i32, ResultReg,
1975 updateValueMap(I, ResultReg);
1984 for (
auto &Opnd :
MI->uses()) {
1986 Reg = Opnd.getReg();
1990 MI->eraseFromParent();
1993 MI =
MRI.getUniqueVRegDef(Reg);
1995 updateValueMap(IntExtVal, ResultReg);
1999 updateValueMap(I, ResultReg);
2003 bool AArch64FastISel::emitStoreRelease(
MVT VT,
unsigned SrcReg,
2008 default:
return false;
2009 case MVT::i8: Opc = AArch64::STLRB;
break;
2010 case MVT::i16: Opc = AArch64::STLRH;
break;
2011 case MVT::i32: Opc = AArch64::STLRW;
break;
2012 case MVT::i64: Opc = AArch64::STLRX;
break;
2018 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
2025 bool AArch64FastISel::emitStore(
MVT VT,
unsigned SrcReg,
Address Addr,
2027 if (!TLI.allowsMisalignedMemoryAccesses(VT))
2031 if (!simplifyAddress(Addr, VT))
2040 bool UseScaled =
true;
2041 if ((Addr.getOffset() < 0) || (Addr.getOffset() & (ScaleFactor - 1))) {
2046 static const unsigned OpcTable[4][6] = {
2047 { AArch64::STURBBi, AArch64::STURHHi, AArch64::STURWi, AArch64::STURXi,
2048 AArch64::STURSi, AArch64::STURDi },
2049 { AArch64::STRBBui, AArch64::STRHHui, AArch64::STRWui, AArch64::STRXui,
2050 AArch64::STRSui, AArch64::STRDui },
2051 { AArch64::STRBBroX, AArch64::STRHHroX, AArch64::STRWroX, AArch64::STRXroX,
2052 AArch64::STRSroX, AArch64::STRDroX },
2053 { AArch64::STRBBroW, AArch64::STRHHroW, AArch64::STRWroW, AArch64::STRXroW,
2054 AArch64::STRSroW, AArch64::STRDroW }
2058 bool VTIsi1 =
false;
2059 bool UseRegOffset = Addr.isRegBase() && !Addr.getOffset() && Addr.getReg() &&
2060 Addr.getOffsetReg();
2061 unsigned Idx = UseRegOffset ? 2 : UseScaled ? 1 : 0;
2069 case MVT::i8: Opc = OpcTable[Idx][0];
break;
2070 case MVT::i16: Opc = OpcTable[Idx][1];
break;
2071 case MVT::i32: Opc = OpcTable[Idx][2];
break;
2072 case MVT::i64: Opc = OpcTable[Idx][3];
break;
2073 case MVT::f32: Opc = OpcTable[Idx][4];
break;
2074 case MVT::f64: Opc = OpcTable[Idx][5];
break;
2078 if (VTIsi1 && SrcReg != AArch64::WZR) {
2079 unsigned ANDReg = emitAnd_ri(
MVT::i32, SrcReg,
false, 1);
2080 assert(ANDReg &&
"Unexpected AND instruction emission failure.");
2087 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II).
addReg(SrcReg);
2093 bool AArch64FastISel::selectStore(
const Instruction *I) {
2099 if (!isTypeSupported(Op0->
getType(), VT,
true))
2103 if (TLI.supportSwiftError()) {
2106 if (
const Argument *Arg = dyn_cast<Argument>(PtrV)) {
2107 if (Arg->hasSwiftErrorAttr())
2111 if (
const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
2112 if (Alloca->isSwiftError())
2119 unsigned SrcReg = 0;
2120 if (
const auto *CI = dyn_cast<ConstantInt>(Op0)) {
2122 SrcReg = (VT ==
MVT::i64) ? AArch64::XZR : AArch64::WZR;
2123 }
else if (
const auto *CF = dyn_cast<ConstantFP>(Op0)) {
2124 if (CF->isZero() && !CF->isNegative()) {
2126 SrcReg = (VT ==
MVT::i64) ? AArch64::XZR : AArch64::WZR;
2131 SrcReg = getRegForValue(Op0);
2136 auto *SI = cast<StoreInst>(
I);
2139 if (SI->isAtomic()) {
2144 unsigned AddrReg = getRegForValue(PtrV);
2145 return emitStoreRelease(VT, SrcReg, AddrReg,
2146 createMachineMemOperandFor(I));
2152 if (!computeAddress(PtrV, Addr, Op0->
getType()))
2155 if (!emitStore(VT, SrcReg, Addr, createMachineMemOperandFor(I)))
2207 bool AArch64FastISel::emitCompareAndBranch(
const BranchInst *BI) {
2216 if (!isTypeSupported(LHS->
getType(), VT))
2227 if (FuncInfo.MBB->isLayoutSuccessor(TBB)) {
2234 switch (Predicate) {
2239 if (isa<Constant>(LHS) && cast<Constant>(LHS)->isNullValue())
2242 if (!isa<Constant>(RHS) || !cast<Constant>(RHS)->isNullValue())
2245 if (
const auto *AI = dyn_cast<BinaryOperator>(LHS))
2250 if (
const auto *C = dyn_cast<ConstantInt>(AndLHS))
2251 if (C->getValue().isPowerOf2())
2254 if (
const auto *C = dyn_cast<ConstantInt>(AndRHS))
2255 if (C->getValue().isPowerOf2()) {
2256 TestBit = C->getValue().logBase2();
2268 if (!isa<Constant>(RHS) || !cast<Constant>(RHS)->isNullValue())
2276 if (!isa<ConstantInt>(RHS))
2279 if (cast<ConstantInt>(RHS)->getValue() !=
APInt(BW, -1,
true))
2287 static const unsigned OpcTable[2][2][2] = {
2288 { {AArch64::CBZW, AArch64::CBZX },
2289 {AArch64::CBNZW, AArch64::CBNZX} },
2290 { {AArch64::TBZW, AArch64::TBZX },
2291 {AArch64::TBNZW, AArch64::TBNZX} }
2294 bool IsBitTest = TestBit != -1;
2295 bool Is64Bit = BW == 64;
2296 if (TestBit < 32 && TestBit >= 0)
2299 unsigned Opc = OpcTable[IsBitTest][IsCmpNE][Is64Bit];
2302 unsigned SrcReg = getRegForValue(LHS);
2305 bool SrcIsKill = hasTrivialKill(LHS);
2307 if (BW == 64 && !Is64Bit)
2308 SrcReg = fastEmitInst_extractsubreg(
MVT::i32, SrcReg, SrcIsKill,
2311 if ((BW < 32) && !IsBitTest)
2312 SrcReg = emitIntExt(VT, SrcReg,
MVT::i32,
true);
2317 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(Opc))
2323 finishCondBranch(BI->
getParent(), TBB, FBB);
2327 bool AArch64FastISel::selectBranch(
const Instruction *I) {
2339 if (CI->
hasOneUse() && isValueAvailable(CI)) {
2342 switch (Predicate) {
2346 fastEmitBranch(FBB, DbgLoc);
2349 fastEmitBranch(TBB, DbgLoc);
2354 if (emitCompareAndBranch(BI))
2358 if (FuncInfo.MBB->isLayoutSuccessor(TBB)) {
2371 switch (Predicate) {
2387 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(AArch64::Bcc))
2393 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(AArch64::Bcc))
2397 finishCondBranch(BI->
getParent(), TBB, FBB);
2400 }
else if (
const auto *CI = dyn_cast<ConstantInt>(BI->
getCondition())) {
2401 uint64_t Imm = CI->getZExtValue();
2412 FuncInfo.MBB->addSuccessorWithoutProb(Target);
2424 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(AArch64::Bcc))
2428 finishCondBranch(BI->
getParent(), TBB, FBB);
2436 bool CondRegIsKill = hasTrivialKill(BI->
getCondition());
2439 unsigned Opcode = AArch64::TBNZW;
2440 if (FuncInfo.MBB->isLayoutSuccessor(TBB)) {
2442 Opcode = AArch64::TBZW;
2446 unsigned ConstrainedCondReg
2448 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
2453 finishCondBranch(BI->
getParent(), TBB, FBB);
2457 bool AArch64FastISel::selectIndirectBr(
const Instruction *I) {
2459 unsigned AddrReg = getRegForValue(BI->
getOperand(0));
2466 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II).
addReg(AddrReg);
2470 FuncInfo.MBB->addSuccessor(FuncInfo.MBBMap[Succ]);
2475 bool AArch64FastISel::selectCmp(
const Instruction *I) {
2476 const CmpInst *CI = cast<CmpInst>(
I);
2484 unsigned ResultReg = 0;
2485 switch (Predicate) {
2489 ResultReg = createResultReg(&AArch64::GPR32RegClass);
2490 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
2491 TII.get(TargetOpcode::COPY), ResultReg)
2500 updateValueMap(I, ResultReg);
2508 ResultReg = createResultReg(&AArch64::GPR32RegClass);
2512 static unsigned CondCodeTable[2][2] = {
2517 switch (Predicate) {
2521 CondCodes = &CondCodeTable[0][0];
2524 CondCodes = &CondCodeTable[1][0];
2529 unsigned TmpReg1 = createResultReg(&AArch64::GPR32RegClass);
2530 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(AArch64::CSINCWr),
2534 .addImm(CondCodes[0]);
2535 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(AArch64::CSINCWr),
2539 .addImm(CondCodes[1]);
2541 updateValueMap(I, ResultReg);
2549 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(AArch64::CSINCWr),
2553 .addImm(invertedCC);
2555 updateValueMap(I, ResultReg);
2561 bool AArch64FastISel::optimizeSelect(
const SelectInst *SI) {
2565 const Value *Src1Val, *Src2Val;
2567 bool NeedExtraOp =
false;
2568 if (
auto *CI = dyn_cast<ConstantInt>(SI->
getTrueValue())) {
2572 Opc = AArch64::ORRWrr;
2577 Opc = AArch64::BICWrr;
2579 }
else if (
auto *CI = dyn_cast<ConstantInt>(SI->
getFalseValue())) {
2583 Opc = AArch64::ORRWrr;
2589 Opc = AArch64::ANDWrr;
2596 unsigned Src1Reg = getRegForValue(Src1Val);
2599 bool Src1IsKill = hasTrivialKill(Src1Val);
2601 unsigned Src2Reg = getRegForValue(Src2Val);
2604 bool Src2IsKill = hasTrivialKill(Src2Val);
2610 unsigned ResultReg = fastEmitInst_rr(Opc, &AArch64::GPR32RegClass, Src1Reg,
2611 Src1IsKill, Src2Reg, Src2IsKill);
2612 updateValueMap(SI, ResultReg);
2616 bool AArch64FastISel::selectSelect(
const Instruction *I) {
2617 assert(isa<SelectInst>(I) &&
"Expected a select instruction.");
2619 if (!isTypeSupported(I->
getType(), VT))
2631 Opc = AArch64::CSELWr;
2632 RC = &AArch64::GPR32RegClass;
2635 Opc = AArch64::CSELXr;
2636 RC = &AArch64::GPR64RegClass;
2639 Opc = AArch64::FCSELSrrr;
2640 RC = &AArch64::FPR32RegClass;
2643 Opc = AArch64::FCSELDrrr;
2644 RC = &AArch64::FPR64RegClass;
2653 if (optimizeSelect(SI))
2657 if (foldXALUIntrinsic(CC, I, Cond)) {
2659 unsigned CondReg = getRegForValue(Cond);
2662 }
else if (isa<CmpInst>(Cond) && cast<CmpInst>(Cond)->hasOneUse() &&
2663 isValueAvailable(Cond)) {
2664 const auto *Cmp = cast<CmpInst>(Cond);
2667 const Value *FoldSelect =
nullptr;
2668 switch (Predicate) {
2680 unsigned SrcReg = getRegForValue(FoldSelect);
2683 unsigned UseReg = lookUpRegForValue(SI);
2685 MRI.clearKillFlags(UseReg);
2687 updateValueMap(I, SrcReg);
2692 if (!
emitCmp(Cmp->getOperand(0), Cmp->getOperand(1), Cmp->isUnsigned()))
2697 switch (Predicate) {
2711 unsigned CondReg = getRegForValue(Cond);
2714 bool CondIsKill = hasTrivialKill(Cond);
2720 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II,
2732 if (!Src1Reg || !Src2Reg)
2736 Src2Reg = fastEmitInst_rri(Opc, RC, Src1Reg, Src1IsKill, Src2Reg,
2737 Src2IsKill, ExtraCC);
2740 unsigned ResultReg = fastEmitInst_rri(Opc, RC, Src1Reg, Src1IsKill, Src2Reg,
2742 updateValueMap(I, ResultReg);
2746 bool AArch64FastISel::selectFPExt(
const Instruction *I) {
2751 unsigned Op = getRegForValue(V);
2755 unsigned ResultReg = createResultReg(&AArch64::FPR64RegClass);
2756 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(AArch64::FCVTDSr),
2757 ResultReg).addReg(Op);
2758 updateValueMap(I, ResultReg);
2762 bool AArch64FastISel::selectFPTrunc(
const Instruction *I) {
2767 unsigned Op = getRegForValue(V);
2771 unsigned ResultReg = createResultReg(&AArch64::FPR32RegClass);
2772 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(AArch64::FCVTSDr),
2773 ResultReg).addReg(Op);
2774 updateValueMap(I, ResultReg);
2784 unsigned SrcReg = getRegForValue(I->
getOperand(0));
2795 Opc = (DestVT ==
MVT::i32) ? AArch64::FCVTZSUWDr : AArch64::FCVTZSUXDr;
2797 Opc = (DestVT ==
MVT::i32) ? AArch64::FCVTZUUWDr : AArch64::FCVTZUUXDr;
2800 Opc = (DestVT ==
MVT::i32) ? AArch64::FCVTZSUWSr : AArch64::FCVTZSUXSr;
2802 Opc = (DestVT ==
MVT::i32) ? AArch64::FCVTZUUWSr : AArch64::FCVTZUUXSr;
2804 unsigned ResultReg = createResultReg(
2805 DestVT ==
MVT::i32 ? &AArch64::GPR32RegClass : &AArch64::GPR64RegClass);
2806 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(Opc), ResultReg)
2808 updateValueMap(I, ResultReg);
2812 bool AArch64FastISel::selectIntToFP(
const Instruction *I,
bool Signed) {
2817 "Unexpected value type.");
2819 unsigned SrcReg = getRegForValue(I->
getOperand(0));
2822 bool SrcIsKill = hasTrivialKill(I->
getOperand(0));
2838 Opc = (DestVT ==
MVT::f32) ? AArch64::SCVTFUXSri : AArch64::SCVTFUXDri;
2840 Opc = (DestVT ==
MVT::f32) ? AArch64::UCVTFUXSri : AArch64::UCVTFUXDri;
2843 Opc = (DestVT ==
MVT::f32) ? AArch64::SCVTFUWSri : AArch64::SCVTFUWDri;
2845 Opc = (DestVT ==
MVT::f32) ? AArch64::UCVTFUWSri : AArch64::UCVTFUWDri;
2848 unsigned ResultReg = fastEmitInst_r(Opc, TLI.getRegClassFor(DestVT), SrcReg,
2850 updateValueMap(I, ResultReg);
2854 bool AArch64FastISel::fastLowerArguments() {
2855 if (!FuncInfo.CanLowerReturn)
2867 unsigned GPRCnt = 0;
2868 unsigned FPRCnt = 0;
2870 for (
auto const &Arg : F->
args()) {
2881 Type *ArgTy = Arg.getType();
2885 EVT ArgVT = TLI.getValueType(DL, ArgTy);
2894 (!Subtarget->hasNEON() || !Subtarget->isLittleEndian()))
2905 if (GPRCnt > 8 || FPRCnt > 8)
2909 static const MCPhysReg Registers[6][8] = {
2910 { AArch64::W0, AArch64::W1, AArch64::W2, AArch64::W3, AArch64::W4,
2911 AArch64::W5, AArch64::W6, AArch64::W7 },
2912 { AArch64::X0, AArch64::X1, AArch64::X2, AArch64::X3, AArch64::X4,
2913 AArch64::X5, AArch64::X6, AArch64::X7 },
2914 { AArch64::H0, AArch64::H1, AArch64::H2, AArch64::H3, AArch64::H4,
2915 AArch64::H5, AArch64::H6, AArch64::H7 },
2916 { AArch64::S0, AArch64::S1, AArch64::S2, AArch64::S3, AArch64::S4,
2917 AArch64::S5, AArch64::S6, AArch64::S7 },
2918 { AArch64::D0, AArch64::D1, AArch64::D2, AArch64::D3, AArch64::D4,
2919 AArch64::D5, AArch64::D6, AArch64::D7 },
2920 { AArch64::Q0, AArch64::Q1, AArch64::Q2, AArch64::Q3, AArch64::Q4,
2921 AArch64::Q5, AArch64::Q6, AArch64::Q7 }
2924 unsigned GPRIdx = 0;
2925 unsigned FPRIdx = 0;
2926 for (
auto const &Arg : F->
args()) {
2927 MVT VT = TLI.getSimpleValueType(DL, Arg.getType());
2930 if (VT >=
MVT::i1 && VT <= MVT::i32) {
2931 SrcReg = Registers[0][GPRIdx++];
2932 RC = &AArch64::GPR32RegClass;
2935 SrcReg = Registers[1][GPRIdx++];
2936 RC = &AArch64::GPR64RegClass;
2938 SrcReg = Registers[2][FPRIdx++];
2939 RC = &AArch64::FPR16RegClass;
2941 SrcReg = Registers[3][FPRIdx++];
2942 RC = &AArch64::FPR32RegClass;
2944 SrcReg = Registers[4][FPRIdx++];
2945 RC = &AArch64::FPR64RegClass;
2947 SrcReg = Registers[5][FPRIdx++];
2948 RC = &AArch64::FPR128RegClass;
2952 unsigned DstReg = FuncInfo.MF->addLiveIn(SrcReg, RC);
2956 unsigned ResultReg = createResultReg(RC);
2957 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
2958 TII.get(TargetOpcode::COPY), ResultReg)
2960 updateValueMap(&Arg, ResultReg);
2965 bool AArch64FastISel::processCallArgs(CallLoweringInfo &CLI,
2967 unsigned &NumBytes) {
2971 CCInfo.AnalyzeCallOperands(OutVTs, CLI.OutFlags, CCAssignFnForCall(CC));
2974 NumBytes = CCInfo.getNextStackOffset();
2977 unsigned AdjStackDown =
TII.getCallFrameSetupOpcode();
2978 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(AdjStackDown))
2983 const Value *ArgVal = CLI.OutVals[VA.getValNo()];
2984 MVT ArgVT = OutVTs[VA.getValNo()];
2986 unsigned ArgReg = getRegForValue(ArgVal);
2991 switch (VA.getLocInfo()) {
2995 MVT DestVT = VA.getLocVT();
2997 ArgReg = emitIntExt(SrcVT, ArgReg, DestVT,
false);
3005 MVT DestVT = VA.getLocVT();
3007 ArgReg = emitIntExt(SrcVT, ArgReg, DestVT,
true);
3017 if (VA.isRegLoc() && !VA.needsCustom()) {
3018 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
3019 TII.get(TargetOpcode::COPY), VA.getLocReg()).addReg(ArgReg);
3020 CLI.OutRegs.push_back(VA.getLocReg());
3021 }
else if (VA.needsCustom()) {
3025 assert(VA.isMemLoc() &&
"Assuming store on stack.");
3028 if (isa<UndefValue>(ArgVal))
3034 unsigned BEAlign = 0;
3036 BEAlign = 8 - ArgSize;
3039 Addr.setKind(Address::RegBase);
3040 Addr.setReg(AArch64::SP);
3041 Addr.setOffset(VA.getLocMemOffset() + BEAlign);
3043 unsigned Alignment =
DL.getABITypeAlignment(ArgVal->
getType());
3048 if (!emitStore(ArgVT, ArgReg, Addr, MMO))
3055 bool AArch64FastISel::finishCall(CallLoweringInfo &CLI,
MVT RetVT,
3056 unsigned NumBytes) {
3060 unsigned AdjStackUp =
TII.getCallFrameDestroyOpcode();
3061 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(AdjStackUp))
3062 .addImm(NumBytes).
addImm(0);
3068 CCInfo.AnalyzeCallResult(RetVT, CCAssignFnForCall(CC));
3071 if (RVLocs.
size() != 1)
3075 MVT CopyVT = RVLocs[0].getValVT();
3078 if (CopyVT.
isVector() && !Subtarget->isLittleEndian())
3081 unsigned ResultReg = createResultReg(TLI.getRegClassFor(CopyVT));
3082 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
3083 TII.get(TargetOpcode::COPY), ResultReg)
3084 .addReg(RVLocs[0].getLocReg());
3085 CLI.InRegs.push_back(RVLocs[0].getLocReg());
3087 CLI.ResultReg = ResultReg;
3088 CLI.NumResultRegs = 1;
3094 bool AArch64FastISel::fastLowerCall(CallLoweringInfo &CLI) {
3096 bool IsTailCall = CLI.IsTailCall;
3097 bool IsVarArg = CLI.IsVarArg;
3098 const Value *Callee = CLI.Callee;
3101 if (!Callee && !Symbol)
3123 if (CLI.RetTy->isVoidTy())
3125 else if (!isTypeLegal(CLI.RetTy, RetVT))
3128 for (
auto Flag : CLI.OutFlags)
3130 Flag.isSwiftSelf() ||
Flag.isSwiftError())
3135 OutVTs.
reserve(CLI.OutVals.size());
3137 for (
auto *Val : CLI.OutVals) {
3139 if (!isTypeLegal(Val->getType(), VT) &&
3151 if (Callee && !computeCallAddress(Callee, Addr))
3156 if (!processCallArgs(CLI, OutVTs, NumBytes))
3162 const MCInstrDesc &II =
TII.get(Addr.getReg() ? AArch64::BLR : AArch64::BL);
3163 MIB =
BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II);
3166 else if (Addr.getGlobalValue())
3168 else if (Addr.getReg()) {
3174 unsigned CallReg = 0;
3176 unsigned ADRPReg = createResultReg(&AArch64::GPR64commonRegClass);
3181 CallReg = createResultReg(&AArch64::GPR64RegClass);
3182 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
3183 TII.get(AArch64::LDRXui), CallReg)
3187 }
else if (Addr.getGlobalValue())
3188 CallReg = materializeGV(Addr.getGlobalValue());
3189 else if (Addr.getReg())
3190 CallReg = Addr.getReg();
3197 MIB =
BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II).
addReg(CallReg);
3201 for (
auto Reg : CLI.OutRegs)
3206 MIB.
addRegMask(TRI.getCallPreservedMask(*FuncInfo.MF, CC));
3211 return finishCall(CLI, RetVT, NumBytes);
3214 bool AArch64FastISel::isMemCpySmall(uint64_t Len,
unsigned Alignment) {
3216 return Len / Alignment <= 4;
3221 bool AArch64FastISel::tryEmitSmallMemCpy(
Address Dest,
Address Src,
3222 uint64_t Len,
unsigned Alignment) {
3224 if (!isMemCpySmall(Len, Alignment))
3227 int64_t UnscaledOffset = 0;
3233 if (!Alignment || Alignment >= 8) {
3245 if (Len >= 4 && Alignment == 4)
3247 else if (Len >= 2 && Alignment == 2)
3254 unsigned ResultReg = emitLoad(VT, VT, Src);
3258 if (!emitStore(VT, ResultReg, Dest))
3263 UnscaledOffset += Size;
3266 Dest.setOffset(OrigDest.getOffset() + UnscaledOffset);
3267 Src.setOffset(OrigSrc.getOffset() + UnscaledOffset);
3277 const Value *Cond) {
3278 if (!isa<ExtractValueInst>(Cond))
3281 const auto *EV = cast<ExtractValueInst>(Cond);
3282 if (!isa<IntrinsicInst>(EV->getAggregateOperand()))
3285 const auto *II = cast<IntrinsicInst>(EV->getAggregateOperand());
3287 const Function *Callee = II->getCalledFunction();
3289 cast<StructType>(Callee->
getReturnType())->getTypeAtIndex(0U);
3290 if (!isTypeLegal(RetTy, RetVT))
3293 if (RetVT != MVT::i32 && RetVT !=
MVT::i64)
3296 const Value *LHS = II->getArgOperand(0);
3297 const Value *RHS = II->getArgOperand(1);
3300 if (isa<ConstantInt>(LHS) && !isa<ConstantInt>(RHS) &&
3301 isCommutativeIntrinsic(II))
3309 case Intrinsic::smul_with_overflow:
3310 if (
const auto *C = dyn_cast<ConstantInt>(RHS))
3311 if (C->getValue() == 2)
3312 IID = Intrinsic::sadd_with_overflow;
3314 case Intrinsic::umul_with_overflow:
3315 if (
const auto *C = dyn_cast<ConstantInt>(RHS))
3316 if (C->getValue() == 2)
3317 IID = Intrinsic::uadd_with_overflow;
3325 case Intrinsic::sadd_with_overflow:
3326 case Intrinsic::ssub_with_overflow:
3329 case Intrinsic::uadd_with_overflow:
3332 case Intrinsic::usub_with_overflow:
3335 case Intrinsic::smul_with_overflow:
3336 case Intrinsic::umul_with_overflow:
3342 if (!isValueAvailable(II))
3348 for (
auto Itr = std::prev(Start); Itr !=
End; --Itr) {
3351 if (!isa<ExtractValueInst>(Itr))
3355 const auto *EVI = cast<ExtractValueInst>(Itr);
3356 if (EVI->getAggregateOperand() != II)
3364 bool AArch64FastISel::fastLowerIntrinsicCall(
const IntrinsicInst *II) {
3367 default:
return false;
3368 case Intrinsic::frameaddress: {
3375 unsigned SrcReg =
MRI.createVirtualRegister(&AArch64::GPR64RegClass);
3376 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
3377 TII.get(TargetOpcode::COPY), SrcReg).addReg(FramePtr);
3384 unsigned Depth = cast<ConstantInt>(II->
getOperand(0))->getZExtValue();
3386 DestReg = fastEmitInst_ri(AArch64::LDRXui, &AArch64::GPR64RegClass,
3388 assert(DestReg &&
"Unexpected LDR instruction emission failure.");
3392 updateValueMap(II, SrcReg);
3395 case Intrinsic::memcpy:
3396 case Intrinsic::memmove: {
3397 const auto *MTI = cast<MemTransferInst>(II);
3399 if (MTI->isVolatile())
3405 if (isa<ConstantInt>(MTI->getLength()) && IsMemCpy) {
3408 uint64_t Len = cast<ConstantInt>(MTI->getLength())->getZExtValue();
3409 unsigned Alignment = MTI->getAlignment();
3410 if (isMemCpySmall(Len, Alignment)) {
3412 if (!computeAddress(MTI->getRawDest(), Dest) ||
3413 !computeAddress(MTI->getRawSource(), Src))
3415 if (tryEmitSmallMemCpy(Dest, Src, Len, Alignment))
3420 if (!MTI->getLength()->getType()->isIntegerTy(64))
3423 if (MTI->getSourceAddressSpace() > 255 || MTI->getDestAddressSpace() > 255)
3428 const char *IntrMemName = isa<MemCpyInst>(II) ?
"memcpy" :
"memmove";
3431 case Intrinsic::memset: {
3432 const MemSetInst *MSI = cast<MemSetInst>(II);
3447 case Intrinsic::sin:
3448 case Intrinsic::cos:
3449 case Intrinsic::pow: {
3451 if (!isTypeLegal(II->
getType(), RetVT))
3467 case Intrinsic::sin:
3468 LC = LibCallTable[0][Is64Bit];
3470 case Intrinsic::cos:
3471 LC = LibCallTable[1][Is64Bit];
3473 case Intrinsic::pow:
3474 LC = LibCallTable[2][Is64Bit];
3485 Entry.Ty = Arg->getType();
3486 Args.push_back(Entry);
3489 CallLoweringInfo CLI;
3491 CLI.setCallee(DL, Ctx, TLI.getLibcallCallingConv(LC), II->
getType(),
3492 TLI.getLibcallName(LC), std::move(Args));
3493 if (!lowerCallTo(CLI))
3495 updateValueMap(II, CLI.ResultReg);
3498 case Intrinsic::fabs: {
3500 if (!isTypeLegal(II->
getType(), VT))
3508 Opc = AArch64::FABSSr;
3511 Opc = AArch64::FABSDr;
3514 unsigned SrcReg = getRegForValue(II->
getOperand(0));
3517 bool SrcRegIsKill = hasTrivialKill(II->
getOperand(0));
3518 unsigned ResultReg = createResultReg(TLI.getRegClassFor(VT));
3519 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(Opc), ResultReg)
3521 updateValueMap(II, ResultReg);
3524 case Intrinsic::trap: {
3525 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(AArch64::BRK))
3529 case Intrinsic::sqrt: {
3533 if (!isTypeLegal(RetTy, VT))
3536 unsigned Op0Reg = getRegForValue(II->
getOperand(0));
3539 bool Op0IsKill = hasTrivialKill(II->
getOperand(0));
3541 unsigned ResultReg = fastEmit_r(VT, VT,
ISD::FSQRT, Op0Reg, Op0IsKill);
3545 updateValueMap(II, ResultReg);
3548 case Intrinsic::sadd_with_overflow:
3549 case Intrinsic::uadd_with_overflow:
3550 case Intrinsic::ssub_with_overflow:
3551 case Intrinsic::usub_with_overflow:
3552 case Intrinsic::smul_with_overflow:
3553 case Intrinsic::umul_with_overflow: {
3557 Type *RetTy = Ty->getTypeAtIndex(0U);
3560 if (!isTypeLegal(RetTy, VT))
3563 if (VT != MVT::i32 && VT !=
MVT::i64)
3569 if (isa<ConstantInt>(LHS) && !isa<ConstantInt>(RHS) &&
3570 isCommutativeIntrinsic(II))
3578 case Intrinsic::smul_with_overflow:
3579 if (
const auto *C = dyn_cast<ConstantInt>(RHS))
3580 if (C->getValue() == 2) {
3581 IID = Intrinsic::sadd_with_overflow;
3585 case Intrinsic::umul_with_overflow:
3586 if (
const auto *C = dyn_cast<ConstantInt>(RHS))
3587 if (C->getValue() == 2) {
3588 IID = Intrinsic::uadd_with_overflow;
3594 unsigned ResultReg1 = 0, ResultReg2 = 0, MulReg = 0;
3598 case Intrinsic::sadd_with_overflow:
3599 ResultReg1 = emitAdd(VT, LHS, RHS,
true);
3602 case Intrinsic::uadd_with_overflow:
3603 ResultReg1 = emitAdd(VT, LHS, RHS,
true);
3606 case Intrinsic::ssub_with_overflow:
3607 ResultReg1 = emitSub(VT, LHS, RHS,
true);
3610 case Intrinsic::usub_with_overflow:
3611 ResultReg1 = emitSub(VT, LHS, RHS,
true);
3614 case Intrinsic::smul_with_overflow: {
3616 unsigned LHSReg = getRegForValue(LHS);
3619 bool LHSIsKill = hasTrivialKill(LHS);
3621 unsigned RHSReg = getRegForValue(RHS);
3624 bool RHSIsKill = hasTrivialKill(RHS);
3626 if (VT == MVT::i32) {
3627 MulReg = emitSMULL_rr(
MVT::i64, LHSReg, LHSIsKill, RHSReg, RHSIsKill);
3630 MulReg = fastEmitInst_extractsubreg(VT, MulReg,
true,
3632 ShiftReg = fastEmitInst_extractsubreg(VT, ShiftReg,
true,
3634 emitSubs_rs(VT, ShiftReg,
true, MulReg,
false,
3640 MulReg = emitMul_rr(VT, LHSReg,
false, RHSReg,
3642 unsigned SMULHReg = fastEmit_rr(VT, VT,
ISD::MULHS, LHSReg, LHSIsKill,
3644 emitSubs_rs(VT, SMULHReg,
true, MulReg,
false,
3649 case Intrinsic::umul_with_overflow: {
3651 unsigned LHSReg = getRegForValue(LHS);
3654 bool LHSIsKill = hasTrivialKill(LHS);
3656 unsigned RHSReg = getRegForValue(RHS);
3659 bool RHSIsKill = hasTrivialKill(RHS);
3661 if (VT == MVT::i32) {
3662 MulReg = emitUMULL_rr(
MVT::i64, LHSReg, LHSIsKill, RHSReg, RHSIsKill);
3663 emitSubs_rs(
MVT::i64, AArch64::XZR,
true, MulReg,
3666 MulReg = fastEmitInst_extractsubreg(VT, MulReg,
true,
3672 MulReg = emitMul_rr(VT, LHSReg,
false, RHSReg,
3674 unsigned UMULHReg = fastEmit_rr(VT, VT,
ISD::MULHU, LHSReg, LHSIsKill,
3676 emitSubs_rr(VT, AArch64::XZR,
true, UMULHReg,
3684 ResultReg1 = createResultReg(TLI.getRegClassFor(VT));
3685 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
3686 TII.get(TargetOpcode::COPY), ResultReg1).addReg(MulReg);
3689 ResultReg2 = fastEmitInst_rri(AArch64::CSINCWr, &AArch64::GPR32RegClass,
3690 AArch64::WZR,
true, AArch64::WZR,
3693 assert((ResultReg1 + 1) == ResultReg2 &&
3694 "Nonconsecutive result registers.");
3695 updateValueMap(II, ResultReg1, 2);
3702 bool AArch64FastISel::selectRet(
const Instruction *I) {
3706 if (!FuncInfo.CanLowerReturn)
3712 if (TLI.supportSwiftError() &&
3713 F.getAttributes().hasAttrSomewhere(Attribute::SwiftError))
3716 if (TLI.supportSplitCSR(FuncInfo.MF))
3731 : RetCC_AArch64_AAPCS;
3732 CCInfo.AnalyzeReturn(Outs, RetCC);
3735 if (ValLocs.size() != 1)
3750 unsigned Reg = getRegForValue(RV);
3754 unsigned SrcReg = Reg + VA.
getValNo();
3757 if (!
MRI.getRegClass(SrcReg)->contains(DestReg))
3760 EVT RVEVT = TLI.getValueType(DL, RV->
getType());
3766 !Subtarget->isLittleEndian())
3775 if (RVVT != DestVT) {
3779 if (!Outs[0].Flags.isZExt() && !Outs[0].Flags.isSExt())
3782 bool IsZExt = Outs[0].Flags.isZExt();
3783 SrcReg = emitIntExt(RVVT, SrcReg, DestVT, IsZExt);
3789 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
3790 TII.get(TargetOpcode::COPY), DestReg).addReg(SrcReg);
3797 TII.get(AArch64::RET_ReallyLR));
3798 for (
unsigned RetReg : RetRegs)
3803 bool AArch64FastISel::selectTrunc(
const Instruction *I) {
3808 EVT SrcEVT = TLI.getValueType(DL, SrcTy,
true);
3809 EVT DestEVT = TLI.getValueType(DL, DestTy,
true);
3825 unsigned SrcReg = getRegForValue(Op);
3828 bool SrcIsKill = hasTrivialKill(Op);
3853 unsigned Reg32 = fastEmitInst_extractsubreg(MVT::i32, SrcReg, SrcIsKill,
3856 ResultReg = emitAnd_ri(MVT::i32, Reg32,
true, Mask);
3857 assert(ResultReg &&
"Unexpected AND instruction emission failure.");
3859 ResultReg = createResultReg(&AArch64::GPR32RegClass);
3860 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
3861 TII.get(TargetOpcode::COPY), ResultReg)
3865 updateValueMap(I, ResultReg);
3869 unsigned AArch64FastISel::emiti1Ext(
unsigned SrcReg,
MVT DestVT,
bool IsZExt) {
3872 "Unexpected value type.");
3878 unsigned ResultReg = emitAnd_ri(MVT::i32, SrcReg,
false, 1);
3879 assert(ResultReg &&
"Unexpected AND instruction emission failure.");
3883 unsigned Reg64 =
MRI.createVirtualRegister(&AArch64::GPR64RegClass);
3884 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
3885 TII.get(AArch64::SUBREG_TO_REG), Reg64)
3888 .
addImm(AArch64::sub_32);
3897 return fastEmitInst_rii(AArch64::SBFMWri, &AArch64::GPR32RegClass, SrcReg,
3902 unsigned AArch64FastISel::emitMul_rr(
MVT RetVT,
unsigned Op0,
bool Op0IsKill,
3903 unsigned Op1,
bool Op1IsKill) {
3911 Opc = AArch64::MADDWrrr; ZReg = AArch64::WZR;
break;
3913 Opc = AArch64::MADDXrrr; ZReg = AArch64::XZR;
break;
3917 (RetVT ==
MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
3918 return fastEmitInst_rrr(Opc, RC, Op0, Op0IsKill, Op1, Op1IsKill,
3922 unsigned AArch64FastISel::emitSMULL_rr(
MVT RetVT,
unsigned Op0,
bool Op0IsKill,
3923 unsigned Op1,
bool Op1IsKill) {
3927 return fastEmitInst_rrr(AArch64::SMADDLrrr, &AArch64::GPR64RegClass,
3928 Op0, Op0IsKill, Op1, Op1IsKill,
3929 AArch64::XZR,
true);
3932 unsigned AArch64FastISel::emitUMULL_rr(
MVT RetVT,
unsigned Op0,
bool Op0IsKill,
3933 unsigned Op1,
bool Op1IsKill) {
3937 return fastEmitInst_rrr(AArch64::UMADDLrrr, &AArch64::GPR64RegClass,
3938 Op0, Op0IsKill, Op1, Op1IsKill,
3939 AArch64::XZR,
true);
3942 unsigned AArch64FastISel::emitLSL_rr(
MVT RetVT,
unsigned Op0Reg,
bool Op0IsKill,
3943 unsigned Op1Reg,
bool Op1IsKill) {
3945 bool NeedTrunc =
false;
3949 case MVT::i8: Opc = AArch64::LSLVWr; NeedTrunc =
true; Mask = 0xff;
break;
3950 case MVT::i16: Opc = AArch64::LSLVWr; NeedTrunc =
true; Mask = 0xffff;
break;
3951 case MVT::i32: Opc = AArch64::LSLVWr;
break;
3952 case MVT::i64: Opc = AArch64::LSLVXr;
break;
3956 (RetVT ==
MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
3958 Op1Reg = emitAnd_ri(MVT::i32, Op1Reg, Op1IsKill, Mask);
3961 unsigned ResultReg = fastEmitInst_rr(Opc, RC, Op0Reg, Op0IsKill, Op1Reg,
3964 ResultReg = emitAnd_ri(MVT::i32, ResultReg,
true, Mask);
3968 unsigned AArch64FastISel::emitLSL_ri(
MVT RetVT,
MVT SrcVT,
unsigned Op0,
3969 bool Op0IsKill, uint64_t Shift,
3972 "Unexpected source/return type pair.");
3974 SrcVT == MVT::i32 || SrcVT ==
MVT::i64) &&
3975 "Unexpected source value type.");
3977 RetVT ==
MVT::i64) &&
"Unexpected return value type.");
3979 bool Is64Bit = (RetVT ==
MVT::i64);
3980 unsigned RegSize = Is64Bit ? 64 : 32;
3984 Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
3988 if (RetVT == SrcVT) {
3989 unsigned ResultReg = createResultReg(RC);
3990 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
3991 TII.get(TargetOpcode::COPY), ResultReg)
3995 return emitIntExt(SrcVT, Op0, RetVT, IsZExt);
3999 if (Shift >= DstBits)
4027 unsigned ImmR = RegSize - Shift;
4029 unsigned ImmS = std::min<unsigned>(SrcBits - 1, DstBits - 1 - Shift);
4030 static const unsigned OpcTable[2][2] = {
4031 {AArch64::SBFMWri, AArch64::SBFMXri},
4032 {AArch64::UBFMWri, AArch64::UBFMXri}
4034 unsigned Opc = OpcTable[IsZExt][Is64Bit];
4036 unsigned TmpReg =
MRI.createVirtualRegister(RC);
4037 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
4038 TII.get(AArch64::SUBREG_TO_REG), TmpReg)
4041 .
addImm(AArch64::sub_32);
4045 return fastEmitInst_rii(Opc, RC, Op0, Op0IsKill, ImmR, ImmS);
4048 unsigned AArch64FastISel::emitLSR_rr(
MVT RetVT,
unsigned Op0Reg,
bool Op0IsKill,
4049 unsigned Op1Reg,
bool Op1IsKill) {
4051 bool NeedTrunc =
false;
4055 case MVT::i8: Opc = AArch64::LSRVWr; NeedTrunc =
true; Mask = 0xff;
break;
4056 case MVT::i16: Opc = AArch64::LSRVWr; NeedTrunc =
true; Mask = 0xffff;
break;
4057 case MVT::i32: Opc = AArch64::LSRVWr;
break;
4058 case MVT::i64: Opc = AArch64::LSRVXr;
break;
4062 (RetVT ==
MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
4064 Op0Reg = emitAnd_ri(MVT::i32, Op0Reg, Op0IsKill, Mask);
4065 Op1Reg = emitAnd_ri(MVT::i32, Op1Reg, Op1IsKill, Mask);
4066 Op0IsKill = Op1IsKill =
true;
4068 unsigned ResultReg = fastEmitInst_rr(Opc, RC, Op0Reg, Op0IsKill, Op1Reg,
4071 ResultReg = emitAnd_ri(MVT::i32, ResultReg,
true, Mask);
4075 unsigned AArch64FastISel::emitLSR_ri(
MVT RetVT,
MVT SrcVT,
unsigned Op0,
4076 bool Op0IsKill, uint64_t Shift,
4079 "Unexpected source/return type pair.");
4081 SrcVT == MVT::i32 || SrcVT ==
MVT::i64) &&
4082 "Unexpected source value type.");
4084 RetVT ==
MVT::i64) &&
"Unexpected return value type.");
4086 bool Is64Bit = (RetVT ==
MVT::i64);
4087 unsigned RegSize = Is64Bit ? 64 : 32;
4091 Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
4095 if (RetVT == SrcVT) {
4096 unsigned ResultReg = createResultReg(RC);
4097 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
4098 TII.get(TargetOpcode::COPY), ResultReg)
4102 return emitIntExt(SrcVT, Op0, RetVT, IsZExt);
4106 if (Shift >= DstBits)
4134 if (Shift >= SrcBits && IsZExt)
4140 Op0 = emitIntExt(SrcVT, Op0, RetVT, IsZExt);
4149 unsigned ImmR = std::min<unsigned>(SrcBits - 1, Shift);
4150 unsigned ImmS = SrcBits - 1;
4151 static const unsigned OpcTable[2][2] = {
4152 {AArch64::SBFMWri, AArch64::SBFMXri},
4153 {AArch64::UBFMWri, AArch64::UBFMXri}
4155 unsigned Opc = OpcTable[IsZExt][Is64Bit];
4157 unsigned TmpReg =
MRI.createVirtualRegister(RC);
4158 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
4159 TII.get(AArch64::SUBREG_TO_REG), TmpReg)
4162 .
addImm(AArch64::sub_32);
4166 return fastEmitInst_rii(Opc, RC, Op0, Op0IsKill, ImmR, ImmS);
4169 unsigned AArch64FastISel::emitASR_rr(
MVT RetVT,
unsigned Op0Reg,
bool Op0IsKill,
4170 unsigned Op1Reg,
bool Op1IsKill) {
4172 bool NeedTrunc =
false;
4176 case MVT::i8: Opc = AArch64::ASRVWr; NeedTrunc =
true; Mask = 0xff;
break;
4177 case MVT::i16: Opc = AArch64::ASRVWr; NeedTrunc =
true; Mask = 0xffff;
break;
4178 case MVT::i32: Opc = AArch64::ASRVWr;
break;
4179 case MVT::i64: Opc = AArch64::ASRVXr;
break;
4183 (RetVT ==
MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
4185 Op0Reg = emitIntExt(RetVT, Op0Reg, MVT::i32,
false);
4186 Op1Reg = emitAnd_ri(MVT::i32, Op1Reg, Op1IsKill, Mask);
4187 Op0IsKill = Op1IsKill =
true;
4189 unsigned ResultReg = fastEmitInst_rr(Opc, RC, Op0Reg, Op0IsKill, Op1Reg,
4192 ResultReg = emitAnd_ri(MVT::i32, ResultReg,
true, Mask);
4196 unsigned AArch64FastISel::emitASR_ri(
MVT RetVT,
MVT SrcVT,
unsigned Op0,
4197 bool Op0IsKill, uint64_t Shift,
4200 "Unexpected source/return type pair.");
4202 SrcVT == MVT::i32 || SrcVT ==
MVT::i64) &&
4203 "Unexpected source value type.");
4205 RetVT ==
MVT::i64) &&
"Unexpected return value type.");
4207 bool Is64Bit = (RetVT ==
MVT::i64);
4208 unsigned RegSize = Is64Bit ? 64 : 32;
4212 Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
4216 if (RetVT == SrcVT) {
4217 unsigned ResultReg = createResultReg(RC);
4218 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
4219 TII.get(TargetOpcode::COPY), ResultReg)
4223 return emitIntExt(SrcVT, Op0, RetVT, IsZExt);
4227 if (Shift >= DstBits)
4255 if (Shift >= SrcBits && IsZExt)
4258 unsigned ImmR = std::min<unsigned>(SrcBits - 1, Shift);
4259 unsigned ImmS = SrcBits - 1;
4260 static const unsigned OpcTable[2][2] = {
4261 {AArch64::SBFMWri, AArch64::SBFMXri},
4262 {AArch64::UBFMWri, AArch64::UBFMXri}
4264 unsigned Opc = OpcTable[IsZExt][Is64Bit];
4266 unsigned TmpReg =
MRI.createVirtualRegister(RC);
4267 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
4268 TII.get(AArch64::SUBREG_TO_REG), TmpReg)
4271 .
addImm(AArch64::sub_32);
4275 return fastEmitInst_rii(Opc, RC, Op0, Op0IsKill, ImmR, ImmS);
4278 unsigned AArch64FastISel::emitIntExt(
MVT SrcVT,
unsigned SrcReg,
MVT DestVT,
4287 (DestVT != MVT::i32) && (DestVT !=
MVT::i64)) ||
4289 (SrcVT !=
MVT::i16) && (SrcVT != MVT::i32)))
4299 return emiti1Ext(SrcReg, DestVT, IsZExt);
4302 Opc = IsZExt ? AArch64::UBFMXri : AArch64::SBFMXri;
4304 Opc = IsZExt ? AArch64::UBFMWri : AArch64::SBFMWri;
4309 Opc = IsZExt ? AArch64::UBFMXri : AArch64::SBFMXri;
4311 Opc = IsZExt ? AArch64::UBFMWri : AArch64::SBFMWri;
4316 Opc = IsZExt ? AArch64::UBFMXri : AArch64::SBFMXri;
4325 unsigned Src64 =
MRI.createVirtualRegister(&AArch64::GPR64RegClass);
4326 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
4327 TII.get(AArch64::SUBREG_TO_REG), Src64)
4330 .
addImm(AArch64::sub_32);
4335 (DestVT ==
MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
4336 return fastEmitInst_rii(Opc, RC, SrcReg,
false, 0, Imm);
4343 case AArch64::LDURBBi:
4344 case AArch64::LDURHHi:
4345 case AArch64::LDURWi:
4346 case AArch64::LDRBBui:
4347 case AArch64::LDRHHui:
4348 case AArch64::LDRWui:
4349 case AArch64::LDRBBroX:
4350 case AArch64::LDRHHroX:
4351 case AArch64::LDRWroX:
4352 case AArch64::LDRBBroW:
4353 case AArch64::LDRHHroW:
4354 case AArch64::LDRWroW:
4363 case AArch64::LDURSBWi:
4364 case AArch64::LDURSHWi:
4365 case AArch64::LDURSBXi:
4366 case AArch64::LDURSHXi:
4367 case AArch64::LDURSWi:
4368 case AArch64::LDRSBWui:
4369 case AArch64::LDRSHWui:
4370 case AArch64::LDRSBXui:
4371 case AArch64::LDRSHXui:
4372 case AArch64::LDRSWui:
4373 case AArch64::LDRSBWroX:
4374 case AArch64::LDRSHWroX:
4375 case AArch64::LDRSBXroX:
4376 case AArch64::LDRSHXroX:
4377 case AArch64::LDRSWroX:
4378 case AArch64::LDRSBWroW:
4379 case AArch64::LDRSHWroW:
4380 case AArch64::LDRSBXroW:
4381 case AArch64::LDRSHXroW:
4382 case AArch64::LDRSWroW:
4387 bool AArch64FastISel::optimizeIntExtLoad(
const Instruction *I,
MVT RetVT,
4390 if (!LI || !LI->hasOneUse())
4394 unsigned Reg = lookUpRegForValue(LI);
4404 bool IsZExt = isa<ZExtInst>(
I);
4405 const auto *LoadMI =
MI;
4406 if (LoadMI->getOpcode() == TargetOpcode::COPY &&
4407 LoadMI->getOperand(1).getSubReg() == AArch64::sub_32) {
4409 LoadMI =
MRI.getUniqueVRegDef(LoadReg);
4410 assert(LoadMI &&
"Expected valid instruction");
4416 if (RetVT !=
MVT::i64 || SrcVT > MVT::i32) {
4417 updateValueMap(I, Reg);
4422 unsigned Reg64 = createResultReg(&AArch64::GPR64RegClass);
4423 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
4424 TII.get(AArch64::SUBREG_TO_REG), Reg64)
4427 .
addImm(AArch64::sub_32);
4432 "Expected copy instruction");
4436 updateValueMap(I, Reg);
4440 bool AArch64FastISel::selectIntExt(
const Instruction *I) {
4441 assert((isa<ZExtInst>(I) || isa<SExtInst>(I)) &&
4442 "Unexpected integer extend instruction.");
4445 if (!isTypeSupported(I->
getType(), RetVT))
4452 if (optimizeIntExtLoad(I, RetVT, SrcVT))
4455 unsigned SrcReg = getRegForValue(I->
getOperand(0));
4458 bool SrcIsKill = hasTrivialKill(I->
getOperand(0));
4461 bool IsZExt = isa<ZExtInst>(
I);
4462 if (
const auto *Arg = dyn_cast<Argument>(I->
getOperand(0))) {
4463 if ((IsZExt && Arg->hasZExtAttr()) || (!IsZExt && Arg->hasSExtAttr())) {
4465 unsigned ResultReg = createResultReg(&AArch64::GPR64RegClass);
4466 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
4467 TII.get(AArch64::SUBREG_TO_REG), ResultReg)
4470 .
addImm(AArch64::sub_32);
4477 unsigned UseReg = lookUpRegForValue(I);
4479 MRI.clearKillFlags(UseReg);
4481 updateValueMap(I, SrcReg);
4486 unsigned ResultReg = emitIntExt(SrcVT, SrcReg, RetVT, IsZExt);
4490 updateValueMap(I, ResultReg);
4494 bool AArch64FastISel::selectRem(
const Instruction *I,
unsigned ISDOpcode) {
4495 EVT DestEVT = TLI.getValueType(DL, I->
getType(),
true);
4500 if (DestVT !=
MVT::i64 && DestVT != MVT::i32)
4504 bool Is64bit = (DestVT ==
MVT::i64);
4505 switch (ISDOpcode) {
4509 DivOpc = Is64bit ? AArch64::SDIVXr : AArch64::SDIVWr;
4512 DivOpc = Is64bit ? AArch64::UDIVXr : AArch64::UDIVWr;
4515 unsigned MSubOpc = Is64bit ? AArch64::MSUBXrrr : AArch64::MSUBWrrr;
4516 unsigned Src0Reg = getRegForValue(I->
getOperand(0));
4519 bool Src0IsKill = hasTrivialKill(I->
getOperand(0));
4521 unsigned Src1Reg = getRegForValue(I->
getOperand(1));
4524 bool Src1IsKill = hasTrivialKill(I->
getOperand(1));
4527 (DestVT ==
MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
4528 unsigned QuotReg = fastEmitInst_rr(DivOpc, RC, Src0Reg,
false,
4530 assert(QuotReg &&
"Unexpected DIV instruction emission failure.");
4533 unsigned ResultReg = fastEmitInst_rrr(MSubOpc, RC, QuotReg,
true,
4534 Src1Reg, Src1IsKill, Src0Reg,
4536 updateValueMap(I, ResultReg);
4540 bool AArch64FastISel::selectMul(
const Instruction *I) {
4542 if (!isTypeSupported(I->
getType(), VT,
true))
4550 if (
const auto *C = dyn_cast<ConstantInt>(Src0))
4551 if (C->getValue().isPowerOf2())
4555 if (
const auto *C = dyn_cast<ConstantInt>(Src1))
4556 if (C->getValue().isPowerOf2()) {
4557 uint64_t ShiftVal = C->getValue().logBase2();
4560 if (
const auto *ZExt = dyn_cast<ZExtInst>(Src0)) {
4563 if (isValueAvailable(ZExt) && isTypeSupported(ZExt->getSrcTy(), VT)) {
4566 Src0 = ZExt->getOperand(0);
4569 }
else if (
const auto *SExt = dyn_cast<SExtInst>(Src0)) {
4572 if (isValueAvailable(SExt) && isTypeSupported(SExt->getSrcTy(), VT)) {
4575 Src0 = SExt->getOperand(0);
4580 unsigned Src0Reg = getRegForValue(Src0);
4583 bool Src0IsKill = hasTrivialKill(Src0);
4585 unsigned ResultReg =
4586 emitLSL_ri(VT, SrcVT, Src0Reg, Src0IsKill, ShiftVal, IsZExt);
4589 updateValueMap(I, ResultReg);
4594 unsigned Src0Reg = getRegForValue(I->
getOperand(0));
4597 bool Src0IsKill = hasTrivialKill(I->
getOperand(0));
4599 unsigned Src1Reg = getRegForValue(I->
getOperand(1));
4602 bool Src1IsKill = hasTrivialKill(I->
getOperand(1));
4604 unsigned ResultReg = emitMul_rr(VT, Src0Reg, Src0IsKill, Src1Reg, Src1IsKill);
4609 updateValueMap(I, ResultReg);
4613 bool AArch64FastISel::selectShift(
const Instruction *I) {
4615 if (!isTypeSupported(I->
getType(), RetVT,
true))
4619 return selectOperator(I, I->
getOpcode());
4621 if (
const auto *C = dyn_cast<ConstantInt>(I->
getOperand(1))) {
4622 unsigned ResultReg = 0;
4623 uint64_t ShiftVal = C->getZExtValue();
4625 bool IsZExt = I->
getOpcode() != Instruction::AShr;
4627 if (
const auto *ZExt = dyn_cast<ZExtInst>(Op0)) {
4630 if (isValueAvailable(ZExt) && isTypeSupported(ZExt->getSrcTy(), TmpVT)) {
4633 Op0 = ZExt->getOperand(0);
4636 }
else if (
const auto *SExt = dyn_cast<SExtInst>(Op0)) {
4639 if (isValueAvailable(SExt) && isTypeSupported(SExt->getSrcTy(), TmpVT)) {
4642 Op0 = SExt->getOperand(0);
4647 unsigned Op0Reg = getRegForValue(Op0);
4650 bool Op0IsKill = hasTrivialKill(Op0);
4654 case Instruction::Shl:
4655 ResultReg = emitLSL_ri(RetVT, SrcVT, Op0Reg, Op0IsKill, ShiftVal, IsZExt);
4657 case Instruction::AShr:
4658 ResultReg = emitASR_ri(RetVT, SrcVT, Op0Reg, Op0IsKill, ShiftVal, IsZExt);
4660 case Instruction::LShr:
4661 ResultReg = emitLSR_ri(RetVT, SrcVT, Op0Reg, Op0IsKill, ShiftVal, IsZExt);
4667 updateValueMap(I, ResultReg);
4671 unsigned Op0Reg = getRegForValue(I->
getOperand(0));
4674 bool Op0IsKill = hasTrivialKill(I->
getOperand(0));
4676 unsigned Op1Reg = getRegForValue(I->
getOperand(1));
4679 bool Op1IsKill = hasTrivialKill(I->
getOperand(1));
4681 unsigned ResultReg = 0;
4684 case Instruction::Shl:
4685 ResultReg = emitLSL_rr(RetVT, Op0Reg, Op0IsKill, Op1Reg, Op1IsKill);
4687 case Instruction::AShr:
4688 ResultReg = emitASR_rr(RetVT, Op0Reg, Op0IsKill, Op1Reg, Op1IsKill);
4690 case Instruction::LShr:
4691 ResultReg = emitLSR_rr(RetVT, Op0Reg, Op0IsKill, Op1Reg, Op1IsKill);
4698 updateValueMap(I, ResultReg);
4702 bool AArch64FastISel::selectBitCast(
const Instruction *I) {
4707 if (!isTypeLegal(I->
getType(), RetVT))
4711 if (RetVT ==
MVT::f32 && SrcVT == MVT::i32)
4712 Opc = AArch64::FMOVWSr;
4714 Opc = AArch64::FMOVXDr;
4715 else if (RetVT == MVT::i32 && SrcVT ==
MVT::f32)
4716 Opc = AArch64::FMOVSWr;
4718 Opc = AArch64::FMOVDXr;
4725 case MVT::i32: RC = &AArch64::GPR32RegClass;
break;
4726 case MVT::i64: RC = &AArch64::GPR64RegClass;
break;
4727 case MVT::f32: RC = &AArch64::FPR32RegClass;
break;
4728 case MVT::f64: RC = &AArch64::FPR64RegClass;
break;
4730 unsigned Op0Reg = getRegForValue(I->
getOperand(0));
4733 bool Op0IsKill = hasTrivialKill(I->
getOperand(0));
4734 unsigned ResultReg = fastEmitInst_r(Opc, RC, Op0Reg, Op0IsKill);
4739 updateValueMap(I, ResultReg);
4743 bool AArch64FastISel::selectFRem(
const Instruction *I) {
4745 if (!isTypeLegal(I->
getType(), RetVT))
4767 Entry.Ty = Arg->getType();
4768 Args.push_back(Entry);
4771 CallLoweringInfo CLI;
4773 CLI.setCallee(DL, Ctx, TLI.getLibcallCallingConv(LC), I->
getType(),
4774 TLI.getLibcallName(LC), std::move(Args));
4775 if (!lowerCallTo(CLI))
4777 updateValueMap(I, CLI.ResultReg);
4781 bool AArch64FastISel::selectSDiv(
const Instruction *I) {
4783 if (!isTypeLegal(I->
getType(), VT))
4790 if ((VT != MVT::i32 && VT !=
MVT::i64) || !C ||
4795 unsigned Src0Reg = getRegForValue(I->
getOperand(0));
4798 bool Src0IsKill = hasTrivialKill(I->
getOperand(0));
4800 if (cast<BinaryOperator>(I)->isExact()) {
4801 unsigned ResultReg = emitASR_ri(VT, VT, Src0Reg, Src0IsKill, Lg2);
4804 updateValueMap(I, ResultReg);
4808 int64_t Pow2MinusOne = (1ULL << Lg2) - 1;
4809 unsigned AddReg = emitAdd_ri_(VT, Src0Reg,
false, Pow2MinusOne);
4814 if (!emitICmp_ri(VT, Src0Reg,
false, 0))
4820 SelectOpc = AArch64::CSELXr;
4821 RC = &AArch64::GPR64RegClass;
4823 SelectOpc = AArch64::CSELWr;
4824 RC = &AArch64::GPR32RegClass;
4826 unsigned SelectReg =
4827 fastEmitInst_rri(SelectOpc, RC, AddReg,
true, Src0Reg,
4834 unsigned ZeroReg = (VT ==
MVT::i64) ? AArch64::XZR : AArch64::WZR;
4837 ResultReg = emitAddSub_rs(
false, VT, ZeroReg,
true,
4840 ResultReg = emitASR_ri(VT, VT, SelectReg,
true, Lg2);
4845 updateValueMap(I, ResultReg);
4852 std::pair<unsigned, bool> AArch64FastISel::getRegForGEPIndex(
const Value *Idx) {
4853 unsigned IdxN = getRegForValue(Idx);
4856 return std::pair<unsigned, bool>(0,
false);
4858 bool IdxNIsKill = hasTrivialKill(Idx);
4861 MVT PtrVT = TLI.getPointerTy(DL);
4863 if (IdxVT.
bitsLT(PtrVT)) {
4864 IdxN = emitIntExt(IdxVT.
getSimpleVT(), IdxN, PtrVT,
false);
4866 }
else if (IdxVT.
bitsGT(PtrVT))
4867 llvm_unreachable(
"AArch64 FastISel doesn't support types larger than i64");
4868 return std::pair<unsigned, bool>(IdxN, IdxNIsKill);
4875 bool AArch64FastISel::selectGetElementPtr(
const Instruction *I) {
4879 bool NIsKill = hasTrivialKill(I->
getOperand(0));
4883 uint64_t TotalOffs = 0;
4884 MVT VT = TLI.getPointerTy(DL);
4887 const Value *Idx = GTI.getOperand();
4888 if (
auto *StTy = GTI.getStructTypeOrNull()) {
4889 unsigned Field = cast<ConstantInt>(Idx)->getZExtValue();
4892 TotalOffs +=
DL.getStructLayout(StTy)->getElementOffset(Field);
4894 Type *Ty = GTI.getIndexedType();
4897 if (
const auto *CI = dyn_cast<ConstantInt>(Idx)) {
4902 DL.getTypeAllocSize(Ty) * cast<ConstantInt>(CI)->getSExtValue();
4906 N = emitAdd_ri_(VT, N, NIsKill, TotalOffs);
4914 uint64_t ElementSize =
DL.getTypeAllocSize(Ty);
4915 std::pair<unsigned, bool> Pair = getRegForGEPIndex(Idx);
4916 unsigned IdxN = Pair.first;
4917 bool IdxNIsKill = Pair.second;
4921 if (ElementSize != 1) {
4922 unsigned C = fastEmit_i(VT, VT,
ISD::Constant, ElementSize);
4925 IdxN = emitMul_rr(VT, IdxN, IdxNIsKill, C,
true);
4930 N = fastEmit_rr(VT, VT,
ISD::ADD, N, NIsKill, IdxN, IdxNIsKill);
4936 N = emitAdd_ri_(VT, N, NIsKill, TotalOffs);
4940 updateValueMap(I, N);
4946 "cmpxchg survived AtomicExpand at optlevel > -O0");
4948 auto *RetPairTy = cast<StructType>(I->
getType());
4949 Type *RetTy = RetPairTy->getTypeAtIndex(0U);
4950 assert(RetPairTy->getTypeAtIndex(1U)->isIntegerTy(1) &&
4951 "cmpxchg has a non-i1 status result");
4954 if (!isTypeLegal(RetTy, VT))
4958 unsigned Opc, CmpOpc;
4961 if (VT == MVT::i32) {
4962 Opc = AArch64::CMP_SWAP_32;
4963 CmpOpc = AArch64::SUBSWrs;
4964 ResRC = &AArch64::GPR32RegClass;
4966 Opc = AArch64::CMP_SWAP_64;
4967 CmpOpc = AArch64::SUBSXrs;
4968 ResRC = &AArch64::GPR64RegClass;
4982 const unsigned ResultReg1 = createResultReg(ResRC);
4983 const unsigned ResultReg2 = createResultReg(&AArch64::GPR32RegClass);
4984 const unsigned ScratchReg = createResultReg(&AArch64::GPR32RegClass);
4987 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
4994 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(CmpOpc))
4995 .addDef(VT == MVT::i32 ? AArch64::WZR : AArch64::XZR)
5000 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(AArch64::CSINCWr))
5006 assert((ResultReg1 + 1) == ResultReg2 &&
"Nonconsecutive result registers.");
5007 updateValueMap(I, ResultReg1, 2);
5011 bool AArch64FastISel::fastSelectInstruction(
const Instruction *I) {
5016 case Instruction::Sub:
5017 return selectAddSub(I);
5018 case Instruction::Mul:
5019 return selectMul(I);
5020 case Instruction::SDiv:
5021 return selectSDiv(I);
5022 case Instruction::SRem:
5026 case Instruction::URem:
5030 case Instruction::Shl:
5031 case Instruction::LShr:
5032 case Instruction::AShr:
5033 return selectShift(I);
5037 return selectLogicalOp(I);
5038 case Instruction::Br:
5039 return selectBranch(I);
5040 case Instruction::IndirectBr:
5041 return selectIndirectBr(I);
5042 case Instruction::BitCast:
5044 return selectBitCast(I);
5046 case Instruction::FPToSI:
5048 return selectFPToInt(I,
true);
5050 case Instruction::FPToUI:
5051 return selectFPToInt(I,
false);
5052 case Instruction::ZExt:
5053 case Instruction::SExt:
5054 return selectIntExt(I);
5055 case Instruction::Trunc:
5057 return selectTrunc(I);
5059 case Instruction::FPExt:
5060 return selectFPExt(I);
5061 case Instruction::FPTrunc:
5062 return selectFPTrunc(I);
5063 case Instruction::SIToFP:
5065 return selectIntToFP(I,
true);
5067 case Instruction::UIToFP:
5068 return selectIntToFP(I,
false);
5070 return selectLoad(I);
5072 return selectStore(I);
5073 case Instruction::FCmp:
5074 case Instruction::ICmp:
5075 return selectCmp(I);
5077 return selectSelect(I);
5079 return selectRet(I);
5080 case Instruction::FRem:
5081 return selectFRem(I);
5082 case Instruction::GetElementPtr:
5083 return selectGetElementPtr(I);
5084 case Instruction::AtomicCmpXchg:
5085 return selectAtomicCmpXchg(cast<AtomicCmpXchgInst>(I));
5089 return selectOperator(I, I->
getOpcode());
5091 (void)&CC_AArch64_DarwinPCS_VarArg;
5097 return new AArch64FastISel(FuncInfo, LibInfo);
void setFrameAddressIsTaken(bool T)
Return a value (possibly void), from a function.
static MVT getIntegerVT(unsigned BitWidth)
void push_back(const T &Elt)
This class is the base class for the comparison instructions.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
LLVM Argument representation.
uint64_t getZExtValue() const
Get zero extended value.
LocInfo getLocInfo() const
MO_PAGE - A symbol operand with this flag represents the pc-relative offset of the 4K page containing...
unsigned constrainOperandRegClass(const MachineFunction &MF, const TargetRegisterInfo &TRI, MachineRegisterInfo &MRI, const TargetInstrInfo &TII, const RegisterBankInfo &RBI, MachineInstr &InsertPt, const MCInstrDesc &II, unsigned Reg, unsigned OpIdx)
Try to constrain Reg so that it is usable by argument OpIdx of the provided MCInstrDesc II...
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo)
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
an instruction that atomically checks whether a specified value is in a memory location, and, if it is, stores a new value there.
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
unsigned getNumOperands() const
static CondCode getInvertedCondCode(CondCode Code)
Describe properties that are true of each instruction in the target description file.
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE, etc.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit...
bool CCAssignFn(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
CCAssignFn - This function assigns a location for Val, updating State to reflect the change...
gep_type_iterator gep_type_end(const User *GEP)
0 1 0 0 True if ordered and less than
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
unsigned getSizeInBits() const
bool isDoubleTy() const
Return true if this is 'double', a 64-bit IEEE fp type.
1 1 1 0 True if unordered or not equal
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
This class wraps the llvm.memset intrinsic.
Type * getReturnType() const
Returns the type of the ret val.
const Function * getParent() const
Return the enclosing method, or null if none.
Value * getNewValOperand()
An instruction for reading from memory.
void reserve(size_type N)
bool hasAttribute(unsigned Index, Attribute::AttrKind Kind) const
Return true if the attribute exists at the given index.
unsigned getValNo() const
const MachineInstrBuilder & addDef(unsigned RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
bool bitsLT(EVT VT) const
bitsLT - Return true if this has less bits than VT.
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
unsigned getFrameRegister(const MachineFunction &MF) const override
bool isNegative() const
Determine sign of this APInt.
bool isVector() const
isVector - Return true if this is a vector value type.
1 0 0 1 True if unordered or equal
Used to lazily calculate structure layout information for a target machine, based on the DataLayout s...
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
bool isUnconditional() const
A description of a memory reference used in the backend.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
This class represents the LLVM 'select' instruction.
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
struct fuzzer::@269 Flags
const HexagonInstrInfo * TII
Class to represent struct types.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
unsigned getNumArgOperands() const
Return the number of call arguments.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
Reg
All possible values of the reg field in the ModR/M byte.
0 1 0 1 True if ordered and less than or equal
APInt bitcastToAPInt() const
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
static bool isLogicalImmediate(uint64_t imm, unsigned regSize)
isLogicalImmediate - Return true if the immediate is valid for a logical immediate instruction of the...
unsigned getStoreSize() const
getStoreSize - Return the number of bytes overwritten by a store of the specified value type...
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
AtomicOrdering
Atomic ordering for LLVM's memory model.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Context object for machine code objects.
Value * getPointerOperand()
unsigned getLocReg() const
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
A constant value that is initialized with an expression using other constant values.
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
MO_GOT - This flag indicates that a symbol operand represents the address of the GOT entry for the sy...
void GetReturnInfo(Type *ReturnType, AttributeSet attr, SmallVectorImpl< ISD::OutputArg > &Outs, const TargetLowering &TLI, const DataLayout &DL)
Given an LLVM IR type and return type attributes, compute the return value EVTs and flags...
static bool isZExtLoad(const MachineInstr *LI)
Simple integer binary arithmetic operators.
BasicBlock * getSuccessor(unsigned i) const
static GCRegistry::Add< OcamlGC > B("ocaml","ocaml 3.10-compatible GC")
static Error getOffset(const SymbolRef &Sym, SectionRef Sec, uint64_t &Result)
bool isArrayTy() const
True if this is an instance of ArrayType.
bool isZero() const
Return true if the value is positive or negative zero.
static MachinePointerInfo getStack(MachineFunction &MF, int64_t Offset)
Stack pointer relative access.
unsigned getKillRegState(bool B)
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
Flag
These should be considered private to the implementation of the MCInstrDesc class.
uint64_t getElementOffset(unsigned Idx) const
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
static unsigned selectBinaryOp(unsigned GenericOpc, unsigned RegBankID, unsigned OpSize)
Select the AArch64 opcode for the basic binary operation GenericOpc (such as G_OR or G_ADD)...
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
Control flow instructions. These all have token chains.
unsigned const MachineRegisterInfo * MRI
static int getFP32Imm(const APInt &Imm)
getFP32Imm - Return an 8-bit floating-point version of the 32-bit floating-point value.
MVT - Machine Value Type.
The instances of the Type class are immutable: once they are created, they are never changed...
This is an important class for using LLVM in a threaded context.
Conditional or Unconditional Branch instruction.
C - The default llvm calling convention, compatible with C.
bool isVectorTy() const
True if this is an instance of VectorType.
This is an important base class in LLVM.
PointerType * getType() const
Overload to return most specific pointer type.
bool isVector() const
isVector - Return true if this is a vector value type.
const Value * getCondition() const
Value * getCompareOperand()
bool isFloatingPoint() const
isFloatingPoint - Return true if this is a FP, or a vector FP type.
const MachineOperand & getOperand(unsigned i) const
Indirect Branch Instruction.
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
APInt Or(const APInt &LHS, const APInt &RHS)
Bitwise OR function for APInt.
static bool isIntExtFree(const Instruction *I)
Check if the sign-/zero-extend will be a noop.
ConstantFP - Floating Point Values [float, double].
APInt Xor(const APInt &LHS, const APInt &RHS)
Bitwise XOR function for APInt.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
constexpr bool isPowerOf2_64(uint64_t Value)
isPowerOf2_64 - This function returns true if the argument is a power of two 0 (64 bit edition...
static bool isSExtLoad(const MachineInstr *LI)
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
static const unsigned End
static AArch64CC::CondCode getCompareCC(CmpInst::Predicate Pred)
Value * getOperand(unsigned i) const
0 1 1 1 True if ordered (no nans)
bool isThreadLocal() const
If the value is "Thread Local", its value isn't shared by the threads.
1 1 1 1 Always true (always folded)
static unsigned getImplicitScaleFactor(MVT VT)
Determine the implicit scale factor that is applied by a memory operation for a given value type...
unsigned getSubReg() const
EVT - Extended Value Type.
LLVMContext & getContext() const
All values hold a context through their type.
const Value * getTrueValue() const
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
1 1 0 1 True if unordered, less than, or equal
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
static bool isReleaseOrStronger(AtomicOrdering ao)
The memory access writes data.
bool bitsGT(EVT VT) const
bitsGT - Return true if this has more bits than VT.
unsigned countTrailingZeros() const
Count the number of trailing zero bits.
0 0 1 0 True if ordered and greater than
static uint64_t encodeLogicalImmediate(uint64_t imm, unsigned regSize)
encodeLogicalImmediate - Return the encoded immediate value for a logical immediate instruction of th...
static int getFP64Imm(const APInt &Imm)
getFP64Imm - Return an 8-bit floating-point version of the 64-bit floating-point value.
static AArch64_AM::ShiftExtendType getExtendType(unsigned Imm)
getExtendType - Extract the extend type for operands of arithmetic ops.
Iterator for intrusive lists based on ilist_node.
CCState - This class holds information needed while lowering arguments and return values...
This is the shared class of boolean and integer constants.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
1 1 0 0 True if unordered or less than
Type * getType() const
All values are typed, get the type of this value.
Provides information about what library functions are available for the current target.
CCValAssign - Represent assignment of one arg/retval to a location.
Value * getLength() const
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Function * getCalledFunction() const
Return the function called, or null if this is an indirect function invocation.
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
bool is64BitVector() const
is64BitVector - Return true if this is a 64-bit vector type.
const MachineInstrBuilder & addFrameIndex(int Idx) const
bool isNullValue() const
Return true if this is the value that would be returned by getNullValue.
static GCRegistry::Add< ShadowStackGC > C("shadow-stack","Very portable GC for uncooperative code generators")
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
AttributeSet getAttributes() const
Return the attribute list for this Function.
bool is128BitVector() const
is128BitVector - Return true if this is a 128-bit vector type.
Value * getArgOperand(unsigned i) const
getArgOperand/setArgOperand - Return/set the i-th call argument.
bool selectBitCast(const User *I)
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
Target - Wrapper for Target specific information.
Class for arbitrary precision integers.
static unsigned getReg(const void *D, unsigned RC, unsigned RegNo)
bool isIntegerTy() const
True if this is an instance of IntegerType.
This file defines the FastISel class.
static unsigned getArithExtendImm(AArch64_AM::ShiftExtendType ET, unsigned Imm)
getArithExtendImm - Encode the extend type and shift amount for an arithmetic instruction: imm: 3-bit...
APInt And(const APInt &LHS, const APInt &RHS)
Bitwise AND function for APInt.
const MachineInstrBuilder & addGlobalAddress(const GlobalValue *GV, int64_t Offset=0, unsigned char TargetFlags=0) const
Flags
Flags values. These may be or'd together.
bool isStructTy() const
True if this is an instance of StructType.
The memory access reads data.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
Representation of each machine instruction.
bool isNegative() const
Return true if the sign bit is set.
PointerType * getType() const
Global values are always pointers.
Value * getCondition() const
Bitwise operators - logical and, logical or, logical xor.
unsigned greater or equal
static unsigned UseReg(const MachineOperand &MO)
const MachineInstrBuilder & addConstantPoolIndex(unsigned Idx, int Offset=0, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
bool hasOneUse() const
Return true if there is exactly one user of this value.
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
MO_PAGEOFF - A symbol operand with this flag represents the offset of that symbol within a 4K page...
0 1 1 0 True if ordered and operands are unequal
iterator find(const KeyT &Val)
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
1 0 1 0 True if unordered or greater than
static EVT getEVT(Type *Ty, bool HandleUnknown=false)
getEVT - Return the value type corresponding to the specified type.
const APFloat & getValueAPF() const
unsigned getReg() const
getReg - Returns the register number.
iterator_range< op_iterator > arg_operands()
Iteration adapter for range-for loops.
bool isUnsigned() const
Determine if this instruction is using an unsigned comparison.
static SDValue emitCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
bool isSimple() const
isSimple - Test if the given EVT is simple (as opposed to being extended).
const MachineInstrBuilder & addUse(unsigned RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
0 0 0 1 True if ordered and equal
LLVM Value Representation.
1 0 1 1 True if unordered, greater than, or equal
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned char TargetFlags=0) const
unsigned getDestAddressSpace() const
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
static const unsigned FramePtr
const Value * getFalseValue() const
MO_NC - Indicates whether the linker is expected to check the symbol reference for overflow...
TRUNCATE - Completely drop the high bits.
const MachineInstrBuilder & addRegMask(const uint32_t *Mask) const
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
0 0 1 1 True if ordered and greater than or equal
const MachineInstrBuilder & addReg(unsigned RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
bool isVarArg() const
isVarArg - Return true if this function takes a variable number of arguments.
const BasicBlock * getParent() const
MVT getSimpleVT() const
getSimpleVT - Return the SimpleValueType held in the specified simple EVT.
iterator_range< arg_iterator > args()
0 0 0 0 Always false (always folded)
A wrapper class for inspecting calls to intrinsic functions.
an instruction to allocate memory on the stack
static bool isMulPowOf2(const Value *I)
Check if the multiply is by a power-of-2 constant.
gep_type_iterator gep_type_begin(const User *GEP)
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
unsigned getVectorNumElements() const
getVectorNumElements - Given a vector type, return the number of elements it contains.