45 class AArch64FastISel final :
public FastISel {
67 OffsetReg(0), Shift(0), Offset(0), GV(nullptr) { Base.Reg = 0; }
68 void setKind(BaseKind K) {
Kind = K; }
69 BaseKind getKind()
const {
return Kind; }
72 bool isRegBase()
const {
return Kind == RegBase; }
73 bool isFIBase()
const {
return Kind == FrameIndexBase; }
74 void setReg(
unsigned Reg) {
75 assert(isRegBase() &&
"Invalid base register access!");
79 assert(isRegBase() &&
"Invalid base register access!");
82 void setOffsetReg(
unsigned Reg) {
85 unsigned getOffsetReg()
const {
88 void setFI(
unsigned FI) {
89 assert(isFIBase() &&
"Invalid base frame index access!");
92 unsigned getFI()
const {
93 assert(isFIBase() &&
"Invalid base frame index access!");
96 void setOffset(int64_t O) { Offset = O; }
98 void setShift(
unsigned S) { Shift = S; }
99 unsigned getShift() {
return Shift; }
110 bool fastLowerArguments()
override;
111 bool fastLowerCall(CallLoweringInfo &CLI)
override;
112 bool fastLowerIntrinsicCall(
const IntrinsicInst *II)
override;
128 bool selectRem(
const Instruction *
I,
unsigned ISDOpcode);
140 bool isTypeLegal(
Type *Ty,
MVT &VT);
141 bool isTypeSupported(
Type *Ty,
MVT &VT,
bool IsVectorAllowed =
false);
142 bool isValueAvailable(
const Value *V)
const;
147 unsigned Flags,
unsigned ScaleFactor,
149 bool isMemCpySmall(uint64_t Len,
unsigned Alignment);
156 std::pair<unsigned, bool> getRegForGEPIndex(
const Value *Idx);
159 unsigned emitAddSub(
bool UseAdd,
MVT RetVT,
const Value *LHS,
160 const Value *RHS,
bool SetFlags =
false,
161 bool WantResult =
true,
bool IsZExt =
false);
162 unsigned emitAddSub_rr(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
163 bool LHSIsKill,
unsigned RHSReg,
bool RHSIsKill,
164 bool SetFlags =
false,
bool WantResult =
true);
165 unsigned emitAddSub_ri(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
166 bool LHSIsKill, uint64_t Imm,
bool SetFlags =
false,
167 bool WantResult =
true);
168 unsigned emitAddSub_rs(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
169 bool LHSIsKill,
unsigned RHSReg,
bool RHSIsKill,
171 uint64_t ShiftImm,
bool SetFlags =
false,
172 bool WantResult =
true);
173 unsigned emitAddSub_rx(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
174 bool LHSIsKill,
unsigned RHSReg,
bool RHSIsKill,
176 uint64_t ShiftImm,
bool SetFlags =
false,
177 bool WantResult =
true);
180 bool emitCompareAndBranch(
const BranchInst *BI);
182 bool emitICmp(
MVT RetVT,
const Value *LHS,
const Value *RHS,
bool IsZExt);
183 bool emitICmp_ri(
MVT RetVT,
unsigned LHSReg,
bool LHSIsKill, uint64_t Imm);
184 bool emitFCmp(
MVT RetVT,
const Value *LHS,
const Value *RHS);
185 unsigned emitLoad(
MVT VT,
MVT ResultVT,
Address Addr,
bool WantZExt =
true,
187 bool emitStore(
MVT VT,
unsigned SrcReg,
Address Addr,
189 unsigned emitIntExt(
MVT SrcVT,
unsigned SrcReg,
MVT DestVT,
bool isZExt);
190 unsigned emiti1Ext(
unsigned SrcReg,
MVT DestVT,
bool isZExt);
191 unsigned emitAdd(
MVT RetVT,
const Value *LHS,
const Value *RHS,
192 bool SetFlags =
false,
bool WantResult =
true,
193 bool IsZExt =
false);
194 unsigned emitAdd_ri_(
MVT VT,
unsigned Op0,
bool Op0IsKill, int64_t Imm);
195 unsigned emitSub(
MVT RetVT,
const Value *LHS,
const Value *RHS,
196 bool SetFlags =
false,
bool WantResult =
true,
197 bool IsZExt =
false);
198 unsigned emitSubs_rr(
MVT RetVT,
unsigned LHSReg,
bool LHSIsKill,
199 unsigned RHSReg,
bool RHSIsKill,
bool WantResult =
true);
200 unsigned emitSubs_rs(
MVT RetVT,
unsigned LHSReg,
bool LHSIsKill,
201 unsigned RHSReg,
bool RHSIsKill,
203 bool WantResult =
true);
204 unsigned emitLogicalOp(
unsigned ISDOpc,
MVT RetVT,
const Value *LHS,
206 unsigned emitLogicalOp_ri(
unsigned ISDOpc,
MVT RetVT,
unsigned LHSReg,
207 bool LHSIsKill, uint64_t Imm);
208 unsigned emitLogicalOp_rs(
unsigned ISDOpc,
MVT RetVT,
unsigned LHSReg,
209 bool LHSIsKill,
unsigned RHSReg,
bool RHSIsKill,
211 unsigned emitAnd_ri(
MVT RetVT,
unsigned LHSReg,
bool LHSIsKill, uint64_t Imm);
212 unsigned emitMul_rr(
MVT RetVT,
unsigned Op0,
bool Op0IsKill,
213 unsigned Op1,
bool Op1IsKill);
214 unsigned emitSMULL_rr(
MVT RetVT,
unsigned Op0,
bool Op0IsKill,
215 unsigned Op1,
bool Op1IsKill);
216 unsigned emitUMULL_rr(
MVT RetVT,
unsigned Op0,
bool Op0IsKill,
217 unsigned Op1,
bool Op1IsKill);
218 unsigned emitLSL_rr(
MVT RetVT,
unsigned Op0Reg,
bool Op0IsKill,
219 unsigned Op1Reg,
bool Op1IsKill);
220 unsigned emitLSL_ri(
MVT RetVT,
MVT SrcVT,
unsigned Op0Reg,
bool Op0IsKill,
221 uint64_t Imm,
bool IsZExt =
true);
222 unsigned emitLSR_rr(
MVT RetVT,
unsigned Op0Reg,
bool Op0IsKill,
223 unsigned Op1Reg,
bool Op1IsKill);
224 unsigned emitLSR_ri(
MVT RetVT,
MVT SrcVT,
unsigned Op0Reg,
bool Op0IsKill,
225 uint64_t Imm,
bool IsZExt =
true);
226 unsigned emitASR_rr(
MVT RetVT,
unsigned Op0Reg,
bool Op0IsKill,
227 unsigned Op1Reg,
bool Op1IsKill);
228 unsigned emitASR_ri(
MVT RetVT,
MVT SrcVT,
unsigned Op0Reg,
bool Op0IsKill,
229 uint64_t Imm,
bool IsZExt =
false);
240 bool finishCall(CallLoweringInfo &CLI,
MVT RetVT,
unsigned NumBytes);
244 unsigned fastMaterializeAlloca(
const AllocaInst *AI)
override;
245 unsigned fastMaterializeConstant(
const Constant *
C)
override;
246 unsigned fastMaterializeFloatZero(
const ConstantFP* CF)
override;
256 bool fastSelectInstruction(
const Instruction *
I)
override;
258 #include "AArch64GenFastISel.inc"
263 #include "AArch64GenCallingConv.inc"
267 assert((isa<ZExtInst>(I) || isa<SExtInst>(I)) &&
268 "Unexpected integer extend instruction.");
270 "Unexpected value type.");
271 bool IsZExt = isa<ZExtInst>(
I);
273 if (
const auto *LI = dyn_cast<LoadInst>(I->
getOperand(0)))
277 if (
const auto *Arg = dyn_cast<Argument>(I->
getOperand(0)))
278 if ((IsZExt && Arg->hasZExtAttr()) || (!IsZExt && Arg->hasSExtAttr()))
306 return CC_AArch64_WebKit_JS;
308 return CC_AArch64_GHC;
309 return Subtarget->isTargetDarwin() ? CC_AArch64_DarwinPCS : CC_AArch64_AAPCS;
312 unsigned AArch64FastISel::fastMaterializeAlloca(
const AllocaInst *AI) {
314 "Alloca should always return a pointer.");
317 if (!FuncInfo.StaticAllocaMap.count(AI))
321 FuncInfo.StaticAllocaMap.find(AI);
323 if (SI != FuncInfo.StaticAllocaMap.
end()) {
324 unsigned ResultReg = createResultReg(&AArch64::GPR64spRegClass);
325 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(AArch64::ADDXri),
327 .addFrameIndex(SI->second)
336 unsigned AArch64FastISel::materializeInt(
const ConstantInt *CI,
MVT VT) {
345 : &AArch64::GPR32RegClass;
346 unsigned ZeroReg = (VT ==
MVT::i64) ? AArch64::XZR : AArch64::WZR;
347 unsigned ResultReg = createResultReg(RC);
353 unsigned AArch64FastISel::materializeFP(
const ConstantFP *CFP,
MVT VT) {
357 return fastMaterializeFloatZero(CFP);
366 if (TLI.isFPImmLegal(Val, VT)) {
369 assert((Imm != -1) &&
"Cannot encode floating-point constant.");
370 unsigned Opc = Is64Bit ? AArch64::FMOVDi : AArch64::FMOVSi;
371 return fastEmitInst_i(Opc, TLI.getRegClassFor(VT), Imm);
376 unsigned Opc1 = Is64Bit ? AArch64::MOVi64imm : AArch64::MOVi32imm;
378 &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
380 unsigned TmpReg = createResultReg(RC);
381 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(Opc1), TmpReg)
384 unsigned ResultReg = createResultReg(TLI.getRegClassFor(VT));
385 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
396 Align =
DL.getTypeAllocSize(CFP->
getType());
398 unsigned CPI = MCP.getConstantPoolIndex(cast<Constant>(CFP), Align);
399 unsigned ADRPReg = createResultReg(&AArch64::GPR64commonRegClass);
403 unsigned Opc = Is64Bit ? AArch64::LDRDui : AArch64::LDRSui;
404 unsigned ResultReg = createResultReg(TLI.getRegClassFor(VT));
405 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(Opc), ResultReg)
411 unsigned AArch64FastISel::materializeGV(
const GlobalValue *GV) {
421 unsigned char OpFlags = Subtarget->ClassifyGlobalReference(GV,
TM);
423 EVT DestEVT = TLI.getValueType(DL, GV->
getType(),
true);
427 unsigned ADRPReg = createResultReg(&AArch64::GPR64commonRegClass);
436 ResultReg = createResultReg(&AArch64::GPR64RegClass);
437 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(AArch64::LDRXui),
451 ResultReg = createResultReg(&AArch64::GPR64spRegClass);
452 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(AArch64::ADDXri),
461 unsigned AArch64FastISel::fastMaterializeConstant(
const Constant *C) {
462 EVT CEVT = TLI.getValueType(DL, C->
getType(),
true);
469 if (
const auto *CI = dyn_cast<ConstantInt>(C))
470 return materializeInt(CI, VT);
471 else if (
const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
472 return materializeFP(CFP, VT);
473 else if (
const GlobalValue *GV = dyn_cast<GlobalValue>(C))
474 return materializeGV(GV);
479 unsigned AArch64FastISel::fastMaterializeFloatZero(
const ConstantFP* CFP) {
481 "Floating-point constant is not a positive zero.");
483 if (!isTypeLegal(CFP->
getType(), VT))
490 unsigned ZReg = Is64Bit ? AArch64::XZR : AArch64::WZR;
491 unsigned Opc = Is64Bit ? AArch64::FMOVXDr : AArch64::FMOVWSr;
492 return fastEmitInst_r(Opc, TLI.getRegClassFor(VT), ZReg,
true);
497 if (
const auto *
MI = dyn_cast<MulOperator>(I)) {
498 if (
const auto *C = dyn_cast<ConstantInt>(
MI->getOperand(0)))
499 if (C->getValue().isPowerOf2())
501 if (
const auto *C = dyn_cast<ConstantInt>(
MI->getOperand(1)))
502 if (C->getValue().isPowerOf2())
509 bool AArch64FastISel::computeAddress(
const Value *Obj,
Address &Addr,
Type *Ty)
511 const User *U =
nullptr;
512 unsigned Opcode = Instruction::UserOp1;
513 if (
const Instruction *
I = dyn_cast<Instruction>(Obj)) {
516 if (FuncInfo.StaticAllocaMap.count(static_cast<const AllocaInst *>(Obj)) ||
517 FuncInfo.MBBMap[
I->getParent()] == FuncInfo.MBB) {
518 Opcode =
I->getOpcode();
521 }
else if (
const ConstantExpr *C = dyn_cast<ConstantExpr>(Obj)) {
522 Opcode = C->getOpcode();
527 if (Ty->getAddressSpace() > 255)
535 case Instruction::BitCast: {
537 return computeAddress(U->
getOperand(0), Addr, Ty);
539 case Instruction::IntToPtr: {
542 TLI.getPointerTy(DL))
543 return computeAddress(U->
getOperand(0), Addr, Ty);
546 case Instruction::PtrToInt: {
548 if (TLI.getValueType(DL, U->
getType()) == TLI.getPointerTy(DL))
549 return computeAddress(U->
getOperand(0), Addr, Ty);
552 case Instruction::GetElementPtr: {
554 uint64_t TmpOffset = Addr.getOffset();
561 const Value *Op = *i;
562 if (
StructType *STy = dyn_cast<StructType>(*GTI)) {
564 unsigned Idx = cast<ConstantInt>(Op)->getZExtValue();
569 if (
const ConstantInt *CI = dyn_cast<ConstantInt>(Op)) {
574 if (canFoldAddIntoGEP(U, Op)) {
577 cast<ConstantInt>(cast<AddOperator>(Op)->getOperand(1));
580 Op = cast<AddOperator>(Op)->getOperand(0);
584 goto unsupported_gep;
590 Addr.setOffset(TmpOffset);
591 if (computeAddress(U->
getOperand(0), Addr, Ty))
600 case Instruction::Alloca: {
603 FuncInfo.StaticAllocaMap.
find(AI);
604 if (SI != FuncInfo.StaticAllocaMap.end()) {
605 Addr.setKind(Address::FrameIndexBase);
606 Addr.setFI(SI->second);
611 case Instruction::Add: {
616 if (isa<ConstantInt>(LHS))
619 if (
const ConstantInt *CI = dyn_cast<ConstantInt>(RHS)) {
621 return computeAddress(LHS, Addr, Ty);
625 if (computeAddress(LHS, Addr, Ty) && computeAddress(RHS, Addr, Ty))
631 case Instruction::Sub: {
636 if (
const ConstantInt *CI = dyn_cast<ConstantInt>(RHS)) {
638 return computeAddress(LHS, Addr, Ty);
642 case Instruction::Shl: {
643 if (Addr.getOffsetReg())
651 if (Val < 1 || Val > 3)
654 uint64_t NumBytes = 0;
656 uint64_t NumBits =
DL.getTypeSizeInBits(Ty);
657 NumBytes = NumBits / 8;
662 if (NumBytes != (1ULL << Val))
669 if (
const auto *
I = dyn_cast<Instruction>(Src)) {
670 if (FuncInfo.MBBMap[
I->getParent()] == FuncInfo.MBB) {
672 if (
const auto *ZE = dyn_cast<ZExtInst>(
I)) {
674 ZE->getOperand(0)->getType()->isIntegerTy(32)) {
676 Src = ZE->getOperand(0);
678 }
else if (
const auto *SE = dyn_cast<SExtInst>(
I)) {
680 SE->getOperand(0)->getType()->isIntegerTy(32)) {
682 Src = SE->getOperand(0);
688 if (
const auto *AI = dyn_cast<BinaryOperator>(Src))
693 if (
const auto *C = dyn_cast<ConstantInt>(LHS))
694 if (C->getValue() == 0xffffffff)
697 if (
const auto *C = dyn_cast<ConstantInt>(RHS))
698 if (C->getValue() == 0xffffffff) {
700 unsigned Reg = getRegForValue(LHS);
703 bool RegIsKill = hasTrivialKill(LHS);
704 Reg = fastEmitInst_extractsubreg(
MVT::i32, Reg, RegIsKill,
706 Addr.setOffsetReg(Reg);
711 unsigned Reg = getRegForValue(Src);
714 Addr.setOffsetReg(Reg);
717 case Instruction::Mul: {
718 if (Addr.getOffsetReg())
728 if (
const auto *C = dyn_cast<ConstantInt>(LHS))
729 if (C->getValue().isPowerOf2())
732 assert(isa<ConstantInt>(RHS) &&
"Expected an ConstantInt.");
733 const auto *C = cast<ConstantInt>(RHS);
734 unsigned Val = C->getValue().logBase2();
735 if (Val < 1 || Val > 3)
738 uint64_t NumBytes = 0;
740 uint64_t NumBits =
DL.getTypeSizeInBits(Ty);
741 NumBytes = NumBits / 8;
746 if (NumBytes != (1ULL << Val))
752 const Value *Src = LHS;
753 if (
const auto *
I = dyn_cast<Instruction>(Src)) {
754 if (FuncInfo.MBBMap[
I->getParent()] == FuncInfo.MBB) {
756 if (
const auto *ZE = dyn_cast<ZExtInst>(
I)) {
758 ZE->getOperand(0)->getType()->isIntegerTy(32)) {
760 Src = ZE->getOperand(0);
762 }
else if (
const auto *SE = dyn_cast<SExtInst>(
I)) {
764 SE->getOperand(0)->getType()->isIntegerTy(32)) {
766 Src = SE->getOperand(0);
772 unsigned Reg = getRegForValue(Src);
775 Addr.setOffsetReg(Reg);
779 if (Addr.getOffsetReg())
782 if (!Ty ||
DL.getTypeSizeInBits(Ty) != 8)
788 if (
const auto *C = dyn_cast<ConstantInt>(LHS))
789 if (C->getValue() == 0xffffffff)
792 if (
const auto *C = dyn_cast<ConstantInt>(RHS))
793 if (C->getValue() == 0xffffffff) {
798 unsigned Reg = getRegForValue(LHS);
801 bool RegIsKill = hasTrivialKill(LHS);
802 Reg = fastEmitInst_extractsubreg(
MVT::i32, Reg, RegIsKill,
804 Addr.setOffsetReg(Reg);
809 case Instruction::SExt:
810 case Instruction::ZExt: {
811 if (!Addr.getReg() || Addr.getOffsetReg())
814 const Value *Src =
nullptr;
816 if (
const auto *ZE = dyn_cast<ZExtInst>(U)) {
817 if (!
isIntExtFree(ZE) && ZE->getOperand(0)->getType()->isIntegerTy(32)) {
819 Src = ZE->getOperand(0);
821 }
else if (
const auto *SE = dyn_cast<SExtInst>(U)) {
822 if (!
isIntExtFree(SE) && SE->getOperand(0)->getType()->isIntegerTy(32)) {
824 Src = SE->getOperand(0);
832 unsigned Reg = getRegForValue(Src);
835 Addr.setOffsetReg(Reg);
840 if (Addr.isRegBase() && !Addr.getReg()) {
841 unsigned Reg = getRegForValue(Obj);
848 if (!Addr.getOffsetReg()) {
849 unsigned Reg = getRegForValue(Obj);
852 Addr.setOffsetReg(Reg);
859 bool AArch64FastISel::computeCallAddress(
const Value *V,
Address &Addr) {
860 const User *U =
nullptr;
861 unsigned Opcode = Instruction::UserOp1;
864 if (
const auto *
I = dyn_cast<Instruction>(V)) {
865 Opcode =
I->getOpcode();
867 InMBB =
I->getParent() == FuncInfo.MBB->getBasicBlock();
868 }
else if (
const auto *C = dyn_cast<ConstantExpr>(V)) {
869 Opcode = C->getOpcode();
875 case Instruction::BitCast:
878 return computeCallAddress(U->
getOperand(0), Addr);
880 case Instruction::IntToPtr:
884 TLI.getPointerTy(DL))
885 return computeCallAddress(U->
getOperand(0), Addr);
887 case Instruction::PtrToInt:
889 if (InMBB && TLI.getValueType(DL, U->
getType()) == TLI.getPointerTy(DL))
890 return computeCallAddress(U->
getOperand(0), Addr);
894 if (
const GlobalValue *GV = dyn_cast<GlobalValue>(V)) {
895 Addr.setGlobalValue(GV);
900 if (!Addr.getGlobalValue()) {
901 Addr.setReg(getRegForValue(V));
902 return Addr.getReg() != 0;
909 bool AArch64FastISel::isTypeLegal(
Type *Ty,
MVT &VT) {
910 EVT evt = TLI.getValueType(DL, Ty,
true);
923 return TLI.isTypeLegal(VT);
930 bool AArch64FastISel::isTypeSupported(
Type *Ty,
MVT &VT,
bool IsVectorAllowed) {
934 if (isTypeLegal(Ty, VT))
945 bool AArch64FastISel::isValueAvailable(
const Value *V)
const {
946 if (!isa<Instruction>(V))
949 const auto *
I = cast<Instruction>(V);
950 if (FuncInfo.MBBMap[
I->getParent()] == FuncInfo.MBB)
956 bool AArch64FastISel::simplifyAddress(
Address &Addr,
MVT VT) {
961 bool ImmediateOffsetNeedsLowering =
false;
962 bool RegisterOffsetNeedsLowering =
false;
963 int64_t Offset = Addr.getOffset();
964 if (((Offset < 0) || (Offset & (ScaleFactor - 1))) && !isInt<9>(Offset))
965 ImmediateOffsetNeedsLowering =
true;
966 else if (Offset > 0 && !(Offset & (ScaleFactor - 1)) &&
967 !isUInt<12>(Offset / ScaleFactor))
968 ImmediateOffsetNeedsLowering =
true;
973 if (!ImmediateOffsetNeedsLowering && Addr.getOffset() && Addr.getOffsetReg())
974 RegisterOffsetNeedsLowering =
true;
977 if (Addr.isRegBase() && Addr.getOffsetReg() && !Addr.getReg())
978 RegisterOffsetNeedsLowering =
true;
983 if ((ImmediateOffsetNeedsLowering || Addr.getOffsetReg()) && Addr.isFIBase())
985 unsigned ResultReg = createResultReg(&AArch64::GPR64spRegClass);
986 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(AArch64::ADDXri),
988 .addFrameIndex(Addr.getFI())
991 Addr.setKind(Address::RegBase);
992 Addr.setReg(ResultReg);
995 if (RegisterOffsetNeedsLowering) {
996 unsigned ResultReg = 0;
1000 ResultReg = emitAddSub_rx(
true,
MVT::i64, Addr.getReg(),
1001 false, Addr.getOffsetReg(),
1002 false, Addr.getExtendType(),
1005 ResultReg = emitAddSub_rs(
true,
MVT::i64, Addr.getReg(),
1006 false, Addr.getOffsetReg(),
1012 false, Addr.getShift(),
1016 false, Addr.getShift(),
1020 false, Addr.getShift());
1025 Addr.setReg(ResultReg);
1026 Addr.setOffsetReg(0);
1033 if (ImmediateOffsetNeedsLowering) {
1037 ResultReg = emitAdd_ri_(
MVT::i64, Addr.getReg(),
false, Offset);
1043 Addr.setReg(ResultReg);
1049 void AArch64FastISel::addLoadStoreOperands(
Address &Addr,
1052 unsigned ScaleFactor,
1054 int64_t Offset = Addr.getOffset() / ScaleFactor;
1056 if (Addr.isFIBase()) {
1057 int FI = Addr.getFI();
1060 MMO = FuncInfo.MF->getMachineMemOperand(
1062 MFI.getObjectSize(FI), MFI.getObjectAlignment(FI));
1066 assert(Addr.isRegBase() &&
"Unexpected address kind.");
1070 constrainOperandRegClass(II, Addr.getReg(), II.getNumDefs()+Idx));
1072 constrainOperandRegClass(II, Addr.getOffsetReg(), II.getNumDefs()+Idx+1));
1073 if (Addr.getOffsetReg()) {
1074 assert(Addr.getOffset() == 0 &&
"Unexpected offset");
1077 MIB.
addReg(Addr.getReg());
1078 MIB.
addReg(Addr.getOffsetReg());
1080 MIB.
addImm(Addr.getShift() != 0);
1082 MIB.
addReg(Addr.getReg()).addImm(Offset);
1089 unsigned AArch64FastISel::emitAddSub(
bool UseAdd,
MVT RetVT,
const Value *LHS,
1090 const Value *RHS,
bool SetFlags,
1091 bool WantResult,
bool IsZExt) {
1093 bool NeedExtend =
false;
1116 if (UseAdd && isa<Constant>(LHS) && !isa<Constant>(RHS))
1120 if (UseAdd && LHS->
hasOneUse() && isValueAvailable(LHS))
1125 if (UseAdd && LHS->
hasOneUse() && isValueAvailable(LHS))
1126 if (
const auto *SI = dyn_cast<BinaryOperator>(LHS))
1127 if (isa<ConstantInt>(SI->getOperand(1)))
1128 if (SI->getOpcode() == Instruction::Shl ||
1129 SI->getOpcode() == Instruction::LShr ||
1130 SI->getOpcode() == Instruction::AShr )
1133 unsigned LHSReg = getRegForValue(LHS);
1136 bool LHSIsKill = hasTrivialKill(LHS);
1139 LHSReg = emitIntExt(SrcVT, LHSReg, RetVT, IsZExt);
1141 unsigned ResultReg = 0;
1142 if (
const auto *C = dyn_cast<ConstantInt>(RHS)) {
1143 uint64_t Imm = IsZExt ? C->getZExtValue() : C->getSExtValue();
1144 if (C->isNegative())
1145 ResultReg = emitAddSub_ri(!UseAdd, RetVT, LHSReg, LHSIsKill, -Imm,
1146 SetFlags, WantResult);
1148 ResultReg = emitAddSub_ri(UseAdd, RetVT, LHSReg, LHSIsKill, Imm, SetFlags,
1150 }
else if (
const auto *C = dyn_cast<Constant>(RHS))
1152 ResultReg = emitAddSub_ri(UseAdd, RetVT, LHSReg, LHSIsKill, 0, SetFlags,
1160 isValueAvailable(RHS)) {
1161 if (
const auto *SI = dyn_cast<BinaryOperator>(RHS))
1162 if (
const auto *C = dyn_cast<ConstantInt>(SI->getOperand(1)))
1163 if ((SI->getOpcode() == Instruction::Shl) && (C->getZExtValue() < 4)) {
1164 unsigned RHSReg = getRegForValue(SI->getOperand(0));
1167 bool RHSIsKill = hasTrivialKill(SI->getOperand(0));
1168 return emitAddSub_rx(UseAdd, RetVT, LHSReg, LHSIsKill, RHSReg,
1169 RHSIsKill, ExtendType, C->getZExtValue(),
1170 SetFlags, WantResult);
1172 unsigned RHSReg = getRegForValue(RHS);
1175 bool RHSIsKill = hasTrivialKill(RHS);
1176 return emitAddSub_rx(UseAdd, RetVT, LHSReg, LHSIsKill, RHSReg, RHSIsKill,
1177 ExtendType, 0, SetFlags, WantResult);
1181 if (RHS->
hasOneUse() && isValueAvailable(RHS))
1183 const Value *MulLHS = cast<MulOperator>(RHS)->getOperand(0);
1184 const Value *MulRHS = cast<MulOperator>(RHS)->getOperand(1);
1186 if (
const auto *C = dyn_cast<ConstantInt>(MulLHS))
1187 if (C->getValue().isPowerOf2())
1190 assert(isa<ConstantInt>(MulRHS) &&
"Expected a ConstantInt.");
1191 uint64_t ShiftVal = cast<ConstantInt>(MulRHS)->getValue().logBase2();
1192 unsigned RHSReg = getRegForValue(MulLHS);
1195 bool RHSIsKill = hasTrivialKill(MulLHS);
1196 return emitAddSub_rs(UseAdd, RetVT, LHSReg, LHSIsKill, RHSReg, RHSIsKill,
1201 if (RHS->
hasOneUse() && isValueAvailable(RHS))
1202 if (
const auto *SI = dyn_cast<BinaryOperator>(RHS)) {
1203 if (
const auto *C = dyn_cast<ConstantInt>(SI->getOperand(1))) {
1205 switch (SI->getOpcode()) {
1211 uint64_t ShiftVal = C->getZExtValue();
1213 unsigned RHSReg = getRegForValue(SI->getOperand(0));
1216 bool RHSIsKill = hasTrivialKill(SI->getOperand(0));
1217 return emitAddSub_rs(UseAdd, RetVT, LHSReg, LHSIsKill, RHSReg,
1218 RHSIsKill, ShiftType, ShiftVal, SetFlags,
1224 unsigned RHSReg = getRegForValue(RHS);
1227 bool RHSIsKill = hasTrivialKill(RHS);
1230 RHSReg = emitIntExt(SrcVT, RHSReg, RetVT, IsZExt);
1232 return emitAddSub_rr(UseAdd, RetVT, LHSReg, LHSIsKill, RHSReg, RHSIsKill,
1233 SetFlags, WantResult);
1236 unsigned AArch64FastISel::emitAddSub_rr(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
1237 bool LHSIsKill,
unsigned RHSReg,
1238 bool RHSIsKill,
bool SetFlags,
1240 assert(LHSReg && RHSReg &&
"Invalid register number.");
1245 static const unsigned OpcTable[2][2][2] = {
1246 { { AArch64::SUBWrr, AArch64::SUBXrr },
1247 { AArch64::ADDWrr, AArch64::ADDXrr } },
1248 { { AArch64::SUBSWrr, AArch64::SUBSXrr },
1249 { AArch64::ADDSWrr, AArch64::ADDSXrr } }
1252 unsigned Opc = OpcTable[SetFlags][UseAdd][Is64Bit];
1254 Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
1257 ResultReg = createResultReg(RC);
1259 ResultReg = Is64Bit ? AArch64::XZR : AArch64::WZR;
1262 LHSReg = constrainOperandRegClass(II, LHSReg, II.
getNumDefs());
1263 RHSReg = constrainOperandRegClass(II, RHSReg, II.
getNumDefs() + 1);
1264 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
1270 unsigned AArch64FastISel::emitAddSub_ri(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
1271 bool LHSIsKill, uint64_t Imm,
1272 bool SetFlags,
bool WantResult) {
1273 assert(LHSReg &&
"Invalid register number.");
1279 if (isUInt<12>(Imm))
1281 else if ((Imm & 0xfff000) == Imm) {
1287 static const unsigned OpcTable[2][2][2] = {
1288 { { AArch64::SUBWri, AArch64::SUBXri },
1289 { AArch64::ADDWri, AArch64::ADDXri } },
1290 { { AArch64::SUBSWri, AArch64::SUBSXri },
1291 { AArch64::ADDSWri, AArch64::ADDSXri } }
1294 unsigned Opc = OpcTable[SetFlags][UseAdd][Is64Bit];
1297 RC = Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
1299 RC = Is64Bit ? &AArch64::GPR64spRegClass : &AArch64::GPR32spRegClass;
1302 ResultReg = createResultReg(RC);
1304 ResultReg = Is64Bit ? AArch64::XZR : AArch64::WZR;
1307 LHSReg = constrainOperandRegClass(II, LHSReg, II.
getNumDefs());
1308 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
1315 unsigned AArch64FastISel::emitAddSub_rs(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
1316 bool LHSIsKill,
unsigned RHSReg,
1319 uint64_t ShiftImm,
bool SetFlags,
1321 assert(LHSReg && RHSReg &&
"Invalid register number.");
1326 static const unsigned OpcTable[2][2][2] = {
1327 { { AArch64::SUBWrs, AArch64::SUBXrs },
1328 { AArch64::ADDWrs, AArch64::ADDXrs } },
1329 { { AArch64::SUBSWrs, AArch64::SUBSXrs },
1330 { AArch64::ADDSWrs, AArch64::ADDSXrs } }
1333 unsigned Opc = OpcTable[SetFlags][UseAdd][Is64Bit];
1335 Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
1338 ResultReg = createResultReg(RC);
1340 ResultReg = Is64Bit ? AArch64::XZR : AArch64::WZR;
1343 LHSReg = constrainOperandRegClass(II, LHSReg, II.
getNumDefs());
1344 RHSReg = constrainOperandRegClass(II, RHSReg, II.
getNumDefs() + 1);
1345 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
1352 unsigned AArch64FastISel::emitAddSub_rx(
bool UseAdd,
MVT RetVT,
unsigned LHSReg,
1353 bool LHSIsKill,
unsigned RHSReg,
1356 uint64_t ShiftImm,
bool SetFlags,
1358 assert(LHSReg && RHSReg &&
"Invalid register number.");
1363 static const unsigned OpcTable[2][2][2] = {
1364 { { AArch64::SUBWrx, AArch64::SUBXrx },
1365 { AArch64::ADDWrx, AArch64::ADDXrx } },
1366 { { AArch64::SUBSWrx, AArch64::SUBSXrx },
1367 { AArch64::ADDSWrx, AArch64::ADDSXrx } }
1370 unsigned Opc = OpcTable[SetFlags][UseAdd][Is64Bit];
1373 RC = Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
1375 RC = Is64Bit ? &AArch64::GPR64spRegClass : &AArch64::GPR32spRegClass;
1378 ResultReg = createResultReg(RC);
1380 ResultReg = Is64Bit ? AArch64::XZR : AArch64::WZR;
1383 LHSReg = constrainOperandRegClass(II, LHSReg, II.
getNumDefs());
1384 RHSReg = constrainOperandRegClass(II, RHSReg, II.
getNumDefs() + 1);
1385 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
1394 EVT EVT = TLI.getValueType(DL, Ty,
true);
1407 return emitICmp(VT, LHS, RHS, IsZExt);
1410 return emitFCmp(VT, LHS, RHS);
1414 bool AArch64FastISel::emitICmp(
MVT RetVT,
const Value *LHS,
const Value *RHS,
1416 return emitSub(RetVT, LHS, RHS,
true,
false,
1420 bool AArch64FastISel::emitICmp_ri(
MVT RetVT,
unsigned LHSReg,
bool LHSIsKill,
1422 return emitAddSub_ri(
false, RetVT, LHSReg, LHSIsKill, Imm,
1426 bool AArch64FastISel::emitFCmp(
MVT RetVT,
const Value *LHS,
const Value *RHS) {
1432 bool UseImm =
false;
1433 if (
const auto *CFP = dyn_cast<ConstantFP>(RHS))
1437 unsigned LHSReg = getRegForValue(LHS);
1440 bool LHSIsKill = hasTrivialKill(LHS);
1443 unsigned Opc = (RetVT ==
MVT::f64) ? AArch64::FCMPDri : AArch64::FCMPSri;
1444 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(Opc))
1449 unsigned RHSReg = getRegForValue(RHS);
1452 bool RHSIsKill = hasTrivialKill(RHS);
1454 unsigned Opc = (RetVT ==
MVT::f64) ? AArch64::FCMPDrr : AArch64::FCMPSrr;
1455 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(Opc))
1461 unsigned AArch64FastISel::emitAdd(
MVT RetVT,
const Value *LHS,
const Value *RHS,
1462 bool SetFlags,
bool WantResult,
bool IsZExt) {
1463 return emitAddSub(
true, RetVT, LHS, RHS, SetFlags, WantResult,
1472 unsigned AArch64FastISel::emitAdd_ri_(
MVT VT,
unsigned Op0,
bool Op0IsKill,
1476 ResultReg = emitAddSub_ri(
false, VT, Op0, Op0IsKill, -Imm);
1478 ResultReg = emitAddSub_ri(
true, VT, Op0, Op0IsKill, Imm);
1487 ResultReg = emitAddSub_rr(
true, VT, Op0, Op0IsKill, CReg,
true);
1491 unsigned AArch64FastISel::emitSub(
MVT RetVT,
const Value *LHS,
const Value *RHS,
1492 bool SetFlags,
bool WantResult,
bool IsZExt) {
1493 return emitAddSub(
false, RetVT, LHS, RHS, SetFlags, WantResult,
1497 unsigned AArch64FastISel::emitSubs_rr(
MVT RetVT,
unsigned LHSReg,
1498 bool LHSIsKill,
unsigned RHSReg,
1499 bool RHSIsKill,
bool WantResult) {
1500 return emitAddSub_rr(
false, RetVT, LHSReg, LHSIsKill, RHSReg,
1501 RHSIsKill,
true, WantResult);
1504 unsigned AArch64FastISel::emitSubs_rs(
MVT RetVT,
unsigned LHSReg,
1505 bool LHSIsKill,
unsigned RHSReg,
1508 uint64_t ShiftImm,
bool WantResult) {
1509 return emitAddSub_rs(
false, RetVT, LHSReg, LHSIsKill, RHSReg,
1510 RHSIsKill, ShiftType, ShiftImm,
true,
1514 unsigned AArch64FastISel::emitLogicalOp(
unsigned ISDOpc,
MVT RetVT,
1517 if (isa<ConstantInt>(LHS) && !isa<ConstantInt>(RHS))
1521 if (LHS->
hasOneUse() && isValueAvailable(LHS))
1526 if (LHS->
hasOneUse() && isValueAvailable(LHS))
1527 if (
const auto *SI = dyn_cast<ShlOperator>(LHS))
1528 if (isa<ConstantInt>(SI->getOperand(1)))
1531 unsigned LHSReg = getRegForValue(LHS);
1534 bool LHSIsKill = hasTrivialKill(LHS);
1536 unsigned ResultReg = 0;
1537 if (
const auto *C = dyn_cast<ConstantInt>(RHS)) {
1538 uint64_t Imm = C->getZExtValue();
1539 ResultReg = emitLogicalOp_ri(ISDOpc, RetVT, LHSReg, LHSIsKill, Imm);
1545 if (RHS->
hasOneUse() && isValueAvailable(RHS))
1547 const Value *MulLHS = cast<MulOperator>(RHS)->getOperand(0);
1548 const Value *MulRHS = cast<MulOperator>(RHS)->getOperand(1);
1550 if (
const auto *C = dyn_cast<ConstantInt>(MulLHS))
1551 if (C->getValue().isPowerOf2())
1554 assert(isa<ConstantInt>(MulRHS) &&
"Expected a ConstantInt.");
1555 uint64_t ShiftVal = cast<ConstantInt>(MulRHS)->getValue().logBase2();
1557 unsigned RHSReg = getRegForValue(MulLHS);
1560 bool RHSIsKill = hasTrivialKill(MulLHS);
1561 return emitLogicalOp_rs(ISDOpc, RetVT, LHSReg, LHSIsKill, RHSReg,
1562 RHSIsKill, ShiftVal);
1566 if (RHS->
hasOneUse() && isValueAvailable(RHS))
1567 if (
const auto *SI = dyn_cast<ShlOperator>(RHS))
1568 if (
const auto *C = dyn_cast<ConstantInt>(SI->getOperand(1))) {
1569 uint64_t ShiftVal = C->getZExtValue();
1570 unsigned RHSReg = getRegForValue(SI->getOperand(0));
1573 bool RHSIsKill = hasTrivialKill(SI->getOperand(0));
1574 return emitLogicalOp_rs(ISDOpc, RetVT, LHSReg, LHSIsKill, RHSReg,
1575 RHSIsKill, ShiftVal);
1578 unsigned RHSReg = getRegForValue(RHS);
1581 bool RHSIsKill = hasTrivialKill(RHS);
1584 ResultReg = fastEmit_rr(VT, VT, ISDOpc, LHSReg, LHSIsKill, RHSReg, RHSIsKill);
1586 uint64_t Mask = (RetVT ==
MVT::i8) ? 0xff : 0xffff;
1587 ResultReg = emitAnd_ri(
MVT::i32, ResultReg,
true, Mask);
1592 unsigned AArch64FastISel::emitLogicalOp_ri(
unsigned ISDOpc,
MVT RetVT,
1593 unsigned LHSReg,
bool LHSIsKill,
1596 "ISD nodes are not consecutive!");
1597 static const unsigned OpcTable[3][2] = {
1598 { AArch64::ANDWri, AArch64::ANDXri },
1599 { AArch64::ORRWri, AArch64::ORRXri },
1600 { AArch64::EORWri, AArch64::EORXri }
1613 Opc = OpcTable[Idx][0];
1614 RC = &AArch64::GPR32spRegClass;
1619 Opc = OpcTable[ISDOpc -
ISD::AND][1];
1620 RC = &AArch64::GPR64spRegClass;
1628 unsigned ResultReg =
1629 fastEmitInst_ri(Opc, RC, LHSReg, LHSIsKill,
1632 uint64_t Mask = (RetVT ==
MVT::i8) ? 0xff : 0xffff;
1633 ResultReg = emitAnd_ri(
MVT::i32, ResultReg,
true, Mask);
1638 unsigned AArch64FastISel::emitLogicalOp_rs(
unsigned ISDOpc,
MVT RetVT,
1639 unsigned LHSReg,
bool LHSIsKill,
1640 unsigned RHSReg,
bool RHSIsKill,
1641 uint64_t ShiftImm) {
1643 "ISD nodes are not consecutive!");
1644 static const unsigned OpcTable[3][2] = {
1645 { AArch64::ANDWrs, AArch64::ANDXrs },
1646 { AArch64::ORRWrs, AArch64::ORRXrs },
1647 { AArch64::EORWrs, AArch64::EORXrs }
1658 Opc = OpcTable[ISDOpc -
ISD::AND][0];
1659 RC = &AArch64::GPR32RegClass;
1662 Opc = OpcTable[ISDOpc -
ISD::AND][1];
1663 RC = &AArch64::GPR64RegClass;
1666 unsigned ResultReg =
1667 fastEmitInst_rri(Opc, RC, LHSReg, LHSIsKill, RHSReg, RHSIsKill,
1670 uint64_t Mask = (RetVT ==
MVT::i8) ? 0xff : 0xffff;
1671 ResultReg = emitAnd_ri(
MVT::i32, ResultReg,
true, Mask);
1676 unsigned AArch64FastISel::emitAnd_ri(
MVT RetVT,
unsigned LHSReg,
bool LHSIsKill,
1678 return emitLogicalOp_ri(
ISD::AND, RetVT, LHSReg, LHSIsKill, Imm);
1681 unsigned AArch64FastISel::emitLoad(
MVT VT,
MVT RetVT,
Address Addr,
1683 if (!TLI.allowsMisalignedMemoryAccesses(VT))
1687 if (!simplifyAddress(Addr, VT))
1696 bool UseScaled =
true;
1697 if ((Addr.getOffset() < 0) || (Addr.getOffset() & (ScaleFactor - 1))) {
1702 static const unsigned GPOpcTable[2][8][4] = {
1704 { { AArch64::LDURSBWi, AArch64::LDURSHWi, AArch64::LDURWi,
1706 { AArch64::LDURSBXi, AArch64::LDURSHXi, AArch64::LDURSWi,
1708 { AArch64::LDRSBWui, AArch64::LDRSHWui, AArch64::LDRWui,
1710 { AArch64::LDRSBXui, AArch64::LDRSHXui, AArch64::LDRSWui,
1712 { AArch64::LDRSBWroX, AArch64::LDRSHWroX, AArch64::LDRWroX,
1714 { AArch64::LDRSBXroX, AArch64::LDRSHXroX, AArch64::LDRSWroX,
1716 { AArch64::LDRSBWroW, AArch64::LDRSHWroW, AArch64::LDRWroW,
1718 { AArch64::LDRSBXroW, AArch64::LDRSHXroW, AArch64::LDRSWroW,
1722 { { AArch64::LDURBBi, AArch64::LDURHHi, AArch64::LDURWi,
1724 { AArch64::LDURBBi, AArch64::LDURHHi, AArch64::LDURWi,
1726 { AArch64::LDRBBui, AArch64::LDRHHui, AArch64::LDRWui,
1728 { AArch64::LDRBBui, AArch64::LDRHHui, AArch64::LDRWui,
1730 { AArch64::LDRBBroX, AArch64::LDRHHroX, AArch64::LDRWroX,
1732 { AArch64::LDRBBroX, AArch64::LDRHHroX, AArch64::LDRWroX,
1734 { AArch64::LDRBBroW, AArch64::LDRHHroW, AArch64::LDRWroW,
1736 { AArch64::LDRBBroW, AArch64::LDRHHroW, AArch64::LDRWroW,
1741 static const unsigned FPOpcTable[4][2] = {
1742 { AArch64::LDURSi, AArch64::LDURDi },
1743 { AArch64::LDRSui, AArch64::LDRDui },
1744 { AArch64::LDRSroX, AArch64::LDRDroX },
1745 { AArch64::LDRSroW, AArch64::LDRDroW }
1750 bool UseRegOffset = Addr.isRegBase() && !Addr.getOffset() && Addr.getReg() &&
1751 Addr.getOffsetReg();
1752 unsigned Idx = UseRegOffset ? 2 : UseScaled ? 1 : 0;
1757 bool IsRet64Bit = RetVT ==
MVT::i64;
1763 Opc = GPOpcTable[WantZExt][2 * Idx + IsRet64Bit][0];
1764 RC = (IsRet64Bit && !WantZExt) ?
1765 &AArch64::GPR64RegClass: &AArch64::GPR32RegClass;
1768 Opc = GPOpcTable[WantZExt][2 * Idx + IsRet64Bit][1];
1769 RC = (IsRet64Bit && !WantZExt) ?
1770 &AArch64::GPR64RegClass: &AArch64::GPR32RegClass;
1773 Opc = GPOpcTable[WantZExt][2 * Idx + IsRet64Bit][2];
1774 RC = (IsRet64Bit && !WantZExt) ?
1775 &AArch64::GPR64RegClass: &AArch64::GPR32RegClass;
1778 Opc = GPOpcTable[WantZExt][2 * Idx + IsRet64Bit][3];
1779 RC = &AArch64::GPR64RegClass;
1782 Opc = FPOpcTable[Idx][0];
1783 RC = &AArch64::FPR32RegClass;
1786 Opc = FPOpcTable[Idx][1];
1787 RC = &AArch64::FPR64RegClass;
1792 unsigned ResultReg = createResultReg(RC);
1794 TII.get(Opc), ResultReg);
1799 unsigned ANDReg = emitAnd_ri(
MVT::i32, ResultReg,
true, 1);
1800 assert(ANDReg &&
"Unexpected AND instruction emission failure.");
1807 unsigned Reg64 = createResultReg(&AArch64::GPR64RegClass);
1808 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1812 .
addImm(AArch64::sub_32);
1818 bool AArch64FastISel::selectAddSub(
const Instruction *
I) {
1820 if (!isTypeSupported(I->
getType(), VT,
true))
1824 return selectOperator(I, I->
getOpcode());
1830 case Instruction::Add:
1833 case Instruction::Sub:
1840 updateValueMap(I, ResultReg);
1844 bool AArch64FastISel::selectLogicalOp(
const Instruction *I) {
1846 if (!isTypeSupported(I->
getType(), VT,
true))
1850 return selectOperator(I, I->
getOpcode());
1869 updateValueMap(I, ResultReg);
1873 bool AArch64FastISel::selectLoad(
const Instruction *I) {
1878 if (!isTypeSupported(I->
getType(), VT,
true) ||
1879 cast<LoadInst>(I)->isAtomic())
1888 bool WantZExt =
true;
1890 const Value *IntExtVal =
nullptr;
1892 if (
const auto *ZE = dyn_cast<ZExtInst>(I->
use_begin()->getUser())) {
1893 if (isTypeSupported(ZE->getType(), RetVT))
1897 }
else if (
const auto *SE = dyn_cast<SExtInst>(I->
use_begin()->getUser())) {
1898 if (isTypeSupported(SE->getType(), RetVT))
1906 unsigned ResultReg =
1907 emitLoad(VT, RetVT, Addr, WantZExt, createMachineMemOperandFor(I));
1927 unsigned Reg = lookUpRegForValue(IntExtVal);
1928 auto *
MI = MRI.getUniqueVRegDef(Reg);
1933 std::prev(FuncInfo.InsertPt)->eraseFromParent();
1934 ResultReg = std::prev(FuncInfo.InsertPt)->getOperand(0).getReg();
1936 ResultReg = fastEmitInst_extractsubreg(
MVT::i32, ResultReg,
1940 updateValueMap(I, ResultReg);
1949 for (
auto &Opnd :
MI->uses()) {
1951 Reg = Opnd.getReg();
1955 MI->eraseFromParent();
1958 MI = MRI.getUniqueVRegDef(Reg);
1960 updateValueMap(IntExtVal, ResultReg);
1964 updateValueMap(I, ResultReg);
1968 bool AArch64FastISel::emitStore(
MVT VT,
unsigned SrcReg,
Address Addr,
1970 if (!TLI.allowsMisalignedMemoryAccesses(VT))
1974 if (!simplifyAddress(Addr, VT))
1983 bool UseScaled =
true;
1984 if ((Addr.getOffset() < 0) || (Addr.getOffset() & (ScaleFactor - 1))) {
1989 static const unsigned OpcTable[4][6] = {
1990 { AArch64::STURBBi, AArch64::STURHHi, AArch64::STURWi, AArch64::STURXi,
1991 AArch64::STURSi, AArch64::STURDi },
1992 { AArch64::STRBBui, AArch64::STRHHui, AArch64::STRWui, AArch64::STRXui,
1993 AArch64::STRSui, AArch64::STRDui },
1994 { AArch64::STRBBroX, AArch64::STRHHroX, AArch64::STRWroX, AArch64::STRXroX,
1995 AArch64::STRSroX, AArch64::STRDroX },
1996 { AArch64::STRBBroW, AArch64::STRHHroW, AArch64::STRWroW, AArch64::STRXroW,
1997 AArch64::STRSroW, AArch64::STRDroW }
2001 bool VTIsi1 =
false;
2002 bool UseRegOffset = Addr.isRegBase() && !Addr.getOffset() && Addr.getReg() &&
2003 Addr.getOffsetReg();
2004 unsigned Idx = UseRegOffset ? 2 : UseScaled ? 1 : 0;
2012 case MVT::i8: Opc = OpcTable[Idx][0];
break;
2013 case MVT::i16: Opc = OpcTable[Idx][1];
break;
2014 case MVT::i32: Opc = OpcTable[Idx][2];
break;
2015 case MVT::i64: Opc = OpcTable[Idx][3];
break;
2016 case MVT::f32: Opc = OpcTable[Idx][4];
break;
2017 case MVT::f64: Opc = OpcTable[Idx][5];
break;
2021 if (VTIsi1 && SrcReg != AArch64::WZR) {
2022 unsigned ANDReg = emitAnd_ri(
MVT::i32, SrcReg,
false, 1);
2023 assert(ANDReg &&
"Unexpected AND instruction emission failure.");
2028 SrcReg = constrainOperandRegClass(II, SrcReg, II.
getNumDefs());
2030 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II).
addReg(SrcReg);
2036 bool AArch64FastISel::selectStore(
const Instruction *I) {
2042 if (!isTypeSupported(Op0->
getType(), VT,
true) ||
2043 cast<StoreInst>(I)->isAtomic())
2048 unsigned SrcReg = 0;
2049 if (
const auto *CI = dyn_cast<ConstantInt>(Op0)) {
2051 SrcReg = (VT ==
MVT::i64) ? AArch64::XZR : AArch64::WZR;
2052 }
else if (
const auto *CF = dyn_cast<ConstantFP>(Op0)) {
2053 if (CF->isZero() && !CF->isNegative()) {
2055 SrcReg = (VT ==
MVT::i64) ? AArch64::XZR : AArch64::WZR;
2060 SrcReg = getRegForValue(Op0);
2070 if (!emitStore(VT, SrcReg, Addr, createMachineMemOperandFor(I)))
2122 bool AArch64FastISel::emitCompareAndBranch(
const BranchInst *BI) {
2123 assert(isa<CmpInst>(BI->
getCondition()) &&
"Expected cmp instruction");
2131 if (!isTypeSupported(LHS->
getType(), VT))
2142 if (FuncInfo.MBB->isLayoutSuccessor(TBB)) {
2149 switch (Predicate) {
2154 if (isa<Constant>(LHS) && cast<Constant>(LHS)->isNullValue())
2157 if (!isa<Constant>(RHS) || !cast<Constant>(RHS)->isNullValue())
2160 if (
const auto *AI = dyn_cast<BinaryOperator>(LHS))
2165 if (
const auto *C = dyn_cast<ConstantInt>(AndLHS))
2166 if (C->getValue().isPowerOf2())
2169 if (
const auto *C = dyn_cast<ConstantInt>(AndRHS))
2170 if (C->getValue().isPowerOf2()) {
2171 TestBit = C->getValue().logBase2();
2183 if (!isa<Constant>(RHS) || !cast<Constant>(RHS)->isNullValue())
2191 if (!isa<ConstantInt>(RHS))
2194 if (cast<ConstantInt>(RHS)->getValue() !=
APInt(BW, -1,
true))
2202 static const unsigned OpcTable[2][2][2] = {
2203 { {AArch64::CBZW, AArch64::CBZX },
2204 {AArch64::CBNZW, AArch64::CBNZX} },
2205 { {AArch64::TBZW, AArch64::TBZX },
2206 {AArch64::TBNZW, AArch64::TBNZX} }
2209 bool IsBitTest = TestBit != -1;
2210 bool Is64Bit = BW == 64;
2211 if (TestBit < 32 && TestBit >= 0)
2214 unsigned Opc = OpcTable[IsBitTest][IsCmpNE][Is64Bit];
2217 unsigned SrcReg = getRegForValue(LHS);
2220 bool SrcIsKill = hasTrivialKill(LHS);
2222 if (BW == 64 && !Is64Bit)
2223 SrcReg = fastEmitInst_extractsubreg(
MVT::i32, SrcReg, SrcIsKill,
2226 if ((BW < 32) && !IsBitTest)
2227 SrcReg = emitIntExt(VT, SrcReg,
MVT::i32,
true);
2230 SrcReg = constrainOperandRegClass(II, SrcReg, II.
getNumDefs());
2232 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(Opc))
2239 uint32_t BranchWeight = 0;
2241 BranchWeight = FuncInfo.BPI->getEdgeWeight(BI->
getParent(),
2243 FuncInfo.MBB->addSuccessor(TBB, BranchWeight);
2244 fastEmitBranch(FBB, DbgLoc);
2249 bool AArch64FastISel::selectBranch(
const Instruction *I) {
2262 if (CI->
hasOneUse() && isValueAvailable(CI)) {
2265 switch (Predicate) {
2269 fastEmitBranch(FBB, DbgLoc);
2272 fastEmitBranch(TBB, DbgLoc);
2277 if (emitCompareAndBranch(BI))
2281 if (FuncInfo.MBB->isLayoutSuccessor(TBB)) {
2294 switch (Predicate) {
2306 assert((CC !=
AArch64CC::AL) &&
"Unexpected condition code.");
2310 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(AArch64::Bcc))
2316 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(AArch64::Bcc))
2321 uint32_t BranchWeight = 0;
2323 BranchWeight = FuncInfo.BPI->getEdgeWeight(BI->
getParent(),
2325 FuncInfo.MBB->addSuccessor(TBB, BranchWeight);
2327 fastEmitBranch(FBB, DbgLoc);
2332 if (TI->hasOneUse() && isValueAvailable(TI) &&
2333 isTypeSupported(TI->getOperand(0)->getType(), SrcVT)) {
2334 unsigned CondReg = getRegForValue(TI->getOperand(0));
2337 bool CondIsKill = hasTrivialKill(TI->getOperand(0));
2341 CondReg = fastEmitInst_extractsubreg(
MVT::i32, CondReg, CondIsKill,
2346 unsigned ANDReg = emitAnd_ri(
MVT::i32, CondReg, CondIsKill, 1);
2347 assert(ANDReg &&
"Unexpected AND instruction emission failure.");
2348 emitICmp_ri(
MVT::i32, ANDReg,
true, 0);
2350 if (FuncInfo.MBB->isLayoutSuccessor(TBB)) {
2354 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(AArch64::Bcc))
2359 uint32_t BranchWeight = 0;
2361 BranchWeight = FuncInfo.BPI->getEdgeWeight(BI->
getParent(),
2363 FuncInfo.MBB->addSuccessor(TBB, BranchWeight);
2365 fastEmitBranch(FBB, DbgLoc);
2368 }
else if (
const auto *CI = dyn_cast<ConstantInt>(BI->
getCondition())) {
2369 uint64_t Imm = CI->getZExtValue();
2371 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(AArch64::B))
2375 uint32_t BranchWeight = 0;
2377 BranchWeight = FuncInfo.BPI->getEdgeWeight(BI->
getParent(),
2379 FuncInfo.MBB->addSuccessor(Target, BranchWeight);
2381 }
else if (foldXALUIntrinsic(CC, I, BI->
getCondition())) {
2389 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(AArch64::Bcc))
2394 uint32_t BranchWeight = 0;
2396 BranchWeight = FuncInfo.BPI->getEdgeWeight(BI->
getParent(),
2398 FuncInfo.MBB->addSuccessor(TBB, BranchWeight);
2400 fastEmitBranch(FBB, DbgLoc);
2407 bool CondRegIsKill = hasTrivialKill(BI->
getCondition());
2416 emitICmp_ri(
MVT::i32, CondReg, CondRegIsKill, 0);
2418 if (FuncInfo.MBB->isLayoutSuccessor(TBB)) {
2423 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(AArch64::Bcc))
2428 uint32_t BranchWeight = 0;
2430 BranchWeight = FuncInfo.BPI->getEdgeWeight(BI->
getParent(),
2432 FuncInfo.MBB->addSuccessor(TBB, BranchWeight);
2434 fastEmitBranch(FBB, DbgLoc);
2438 bool AArch64FastISel::selectIndirectBr(
const Instruction *I) {
2440 unsigned AddrReg = getRegForValue(BI->
getOperand(0));
2446 AddrReg = constrainOperandRegClass(II, AddrReg, II.
getNumDefs());
2447 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II).
addReg(AddrReg);
2451 FuncInfo.MBB->addSuccessor(FuncInfo.MBBMap[BI->
getSuccessor(i)]);
2456 bool AArch64FastISel::selectCmp(
const Instruction *I) {
2457 const CmpInst *CI = cast<CmpInst>(
I);
2461 unsigned ResultReg = 0;
2462 switch (Predicate) {
2466 ResultReg = createResultReg(&AArch64::GPR32RegClass);
2467 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
2477 updateValueMap(I, ResultReg);
2485 ResultReg = createResultReg(&AArch64::GPR32RegClass);
2489 static unsigned CondCodeTable[2][2] = {
2494 switch (Predicate) {
2498 CondCodes = &CondCodeTable[0][0];
2501 CondCodes = &CondCodeTable[1][0];
2506 unsigned TmpReg1 = createResultReg(&AArch64::GPR32RegClass);
2507 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(AArch64::CSINCWr),
2511 .addImm(CondCodes[0]);
2512 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(AArch64::CSINCWr),
2516 .addImm(CondCodes[1]);
2518 updateValueMap(I, ResultReg);
2524 assert((CC !=
AArch64CC::AL) &&
"Unexpected condition code.");
2526 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(AArch64::CSINCWr),
2530 .addImm(invertedCC);
2532 updateValueMap(I, ResultReg);
2538 bool AArch64FastISel::optimizeSelect(
const SelectInst *SI) {
2542 const Value *Src1Val, *Src2Val;
2544 bool NeedExtraOp =
false;
2545 if (
auto *CI = dyn_cast<ConstantInt>(SI->
getTrueValue())) {
2549 Opc = AArch64::ORRWrr;
2551 assert(CI->isZero());
2554 Opc = AArch64::BICWrr;
2556 }
else if (
auto *CI = dyn_cast<ConstantInt>(SI->
getFalseValue())) {
2560 Opc = AArch64::ORRWrr;
2563 assert(CI->isZero());
2566 Opc = AArch64::ANDWrr;
2573 unsigned Src1Reg = getRegForValue(Src1Val);
2576 bool Src1IsKill = hasTrivialKill(Src1Val);
2578 unsigned Src2Reg = getRegForValue(Src2Val);
2581 bool Src2IsKill = hasTrivialKill(Src2Val);
2587 unsigned ResultReg = fastEmitInst_rr(Opc, &AArch64::GPR32RegClass, Src1Reg,
2588 Src1IsKill, Src2Reg, Src2IsKill);
2589 updateValueMap(SI, ResultReg);
2593 bool AArch64FastISel::selectSelect(
const Instruction *I) {
2594 assert(isa<SelectInst>(I) &&
"Expected a select instruction.");
2596 if (!isTypeSupported(I->
getType(), VT))
2608 Opc = AArch64::CSELWr;
2609 RC = &AArch64::GPR32RegClass;
2612 Opc = AArch64::CSELXr;
2613 RC = &AArch64::GPR64RegClass;
2616 Opc = AArch64::FCSELSrrr;
2617 RC = &AArch64::FPR32RegClass;
2620 Opc = AArch64::FCSELDrrr;
2621 RC = &AArch64::FPR64RegClass;
2630 if (optimizeSelect(SI))
2634 if (foldXALUIntrinsic(CC, I, Cond)) {
2636 unsigned CondReg = getRegForValue(Cond);
2639 }
else if (isa<CmpInst>(Cond) && cast<CmpInst>(Cond)->hasOneUse() &&
2640 isValueAvailable(Cond)) {
2641 const auto *Cmp = cast<CmpInst>(Cond);
2644 const Value *FoldSelect =
nullptr;
2645 switch (Predicate) {
2657 unsigned SrcReg = getRegForValue(FoldSelect);
2660 unsigned UseReg = lookUpRegForValue(SI);
2662 MRI.clearKillFlags(UseReg);
2664 updateValueMap(I, SrcReg);
2669 if (!
emitCmp(Cmp->getOperand(0), Cmp->getOperand(1), Cmp->isUnsigned()))
2674 switch (Predicate) {
2686 assert((CC !=
AArch64CC::AL) &&
"Unexpected condition code.");
2688 unsigned CondReg = getRegForValue(Cond);
2691 bool CondIsKill = hasTrivialKill(Cond);
2694 CondReg = constrainOperandRegClass(II, CondReg, 1);
2697 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II,
2709 if (!Src1Reg || !Src2Reg)
2713 Src2Reg = fastEmitInst_rri(Opc, RC, Src1Reg, Src1IsKill, Src2Reg,
2714 Src2IsKill, ExtraCC);
2717 unsigned ResultReg = fastEmitInst_rri(Opc, RC, Src1Reg, Src1IsKill, Src2Reg,
2719 updateValueMap(I, ResultReg);
2723 bool AArch64FastISel::selectFPExt(
const Instruction *I) {
2728 unsigned Op = getRegForValue(V);
2732 unsigned ResultReg = createResultReg(&AArch64::FPR64RegClass);
2733 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(AArch64::FCVTDSr),
2734 ResultReg).addReg(Op);
2735 updateValueMap(I, ResultReg);
2739 bool AArch64FastISel::selectFPTrunc(
const Instruction *I) {
2744 unsigned Op = getRegForValue(V);
2748 unsigned ResultReg = createResultReg(&AArch64::FPR32RegClass);
2749 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(AArch64::FCVTSDr),
2750 ResultReg).addReg(Op);
2751 updateValueMap(I, ResultReg);
2761 unsigned SrcReg = getRegForValue(I->
getOperand(0));
2772 Opc = (DestVT ==
MVT::i32) ? AArch64::FCVTZSUWDr : AArch64::FCVTZSUXDr;
2774 Opc = (DestVT ==
MVT::i32) ? AArch64::FCVTZUUWDr : AArch64::FCVTZUUXDr;
2777 Opc = (DestVT ==
MVT::i32) ? AArch64::FCVTZSUWSr : AArch64::FCVTZSUXSr;
2779 Opc = (DestVT ==
MVT::i32) ? AArch64::FCVTZUUWSr : AArch64::FCVTZUUXSr;
2781 unsigned ResultReg = createResultReg(
2782 DestVT ==
MVT::i32 ? &AArch64::GPR32RegClass : &AArch64::GPR64RegClass);
2783 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(Opc), ResultReg)
2785 updateValueMap(I, ResultReg);
2789 bool AArch64FastISel::selectIntToFP(
const Instruction *I,
bool Signed) {
2794 "Unexpected value type.");
2796 unsigned SrcReg = getRegForValue(I->
getOperand(0));
2799 bool SrcIsKill = hasTrivialKill(I->
getOperand(0));
2815 Opc = (DestVT ==
MVT::f32) ? AArch64::SCVTFUXSri : AArch64::SCVTFUXDri;
2817 Opc = (DestVT ==
MVT::f32) ? AArch64::UCVTFUXSri : AArch64::UCVTFUXDri;
2820 Opc = (DestVT ==
MVT::f32) ? AArch64::SCVTFUWSri : AArch64::SCVTFUWDri;
2822 Opc = (DestVT ==
MVT::f32) ? AArch64::UCVTFUWSri : AArch64::UCVTFUWDri;
2825 unsigned ResultReg = fastEmitInst_r(Opc, TLI.getRegClassFor(DestVT), SrcReg,
2827 updateValueMap(I, ResultReg);
2831 bool AArch64FastISel::fastLowerArguments() {
2832 if (!FuncInfo.CanLowerReturn)
2844 unsigned GPRCnt = 0;
2845 unsigned FPRCnt = 0;
2847 for (
auto const &Arg : F->
args()) {
2856 Type *ArgTy = Arg.getType();
2860 EVT ArgVT = TLI.getValueType(DL, ArgTy);
2869 (!Subtarget->hasNEON() || !Subtarget->isLittleEndian()))
2880 if (GPRCnt > 8 || FPRCnt > 8)
2884 static const MCPhysReg Registers[6][8] = {
2885 { AArch64::W0, AArch64::W1, AArch64::W2, AArch64::W3, AArch64::W4,
2886 AArch64::W5, AArch64::W6, AArch64::W7 },
2887 { AArch64::X0, AArch64::X1, AArch64::X2, AArch64::X3, AArch64::X4,
2888 AArch64::X5, AArch64::X6, AArch64::X7 },
2889 { AArch64::H0, AArch64::H1, AArch64::H2, AArch64::H3, AArch64::H4,
2890 AArch64::H5, AArch64::H6, AArch64::H7 },
2891 { AArch64::S0, AArch64::S1, AArch64::S2, AArch64::S3, AArch64::S4,
2892 AArch64::S5, AArch64::S6, AArch64::S7 },
2893 { AArch64::D0, AArch64::D1, AArch64::D2, AArch64::D3, AArch64::D4,
2894 AArch64::D5, AArch64::D6, AArch64::D7 },
2895 { AArch64::Q0, AArch64::Q1, AArch64::Q2, AArch64::Q3, AArch64::Q4,
2896 AArch64::Q5, AArch64::Q6, AArch64::Q7 }
2899 unsigned GPRIdx = 0;
2900 unsigned FPRIdx = 0;
2901 for (
auto const &Arg : F->
args()) {
2902 MVT VT = TLI.getSimpleValueType(DL, Arg.getType());
2905 if (VT >=
MVT::i1 && VT <= MVT::i32) {
2906 SrcReg = Registers[0][GPRIdx++];
2907 RC = &AArch64::GPR32RegClass;
2910 SrcReg = Registers[1][GPRIdx++];
2911 RC = &AArch64::GPR64RegClass;
2913 SrcReg = Registers[2][FPRIdx++];
2914 RC = &AArch64::FPR16RegClass;
2916 SrcReg = Registers[3][FPRIdx++];
2917 RC = &AArch64::FPR32RegClass;
2919 SrcReg = Registers[4][FPRIdx++];
2920 RC = &AArch64::FPR64RegClass;
2922 SrcReg = Registers[5][FPRIdx++];
2923 RC = &AArch64::FPR128RegClass;
2927 unsigned DstReg = FuncInfo.MF->addLiveIn(SrcReg, RC);
2931 unsigned ResultReg = createResultReg(RC);
2932 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
2935 updateValueMap(&Arg, ResultReg);
2940 bool AArch64FastISel::processCallArgs(CallLoweringInfo &CLI,
2942 unsigned &NumBytes) {
2945 CCState CCInfo(CC,
false, *FuncInfo.MF, ArgLocs, *Context);
2946 CCInfo.AnalyzeCallOperands(OutVTs, CLI.OutFlags, CCAssignFnForCall(CC));
2949 NumBytes = CCInfo.getNextStackOffset();
2952 unsigned AdjStackDown =
TII.getCallFrameSetupOpcode();
2953 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(AdjStackDown))
2957 for (
unsigned i = 0, e = ArgLocs.
size(); i != e; ++i) {
2962 unsigned ArgReg = getRegForValue(ArgVal);
2973 ArgReg = emitIntExt(SrcVT, ArgReg, DestVT,
false);
2983 ArgReg = emitIntExt(SrcVT, ArgReg, DestVT,
true);
2994 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
3001 assert(VA.
isMemLoc() &&
"Assuming store on stack.");
3004 if (isa<UndefValue>(ArgVal))
3010 unsigned BEAlign = 0;
3011 if (ArgSize < 8 && !Subtarget->isLittleEndian())
3012 BEAlign = 8 - ArgSize;
3015 Addr.setKind(Address::RegBase);
3016 Addr.setReg(AArch64::SP);
3019 unsigned Alignment =
DL.getABITypeAlignment(ArgVal->
getType());
3024 if (!emitStore(ArgVT, ArgReg, Addr, MMO))
3031 bool AArch64FastISel::finishCall(CallLoweringInfo &CLI,
MVT RetVT,
3032 unsigned NumBytes) {
3036 unsigned AdjStackUp =
TII.getCallFrameDestroyOpcode();
3037 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(AdjStackUp))
3038 .addImm(NumBytes).
addImm(0);
3043 CCState CCInfo(CC,
false, *FuncInfo.MF, RVLocs, *Context);
3044 CCInfo.AnalyzeCallResult(RetVT, CCAssignFnForCall(CC));
3047 if (RVLocs.
size() != 1)
3051 MVT CopyVT = RVLocs[0].getValVT();
3054 if (CopyVT.
isVector() && !Subtarget->isLittleEndian())
3057 unsigned ResultReg = createResultReg(TLI.getRegClassFor(CopyVT));
3058 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
3060 .addReg(RVLocs[0].getLocReg());
3061 CLI.InRegs.push_back(RVLocs[0].getLocReg());
3063 CLI.ResultReg = ResultReg;
3064 CLI.NumResultRegs = 1;
3070 bool AArch64FastISel::fastLowerCall(CallLoweringInfo &CLI) {
3072 bool IsTailCall = CLI.IsTailCall;
3073 bool IsVarArg = CLI.IsVarArg;
3074 const Value *Callee = CLI.Callee;
3077 if (!Callee && !Symbol)
3099 if (CLI.RetTy->isVoidTy())
3101 else if (!isTypeLegal(CLI.RetTy, RetVT))
3104 for (
auto Flag : CLI.OutFlags)
3110 OutVTs.
reserve(CLI.OutVals.size());
3112 for (
auto *Val : CLI.OutVals) {
3114 if (!isTypeLegal(Val->getType(), VT) &&
3126 if (Callee && !computeCallAddress(Callee, Addr))
3131 if (!processCallArgs(CLI, OutVTs, NumBytes))
3137 const MCInstrDesc &II =
TII.get(Addr.getReg() ? AArch64::BLR : AArch64::BL);
3138 MIB =
BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II);
3141 else if (Addr.getGlobalValue())
3143 else if (Addr.getReg()) {
3144 unsigned Reg = constrainOperandRegClass(II, Addr.getReg(), 0);
3149 unsigned CallReg = 0;
3151 unsigned ADRPReg = createResultReg(&AArch64::GPR64commonRegClass);
3156 CallReg = createResultReg(&AArch64::GPR64RegClass);
3157 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
3158 TII.get(AArch64::LDRXui), CallReg)
3162 }
else if (Addr.getGlobalValue())
3163 CallReg = materializeGV(Addr.getGlobalValue());
3164 else if (Addr.getReg())
3165 CallReg = Addr.getReg();
3171 CallReg = constrainOperandRegClass(II, CallReg, 0);
3172 MIB =
BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II).
addReg(CallReg);
3176 for (
auto Reg : CLI.OutRegs)
3181 MIB.
addRegMask(TRI.getCallPreservedMask(*FuncInfo.MF, CC));
3186 return finishCall(CLI, RetVT, NumBytes);
3189 bool AArch64FastISel::isMemCpySmall(uint64_t Len,
unsigned Alignment) {
3191 return Len / Alignment <= 4;
3196 bool AArch64FastISel::tryEmitSmallMemCpy(
Address Dest,
Address Src,
3197 uint64_t Len,
unsigned Alignment) {
3199 if (!isMemCpySmall(Len, Alignment))
3202 int64_t UnscaledOffset = 0;
3208 if (!Alignment || Alignment >= 8) {
3220 if (Len >= 4 && Alignment == 4)
3222 else if (Len >= 2 && Alignment == 2)
3229 unsigned ResultReg = emitLoad(VT, VT, Src);
3233 if (!emitStore(VT, ResultReg, Dest))
3238 UnscaledOffset += Size;
3241 Dest.setOffset(OrigDest.getOffset() + UnscaledOffset);
3242 Src.setOffset(OrigSrc.getOffset() + UnscaledOffset);
3252 const Value *Cond) {
3253 if (!isa<ExtractValueInst>(Cond))
3256 const auto *EV = cast<ExtractValueInst>(Cond);
3257 if (!isa<IntrinsicInst>(EV->getAggregateOperand()))
3260 const auto *II = cast<IntrinsicInst>(EV->getAggregateOperand());
3262 const Function *Callee = II->getCalledFunction();
3264 cast<StructType>(Callee->
getReturnType())->getTypeAtIndex(0U);
3265 if (!isTypeLegal(RetTy, RetVT))
3268 if (RetVT != MVT::i32 && RetVT !=
MVT::i64)
3271 const Value *LHS = II->getArgOperand(0);
3272 const Value *RHS = II->getArgOperand(1);
3275 if (isa<ConstantInt>(LHS) && !isa<ConstantInt>(RHS) &&
3276 isCommutativeIntrinsic(II))
3284 case Intrinsic::smul_with_overflow:
3285 if (
const auto *C = dyn_cast<ConstantInt>(RHS))
3286 if (C->getValue() == 2)
3287 IID = Intrinsic::sadd_with_overflow;
3289 case Intrinsic::umul_with_overflow:
3290 if (
const auto *C = dyn_cast<ConstantInt>(RHS))
3291 if (C->getValue() == 2)
3292 IID = Intrinsic::uadd_with_overflow;
3300 case Intrinsic::sadd_with_overflow:
3301 case Intrinsic::ssub_with_overflow:
3304 case Intrinsic::uadd_with_overflow:
3307 case Intrinsic::usub_with_overflow:
3310 case Intrinsic::smul_with_overflow:
3311 case Intrinsic::umul_with_overflow:
3317 if (!isValueAvailable(II))
3323 for (
auto Itr = std::prev(Start); Itr != End; --Itr) {
3326 if (!isa<ExtractValueInst>(Itr))
3330 const auto *EVI = cast<ExtractValueInst>(Itr);
3331 if (EVI->getAggregateOperand() != II)
3339 bool AArch64FastISel::fastLowerIntrinsicCall(
const IntrinsicInst *II) {
3342 default:
return false;
3343 case Intrinsic::frameaddress: {
3350 unsigned SrcReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
3351 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
3359 unsigned Depth = cast<ConstantInt>(II->
getOperand(0))->getZExtValue();
3361 DestReg = fastEmitInst_ri(AArch64::LDRXui, &AArch64::GPR64RegClass,
3363 assert(DestReg &&
"Unexpected LDR instruction emission failure.");
3367 updateValueMap(II, SrcReg);
3370 case Intrinsic::memcpy:
3371 case Intrinsic::memmove: {
3372 const auto *MTI = cast<MemTransferInst>(II);
3374 if (MTI->isVolatile())
3380 if (isa<ConstantInt>(MTI->getLength()) && IsMemCpy) {
3383 uint64_t Len = cast<ConstantInt>(MTI->getLength())->getZExtValue();
3384 unsigned Alignment = MTI->getAlignment();
3385 if (isMemCpySmall(Len, Alignment)) {
3387 if (!computeAddress(MTI->getRawDest(), Dest) ||
3388 !computeAddress(MTI->getRawSource(), Src))
3390 if (tryEmitSmallMemCpy(Dest, Src, Len, Alignment))
3395 if (!MTI->getLength()->getType()->isIntegerTy(64))
3398 if (MTI->getSourceAddressSpace() > 255 || MTI->getDestAddressSpace() > 255)
3403 const char *IntrMemName = isa<MemCpyInst>(II) ?
"memcpy" :
"memmove";
3406 case Intrinsic::memset: {
3407 const MemSetInst *MSI = cast<MemSetInst>(II);
3422 case Intrinsic::sin:
3423 case Intrinsic::cos:
3424 case Intrinsic::pow: {
3426 if (!isTypeLegal(II->
getType(), RetVT))
3442 case Intrinsic::sin:
3443 LC = LibCallTable[0][Is64Bit];
3445 case Intrinsic::cos:
3446 LC = LibCallTable[1][Is64Bit];
3448 case Intrinsic::pow:
3449 LC = LibCallTable[2][Is64Bit];
3460 Entry.Ty = Arg->getType();
3461 Args.push_back(Entry);
3464 CallLoweringInfo CLI;
3466 CLI.setCallee(DL, Ctx, TLI.getLibcallCallingConv(LC), II->
getType(),
3467 TLI.getLibcallName(LC), std::move(Args));
3468 if (!lowerCallTo(CLI))
3470 updateValueMap(II, CLI.ResultReg);
3473 case Intrinsic::fabs: {
3475 if (!isTypeLegal(II->
getType(), VT))
3483 Opc = AArch64::FABSSr;
3486 Opc = AArch64::FABSDr;
3489 unsigned SrcReg = getRegForValue(II->
getOperand(0));
3492 bool SrcRegIsKill = hasTrivialKill(II->
getOperand(0));
3493 unsigned ResultReg = createResultReg(TLI.getRegClassFor(VT));
3494 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(Opc), ResultReg)
3496 updateValueMap(II, ResultReg);
3499 case Intrinsic::trap: {
3500 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(AArch64::BRK))
3504 case Intrinsic::sqrt: {
3508 if (!isTypeLegal(RetTy, VT))
3511 unsigned Op0Reg = getRegForValue(II->
getOperand(0));
3514 bool Op0IsKill = hasTrivialKill(II->
getOperand(0));
3516 unsigned ResultReg = fastEmit_r(VT, VT,
ISD::FSQRT, Op0Reg, Op0IsKill);
3520 updateValueMap(II, ResultReg);
3523 case Intrinsic::sadd_with_overflow:
3524 case Intrinsic::uadd_with_overflow:
3525 case Intrinsic::ssub_with_overflow:
3526 case Intrinsic::usub_with_overflow:
3527 case Intrinsic::smul_with_overflow:
3528 case Intrinsic::umul_with_overflow: {
3532 Type *RetTy = Ty->getTypeAtIndex(0U);
3535 if (!isTypeLegal(RetTy, VT))
3538 if (VT != MVT::i32 && VT !=
MVT::i64)
3544 if (isa<ConstantInt>(LHS) && !isa<ConstantInt>(RHS) &&
3545 isCommutativeIntrinsic(II))
3553 case Intrinsic::smul_with_overflow:
3554 if (
const auto *C = dyn_cast<ConstantInt>(RHS))
3555 if (C->getValue() == 2) {
3556 IID = Intrinsic::sadd_with_overflow;
3560 case Intrinsic::umul_with_overflow:
3561 if (
const auto *C = dyn_cast<ConstantInt>(RHS))
3562 if (C->getValue() == 2) {
3563 IID = Intrinsic::uadd_with_overflow;
3569 unsigned ResultReg1 = 0, ResultReg2 = 0, MulReg = 0;
3573 case Intrinsic::sadd_with_overflow:
3574 ResultReg1 = emitAdd(VT, LHS, RHS,
true);
3577 case Intrinsic::uadd_with_overflow:
3578 ResultReg1 = emitAdd(VT, LHS, RHS,
true);
3581 case Intrinsic::ssub_with_overflow:
3582 ResultReg1 = emitSub(VT, LHS, RHS,
true);
3585 case Intrinsic::usub_with_overflow:
3586 ResultReg1 = emitSub(VT, LHS, RHS,
true);
3589 case Intrinsic::smul_with_overflow: {
3591 unsigned LHSReg = getRegForValue(LHS);
3594 bool LHSIsKill = hasTrivialKill(LHS);
3596 unsigned RHSReg = getRegForValue(RHS);
3599 bool RHSIsKill = hasTrivialKill(RHS);
3601 if (VT == MVT::i32) {
3602 MulReg = emitSMULL_rr(
MVT::i64, LHSReg, LHSIsKill, RHSReg, RHSIsKill);
3605 MulReg = fastEmitInst_extractsubreg(VT, MulReg,
true,
3607 ShiftReg = fastEmitInst_extractsubreg(VT, ShiftReg,
true,
3609 emitSubs_rs(VT, ShiftReg,
true, MulReg,
false,
3612 assert(VT ==
MVT::i64 &&
"Unexpected value type.");
3615 MulReg = emitMul_rr(VT, LHSReg,
false, RHSReg,
3617 unsigned SMULHReg = fastEmit_rr(VT, VT,
ISD::MULHS, LHSReg, LHSIsKill,
3619 emitSubs_rs(VT, SMULHReg,
true, MulReg,
false,
3624 case Intrinsic::umul_with_overflow: {
3626 unsigned LHSReg = getRegForValue(LHS);
3629 bool LHSIsKill = hasTrivialKill(LHS);
3631 unsigned RHSReg = getRegForValue(RHS);
3634 bool RHSIsKill = hasTrivialKill(RHS);
3636 if (VT == MVT::i32) {
3637 MulReg = emitUMULL_rr(
MVT::i64, LHSReg, LHSIsKill, RHSReg, RHSIsKill);
3638 emitSubs_rs(
MVT::i64, AArch64::XZR,
true, MulReg,
3641 MulReg = fastEmitInst_extractsubreg(VT, MulReg,
true,
3644 assert(VT ==
MVT::i64 &&
"Unexpected value type.");
3647 MulReg = emitMul_rr(VT, LHSReg,
false, RHSReg,
3649 unsigned UMULHReg = fastEmit_rr(VT, VT,
ISD::MULHU, LHSReg, LHSIsKill,
3651 emitSubs_rr(VT, AArch64::XZR,
true, UMULHReg,
3659 ResultReg1 = createResultReg(TLI.getRegClassFor(VT));
3660 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
3664 ResultReg2 = fastEmitInst_rri(AArch64::CSINCWr, &AArch64::GPR32RegClass,
3665 AArch64::WZR,
true, AArch64::WZR,
3668 assert((ResultReg1 + 1) == ResultReg2 &&
3669 "Nonconsecutive result registers.");
3670 updateValueMap(II, ResultReg1, 2);
3677 bool AArch64FastISel::selectRet(
const Instruction *I) {
3681 if (!FuncInfo.CanLowerReturn)
3699 : RetCC_AArch64_AAPCS;
3700 CCInfo.AnalyzeReturn(Outs, RetCC);
3703 if (ValLocs.size() != 1)
3718 unsigned Reg = getRegForValue(RV);
3722 unsigned SrcReg = Reg + VA.
getValNo();
3725 if (!MRI.getRegClass(SrcReg)->contains(DestReg))
3728 EVT RVEVT = TLI.getValueType(DL, RV->
getType());
3734 !Subtarget->isLittleEndian())
3743 if (RVVT != DestVT) {
3747 if (!Outs[0].Flags.isZExt() && !Outs[0].Flags.isSExt())
3750 bool IsZExt = Outs[0].Flags.isZExt();
3751 SrcReg = emitIntExt(RVVT, SrcReg, DestVT, IsZExt);
3757 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
3765 TII.get(AArch64::RET_ReallyLR));
3766 for (
unsigned i = 0, e = RetRegs.
size(); i != e; ++i)
3771 bool AArch64FastISel::selectTrunc(
const Instruction *I) {
3776 EVT SrcEVT = TLI.getValueType(DL, SrcTy,
true);
3777 EVT DestEVT = TLI.getValueType(DL, DestTy,
true);
3793 unsigned SrcReg = getRegForValue(Op);
3796 bool SrcIsKill = hasTrivialKill(Op);
3821 unsigned Reg32 = fastEmitInst_extractsubreg(MVT::i32, SrcReg, SrcIsKill,
3824 ResultReg = emitAnd_ri(MVT::i32, Reg32,
true, Mask);
3825 assert(ResultReg &&
"Unexpected AND instruction emission failure.");
3827 ResultReg = createResultReg(&AArch64::GPR32RegClass);
3828 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
3833 updateValueMap(I, ResultReg);
3837 unsigned AArch64FastISel::emiti1Ext(
unsigned SrcReg,
MVT DestVT,
bool IsZExt) {
3838 assert((DestVT ==
MVT::i8 || DestVT ==
MVT::i16 || DestVT == MVT::i32 ||
3840 "Unexpected value type.");
3846 unsigned ResultReg = emitAnd_ri(MVT::i32, SrcReg,
false, 1);
3847 assert(ResultReg &&
"Unexpected AND instruction emission failure.");
3851 unsigned Reg64 = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
3852 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
3856 .
addImm(AArch64::sub_32);
3865 return fastEmitInst_rii(AArch64::SBFMWri, &AArch64::GPR32RegClass, SrcReg,
3870 unsigned AArch64FastISel::emitMul_rr(
MVT RetVT,
unsigned Op0,
bool Op0IsKill,
3871 unsigned Op1,
bool Op1IsKill) {
3879 Opc = AArch64::MADDWrrr; ZReg = AArch64::WZR;
break;
3881 Opc = AArch64::MADDXrrr; ZReg = AArch64::XZR;
break;
3885 (RetVT ==
MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
3886 return fastEmitInst_rrr(Opc, RC, Op0, Op0IsKill, Op1, Op1IsKill,
3890 unsigned AArch64FastISel::emitSMULL_rr(
MVT RetVT,
unsigned Op0,
bool Op0IsKill,
3891 unsigned Op1,
bool Op1IsKill) {
3895 return fastEmitInst_rrr(AArch64::SMADDLrrr, &AArch64::GPR64RegClass,
3896 Op0, Op0IsKill, Op1, Op1IsKill,
3897 AArch64::XZR,
true);
3900 unsigned AArch64FastISel::emitUMULL_rr(
MVT RetVT,
unsigned Op0,
bool Op0IsKill,
3901 unsigned Op1,
bool Op1IsKill) {
3905 return fastEmitInst_rrr(AArch64::UMADDLrrr, &AArch64::GPR64RegClass,
3906 Op0, Op0IsKill, Op1, Op1IsKill,
3907 AArch64::XZR,
true);
3910 unsigned AArch64FastISel::emitLSL_rr(
MVT RetVT,
unsigned Op0Reg,
bool Op0IsKill,
3911 unsigned Op1Reg,
bool Op1IsKill) {
3913 bool NeedTrunc =
false;
3917 case MVT::i8: Opc = AArch64::LSLVWr; NeedTrunc =
true; Mask = 0xff;
break;
3918 case MVT::i16: Opc = AArch64::LSLVWr; NeedTrunc =
true; Mask = 0xffff;
break;
3919 case MVT::i32: Opc = AArch64::LSLVWr;
break;
3920 case MVT::i64: Opc = AArch64::LSLVXr;
break;
3924 (RetVT ==
MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
3926 Op1Reg = emitAnd_ri(MVT::i32, Op1Reg, Op1IsKill, Mask);
3929 unsigned ResultReg = fastEmitInst_rr(Opc, RC, Op0Reg, Op0IsKill, Op1Reg,
3932 ResultReg = emitAnd_ri(MVT::i32, ResultReg,
true, Mask);
3936 unsigned AArch64FastISel::emitLSL_ri(
MVT RetVT,
MVT SrcVT,
unsigned Op0,
3937 bool Op0IsKill, uint64_t Shift,
3940 "Unexpected source/return type pair.");
3942 SrcVT == MVT::i32 || SrcVT ==
MVT::i64) &&
3943 "Unexpected source value type.");
3945 RetVT ==
MVT::i64) &&
"Unexpected return value type.");
3947 bool Is64Bit = (RetVT ==
MVT::i64);
3948 unsigned RegSize = Is64Bit ? 64 : 32;
3952 Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
3956 if (RetVT == SrcVT) {
3957 unsigned ResultReg = createResultReg(RC);
3958 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
3963 return emitIntExt(SrcVT, Op0, RetVT, IsZExt);
3967 if (Shift >= DstBits)
3995 unsigned ImmR = RegSize - Shift;
3997 unsigned ImmS = std::min<unsigned>(SrcBits - 1, DstBits - 1 - Shift);
3998 static const unsigned OpcTable[2][2] = {
3999 {AArch64::SBFMWri, AArch64::SBFMXri},
4000 {AArch64::UBFMWri, AArch64::UBFMXri}
4002 unsigned Opc = OpcTable[IsZExt][Is64Bit];
4004 unsigned TmpReg = MRI.createVirtualRegister(RC);
4005 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
4009 .
addImm(AArch64::sub_32);
4013 return fastEmitInst_rii(Opc, RC, Op0, Op0IsKill, ImmR, ImmS);
4016 unsigned AArch64FastISel::emitLSR_rr(
MVT RetVT,
unsigned Op0Reg,
bool Op0IsKill,
4017 unsigned Op1Reg,
bool Op1IsKill) {
4019 bool NeedTrunc =
false;
4023 case MVT::i8: Opc = AArch64::LSRVWr; NeedTrunc =
true; Mask = 0xff;
break;
4024 case MVT::i16: Opc = AArch64::LSRVWr; NeedTrunc =
true; Mask = 0xffff;
break;
4025 case MVT::i32: Opc = AArch64::LSRVWr;
break;
4026 case MVT::i64: Opc = AArch64::LSRVXr;
break;
4030 (RetVT ==
MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
4032 Op0Reg = emitAnd_ri(MVT::i32, Op0Reg, Op0IsKill, Mask);
4033 Op1Reg = emitAnd_ri(MVT::i32, Op1Reg, Op1IsKill, Mask);
4034 Op0IsKill = Op1IsKill =
true;
4036 unsigned ResultReg = fastEmitInst_rr(Opc, RC, Op0Reg, Op0IsKill, Op1Reg,
4039 ResultReg = emitAnd_ri(MVT::i32, ResultReg,
true, Mask);
4043 unsigned AArch64FastISel::emitLSR_ri(
MVT RetVT,
MVT SrcVT,
unsigned Op0,
4044 bool Op0IsKill, uint64_t Shift,
4047 "Unexpected source/return type pair.");
4049 SrcVT == MVT::i32 || SrcVT ==
MVT::i64) &&
4050 "Unexpected source value type.");
4052 RetVT ==
MVT::i64) &&
"Unexpected return value type.");
4054 bool Is64Bit = (RetVT ==
MVT::i64);
4055 unsigned RegSize = Is64Bit ? 64 : 32;
4059 Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
4063 if (RetVT == SrcVT) {
4064 unsigned ResultReg = createResultReg(RC);
4065 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
4070 return emitIntExt(SrcVT, Op0, RetVT, IsZExt);
4074 if (Shift >= DstBits)
4102 if (Shift >= SrcBits && IsZExt)
4108 Op0 = emitIntExt(SrcVT, Op0, RetVT, IsZExt);
4117 unsigned ImmR = std::min<unsigned>(SrcBits - 1, Shift);
4118 unsigned ImmS = SrcBits - 1;
4119 static const unsigned OpcTable[2][2] = {
4120 {AArch64::SBFMWri, AArch64::SBFMXri},
4121 {AArch64::UBFMWri, AArch64::UBFMXri}
4123 unsigned Opc = OpcTable[IsZExt][Is64Bit];
4125 unsigned TmpReg = MRI.createVirtualRegister(RC);
4126 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
4130 .
addImm(AArch64::sub_32);
4134 return fastEmitInst_rii(Opc, RC, Op0, Op0IsKill, ImmR, ImmS);
4137 unsigned AArch64FastISel::emitASR_rr(
MVT RetVT,
unsigned Op0Reg,
bool Op0IsKill,
4138 unsigned Op1Reg,
bool Op1IsKill) {
4140 bool NeedTrunc =
false;
4144 case MVT::i8: Opc = AArch64::ASRVWr; NeedTrunc =
true; Mask = 0xff;
break;
4145 case MVT::i16: Opc = AArch64::ASRVWr; NeedTrunc =
true; Mask = 0xffff;
break;
4146 case MVT::i32: Opc = AArch64::ASRVWr;
break;
4147 case MVT::i64: Opc = AArch64::ASRVXr;
break;
4151 (RetVT ==
MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
4153 Op0Reg = emitIntExt(RetVT, Op0Reg, MVT::i32,
false);
4154 Op1Reg = emitAnd_ri(MVT::i32, Op1Reg, Op1IsKill, Mask);
4155 Op0IsKill = Op1IsKill =
true;
4157 unsigned ResultReg = fastEmitInst_rr(Opc, RC, Op0Reg, Op0IsKill, Op1Reg,
4160 ResultReg = emitAnd_ri(MVT::i32, ResultReg,
true, Mask);
4164 unsigned AArch64FastISel::emitASR_ri(
MVT RetVT,
MVT SrcVT,
unsigned Op0,
4165 bool Op0IsKill, uint64_t Shift,
4168 "Unexpected source/return type pair.");
4170 SrcVT == MVT::i32 || SrcVT ==
MVT::i64) &&
4171 "Unexpected source value type.");
4173 RetVT ==
MVT::i64) &&
"Unexpected return value type.");
4175 bool Is64Bit = (RetVT ==
MVT::i64);
4176 unsigned RegSize = Is64Bit ? 64 : 32;
4180 Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
4184 if (RetVT == SrcVT) {
4185 unsigned ResultReg = createResultReg(RC);
4186 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
4191 return emitIntExt(SrcVT, Op0, RetVT, IsZExt);
4195 if (Shift >= DstBits)
4223 if (Shift >= SrcBits && IsZExt)
4226 unsigned ImmR = std::min<unsigned>(SrcBits - 1, Shift);
4227 unsigned ImmS = SrcBits - 1;
4228 static const unsigned OpcTable[2][2] = {
4229 {AArch64::SBFMWri, AArch64::SBFMXri},
4230 {AArch64::UBFMWri, AArch64::UBFMXri}
4232 unsigned Opc = OpcTable[IsZExt][Is64Bit];
4234 unsigned TmpReg = MRI.createVirtualRegister(RC);
4235 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
4239 .
addImm(AArch64::sub_32);
4243 return fastEmitInst_rii(Opc, RC, Op0, Op0IsKill, ImmR, ImmS);
4246 unsigned AArch64FastISel::emitIntExt(
MVT SrcVT,
unsigned SrcReg,
MVT DestVT,
4248 assert(DestVT !=
MVT::i1 &&
"ZeroExt/SignExt an i1?");
4255 (DestVT != MVT::i32) && (DestVT !=
MVT::i64)) ||
4257 (SrcVT !=
MVT::i16) && (SrcVT != MVT::i32)))
4267 return emiti1Ext(SrcReg, DestVT, IsZExt);
4270 Opc = IsZExt ? AArch64::UBFMXri : AArch64::SBFMXri;
4272 Opc = IsZExt ? AArch64::UBFMWri : AArch64::SBFMWri;
4277 Opc = IsZExt ? AArch64::UBFMXri : AArch64::SBFMXri;
4279 Opc = IsZExt ? AArch64::UBFMWri : AArch64::SBFMWri;
4283 assert(DestVT ==
MVT::i64 &&
"IntExt i32 to i32?!?");
4284 Opc = IsZExt ? AArch64::UBFMXri : AArch64::SBFMXri;
4293 unsigned Src64 = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
4294 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
4298 .
addImm(AArch64::sub_32);
4303 (DestVT ==
MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
4304 return fastEmitInst_rii(Opc, RC, SrcReg,
false, 0, Imm);
4311 case AArch64::LDURBBi:
4312 case AArch64::LDURHHi:
4313 case AArch64::LDURWi:
4314 case AArch64::LDRBBui:
4315 case AArch64::LDRHHui:
4316 case AArch64::LDRWui:
4317 case AArch64::LDRBBroX:
4318 case AArch64::LDRHHroX:
4319 case AArch64::LDRWroX:
4320 case AArch64::LDRBBroW:
4321 case AArch64::LDRHHroW:
4322 case AArch64::LDRWroW:
4331 case AArch64::LDURSBWi:
4332 case AArch64::LDURSHWi:
4333 case AArch64::LDURSBXi:
4334 case AArch64::LDURSHXi:
4335 case AArch64::LDURSWi:
4336 case AArch64::LDRSBWui:
4337 case AArch64::LDRSHWui:
4338 case AArch64::LDRSBXui:
4339 case AArch64::LDRSHXui:
4340 case AArch64::LDRSWui:
4341 case AArch64::LDRSBWroX:
4342 case AArch64::LDRSHWroX:
4343 case AArch64::LDRSBXroX:
4344 case AArch64::LDRSHXroX:
4345 case AArch64::LDRSWroX:
4346 case AArch64::LDRSBWroW:
4347 case AArch64::LDRSHWroW:
4348 case AArch64::LDRSBXroW:
4349 case AArch64::LDRSHXroW:
4350 case AArch64::LDRSWroW:
4355 bool AArch64FastISel::optimizeIntExtLoad(
const Instruction *I,
MVT RetVT,
4358 if (!LI || !LI->hasOneUse())
4362 unsigned Reg = lookUpRegForValue(LI);
4372 bool IsZExt = isa<ZExtInst>(
I);
4373 const auto *LoadMI =
MI;
4375 LoadMI->getOperand(1).getSubReg() == AArch64::sub_32) {
4377 LoadMI = MRI.getUniqueVRegDef(LoadReg);
4378 assert(LoadMI &&
"Expected valid instruction");
4384 if (RetVT !=
MVT::i64 || SrcVT > MVT::i32) {
4385 updateValueMap(I, Reg);
4390 unsigned Reg64 = createResultReg(&AArch64::GPR64RegClass);
4391 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
4395 .
addImm(AArch64::sub_32);
4400 "Expected copy instruction");
4404 updateValueMap(I, Reg);
4408 bool AArch64FastISel::selectIntExt(
const Instruction *I) {
4409 assert((isa<ZExtInst>(I) || isa<SExtInst>(I)) &&
4410 "Unexpected integer extend instruction.");
4413 if (!isTypeSupported(I->
getType(), RetVT))
4420 if (optimizeIntExtLoad(I, RetVT, SrcVT))
4423 unsigned SrcReg = getRegForValue(I->
getOperand(0));
4426 bool SrcIsKill = hasTrivialKill(I->
getOperand(0));
4429 bool IsZExt = isa<ZExtInst>(
I);
4430 if (
const auto *Arg = dyn_cast<Argument>(I->
getOperand(0))) {
4431 if ((IsZExt && Arg->hasZExtAttr()) || (!IsZExt && Arg->hasSExtAttr())) {
4433 unsigned ResultReg = createResultReg(&AArch64::GPR64RegClass);
4434 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
4438 .
addImm(AArch64::sub_32);
4445 unsigned UseReg = lookUpRegForValue(I);
4447 MRI.clearKillFlags(UseReg);
4449 updateValueMap(I, SrcReg);
4454 unsigned ResultReg = emitIntExt(SrcVT, SrcReg, RetVT, IsZExt);
4458 updateValueMap(I, ResultReg);
4462 bool AArch64FastISel::selectRem(
const Instruction *I,
unsigned ISDOpcode) {
4463 EVT DestEVT = TLI.getValueType(DL, I->
getType(),
true);
4468 if (DestVT !=
MVT::i64 && DestVT != MVT::i32)
4472 bool Is64bit = (DestVT ==
MVT::i64);
4473 switch (ISDOpcode) {
4477 DivOpc = Is64bit ? AArch64::SDIVXr : AArch64::SDIVWr;
4480 DivOpc = Is64bit ? AArch64::UDIVXr : AArch64::UDIVWr;
4483 unsigned MSubOpc = Is64bit ? AArch64::MSUBXrrr : AArch64::MSUBWrrr;
4484 unsigned Src0Reg = getRegForValue(I->
getOperand(0));
4487 bool Src0IsKill = hasTrivialKill(I->
getOperand(0));
4489 unsigned Src1Reg = getRegForValue(I->
getOperand(1));
4492 bool Src1IsKill = hasTrivialKill(I->
getOperand(1));
4495 (DestVT ==
MVT::i64) ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
4496 unsigned QuotReg = fastEmitInst_rr(DivOpc, RC, Src0Reg,
false,
4498 assert(QuotReg &&
"Unexpected DIV instruction emission failure.");
4501 unsigned ResultReg = fastEmitInst_rrr(MSubOpc, RC, QuotReg,
true,
4502 Src1Reg, Src1IsKill, Src0Reg,
4504 updateValueMap(I, ResultReg);
4508 bool AArch64FastISel::selectMul(
const Instruction *I) {
4510 if (!isTypeSupported(I->
getType(), VT,
true))
4514 return selectBinaryOp(I,
ISD::MUL);
4518 if (
const auto *C = dyn_cast<ConstantInt>(Src0))
4519 if (C->getValue().isPowerOf2())
4523 if (
const auto *C = dyn_cast<ConstantInt>(Src1))
4524 if (C->getValue().isPowerOf2()) {
4525 uint64_t ShiftVal = C->getValue().logBase2();
4528 if (
const auto *ZExt = dyn_cast<ZExtInst>(Src0)) {
4531 if (isValueAvailable(ZExt) && isTypeSupported(ZExt->getSrcTy(), VT)) {
4534 Src0 = ZExt->getOperand(0);
4537 }
else if (
const auto *SExt = dyn_cast<SExtInst>(Src0)) {
4540 if (isValueAvailable(SExt) && isTypeSupported(SExt->getSrcTy(), VT)) {
4543 Src0 = SExt->getOperand(0);
4548 unsigned Src0Reg = getRegForValue(Src0);
4551 bool Src0IsKill = hasTrivialKill(Src0);
4553 unsigned ResultReg =
4554 emitLSL_ri(VT, SrcVT, Src0Reg, Src0IsKill, ShiftVal, IsZExt);
4557 updateValueMap(I, ResultReg);
4562 unsigned Src0Reg = getRegForValue(I->
getOperand(0));
4565 bool Src0IsKill = hasTrivialKill(I->
getOperand(0));
4567 unsigned Src1Reg = getRegForValue(I->
getOperand(1));
4570 bool Src1IsKill = hasTrivialKill(I->
getOperand(1));
4572 unsigned ResultReg = emitMul_rr(VT, Src0Reg, Src0IsKill, Src1Reg, Src1IsKill);
4577 updateValueMap(I, ResultReg);
4581 bool AArch64FastISel::selectShift(
const Instruction *I) {
4583 if (!isTypeSupported(I->
getType(), RetVT,
true))
4587 return selectOperator(I, I->
getOpcode());
4589 if (
const auto *C = dyn_cast<ConstantInt>(I->
getOperand(1))) {
4590 unsigned ResultReg = 0;
4591 uint64_t ShiftVal = C->getZExtValue();
4593 bool IsZExt = I->
getOpcode() != Instruction::AShr;
4595 if (
const auto *ZExt = dyn_cast<ZExtInst>(Op0)) {
4598 if (isValueAvailable(ZExt) && isTypeSupported(ZExt->getSrcTy(), TmpVT)) {
4601 Op0 = ZExt->getOperand(0);
4604 }
else if (
const auto *SExt = dyn_cast<SExtInst>(Op0)) {
4607 if (isValueAvailable(SExt) && isTypeSupported(SExt->getSrcTy(), TmpVT)) {
4610 Op0 = SExt->getOperand(0);
4615 unsigned Op0Reg = getRegForValue(Op0);
4618 bool Op0IsKill = hasTrivialKill(Op0);
4622 case Instruction::Shl:
4623 ResultReg = emitLSL_ri(RetVT, SrcVT, Op0Reg, Op0IsKill, ShiftVal, IsZExt);
4625 case Instruction::AShr:
4626 ResultReg = emitASR_ri(RetVT, SrcVT, Op0Reg, Op0IsKill, ShiftVal, IsZExt);
4628 case Instruction::LShr:
4629 ResultReg = emitLSR_ri(RetVT, SrcVT, Op0Reg, Op0IsKill, ShiftVal, IsZExt);
4635 updateValueMap(I, ResultReg);
4639 unsigned Op0Reg = getRegForValue(I->
getOperand(0));
4642 bool Op0IsKill = hasTrivialKill(I->
getOperand(0));
4644 unsigned Op1Reg = getRegForValue(I->
getOperand(1));
4647 bool Op1IsKill = hasTrivialKill(I->
getOperand(1));
4649 unsigned ResultReg = 0;
4652 case Instruction::Shl:
4653 ResultReg = emitLSL_rr(RetVT, Op0Reg, Op0IsKill, Op1Reg, Op1IsKill);
4655 case Instruction::AShr:
4656 ResultReg = emitASR_rr(RetVT, Op0Reg, Op0IsKill, Op1Reg, Op1IsKill);
4658 case Instruction::LShr:
4659 ResultReg = emitLSR_rr(RetVT, Op0Reg, Op0IsKill, Op1Reg, Op1IsKill);
4666 updateValueMap(I, ResultReg);
4670 bool AArch64FastISel::selectBitCast(
const Instruction *I) {
4675 if (!isTypeLegal(I->
getType(), RetVT))
4679 if (RetVT ==
MVT::f32 && SrcVT == MVT::i32)
4680 Opc = AArch64::FMOVWSr;
4682 Opc = AArch64::FMOVXDr;
4683 else if (RetVT == MVT::i32 && SrcVT ==
MVT::f32)
4684 Opc = AArch64::FMOVSWr;
4686 Opc = AArch64::FMOVDXr;
4693 case MVT::i32: RC = &AArch64::GPR32RegClass;
break;
4694 case MVT::i64: RC = &AArch64::GPR64RegClass;
break;
4695 case MVT::f32: RC = &AArch64::FPR32RegClass;
break;
4696 case MVT::f64: RC = &AArch64::FPR64RegClass;
break;
4698 unsigned Op0Reg = getRegForValue(I->
getOperand(0));
4701 bool Op0IsKill = hasTrivialKill(I->
getOperand(0));
4702 unsigned ResultReg = fastEmitInst_r(Opc, RC, Op0Reg, Op0IsKill);
4707 updateValueMap(I, ResultReg);
4711 bool AArch64FastISel::selectFRem(
const Instruction *I) {
4713 if (!isTypeLegal(I->
getType(), RetVT))
4735 Entry.Ty = Arg->getType();
4736 Args.push_back(Entry);
4739 CallLoweringInfo CLI;
4741 CLI.setCallee(DL, Ctx, TLI.getLibcallCallingConv(LC), I->
getType(),
4742 TLI.getLibcallName(LC), std::move(Args));
4743 if (!lowerCallTo(CLI))
4745 updateValueMap(I, CLI.ResultReg);
4749 bool AArch64FastISel::selectSDiv(
const Instruction *I) {
4751 if (!isTypeLegal(I->
getType(), VT))
4758 if ((VT != MVT::i32 && VT !=
MVT::i64) || !C ||
4763 unsigned Src0Reg = getRegForValue(I->
getOperand(0));
4766 bool Src0IsKill = hasTrivialKill(I->
getOperand(0));
4768 if (cast<BinaryOperator>(I)->isExact()) {
4769 unsigned ResultReg = emitASR_ri(VT, VT, Src0Reg, Src0IsKill, Lg2);
4772 updateValueMap(I, ResultReg);
4776 int64_t Pow2MinusOne = (1ULL << Lg2) - 1;
4777 unsigned AddReg = emitAdd_ri_(VT, Src0Reg,
false, Pow2MinusOne);
4782 if (!emitICmp_ri(VT, Src0Reg,
false, 0))
4788 SelectOpc = AArch64::CSELXr;
4789 RC = &AArch64::GPR64RegClass;
4791 SelectOpc = AArch64::CSELWr;
4792 RC = &AArch64::GPR32RegClass;
4794 unsigned SelectReg =
4795 fastEmitInst_rri(SelectOpc, RC, AddReg,
true, Src0Reg,
4802 unsigned ZeroReg = (VT ==
MVT::i64) ? AArch64::XZR : AArch64::WZR;
4805 ResultReg = emitAddSub_rs(
false, VT, ZeroReg,
true,
4808 ResultReg = emitASR_ri(VT, VT, SelectReg,
true, Lg2);
4813 updateValueMap(I, ResultReg);
4820 std::pair<unsigned, bool> AArch64FastISel::getRegForGEPIndex(
const Value *Idx) {
4821 unsigned IdxN = getRegForValue(Idx);
4824 return std::pair<unsigned, bool>(0,
false);
4826 bool IdxNIsKill = hasTrivialKill(Idx);
4829 MVT PtrVT = TLI.getPointerTy(DL);
4831 if (IdxVT.
bitsLT(PtrVT)) {
4832 IdxN = emitIntExt(IdxVT.
getSimpleVT(), IdxN, PtrVT,
false);
4834 }
else if (IdxVT.
bitsGT(PtrVT))
4835 llvm_unreachable(
"AArch64 FastISel doesn't support types larger than i64");
4836 return std::pair<unsigned, bool>(IdxN, IdxNIsKill);
4843 bool AArch64FastISel::selectGetElementPtr(
const Instruction *I) {
4847 bool NIsKill = hasTrivialKill(I->
getOperand(0));
4851 uint64_t TotalOffs = 0;
4853 MVT VT = TLI.getPointerTy(DL);
4854 for (
auto OI = std::next(I->
op_begin()), E = I->
op_end(); OI != E; ++OI) {
4855 const Value *Idx = *OI;
4856 if (
auto *StTy = dyn_cast<StructType>(Ty)) {
4857 unsigned Field = cast<ConstantInt>(Idx)->getZExtValue();
4860 TotalOffs +=
DL.getStructLayout(StTy)->getElementOffset(Field);
4861 Ty = StTy->getElementType(Field);
4863 Ty = cast<SequentialType>(Ty)->getElementType();
4865 if (
const auto *CI = dyn_cast<ConstantInt>(Idx)) {
4870 DL.getTypeAllocSize(Ty) * cast<ConstantInt>(CI)->getSExtValue();
4874 N = emitAdd_ri_(VT, N, NIsKill, TotalOffs);
4882 uint64_t ElementSize =
DL.getTypeAllocSize(Ty);
4883 std::pair<unsigned, bool> Pair = getRegForGEPIndex(Idx);
4884 unsigned IdxN = Pair.first;
4885 bool IdxNIsKill = Pair.second;
4889 if (ElementSize != 1) {
4890 unsigned C = fastEmit_i(VT, VT,
ISD::Constant, ElementSize);
4893 IdxN = emitMul_rr(VT, IdxN, IdxNIsKill, C,
true);
4898 N = fastEmit_rr(VT, VT,
ISD::ADD, N, NIsKill, IdxN, IdxNIsKill);
4904 N = emitAdd_ri_(VT, N, NIsKill, TotalOffs);
4908 updateValueMap(I, N);
4912 bool AArch64FastISel::fastSelectInstruction(
const Instruction *I) {
4916 case Instruction::Add:
4917 case Instruction::Sub:
4918 return selectAddSub(I);
4919 case Instruction::Mul:
4920 return selectMul(I);
4921 case Instruction::SDiv:
4922 return selectSDiv(I);
4923 case Instruction::SRem:
4927 case Instruction::URem:
4931 case Instruction::Shl:
4932 case Instruction::LShr:
4933 case Instruction::AShr:
4934 return selectShift(I);
4938 return selectLogicalOp(I);
4939 case Instruction::Br:
4940 return selectBranch(I);
4941 case Instruction::IndirectBr:
4942 return selectIndirectBr(I);
4943 case Instruction::BitCast:
4945 return selectBitCast(I);
4947 case Instruction::FPToSI:
4949 return selectFPToInt(I,
true);
4951 case Instruction::FPToUI:
4952 return selectFPToInt(I,
false);
4953 case Instruction::ZExt:
4954 case Instruction::SExt:
4955 return selectIntExt(I);
4956 case Instruction::Trunc:
4958 return selectTrunc(I);
4960 case Instruction::FPExt:
4961 return selectFPExt(I);
4962 case Instruction::FPTrunc:
4963 return selectFPTrunc(I);
4964 case Instruction::SIToFP:
4966 return selectIntToFP(I,
true);
4968 case Instruction::UIToFP:
4969 return selectIntToFP(I,
false);
4971 return selectLoad(I);
4973 return selectStore(I);
4974 case Instruction::FCmp:
4975 case Instruction::ICmp:
4976 return selectCmp(I);
4978 return selectSelect(I);
4980 return selectRet(I);
4981 case Instruction::FRem:
4982 return selectFRem(I);
4983 case Instruction::GetElementPtr:
4984 return selectGetElementPtr(I);
4988 return selectOperator(I, I->
getOpcode());
4990 (void)&CC_AArch64_DarwinPCS_VarArg;
4996 return new AArch64FastISel(FuncInfo, LibInfo);
void setFrameAddressIsTaken(bool T)
ReturnInst - Return a value (possibly void), from a function.
static MVT getIntegerVT(unsigned BitWidth)
void push_back(const T &Elt)
The memory access reads data.
Type * getIndexedType() const
This class is the base class for the comparison instructions.
The memory access writes data.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
uint64_t getZExtValue() const
Get zero extended value.
LocInfo getLocInfo() const
MO_PAGE - A symbol operand with this flag represents the pc-relative offset of the 4K page containing...
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo)
Force argument to be passed in register.
Intrinsic::ID getIntrinsicID() const
getIntrinsicID - Return the intrinsic ID of this intrinsic.
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
unsigned getNumOperands() const
Nested function static chain.
static CondCode getInvertedCondCode(CondCode Code)
Describe properties that are true of each instruction in the target description file.
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE, etc.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit...
bool CCAssignFn(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
CCAssignFn - This function assigns a location for Val, updating State to reflect the change...
0 1 0 0 True if ordered and less than
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
unsigned getSizeInBits() const
bool isDoubleTy() const
isDoubleTy - Return true if this is 'double', a 64-bit IEEE fp type.
1 1 1 0 True if unordered or not equal
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
MemSetInst - This class wraps the llvm.memset intrinsic.
Type * getReturnType() const
const Function * getParent() const
Return the enclosing method, or null if none.
LoadInst - an instruction for reading from memory.
void reserve(size_type N)
bool hasAttribute(unsigned Index, Attribute::AttrKind Kind) const
Return true if the attribute exists at the given index.
static std::error_code getOffset(const SymbolRef &Sym, SectionRef Sec, uint64_t &Result)
unsigned getValNo() const
bool bitsLT(EVT VT) const
bitsLT - Return true if this has less bits than VT.
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
static MachinePointerInfo getFixedStack(int FI, int64_t offset=0)
getFixedStack - Return a MachinePointerInfo record that refers to the the specified FrameIndex...
bool isNegative() const
Determine sign of this APInt.
COPY - Target-independent register copy.
bool isVector() const
isVector - Return true if this is a vector value type.
1 0 0 1 True if unordered or equal
Used to lazily calculate structure layout information for a target machine, based on the DataLayout s...
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
bool isUnconditional() const
MachineMemOperand - A description of a memory reference used in the backend.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
SelectInst - This class represents the LLVM 'select' instruction.
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
const HexagonInstrInfo * TII
StructType - Class to represent struct types.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
A Use represents the edge between a Value definition and its users.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
unsigned getNumArgOperands() const
getNumArgOperands - Return the number of call arguments.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
static SDValue emitCmp(SelectionDAG &DAG, SDLoc DL, Comparison &C)
Reg
All possible values of the reg field in the ModR/M byte.
0 1 0 1 True if ordered and less than or equal
bool isSized(SmallPtrSetImpl< const Type * > *Visited=nullptr) const
isSized - Return true if it makes sense to take the size of this type.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
static bool isLogicalImmediate(uint64_t imm, unsigned regSize)
isLogicalImmediate - Return true if the immediate is valid for a logical immediate instruction of the...
unsigned getStoreSize() const
getStoreSize - Return the number of bytes overwritten by a store of the specified value type...
const MachineInstrBuilder & addImm(int64_t Val) const
addImm - Add a new immediate operand.
MO_CONSTPOOL - This flag indicates that a symbol operand represents the address of a constant pool en...
Hidden pointer to structure to return.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Context object for machine code objects.
unsigned getLocReg() const
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
ConstantExpr - a constant value that is initialized with an expression using other constant values...
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
MO_GOT - This flag indicates that a symbol operand represents the address of the GOT entry for the sy...
void GetReturnInfo(Type *ReturnType, AttributeSet attr, SmallVectorImpl< ISD::OutputArg > &Outs, const TargetLowering &TLI, const DataLayout &DL)
Given an LLVM IR type and return type attributes, compute the return value EVTs and flags...
static bool isZExtLoad(const MachineInstr *LI)
Simple integer binary arithmetic operators.
BasicBlock * getSuccessor(unsigned i) const
bool isArrayTy() const
isArrayTy - True if this is an instance of ArrayType.
bool isZero() const
isZero - Return true if the value is positive or negative zero.
This class represents a truncation of integer types.
PointerType - Class to represent pointers.
unsigned getKillRegState(bool B)
const BasicBlock * getBasicBlock() const
getBasicBlock - Return the LLVM basic block that this instance corresponded to originally.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
Flag
These should be considered private to the implementation of the MCInstrDesc class.
uint64_t getElementOffset(unsigned Idx) const
A self-contained host- and target-independent arbitrary-precision floating-point software implementat...
Control flow instructions. These all have token chains.
static int getFP32Imm(const APInt &Imm)
getFP32Imm - Return an 8-bit floating-point version of the 32-bit floating-point value.
MVT - Machine Value Type.
The instances of the Type class are immutable: once they are created, they are never changed...
This is an important class for using LLVM in a threaded context.
BranchInst - Conditional or Unconditional Branch instruction.
bool isVectorTy() const
isVectorTy - True if this is an instance of VectorType.
This is an important base class in LLVM.
PointerType * getType() const
getType - Overload to return most specific pointer type
bool isVector() const
isVector - Return true if this is a vector value type.
const Value * getCondition() const
bool isFloatingPoint() const
isFloatingPoint - Return true if this is a FP, or a vector FP type.
const MachineOperand & getOperand(unsigned i) const
IndirectBrInst - Indirect Branch Instruction.
bool isFloatTy() const
isFloatTy - Return true if this is 'float', a 32-bit IEEE fp type.
APInt Or(const APInt &LHS, const APInt &RHS)
Bitwise OR function for APInt.
static bool isIntExtFree(const Instruction *I)
Check if the sign-/zero-extend will be a noop.
ConstantFP - Floating Point Values [float, double].
APInt Xor(const APInt &LHS, const APInt &RHS)
Bitwise XOR function for APInt.
const DebugLoc & getDebugLoc() const
getDebugLoc - Return the debug location for this node as a DebugLoc.
static bool isSExtLoad(const MachineInstr *LI)
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
static AArch64CC::CondCode getCompareCC(CmpInst::Predicate Pred)
Value * getOperand(unsigned i) const
0 1 1 1 True if ordered (no nans)
unsigned getFrameRegister(const MachineFunction &MF) const override
bool isThreadLocal() const
If the value is "Thread Local", its value isn't shared by the threads.
MachineInstrBuilder BuildMI(MachineFunction &MF, DebugLoc DL, const MCInstrDesc &MCID)
BuildMI - Builder interface.
1 1 1 1 Always true (always folded)
static unsigned getImplicitScaleFactor(MVT VT)
Determine the implicit scale factor that is applied by a memory operation for a given value type...
unsigned getSubReg() const
EVT - Extended Value Type.
LLVMContext & getContext() const
All values hold a context through their type.
const Value * getTrueValue() const
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
1 1 0 1 True if unordered, less than, or equal
unsigned getNumSuccessors() const
BasicBlock * getSuccessor(unsigned i) const
bool bitsGT(EVT VT) const
bitsGT - Return true if this has more bits than VT.
unsigned countTrailingZeros() const
Count the number of trailing zero bits.
0 0 1 0 True if ordered and greater than
static uint64_t encodeLogicalImmediate(uint64_t imm, unsigned regSize)
encodeLogicalImmediate - Return the encoded immediate value for a logical immediate instruction of th...
static int getFP64Imm(const APInt &Imm)
getFP64Imm - Return an 8-bit floating-point version of the 64-bit floating-point value.
static AArch64_AM::ShiftExtendType getExtendType(unsigned Imm)
getExtendType - Extract the extend type for operands of arithmetic ops.
CCState - This class holds information needed while lowering arguments and return values...
This is the shared class of boolean and integer constants.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
1 1 0 0 True if unordered or less than
Type * getType() const
All values are typed, get the type of this value.
Provides information about what library functions are available for the current target.
CCValAssign - Represent assignment of one arg/retval to a location.
Value * getLength() const
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Function * getCalledFunction() const
getCalledFunction - Return the function called, or null if this is an indirect function invocation...
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
bool is64BitVector() const
is64BitVector - Return true if this is a 64-bit vector type.
const MachineInstrBuilder & addFrameIndex(int Idx) const
bool isNullValue() const
isNullValue - Return true if this is the value that would be returned by getNullValue.
static MachinePointerInfo getStack(int64_t Offset)
getStack - stack pointer relative access.
static cl::opt< AlignMode > Align(cl::desc("Load/store alignment support"), cl::Hidden, cl::init(NoStrictAlign), cl::values(clEnumValN(StrictAlign,"aarch64-strict-align","Disallow all unaligned memory accesses"), clEnumValN(NoStrictAlign,"aarch64-no-strict-align","Allow unaligned memory accesses"), clEnumValEnd))
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
AttributeSet getAttributes() const
Return the attribute list for this Function.
bool is128BitVector() const
is128BitVector - Return true if this is a 128-bit vector type.
Value * getArgOperand(unsigned i) const
getArgOperand/setArgOperand - Return/set the i-th call argument.
bool selectBitCast(const User *I)
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
Target - Wrapper for Target specific information.
Class for arbitrary precision integers.
static unsigned getReg(const void *D, unsigned RC, unsigned RegNo)
bool isIntegerTy() const
isIntegerTy - True if this is an instance of IntegerType.
This file defines the FastISel class.
bool isPowerOf2_64(uint64_t Value)
isPowerOf2_64 - This function returns true if the argument is a power of two 0 (64 bit edition...
APInt bitcastToAPInt() const
LLVM_ATTRIBUTE_UNUSED_RESULT std::enable_if< !is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
static unsigned getArithExtendImm(AArch64_AM::ShiftExtendType ET, unsigned Imm)
getArithExtendImm - Encode the extend type and shift amount for an arithmetic instruction: imm: 3-bit...
APInt And(const APInt &LHS, const APInt &RHS)
Bitwise AND function for APInt.
const MachineInstrBuilder & addGlobalAddress(const GlobalValue *GV, int64_t Offset=0, unsigned char TargetFlags=0) const
bool isStructTy() const
isStructTy - True if this is an instance of StructType.
Representation of each machine instruction.
bool isNegative() const
isNegative - Return true if the sign bit is set.
PointerType * getType() const
Global values are always pointers.
Value * getCondition() const
Bitwise operators - logical and, logical or, logical xor.
unsigned greater or equal
static unsigned UseReg(const MachineOperand &MO)
const MachineInstrBuilder & addConstantPoolIndex(unsigned Idx, int Offset=0, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
bool hasOneUse() const
Return true if there is exactly one user of this value.
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
MO_PAGEOFF - A symbol operand with this flag represents the offset of that symbol within a 4K page...
0 1 1 0 True if ordered and operands are unequal
iterator find(const KeyT &Val)
1 0 1 0 True if unordered or greater than
static EVT getEVT(Type *Ty, bool HandleUnknown=false)
getEVT - Return the value type corresponding to the specified type.
const APFloat & getValueAPF() const
unsigned getReg() const
getReg - Returns the register number.
iterator_range< op_iterator > arg_operands()
arg_operands - iteration adapter for range-for loops.
bool isUnsigned() const
Determine if this instruction is using an unsigned comparison.
const ARM::ArchExtKind Kind
bool isSimple() const
isSimple - Test if the given EVT is simple (as opposed to being extended).
0 0 0 1 True if ordered and equal
LLVM Value Representation.
1 0 1 1 True if unordered, greater than, or equal
unsigned getOpcode() const
getOpcode() returns a member of one of the enums like Instruction::Add.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned char TargetFlags=0) const
unsigned getDestAddressSpace() const
static const unsigned FramePtr
C - The default llvm calling convention, compatible with C.
const Value * getFalseValue() const
MO_NC - Indicates whether the linker is expected to check the symbol reference for overflow...
SUBREG_TO_REG - This instruction is similar to INSERT_SUBREG except that the first operand is an imme...
unsigned getLocMemOffset() const
TRUNCATE - Completely drop the high bits.
const MachineInstrBuilder & addRegMask(const uint32_t *Mask) const
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
0 0 1 1 True if ordered and greater than or equal
const MachineInstrBuilder & addReg(unsigned RegNo, unsigned flags=0, unsigned SubReg=0) const
addReg - Add a new virtual register operand...
bool isVarArg() const
isVarArg - Return true if this function takes a variable number of arguments.
const BasicBlock * getParent() const
MVT getSimpleVT() const
getSimpleVT - Return the SimpleValueType held in the specified simple EVT.
iterator_range< arg_iterator > args()
0 0 0 0 Always false (always folded)
IntrinsicInst - A useful wrapper class for inspecting calls to intrinsic functions.
AllocaInst - an instruction to allocate memory on the stack.
static bool isMulPowOf2(const Value *I)
Check if the multiply is by a power-of-2 constant.
gep_type_iterator gep_type_begin(const User *GEP)
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
unsigned getVectorNumElements() const
getVectorNumElements - Given a vector type, return the number of elements it contains.