48 class X86FastISel final :
public FastISel {
65 X86ScalarSSEf64 = Subtarget->
hasSSE2();
66 X86ScalarSSEf32 = Subtarget->hasSSE1();
69 bool fastSelectInstruction(
const Instruction *
I)
override;
78 bool fastLowerArguments()
override;
79 bool fastLowerCall(CallLoweringInfo &CLI)
override;
80 bool fastLowerIntrinsicCall(
const IntrinsicInst *II)
override;
82 #include "X86GenFastISel.inc"
88 unsigned &ResultReg,
unsigned Alignment = 1);
92 bool X86FastEmitStore(
EVT VT,
unsigned ValReg,
bool ValIsKill,
128 bool X86SelectFPExtOrFPTrunc(
const Instruction *
I,
unsigned Opc,
136 return Subtarget->getInstrInfo();
147 unsigned fastMaterializeConstant(
const Constant *
C)
override;
149 unsigned fastMaterializeAlloca(
const AllocaInst *
C)
override;
151 unsigned fastMaterializeFloatZero(
const ConstantFP *CF)
override;
155 bool isScalarFPTypeInSSEReg(
EVT VT)
const {
156 return (VT ==
MVT::f64 && X86ScalarSSEf64) ||
157 (VT ==
MVT::f32 && X86ScalarSSEf32);
160 bool isTypeLegal(
Type *Ty,
MVT &VT,
bool AllowI1 =
false);
162 bool IsMemcpySmall(uint64_t Len);
176 static std::pair<X86::CondCode, bool>
179 bool NeedSwap =
false;
211 return std::make_pair(CC, NeedSwap);
214 static std::pair<unsigned, bool>
217 bool NeedSwap =
false;
246 return std::make_pair(CC, NeedSwap);
267 if (!isa<ExtractValueInst>(Cond))
270 const auto *EV = cast<ExtractValueInst>(Cond);
271 if (!isa<IntrinsicInst>(EV->getAggregateOperand()))
274 const auto *II = cast<IntrinsicInst>(EV->getAggregateOperand());
276 const Function *Callee = II->getCalledFunction();
278 cast<StructType>(Callee->
getReturnType())->getTypeAtIndex(0U);
279 if (!isTypeLegal(RetTy, RetVT))
286 switch (II->getIntrinsicID()) {
287 default:
return false;
288 case Intrinsic::sadd_with_overflow:
289 case Intrinsic::ssub_with_overflow:
290 case Intrinsic::smul_with_overflow:
291 case Intrinsic::umul_with_overflow: TmpCC =
X86::COND_O;
break;
292 case Intrinsic::uadd_with_overflow:
293 case Intrinsic::usub_with_overflow: TmpCC =
X86::COND_B;
break;
303 for (
auto Itr = std::prev(Start); Itr != End; --Itr) {
306 if (!isa<ExtractValueInst>(Itr))
310 const auto *EVI = cast<ExtractValueInst>(Itr);
311 if (EVI->getAggregateOperand() != II)
319 bool X86FastISel::isTypeLegal(
Type *Ty,
MVT &VT,
bool AllowI1) {
320 EVT evt = TLI.getValueType(DL, Ty,
true);
328 if (VT ==
MVT::f64 && !X86ScalarSSEf64)
330 if (VT ==
MVT::f32 && !X86ScalarSSEf32)
339 return (AllowI1 && VT ==
MVT::i1) || TLI.isTypeLegal(VT);
342 #include "X86GenCallingConv.inc"
349 unsigned Alignment) {
354 default:
return false;
358 RC = &X86::GR8RegClass;
362 RC = &X86::GR16RegClass;
366 RC = &X86::GR32RegClass;
371 RC = &X86::GR64RegClass;
374 if (X86ScalarSSEf32) {
375 Opc = Subtarget->hasAVX() ? X86::VMOVSSrm : X86::MOVSSrm;
376 RC = &X86::FR32RegClass;
379 RC = &X86::RFP32RegClass;
383 if (X86ScalarSSEf64) {
384 Opc = Subtarget->hasAVX() ? X86::VMOVSDrm : X86::MOVSDrm;
385 RC = &X86::FR64RegClass;
388 RC = &X86::RFP64RegClass;
396 Opc = Subtarget->hasAVX() ? X86::VMOVAPSrm : X86::MOVAPSrm;
398 Opc = Subtarget->hasAVX() ? X86::VMOVUPSrm : X86::MOVUPSrm;
399 RC = &X86::VR128RegClass;
403 Opc = Subtarget->hasAVX() ? X86::VMOVAPDrm : X86::MOVAPDrm;
405 Opc = Subtarget->hasAVX() ? X86::VMOVUPDrm : X86::MOVUPDrm;
406 RC = &X86::VR128RegClass;
413 Opc = Subtarget->hasAVX() ? X86::VMOVDQArm : X86::MOVDQArm;
415 Opc = Subtarget->hasAVX() ? X86::VMOVDQUrm : X86::MOVDQUrm;
416 RC = &X86::VR128RegClass;
420 ResultReg = createResultReg(RC);
422 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(Opc), ResultReg);
433 bool X86FastISel::X86FastEmitStore(
EVT VT,
unsigned ValReg,
bool ValIsKill,
440 default:
return false;
443 unsigned AndResult = createResultReg(&X86::GR8RegClass);
444 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
445 TII.get(X86::AND8ri), AndResult)
450 case MVT::i8: Opc = X86::MOV8mr;
break;
451 case MVT::i16: Opc = X86::MOV16mr;
break;
452 case MVT::i32: Opc = X86::MOV32mr;
break;
453 case MVT::i64: Opc = X86::MOV64mr;
break;
455 Opc = X86ScalarSSEf32 ?
456 (Subtarget->hasAVX() ? X86::VMOVSSmr : X86::MOVSSmr) : X86::ST_Fp32m;
459 Opc = X86ScalarSSEf64 ?
460 (Subtarget->hasAVX() ? X86::VMOVSDmr : X86::MOVSDmr) : X86::ST_Fp64m;
464 Opc = Subtarget->hasAVX() ? X86::VMOVAPSmr : X86::MOVAPSmr;
466 Opc = Subtarget->hasAVX() ? X86::VMOVUPSmr : X86::MOVUPSmr;
470 Opc = Subtarget->hasAVX() ? X86::VMOVAPDmr : X86::MOVAPDmr;
472 Opc = Subtarget->hasAVX() ? X86::VMOVUPDmr : X86::MOVUPDmr;
479 Opc = Subtarget->hasAVX() ? X86::VMOVDQAmr : X86::MOVDQAmr;
481 Opc = Subtarget->hasAVX() ? X86::VMOVDQUmr : X86::MOVDQUmr;
486 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(Opc));
494 bool X86FastISel::X86FastEmitStore(
EVT VT,
const Value *Val,
498 if (isa<ConstantPointerNull>(Val))
502 if (
const ConstantInt *CI = dyn_cast<ConstantInt>(Val)) {
508 case MVT::i8: Opc = X86::MOV8mi;
break;
509 case MVT::i16: Opc = X86::MOV16mi;
break;
510 case MVT::i32: Opc = X86::MOV32mi;
break;
514 Opc = X86::MOV64mi32;
520 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(Opc));
522 : CI->getZExtValue());
529 unsigned ValReg = getRegForValue(Val);
533 bool ValKill = hasTrivialKill(Val);
534 return X86FastEmitStore(VT, ValReg, ValKill, AM, MMO, Aligned);
541 unsigned Src,
EVT SrcVT,
542 unsigned &ResultReg) {
554 if (
const GlobalValue *GV = dyn_cast<GlobalValue>(V)) {
560 if (GV->isThreadLocal())
566 if (!Subtarget->isPICStyleRIPRel() ||
572 unsigned char GVFlags = Subtarget->ClassifyGlobalReference(GV,
TM);
577 AM.
Base.
Reg = getInstrInfo()->getGlobalBaseReg(FuncInfo.MF);
583 if (Subtarget->isPICStyleRIPRel()) {
597 if (I != LocalValueMap.
end() && I->second != 0) {
609 SavePoint SaveInsertPt = enterLocalValueArea();
611 if (TLI.getPointerTy(DL) ==
MVT::i64) {
613 RC = &X86::GR64RegClass;
615 if (Subtarget->isPICStyleRIPRel())
619 RC = &X86::GR32RegClass;
622 LoadReg = createResultReg(RC);
624 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(Opc), LoadReg);
628 leaveLocalValueArea(SaveInsertPt);
631 LocalValueMap[V] = LoadReg;
643 if (!AM.
GV || !Subtarget->isPICStyleRIPRel()) {
645 AM.
Base.
Reg = getRegForValue(V);
649 assert(AM.
Scale == 1 &&
"Scale with no index!");
663 const User *U =
nullptr;
664 unsigned Opcode = Instruction::UserOp1;
665 if (
const Instruction *I = dyn_cast<Instruction>(V)) {
669 if (FuncInfo.StaticAllocaMap.count(static_cast<const AllocaInst *>(V)) ||
670 FuncInfo.MBBMap[I->getParent()] == FuncInfo.MBB) {
671 Opcode = I->getOpcode();
674 }
else if (
const ConstantExpr *C = dyn_cast<ConstantExpr>(V)) {
675 Opcode =
C->getOpcode();
680 if (Ty->getAddressSpace() > 255)
687 case Instruction::BitCast:
689 return X86SelectAddress(U->
getOperand(0), AM);
691 case Instruction::IntToPtr:
694 TLI.getPointerTy(DL))
695 return X86SelectAddress(U->
getOperand(0), AM);
698 case Instruction::PtrToInt:
700 if (TLI.getValueType(DL, U->
getType()) == TLI.getPointerTy(DL))
701 return X86SelectAddress(U->
getOperand(0), AM);
704 case Instruction::Alloca: {
708 FuncInfo.StaticAllocaMap.find(A);
709 if (SI != FuncInfo.StaticAllocaMap.end()) {
717 case Instruction::Add: {
720 uint64_t Disp = (int32_t)AM.
Disp + (uint64_t)CI->getSExtValue();
723 AM.
Disp = (uint32_t)Disp;
724 return X86SelectAddress(U->
getOperand(0), AM);
730 case Instruction::GetElementPtr: {
734 uint64_t Disp = (int32_t)AM.
Disp;
736 unsigned Scale = AM.
Scale;
741 i != e; ++i, ++GTI) {
742 const Value *Op = *i;
743 if (
StructType *STy = dyn_cast<StructType>(*GTI)) {
751 uint64_t S =
DL.getTypeAllocSize(GTI.getIndexedType());
753 if (
const ConstantInt *CI = dyn_cast<ConstantInt>(Op)) {
755 Disp += CI->getSExtValue() * S;
758 if (canFoldAddIntoGEP(U, Op)) {
761 cast<ConstantInt>(cast<AddOperator>(Op)->getOperand(1));
764 Op = cast<AddOperator>(Op)->getOperand(0);
768 (!AM.
GV || !Subtarget->isPICStyleRIPRel()) &&
769 (S == 1 || S == 2 || S == 4 || S == 8)) {
772 IndexReg = getRegForGEPIndex(Op).first;
778 goto unsupported_gep;
788 AM.
Disp = (uint32_t)Disp;
792 dyn_cast<GetElementPtrInst>(U->
getOperand(0))) {
797 }
else if (X86SelectAddress(U->
getOperand(0), AM)) {
807 if (handleConstantAddresses(*I, AM))
817 return handleConstantAddresses(V, AM);
823 const User *U =
nullptr;
824 unsigned Opcode = Instruction::UserOp1;
853 InMBB = I->
getParent() == FuncInfo.MBB->getBasicBlock();
854 }
else if (
const ConstantExpr *C = dyn_cast<ConstantExpr>(V)) {
855 Opcode =
C->getOpcode();
861 case Instruction::BitCast:
864 return X86SelectCallAddress(U->
getOperand(0), AM);
867 case Instruction::IntToPtr:
871 TLI.getPointerTy(DL))
872 return X86SelectCallAddress(U->
getOperand(0), AM);
875 case Instruction::PtrToInt:
877 if (InMBB && TLI.getValueType(DL, U->
getType()) == TLI.getPointerTy(DL))
878 return X86SelectCallAddress(U->
getOperand(0), AM);
883 if (
const GlobalValue *GV = dyn_cast<GlobalValue>(V)) {
889 if (Subtarget->isPICStyleRIPRel() &&
894 if (GV->hasDLLImportStorageClass())
899 if (GVar->isThreadLocal())
907 if (Subtarget->isPICStyleRIPRel()) {
912 }
else if (Subtarget->isPICStyleStubPIC()) {
914 }
else if (Subtarget->isPICStyleGOT()) {
922 if (!AM.
GV || !Subtarget->isPICStyleRIPRel()) {
924 AM.
Base.
Reg = getRegForValue(V);
928 assert(AM.
Scale == 1 &&
"Scale with no index!");
939 bool X86FastISel::X86SelectStore(
const Instruction *I) {
950 if (!isTypeLegal(Val->
getType(), VT,
true))
954 unsigned ABIAlignment =
DL.getABITypeAlignment(Val->
getType());
956 Alignment = ABIAlignment;
957 bool Aligned = Alignment >= ABIAlignment;
960 if (!X86SelectAddress(Ptr, AM))
963 return X86FastEmitStore(VT, Val, AM, createMachineMemOperandFor(I), Aligned);
967 bool X86FastISel::X86SelectRet(
const Instruction *I) {
973 if (!FuncInfo.CanLowerReturn)
983 if (Subtarget->isCallingConvWin64(CC))
1012 unsigned Reg = getRegForValue(RV);
1017 if (ValLocs.size() != 1)
1034 unsigned SrcReg = Reg + VA.
getValNo();
1035 EVT SrcVT = TLI.getValueType(DL, RV->
getType());
1038 if (SrcVT != DstVT) {
1042 if (!Outs[0].
Flags.isZExt() && !Outs[0].Flags.isSExt())
1045 assert(DstVT ==
MVT::i32 &&
"X86 should always ext to i32");
1048 if (Outs[0].
Flags.isSExt())
1050 SrcReg = fastEmitZExtFromI1(
MVT::i8, SrcReg,
false);
1065 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1076 if (F.hasStructRetAttr() &&
1077 (Subtarget->is64Bit() || Subtarget->isTargetKnownWindowsMSVC())) {
1080 "SRetReturnReg should have been set in LowerFormalArguments()!");
1081 unsigned RetReg = Subtarget->is64Bit() ? X86::RAX : X86::EAX;
1082 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1089 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1090 TII.get(Subtarget->is64Bit() ? X86::RETQ : X86::RETL));
1091 for (
unsigned i = 0, e = RetRegs.
size(); i != e; ++i)
1098 bool X86FastISel::X86SelectLoad(
const Instruction *I) {
1106 if (!isTypeLegal(LI->
getType(), VT,
true))
1112 if (!X86SelectAddress(Ptr, AM))
1116 unsigned ABIAlignment =
DL.getABITypeAlignment(LI->
getType());
1118 Alignment = ABIAlignment;
1120 unsigned ResultReg = 0;
1121 if (!X86FastEmitLoad(VT, AM, createMachineMemOperandFor(LI), ResultReg,
1125 updateValueMap(I, ResultReg);
1130 bool HasAVX = Subtarget->
hasAVX();
1131 bool X86ScalarSSEf32 = Subtarget->
hasSSE1();
1132 bool X86ScalarSSEf64 = Subtarget->
hasSSE2();
1136 case MVT::i8:
return X86::CMP8rr;
1137 case MVT::i16:
return X86::CMP16rr;
1138 case MVT::i32:
return X86::CMP32rr;
1139 case MVT::i64:
return X86::CMP64rr;
1141 return X86ScalarSSEf32 ? (HasAVX ? X86::VUCOMISSrr : X86::UCOMISSrr) : 0;
1143 return X86ScalarSSEf64 ? (HasAVX ? X86::VUCOMISDrr : X86::UCOMISDrr) : 0;
1159 return X86::CMP16ri8;
1160 return X86::CMP16ri;
1163 return X86::CMP32ri8;
1164 return X86::CMP32ri;
1167 return X86::CMP64ri8;
1171 return X86::CMP64ri32;
1176 bool X86FastISel::X86FastEmitCompare(
const Value *Op0,
const Value *Op1,
1178 unsigned Op0Reg = getRegForValue(Op0);
1179 if (Op0Reg == 0)
return false;
1182 if (isa<ConstantPointerNull>(Op1))
1188 if (
const ConstantInt *Op1C = dyn_cast<ConstantInt>(Op1)) {
1190 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, CurDbgLoc,
TII.get(CompareImmOpc))
1192 .
addImm(Op1C->getSExtValue());
1198 if (CompareOpc == 0)
return false;
1200 unsigned Op1Reg = getRegForValue(Op1);
1201 if (Op1Reg == 0)
return false;
1202 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, CurDbgLoc,
TII.get(CompareOpc))
1209 bool X86FastISel::X86SelectCmp(
const Instruction *I) {
1210 const CmpInst *CI = cast<CmpInst>(
I);
1218 unsigned ResultReg = 0;
1219 switch (Predicate) {
1222 ResultReg = createResultReg(&X86::GR32RegClass);
1223 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(X86::MOV32r0),
1225 ResultReg = fastEmitInst_extractsubreg(
MVT::i8, ResultReg,
true,
1232 ResultReg = createResultReg(&X86::GR8RegClass);
1233 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(X86::MOV8ri),
1234 ResultReg).addImm(1);
1240 updateValueMap(I, ResultReg);
1252 if (RHSC && RHSC->isNullValue())
1257 static unsigned SETFOpcTable[2][3] = {
1258 { X86::SETEr, X86::SETNPr, X86::AND8rr },
1259 { X86::SETNEr, X86::SETPr, X86::OR8rr }
1261 unsigned *SETFOpc =
nullptr;
1262 switch (Predicate) {
1268 ResultReg = createResultReg(&X86::GR8RegClass);
1270 if (!X86FastEmitCompare(LHS, RHS, VT, I->
getDebugLoc()))
1273 unsigned FlagReg1 = createResultReg(&X86::GR8RegClass);
1274 unsigned FlagReg2 = createResultReg(&X86::GR8RegClass);
1275 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(SETFOpc[0]),
1277 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(SETFOpc[1]),
1279 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(SETFOpc[2]),
1280 ResultReg).addReg(FlagReg1).
addReg(FlagReg2);
1281 updateValueMap(I, ResultReg);
1295 if (!X86FastEmitCompare(LHS, RHS, VT, I->
getDebugLoc()))
1298 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(Opc), ResultReg);
1299 updateValueMap(I, ResultReg);
1303 bool X86FastISel::X86SelectZExt(
const Instruction *I) {
1304 EVT DstVT = TLI.getValueType(DL, I->
getType());
1305 if (!TLI.isTypeLegal(DstVT))
1308 unsigned ResultReg = getRegForValue(I->
getOperand(0));
1316 ResultReg = fastEmitZExtFromI1(
MVT::i8, ResultReg,
false);
1328 case MVT::i8: MovInst = X86::MOVZX32rr8;
break;
1329 case MVT::i16: MovInst = X86::MOVZX32rr16;
break;
1330 case MVT::i32: MovInst = X86::MOV32rr;
break;
1334 unsigned Result32 = createResultReg(&X86::GR32RegClass);
1335 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(MovInst), Result32)
1338 ResultReg = createResultReg(&X86::GR64RegClass);
1342 }
else if (DstVT !=
MVT::i8) {
1349 updateValueMap(I, ResultReg);
1353 bool X86FastISel::X86SelectBranch(
const Instruction *I) {
1370 switch (Predicate) {
1385 if (CmpRHSC && CmpRHSC->isNullValue())
1390 if (FuncInfo.MBB->isLayoutSuccessor(TrueMBB)) {
1400 bool NeedExtraBranch =
false;
1401 switch (Predicate) {
1406 NeedExtraBranch =
true;
1421 if (!X86FastEmitCompare(CmpLHS, CmpRHS, VT, CI->
getDebugLoc()))
1424 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(BranchOpc))
1429 if (NeedExtraBranch) {
1430 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(X86::JP_1))
1435 uint32_t BranchWeight = 0;
1437 BranchWeight = FuncInfo.BPI->getEdgeWeight(BI->
getParent(),
1438 TrueMBB->getBasicBlock());
1439 FuncInfo.MBB->addSuccessor(TrueMBB, BranchWeight);
1443 fastEmitBranch(FalseMBB, DbgLoc);
1451 if (TI->hasOneUse() && TI->getParent() == I->
getParent() &&
1452 isTypeLegal(TI->getOperand(0)->getType(), SourceVT)) {
1453 unsigned TestOpc = 0;
1456 case MVT::i8: TestOpc = X86::TEST8ri;
break;
1457 case MVT::i16: TestOpc = X86::TEST16ri;
break;
1458 case MVT::i32: TestOpc = X86::TEST32ri;
break;
1459 case MVT::i64: TestOpc = X86::TEST64ri32;
break;
1462 unsigned OpReg = getRegForValue(TI->getOperand(0));
1463 if (OpReg == 0)
return false;
1464 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(TestOpc))
1465 .addReg(OpReg).
addImm(1);
1467 unsigned JmpOpc = X86::JNE_1;
1468 if (FuncInfo.MBB->isLayoutSuccessor(TrueMBB)) {
1473 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(JmpOpc))
1475 fastEmitBranch(FalseMBB, DbgLoc);
1476 uint32_t BranchWeight = 0;
1478 BranchWeight = FuncInfo.BPI->getEdgeWeight(BI->
getParent(),
1479 TrueMBB->getBasicBlock());
1480 FuncInfo.MBB->addSuccessor(TrueMBB, BranchWeight);
1484 }
else if (foldX86XALUIntrinsic(CC, BI, BI->
getCondition())) {
1493 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(BranchOpc))
1495 fastEmitBranch(FalseMBB, DbgLoc);
1496 uint32_t BranchWeight = 0;
1498 BranchWeight = FuncInfo.BPI->getEdgeWeight(BI->
getParent(),
1499 TrueMBB->getBasicBlock());
1500 FuncInfo.MBB->addSuccessor(TrueMBB, BranchWeight);
1508 if (OpReg == 0)
return false;
1510 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(X86::TEST8ri))
1511 .addReg(OpReg).
addImm(1);
1512 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(X86::JNE_1))
1514 fastEmitBranch(FalseMBB, DbgLoc);
1515 uint32_t BranchWeight = 0;
1517 BranchWeight = FuncInfo.BPI->getEdgeWeight(BI->
getParent(),
1518 TrueMBB->getBasicBlock());
1519 FuncInfo.MBB->addSuccessor(TrueMBB, BranchWeight);
1523 bool X86FastISel::X86SelectShift(
const Instruction *I) {
1524 unsigned CReg = 0, OpReg = 0;
1528 RC = &X86::GR8RegClass;
1530 case Instruction::LShr: OpReg = X86::SHR8rCL;
break;
1531 case Instruction::AShr: OpReg = X86::SAR8rCL;
break;
1532 case Instruction::Shl: OpReg = X86::SHL8rCL;
break;
1533 default:
return false;
1537 RC = &X86::GR16RegClass;
1539 case Instruction::LShr: OpReg = X86::SHR16rCL;
break;
1540 case Instruction::AShr: OpReg = X86::SAR16rCL;
break;
1541 case Instruction::Shl: OpReg = X86::SHL16rCL;
break;
1542 default:
return false;
1546 RC = &X86::GR32RegClass;
1548 case Instruction::LShr: OpReg = X86::SHR32rCL;
break;
1549 case Instruction::AShr: OpReg = X86::SAR32rCL;
break;
1550 case Instruction::Shl: OpReg = X86::SHL32rCL;
break;
1551 default:
return false;
1555 RC = &X86::GR64RegClass;
1557 case Instruction::LShr: OpReg = X86::SHR64rCL;
break;
1558 case Instruction::AShr: OpReg = X86::SAR64rCL;
break;
1559 case Instruction::Shl: OpReg = X86::SHL64rCL;
break;
1560 default:
return false;
1567 if (!isTypeLegal(I->
getType(), VT))
1570 unsigned Op0Reg = getRegForValue(I->
getOperand(0));
1571 if (Op0Reg == 0)
return false;
1573 unsigned Op1Reg = getRegForValue(I->
getOperand(1));
1574 if (Op1Reg == 0)
return false;
1576 CReg).addReg(Op1Reg);
1580 if (CReg != X86::CL)
1581 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1585 unsigned ResultReg = createResultReg(RC);
1586 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(OpReg), ResultReg)
1588 updateValueMap(I, ResultReg);
1592 bool X86FastISel::X86SelectDivRem(
const Instruction *I) {
1593 const static unsigned NumTypes = 4;
1594 const static unsigned NumOps = 4;
1595 const static bool S =
true;
1596 const static bool U =
false;
1607 const static struct DivRemEntry {
1613 struct DivRemResult {
1615 unsigned OpSignExtend;
1619 unsigned DivRemResultReg;
1621 } ResultTable[NumOps];
1622 } OpTable[NumTypes] = {
1623 { &X86::GR8RegClass, X86::AX, 0, {
1624 { X86::IDIV8r, 0, X86::MOVSX16rr8, X86::AL, S },
1625 { X86::IDIV8r, 0, X86::MOVSX16rr8, X86::AH, S },
1626 { X86::DIV8r, 0, X86::MOVZX16rr8, X86::AL, U },
1627 { X86::DIV8r, 0, X86::MOVZX16rr8, X86::AH, U },
1630 { &X86::GR16RegClass, X86::AX, X86::DX, {
1631 { X86::IDIV16r, X86::CWD, Copy, X86::AX, S },
1632 { X86::IDIV16r, X86::CWD, Copy, X86::DX, S },
1633 { X86::DIV16r, X86::MOV32r0, Copy, X86::AX, U },
1634 { X86::DIV16r, X86::MOV32r0, Copy, X86::DX, U },
1637 { &X86::GR32RegClass, X86::EAX, X86::EDX, {
1638 { X86::IDIV32r, X86::CDQ, Copy, X86::EAX, S },
1639 { X86::IDIV32r, X86::CDQ, Copy, X86::EDX, S },
1640 { X86::DIV32r, X86::MOV32r0, Copy, X86::EAX, U },
1641 { X86::DIV32r, X86::MOV32r0, Copy, X86::EDX, U },
1644 { &X86::GR64RegClass, X86::RAX, X86::RDX, {
1645 { X86::IDIV64r, X86::CQO, Copy, X86::RAX, S },
1646 { X86::IDIV64r, X86::CQO, Copy, X86::RDX, S },
1647 { X86::DIV64r, X86::MOV32r0, Copy, X86::RAX, U },
1648 { X86::DIV64r, X86::MOV32r0, Copy, X86::RDX, U },
1654 if (!isTypeLegal(I->
getType(), VT))
1657 unsigned TypeIndex, OpIndex;
1659 default:
return false;
1660 case MVT::i8: TypeIndex = 0;
break;
1661 case MVT::i16: TypeIndex = 1;
break;
1662 case MVT::i32: TypeIndex = 2;
break;
1664 if (!Subtarget->is64Bit())
1671 case Instruction::SDiv: OpIndex = 0;
break;
1672 case Instruction::SRem: OpIndex = 1;
break;
1673 case Instruction::UDiv: OpIndex = 2;
break;
1674 case Instruction::URem: OpIndex = 3;
break;
1677 const DivRemEntry &TypeEntry = OpTable[TypeIndex];
1678 const DivRemEntry::DivRemResult &OpEntry = TypeEntry.ResultTable[OpIndex];
1679 unsigned Op0Reg = getRegForValue(I->
getOperand(0));
1682 unsigned Op1Reg = getRegForValue(I->
getOperand(1));
1687 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1688 TII.get(OpEntry.OpCopy), TypeEntry.LowInReg).addReg(Op0Reg);
1690 if (OpEntry.OpSignExtend) {
1691 if (OpEntry.IsOpSigned)
1692 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1693 TII.get(OpEntry.OpSignExtend));
1695 unsigned Zero32 = createResultReg(&X86::GR32RegClass);
1696 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1697 TII.get(X86::MOV32r0), Zero32);
1703 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1704 TII.get(Copy), TypeEntry.HighInReg)
1705 .addReg(Zero32, 0, X86::sub_16bit);
1707 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1708 TII.get(Copy), TypeEntry.HighInReg)
1711 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1718 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1719 TII.get(OpEntry.OpDivRem)).addReg(Op1Reg);
1728 unsigned ResultReg = 0;
1729 if ((I->
getOpcode() == Instruction::SRem ||
1731 OpEntry.DivRemResultReg == X86::AH && Subtarget->is64Bit()) {
1732 unsigned SourceSuperReg = createResultReg(&X86::GR16RegClass);
1733 unsigned ResultSuperReg = createResultReg(&X86::GR16RegClass);
1734 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1735 TII.get(Copy), SourceSuperReg).addReg(X86::AX);
1738 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(X86::SHR16ri),
1739 ResultSuperReg).addReg(SourceSuperReg).
addImm(8);
1742 ResultReg = fastEmitInst_extractsubreg(
MVT::i8, ResultSuperReg,
1743 true, X86::sub_8bit);
1747 ResultReg = createResultReg(TypeEntry.RC);
1748 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(Copy), ResultReg)
1749 .addReg(OpEntry.DivRemResultReg);
1751 updateValueMap(I, ResultReg);
1758 bool X86FastISel::X86FastEmitCMoveSelect(
MVT RetVT,
const Instruction *I) {
1760 if (!Subtarget->hasCMov())
1764 if (RetVT < MVT::i16 || RetVT >
MVT::i64)
1769 bool NeedTest =
true;
1780 static unsigned SETFOpcTable[2][3] = {
1781 { X86::SETNPr, X86::SETEr , X86::TEST8rr },
1782 { X86::SETPr, X86::SETNEr, X86::OR8rr }
1784 unsigned *SETFOpc =
nullptr;
1785 switch (Predicate) {
1788 SETFOpc = &SETFOpcTable[0][0];
1792 SETFOpc = &SETFOpcTable[1][0];
1806 EVT CmpVT = TLI.getValueType(DL, CmpLHS->
getType());
1808 if (!X86FastEmitCompare(CmpLHS, CmpRHS, CmpVT, CI->
getDebugLoc()))
1812 unsigned FlagReg1 = createResultReg(&X86::GR8RegClass);
1813 unsigned FlagReg2 = createResultReg(&X86::GR8RegClass);
1814 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(SETFOpc[0]),
1816 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(SETFOpc[1]),
1818 auto const &II =
TII.get(SETFOpc[2]);
1819 if (II.getNumDefs()) {
1820 unsigned TmpReg = createResultReg(&X86::GR8RegClass);
1821 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, TmpReg)
1824 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
1829 }
else if (foldX86XALUIntrinsic(CC, I, Cond)) {
1832 unsigned TmpReg = getRegForValue(Cond);
1845 unsigned CondReg = getRegForValue(Cond);
1848 bool CondIsKill = hasTrivialKill(Cond);
1850 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(X86::TEST8ri))
1857 unsigned RHSReg = getRegForValue(RHS);
1858 bool RHSIsKill = hasTrivialKill(RHS);
1860 unsigned LHSReg = getRegForValue(LHS);
1861 bool LHSIsKill = hasTrivialKill(LHS);
1863 if (!LHSReg || !RHSReg)
1867 unsigned ResultReg = fastEmitInst_rr(Opc, RC, RHSReg, RHSIsKill,
1869 updateValueMap(I, ResultReg);
1878 bool X86FastISel::X86FastEmitSSESelect(
MVT RetVT,
const Instruction *I) {
1887 !((Subtarget->hasSSE1() && RetVT ==
MVT::f32) ||
1888 (Subtarget->hasSSE2() && RetVT ==
MVT::f64)))
1900 if (CmpRHSC && CmpRHSC->isNullValue())
1914 static unsigned OpcTable[2][4] = {
1915 { X86::CMPSSrr, X86::FsANDPSrr, X86::FsANDNPSrr, X86::FsORPSrr },
1916 { X86::CMPSDrr, X86::FsANDPDrr, X86::FsANDNPDrr, X86::FsORPDrr }
1919 unsigned *Opc =
nullptr;
1921 default:
return false;
1922 case MVT::f32: Opc = &OpcTable[0][0];
break;
1923 case MVT::f64: Opc = &OpcTable[1][0];
break;
1929 unsigned LHSReg = getRegForValue(LHS);
1930 bool LHSIsKill = hasTrivialKill(LHS);
1932 unsigned RHSReg = getRegForValue(RHS);
1933 bool RHSIsKill = hasTrivialKill(RHS);
1935 unsigned CmpLHSReg = getRegForValue(CmpLHS);
1936 bool CmpLHSIsKill = hasTrivialKill(CmpLHS);
1938 unsigned CmpRHSReg = getRegForValue(CmpRHS);
1939 bool CmpRHSIsKill = hasTrivialKill(CmpRHS);
1941 if (!LHSReg || !RHSReg || !CmpLHS || !CmpRHS)
1947 if (Subtarget->hasAVX()) {
1953 unsigned CmpOpcode =
1955 unsigned BlendOpcode =
1958 unsigned CmpReg = fastEmitInst_rri(CmpOpcode, RC, CmpLHSReg, CmpLHSIsKill,
1959 CmpRHSReg, CmpRHSIsKill, CC);
1960 ResultReg = fastEmitInst_rrr(BlendOpcode, RC, RHSReg, RHSIsKill,
1961 LHSReg, LHSIsKill, CmpReg,
true);
1963 unsigned CmpReg = fastEmitInst_rri(Opc[0], RC, CmpLHSReg, CmpLHSIsKill,
1964 CmpRHSReg, CmpRHSIsKill, CC);
1965 unsigned AndReg = fastEmitInst_rr(Opc[1], RC, CmpReg,
false,
1967 unsigned AndNReg = fastEmitInst_rr(Opc[2], RC, CmpReg,
true,
1969 ResultReg = fastEmitInst_rr(Opc[3], RC, AndNReg,
true,
1972 updateValueMap(I, ResultReg);
1976 bool X86FastISel::X86FastEmitPseudoSelect(
MVT RetVT,
const Instruction *I) {
1981 default:
return false;
1982 case MVT::i8: Opc = X86::CMOV_GR8;
break;
1983 case MVT::i16: Opc = X86::CMOV_GR16;
break;
1984 case MVT::i32: Opc = X86::CMOV_GR32;
break;
1985 case MVT::f32: Opc = X86::CMOV_FR32;
break;
1986 case MVT::f64: Opc = X86::CMOV_FR64;
break;
2008 EVT CmpVT = TLI.getValueType(DL, CmpLHS->
getType());
2009 if (!X86FastEmitCompare(CmpLHS, CmpRHS, CmpVT, CI->
getDebugLoc()))
2012 unsigned CondReg = getRegForValue(Cond);
2015 bool CondIsKill = hasTrivialKill(Cond);
2016 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(X86::TEST8ri))
2023 unsigned LHSReg = getRegForValue(LHS);
2024 bool LHSIsKill = hasTrivialKill(LHS);
2026 unsigned RHSReg = getRegForValue(RHS);
2027 bool RHSIsKill = hasTrivialKill(RHS);
2029 if (!LHSReg || !RHSReg)
2034 unsigned ResultReg =
2035 fastEmitInst_rri(Opc, RC, RHSReg, RHSIsKill, LHSReg, LHSIsKill, CC);
2036 updateValueMap(I, ResultReg);
2040 bool X86FastISel::X86SelectSelect(
const Instruction *I) {
2042 if (!isTypeLegal(I->
getType(), RetVT))
2046 if (
const auto *CI = dyn_cast<CmpInst>(I->
getOperand(0))) {
2048 const Value *Opnd =
nullptr;
2049 switch (Predicate) {
2056 unsigned OpReg = getRegForValue(Opnd);
2059 bool OpIsKill = hasTrivialKill(Opnd);
2061 unsigned ResultReg = createResultReg(RC);
2062 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
2065 updateValueMap(I, ResultReg);
2071 if (X86FastEmitCMoveSelect(RetVT, I))
2075 if (X86FastEmitSSESelect(RetVT, I))
2080 if (X86FastEmitPseudoSelect(RetVT, I))
2086 bool X86FastISel::X86SelectSIToFP(
const Instruction *I) {
2090 if (!Subtarget->hasAVX())
2097 unsigned OpReg = getRegForValue(I->
getOperand(0));
2106 Opcode = X86::VCVTSI2SDrr;
2107 RC = &X86::FR64RegClass;
2110 Opcode = X86::VCVTSI2SSrr;
2111 RC = &X86::FR32RegClass;
2115 unsigned ImplicitDefReg = createResultReg(RC);
2116 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
2118 unsigned ResultReg =
2119 fastEmitInst_rr(Opcode, RC, ImplicitDefReg,
true, OpReg,
false);
2120 updateValueMap(I, ResultReg);
2125 bool X86FastISel::X86SelectFPExtOrFPTrunc(
const Instruction *I,
2128 assert((I->
getOpcode() == Instruction::FPExt ||
2129 I->
getOpcode() == Instruction::FPTrunc) &&
2130 "Instruction must be an FPExt or FPTrunc!");
2132 unsigned OpReg = getRegForValue(I->
getOperand(0));
2136 unsigned ResultReg = createResultReg(RC);
2138 MIB =
BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(TargetOpc),
2140 if (Subtarget->hasAVX())
2143 updateValueMap(I, ResultReg);
2147 bool X86FastISel::X86SelectFPExt(
const Instruction *I) {
2151 unsigned Opc = Subtarget->hasAVX() ? X86::VCVTSS2SDrr : X86::CVTSS2SDrr;
2152 return X86SelectFPExtOrFPTrunc(I, Opc, &X86::FR64RegClass);
2158 bool X86FastISel::X86SelectFPTrunc(
const Instruction *I) {
2162 unsigned Opc = Subtarget->hasAVX() ? X86::VCVTSD2SSrr : X86::CVTSD2SSrr;
2163 return X86SelectFPExtOrFPTrunc(I, Opc, &X86::FR32RegClass);
2169 bool X86FastISel::X86SelectTrunc(
const Instruction *I) {
2171 EVT DstVT = TLI.getValueType(DL, I->
getType());
2176 if (!TLI.isTypeLegal(SrcVT))
2179 unsigned InputReg = getRegForValue(I->
getOperand(0));
2186 updateValueMap(I, InputReg);
2190 bool KillInputReg =
false;
2191 if (!Subtarget->is64Bit()) {
2195 (SrcVT ==
MVT::i16) ? &X86::GR16_ABCDRegClass : &X86::GR32_ABCDRegClass;
2196 unsigned CopyReg = createResultReg(CopyRC);
2197 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
2200 KillInputReg =
true;
2204 unsigned ResultReg = fastEmitInst_extractsubreg(
MVT::i8,
2205 InputReg, KillInputReg,
2210 updateValueMap(I, ResultReg);
2214 bool X86FastISel::IsMemcpySmall(uint64_t Len) {
2215 return Len <= (Subtarget->is64Bit() ? 32 : 16);
2222 if (!IsMemcpySmall(Len))
2225 bool i64Legal = Subtarget->is64Bit();
2230 if (Len >= 8 && i64Legal)
2240 bool RV = X86FastEmitLoad(VT, SrcAM,
nullptr, Reg);
2241 RV &= X86FastEmitStore(VT, Reg,
true, DestAM);
2242 assert(RV &&
"Failed to emit load or store??");
2246 DestAM.
Disp += Size;
2253 bool X86FastISel::fastLowerIntrinsicCall(
const IntrinsicInst *II) {
2256 default:
return false;
2257 case Intrinsic::convert_from_fp16:
2258 case Intrinsic::convert_to_fp16: {
2259 if (Subtarget->useSoftFloat() || !Subtarget->hasF16C())
2263 unsigned InputReg = getRegForValue(Op);
2268 bool IsFloatToHalf = II->
getIntrinsicID() == Intrinsic::convert_to_fp16;
2269 if (IsFloatToHalf) {
2277 unsigned ResultReg = 0;
2279 if (IsFloatToHalf) {
2285 InputReg = fastEmitInst_ri(X86::VCVTPS2PHrr, RC, InputReg,
false, 0);
2288 ResultReg = createResultReg(&X86::GR32RegClass);
2289 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
2290 TII.get(X86::VMOVPDI2DIrr), ResultReg)
2294 unsigned RegIdx = X86::sub_16bit;
2295 ResultReg = fastEmitInst_extractsubreg(
MVT::i16, ResultReg,
true, RegIdx);
2306 InputReg = fastEmitInst_r(X86::VCVTPH2PSrr, RC, InputReg,
true);
2310 ResultReg = createResultReg(&X86::FR32RegClass);
2311 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
2316 updateValueMap(II, ResultReg);
2319 case Intrinsic::frameaddress: {
2327 if (!isTypeLegal(RetTy, VT))
2335 case MVT::i32: Opc = X86::MOV32rm; RC = &X86::GR32RegClass;
break;
2336 case MVT::i64: Opc = X86::MOV64rm; RC = &X86::GR64RegClass;
break;
2346 assert(((FrameReg == X86::RBP && VT == MVT::i64) ||
2347 (FrameReg == X86::EBP && VT ==
MVT::i32)) &&
2348 "Invalid Frame Register!");
2353 unsigned SrcReg = createResultReg(RC);
2354 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
2363 unsigned Depth = cast<ConstantInt>(II->
getOperand(0))->getZExtValue();
2365 DestReg = createResultReg(RC);
2367 TII.get(Opc), DestReg), SrcReg);
2371 updateValueMap(II, SrcReg);
2374 case Intrinsic::memcpy: {
2375 const MemCpyInst *MCI = cast<MemCpyInst>(II);
2380 if (isa<ConstantInt>(MCI->
getLength())) {
2383 uint64_t Len = cast<ConstantInt>(MCI->
getLength())->getZExtValue();
2384 if (IsMemcpySmall(Len)) {
2386 if (!X86SelectAddress(MCI->
getRawDest(), DestAM) ||
2389 TryEmitSmallMemcpy(DestAM, SrcAM, Len);
2394 unsigned SizeWidth = Subtarget->is64Bit() ? 64 : 32;
2403 case Intrinsic::memset: {
2404 const MemSetInst *MSI = cast<MemSetInst>(II);
2409 unsigned SizeWidth = Subtarget->is64Bit() ? 64 : 32;
2418 case Intrinsic::stackprotector: {
2420 EVT PtrTy = TLI.getPointerTy(DL);
2429 if (!X86SelectAddress(Slot, AM))
return false;
2430 if (!X86FastEmitStore(PtrTy, Op1, AM))
return false;
2433 case Intrinsic::dbg_declare: {
2436 assert(DI->
getAddress() &&
"Null address should be checked earlier!");
2443 "Expected inlined-at fields to agree");
2450 case Intrinsic::trap: {
2454 case Intrinsic::sqrt: {
2455 if (!Subtarget->hasSSE1())
2458 Type *RetTy = II->getCalledFunction()->getReturnType();
2461 if (!isTypeLegal(RetTy, VT))
2467 static const unsigned SqrtOpc[2][2] = {
2468 {X86::SQRTSSr, X86::VSQRTSSr},
2469 {X86::SQRTSDr, X86::VSQRTSDr}
2471 bool HasAVX = Subtarget->hasAVX();
2475 default:
return false;
2476 case MVT::f32: Opc = SqrtOpc[0][HasAVX]; RC = &X86::FR32RegClass;
break;
2477 case MVT::f64: Opc = SqrtOpc[1][HasAVX]; RC = &X86::FR64RegClass;
break;
2480 const Value *SrcVal = II->getArgOperand(0);
2481 unsigned SrcReg = getRegForValue(SrcVal);
2486 unsigned ImplicitDefReg = 0;
2488 ImplicitDefReg = createResultReg(RC);
2489 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
2493 unsigned ResultReg = createResultReg(RC);
2495 MIB =
BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(Opc),
2499 MIB.
addReg(ImplicitDefReg);
2503 updateValueMap(II, ResultReg);
2506 case Intrinsic::sadd_with_overflow:
2507 case Intrinsic::uadd_with_overflow:
2508 case Intrinsic::ssub_with_overflow:
2509 case Intrinsic::usub_with_overflow:
2510 case Intrinsic::smul_with_overflow:
2511 case Intrinsic::umul_with_overflow: {
2514 const Function *Callee = II->getCalledFunction();
2516 Type *RetTy = Ty->getTypeAtIndex(0U);
2517 Type *CondTy = Ty->getTypeAtIndex(1);
2520 if (!isTypeLegal(RetTy, VT))
2523 if (VT < MVT::i8 || VT > MVT::i64)
2526 const Value *LHS = II->getArgOperand(0);
2527 const Value *RHS = II->getArgOperand(1);
2530 if (isa<ConstantInt>(LHS) && !isa<ConstantInt>(RHS) &&
2531 isCommutativeIntrinsic(II))
2534 bool UseIncDec =
false;
2535 if (isa<ConstantInt>(RHS) && cast<ConstantInt>(RHS)->isOne())
2538 unsigned BaseOpc, CondOpc;
2539 switch (II->getIntrinsicID()) {
2541 case Intrinsic::sadd_with_overflow:
2543 CondOpc = X86::SETOr;
2545 case Intrinsic::uadd_with_overflow:
2546 BaseOpc =
ISD::ADD; CondOpc = X86::SETBr;
break;
2547 case Intrinsic::ssub_with_overflow:
2548 BaseOpc = UseIncDec ? unsigned(
X86ISD::DEC) : unsigned(ISD::
SUB);
2549 CondOpc = X86::SETOr;
2551 case Intrinsic::usub_with_overflow:
2552 BaseOpc =
ISD::SUB; CondOpc = X86::SETBr;
break;
2553 case Intrinsic::smul_with_overflow:
2555 case Intrinsic::umul_with_overflow:
2559 unsigned LHSReg = getRegForValue(LHS);
2562 bool LHSIsKill = hasTrivialKill(LHS);
2564 unsigned ResultReg = 0;
2566 if (
const auto *CI = dyn_cast<ConstantInt>(RHS)) {
2567 static const unsigned Opc[2][4] = {
2568 { X86::INC8r, X86::INC16r, X86::INC32r, X86::INC64r },
2569 { X86::DEC8r, X86::DEC16r, X86::DEC32r, X86::DEC64r }
2573 ResultReg = createResultReg(TLI.getRegClassFor(VT));
2575 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
2579 ResultReg = fastEmit_ri(VT, VT, BaseOpc, LHSReg, LHSIsKill,
2580 CI->getZExtValue());
2586 RHSReg = getRegForValue(RHS);
2589 RHSIsKill = hasTrivialKill(RHS);
2590 ResultReg = fastEmit_rr(VT, VT, BaseOpc, LHSReg, LHSIsKill, RHSReg,
2597 static const unsigned MULOpc[] =
2598 { X86::MUL8r, X86::MUL16r, X86::MUL32r, X86::MUL64r };
2599 static const unsigned Reg[] = { X86::AL, X86::AX, X86::EAX, X86::RAX };
2602 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
2606 TLI.getRegClassFor(VT), RHSReg, RHSIsKill);
2608 static const unsigned MULOpc[] =
2609 { X86::IMUL8r, X86::IMUL16rr, X86::IMUL32rr, X86::IMUL64rr };
2613 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
2616 ResultReg = fastEmitInst_r(MULOpc[0], TLI.getRegClassFor(VT), RHSReg,
2620 TLI.getRegClassFor(VT), LHSReg, LHSIsKill,
2627 unsigned ResultReg2 = FuncInfo.CreateRegs(CondTy);
2628 assert((ResultReg+1) == ResultReg2 &&
"Nonconsecutive result registers.");
2629 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(CondOpc),
2632 updateValueMap(II, ResultReg, 2);
2635 case Intrinsic::x86_sse_cvttss2si:
2636 case Intrinsic::x86_sse_cvttss2si64:
2637 case Intrinsic::x86_sse2_cvttsd2si:
2638 case Intrinsic::x86_sse2_cvttsd2si64: {
2640 switch (II->getIntrinsicID()) {
2642 case Intrinsic::x86_sse_cvttss2si:
2643 case Intrinsic::x86_sse_cvttss2si64:
2644 if (!Subtarget->hasSSE1())
2646 IsInputDouble =
false;
2648 case Intrinsic::x86_sse2_cvttsd2si:
2649 case Intrinsic::x86_sse2_cvttsd2si64:
2650 if (!Subtarget->hasSSE2())
2652 IsInputDouble =
true;
2656 Type *RetTy = II->getCalledFunction()->getReturnType();
2658 if (!isTypeLegal(RetTy, VT))
2661 static const unsigned CvtOpc[2][2][2] = {
2662 { { X86::CVTTSS2SIrr, X86::VCVTTSS2SIrr },
2663 { X86::CVTTSS2SI64rr, X86::VCVTTSS2SI64rr } },
2664 { { X86::CVTTSD2SIrr, X86::VCVTTSD2SIrr },
2665 { X86::CVTTSD2SI64rr, X86::VCVTTSD2SI64rr } }
2667 bool HasAVX = Subtarget->hasAVX();
2671 case MVT::i32: Opc = CvtOpc[IsInputDouble][0][HasAVX];
break;
2672 case MVT::i64: Opc = CvtOpc[IsInputDouble][1][HasAVX];
break;
2676 const Value *Op = II->getArgOperand(0);
2677 while (
auto *
IE = dyn_cast<InsertElementInst>(Op)) {
2678 const Value *Index =
IE->getOperand(2);
2679 if (!isa<ConstantInt>(Index))
2681 unsigned Idx = cast<ConstantInt>(Index)->getZExtValue();
2684 Op =
IE->getOperand(1);
2687 Op =
IE->getOperand(0);
2690 unsigned Reg = getRegForValue(Op);
2694 unsigned ResultReg = createResultReg(TLI.getRegClassFor(VT));
2695 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(Opc), ResultReg)
2698 updateValueMap(II, ResultReg);
2704 bool X86FastISel::fastLowerArguments() {
2705 if (!FuncInfo.CanLowerReturn)
2716 if (Subtarget->isCallingConvWin64(CC))
2719 if (!Subtarget->is64Bit())
2723 unsigned GPRCnt = 0;
2724 unsigned FPRCnt = 0;
2726 for (
auto const &Arg : F->
args()) {
2735 Type *ArgTy = Arg.getType();
2739 EVT ArgVT = TLI.getValueType(DL, ArgTy);
2740 if (!ArgVT.
isSimple())
return false;
2742 default:
return false;
2749 if (!Subtarget->hasSSE1())
2762 static const MCPhysReg GPR32ArgRegs[] = {
2763 X86::EDI, X86::ESI, X86::EDX, X86::ECX, X86::R8D, X86::R9D
2765 static const MCPhysReg GPR64ArgRegs[] = {
2766 X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8 , X86::R9
2769 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,
2770 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7
2773 unsigned GPRIdx = 0;
2774 unsigned FPRIdx = 0;
2775 for (
auto const &Arg : F->
args()) {
2776 MVT VT = TLI.getSimpleValueType(DL, Arg.getType());
2781 case MVT::i32: SrcReg = GPR32ArgRegs[GPRIdx++];
break;
2782 case MVT::i64: SrcReg = GPR64ArgRegs[GPRIdx++];
break;
2784 case MVT::f64: SrcReg = XMMArgRegs[FPRIdx++];
break;
2786 unsigned DstReg = FuncInfo.MF->addLiveIn(SrcReg, RC);
2790 unsigned ResultReg = createResultReg(RC);
2791 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
2794 updateValueMap(&Arg, ResultReg);
2816 bool X86FastISel::fastLowerCall(CallLoweringInfo &CLI) {
2817 auto &OutVals = CLI.OutVals;
2818 auto &OutFlags = CLI.OutFlags;
2819 auto &OutRegs = CLI.OutRegs;
2820 auto &
Ins = CLI.Ins;
2821 auto &InRegs = CLI.InRegs;
2823 bool &IsTailCall = CLI.IsTailCall;
2824 bool IsVarArg = CLI.IsVarArg;
2825 const Value *Callee = CLI.Callee;
2828 bool Is64Bit = Subtarget->is64Bit();
2829 bool IsWin64 = Subtarget->isCallingConvWin64(CC);
2833 default:
return false;
2854 if (IsVarArg && IsWin64)
2858 if (CLI.CS && CLI.CS->hasInAllocaArgument())
2863 TM.Options.GuaranteedTailCallOpt))
2872 for (
int i = 0, e = OutVals.size(); i != e; ++i) {
2873 Value *&Val = OutVals[i];
2875 if (
auto *CI = dyn_cast<ConstantInt>(Val)) {
2876 if (CI->getBitWidth() < 32) {
2889 if (TI && TI->getType()->isIntegerTy(1) && CLI.CS &&
2890 (TI->getParent() == CLI.CS->getInstruction()->getParent()) &&
2892 Value *PrevVal = TI->getOperand(0);
2893 ResultReg = getRegForValue(PrevVal);
2898 if (!isTypeLegal(PrevVal->
getType(), VT))
2902 fastEmit_ri(VT, VT,
ISD::AND, ResultReg, hasTrivialKill(PrevVal), 1);
2904 if (!isTypeLegal(Val->
getType(), VT))
2906 ResultReg = getRegForValue(Val);
2918 CCState CCInfo(CC, IsVarArg, *FuncInfo.MF, ArgLocs, CLI.RetTy->getContext());
2924 CCInfo.AnalyzeCallOperands(OutVTs, OutFlags, CC_X86);
2927 unsigned NumBytes = CCInfo.getNextStackOffset();
2930 unsigned AdjStackDown =
TII.getCallFrameSetupOpcode();
2931 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(AdjStackDown))
2932 .addImm(NumBytes).
addImm(0);
2936 for (
unsigned i = 0, e = ArgLocs.
size(); i != e; ++i) {
2944 unsigned ArgReg = ArgRegs[VA.
getValNo()];
2951 "Unexpected extend");
2954 assert(Emitted &&
"Failed to emit a sext!"); (void)Emitted;
2960 "Unexpected extend");
2963 assert(Emitted &&
"Failed to emit a zext!"); (void)Emitted;
2969 "Unexpected extend");
2979 assert(Emitted &&
"Failed to emit a aext!"); (void)Emitted;
2986 assert(ArgReg &&
"Failed to emit a bitcast!");
3006 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
3013 if (isa<UndefValue>(ArgVal))
3019 AM.
Disp = LocMemOffset;
3021 unsigned Alignment =
DL.getABITypeAlignment(ArgVal->
getType());
3024 ArgVT.getStoreSize(), Alignment);
3028 if (!TryEmitSmallMemcpy(AM, SrcAM, Flags.
getByValSize()))
3030 }
else if (isa<ConstantInt>(ArgVal) || isa<ConstantPointerNull>(ArgVal)) {
3034 if (!X86FastEmitStore(ArgVT, ArgVal, AM, MMO))
3037 bool ValIsKill = hasTrivialKill(ArgVal);
3038 if (!X86FastEmitStore(ArgVT, ArgReg, ValIsKill, AM, MMO))
3046 if (Subtarget->isPICStyleGOT()) {
3047 unsigned Base = getInstrInfo()->getGlobalBaseReg(FuncInfo.MF);
3048 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
3052 if (Is64Bit && IsVarArg && !IsWin64) {
3063 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,
3064 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7
3066 unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs);
3067 assert((Subtarget->hasSSE1() || !NumXMMRegs)
3068 &&
"SSE registers cannot be used when SSE is disabled");
3069 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(X86::MOV8ri),
3070 X86::AL).addImm(NumXMMRegs);
3076 if (!X86SelectCallAddress(Callee, CalleeAM))
3079 unsigned CalleeOp = 0;
3081 if (CalleeAM.
GV !=
nullptr) {
3083 }
else if (CalleeAM.
Base.
Reg != 0) {
3092 unsigned CallOpc = Is64Bit ? X86::CALL64r : X86::CALL32r;
3093 MIB =
BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(CallOpc))
3097 assert(GV &&
"Not a direct call");
3098 unsigned CallOpc = Is64Bit ? X86::CALL64pcrel32 : X86::CALLpcrel32;
3101 unsigned char OpFlags = 0;
3107 if (Subtarget->isTargetELF() &&
3111 }
else if (Subtarget->isPICStyleStubAny() &&
3113 (!Subtarget->getTargetTriple().isMacOSX() ||
3114 Subtarget->getTargetTriple().isMacOSXVersionLT(10, 5))) {
3121 MIB =
BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(CallOpc));
3123 MIB.
addSym(Symbol, OpFlags);
3130 MIB.
addRegMask(TRI.getCallPreservedMask(*FuncInfo.MF, CC));
3133 if (Subtarget->isPICStyleGOT())
3136 if (Is64Bit && IsVarArg && !IsWin64)
3140 for (
auto Reg : OutRegs)
3144 unsigned NumBytesForCalleeToPop =
3146 unsigned AdjStackUp =
TII.getCallFrameDestroyOpcode();
3147 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(AdjStackUp))
3148 .addImm(NumBytes).
addImm(NumBytesForCalleeToPop);
3152 CCState CCRetInfo(CC, IsVarArg, *FuncInfo.MF, RVLocs,
3153 CLI.RetTy->getContext());
3157 unsigned ResultReg = FuncInfo.CreateRegs(CLI.RetTy);
3158 for (
unsigned i = 0; i != RVLocs.
size(); ++i) {
3161 unsigned CopyReg = ResultReg + i;
3165 ((Is64Bit ||
Ins[i].Flags.
isInReg()) && !Subtarget->hasSSE1())) {
3172 isScalarFPTypeInSSEReg(VA.
getValVT())) {
3174 CopyReg = createResultReg(&X86::RFP80RegClass);
3178 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
3187 unsigned Opc = ResVT ==
MVT::f32 ? X86::ST_Fp80m32 : X86::ST_Fp80m64;
3193 Opc = ResVT ==
MVT::f32 ? X86::MOVSSrm : X86::MOVSDrm;
3195 TII.get(Opc), ResultReg + i), FI);
3199 CLI.ResultReg = ResultReg;
3200 CLI.NumResultRegs = RVLocs.
size();
3207 X86FastISel::fastSelectInstruction(
const Instruction *I) {
3211 return X86SelectLoad(I);
3213 return X86SelectStore(I);
3215 return X86SelectRet(I);
3216 case Instruction::ICmp:
3217 case Instruction::FCmp:
3218 return X86SelectCmp(I);
3219 case Instruction::ZExt:
3220 return X86SelectZExt(I);
3221 case Instruction::Br:
3222 return X86SelectBranch(I);
3223 case Instruction::LShr:
3224 case Instruction::AShr:
3225 case Instruction::Shl:
3226 return X86SelectShift(I);
3227 case Instruction::SDiv:
3228 case Instruction::UDiv:
3229 case Instruction::SRem:
3230 case Instruction::URem:
3231 return X86SelectDivRem(I);
3233 return X86SelectSelect(I);
3234 case Instruction::Trunc:
3235 return X86SelectTrunc(I);
3236 case Instruction::FPExt:
3237 return X86SelectFPExt(I);
3238 case Instruction::FPTrunc:
3239 return X86SelectFPTrunc(I);
3240 case Instruction::SIToFP:
3241 return X86SelectSIToFP(I);
3242 case Instruction::IntToPtr:
3243 case Instruction::PtrToInt: {
3245 EVT DstVT = TLI.getValueType(DL, I->
getType());
3247 return X86SelectZExt(I);
3249 return X86SelectTrunc(I);
3250 unsigned Reg = getRegForValue(I->
getOperand(0));
3251 if (Reg == 0)
return false;
3252 updateValueMap(I, Reg);
3260 unsigned X86FastISel::X86MaterializeInt(
const ConstantInt *CI,
MVT VT) {
3266 unsigned SrcReg = fastEmitInst_(X86::MOV32r0, &X86::GR32RegClass);
3271 return fastEmitInst_extractsubreg(
MVT::i8, SrcReg,
true,
3274 return fastEmitInst_extractsubreg(
MVT::i16, SrcReg,
true,
3279 unsigned ResultReg = createResultReg(&X86::GR64RegClass);
3280 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
3292 case MVT::i8: Opc = X86::MOV8ri;
break;
3293 case MVT::i16: Opc = X86::MOV16ri;
break;
3294 case MVT::i32: Opc = X86::MOV32ri;
break;
3299 Opc = X86::MOV64ri32;
3305 if (VT == MVT::i64 && Opc == X86::MOV32ri) {
3306 unsigned SrcReg = fastEmitInst_i(Opc, &X86::GR32RegClass, Imm);
3307 unsigned ResultReg = createResultReg(&X86::GR64RegClass);
3308 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
3313 return fastEmitInst_i(Opc, TLI.getRegClassFor(VT), Imm);
3316 unsigned X86FastISel::X86MaterializeFP(
const ConstantFP *CFP,
MVT VT) {
3318 return fastMaterializeFloatZero(CFP);
3331 if (X86ScalarSSEf32) {
3332 Opc = Subtarget->hasAVX() ? X86::VMOVSSrm : X86::MOVSSrm;
3333 RC = &X86::FR32RegClass;
3335 Opc = X86::LD_Fp32m;
3336 RC = &X86::RFP32RegClass;
3340 if (X86ScalarSSEf64) {
3341 Opc = Subtarget->hasAVX() ? X86::VMOVSDrm : X86::MOVSDrm;
3342 RC = &X86::FR64RegClass;
3344 Opc = X86::LD_Fp64m;
3345 RC = &X86::RFP64RegClass;
3357 Align =
DL.getTypeAllocSize(CFP->
getType());
3361 unsigned PICBase = 0;
3362 unsigned char OpFlag = 0;
3363 if (Subtarget->isPICStyleStubPIC()) {
3365 PICBase = getInstrInfo()->getGlobalBaseReg(FuncInfo.MF);
3366 }
else if (Subtarget->isPICStyleGOT()) {
3368 PICBase = getInstrInfo()->getGlobalBaseReg(FuncInfo.MF);
3369 }
else if (Subtarget->isPICStyleRIPRel() &&
3375 unsigned CPI = MCP.getConstantPoolIndex(CFP, Align);
3376 unsigned ResultReg = createResultReg(RC);
3379 unsigned AddrReg = createResultReg(&X86::GR64RegClass);
3380 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(X86::MOV64ri),
3382 .addConstantPoolIndex(CPI, 0, OpFlag);
3384 TII.get(Opc), ResultReg);
3394 TII.get(Opc), ResultReg),
3395 CPI, PICBase, OpFlag);
3399 unsigned X86FastISel::X86MaterializeGV(
const GlobalValue *GV,
MVT VT) {
3406 if (X86SelectAddress(GV, AM)) {
3413 unsigned ResultReg = createResultReg(TLI.getRegClassFor(VT));
3415 TLI.getPointerTy(DL) ==
MVT::i64) {
3418 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(X86::MOV64ri),
3420 .addGlobalAddress(GV);
3424 ? (Subtarget->isTarget64BitILP32() ? X86::LEA64_32r : X86::LEA32r)
3427 TII.get(Opc), ResultReg), AM);
3434 unsigned X86FastISel::fastMaterializeConstant(
const Constant *C) {
3435 EVT CEVT = TLI.getValueType(DL, C->
getType(),
true);
3442 if (
const auto *CI = dyn_cast<ConstantInt>(C))
3443 return X86MaterializeInt(CI, VT);
3444 else if (
const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
3445 return X86MaterializeFP(CFP, VT);
3446 else if (
const GlobalValue *GV = dyn_cast<GlobalValue>(C))
3447 return X86MaterializeGV(GV, VT);
3452 unsigned X86FastISel::fastMaterializeAlloca(
const AllocaInst *C) {
3460 if (!FuncInfo.StaticAllocaMap.count(C))
3462 assert(C->
isStaticAlloca() &&
"dynamic alloca in the static alloca map?");
3465 if (!X86SelectAddress(C, AM))
3469 ? (Subtarget->isTarget64BitILP32() ? X86::LEA64_32r : X86::LEA32r)
3472 unsigned ResultReg = createResultReg(RC);
3474 TII.get(Opc), ResultReg), AM);
3478 unsigned X86FastISel::fastMaterializeFloatZero(
const ConstantFP *CF) {
3480 if (!isTypeLegal(CF->
getType(), VT))
3489 if (X86ScalarSSEf32) {
3490 Opc = X86::FsFLD0SS;
3491 RC = &X86::FR32RegClass;
3493 Opc = X86::LD_Fp032;
3494 RC = &X86::RFP32RegClass;
3498 if (X86ScalarSSEf64) {
3499 Opc = X86::FsFLD0SD;
3500 RC = &X86::FR64RegClass;
3502 Opc = X86::LD_Fp064;
3503 RC = &X86::RFP64RegClass;
3511 unsigned ResultReg = createResultReg(RC);
3512 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
TII.get(Opc), ResultReg);
3517 bool X86FastISel::tryToFoldLoadIntoMI(
MachineInstr *
MI,
unsigned OpNo,
3521 if (!X86SelectAddress(Ptr, AM))
3526 unsigned Size =
DL.getTypeAllocSize(LI->
getType());
3530 Alignment =
DL.getABITypeAlignment(LI->
getType());
3536 *FuncInfo.MF, MI, OpNo, AddrOps, FuncInfo.InsertPt, Size, Alignment,
3546 unsigned OperandNo = 0;
3553 unsigned IndexReg = constrainOperandRegClass(Result->
getDesc(),
3555 if (IndexReg == MO.
getReg())
3560 Result->
addMemOperand(*FuncInfo.MF, createMachineMemOperandFor(LI));
3569 return new X86FastISel(funcInfo, libInfo);
bool isInt< 32 >(int64_t x)
void setFrameAddressIsTaken(bool T)
unsigned GetCondBranchFromCond(CondCode CC)
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
ReturnInst - Return a value (possibly void), from a function.
Value * getValueOperand()
const MachineInstrBuilder & addMetadata(const MDNode *MD) const
void AnalyzeCallResult(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeCallResult - Analyze the return values of a call, incorporating info about the passed values i...
void push_back(const T &Elt)
The memory access reads data.
mop_iterator operands_end()
This class is the base class for the comparison instructions.
The C convention as specified in the x86-64 supplement to the System V ABI, used on most non-Windows ...
The memory access writes data.
LocInfo getLocInfo() const
static const MachineInstrBuilder & addConstantPoolReference(const MachineInstrBuilder &MIB, unsigned CPI, unsigned GlobalBaseReg, unsigned char OpFlags)
addConstantPoolReference - This function is used to add a reference to the base of a constant value s...
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Force argument to be passed in register.
Intrinsic::ID getIntrinsicID() const
getIntrinsicID - Return the intrinsic ID of this intrinsic.
unsigned getNumOperands() const
Nested function static chain.
Describe properties that are true of each instruction in the target description file.
static unsigned computeBytesPoppedByCallee(const X86Subtarget *Subtarget, CallingConv::ID CC, ImmutableCallSite *CS)
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE, etc.
MachineInstr * foldMemoryOperandImpl(MachineFunction &MF, MachineInstr *MI, ArrayRef< unsigned > Ops, MachineBasicBlock::iterator InsertPt, int FrameIndex) const override
foldMemoryOperand - If this target supports it, fold a load or store of the specified stack slot into...
0 1 0 0 True if ordered and less than
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
unsigned getSizeInBits() const
bool isDoubleTy() const
isDoubleTy - Return true if this is 'double', a 64-bit IEEE fp type.
unsigned getByValSize() const
1 1 1 0 True if unordered or not equal
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
MemSetInst - This class wraps the llvm.memset intrinsic.
Type * getReturnType() const
const Function * getParent() const
Return the enclosing method, or null if none.
LoadInst - an instruction for reading from memory.
static MachinePointerInfo getConstantPool()
getConstantPool - Return a MachinePointerInfo record that refers to the constant pool.
bool hasAttribute(unsigned Index, Attribute::AttrKind Kind) const
Return true if the attribute exists at the given index.
unsigned getValNo() const
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
bool bitsLT(EVT VT) const
bitsLT - Return true if this has less bits than VT.
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(const char *reason, bool gen_crash_diag=true)
Reports a serious error, calling any installed error handler.
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
static Constant * getNullValue(Type *Ty)
X86MachineFunctionInfo - This class is derived from MachineFunction and contains private X86 target-s...
COPY - Target-independent register copy.
1 0 0 1 True if unordered or equal
unsigned getSize() const
getSize - Return the size of the register in bytes, which is also the size of a stack slot allocated ...
Used to lazily calculate structure layout information for a target machine, based on the DataLayout s...
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
MachineMemOperand - A description of a memory reference used in the backend.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
const HexagonInstrInfo * TII
bool hasDefaultVisibility() const
const MCAsmInfo * getMCAsmInfo() const
Return target specific asm information.
StructType - Class to represent struct types.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
A Use represents the edge between a Value definition and its users.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
bool isReg() const
isReg - Tests if this is a MO_Register operand.
static bool isGlobalStubReference(unsigned char TargetFlag)
isGlobalStubReference - Return true if the specified TargetFlag operand is a reference to a stub for ...
unsigned getNumArgOperands() const
getNumArgOperands - Return the number of call arguments.
bool isInt< 8 >(int64_t x)
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
Reg
All possible values of the reg field in the ModR/M byte.
0 1 0 1 True if ordered and less than or equal
static Constant * getSExt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
const Triple & getTargetTriple() const
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
bool isStrongDefinitionForLinker() const
Returns true if this global's definition will be the one chosen by the linker.
const MachineInstrBuilder & addImm(int64_t Val) const
addImm - Add a new immediate operand.
Hidden pointer to structure to return.
unsigned getNumOperands() const
Access to explicit operands of the instruction.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
static bool isGlobalRelativeToPICBase(unsigned char TargetFlag)
isGlobalRelativeToPICBase - Return true if the specified global value reference is relative to a 32-b...
static Constant * getZExt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
unsigned getLocReg() const
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
ConstantExpr - a constant value that is initialized with an expression using other constant values...
void GetReturnInfo(Type *ReturnType, AttributeSet attr, SmallVectorImpl< ISD::OutputArg > &Outs, const TargetLowering &TLI, const DataLayout &DL)
Given an LLVM IR type and return type attributes, compute the return value EVTs and flags...
bool is64Bit() const
Is this x86_64? (disregarding specific ABI / programming model)
Simple integer binary arithmetic operators.
This instruction compares its operands according to the predicate given to the constructor.
BasicBlock * getSuccessor(unsigned i) const
StoreInst - an instruction for storing to memory.
bool isArrayTy() const
isArrayTy - True if this is an instance of ArrayType.
void addMemOperand(MachineFunction &MF, MachineMemOperand *MO)
Add a MachineMemOperand to the machine instruction.
void setStackProtectorIndex(int I)
This class represents a truncation of integer types.
bool isAtomic() const
isAtomic - Return true if this instruction has an AtomicOrdering of unordered or higher.
PointerType - Class to represent pointers.
unsigned getKillRegState(bool B)
MO_DARWIN_STUB - On a symbol operand "FOO", this indicates that the reference is actually to the "FOO...
uint64_t getElementOffset(unsigned Idx) const
DBG_VALUE - a mapping of the llvm.dbg.value intrinsic.
IMPLICIT_DEF - This is the MachineInstr-level equivalent of undef.
GetElementPtrInst - an instruction for type-safe pointer arithmetic to access elements of arrays and ...
bool isInteger() const
isInteger - Return true if this is an integer, or a vector integer type.
unsigned getAlignment() const
getAlignment - Return the alignment of the access that is being performed
MVT - Machine Value Type.
The instances of the Type class are immutable: once they are created, they are never changed...
DIExpression * getExpression() const
BranchInst - Conditional or Unconditional Branch instruction.
bool isVectorTy() const
isVectorTy - True if this is an instance of VectorType.
This is an important base class in LLVM.
bool isVector() const
isVector - Return true if this is a vector value type.
bool isFloatTy() const
isFloatTy - Return true if this is 'float', a 32-bit IEEE fp type.
ConstantFP - Floating Point Values [float, double].
Value * getRawDest() const
const DebugLoc & getDebugLoc() const
getDebugLoc - Return the debug location for this node as a DebugLoc.
static std::pair< unsigned, bool > getX86SSEConditionCode(CmpInst::Predicate Predicate)
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
TRAP - Trapping instruction.
Value * getOperand(unsigned i) const
0 1 1 1 True if ordered (no nans)
Value * getPointerOperand()
Predicate getPredicate() const
Return the predicate for this instruction.
MachineInstrBuilder BuildMI(MachineFunction &MF, DebugLoc DL, const MCInstrDesc &MCID)
BuildMI - Builder interface.
1 1 1 1 Always true (always folded)
unsigned getSETFromCond(CondCode CC, bool HasMemoryOperand=false)
Return a set opcode for the given condition and whether it has a memory operand.
EVT - Extended Value Type.
LLVMContext & getContext() const
All values hold a context through their type.
1 1 0 1 True if unordered, less than, or equal
unsigned getStackRegister() const
static const MachineInstrBuilder & addFrameReference(const MachineInstrBuilder &MIB, int FI, int Offset=0, bool mem=true)
addFrameReference - This function is used to add a reference to the base of an abstract object on the...
bool bitsGT(EVT VT) const
bitsGT - Return true if this has more bits than VT.
0 0 1 0 True if ordered and greater than
CCState - This class holds information needed while lowering arguments and return values...
This is the shared class of boolean and integer constants.
bool paramHasAttr(unsigned i, Attribute::AttrKind A) const
Return true if the call or the callee has the given attribute.
MachineOperand class - Representation of each machine instruction operand.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
1 1 0 0 True if unordered or less than
bool isStaticAlloca() const
isStaticAlloca - Return true if this alloca is in the entry block of the function and is a constant s...
Type * getType() const
All values are typed, get the type of this value.
Provides information about what library functions are available for the current target.
CCValAssign - Represent assignment of one arg/retval to a location.
enum llvm::X86AddressMode::@294 BaseType
Value * getLength() const
void getFullAddress(SmallVectorImpl< MachineOperand > &MO)
MemCpyInst - This class wraps the llvm.memcpy intrinsic.
Function * getCalledFunction() const
getCalledFunction - Return the function called, or null if this is an indirect function invocation...
MachineFrameInfo * getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
X86_FastCall - 'fast' analog of X86_StdCall.
bool isNullValue() const
isNullValue - Return true if this is the value that would be returned by getNullValue.
static MachinePointerInfo getStack(int64_t Offset)
getStack - stack pointer relative access.
static cl::opt< AlignMode > Align(cl::desc("Load/store alignment support"), cl::Hidden, cl::init(NoStrictAlign), cl::values(clEnumValN(StrictAlign,"aarch64-strict-align","Disallow all unaligned memory accesses"), clEnumValN(NoStrictAlign,"aarch64-no-strict-align","Allow unaligned memory accesses"), clEnumValEnd))
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
AttributeSet getAttributes() const
Return the attribute list for this Function.
Value * getArgOperand(unsigned i) const
getArgOperand/setArgOperand - Return/set the i-th call argument.
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
bool isUInt< 32 >(uint64_t x)
bool isIntegerTy() const
isIntegerTy - True if this is an instance of IntegerType.
This file defines the FastISel class.
unsigned getCMovFromCond(CondCode CC, unsigned RegBytes, bool HasMemoryOperand=false)
Return a cmov opcode for the given condition, register size in bytes, and operand type...
ZERO_EXTEND - Used for integer types, zeroing the new bits.
LLVM_ATTRIBUTE_UNUSED_RESULT std::enable_if< !is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
ANY_EXTEND - Used for integer types. The high bits are undefined.
KILL - This instruction is a noop that is used only to adjust the liveness of registers.
const MachineInstrBuilder & addGlobalAddress(const GlobalValue *GV, int64_t Offset=0, unsigned char TargetFlags=0) const
bool isStructTy() const
isStructTy - True if this is an instance of StructType.
Representation of each machine instruction.
static const MachineInstrBuilder & addDirectMem(const MachineInstrBuilder &MIB, unsigned Reg)
addDirectMem - This function is used to add a direct memory reference to the current instruction – th...
MO_GOTOFF - On a symbol operand this indicates that the immediate is the offset to the location of th...
Value * getCondition() const
Bitwise operators - logical and, logical or, logical xor.
unsigned getSRetReturnReg() const
static IntegerType * getInt32Ty(LLVMContext &C)
unsigned getBytesToPopOnReturn() const
unsigned greater or equal
unsigned getAlignment() const
getAlignment - Return the alignment of the access that is being performed
ImmutableCallSite - establish a view to a call site for examination.
unsigned getSizeInBits() const
getSizeInBits - Return the size of the specified value type in bits.
void setReg(unsigned Reg)
Change the register this operand corresponds to.
bool isOSMSVCRT() const
Is this a "Windows" OS targeting a "MSVCRT.dll" environment.
void AnalyzeReturn(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeReturn - Analyze the returned values of a return, incorporating info about the result values i...
Fast - This calling convention attempts to make calls as fast as possible (e.g.
bool hasOneUse() const
Return true if there is exactly one user of this value.
static std::pair< X86::CondCode, bool > getX86ConditionCode(CmpInst::Predicate Predicate)
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
0 1 1 0 True if ordered and operands are unequal
const TargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
1 0 1 0 True if unordered or greater than
int CreateStackObject(uint64_t Size, unsigned Alignment, bool isSS, const AllocaInst *Alloca=nullptr)
Create a new statically sized stack object, returning a nonnegative identifier to represent it...
bool hasLocalLinkage() const
MO_PLT - On a symbol operand this indicates that the immediate is offset to the PLT entry of symbol n...
bool isCalleePop(CallingConv::ID CallingConv, bool is64Bit, bool IsVarArg, bool TailCallOpt)
Determines whether the callee is required to pop its own arguments.
Value * getRawSource() const
get* - Return the arguments to the instruction.
unsigned getReg() const
getReg - Returns the register number.
DILocalVariable * getVariable() const
reverse_iterator rbegin()
bool isSimple() const
isSimple - Test if the given EVT is simple (as opposed to being extended).
0 0 0 1 True if ordered and equal
LLVM Value Representation.
mop_iterator operands_begin()
1 0 1 1 True if unordered, greater than, or equal
unsigned getOpcode() const
getOpcode() returns a member of one of the enums like Instruction::Add.
Value * getAddress() const
DbgDeclareInst - This represents the llvm.dbg.declare instruction.
static unsigned X86ChooseCmpImmediateOpcode(EVT VT, const ConstantInt *RHSC)
If we have a comparison with RHS as the RHS of the comparison, return an opcode that works for the co...
unsigned getDestAddressSpace() const
The C convention as implemented on Windows/x86-64.
MO_PIC_BASE_OFFSET - On a symbol operand this indicates that the immediate should get the value of th...
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this variable.
C - The default llvm calling convention, compatible with C.
SUBREG_TO_REG - This instruction is similar to INSERT_SUBREG except that the first operand is an imme...
unsigned getLocMemOffset() const
X86AddressMode - This struct holds a generalized full x86 address mode.
const MachineInstrBuilder & addRegMask(const uint32_t *Mask) const
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
0 0 1 1 True if ordered and greater than or equal
union llvm::X86AddressMode::@295 Base
const MachineInstrBuilder & addReg(unsigned RegNo, unsigned flags=0, unsigned SubReg=0) const
addReg - Add a new virtual register operand...
DbgDeclareInst - This represents the llvm.dbg.declare instruction.
unsigned getSourceAddressSpace() const
bool isVarArg() const
isVarArg - Return true if this function takes a variable number of arguments.
Value * getPointerOperand()
unsigned AllocateStack(unsigned Size, unsigned Align)
AllocateStack - Allocate a chunk of stack space with the specified size and alignment.
const BasicBlock * getParent() const
static const MachineInstrBuilder & addFullAddress(const MachineInstrBuilder &MIB, const X86AddressMode &AM)
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo)
MVT getSimpleVT() const
getSimpleVT - Return the SimpleValueType held in the specified simple EVT.
iterator_range< arg_iterator > args()
0 0 0 0 Always false (always folded)
IntrinsicInst - A useful wrapper class for inspecting calls to intrinsic functions.
static unsigned X86ChooseCmpOpcode(EVT VT, const X86Subtarget *Subtarget)
AllocaInst - an instruction to allocate memory on the stack.
gep_type_iterator gep_type_begin(const User *GEP)
unsigned getPtrSizedFrameRegister(const MachineFunction &MF) const
bool usesWindowsCFI() const
bool contains(unsigned Reg) const
contains - Return true if the specified register is included in this register class.