66#include "llvm/IR/IntrinsicsAMDGPU.h"
96#define DEBUG_TYPE "irtranslator"
102 cl::desc(
"Should enable CSE in irtranslator"),
120 MF.getProperties().set(MachineFunctionProperties::Property::FailedISel);
124 if (!R.getLocation().isValid() || TPC.isGlobalISelAbortEnabled())
125 R << (
" (in function: " + MF.getName() +
")").str();
127 if (TPC.isGlobalISelAbortEnabled())
144 DILocationVerifier() =
default;
145 ~DILocationVerifier() =
default;
147 const Instruction *getCurrentInst()
const {
return CurrInst; }
148 void setCurrentInst(
const Instruction *Inst) { CurrInst = Inst; }
155 assert(getCurrentInst() &&
"Inserted instruction without a current MI");
160 <<
" was copied to " <<
MI);
166 (
MI.getParent()->isEntryBlock() && !
MI.getDebugLoc()) ||
167 (
MI.isDebugInstr())) &&
168 "Line info was not transferred to all instructions");
191IRTranslator::allocateVRegs(
const Value &Val) {
192 auto VRegsIt = VMap.findVRegs(Val);
193 if (VRegsIt != VMap.vregs_end())
194 return *VRegsIt->second;
195 auto *Regs = VMap.getVRegs(Val);
196 auto *Offsets = VMap.getOffsets(Val);
199 Offsets->empty() ? Offsets :
nullptr);
200 for (
unsigned i = 0; i < SplitTys.
size(); ++i)
206 auto VRegsIt = VMap.findVRegs(Val);
207 if (VRegsIt != VMap.vregs_end())
208 return *VRegsIt->second;
211 return *VMap.getVRegs(Val);
214 auto *VRegs = VMap.getVRegs(Val);
215 auto *Offsets = VMap.getOffsets(Val);
218 "Don't know how to create an empty vreg");
222 Offsets->empty() ? Offsets :
nullptr);
224 if (!isa<Constant>(Val)) {
225 for (
auto Ty : SplitTys)
232 auto &
C = cast<Constant>(Val);
234 while (
auto Elt =
C.getAggregateElement(
Idx++)) {
235 auto EltRegs = getOrCreateVRegs(*Elt);
236 llvm::copy(EltRegs, std::back_inserter(*VRegs));
239 assert(SplitTys.size() == 1 &&
"unexpectedly split LLT");
241 bool Success = translate(cast<Constant>(Val), VRegs->front());
246 R <<
"unable to translate constant: " <<
ore::NV(
"Type", Val.
getType());
255int IRTranslator::getOrCreateFrameIndex(
const AllocaInst &AI) {
256 auto MapEntry = FrameIndices.find(&AI);
257 if (MapEntry != FrameIndices.end())
258 return MapEntry->second;
262 ElementSize * cast<ConstantInt>(AI.
getArraySize())->getZExtValue();
265 Size = std::max<uint64_t>(
Size, 1u);
267 int &FI = FrameIndices[&AI];
273 if (
const StoreInst *SI = dyn_cast<StoreInst>(&
I))
274 return SI->getAlign();
275 if (
const LoadInst *LI = dyn_cast<LoadInst>(&
I))
276 return LI->getAlign();
283 R <<
"unable to translate memop: " <<
ore::NV(
"Opcode", &
I);
290 assert(
MBB &&
"BasicBlock was not encountered before");
295 assert(NewPred &&
"new predecessor must be a real MachineBasicBlock");
296 MachinePreds[Edge].push_back(NewPred);
299bool IRTranslator::translateBinaryOp(
unsigned Opcode,
const User &U,
305 Register Op0 = getOrCreateVReg(*
U.getOperand(0));
306 Register Op1 = getOrCreateVReg(*
U.getOperand(1));
309 if (isa<Instruction>(U)) {
318bool IRTranslator::translateUnaryOp(
unsigned Opcode,
const User &U,
320 Register Op0 = getOrCreateVReg(*
U.getOperand(0));
323 if (isa<Instruction>(U)) {
332 return translateUnaryOp(TargetOpcode::G_FNEG, U, MIRBuilder);
335bool IRTranslator::translateCompare(
const User &U,
337 auto *CI = dyn_cast<CmpInst>(&U);
338 Register Op0 = getOrCreateVReg(*
U.getOperand(0));
339 Register Op1 = getOrCreateVReg(*
U.getOperand(1));
343 cast<ConstantExpr>(U).getPredicate());
345 MIRBuilder.
buildICmp(Pred, Res, Op0, Op1);
356 MIRBuilder.
buildFCmp(Pred, Res, Op0, Op1, Flags);
370 VRegs = getOrCreateVRegs(*Ret);
381 return CLI->
lowerReturn(MIRBuilder, Ret, VRegs, FuncInfo, SwiftErrorVReg);
384void IRTranslator::emitBranchForMergedCondition(
390 if (
const CmpInst *BOp = dyn_cast<CmpInst>(
Cond)) {
393 Condition = InvertCond ? IC->getInversePredicate() : IC->getPredicate();
396 Condition = InvertCond ?
FC->getInversePredicate() :
FC->getPredicate();
400 BOp->getOperand(1),
nullptr,
TBB, FBB, CurBB,
401 CurBuilder->getDebugLoc(), TProb, FProb);
402 SL->SwitchCases.push_back(CB);
410 nullptr,
TBB, FBB, CurBB, CurBuilder->getDebugLoc(), TProb, FProb);
411 SL->SwitchCases.push_back(CB);
416 return I->getParent() == BB;
420void IRTranslator::findMergedConditions(
425 using namespace PatternMatch;
426 assert((Opc == Instruction::And || Opc == Instruction::Or) &&
427 "Expected Opc to be AND/OR");
433 findMergedConditions(NotCond,
TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
439 const Value *BOpOp0, *BOpOp1;
453 if (BOpc == Instruction::And)
454 BOpc = Instruction::Or;
455 else if (BOpc == Instruction::Or)
456 BOpc = Instruction::And;
462 bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->
hasOneUse();
466 emitBranchForMergedCondition(
Cond,
TBB, FBB, CurBB, SwitchBB, TProb, FProb,
477 if (Opc == Instruction::Or) {
498 auto NewTrueProb = TProb / 2;
499 auto NewFalseProb = TProb / 2 + FProb;
501 findMergedConditions(BOpOp0,
TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb,
502 NewFalseProb, InvertCond);
508 findMergedConditions(BOpOp1,
TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
509 Probs[1], InvertCond);
511 assert(Opc == Instruction::And &&
"Unknown merge op!");
531 auto NewTrueProb = TProb + FProb / 2;
532 auto NewFalseProb = FProb / 2;
534 findMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb,
535 NewFalseProb, InvertCond);
541 findMergedConditions(BOpOp1,
TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
542 Probs[1], InvertCond);
546bool IRTranslator::shouldEmitAsBranches(
547 const std::vector<SwitchCG::CaseBlock> &Cases) {
549 if (Cases.size() != 2)
554 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
555 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
556 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
557 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
563 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
564 Cases[0].PredInfo.Pred == Cases[1].PredInfo.Pred &&
565 isa<Constant>(Cases[0].CmpRHS) &&
566 cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
568 Cases[0].TrueBB == Cases[1].ThisBB)
571 Cases[0].FalseBB == Cases[1].ThisBB)
579 const BranchInst &BrInst = cast<BranchInst>(U);
580 auto &CurMBB = MIRBuilder.
getMBB();
586 !CurMBB.isLayoutSuccessor(Succ0MBB))
591 CurMBB.addSuccessor(&getMBB(*Succ));
617 using namespace PatternMatch;
618 const Instruction *CondI = dyn_cast<Instruction>(CondVal);
620 !BrInst.
hasMetadata(LLVMContext::MD_unpredictable)) {
623 const Value *BOp0, *BOp1;
625 Opcode = Instruction::And;
627 Opcode = Instruction::Or;
631 findMergedConditions(CondI, Succ0MBB, Succ1MBB, &CurMBB, &CurMBB, Opcode,
632 getEdgeProbability(&CurMBB, Succ0MBB),
633 getEdgeProbability(&CurMBB, Succ1MBB),
635 assert(SL->SwitchCases[0].ThisBB == &CurMBB &&
"Unexpected lowering!");
638 if (shouldEmitAsBranches(SL->SwitchCases)) {
640 emitSwitchCase(SL->SwitchCases[0], &CurMBB, *CurBuilder);
641 SL->SwitchCases.erase(SL->SwitchCases.begin());
647 for (
unsigned I = 1,
E = SL->SwitchCases.size();
I !=
E; ++
I)
648 MF->
erase(SL->SwitchCases[
I].ThisBB);
650 SL->SwitchCases.clear();
657 nullptr, Succ0MBB, Succ1MBB, &CurMBB,
658 CurBuilder->getDebugLoc());
662 emitSwitchCase(CB, &CurMBB, *CurBuilder);
670 Src->addSuccessorWithoutProb(Dst);
674 Prob = getEdgeProbability(Src, Dst);
675 Src->addSuccessor(Dst, Prob);
681 const BasicBlock *SrcBB = Src->getBasicBlock();
682 const BasicBlock *DstBB = Dst->getBasicBlock();
686 auto SuccSize = std::max<uint32_t>(
succ_size(SrcBB), 1);
693 using namespace SwitchCG;
698 Clusters.reserve(
SI.getNumCases());
699 for (
const auto &
I :
SI.cases()) {
701 assert(Succ &&
"Could not find successor mbb in mapping");
706 Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob));
719 if (Clusters.empty()) {
726 SL->findJumpTables(Clusters, &SI, std::nullopt, DefaultMBB,
nullptr,
nullptr);
727 SL->findBitTestClusters(Clusters, &SI);
730 dbgs() <<
"Case clusters: ";
731 for (
const CaseCluster &
C : Clusters) {
732 if (
C.Kind == CC_JumpTable)
734 if (
C.Kind == CC_BitTests)
737 C.Low->getValue().print(
dbgs(),
true);
738 if (
C.Low !=
C.High) {
740 C.High->getValue().print(
dbgs(),
true);
747 assert(!Clusters.empty());
751 auto DefaultProb = getEdgeProbability(SwitchMBB, DefaultMBB);
752 WorkList.push_back({SwitchMBB,
First,
Last,
nullptr,
nullptr, DefaultProb});
754 while (!WorkList.empty()) {
755 SwitchWorkListItem
W = WorkList.pop_back_val();
757 unsigned NumClusters =
W.LastCluster -
W.FirstCluster + 1;
759 if (NumClusters > 3 &&
762 splitWorkItem(WorkList, W,
SI.getCondition(), SwitchMBB, MIB);
766 if (!lowerSwitchWorkItem(W,
SI.getCondition(), SwitchMBB, DefaultMBB, MIB))
776 using namespace SwitchCG;
777 assert(
W.FirstCluster->Low->getValue().slt(
W.LastCluster->Low->getValue()) &&
778 "Clusters not sorted?");
779 assert(
W.LastCluster -
W.FirstCluster + 1 >= 2 &&
"Too small to split!");
781 auto [LastLeft, FirstRight, LeftProb, RightProb] =
782 SL->computeSplitWorkItemInfo(W);
787 assert(PivotCluster >
W.FirstCluster);
788 assert(PivotCluster <=
W.LastCluster);
803 if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
804 FirstLeft->Low ==
W.GE &&
805 (FirstLeft->High->getValue() + 1LL) == Pivot->
getValue()) {
806 LeftMBB = FirstLeft->MBB;
811 {LeftMBB, FirstLeft, LastLeft,
W.GE, Pivot,
W.DefaultProb / 2});
818 if (FirstRight == LastRight && FirstRight->Kind == CC_Range &&
W.LT &&
819 (FirstRight->High->getValue() + 1ULL) ==
W.LT->getValue()) {
820 RightMBB = FirstRight->MBB;
825 {RightMBB, FirstRight, LastRight, Pivot,
W.LT,
W.DefaultProb / 2});
833 if (
W.MBB == SwitchMBB)
834 emitSwitchCase(CB, SwitchMBB, MIB);
836 SL->SwitchCases.push_back(CB);
842 assert(
JT.Reg != -1U &&
"Should lower JT Header first!");
864 Register SwitchOpReg = getOrCreateVReg(SValue);
866 auto Sub = MIB.
buildSub({SwitchTy}, SwitchOpReg, FirstCst);
874 JT.Reg = Sub.getReg(0);
885 auto Cst = getOrCreateVReg(
922 const auto *CI = dyn_cast<ConstantInt>(CB.
CmpRHS);
940 "Can only handle SLE ranges");
946 if (cast<ConstantInt>(CB.
CmpLHS)->isMinValue(
true)) {
952 auto Sub = MIB.
buildSub({CmpTy}, CmpOpReg, CondLHS);
987 bool FallthroughUnreachable) {
988 using namespace SwitchCG;
991 JumpTableHeader *JTH = &SL->JTCases[
I->JTCasesIndex].first;
997 CurMF->
insert(BBI, JumpMBB);
1007 auto JumpProb =
I->Prob;
1008 auto FallthroughProb = UnhandledProbs;
1016 if (*SI == DefaultMBB) {
1017 JumpProb += DefaultProb / 2;
1018 FallthroughProb -= DefaultProb / 2;
1023 addMachineCFGPred({SwitchMBB->
getBasicBlock(), (*SI)->getBasicBlock()},
1028 if (FallthroughUnreachable)
1029 JTH->FallthroughUnreachable =
true;
1031 if (!JTH->FallthroughUnreachable)
1032 addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
1033 addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
1038 JTH->HeaderBB = CurMBB;
1039 JT->Default = Fallthrough;
1042 if (CurMBB == SwitchMBB) {
1043 if (!emitJumpTableHeader(*JT, *JTH, CurMBB))
1045 JTH->Emitted =
true;
1052 bool FallthroughUnreachable,
1057 using namespace SwitchCG;
1060 if (
I->Low ==
I->High) {
1076 CaseBlock CB(Pred, FallthroughUnreachable, LHS, RHS, MHS,
I->MBB, Fallthrough,
1079 emitSwitchCase(CB, SwitchMBB, MIB);
1089 Register SwitchOpReg = getOrCreateVReg(*
B.SValue);
1093 auto RangeSub = MIB.
buildSub(SwitchOpTy, SwitchOpReg, MinValReg);
1098 LLT MaskTy = SwitchOpTy;
1104 for (
unsigned I = 0,
E =
B.Cases.size();
I !=
E; ++
I) {
1114 if (SwitchOpTy != MaskTy)
1122 if (!
B.FallthroughUnreachable)
1123 addSuccessorWithProb(SwitchBB,
B.Default,
B.DefaultProb);
1124 addSuccessorWithProb(SwitchBB,
MBB,
B.Prob);
1128 if (!
B.FallthroughUnreachable) {
1132 RangeSub, RangeCst);
1152 if (PopCount == 1) {
1155 auto MaskTrailingZeros =
1160 }
else if (PopCount == BB.
Range) {
1162 auto MaskTrailingOnes =
1169 auto SwitchVal = MIB.
buildShl(SwitchTy, CstOne, Reg);
1173 auto AndOp = MIB.
buildAnd(SwitchTy, SwitchVal, CstMask);
1180 addSuccessorWithProb(SwitchBB,
B.TargetBB,
B.ExtraProb);
1182 addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
1200bool IRTranslator::lowerBitTestWorkItem(
1206 bool FallthroughUnreachable) {
1207 using namespace SwitchCG;
1210 BitTestBlock *BTB = &SL->BitTestCases[
I->BTCasesIndex];
1212 for (BitTestCase &BTC : BTB->Cases)
1213 CurMF->
insert(BBI, BTC.ThisBB);
1216 BTB->Parent = CurMBB;
1217 BTB->Default = Fallthrough;
1219 BTB->DefaultProb = UnhandledProbs;
1223 if (!BTB->ContiguousRange) {
1224 BTB->Prob += DefaultProb / 2;
1225 BTB->DefaultProb -= DefaultProb / 2;
1228 if (FallthroughUnreachable)
1229 BTB->FallthroughUnreachable =
true;
1232 if (CurMBB == SwitchMBB) {
1233 emitBitTestHeader(*BTB, SwitchMBB);
1234 BTB->Emitted =
true;
1244 using namespace SwitchCG;
1248 if (++BBI != FuncInfo.
MF->
end())
1257 [](
const CaseCluster &a,
const CaseCluster &b) {
1258 return a.Prob != b.Prob
1260 : a.Low->getValue().slt(b.Low->getValue());
1265 for (CaseClusterIt
I =
W.LastCluster;
I >
W.FirstCluster;) {
1267 if (
I->Prob >
W.LastCluster->Prob)
1269 if (
I->Kind == CC_Range &&
I->MBB == NextMBB) {
1279 for (CaseClusterIt
I =
W.FirstCluster;
I <=
W.LastCluster; ++
I)
1280 UnhandledProbs +=
I->Prob;
1283 for (CaseClusterIt
I =
W.FirstCluster,
E =
W.LastCluster;
I <=
E; ++
I) {
1284 bool FallthroughUnreachable =
false;
1286 if (
I ==
W.LastCluster) {
1288 Fallthrough = DefaultMBB;
1289 FallthroughUnreachable = isa<UnreachableInst>(
1293 CurMF->
insert(BBI, Fallthrough);
1295 UnhandledProbs -=
I->Prob;
1299 if (!lowerBitTestWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
1300 DefaultProb, UnhandledProbs,
I, Fallthrough,
1301 FallthroughUnreachable)) {
1309 if (!lowerJumpTableWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
1310 UnhandledProbs,
I, Fallthrough,
1311 FallthroughUnreachable)) {
1318 if (!lowerSwitchRangeWorkItem(
I,
Cond, Fallthrough,
1319 FallthroughUnreachable, UnhandledProbs,
1320 CurMBB, MIB, SwitchMBB)) {
1327 CurMBB = Fallthrough;
1333bool IRTranslator::translateIndirectBr(
const User &U,
1347 if (!AddedSuccessors.
insert(Succ).second)
1356 if (
auto Arg = dyn_cast<Argument>(V))
1357 return Arg->hasSwiftErrorAttr();
1358 if (
auto AI = dyn_cast<AllocaInst>(V))
1364 const LoadInst &LI = cast<LoadInst>(U);
1380 assert(Regs.
size() == 1 &&
"swifterror should be single pointer");
1398 for (
unsigned i = 0; i < Regs.
size(); ++i) {
1403 Align BaseAlign = getMemOpAlign(LI);
1427 assert(Vals.
size() == 1 &&
"swifterror should be single pointer");
1430 SI.getPointerOperand());
1437 for (
unsigned i = 0; i < Vals.
size(); ++i) {
1442 Align BaseAlign = getMemOpAlign(SI);
1446 SI.getSyncScopeID(),
SI.getOrdering());
1453 const Value *Src = U.getOperand(0);
1462 for (
auto Idx : EVI->indices())
1464 }
else if (
const InsertValueInst *IVI = dyn_cast<InsertValueInst>(&U)) {
1465 for (
auto Idx : IVI->indices())
1468 for (
unsigned i = 1; i < U.getNumOperands(); ++i)
1473 DL.getIndexedOffsetInType(Src->getType(), Indices));
1476bool IRTranslator::translateExtractValue(
const User &U,
1478 const Value *Src =
U.getOperand(0);
1483 auto &DstRegs = allocateVRegs(U);
1485 for (
unsigned i = 0; i < DstRegs.size(); ++i)
1486 DstRegs[i] = SrcRegs[
Idx++];
1491bool IRTranslator::translateInsertValue(
const User &U,
1493 const Value *Src =
U.getOperand(0);
1495 auto &DstRegs = allocateVRegs(U);
1499 auto *InsertedIt = InsertedRegs.
begin();
1501 for (
unsigned i = 0; i < DstRegs.size(); ++i) {
1502 if (DstOffsets[i] >=
Offset && InsertedIt != InsertedRegs.
end())
1503 DstRegs[i] = *InsertedIt++;
1505 DstRegs[i] = SrcRegs[i];
1511bool IRTranslator::translateSelect(
const User &U,
1513 Register Tst = getOrCreateVReg(*
U.getOperand(0));
1519 if (
const SelectInst *SI = dyn_cast<SelectInst>(&U))
1522 for (
unsigned i = 0; i < ResRegs.
size(); ++i) {
1523 MIRBuilder.
buildSelect(ResRegs[i], Tst, Op0Regs[i], Op1Regs[i], Flags);
1529bool IRTranslator::translateCopy(
const User &U,
const Value &V,
1532 auto &Regs = *VMap.getVRegs(U);
1534 Regs.push_back(Src);
1535 VMap.getOffsets(U)->push_back(0);
1544bool IRTranslator::translateBitCast(
const User &U,
1551 if (isa<ConstantInt>(
U.getOperand(0)))
1552 return translateCast(TargetOpcode::G_CONSTANT_FOLD_BARRIER, U,
1554 return translateCopy(U, *
U.getOperand(0), MIRBuilder);
1557 return translateCast(TargetOpcode::G_BITCAST, U, MIRBuilder);
1560bool IRTranslator::translateCast(
unsigned Opcode,
const User &U,
1562 if (
U.getType()->getScalarType()->isBFloatTy() ||
1563 U.getOperand(0)->getType()->getScalarType()->isBFloatTy())
1571bool IRTranslator::translateGetElementPtr(
const User &U,
1573 Value &Op0 = *
U.getOperand(0);
1574 Register BaseReg = getOrCreateVReg(Op0);
1581 if (isa<Instruction>(U)) {
1588 unsigned VectorWidth = 0;
1592 bool WantSplatVector =
false;
1593 if (
auto *VT = dyn_cast<VectorType>(
U.getType())) {
1594 VectorWidth = cast<FixedVectorType>(VT)->getNumElements();
1596 WantSplatVector = VectorWidth > 1;
1601 if (WantSplatVector && !PtrTy.
isVector()) {
1602 BaseReg = MIRBuilder
1615 const Value *
Idx = GTI.getOperand();
1616 if (
StructType *StTy = GTI.getStructTypeOrNull()) {
1617 unsigned Field = cast<Constant>(
Idx)->getUniqueInteger().getZExtValue();
1621 uint64_t ElementSize = GTI.getSequentialElementStride(*DL);
1625 if (
const auto *CI = dyn_cast<ConstantInt>(
Idx)) {
1626 if (std::optional<int64_t> Val = CI->getValue().trySExtValue()) {
1627 Offset += ElementSize * *Val;
1634 BaseReg = MIRBuilder.
buildPtrAdd(PtrTy, BaseReg, OffsetMIB.getReg(0))
1641 if (IdxTy != OffsetTy) {
1642 if (!IdxTy.
isVector() && WantSplatVector) {
1655 if (ElementSize != 1) {
1661 GepOffsetReg = IdxReg;
1671 if (int64_t(
Offset) >= 0 && cast<GEPOperator>(U).isInBounds())
1674 MIRBuilder.
buildPtrAdd(getOrCreateVReg(U), BaseReg, OffsetMIB.getReg(0),
1679 MIRBuilder.
buildCopy(getOrCreateVReg(U), BaseReg);
1683bool IRTranslator::translateMemFunc(
const CallInst &CI,
1688 if (isa<UndefValue>(SrcPtr))
1693 unsigned MinPtrSize = UINT_MAX;
1694 for (
auto AI = CI.
arg_begin(), AE = CI.
arg_end(); std::next(AI) != AE; ++AI) {
1695 Register SrcReg = getOrCreateVReg(**AI);
1698 MinPtrSize = std::min<unsigned>(SrcTy.
getSizeInBits(), MinPtrSize);
1706 if (MRI->
getType(SizeOpReg) != SizeTy)
1720 if (
auto *MCI = dyn_cast<MemCpyInst>(&CI)) {
1721 DstAlign = MCI->getDestAlign().valueOrOne();
1722 SrcAlign = MCI->getSourceAlign().valueOrOne();
1723 CopySize = dyn_cast<ConstantInt>(MCI->getArgOperand(2));
1724 }
else if (
auto *MCI = dyn_cast<MemCpyInlineInst>(&CI)) {
1725 DstAlign = MCI->getDestAlign().valueOrOne();
1726 SrcAlign = MCI->getSourceAlign().valueOrOne();
1727 CopySize = dyn_cast<ConstantInt>(MCI->getArgOperand(2));
1728 }
else if (
auto *MMI = dyn_cast<MemMoveInst>(&CI)) {
1729 DstAlign = MMI->getDestAlign().valueOrOne();
1730 SrcAlign = MMI->getSourceAlign().valueOrOne();
1731 CopySize = dyn_cast<ConstantInt>(MMI->getArgOperand(2));
1733 auto *MSI = cast<MemSetInst>(&CI);
1734 DstAlign = MSI->getDestAlign().valueOrOne();
1737 if (Opcode != TargetOpcode::G_MEMCPY_INLINE) {
1753 if (AA && CopySize &&
1764 ICall.addMemOperand(
1766 StoreFlags, 1, DstAlign, AAInfo));
1767 if (Opcode != TargetOpcode::G_MEMSET)
1774bool IRTranslator::translateVectorInterleave2Intrinsic(
1777 "This function can only be called on the interleave2 intrinsic!");
1781 Register Res = getOrCreateVReg(CI);
1790bool IRTranslator::translateVectorDeinterleave2Intrinsic(
1793 "This function can only be called on the deinterleave2 intrinsic!");
1809void IRTranslator::getStackGuard(
Register DstReg,
1814 MIRBuilder.
buildInstr(TargetOpcode::LOAD_STACK_GUARD, {DstReg}, {});
1820 unsigned AddrSpace =
Global->getType()->getPointerAddressSpace();
1828 MIB.setMemRefs({
MemRef});
1831bool IRTranslator::translateOverflowIntrinsic(
const CallInst &CI,
unsigned Op,
1835 Op, {ResRegs[0], ResRegs[1]},
1841bool IRTranslator::translateFixedPointIntrinsic(
unsigned Op,
const CallInst &CI,
1843 Register Dst = getOrCreateVReg(CI);
1847 MIRBuilder.
buildInstr(
Op, {Dst}, { Src0, Src1, Scale });
1855 case Intrinsic::bswap:
1856 return TargetOpcode::G_BSWAP;
1857 case Intrinsic::bitreverse:
1858 return TargetOpcode::G_BITREVERSE;
1859 case Intrinsic::fshl:
1860 return TargetOpcode::G_FSHL;
1861 case Intrinsic::fshr:
1862 return TargetOpcode::G_FSHR;
1863 case Intrinsic::ceil:
1864 return TargetOpcode::G_FCEIL;
1865 case Intrinsic::cos:
1866 return TargetOpcode::G_FCOS;
1867 case Intrinsic::ctpop:
1868 return TargetOpcode::G_CTPOP;
1869 case Intrinsic::exp:
1870 return TargetOpcode::G_FEXP;
1871 case Intrinsic::exp2:
1872 return TargetOpcode::G_FEXP2;
1873 case Intrinsic::exp10:
1874 return TargetOpcode::G_FEXP10;
1875 case Intrinsic::fabs:
1876 return TargetOpcode::G_FABS;
1877 case Intrinsic::copysign:
1878 return TargetOpcode::G_FCOPYSIGN;
1879 case Intrinsic::minnum:
1880 return TargetOpcode::G_FMINNUM;
1881 case Intrinsic::maxnum:
1882 return TargetOpcode::G_FMAXNUM;
1883 case Intrinsic::minimum:
1884 return TargetOpcode::G_FMINIMUM;
1885 case Intrinsic::maximum:
1886 return TargetOpcode::G_FMAXIMUM;
1887 case Intrinsic::canonicalize:
1888 return TargetOpcode::G_FCANONICALIZE;
1889 case Intrinsic::floor:
1890 return TargetOpcode::G_FFLOOR;
1891 case Intrinsic::fma:
1892 return TargetOpcode::G_FMA;
1893 case Intrinsic::log:
1894 return TargetOpcode::G_FLOG;
1895 case Intrinsic::log2:
1896 return TargetOpcode::G_FLOG2;
1897 case Intrinsic::log10:
1898 return TargetOpcode::G_FLOG10;
1899 case Intrinsic::ldexp:
1900 return TargetOpcode::G_FLDEXP;
1901 case Intrinsic::nearbyint:
1902 return TargetOpcode::G_FNEARBYINT;
1903 case Intrinsic::pow:
1904 return TargetOpcode::G_FPOW;
1905 case Intrinsic::powi:
1906 return TargetOpcode::G_FPOWI;
1907 case Intrinsic::rint:
1908 return TargetOpcode::G_FRINT;
1909 case Intrinsic::round:
1910 return TargetOpcode::G_INTRINSIC_ROUND;
1911 case Intrinsic::roundeven:
1912 return TargetOpcode::G_INTRINSIC_ROUNDEVEN;
1913 case Intrinsic::sin:
1914 return TargetOpcode::G_FSIN;
1915 case Intrinsic::sqrt:
1916 return TargetOpcode::G_FSQRT;
1917 case Intrinsic::trunc:
1918 return TargetOpcode::G_INTRINSIC_TRUNC;
1919 case Intrinsic::readcyclecounter:
1920 return TargetOpcode::G_READCYCLECOUNTER;
1921 case Intrinsic::readsteadycounter:
1922 return TargetOpcode::G_READSTEADYCOUNTER;
1923 case Intrinsic::ptrmask:
1924 return TargetOpcode::G_PTRMASK;
1925 case Intrinsic::lrint:
1926 return TargetOpcode::G_INTRINSIC_LRINT;
1928 case Intrinsic::vector_reduce_fmin:
1929 return TargetOpcode::G_VECREDUCE_FMIN;
1930 case Intrinsic::vector_reduce_fmax:
1931 return TargetOpcode::G_VECREDUCE_FMAX;
1932 case Intrinsic::vector_reduce_fminimum:
1933 return TargetOpcode::G_VECREDUCE_FMINIMUM;
1934 case Intrinsic::vector_reduce_fmaximum:
1935 return TargetOpcode::G_VECREDUCE_FMAXIMUM;
1936 case Intrinsic::vector_reduce_add:
1937 return TargetOpcode::G_VECREDUCE_ADD;
1938 case Intrinsic::vector_reduce_mul:
1939 return TargetOpcode::G_VECREDUCE_MUL;
1940 case Intrinsic::vector_reduce_and:
1941 return TargetOpcode::G_VECREDUCE_AND;
1942 case Intrinsic::vector_reduce_or:
1943 return TargetOpcode::G_VECREDUCE_OR;
1944 case Intrinsic::vector_reduce_xor:
1945 return TargetOpcode::G_VECREDUCE_XOR;
1946 case Intrinsic::vector_reduce_smax:
1947 return TargetOpcode::G_VECREDUCE_SMAX;
1948 case Intrinsic::vector_reduce_smin:
1949 return TargetOpcode::G_VECREDUCE_SMIN;
1950 case Intrinsic::vector_reduce_umax:
1951 return TargetOpcode::G_VECREDUCE_UMAX;
1952 case Intrinsic::vector_reduce_umin:
1953 return TargetOpcode::G_VECREDUCE_UMIN;
1954 case Intrinsic::lround:
1955 return TargetOpcode::G_LROUND;
1956 case Intrinsic::llround:
1957 return TargetOpcode::G_LLROUND;
1958 case Intrinsic::get_fpenv:
1959 return TargetOpcode::G_GET_FPENV;
1960 case Intrinsic::get_fpmode:
1961 return TargetOpcode::G_GET_FPMODE;
1966bool IRTranslator::translateSimpleIntrinsic(
const CallInst &CI,
1970 unsigned Op = getSimpleIntrinsicOpcode(
ID);
1978 for (
const auto &Arg : CI.
args())
1981 MIRBuilder.
buildInstr(
Op, {getOrCreateVReg(CI)}, VRegs,
1989 case Intrinsic::experimental_constrained_fadd:
1990 return TargetOpcode::G_STRICT_FADD;
1991 case Intrinsic::experimental_constrained_fsub:
1992 return TargetOpcode::G_STRICT_FSUB;
1993 case Intrinsic::experimental_constrained_fmul:
1994 return TargetOpcode::G_STRICT_FMUL;
1995 case Intrinsic::experimental_constrained_fdiv:
1996 return TargetOpcode::G_STRICT_FDIV;
1997 case Intrinsic::experimental_constrained_frem:
1998 return TargetOpcode::G_STRICT_FREM;
1999 case Intrinsic::experimental_constrained_fma:
2000 return TargetOpcode::G_STRICT_FMA;
2001 case Intrinsic::experimental_constrained_sqrt:
2002 return TargetOpcode::G_STRICT_FSQRT;
2003 case Intrinsic::experimental_constrained_ldexp:
2004 return TargetOpcode::G_STRICT_FLDEXP;
2010bool IRTranslator::translateConstrainedFPIntrinsic(
2033std::optional<MCRegister> IRTranslator::getArgPhysReg(
Argument &Arg) {
2034 auto VRegs = getOrCreateVRegs(Arg);
2035 if (VRegs.
size() != 1)
2036 return std::nullopt;
2040 if (!VRegDef || !VRegDef->isCopy())
2041 return std::nullopt;
2045bool IRTranslator::translateIfEntryValueArgument(
bool isDeclare,
Value *Val,
2050 auto *Arg = dyn_cast<Argument>(Val);
2057 std::optional<MCRegister> PhysReg = getArgPhysReg(*Arg);
2059 LLVM_DEBUG(
dbgs() <<
"Dropping dbg." << (isDeclare ?
"declare" :
"value")
2060 <<
": expression is entry_value but "
2061 <<
"couldn't find a physical register\n");
2079 if (
auto *
MI = dyn_cast<AnyMemIntrinsic>(&CI)) {
2080 if (ORE->enabled()) {
2090 if (translateSimpleIntrinsic(CI,
ID, MIRBuilder))
2096 case Intrinsic::lifetime_start:
2097 case Intrinsic::lifetime_end: {
2102 unsigned Op =
ID == Intrinsic::lifetime_start ? TargetOpcode::LIFETIME_START
2103 : TargetOpcode::LIFETIME_END;
2112 for (
const Value *V : Allocas) {
2113 const AllocaInst *AI = dyn_cast<AllocaInst>(V);
2124 case Intrinsic::dbg_declare: {
2131 case Intrinsic::dbg_label: {
2137 "Expected inlined-at fields to agree");
2142 case Intrinsic::vaend:
2146 case Intrinsic::vastart: {
2151 MIRBuilder.
buildInstr(TargetOpcode::G_VASTART, {}, {getOrCreateVReg(*
Ptr)})
2154 ListSize, Alignment));
2157 case Intrinsic::dbg_assign:
2164 case Intrinsic::dbg_value: {
2171 case Intrinsic::uadd_with_overflow:
2172 return translateOverflowIntrinsic(CI, TargetOpcode::G_UADDO, MIRBuilder);
2173 case Intrinsic::sadd_with_overflow:
2174 return translateOverflowIntrinsic(CI, TargetOpcode::G_SADDO, MIRBuilder);
2175 case Intrinsic::usub_with_overflow:
2176 return translateOverflowIntrinsic(CI, TargetOpcode::G_USUBO, MIRBuilder);
2177 case Intrinsic::ssub_with_overflow:
2178 return translateOverflowIntrinsic(CI, TargetOpcode::G_SSUBO, MIRBuilder);
2179 case Intrinsic::umul_with_overflow:
2180 return translateOverflowIntrinsic(CI, TargetOpcode::G_UMULO, MIRBuilder);
2181 case Intrinsic::smul_with_overflow:
2182 return translateOverflowIntrinsic(CI, TargetOpcode::G_SMULO, MIRBuilder);
2183 case Intrinsic::uadd_sat:
2184 return translateBinaryOp(TargetOpcode::G_UADDSAT, CI, MIRBuilder);
2185 case Intrinsic::sadd_sat:
2186 return translateBinaryOp(TargetOpcode::G_SADDSAT, CI, MIRBuilder);
2187 case Intrinsic::usub_sat:
2188 return translateBinaryOp(TargetOpcode::G_USUBSAT, CI, MIRBuilder);
2189 case Intrinsic::ssub_sat:
2190 return translateBinaryOp(TargetOpcode::G_SSUBSAT, CI, MIRBuilder);
2191 case Intrinsic::ushl_sat:
2192 return translateBinaryOp(TargetOpcode::G_USHLSAT, CI, MIRBuilder);
2193 case Intrinsic::sshl_sat:
2194 return translateBinaryOp(TargetOpcode::G_SSHLSAT, CI, MIRBuilder);
2195 case Intrinsic::umin:
2196 return translateBinaryOp(TargetOpcode::G_UMIN, CI, MIRBuilder);
2197 case Intrinsic::umax:
2198 return translateBinaryOp(TargetOpcode::G_UMAX, CI, MIRBuilder);
2199 case Intrinsic::smin:
2200 return translateBinaryOp(TargetOpcode::G_SMIN, CI, MIRBuilder);
2201 case Intrinsic::smax:
2202 return translateBinaryOp(TargetOpcode::G_SMAX, CI, MIRBuilder);
2203 case Intrinsic::abs:
2205 return translateUnaryOp(TargetOpcode::G_ABS, CI, MIRBuilder);
2206 case Intrinsic::smul_fix:
2207 return translateFixedPointIntrinsic(TargetOpcode::G_SMULFIX, CI, MIRBuilder);
2208 case Intrinsic::umul_fix:
2209 return translateFixedPointIntrinsic(TargetOpcode::G_UMULFIX, CI, MIRBuilder);
2210 case Intrinsic::smul_fix_sat:
2211 return translateFixedPointIntrinsic(TargetOpcode::G_SMULFIXSAT, CI, MIRBuilder);
2212 case Intrinsic::umul_fix_sat:
2213 return translateFixedPointIntrinsic(TargetOpcode::G_UMULFIXSAT, CI, MIRBuilder);
2214 case Intrinsic::sdiv_fix:
2215 return translateFixedPointIntrinsic(TargetOpcode::G_SDIVFIX, CI, MIRBuilder);
2216 case Intrinsic::udiv_fix:
2217 return translateFixedPointIntrinsic(TargetOpcode::G_UDIVFIX, CI, MIRBuilder);
2218 case Intrinsic::sdiv_fix_sat:
2219 return translateFixedPointIntrinsic(TargetOpcode::G_SDIVFIXSAT, CI, MIRBuilder);
2220 case Intrinsic::udiv_fix_sat:
2221 return translateFixedPointIntrinsic(TargetOpcode::G_UDIVFIXSAT, CI, MIRBuilder);
2222 case Intrinsic::fmuladd: {
2224 Register Dst = getOrCreateVReg(CI);
2233 MIRBuilder.
buildFMA(Dst, Op0, Op1, Op2,
2244 case Intrinsic::convert_from_fp16:
2250 case Intrinsic::convert_to_fp16:
2256 case Intrinsic::frexp: {
2263 case Intrinsic::memcpy_inline:
2264 return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMCPY_INLINE);
2265 case Intrinsic::memcpy:
2266 return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMCPY);
2267 case Intrinsic::memmove:
2268 return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMMOVE);
2269 case Intrinsic::memset:
2270 return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMSET);
2271 case Intrinsic::eh_typeid_for: {
2278 case Intrinsic::objectsize:
2281 case Intrinsic::is_constant:
2284 case Intrinsic::stackguard:
2285 getStackGuard(getOrCreateVReg(CI), MIRBuilder);
2287 case Intrinsic::stackprotector: {
2292 getStackGuard(GuardVal, MIRBuilder);
2297 int FI = getOrCreateFrameIndex(*Slot);
2301 GuardVal, getOrCreateVReg(*Slot),
2308 case Intrinsic::stacksave: {
2309 MIRBuilder.
buildInstr(TargetOpcode::G_STACKSAVE, {getOrCreateVReg(CI)}, {});
2312 case Intrinsic::stackrestore: {
2313 MIRBuilder.
buildInstr(TargetOpcode::G_STACKRESTORE, {},
2317 case Intrinsic::cttz:
2318 case Intrinsic::ctlz: {
2320 bool isTrailing =
ID == Intrinsic::cttz;
2321 unsigned Opcode = isTrailing
2322 ? Cst->
isZero() ? TargetOpcode::G_CTTZ
2323 : TargetOpcode::G_CTTZ_ZERO_UNDEF
2324 : Cst->
isZero() ? TargetOpcode::G_CTLZ
2325 : TargetOpcode::G_CTLZ_ZERO_UNDEF;
2326 MIRBuilder.
buildInstr(Opcode, {getOrCreateVReg(CI)},
2330 case Intrinsic::invariant_start: {
2336 case Intrinsic::invariant_end:
2338 case Intrinsic::expect:
2339 case Intrinsic::annotation:
2340 case Intrinsic::ptr_annotation:
2341 case Intrinsic::launder_invariant_group:
2342 case Intrinsic::strip_invariant_group: {
2344 MIRBuilder.
buildCopy(getOrCreateVReg(CI),
2348 case Intrinsic::assume:
2349 case Intrinsic::experimental_noalias_scope_decl:
2350 case Intrinsic::var_annotation:
2351 case Intrinsic::sideeffect:
2354 case Intrinsic::read_volatile_register:
2355 case Intrinsic::read_register: {
2358 .
buildInstr(TargetOpcode::G_READ_REGISTER, {getOrCreateVReg(CI)}, {})
2359 .addMetadata(cast<MDNode>(cast<MetadataAsValue>(Arg)->getMetadata()));
2362 case Intrinsic::write_register: {
2364 MIRBuilder.
buildInstr(TargetOpcode::G_WRITE_REGISTER)
2365 .
addMetadata(cast<MDNode>(cast<MetadataAsValue>(Arg)->getMetadata()))
2369 case Intrinsic::localescape: {
2377 if (isa<ConstantPointerNull>(Arg))
2380 int FI = getOrCreateFrameIndex(*cast<AllocaInst>(Arg));
2396 case Intrinsic::vector_reduce_fadd:
2397 case Intrinsic::vector_reduce_fmul: {
2400 Register Dst = getOrCreateVReg(CI);
2406 Opc =
ID == Intrinsic::vector_reduce_fadd
2407 ? TargetOpcode::G_VECREDUCE_SEQ_FADD
2408 : TargetOpcode::G_VECREDUCE_SEQ_FMUL;
2409 MIRBuilder.
buildInstr(Opc, {Dst}, {ScalarSrc, VecSrc},
2416 if (
ID == Intrinsic::vector_reduce_fadd) {
2417 Opc = TargetOpcode::G_VECREDUCE_FADD;
2418 ScalarOpc = TargetOpcode::G_FADD;
2420 Opc = TargetOpcode::G_VECREDUCE_FMUL;
2421 ScalarOpc = TargetOpcode::G_FMUL;
2426 MIRBuilder.
buildInstr(ScalarOpc, {Dst}, {ScalarSrc, Rdx},
2431 case Intrinsic::trap:
2432 case Intrinsic::debugtrap:
2433 case Intrinsic::ubsantrap: {
2436 if (TrapFuncName.
empty())
2439 if (
ID == Intrinsic::ubsantrap) {
2446 return CLI->
lowerCall(MIRBuilder, Info);
2448 case Intrinsic::amdgcn_cs_chain:
2449 return translateCallBase(CI, MIRBuilder);
2450 case Intrinsic::fptrunc_round: {
2455 std::optional<RoundingMode> RoundMode =
2460 .
buildInstr(TargetOpcode::G_INTRINSIC_FPTRUNC_ROUND,
2461 {getOrCreateVReg(CI)},
2463 .addImm((
int)*RoundMode);
2467 case Intrinsic::is_fpclass: {
2472 .
buildInstr(TargetOpcode::G_IS_FPCLASS, {getOrCreateVReg(CI)},
2473 {getOrCreateVReg(*FpValue)})
2478 case Intrinsic::set_fpenv: {
2480 MIRBuilder.
buildInstr(TargetOpcode::G_SET_FPENV, {},
2481 {getOrCreateVReg(*FPEnv)});
2484 case Intrinsic::reset_fpenv: {
2485 MIRBuilder.
buildInstr(TargetOpcode::G_RESET_FPENV, {}, {});
2488 case Intrinsic::set_fpmode: {
2490 MIRBuilder.
buildInstr(TargetOpcode::G_SET_FPMODE, {},
2491 { getOrCreateVReg(*FPState) });
2494 case Intrinsic::reset_fpmode: {
2495 MIRBuilder.
buildInstr(TargetOpcode::G_RESET_FPMODE, {}, {});
2498 case Intrinsic::prefetch: {
2500 unsigned RW = cast<ConstantInt>(CI.
getOperand(1))->getZExtValue();
2501 unsigned Locality = cast<ConstantInt>(CI.
getOperand(2))->getZExtValue();
2502 unsigned CacheType = cast<ConstantInt>(CI.
getOperand(3))->getZExtValue();
2514 case Intrinsic::experimental_vector_interleave2:
2515 case Intrinsic::experimental_vector_deinterleave2: {
2522 if (CI.
getIntrinsicID() == Intrinsic::experimental_vector_interleave2)
2523 return translateVectorInterleave2Intrinsic(CI, MIRBuilder);
2525 return translateVectorDeinterleave2Intrinsic(CI, MIRBuilder);
2528#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
2529 case Intrinsic::INTRINSIC:
2530#include "llvm/IR/ConstrainedOps.def"
2531 return translateConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(CI),
2538bool IRTranslator::translateInlineAsm(
const CallBase &CB,
2545 dbgs() <<
"Inline asm lowering is not supported for this target yet\n");
2550 MIRBuilder, CB, [&](
const Value &Val) {
return getOrCreateVRegs(Val); });
2553bool IRTranslator::translateCallBase(
const CallBase &CB,
2560 for (
const auto &Arg : CB.
args()) {
2562 assert(SwiftInVReg == 0 &&
"Expected only one swift error argument");
2566 &CB, &MIRBuilder.
getMBB(), Arg));
2572 Args.push_back(getOrCreateVRegs(*Arg));
2575 if (
auto *CI = dyn_cast<CallInst>(&CB)) {
2576 if (ORE->enabled()) {
2588 CLI->
lowerCall(MIRBuilder, CB, Res, Args, SwiftErrorVReg,
2593 assert(!HasTailCall &&
"Can't tail call return twice from block?");
2602 const CallInst &CI = cast<CallInst>(U);
2608 if (
F && (
F->hasDLLImportStorageClass() ||
2610 F->hasExternalWeakLinkage())))
2618 if (isa<GCStatepointInst, GCRelocateInst, GCResultInst>(U))
2622 return translateInlineAsm(CI, MIRBuilder);
2627 if (
F &&
F->isIntrinsic()) {
2628 ID =
F->getIntrinsicID();
2634 return translateCallBase(CI, MIRBuilder);
2638 if (translateKnownIntrinsic(CI,
ID, MIRBuilder))
2643 ResultRegs = getOrCreateVRegs(CI);
2648 if (isa<FPMathOperator>(CI))
2655 if (
ConstantInt *CI = dyn_cast<ConstantInt>(Arg.value())) {
2658 assert(CI->getBitWidth() <= 64 &&
2659 "large intrinsic immediates not handled");
2660 MIB.
addImm(CI->getSExtValue());
2662 MIB.
addFPImm(cast<ConstantFP>(Arg.value()));
2664 }
else if (
auto *MDVal = dyn_cast<MetadataAsValue>(Arg.value())) {
2665 auto *MD = MDVal->getMetadata();
2666 auto *MDN = dyn_cast<MDNode>(MD);
2668 if (
auto *ConstMD = dyn_cast<ConstantAsMetadata>(MD))
2676 if (VRegs.
size() > 1)
2687 DL->getABITypeAlign(
Info.memVT.getTypeForEVT(
F->getContext())));
2688 LLT MemTy =
Info.memVT.isSimple()
2690 :
LLT::scalar(
Info.memVT.getStoreSizeInBits());
2697 else if (
Info.fallbackAddressSpace)
2706bool IRTranslator::findUnwindDestinations(
2726 if (isa<LandingPadInst>(Pad)) {
2728 UnwindDests.emplace_back(&getMBB(*EHPadBB), Prob);
2731 if (isa<CleanupPadInst>(Pad)) {
2734 UnwindDests.emplace_back(&getMBB(*EHPadBB), Prob);
2735 UnwindDests.
back().first->setIsEHScopeEntry();
2736 UnwindDests.back().first->setIsEHFuncletEntry();
2739 if (
auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
2741 for (
const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2742 UnwindDests.emplace_back(&getMBB(*CatchPadBB), Prob);
2744 if (IsMSVCCXX || IsCoreCLR)
2745 UnwindDests.back().first->setIsEHFuncletEntry();
2747 UnwindDests.back().first->setIsEHScopeEntry();
2749 NewEHPadBB = CatchSwitch->getUnwindDest();
2755 if (BPI && NewEHPadBB)
2757 EHPadBB = NewEHPadBB;
2762bool IRTranslator::translateInvoke(
const User &U,
2770 const Function *Fn =
I.getCalledFunction();
2795 bool LowerInlineAsm =
I.isInlineAsm();
2796 bool NeedEHLabel =
true;
2802 MIRBuilder.
buildInstr(TargetOpcode::G_INVOKE_REGION_START);
2803 BeginSymbol =
Context.createTempSymbol();
2807 if (LowerInlineAsm) {
2808 if (!translateInlineAsm(
I, MIRBuilder))
2810 }
else if (!translateCallBase(
I, MIRBuilder))
2815 EndSymbol =
Context.createTempSymbol();
2826 if (!findUnwindDestinations(EHPadBB, EHPadBBProb, UnwindDests))
2830 &ReturnMBB = getMBB(*ReturnBB);
2832 addSuccessorWithProb(InvokeMBB, &ReturnMBB);
2833 for (
auto &UnwindDest : UnwindDests) {
2834 UnwindDest.first->setIsEHPad();
2835 addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
2840 assert(BeginSymbol &&
"Expected a begin symbol!");
2841 assert(EndSymbol &&
"Expected an end symbol!");
2842 MF->
addInvoke(&EHPadMBB, BeginSymbol, EndSymbol);
2845 MIRBuilder.
buildBr(ReturnMBB);
2849bool IRTranslator::translateCallBr(
const User &U,
2855bool IRTranslator::translateLandingPad(
const User &U,
2879 MIRBuilder.
buildInstr(TargetOpcode::EH_LABEL)
2885 if (
auto *RegMask =
TRI.getCustomEHPadPreservedMask(*MF))
2893 for (
Type *Ty : cast<StructType>(LP.
getType())->elements())
2895 assert(Tys.
size() == 2 &&
"Only two-valued landingpads are supported");
2904 MIRBuilder.
buildCopy(ResRegs[0], ExceptionReg);
2912 MIRBuilder.
buildCopy(PtrVReg, SelectorReg);
2913 MIRBuilder.
buildCast(ResRegs[1], PtrVReg);
2918bool IRTranslator::translateAlloca(
const User &U,
2920 auto &AI = cast<AllocaInst>(U);
2926 Register Res = getOrCreateVReg(AI);
2927 int FI = getOrCreateFrameIndex(AI);
2940 if (MRI->
getType(NumElts) != IntPtrTy) {
2950 getOrCreateVReg(*ConstantInt::get(IntPtrIRTy,
DL->getTypeAllocSize(Ty)));
2951 MIRBuilder.
buildMul(AllocSize, NumElts, TySize);
2958 auto AllocAdd = MIRBuilder.
buildAdd(IntPtrTy, AllocSize, SAMinusOne,
2962 auto AlignedAlloc = MIRBuilder.
buildAnd(IntPtrTy, AllocAdd, AlignCst);
2965 if (Alignment <= StackAlign)
2966 Alignment =
Align(1);
2979 MIRBuilder.
buildInstr(TargetOpcode::G_VAARG, {getOrCreateVReg(U)},
2980 {getOrCreateVReg(*
U.getOperand(0)),
2981 DL->getABITypeAlign(
U.getType()).value()});
2989 auto &UI = cast<UnreachableInst>(U);
2993 if (&UI != &BB.
front()) {
2996 if (
const CallInst *Call = dyn_cast<CallInst>(&*PredI)) {
2997 if (
Call->doesNotReturn())
3007bool IRTranslator::translateInsertElement(
const User &U,
3011 if (
auto *FVT = dyn_cast<FixedVectorType>(
U.getType());
3012 FVT && FVT->getNumElements() == 1)
3013 return translateCopy(U, *
U.getOperand(1), MIRBuilder);
3016 Register Val = getOrCreateVReg(*
U.getOperand(0));
3017 Register Elt = getOrCreateVReg(*
U.getOperand(1));
3023bool IRTranslator::translateExtractElement(
const User &U,
3027 if (cast<FixedVectorType>(
U.getOperand(0)->getType())->getNumElements() == 1)
3028 return translateCopy(U, *
U.getOperand(0), MIRBuilder);
3031 Register Val = getOrCreateVReg(*
U.getOperand(0));
3034 if (
auto *CI = dyn_cast<ConstantInt>(
U.getOperand(1))) {
3035 if (CI->getBitWidth() != PreferredVecIdxWidth) {
3036 APInt NewIdx = CI->getValue().zextOrTrunc(PreferredVecIdxWidth);
3037 auto *NewIdxCI = ConstantInt::get(CI->
getContext(), NewIdx);
3038 Idx = getOrCreateVReg(*NewIdxCI);
3042 Idx = getOrCreateVReg(*
U.getOperand(1));
3051bool IRTranslator::translateShuffleVector(
const User &U,
3057 if (
U.getOperand(0)->getType()->isScalableTy()) {
3058 Value *Op0 =
U.getOperand(0);
3061 getOrCreateVReg(*Op0), 0);
3067 if (
auto *SVI = dyn_cast<ShuffleVectorInst>(&U))
3068 Mask = SVI->getShuffleMask();
3070 Mask = cast<ConstantExpr>(U).getShuffleMask();
3073 .
buildInstr(TargetOpcode::G_SHUFFLE_VECTOR, {getOrCreateVReg(U)},
3074 {getOrCreateVReg(*
U.getOperand(0)),
3075 getOrCreateVReg(*
U.getOperand(1))})
3076 .addShuffleMask(MaskAlloc);
3081 const PHINode &PI = cast<PHINode>(U);
3084 for (
auto Reg : getOrCreateVRegs(PI)) {
3085 auto MIB = MIRBuilder.
buildInstr(TargetOpcode::G_PHI, {
Reg}, {});
3089 PendingPHIs.emplace_back(&PI, std::move(Insts));
3093bool IRTranslator::translateAtomicCmpXchg(
const User &U,
3099 auto Res = getOrCreateVRegs(
I);
3103 Register Cmp = getOrCreateVReg(*
I.getCompareOperand());
3104 Register NewVal = getOrCreateVReg(*
I.getNewValOperand());
3107 OldValRes, SuccessRes,
Addr, Cmp, NewVal,
3110 getMemOpAlign(
I),
I.getAAMetadata(),
nullptr,
I.getSyncScopeID(),
3111 I.getSuccessOrdering(),
I.getFailureOrdering()));
3115bool IRTranslator::translateAtomicRMW(
const User &U,
3122 Register Val = getOrCreateVReg(*
I.getValOperand());
3124 unsigned Opcode = 0;
3125 switch (
I.getOperation()) {
3129 Opcode = TargetOpcode::G_ATOMICRMW_XCHG;
3132 Opcode = TargetOpcode::G_ATOMICRMW_ADD;
3135 Opcode = TargetOpcode::G_ATOMICRMW_SUB;
3138 Opcode = TargetOpcode::G_ATOMICRMW_AND;
3141 Opcode = TargetOpcode::G_ATOMICRMW_NAND;
3144 Opcode = TargetOpcode::G_ATOMICRMW_OR;
3147 Opcode = TargetOpcode::G_ATOMICRMW_XOR;
3150 Opcode = TargetOpcode::G_ATOMICRMW_MAX;
3153 Opcode = TargetOpcode::G_ATOMICRMW_MIN;
3156 Opcode = TargetOpcode::G_ATOMICRMW_UMAX;
3159 Opcode = TargetOpcode::G_ATOMICRMW_UMIN;
3162 Opcode = TargetOpcode::G_ATOMICRMW_FADD;
3165 Opcode = TargetOpcode::G_ATOMICRMW_FSUB;
3168 Opcode = TargetOpcode::G_ATOMICRMW_FMAX;
3171 Opcode = TargetOpcode::G_ATOMICRMW_FMIN;
3174 Opcode = TargetOpcode::G_ATOMICRMW_UINC_WRAP;
3177 Opcode = TargetOpcode::G_ATOMICRMW_UDEC_WRAP;
3182 Opcode, Res,
Addr, Val,
3184 Flags, MRI->
getType(Val), getMemOpAlign(
I),
3185 I.getAAMetadata(),
nullptr,
I.getSyncScopeID(),
3190bool IRTranslator::translateFence(
const User &U,
3192 const FenceInst &Fence = cast<FenceInst>(U);
3198bool IRTranslator::translateFreeze(
const User &U,
3204 "Freeze with different source and destination type?");
3206 for (
unsigned I = 0;
I < DstRegs.
size(); ++
I) {
3213void IRTranslator::finishPendingPhis() {
3219 for (
auto &Phi : PendingPHIs) {
3234 for (
auto *Pred : getMachinePredBBs({IRPred, PI->
getParent()})) {
3238 for (
unsigned j = 0;
j < ValRegs.
size(); ++
j) {
3248void IRTranslator::translateDbgValueRecord(
Value *V,
bool HasArgList,
3254 "Expected inlined-at fields to agree");
3258 if (!V || HasArgList) {
3265 if (
const auto *CI = dyn_cast<Constant>(V)) {
3270 if (
auto *AI = dyn_cast<AllocaInst>(V);
3275 auto ExprOperands =
Expression->getElements();
3276 auto *ExprDerefRemoved =
3282 if (translateIfEntryValueArgument(
false, V, Variable,
Expression, DL,
3285 for (
Register Reg : getOrCreateVRegs(*V)) {
3295void IRTranslator::translateDbgDeclareRecord(
Value *
Address,
bool HasArgList,
3301 LLVM_DEBUG(
dbgs() <<
"Dropping debug info for " << *Variable <<
"\n");
3306 "Expected inlined-at fields to agree");
3307 auto AI = dyn_cast<AllocaInst>(
Address);
3312 getOrCreateFrameIndex(*AI), DL);
3316 if (translateIfEntryValueArgument(
true,
Address, Variable,
3329void IRTranslator::translateDbgInfo(
const Instruction &Inst,
3332 if (
DPLabel *DPL = dyn_cast<DPLabel>(&DR)) {
3334 assert(DPL->getLabel() &&
"Missing label");
3335 assert(DPL->getLabel()->isValidLocationForIntrinsic(
3337 "Expected inlined-at fields to agree");
3341 DPValue &DPV = cast<DPValue>(DR);
3346 translateDbgDeclareRecord(V, DPV.
hasArgList(), Variable,
3349 translateDbgValueRecord(V, DPV.
hasArgList(), Variable,
3354bool IRTranslator::translate(
const Instruction &Inst) {
3356 CurBuilder->setPCSections(Inst.
getMetadata(LLVMContext::MD_pcsections));
3362#define HANDLE_INST(NUM, OPCODE, CLASS) \
3363 case Instruction::OPCODE: \
3364 return translate##OPCODE(Inst, *CurBuilder.get());
3365#include "llvm/IR/Instruction.def"
3374 if (
auto CurrInstDL = CurBuilder->getDL())
3375 EntryBuilder->setDebugLoc(
DebugLoc());
3377 if (
auto CI = dyn_cast<ConstantInt>(&
C))
3378 EntryBuilder->buildConstant(Reg, *CI);
3379 else if (
auto CF = dyn_cast<ConstantFP>(&
C))
3380 EntryBuilder->buildFConstant(Reg, *CF);
3381 else if (isa<UndefValue>(
C))
3382 EntryBuilder->buildUndef(Reg);
3383 else if (isa<ConstantPointerNull>(
C))
3384 EntryBuilder->buildConstant(Reg, 0);
3385 else if (
auto GV = dyn_cast<GlobalValue>(&
C))
3386 EntryBuilder->buildGlobalValue(Reg, GV);
3387 else if (
auto CAZ = dyn_cast<ConstantAggregateZero>(&
C)) {
3388 if (!isa<FixedVectorType>(CAZ->getType()))
3391 unsigned NumElts = CAZ->getElementCount().getFixedValue();
3393 return translateCopy(
C, *CAZ->getElementValue(0u), *EntryBuilder);
3395 for (
unsigned I = 0;
I < NumElts; ++
I) {
3396 Constant &Elt = *CAZ->getElementValue(
I);
3399 EntryBuilder->buildBuildVector(Reg, Ops);
3400 }
else if (
auto CV = dyn_cast<ConstantDataVector>(&
C)) {
3402 if (CV->getNumElements() == 1)
3403 return translateCopy(
C, *CV->getElementAsConstant(0), *EntryBuilder);
3405 for (
unsigned i = 0; i < CV->getNumElements(); ++i) {
3406 Constant &Elt = *CV->getElementAsConstant(i);
3409 EntryBuilder->buildBuildVector(Reg, Ops);
3410 }
else if (
auto CE = dyn_cast<ConstantExpr>(&
C)) {
3411 switch(
CE->getOpcode()) {
3412#define HANDLE_INST(NUM, OPCODE, CLASS) \
3413 case Instruction::OPCODE: \
3414 return translate##OPCODE(*CE, *EntryBuilder.get());
3415#include "llvm/IR/Instruction.def"
3419 }
else if (
auto CV = dyn_cast<ConstantVector>(&
C)) {
3420 if (CV->getNumOperands() == 1)
3421 return translateCopy(
C, *CV->getOperand(0), *EntryBuilder);
3423 for (
unsigned i = 0; i < CV->getNumOperands(); ++i) {
3424 Ops.
push_back(getOrCreateVReg(*CV->getOperand(i)));
3426 EntryBuilder->buildBuildVector(Reg, Ops);
3427 }
else if (
auto *BA = dyn_cast<BlockAddress>(&
C)) {
3428 EntryBuilder->buildBlockAddress(Reg, BA);
3435bool IRTranslator::finalizeBasicBlock(
const BasicBlock &BB,
3437 for (
auto &BTB : SL->BitTestCases) {
3440 emitBitTestHeader(BTB, BTB.Parent);
3443 for (
unsigned j = 0, ej = BTB.Cases.size(); j != ej; ++j) {
3444 UnhandledProb -= BTB.Cases[
j].ExtraProb;
3456 if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
3459 NextMBB = BTB.Cases[
j + 1].TargetBB;
3460 }
else if (j + 1 == ej) {
3462 NextMBB = BTB.Default;
3465 NextMBB = BTB.Cases[
j + 1].ThisBB;
3468 emitBitTestCase(BTB, NextMBB, UnhandledProb, BTB.Reg, BTB.Cases[j],
MBB);
3470 if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
3474 addMachineCFGPred({BTB.Parent->getBasicBlock(),
3475 BTB.Cases[ej - 1].TargetBB->getBasicBlock()},
3478 BTB.Cases.pop_back();
3484 CFGEdge HeaderToDefaultEdge = {BTB.Parent->getBasicBlock(),
3485 BTB.Default->getBasicBlock()};
3486 addMachineCFGPred(HeaderToDefaultEdge, BTB.Parent);
3487 if (!BTB.ContiguousRange) {
3488 addMachineCFGPred(HeaderToDefaultEdge, BTB.Cases.back().ThisBB);
3491 SL->BitTestCases.clear();
3493 for (
auto &JTCase : SL->JTCases) {
3495 if (!JTCase.first.Emitted)
3496 emitJumpTableHeader(JTCase.second, JTCase.first, JTCase.first.HeaderBB);
3498 emitJumpTable(JTCase.second, JTCase.second.MBB);
3500 SL->JTCases.clear();
3502 for (
auto &SwCase : SL->SwitchCases)
3503 emitSwitchCase(SwCase, &CurBuilder->getMBB(), *CurBuilder);
3504 SL->SwitchCases.clear();
3509 bool FunctionBasedInstrumentation =
3511 SPDescriptor.
initialize(&BB, &
MBB, FunctionBasedInstrumentation);
3531 SuccessMBB->
splice(SuccessMBB->
end(), ParentMBB, SplitPoint,
3535 if (!emitSPDescriptorParent(SPDescriptor, ParentMBB))
3540 if (FailureMBB->
empty()) {
3541 if (!emitSPDescriptorFailure(SPDescriptor, FailureMBB))
3553 CurBuilder->setInsertPt(*ParentBB, ParentBB->
end());
3563 Register StackSlotPtr = CurBuilder->buildFrameIndex(PtrTy, FI).getReg(0);
3570 ->buildLoad(PtrMemTy, StackSlotPtr,
3576 LLVM_DEBUG(
dbgs() <<
"Stack protector xor'ing with FP not yet implemented");
3594 assert(FnTy->getNumParams() == 1 &&
"Invalid function signature");
3596 if (GuardCheckFn->hasAttribute(1, Attribute::AttrKind::InReg))
3599 {GuardVal, FnTy->getParamType(0), {
Flags}});
3602 Info.OrigArgs.push_back(GuardArgInfo);
3603 Info.CallConv = GuardCheckFn->getCallingConv();
3606 if (!CLI->
lowerCall(MIRBuilder, Info)) {
3607 LLVM_DEBUG(
dbgs() <<
"Failed to lower call to stack protector check\n");
3619 getStackGuard(Guard, *CurBuilder);
3623 Register GuardPtr = getOrCreateVReg(*IRGuard);
3626 ->buildLoad(PtrMemTy, GuardPtr,
3645 CurBuilder->setInsertPt(*FailureBB, FailureBB->
end());
3655 if (!CLI->
lowerCall(*CurBuilder, Info)) {
3656 LLVM_DEBUG(
dbgs() <<
"Failed to lower call to stack protector fail\n");
3666 if (
TM.getTargetTriple().isPS() ||
TM.getTargetTriple().isWasm()) {
3667 LLVM_DEBUG(
dbgs() <<
"Unhandled trap emission for stack protector fail\n");
3673void IRTranslator::finalizeFunction() {
3676 PendingPHIs.clear();
3678 FrameIndices.clear();
3679 MachinePreds.clear();
3683 EntryBuilder.reset();
3698 const auto *CI = dyn_cast<CallInst>(&
I);
3707 getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
3710 TPC = &getAnalysis<TargetPassConfig>();
3717 EntryBuilder = std::make_unique<CSEMIRBuilder>(CurMF);
3719 EntryBuilder->setCSEInfo(CSEInfo);
3720 CurBuilder = std::make_unique<CSEMIRBuilder>(CurMF);
3721 CurBuilder->setCSEInfo(CSEInfo);
3723 EntryBuilder = std::make_unique<MachineIRBuilder>();
3724 CurBuilder = std::make_unique<MachineIRBuilder>();
3727 CurBuilder->setMF(*MF);
3728 EntryBuilder->setMF(*MF);
3730 DL = &
F.getParent()->getDataLayout();
3731 ORE = std::make_unique<OptimizationRemarkEmitter>(&
F);
3733 TM.resetTargetOptions(
F);
3737 AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
3738 FuncInfo.
BPI = &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
3741 FuncInfo.
BPI =
nullptr;
3744 AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(
3746 LibInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(
F);
3749 SL = std::make_unique<GISelSwitchLowering>(
this, FuncInfo);
3750 SL->init(*TLI,
TM, *
DL);
3752 assert(PendingPHIs.empty() &&
"stale PHIs");
3759 F.getSubprogram(), &
F.getEntryBlock());
3760 R <<
"unable to translate in big endian mode";
3765 auto FinalizeOnReturn =
make_scope_exit([
this]() { finalizeFunction(); });
3770 EntryBuilder->setMBB(*EntryBB);
3772 DebugLoc DbgLoc =
F.getEntryBlock().getFirstNonPHI()->getDebugLoc();
3776 bool IsVarArg =
F.isVarArg();
3777 bool HasMustTailInVarArgFn =
false;
3781 auto *&
MBB = BBToMBB[&BB];
3789 if (!HasMustTailInVarArgFn)
3796 EntryBB->addSuccessor(&getMBB(
F.front()));
3800 F.getSubprogram(), &
F.getEntryBlock());
3801 R <<
"unable to lower function: " <<
ore::NV(
"Prototype",
F.getType());
3809 if (
DL->getTypeStoreSize(Arg.
getType()).isZero())
3814 if (Arg.hasSwiftErrorAttr()) {
3815 assert(VRegs.
size() == 1 &&
"Too many vregs for Swift error");
3822 F.getSubprogram(), &
F.getEntryBlock());
3823 R <<
"unable to lower arguments: " <<
ore::NV(
"Prototype",
F.getType());
3830 if (EnableCSE && CSEInfo)
3844 CurBuilder->setMBB(
MBB);
3845 HasTailCall =
false;
3859 translateDbgInfo(Inst, *CurBuilder.get());
3861 if (translate(Inst))
3866 R <<
"unable to translate instruction: " <<
ore::NV(
"Opcode", &Inst);
3868 if (ORE->allowExtraAnalysis(
"gisel-irtranslator")) {
3869 std::string InstStrStorage;
3873 R <<
": '" << InstStr.
str() <<
"'";
3880 if (!finalizeBasicBlock(*BB,
MBB)) {
3882 BB->getTerminator()->getDebugLoc(), BB);
3883 R <<
"unable to translate basic block";
3893 finishPendingPhis();
3900 assert(EntryBB->succ_size() == 1 &&
3901 "Custom BB used for lowering should have only one successor");
3905 "LLVM-IR entry block has a predecessor!?");
3908 NewEntryBB.
splice(NewEntryBB.
begin(), EntryBB, EntryBB->begin(),
3917 EntryBB->removeSuccessor(&NewEntryBB);
3922 "New entry wasn't next in the list of basic block!");
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu aa AMDGPU Address space based Alias Analysis Wrapper
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Analysis containing CSE Info
Provides analysis for continuously CSEing during GISel passes.
This file implements a version of MachineIRBuilder which CSEs insts within a MachineBasicBlock.
This file describes how to lower LLVM calls to machine code calls.
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This contains common code to allow clients to notify changes to machine instr.
const HexagonInstrInfo * TII
IRTranslator LLVM IR static false void reportTranslationError(MachineFunction &MF, const TargetPassConfig &TPC, OptimizationRemarkEmitter &ORE, OptimizationRemarkMissed &R)
static bool checkForMustTailInVarArgFn(bool IsVarArg, const BasicBlock &BB)
Returns true if a BasicBlock BB within a variadic function contains a variadic musttail call.
static uint64_t getOffsetFromIndices(const User &U, const DataLayout &DL)
static unsigned getConstrainedOpcode(Intrinsic::ID ID)
static cl::opt< bool > EnableCSEInIRTranslator("enable-cse-in-irtranslator", cl::desc("Should enable CSE in irtranslator"), cl::Optional, cl::init(false))
static bool isValInBlock(const Value *V, const BasicBlock *BB)
static bool isSwiftError(const Value *V)
This file declares the IRTranslator pass.
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
This file describes how to lower LLVM inline asm to machine code INLINEASM.
Legalize the Machine IR a function s Machine IR
Implement a low-level type suitable for MachineInstr level instruction selection.
Implement a low-level type suitable for MachineInstr level instruction selection.
This file declares the MachineIRBuilder class.
unsigned const TargetRegisterInfo * TRI
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS_DEPENDENCY(depName)
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
verify safepoint Safepoint IR Verifier
This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...
This file defines the SmallSet class.
This file defines the SmallVector class.
This file describes how to lower LLVM code to machine code.
Target-Independent Code Generator Pass Configuration Options pass.
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.
bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal=false)
Checks whether the given location points to constant memory, or if OrLocal is true whether it points ...
Class for arbitrary precision integers.
an instruction to allocate memory on the stack
bool isSwiftError() const
Return true if this alloca is used as a swifterror argument to a call.
bool isStaticAlloca() const
Return true if this alloca is in the entry block of the function and is a constant size.
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
PointerType * getType() const
Overload to return most specific pointer type.
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
const Value * getArraySize() const
Get the number of elements allocated.
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
An immutable pass that tracks lazily created AssumptionCache objects.
An instruction that atomically checks whether a specified value is in a memory location,...
an instruction that atomically reads a memory location, combines it with another value,...
@ Min
*p = old <signed v ? old : v
@ UIncWrap
Increment one up to a maximum value.
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
@ UMax
*p = old >unsigned v ? old : v
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
@ UDecWrap
Decrement one until a minimum value or zero.
Attribute getFnAttr(Attribute::AttrKind Kind) const
Return the attribute object that exists for the function.
StringRef getValueAsString() const
Return the attribute's value as a string.
LLVM Basic Block Representation.
bool hasAddressTaken() const
Returns true if there are any uses of this basic block other than direct branches,...
InstListType::const_iterator const_iterator
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
const Instruction & front() const
const Function * getParent() const
Return the enclosing method, or null if none.
const Instruction * getFirstNonPHIOrDbg(bool SkipPseudoOp=true) const
Returns a pointer to the first instruction in this block that is not a PHINode or a debug intrinsic,...
const Instruction & back() const
Legacy analysis pass which computes BlockFrequencyInfo.
Conditional or Unconditional Branch instruction.
BasicBlock * getSuccessor(unsigned i) const
bool isUnconditional() const
Value * getCondition() const
Legacy analysis pass which computes BranchProbabilityInfo.
Analysis providing branch probability information.
BranchProbability getEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors) const
Get an edge's probability, relative to other out-edges of the Src.
static BranchProbability getZero()
static void normalizeProbabilities(ProbabilityIter Begin, ProbabilityIter End)
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
bool isInlineAsm() const
Check if this call is an inline asm statement.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const
Determine whether the argument or parameter has the given attribute.
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
unsigned countOperandBundlesOfType(StringRef Name) const
Return the number of operand bundles with the tag Name attached to this instruction.
Value * getCalledOperand() const
Value * getArgOperand(unsigned i) const
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
unsigned arg_size() const
AttributeList getAttributes() const
Return the parameter attributes for this call.
This class represents a function call, abstracting a target machine's calling convention.
bool isMustTailCall() const
bool checkReturnTypeForCallConv(MachineFunction &MF) const
Toplevel function to check the return type based on the target calling convention.
virtual bool lowerFormalArguments(MachineIRBuilder &MIRBuilder, const Function &F, ArrayRef< ArrayRef< Register > > VRegs, FunctionLoweringInfo &FLI) const
This hook must be implemented to lower the incoming (formal) arguments, described by VRegs,...
virtual bool enableBigEndian() const
For targets which want to use big-endian can enable it with enableBigEndian() hook.
virtual bool supportSwiftError() const
virtual bool lowerReturn(MachineIRBuilder &MIRBuilder, const Value *Val, ArrayRef< Register > VRegs, FunctionLoweringInfo &FLI, Register SwiftErrorVReg) const
This hook must be implemented to lower outgoing return values, described by Val, into the specified v...
virtual bool lowerCall(MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info) const
This hook must be implemented to lower the given call instruction, including argument and return valu...
virtual bool fallBackToDAGISel(const MachineFunction &MF) const
This class is the base class for the comparison instructions.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ FCMP_TRUE
1 1 1 1 Always true (always folded)
@ ICMP_SLT
signed less than
@ ICMP_SLE
signed less or equal
@ ICMP_UGT
unsigned greater than
@ ICMP_ULE
unsigned less or equal
@ FCMP_FALSE
0 0 0 0 Always false (always folded)
bool isFPPredicate() const
bool isIntPredicate() const
This is the shared class of boolean and integer constants.
static ConstantInt * getTrue(LLVMContext &Context)
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
const APInt & getValue() const
Return the constant as an APInt value reference.
This is an important base class in LLVM.
static Constant * getAllOnesValue(Type *Ty)
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
This is the common base class for constrained floating point intrinsics.
std::optional< fp::ExceptionBehavior > getExceptionBehavior() const
bool isEntryValue() const
Check if the expression consists of exactly one entry value operand.
static DIExpression * append(const DIExpression *Expr, ArrayRef< uint64_t > Ops)
Append the opcodes Ops to DIExpr.
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this label.
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this variable.
Records a position in IR for a source label (DILabel).
Record of a variable value-assignment, aka a non instruction representation of the dbg....
DIExpression * getExpression() const
Value * getVariableLocationOp(unsigned OpIdx) const
DILocalVariable * getVariable() const
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
unsigned getPointerSizeInBits(unsigned AS=0) const
Layout pointer size, in bits FIXME: The defaults need to be removed once all of the backends/clients ...
const StructLayout * getStructLayout(StructType *Ty) const
Returns a StructLayout object, indicating the alignment of the struct, its size, and the offsets of i...
IntegerType * getIndexType(LLVMContext &C, unsigned AddressSpace) const
Returns the type of a GEP index in AddressSpace.
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
TypeSize getTypeSizeInBits(Type *Ty) const
Size examples:
TypeSize getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type.
Align getPointerABIAlignment(unsigned AS) const
Layout pointer alignment.
This represents the llvm.dbg.declare instruction.
Value * getAddress() const
This represents the llvm.dbg.label instruction.
DILabel * getLabel() const
Base class for non-instruction debug metadata records that have positions within IR.
DebugLoc getDebugLoc() const
This represents the llvm.dbg.value instruction.
Value * getValue(unsigned OpIdx=0) const
DILocalVariable * getVariable() const
DIExpression * getExpression() const
Class representing an expression and its matching format.
This instruction compares its operands according to the predicate given to the constructor.
An instruction for ordering other memory operations.
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this fence instruction.
AtomicOrdering getOrdering() const
Returns the ordering constraint of this fence instruction.
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
BranchProbabilityInfo * BPI
void clear()
clear - Clear out all the function-specific state.
bool CanLowerReturn
CanLowerReturn - true iff the function's return value can be lowered to registers.
bool skipFunction(const Function &F) const
Optional passes call this function to check whether the pass should be skipped.
const BasicBlock & getEntryBlock() const
DISubprogram * getSubprogram() const
Get the attached subprogram.
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Constant * getPersonalityFn() const
Get the personality function associated with this function.
const Function & getFunction() const
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
The actual analysis pass wrapper.
Simple wrapper that does the following.
Abstract class that contains various methods for clients to notify about changes.
Simple wrapper observer that takes several observers, and calls each one for each event.
void removeObserver(GISelChangeObserver *O)
void addObserver(GISelChangeObserver *O)
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
bool hasExternalWeakLinkage() const
bool hasDLLImportStorageClass() const
Module * getParent()
Get the module that this global value is contained inside of...
bool isTailCall(const MachineInstr &MI) const override
This instruction compares its operands according to the predicate given to the constructor.
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
IRTranslator(CodeGenOptLevel OptLevel=CodeGenOptLevel::None)
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
Indirect Branch Instruction.
bool lowerInlineAsm(MachineIRBuilder &MIRBuilder, const CallBase &CB, std::function< ArrayRef< Register >(const Value &Val)> GetOrCreateVRegs) const
Lower the given inline asm call instruction GetOrCreateVRegs is a callback to materialize a register ...
This instruction inserts a struct field of array element value into an aggregate value.
iterator_range< simple_ilist< DbgRecord >::iterator > getDbgRecordRange() const
Return a range over the DbgRecords attached to this instruction.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
bool hasMetadata() const
Return true if this instruction has any metadata attached to it.
const BasicBlock * getParent() const
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
AAMDNodes getAAMetadata() const
Returns the AA metadata for this instruction.
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
bool hasAllowReassoc() const LLVM_READONLY
Determine whether the allow-reassociation flag is set.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
constexpr LLT changeElementType(LLT NewEltTy) const
If this type is a vector, return a vector with the same number of elements but the new element type.
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
constexpr uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
constexpr bool isVector() const
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
constexpr bool isPointer() const
static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
constexpr bool isFixedVector() const
Returns true if the LLT is a fixed vector.
The landingpad instruction holds all of the information necessary to generate correct exception handl...
An instruction for reading from memory.
Value * getPointerOperand()
AtomicOrdering getOrdering() const
Returns the ordering constraint of this load instruction.
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this load instruction.
static LocationSize precise(uint64_t Value)
Context object for machine code objects.
MCSymbol * getOrCreateFrameAllocSymbol(const Twine &FuncName, unsigned Idx)
Gets a symbol that will be defined to the final stack offset of a local variable after codegen.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
unsigned pred_size() const
void normalizeSuccProbs()
Normalize probabilities of all successors so that the sum of them becomes one.
void setAddressTakenIRBlock(BasicBlock *BB)
Set this block to reflect that it corresponds to an IR-level basic block with a BlockAddress.
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void setSuccProbability(succ_iterator I, BranchProbability Prob)
Set successor probability of a given iterator.
succ_iterator succ_begin()
std::vector< MachineBasicBlock * >::iterator succ_iterator
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
void sortUniqueLiveIns()
Sorts and uniques the LiveIns vector.
bool isPredecessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB is a predecessor of this block.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
void setIsEHPad(bool V=true)
Indicates the block is a landing pad.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
bool hasVarSizedObjects() const
This method may be called any time after instruction selection is complete to determine if the stack ...
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
int getStackProtectorIndex() const
Return the index for the stack protector object.
void setStackProtectorIndex(int I)
int CreateVariableSizedObject(Align Alignment, const AllocaInst *Alloca)
Notify the MachineFrameInfo object that a variable sized object has been created.
void setHasMustTailInVarArgFunc(bool B)
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
ArrayRef< int > allocateShuffleMask(ArrayRef< int > Mask)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
unsigned getTypeIDFor(const GlobalValue *TI)
Return the type id for the specified typeinfo. This is function wide.
void push_back(MachineBasicBlock *MBB)
MCContext & getContext() const
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
MCSymbol * addLandingPad(MachineBasicBlock *LandingPad)
Add a new panding pad, and extract the exception handling information from the landingpad instruction...
void deleteMachineBasicBlock(MachineBasicBlock *MBB)
DeleteMachineBasicBlock - Delete the given MachineBasicBlock.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
MachineModuleInfo & getMMI() const
void remove(iterator MBBI)
void setVariableDbgInfo(const DILocalVariable *Var, const DIExpression *Expr, int Slot, const DILocation *Loc)
Collect information used to emit debugging information of a variable in a stack slot.
const MachineBasicBlock & front() const
void addInvoke(MachineBasicBlock *LandingPad, MCSymbol *BeginLabel, MCSymbol *EndLabel)
Provide the begin and end labels of an invoke style call and associate it with a try landing pad bloc...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void erase(iterator MBBI)
void insert(iterator MBBI, MachineBasicBlock *MBB)
Helper class to build MachineInstr.
MachineInstrBuilder buildFMul(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
MachineInstrBuilder buildFreeze(const DstOp &Dst, const SrcOp &Src)
Build and insert Dst = G_FREEZE Src.
MachineInstrBuilder buildBr(MachineBasicBlock &Dest)
Build and insert G_BR Dest.
std::optional< MachineInstrBuilder > materializePtrAdd(Register &Res, Register Op0, const LLT ValueTy, uint64_t Value)
Materialize and insert Res = G_PTR_ADD Op0, (G_CONSTANT Value)
MachineInstrBuilder buildAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_ADD Op0, Op1.
MachineInstrBuilder buildUndef(const DstOp &Res)
Build and insert Res = IMPLICIT_DEF.
MachineInstrBuilder buildFPExt(const DstOp &Res, const SrcOp &Op, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FPEXT Op.
MachineInstrBuilder buildJumpTable(const LLT PtrTy, unsigned JTI)
Build and insert Res = G_JUMP_TABLE JTI.
MachineInstrBuilder buildFence(unsigned Ordering, unsigned Scope)
Build and insert G_FENCE Ordering, Scope.
MachineInstrBuilder buildSelect(const DstOp &Res, const SrcOp &Tst, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert a Res = G_SELECT Tst, Op0, Op1.
MachineInstrBuilder buildFMA(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, const SrcOp &Src2, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FMA Op0, Op1, Op2.
MachineInstrBuilder buildMul(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_MUL Op0, Op1.
MachineInstrBuilder buildAnd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_AND Op0, Op1.
MachineInstrBuilder buildICmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1)
Build and insert a Res = G_ICMP Pred, Op0, Op1.
MachineInstrBuilder buildCast(const DstOp &Dst, const SrcOp &Src)
Build and insert an appropriate cast between two registers of equal size.
MachineBasicBlock::iterator getInsertPt()
Current insertion point for new instructions.
MachineInstrBuilder buildSExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_SEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...
MachineInstrBuilder buildAtomicCmpXchgWithSuccess(Register OldValRes, Register SuccessRes, Register Addr, Register CmpVal, Register NewVal, MachineMemOperand &MMO)
Build and insert OldValRes<def>, SuccessRes<def> = G_ATOMIC_CMPXCHG_WITH_SUCCESS Addr,...
MachineInstrBuilder buildAtomicRMW(unsigned Opcode, const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val, MachineMemOperand &MMO)
Build and insert OldValRes<def> = G_ATOMICRMW_<Opcode> Addr, Val, MMO.
MachineInstrBuilder buildSub(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_SUB Op0, Op1.
MachineInstrBuilder buildIntrinsic(Intrinsic::ID ID, ArrayRef< Register > Res, bool HasSideEffects, bool isConvergent)
Build and insert a G_INTRINSIC instruction.
MachineInstrBuilder buildSplatBuildVector(const DstOp &Res, const SrcOp &Src)
Build and insert Res = G_BUILD_VECTOR with Src replicated to fill the number of elements.
MachineInstrBuilder buildIndirectDbgValue(Register Reg, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instruction expressing the fact that the associated Variable lives in me...
MachineInstrBuilder buildConstDbgValue(const Constant &C, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instructions specifying that Variable is given by C (suitably modified b...
MachineInstrBuilder buildBrCond(const SrcOp &Tst, MachineBasicBlock &Dest)
Build and insert G_BRCOND Tst, Dest.
MachineInstrBuilder buildExtractVectorElement(const DstOp &Res, const SrcOp &Val, const SrcOp &Idx)
Build and insert Res = G_EXTRACT_VECTOR_ELT Val, Idx.
MachineInstrBuilder buildLoad(const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert Res = G_LOAD Addr, MMO.
MachineInstrBuilder buildPtrAdd(const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_PTR_ADD Op0, Op1.
MachineInstrBuilder buildZExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ZEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...
MachineInstrBuilder buildExtractVectorElementConstant(const DstOp &Res, const SrcOp &Val, const int Idx)
Build and insert Res = G_EXTRACT_VECTOR_ELT Val, Idx.
MachineInstrBuilder buildShl(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
MachineInstrBuilder buildStore(const SrcOp &Val, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert G_STORE Val, Addr, MMO.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
MachineInstrBuilder buildFrameIndex(const DstOp &Res, int Idx)
Build and insert Res = G_FRAME_INDEX Idx.
MachineInstrBuilder buildDirectDbgValue(Register Reg, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instruction expressing the fact that the associated Variable lives in Re...
MachineInstrBuilder buildDbgLabel(const MDNode *Label)
Build and insert a DBG_LABEL instructions specifying that Label is given.
MachineInstrBuilder buildBrJT(Register TablePtr, unsigned JTI, Register IndexReg)
Build and insert G_BRJT TablePtr, JTI, IndexReg.
MachineInstrBuilder buildDynStackAlloc(const DstOp &Res, const SrcOp &Size, Align Alignment)
Build and insert Res = G_DYN_STACKALLOC Size, Align.
MachineInstrBuilder buildFIDbgValue(int FI, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instruction expressing the fact that the associated Variable lives in th...
void setDebugLoc(const DebugLoc &DL)
Set the debug location to DL for all the next build instructions.
const MachineBasicBlock & getMBB() const
Getter for the basic block we currently build.
MachineInstrBuilder buildInsertVectorElement(const DstOp &Res, const SrcOp &Val, const SrcOp &Elt, const SrcOp &Idx)
Build and insert Res = G_INSERT_VECTOR_ELT Val, Elt, Idx.
void setMBB(MachineBasicBlock &MBB)
Set the insertion point to the end of MBB.
const DebugLoc & getDebugLoc()
Get the current instruction's debug location.
MachineInstrBuilder buildFFrexp(const DstOp &Fract, const DstOp &Exp, const SrcOp &Src, std::optional< unsigned > Flags=std::nullopt)
Build and insert Fract, Exp = G_FFREXP Src.
MachineInstrBuilder buildFPTrunc(const DstOp &Res, const SrcOp &Op, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FPTRUNC Op.
MachineInstrBuilder buildShuffleVector(const DstOp &Res, const SrcOp &Src1, const SrcOp &Src2, ArrayRef< int > Mask)
Build and insert Res = G_SHUFFLE_VECTOR Src1, Src2, Mask.
MachineInstrBuilder buildInstrNoInsert(unsigned Opcode)
Build but don't insert <empty> = Opcode <empty>.
MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)
Build and insert Res = COPY Op.
MachineInstrBuilder buildPrefetch(const SrcOp &Addr, unsigned RW, unsigned Locality, unsigned CacheType, MachineMemOperand &MMO)
Build and insert G_PREFETCH Addr, RW, Locality, CacheType.
const DataLayout & getDataLayout() const
MachineInstrBuilder buildBrIndirect(Register Tgt)
Build and insert G_BRINDIRECT Tgt.
MachineInstrBuilder buildSplatVector(const DstOp &Res, const SrcOp &Val)
Build and insert Res = G_SPLAT_VECTOR Val.
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
MachineInstrBuilder buildFCmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert a Res = G_FCMP PredOp0, Op1.
MachineInstrBuilder buildFAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FADD Op0, Op1.
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addMetadata(const MDNode *MD) const
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addFPImm(const ConstantFP *Val) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Representation of each machine instruction.
void copyIRFlags(const Instruction &I)
Copy all flags to MachineInst MIFlags.
static uint32_t copyFlagsFromInstruction(const Instruction &I)
const MachineOperand & getOperand(unsigned i) const
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
const MCContext & getContext() const
static MachineOperand CreateES(const char *SymName, unsigned TargetFlags=0)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateGA(const GlobalValue *GV, int64_t Offset, unsigned TargetFlags=0)
MachineInstr * getVRegDef(Register Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
LLT getType(Register Reg) const
Get the low-level type of Reg or LLT{} if Reg is not a generic (target independent) virtual register.
void setRegClass(Register Reg, const TargetRegisterClass *RC)
setRegClass - Set the register class of the specified virtual register.
Register createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
void addPhysRegsUsedFromRegMask(const uint32_t *RegMask)
addPhysRegsUsedFromRegMask - Mark any registers not in RegMask as used.
Representation for a specific memory location.
A Module instance is used to store all the information related to an LLVM module.
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
unsigned getNumIncomingValues() const
Return the number of incoming edges.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
A simple RAII based Delegate installer.
A simple RAII based Observer installer.
Wrapper class representing virtual and physical registers.
MCRegister asMCReg() const
Utility to check-convert this value to a MCRegister.
Return a value (possibly void), from a function.
Value * getReturnValue() const
Convenience accessor. Returns null if there is no return value.
This class represents the LLVM 'select' instruction.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Encapsulates all of the information needed to generate a stack protector check, and signals to isel w...
void initialize(const BasicBlock *BB, MachineBasicBlock *MBB, bool FunctionBasedInstrumentation)
Initialize the stack protector descriptor structure for a new basic block.
MachineBasicBlock * getSuccessMBB()
void resetPerBBState()
Reset state that changes when we handle different basic blocks.
void resetPerFunctionState()
Reset state that only changes when we switch functions.
MachineBasicBlock * getFailureMBB()
MachineBasicBlock * getParentMBB()
bool shouldEmitStackProtector() const
Returns true if all fields of the stack protector descriptor are initialized implying that we should/...
bool shouldEmitFunctionBasedCheckStackProtector() const
bool shouldEmitSDCheck(const BasicBlock &BB) const
void copyToMachineFrameInfo(MachineFrameInfo &MFI) const
An instruction for storing to memory.
StringRef - Represent a constant reference to a string, i.e.
constexpr bool empty() const
empty - Check if the string is empty.
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
TypeSize getElementOffset(unsigned Idx) const
Class to represent struct types.
bool createEntriesInEntryBlock(DebugLoc DbgLoc)
Create initial definitions of swifterror values in the entry block of the current function.
void setFunction(MachineFunction &MF)
Initialize data structures for specified new function.
void setCurrentVReg(const MachineBasicBlock *MBB, const Value *, Register)
Set the swifterror virtual register in the VRegDefMap for this basic block.
Register getOrCreateVRegUseAt(const Instruction *, const MachineBasicBlock *, const Value *)
Get or create the swifterror value virtual register for a use of a swifterror by an instruction.
Register getOrCreateVRegDefAt(const Instruction *, const MachineBasicBlock *, const Value *)
Get or create the swifterror value virtual register for a def of a swifterror by an instruction.
const Value * getFunctionArg() const
Get the (unique) function argument that was marked swifterror, or nullptr if this function has no swi...
void propagateVRegs()
Propagate assigned swifterror vregs through a function, synthesizing PHI nodes when needed to maintai...
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
TargetInstrInfo - Interface to description of machine instruction set.
virtual bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT) const
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
virtual unsigned getVaListSizeInBits(const DataLayout &DL) const
Returns the size of the platform's va_list object.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
virtual bool useStackGuardXorFP() const
If this function returns true, stack protection checks should XOR the frame pointer (or whichever poi...
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
virtual Value * getSDagStackGuard(const Module &M) const
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
bool isJumpExpensive() const
Return true if Flow Control is an expensive operation that should be avoided.
virtual Function * getSSPStackGuardCheck(const Module &M) const
If the target has a standard stack protection check function that performs validation and error handl...
MachineMemOperand::Flags getAtomicMemOperandFlags(const Instruction &AI, const DataLayout &DL) const
virtual bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &, MachineFunction &, unsigned) const
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
MachineMemOperand::Flags getLoadMemOperandFlags(const LoadInst &LI, const DataLayout &DL, AssumptionCache *AC=nullptr, const TargetLibraryInfo *LibInfo=nullptr) const
MachineMemOperand::Flags getStoreMemOperandFlags(const StoreInst &SI, const DataLayout &DL) const
virtual bool fallBackToDAGISel(const Instruction &Inst) const
virtual Register getExceptionPointerRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception address on entry to an ...
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
virtual Register getExceptionSelectorRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception typeid on entry to a la...
virtual MVT getPointerMemTy(const DataLayout &DL, uint32_t AS=0) const
Return the in-memory pointer type for the given address space, defaults to the pointer type from the ...
virtual bool useLoadStackGuardNode() const
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
Primary interface to the complete machine description for the target machine.
virtual const TargetIntrinsicInfo * getIntrinsicInfo() const
If intrinsic information is available, return it. If not, return null.
const Triple & getTargetTriple() const
CodeGenOptLevel getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
unsigned NoTrapAfterNoreturn
Do not emit a trap instruction for 'unreachable' IR instructions behind noreturn calls,...
unsigned TrapUnreachable
Emit target-specific trap instruction for 'unreachable' IR instructions.
Target-Independent Code Generator Pass Configuration Options.
virtual std::unique_ptr< CSEConfigBase > getCSEConfig() const
Returns the CSEConfig object to use for the current optimization level.
virtual bool isGISelCSEEnabled() const
Check whether continuous CSE should be enabled in GISel passes.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const InlineAsmLowering * getInlineAsmLowering() const
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const CallLowering * getCallLowering() const
virtual const TargetFrameLowering * getFrameLowering() const
virtual const TargetInstrInfo * getInstrInfo() const
virtual const TargetLowering * getTargetLowering() const
bool isOSWindows() const
Tests whether the OS is Windows.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
bool isEmptyTy() const
Return true if this type is empty, that is, it has no elements or all of its elements are empty.
TypeID
Definitions of all of the base types for the Type system.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static Type * getVoidTy(LLVMContext &C)
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
bool isAggregateType() const
Return true if the type is an aggregate type.
static IntegerType * getInt32Ty(LLVMContext &C)
bool isTokenTy() const
Return true if this is 'token'.
bool isVoidTy() const
Return true if this is 'void'.
Value * getOperand(unsigned i) const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
LLVMContext & getContext() const
All values hold a context through their type.
constexpr bool isZero() const
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
A raw_ostream that writes to an std::string.
std::string & str()
Returns the string's reference.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ C
The default llvm calling convention, compatible with C.
bool match(Val *V, const Pattern &P)
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
TwoOps_match< Val_t, Idx_t, Instruction::ExtractElement > m_ExtractElt(const Val_t &Val, const Idx_t &Idx)
Matches ExtractElementInst.
OneUse_match< T > m_OneUse(const T &SubPattern)
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
BinaryOp_match< cst_pred_ty< is_all_ones >, ValTy, Instruction::Xor, true > m_Not(const ValTy &V)
Matches a 'Not' as 'xor V, -1' or 'xor -1, V'.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
@ Undef
Value of the register doesn't matter.
SmallVector< SwitchWorkListItem, 4 > SwitchWorkList
std::vector< CaseCluster > CaseClusterVector
void sortAndRangeify(CaseClusterVector &Clusters)
Sort Clusters and merge adjacent cases.
CaseClusterVector::iterator CaseClusterIt
@ CC_Range
A cluster of adjacent case labels with the same destination, or just one case.
@ CC_JumpTable
A cluster of cases suitable for jump table lowering.
@ CC_BitTests
A cluster of cases suitable for bit test lowering.
@ CE
Windows NT (Windows on ARM)
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
ExceptionBehavior
Exception behavior used for floating point operations.
@ ebIgnore
This corresponds to "fpexcept.ignore".
DiagnosticInfoOptimizationBase::Argument NV
NodeAddr< PhiNode * > Phi
This is an optimization pass for GlobalISel generic memory operations.
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
int popcount(T Value) noexcept
Count the number of set bits in a value.
bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
void diagnoseDontCall(const CallInst &CI)
auto successors(const MachineBasicBlock *BB)
MVT getMVTForLLT(LLT Ty)
Get a rough equivalent of an MVT for a given LLT.
gep_type_iterator gep_type_end(const User *GEP)
MachineBasicBlock::iterator findSplitPointForStackProtector(MachineBasicBlock *BB, const TargetInstrInfo &TII)
Find the split point at which to splice the end of BB into its success stack protector check machine ...
LLT getLLTForMVT(MVT Ty)
Get a rough equivalent of an LLT for a given MVT.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Align getKnownAlignment(Value *V, const DataLayout &DL, const Instruction *CxtI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr)
Try to infer an alignment for the specified pointer.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
llvm::SmallVector< int, 16 > createStrideMask(unsigned Start, unsigned Stride, unsigned VF)
Create a stride shuffle mask.
auto reverse(ContainerTy &&C)
void computeValueLLTs(const DataLayout &DL, Type &Ty, SmallVectorImpl< LLT > &ValueTys, SmallVectorImpl< uint64_t > *Offsets=nullptr, uint64_t StartingOffset=0)
computeValueLLTs - Given an LLVM IR type, compute a sequence of LLTs that represent all the individua...
void sort(IteratorTy Start, IteratorTy End)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
CodeGenOptLevel
Code generation optimization level.
@ Global
Append to llvm.global_dtors.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
void getUnderlyingObjects(const Value *V, SmallVectorImpl< const Value * > &Objects, LoopInfo *LI=nullptr, unsigned MaxLookup=6)
This method is similar to getUnderlyingObject except that it can look through phi and select instruct...
void getSelectionDAGFallbackAnalysisUsage(AnalysisUsage &AU)
Modify analysis usage so it preserves passes required for the SelectionDAG fallback.
auto lower_bound(R &&Range, T &&Value)
Provide wrappers to std::lower_bound which take ranges instead of having to pass begin/end explicitly...
llvm::SmallVector< int, 16 > createInterleaveMask(unsigned VF, unsigned NumVecs)
Create an interleave shuffle mask.
bool isAsynchronousEHPersonality(EHPersonality Pers)
Returns true if this personality function catches asynchronous exceptions.
OutputIt copy(R &&Range, OutputIt Out)
std::optional< RoundingMode > convertStrToRoundingMode(StringRef)
Returns a valid RoundingMode enumerator when given a string that is valid as input in constrained int...
gep_type_iterator gep_type_begin(const User *GEP)
GlobalValue * ExtractTypeInfo(Value *V)
ExtractTypeInfo - Returns the type info, possibly bitcast, encoded in V.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
unsigned succ_size(const MachineBasicBlock *BB)
LLT getLLTForType(Type &Ty, const DataLayout &DL)
Construct a low-level type based on an LLVM type.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
Pair of physical register and lane mask.
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
MachineBasicBlock * Parent
This structure is used to communicate between SelectionDAGBuilder and SDISel for the code generation ...
BranchProbability TrueProb
MachineBasicBlock * ThisBB
struct PredInfoPair PredInfo
BranchProbability FalseProb
MachineBasicBlock * TrueBB
MachineBasicBlock * FalseBB