Go to the documentation of this file.
74 #define DEBUG_TYPE "code-extractor"
82 cl::desc(
"Aggregate arguments to code-extracted functions"));
87 bool AllowVarArgs,
bool AllowAlloca) {
89 if (
BB.hasAddressTaken())
98 ToVisit.push_back(&Inst);
100 while (!ToVisit.empty()) {
102 if (!Visited.
insert(Curr).second)
104 if (isa<BlockAddress const>(Curr))
107 if (isa<Instruction>(Curr) && cast<Instruction>(Curr)->
getParent() != &
BB)
110 for (
auto const &U : Curr->
operands()) {
111 if (
auto *UU = dyn_cast<User>(U))
112 ToVisit.push_back(UU);
119 if (isa<AllocaInst>(
I)) {
125 if (
const auto *II = dyn_cast<InvokeInst>(
I)) {
128 if (
auto *UBB = II->getUnwindDest())
129 if (!Result.count(UBB))
136 if (
const auto *CSI = dyn_cast<CatchSwitchInst>(
I)) {
137 if (
auto *UBB = CSI->getUnwindDest())
138 if (!Result.count(UBB))
140 for (
const auto *HBB : CSI->handlers())
141 if (!Result.count(
const_cast<BasicBlock*
>(HBB)))
148 if (
const auto *CPI = dyn_cast<CatchPadInst>(
I)) {
149 for (
const auto *U : CPI->users())
150 if (
const auto *CRI = dyn_cast<CatchReturnInst>(U))
151 if (!Result.count(
const_cast<BasicBlock*
>(CRI->getParent())))
159 if (
const auto *CPI = dyn_cast<CleanupPadInst>(
I)) {
160 for (
const auto *U : CPI->users())
161 if (
const auto *CRI = dyn_cast<CleanupReturnInst>(U))
162 if (!Result.count(
const_cast<BasicBlock*
>(CRI->getParent())))
166 if (
const auto *CRI = dyn_cast<CleanupReturnInst>(
I)) {
167 if (
auto *UBB = CRI->getUnwindDest())
168 if (!Result.count(UBB))
173 if (
const CallInst *CI = dyn_cast<CallInst>(
I)) {
174 if (
const Function *
F = CI->getCalledFunction()) {
175 auto IID =
F->getIntrinsicID();
176 if (IID == Intrinsic::vastart) {
185 if (IID == Intrinsic::eh_typeid_for)
197 bool AllowVarArgs,
bool AllowAlloca) {
198 assert(!BBs.
empty() &&
"The set of blocks to extract must be non-empty");
208 if (!Result.insert(
BB))
212 LLVM_DEBUG(
dbgs() <<
"Region front block: " << Result.front()->getName()
215 for (
auto *
BB : Result) {
220 if (
BB == Result.front()) {
222 LLVM_DEBUG(
dbgs() <<
"The first block cannot be an unwind block\n");
231 if (!Result.count(PBB)) {
232 LLVM_DEBUG(
dbgs() <<
"No blocks in this region may have entries from "
233 "outside the region except for the first block!\n"
234 <<
"Problematic source BB: " <<
BB->getName() <<
"\n"
235 <<
"Problematic destination BB: " << PBB->getName()
247 bool AllowVarArgs,
bool AllowAlloca,
248 BasicBlock *AllocationBlock, std::string Suffix)
250 BPI(BPI), AC(AC), AllocationBlock(AllocationBlock),
251 AllowVarArgs(AllowVarArgs),
260 BPI(BPI), AC(AC), AllocationBlock(nullptr), AllowVarArgs(
false),
270 if (Blocks.
count(
I->getParent()))
279 if (isa<Argument>(V))
return true;
281 if (!Blocks.
count(
I->getParent()))
288 auto hasNonCommonExitSucc = [&](
BasicBlock *Block) {
291 if (Blocks.
count(Succ))
293 if (!CommonExitBlock) {
294 CommonExitBlock = Succ;
297 if (CommonExitBlock != Succ)
303 if (
any_of(Blocks, hasNonCommonExitSucc))
306 return CommonExitBlock;
312 if (
auto *AI = dyn_cast<AllocaInst>(&II))
313 Allocas.push_back(AI);
315 findSideEffectInfoForBlock(
BB);
319 void CodeExtractorAnalysisCache::findSideEffectInfoForBlock(
BasicBlock &
BB) {
321 unsigned Opcode = II.getOpcode();
322 Value *MemAddr =
nullptr;
328 MemAddr =
SI->getPointerOperand();
334 if (isa<Constant>(MemAddr))
337 if (!isa<AllocaInst>(
Base)) {
341 BaseMemAddrs[&
BB].insert(
Base);
353 if (II.mayHaveSideEffects()) {
364 if (SideEffectingBlocks.
count(&
BB))
366 auto It = BaseMemAddrs.find(&
BB);
367 if (It != BaseMemAddrs.end())
368 return It->second.count(
Addr);
374 AllocaInst *AI = cast<AllocaInst>(
Addr->stripInBoundsConstantOffsets());
387 BasicBlock *SinglePredFromOutlineRegion =
nullptr;
389 "Expect a block outside the region!");
391 if (!Blocks.
count(Pred))
393 if (!SinglePredFromOutlineRegion) {
394 SinglePredFromOutlineRegion = Pred;
395 }
else if (SinglePredFromOutlineRegion != Pred) {
396 SinglePredFromOutlineRegion =
nullptr;
401 if (SinglePredFromOutlineRegion)
402 return SinglePredFromOutlineRegion;
408 while (
I !=
BB->end()) {
409 PHINode *Phi = dyn_cast<PHINode>(
I);
421 assert(!getFirstPHI(CommonExitBlock) &&
"Phi not expected");
429 if (Blocks.
count(Pred))
434 Blocks.
insert(CommonExitBlock);
435 OldTargets.push_back(NewExitBlock);
436 return CommonExitBlock;
443 CodeExtractor::LifetimeMarkerInfo
447 LifetimeMarkerInfo
Info;
457 Info.LifeStart = IntrInst;
463 Info.LifeEnd = IntrInst;
468 if (isa<DbgInfoIntrinsic>(IntrInst))
476 if (!
Info.LifeStart || !
Info.LifeEnd)
482 if ((
Info.SinkLifeStart ||
Info.HoistLifeEnd) &&
487 if (
Info.HoistLifeEnd && !ExitBlock)
499 auto moveOrIgnoreLifetimeMarkers =
500 [&](
const LifetimeMarkerInfo &LMI) ->
bool {
503 if (LMI.SinkLifeStart) {
506 SinkCands.
insert(LMI.LifeStart);
508 if (LMI.HoistLifeEnd) {
509 LLVM_DEBUG(
dbgs() <<
"Hoisting lifetime.end: " << *LMI.LifeEnd <<
"\n");
510 HoistCands.
insert(LMI.LifeEnd);
528 LifetimeMarkerInfo MarkerInfo = getLifetimeMarkers(CEAC, AI, ExitBlock);
529 bool Moved = moveOrIgnoreLifetimeMarkers(MarkerInfo);
545 if (U->stripInBoundsConstantOffsets() != AI)
561 << *
Bitcast <<
" in out-of-region lifetime marker "
562 << *IntrInst <<
"\n");
563 LifetimeBitcastUsers.push_back(IntrInst);
573 I->replaceUsesOfWith(
I->getOperand(1), CastI);
580 if (U->stripInBoundsConstantOffsets() == AI) {
582 LifetimeMarkerInfo LMI = getLifetimeMarkers(CEAC,
Bitcast, ExitBlock);
585 BitcastLifetimeInfo.push_back(LMI);
598 if (Bitcasts.empty())
601 LLVM_DEBUG(
dbgs() <<
"Sinking alloca (via bitcast): " << *AI <<
"\n");
603 for (
unsigned I = 0,
E = Bitcasts.size();
I !=
E; ++
I) {
605 const LifetimeMarkerInfo &LMI = BitcastLifetimeInfo[
I];
607 "Unsafe to sink bitcast without lifetime markers");
608 moveOrIgnoreLifetimeMarkers(LMI);
610 LLVM_DEBUG(
dbgs() <<
"Sinking bitcast-of-alloca: " << *BitcastAddr
612 SinkCands.
insert(BitcastAddr);
626 if (AllowVarArgs &&
F->getFunctionType()->isVarArg()) {
627 auto containsVarArgIntrinsic = [](
const Instruction &
I) {
628 if (
const CallInst *CI = dyn_cast<CallInst>(&
I))
630 return Callee->getIntrinsicID() == Intrinsic::vastart ||
631 Callee->getIntrinsicID() == Intrinsic::vaend;
635 for (
auto &
BB : *
F) {
651 for (
auto &OI : II.operands()) {
669 void CodeExtractor::severSplitPHINodesOfEntry(
BasicBlock *&Header) {
670 unsigned NumPredsFromRegion = 0;
671 unsigned NumPredsOutsideRegion = 0;
673 if (Header != &Header->getParent()->getEntryBlock()) {
674 PHINode *PN = dyn_cast<PHINode>(Header->begin());
682 ++NumPredsFromRegion;
684 ++NumPredsOutsideRegion;
688 if (NumPredsOutsideRegion <= 1)
return;
706 if (NumPredsFromRegion) {
719 for (AfterPHIs = OldPred->
begin(); isa<PHINode>(AfterPHIs); ++AfterPHIs) {
720 PHINode *PN = cast<PHINode>(AfterPHIs);
745 void CodeExtractor::severSplitPHINodesOfExits(
750 for (
PHINode &PN : ExitBB->phis()) {
755 IncomingVals.push_back(
i);
760 if (IncomingVals.size() <= 1)
767 ExitBB->getName() +
".split",
768 ExitBB->getParent(), ExitBB);
771 if (Blocks.
count(PredBB))
772 PredBB->getTerminator()->replaceUsesOfWith(ExitBB, NewBB);
781 for (
unsigned i : IncomingVals)
783 for (
unsigned i :
reverse(IncomingVals))
790 void CodeExtractor::splitReturnBlocks() {
794 Block->splitBasicBlock(RI->getIterator(),
Block->getName() +
".ret");
812 Function *CodeExtractor::constructFunction(
const ValueSet &inputs,
813 const ValueSet &outputs,
823 switch (NumExitBlocks) {
830 std::vector<Type *> ParamTy;
831 std::vector<Type *> AggParamTy;
832 ValueSet StructValues;
836 for (
Value *value : inputs) {
838 if (AggregateArgs && !ExcludeArgsFromAggregate.
contains(value)) {
839 AggParamTy.push_back(value->getType());
840 StructValues.insert(value);
842 ParamTy.push_back(value->getType());
848 if (AggregateArgs && !ExcludeArgsFromAggregate.
contains(
output)) {
849 AggParamTy.push_back(
output->getType());
850 StructValues.insert(
output);
857 (ParamTy.size() + AggParamTy.size()) ==
858 (inputs.size() + outputs.size()) &&
859 "Number of scalar and aggregate params does not match inputs, outputs");
860 assert((StructValues.empty() || AggregateArgs) &&
861 "Expeced StructValues only with AggregateArgs set");
864 size_t NumScalarParams = ParamTy.size();
866 if (AggregateArgs && !AggParamTy.empty()) {
872 dbgs() <<
"Function type: " << *RetTy <<
" f(";
873 for (
Type *
i : ParamTy)
874 dbgs() << *
i <<
", ";
879 RetTy, ParamTy, AllowVarArgs && oldFunction->
isVarArg());
881 std::string SuffixToUse =
888 oldFunction->
getName() +
"." + SuffixToUse,
M);
898 if (Attr.isStringAttribute()) {
899 if (Attr.getKindAsString() ==
"thunk")
902 switch (Attr.getKindAsEnum()) {
905 case Attribute::AllocSize:
906 case Attribute::Builtin:
909 case Attribute::Naked:
910 case Attribute::NoBuiltin:
911 case Attribute::NoMerge:
912 case Attribute::NoReturn:
913 case Attribute::NoSync:
914 case Attribute::ReturnsTwice:
915 case Attribute::Speculatable:
916 case Attribute::StackAlignment:
917 case Attribute::WillReturn:
918 case Attribute::AllocKind:
919 case Attribute::PresplitCoroutine:
920 case Attribute::Memory:
923 case Attribute::AlwaysInline:
925 case Attribute::DisableSanitizerInstrumentation:
926 case Attribute::FnRetThunkExtern:
928 case Attribute::NoRecurse:
929 case Attribute::InlineHint:
930 case Attribute::MinSize:
931 case Attribute::NoCallback:
932 case Attribute::NoDuplicate:
933 case Attribute::NoFree:
934 case Attribute::NoImplicitFloat:
935 case Attribute::NoInline:
936 case Attribute::NonLazyBind:
937 case Attribute::NoRedZone:
938 case Attribute::NoUnwind:
939 case Attribute::NoSanitizeBounds:
940 case Attribute::NoSanitizeCoverage:
941 case Attribute::NullPointerIsValid:
942 case Attribute::OptForFuzzing:
943 case Attribute::OptimizeNone:
944 case Attribute::OptimizeForSize:
945 case Attribute::SafeStack:
946 case Attribute::ShadowCallStack:
947 case Attribute::SanitizeAddress:
948 case Attribute::SanitizeMemory:
949 case Attribute::SanitizeThread:
950 case Attribute::SanitizeHWAddress:
951 case Attribute::SanitizeMemTag:
952 case Attribute::SpeculativeLoadHardening:
953 case Attribute::StackProtect:
954 case Attribute::StackProtectReq:
955 case Attribute::StackProtectStrong:
956 case Attribute::StrictFP:
957 case Attribute::UWTable:
958 case Attribute::VScaleRange:
959 case Attribute::NoCfCheck:
960 case Attribute::MustProgress:
961 case Attribute::NoProfile:
962 case Attribute::SkipProfile:
965 case Attribute::Alignment:
966 case Attribute::AllocatedPointer:
967 case Attribute::AllocAlign:
968 case Attribute::ByVal:
969 case Attribute::Dereferenceable:
970 case Attribute::DereferenceableOrNull:
971 case Attribute::ElementType:
972 case Attribute::InAlloca:
973 case Attribute::InReg:
974 case Attribute::Nest:
975 case Attribute::NoAlias:
976 case Attribute::NoCapture:
977 case Attribute::NoUndef:
978 case Attribute::NonNull:
979 case Attribute::Preallocated:
980 case Attribute::ReadNone:
981 case Attribute::ReadOnly:
982 case Attribute::Returned:
983 case Attribute::SExt:
984 case Attribute::StructRet:
985 case Attribute::SwiftError:
986 case Attribute::SwiftSelf:
987 case Attribute::SwiftAsync:
988 case Attribute::ZExt:
989 case Attribute::ImmArg:
990 case Attribute::ByRef:
991 case Attribute::WriteOnly:
1002 newFunction->
insert(newFunction->
end(), newRootNode);
1011 for (
unsigned i = 0,
e = inputs.size(), aggIdx = 0;
i !=
e; ++
i) {
1013 if (AggregateArgs && StructValues.contains(inputs[
i])) {
1019 StructTy, &*AggAI, Idx,
"gep_" + inputs[
i]->
getName(), TI);
1021 "loadgep_" + inputs[
i]->getName(), TI);
1024 RewriteVal = &*ScalarAI++;
1026 std::vector<User *>
Users(inputs[
i]->user_begin(), inputs[
i]->user_end());
1029 if (Blocks.count(inst->getParent()))
1030 inst->replaceUsesOfWith(inputs[
i], RewriteVal);
1034 if (NumScalarParams) {
1036 for (
unsigned i = 0,
e = inputs.size();
i !=
e; ++
i, ++ScalarAI)
1037 if (!StructValues.contains(inputs[
i]))
1039 for (
unsigned i = 0,
e = outputs.size();
i !=
e; ++
i, ++ScalarAI)
1040 if (!StructValues.contains(outputs[
i]))
1048 for (
auto &U :
Users)
1052 if (
I->isTerminator() &&
I->getFunction() == oldFunction &&
1053 !Blocks.count(
I->getParent()))
1054 I->replaceUsesOfWith(header, newHeader);
1070 auto *II = dyn_cast<IntrinsicInst>(&
I);
1071 if (!II || !II->isLifetimeStartOrEnd())
1081 if (II->getIntrinsicID() == Intrinsic::lifetime_start)
1082 LifetimesStart.
insert(Mem);
1083 II->eraseFromParent();
1105 bool InsertBefore) {
1106 for (
Value *Mem : Objects) {
1109 "Input memory not defined in original function");
1110 Value *&MemAsI8Ptr = Bitcasts[Mem];
1112 if (Mem->getType() == Int8PtrTy)
1123 Marker->insertBefore(
Term);
1127 if (!LifetimesStart.
empty()) {
1129 M, llvm::Intrinsic::lifetime_start, Int8PtrTy);
1130 insertMarkers(StartFn, LifetimesStart,
true);
1133 if (!LifetimesEnd.
empty()) {
1135 M, llvm::Intrinsic::lifetime_end, Int8PtrTy);
1136 insertMarkers(EndFn, LifetimesEnd,
false);
1146 ValueSet &outputs) {
1149 std::vector<Value *> params, ReloadOutputs, Reloads;
1150 ValueSet StructValues;
1158 unsigned ScalarInputArgNo = 0;
1161 if (AggregateArgs && !ExcludeArgsFromAggregate.
contains(
input))
1164 params.push_back(
input);
1165 if (
input->isSwiftError())
1166 SwiftErrorArgs.push_back(ScalarInputArgNo);
1172 unsigned ScalarOutputArgNo = 0;
1174 if (AggregateArgs && !ExcludeArgsFromAggregate.
contains(
output)) {
1179 nullptr,
output->getName() +
".loc",
1181 ReloadOutputs.push_back(alloca);
1182 params.push_back(alloca);
1183 ++ScalarOutputArgNo;
1189 unsigned NumAggregatedInputs = 0;
1190 if (AggregateArgs && !StructValues.empty()) {
1191 std::vector<Type *> ArgTypes;
1192 for (
Value *V : StructValues)
1193 ArgTypes.push_back(V->getType());
1198 StructArgTy,
DL.getAllocaAddrSpace(),
nullptr,
"structArg",
1201 params.push_back(
Struct);
1204 for (
unsigned i = 0,
e = StructValues.size();
i !=
e; ++
i) {
1205 if (inputs.contains(StructValues[
i])) {
1211 GEP->insertInto(codeReplacer, codeReplacer->
end());
1213 NumAggregatedInputs++;
1220 NumExitBlocks > 1 ?
"targetBlock" :
"");
1229 call->insertInto(codeReplacer, codeReplacer->
end());
1232 for (
unsigned SwiftErrArgNo : SwiftErrorArgs) {
1233 call->addParamAttr(SwiftErrArgNo, Attribute::SwiftError);
1234 newFunction->
addParamAttr(SwiftErrArgNo, Attribute::SwiftError);
1239 for (
unsigned i = 0,
e = outputs.size(), scalarIdx = 0,
1240 aggIdx = NumAggregatedInputs;
1242 Value *Output =
nullptr;
1243 if (AggregateArgs && StructValues.contains(outputs[
i])) {
1248 StructArgTy,
Struct, Idx,
"gep_reload_" + outputs[
i]->
getName());
1249 GEP->insertInto(codeReplacer, codeReplacer->
end());
1253 Output = ReloadOutputs[scalarIdx];
1259 Reloads.push_back(
load);
1260 std::vector<User *>
Users(outputs[
i]->user_begin(), outputs[
i]->user_end());
1271 codeReplacer, 0, codeReplacer);
1278 std::map<BasicBlock *, BasicBlock *> ExitBlockMap;
1282 unsigned switchVal = 0;
1284 if (Blocks.count(OldTarget))
1286 BasicBlock *&NewTarget = ExitBlockMap[OldTarget];
1293 OldTarget->getName() +
".exitStub",
1295 unsigned SuccNum = switchVal++;
1297 Value *brVal =
nullptr;
1298 assert(NumExitBlocks < 0xffff &&
"too many exit blocks for switch");
1299 switch (NumExitBlocks) {
1325 BasicBlock *NewTarget = ExitBlockMap[OldTarget];
1326 assert(NewTarget &&
"Unknown target block!");
1337 std::advance(ScalarOutputArgBegin, ScalarInputArgNo);
1339 std::advance(AggOutputArgBegin, ScalarInputArgNo + ScalarOutputArgNo);
1341 for (
unsigned i = 0,
e = outputs.size(), aggIdx = NumAggregatedInputs;
i !=
e;
1343 auto *OutI = dyn_cast<Instruction>(outputs[
i]);
1351 if (
auto *InvokeI = dyn_cast<InvokeInst>(OutI))
1352 InsertPt = InvokeI->getNormalDest()->getFirstInsertionPt();
1353 else if (
auto *Phi = dyn_cast<PHINode>(OutI))
1354 InsertPt = Phi->getParent()->getFirstInsertionPt();
1356 InsertPt = std::next(OutI->getIterator());
1360 Blocks.count(InsertBefore->
getParent())) &&
1361 "InsertPt should be in new function");
1362 if (AggregateArgs && StructValues.contains(outputs[
i])) {
1364 "Number of aggregate output arguments should match "
1365 "the number of defined values");
1370 StructArgTy, &*AggOutputArgBegin, Idx,
"gep_" + outputs[
i]->
getName(),
1379 "Number of scalar output arguments should match "
1380 "the number of defined values");
1381 new StoreInst(outputs[
i], &*ScalarOutputArgBegin, InsertBefore);
1382 ++ScalarOutputArgBegin;
1388 switch (NumExitBlocks) {
1437 void CodeExtractor::moveCodeToFunction(
Function *newFunction) {
1441 Block->removeFromParent();
1448 newFuncIt = newFunction->
insert(std::next(newFuncIt), Block);
1452 void CodeExtractor::calculateNewCallTerminatorWeights(
1464 Distribution BranchDist;
1471 BlockNode ExitNode(
i);
1474 BranchDist.addExit(ExitNode, ExitFreq);
1480 if (BranchDist.Total == 0) {
1486 BranchDist.normalize();
1489 for (
unsigned I = 0,
E = BranchDist.Weights.size();
I <
E; ++
I) {
1490 const auto &Weight = BranchDist.Weights[
I];
1493 BranchWeights[Weight.TargetNode.Index] = Weight.Amount;
1495 EdgeProbabilities[Weight.TargetNode.Index] = BP;
1499 LLVMContext::MD_prof,
1510 if (DVI->getFunction() != &
F)
1511 DVI->eraseFromParent();
1534 assert(OldSP->getUnit() &&
"Missing compile unit for subprogram");
1540 DISubprogram::SPFlagOptimized |
1541 DISubprogram::SPFlagLocalToUnit;
1544 0, SPType, 0, DINode::FlagZero, SPFlags);
1557 auto *DII = dyn_cast<DbgInfoIntrinsic>(&
I);
1563 if (
auto *DLI = dyn_cast<DbgLabelInst>(&
I)) {
1564 if (DLI->getDebugLoc().getInlinedAt())
1566 DILabel *OldLabel = DLI->getLabel();
1567 DINode *&NewLabel = RemappedMetadata[OldLabel];
1570 *OldLabel->
getScope(), *NewSP, Ctx, Cache);
1578 auto IsInvalidLocation = [&NewFunc](
Value *Location) {
1582 (!isa<Constant>(Location) && !isa<Instruction>(Location)))
1584 Instruction *LocationInst = dyn_cast<Instruction>(Location);
1585 return LocationInst && LocationInst->
getFunction() != &NewFunc;
1588 auto *DVI = cast<DbgVariableIntrinsic>(DII);
1590 if (
any_of(DVI->location_ops(), IsInvalidLocation)) {
1591 DebugIntrinsicsToDelete.push_back(DVI);
1596 if (!DVI->getDebugLoc().getInlinedAt()) {
1598 DINode *&NewVar = RemappedMetadata[OldVar];
1601 *OldVar->
getScope(), *NewSP, Ctx, Cache);
1604 OldVar->
getType(),
false, DINode::FlagZero,
1607 DVI->setVariable(cast<DILocalVariable>(NewVar));
1611 for (
auto *DII : DebugIntrinsicsToDelete)
1612 DII->eraseFromParent();
1623 auto updateLoopInfoLoc = [&Ctx, &Cache, NewSP](
Metadata *MD) ->
Metadata * {
1624 if (
auto *Loc = dyn_cast_or_null<DILocation>(MD))
1657 assert(BPI &&
"Both BPI and BFI are required to preserve profile info");
1659 if (Blocks.count(Pred))
1670 if (
auto *CI = dyn_cast<CondGuardInst>(&
I)) {
1673 CI->eraseFromParent();
1680 splitReturnBlocks();
1688 if (!Blocks.count(Succ)) {
1698 NumExitBlocks = ExitBlocks.
size();
1706 OldTargets.push_back(OldTarget);
1711 severSplitPHINodesOfEntry(header);
1712 severSplitPHINodesOfExits(ExitBlocks);
1716 "codeRepl", oldFunction,
1731 if (!
I.getDebugLoc())
1733 BranchI->setDebugLoc(
I.getDebugLoc());
1738 BranchI->insertInto(newFuncRoot, newFuncRoot->
end());
1740 ValueSet SinkingCands, HoistingCands;
1742 findAllocas(CEAC, SinkingCands, HoistingCands, CommonExit);
1752 for (
auto *II : SinkingCands) {
1753 if (
auto *AI = dyn_cast<AllocaInst>(II)) {
1755 if (!FirstSunkAlloca)
1756 FirstSunkAlloca = AI;
1759 assert((SinkingCands.empty() || FirstSunkAlloca) &&
1760 "Did not expect a sink candidate without any allocas");
1761 for (
auto *II : SinkingCands) {
1762 if (!isa<AllocaInst>(II)) {
1763 cast<Instruction>(II)->moveAfter(FirstSunkAlloca);
1767 if (!HoistingCands.
empty()) {
1770 for (
auto *II : HoistingCands)
1783 constructFunction(inputs, outputs, header, newFuncRoot, codeReplacer,
1796 emitCallAndSwitchStatement(newFunction, codeReplacer, inputs, outputs);
1798 moveCodeToFunction(newFunction);
1810 if (
BFI && NumExitBlocks > 1)
1811 calculateNewCallTerminatorWeights(codeReplacer, ExitWeights, BPI);
1823 for (
PHINode &PN : ExitBB->phis()) {
1824 Value *IncomingCodeReplacerVal =
nullptr;
1831 if (!IncomingCodeReplacerVal) {
1836 "PHI has two incompatbile incoming values from codeRepl");
1847 return isa<ReturnInst>(
Term) || isa<ResumeInst>(
Term);
1853 newFunction->
dump();
1867 auto *
I = dyn_cast_or_null<CondGuardInst>(AssumeVH);
1872 if (
I->getFunction() != &OldFunc)
1879 auto *AffectedCI = dyn_cast_or_null<CondGuardInst>(AffectedValVH);
1882 if (AffectedCI->getFunction() != &OldFunc)
1884 auto *AssumedInst = cast<Instruction>(AffectedCI->getOperand(0));
1885 if (AssumedInst->getFunction() != &OldFunc)
Move duplicate certain instructions close to their use
@ EndAttrKinds
Sentinal value useful for loops.
This class represents an incoming formal argument to a Function.
static StringRef getName(Value *V)
unsigned getNumSuccessors() const LLVM_READONLY
Return the number of successors that this instruction has.
This is an optimization pass for GlobalISel generic memory operations.
static IntegerType * getInt1Ty(LLVMContext &C)
We currently emits eax Perhaps this is what we really should generate is Is imull three or four cycles eax eax The current instruction priority is based on pattern complexity The former is more complex because it folds a load so the latter will not be emitted Perhaps we should use AddedComplexity to give LEA32r a higher priority We should always try to match LEA first since the LEA matching code does some estimate to determine whether the match is profitable if we care more about code then imull is better It s two bytes shorter than movl leal On a Pentium M
DITypeRefArray getOrCreateTypeArray(ArrayRef< Metadata * > Elements)
Get a DITypeRefArray, create one if required.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Return a value (possibly void), from a function.
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
A parsed version of the target data layout string in and methods for querying it.
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
InstListType::iterator iterator
Instruction iterators...
const Function * getParent() const
Return the enclosing method, or null if none.
void finalizeSubprogram(DISubprogram *SP)
Finalize a specific subprogram - no new variables may be added to this subprogram afterwards.
static Function * getFunction(Constant *C)
Represents a single loop in the control flow graph.
void dump() const
Support for debugging, callable in GDB: V->dump()
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
@ TombstoneKey
Use as Tombstone key for DenseMap of AttrKind.
auto successors(const MachineBasicBlock *BB)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
static SwitchInst * Create(Value *Value, BasicBlock *Default, unsigned NumCases, Instruction *InsertBefore=nullptr)
@ Bitcast
Perform the operation on a different, but equivalently sized type.
DISubprogram * getSubprogram() const
Get the attached subprogram.
static DebugLoc replaceInlinedAtSubprogram(const DebugLoc &DL, DISubprogram &NewSP, LLVMContext &Ctx, DenseMap< const MDNode *, MDNode * > &Cache)
Rebuild the entire inline-at chain by replacing the subprogram at the end of the chain with NewSP.
const BasicBlock & getEntryBlock() const
void findDbgUsers(SmallVectorImpl< DbgVariableIntrinsic * > &DbgInsts, Value *V)
Finds the debug info intrinsics describing a value.
Value * removeIncomingValue(unsigned Idx, bool DeletePHIIfEmpty=true)
Remove an incoming value.
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
void insertBefore(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified instruction.
@ EmptyKey
Use as Empty key for DenseMap of AttrKind.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
bool verifyFunction(const Function &F, raw_ostream *OS=nullptr)
Check a function for errors, useful for use when debugging a pass.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
ArrayRef< T > getArrayRef() const
StringRef getName() const
void unregisterAssumption(CondGuardInst *CI)
Remove an @llvm.assume intrinsic from this function's cache if it has been added to the cache earlier...
The instances of the Type class are immutable: once they are created, they are never changed.
static BranchProbability getZero()
DomTreeNodeBase< NodeT > * getNode(const NodeT *BB) const
getNode - return the (Post)DominatorTree node for the specified basic block.
AttributeSet getFnAttrs() const
The function attributes are returned.
Function::iterator insert(Function::iterator Position, BasicBlock *BB)
Insert BB in the basic block list at Position.
BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="", bool Before=false)
Split the basic block into two basic blocks at the specified instruction.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
user_iterator user_begin()
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
DISubroutineType * createSubroutineType(DITypeRefArray ParameterTypes, DINode::DIFlags Flags=DINode::FlagZero, unsigned CC=0)
Create subroutine type.
Value * getPointerOperand()
std::pair< iterator, bool > insert(const ValueT &V)
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
static IntegerType * getInt32Ty(LLVMContext &C)
bool empty() const
empty - Check if the array is empty.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
LLVM Basic Block Representation.
DILocalVariable * createAutoVariable(DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNo, DIType *Ty, bool AlwaysPreserve=false, DINode::DIFlags Flags=DINode::FlagZero, uint32_t AlignInBits=0)
Create a new descriptor for an auto variable.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
bool remove(const value_type &X)
Remove an item from the set vector.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
bool isLifetimeStartOrEnd() const LLVM_READONLY
Return true if the instruction is a llvm.lifetime.start or llvm.lifetime.end marker.
iterator begin()
Get an iterator to the beginning of the SetVector.
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
CaseIt removeCase(CaseIt I)
This method removes the specified case and its successor from the switch instruction.
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
iterator begin()
Instruction iterator methods.
Analysis providing branch probability information.
MDNode * createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight)
Return metadata containing two branch weights.
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
DISPFlags
Debug info subprogram flags.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
void setName(const Twine &Name)
Change the name of the value.
bool hasPersonalityFn() const
Check whether this function has a personality function.
bool stripDebugInfo(Function &F)
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Analysis containing CSE Info
void changeImmediateDominator(DomTreeNodeBase< NodeT > *N, DomTreeNodeBase< NodeT > *NewIDom)
changeImmediateDominator - This method is used to update the dominator tree information when a node's...
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
The initial backend is deliberately restricted to z10 We should add support for later architectures at some point If an asm ties an i32 r result to an i64 input
@ InternalLinkage
Rename collisions when linking (static functions).
bool empty() const
Determine if the SetVector is empty or not.
unsigned getNumIncomingValues() const
Return the number of incoming edges.
static BranchProbability getUnknown()
Distribution of unscaled probability weight.
AttributeList getAttributes() const
Return the attribute list for this Function.
DILocalScope * getScope() const
Get the local scope for this variable.
auto predecessors(const MachineBasicBlock *BB)
constexpr bool empty() const
empty - Check if the string is empty.
void setSubprogram(DISubprogram *SP)
Set the attached subprogram.
This is the common base class for debug info intrinsics for variables.
Function::ProfileCount ProfileCount
inst_range instructions(Function *F)
An instruction for storing to memory.
bool contains(const key_type &key) const
Check if the SetVector contains the given key.
BasicBlock * getSuccessor(unsigned Idx) const LLVM_READONLY
Return the specified successor. This instruction must be a terminator.
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Type * getReturnType() const
Returns the type of the ret val.
BasicBlock * getSuccessor(unsigned idx) const
void setDefaultDest(BasicBlock *DefaultCase)
@ None
No attributes have been set.
Module * getParent()
Get the module that this global value is contained inside of...
void setIncomingBlock(unsigned i, BasicBlock *BB)
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
This is an important class for using LLVM in a threaded context.
static BranchInst * Create(BasicBlock *IfTrue, Instruction *InsertBefore=nullptr)
uint64_t getFrequency() const
Returns the frequency as a fixpoint number scaled by the entry frequency.
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
S is passed via registers r2 But gcc stores them to the and then reload them to and r3 before issuing the call(r0 contains the address of the format string)
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
unsigned getAddressSpace() const
LLVM currently emits rax rax movq rax rax ret It could narrow the loads and stores to emit rax rax movq rax rax ret The trouble is that there is a TokenFactor between the store and the load
bool isReachableFromEntry(const Use &U) const
Provide an overload for a Use.
bool isVoidTy() const
Return true if this is 'void'.
Value * getCondition() const
void setPersonalityFn(Constant *Fn)
A Module instance is used to store all the information related to an LLVM module.
StringRef getName() const
BranchProbability getEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors) const
Get an edge's probability, relative to other out-edges of the Src.
bool insert(const value_type &X)
Insert a new element into the SetVector.
static M68kRelType getType(unsigned Kind, MCSymbolRefExpr::VariantKind &Modifier, bool &IsPCRel)
static GetElementPtrInst * Create(Type *PointeeType, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
void setEntryCount(ProfileCount Count, const DenseSet< GlobalValue::GUID > *Imports=nullptr)
Set the entry count for this function.
void setEdgeProbability(const BasicBlock *Src, const SmallVectorImpl< BranchProbability > &Probs)
Set the raw probabilities for all edges from the given block.
void setCondition(Value *V)
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Class to represent struct types.
void setSuccessor(unsigned Idx, BasicBlock *BB)
Update the specified successor to point at the provided block.
A cache of @llvm.assume calls within a function.
Tagged DWARF-like metadata node.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Type * getType() const
All values are typed, get the type of this value.
const Function * getFunction() const
Return the function this instruction belongs to.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
static const Function * getParent(const Value *V)
LLVMContext & getContext() const
All values hold a context through their type.
self_iterator getIterator()
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
void updateLoopMetadataDebugLocations(Instruction &I, function_ref< Metadata *(Metadata *)> Updater)
Update the debug locations contained within the MD_loop metadata attached to the instruction I,...
This is the base class for all instructions that perform data casts.
static DILocalScope * cloneScopeForSubprogram(DILocalScope &RootScope, DISubprogram &NewSP, LLVMContext &Ctx, DenseMap< const MDNode *, MDNode * > &Cache)
Traverses the scope chain rooted at RootScope until it hits a Subprogram, recreating the chain with "...
StringRef getName() const
Return a constant reference to the value's name.
An instruction for reading from memory.
bool replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
DomTreeNodeBase< NodeT > * addNewBlock(NodeT *BB, NodeT *DomBB)
Add a new node to the dominator tree information.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
const Value * stripInBoundsOffsets(function_ref< void(const Value *)> Func=[](const Value *) {}) const
Strip off pointer casts and inbounds GEPs.
const Instruction & front() const
amdgpu Simplify well known AMD library false FunctionCallee Callee
LLVMContext & getContext() const
Get the context in which this basic block lives.
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
adds the attribute to the list of attributes for the given arg.
static IntegerType * getInt64Ty(LLVMContext &C)
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
DILocalScope * getScope() const
Get the local scope for this label.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
MIPS Relocation Principles In there are several elements of the llvm::ISD::NodeType enum that deal with addresses and or relocations These are defined in include llvm Target TargetSelectionDAG td JumpTable
static ConstantInt * getSigned(IntegerType *Ty, int64_t V)
Return a ConstantInt with the specified value for the specified type.
size_type count(const key_type &key) const
Count the number of elements of a given key in the SetVector.
static ReturnInst * Create(LLVMContext &C, Value *retVal=nullptr, Instruction *InsertBefore=nullptr)
Constant * getPersonalityFn() const
Get the personality function associated with this function.
A wrapper class for inspecting calls to intrinsic functions.
const BasicBlock & front() const
bool isVarArg() const
isVarArg - Return true if this function takes a variable number of arguments.
static Type * getVoidTy(LLVMContext &C)
DISubprogram * createFunction(DIScope *Scope, StringRef Name, StringRef LinkageName, DIFile *File, unsigned LineNo, DISubroutineType *Ty, unsigned ScopeLine, DINode::DIFlags Flags=DINode::FlagZero, DISubprogram::DISPFlags SPFlags=DISubprogram::SPFlagZero, DITemplateParameterArray TParams=nullptr, DISubprogram *Decl=nullptr, DITypeArray ThrownTypes=nullptr, DINodeArray Annotations=nullptr, StringRef TargetFuncName="")
Create a new descriptor for the specified subprogram.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
const Value * stripInBoundsConstantOffsets() const
Strip off pointer casts and all-constant inbounds GEPs.
MutableArrayRef< ResultElem > assumptions()
Access the list of assumption handles currently tracked for this function.
const BasicBlock * getParent() const
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
iv Induction Variable Users
void addFnAttr(Attribute::AttrKind Kind)
Add function attributes to this function.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
uint32_t getAlignInBits() const
Type * getElementType(unsigned N) const
auto reverse(ContainerTy &&C)
static IntegerType * getInt16Ty(LLVMContext &C)
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
std::string str() const
str - Get the contents as an std::string.
This class represents a function call, abstracting a target machine's calling convention.
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
static CastInst * CreatePointerCast(Value *S, Type *Ty, const Twine &Name, BasicBlock *InsertAtEnd)
Create a BitCast AddrSpaceCast, or a PtrToInt cast instruction.
an instruction to allocate memory on the stack
Class to represent profile counts.
void addCase(ConstantInt *OnVal, BasicBlock *Dest)
Add an entry to the switch instruction.
@ Cold
Attempts to make code in the caller as efficient as possible under the assumption that the call is no...
BasicBlock * SplitBlock(BasicBlock *Old, Instruction *SplitPt, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="", bool Before=false)
Split the specified block at the specified instruction.
Representative of a block.
LLVM Value Representation.
iterator_range< user_iterator > users()
InstListType::const_iterator const_iterator
void moveBefore(Instruction *MovePos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
Class to represent function types.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
iterator insert(iterator I, T &&Elt)
MutableArrayRef< ResultElem > assumptionsFor(const Value *V)
Access the list of assumptions which affect this value.