59 using namespace llvm::PatternMatch;
61 #define DEBUG_TYPE "codegenprepare"
63 STATISTIC(NumBlocksElim,
"Number of blocks eliminated");
64 STATISTIC(NumPHIsElim,
"Number of trivial PHIs eliminated");
65 STATISTIC(NumGEPsElim,
"Number of GEPs converted to casts");
66 STATISTIC(NumCmpUses,
"Number of uses of Cmp expressions replaced with uses of "
68 STATISTIC(NumCastUses,
"Number of uses of Cast expressions replaced with uses "
70 STATISTIC(NumMemoryInsts,
"Number of memory instructions whose address "
71 "computations were sunk");
72 STATISTIC(NumExtsMoved,
"Number of [s|z]ext instructions combined with loads");
73 STATISTIC(NumExtUses,
"Number of uses of [s|z]ext instructions optimized");
75 "Number of and mask instructions added to form ext loads");
76 STATISTIC(NumAndUses,
"Number of uses of and mask instructions optimized");
77 STATISTIC(NumRetsDup,
"Number of return instructions duplicated");
78 STATISTIC(NumDbgValueMoved,
"Number of debug value instructions moved");
79 STATISTIC(NumSelectsExpanded,
"Number of selects turned into branches");
80 STATISTIC(NumAndCmpsMoved,
"Number of and/cmp's pushed into branches");
81 STATISTIC(NumStoreExtractExposed,
"Number of store(extractelement) exposed");
85 cl::desc(
"Disable branch optimizations in CodeGenPrepare"));
89 cl::desc(
"Disable GC optimizations in CodeGenPrepare"));
93 cl::desc(
"Disable select to branch conversion."));
97 cl::desc(
"Address sinking in CGP using GEPs."));
101 cl::desc(
"Enable sinkinig and/cmp into branches."));
105 cl::desc(
"Disable store(extract) optimizations in CodeGenPrepare"));
109 cl::desc(
"Stress test store(extract) optimizations in CodeGenPrepare"));
113 cl::desc(
"Disable ext(promotable(ld)) -> promoted(ext(ld)) optimization in "
118 cl::desc(
"Stress test ext(promotable(ld)) -> promoted(ext(ld)) "
119 "optimization in CodeGenPrepare"));
123 cl::desc(
"Disable protection against removing loop preheaders"));
127 cl::desc(
"Use profile info to add section prefix for hot/cold functions"));
131 cl::desc(
"Skip merging empty blocks if (frequency of empty block) / "
132 "(frequency of destination block) is greater than this ratio"));
136 cl::desc(
"Force store splitting no matter what the target query says."));
142 class TypePromotionTransaction;
150 std::unique_ptr<BlockFrequencyInfo>
BFI;
151 std::unique_ptr<BranchProbabilityInfo> BPI;
163 SetOfInstrs InsertedInsts;
166 InstrToOrigTy PromotedInsts;
183 bool runOnFunction(
Function &
F)
override;
185 StringRef getPassName()
const override {
return "CodeGen Prepare"; }
197 bool eliminateMostlyEmptyBlocks(
Function &
F);
200 void eliminateMostlyEmptyBlock(
BasicBlock *BB);
203 bool optimizeBlock(
BasicBlock &BB,
bool& ModifiedDT);
206 Type *AccessTy,
unsigned AS);
207 bool optimizeInlineAsmInst(
CallInst *CS);
208 bool optimizeCallInst(
CallInst *CI,
bool& ModifiedDT);
215 bool optimizeExtractElementInst(
Instruction *Inst);
216 bool dupRetToEnableTailCallOpts(
BasicBlock *BB);
219 bool extLdPromotion(TypePromotionTransaction &TPT,
LoadInst *&LI,
222 unsigned CreatedInstCost);
230 "Optimize for code generation",
false,
false)
236 return new CodeGenPrepare(TM);
239 bool CodeGenPrepare::runOnFunction(
Function &
F) {
245 bool EverMadeChange =
false;
247 InsertedInsts.clear();
248 PromotedInsts.clear();
254 TLI =
TM->getSubtargetImpl(F)->getTargetLowering();
255 TLInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
256 TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
257 LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
262 getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
271 if (!OptSize && TLI && TLI->isSlowDivBypassed()) {
273 TLI->getBypassSlowDivWidths();
275 while (BB !=
nullptr) {
286 EverMadeChange |= eliminateMostlyEmptyBlocks(F);
291 EverMadeChange |= placeDbgValues(F);
298 EverMadeChange |= sinkAndCmp(F);
299 EverMadeChange |= splitBranchCondition(F);
302 bool MadeChange =
true;
307 bool ModifiedDTOnIteration =
false;
308 MadeChange |= optimizeBlock(*BB, ModifiedDTOnIteration);
311 if (ModifiedDTOnIteration)
314 EverMadeChange |= MadeChange;
325 if (!MadeChange)
continue;
328 II = Successors.begin(),
IE = Successors.end(); II !=
IE; ++II)
334 MadeChange |= !WorkList.
empty();
335 while (!WorkList.
empty()) {
343 II = Successors.begin(),
IE = Successors.end(); II !=
IE; ++II)
350 if (EverMadeChange || MadeChange)
351 MadeChange |= eliminateFallThrough(F);
353 EverMadeChange |= MadeChange;
362 for (
auto &
I : Statepoints)
363 EverMadeChange |= simplifyOffsetableRelocate(*
I);
366 return EverMadeChange;
372 bool CodeGenPrepare::eliminateFallThrough(
Function &F) {
373 bool Changed =
false;
382 if (!SinglePred || SinglePred == BB || BB->
hasAddressTaken())
continue;
387 DEBUG(
dbgs() <<
"To merge:\n"<< *SinglePred <<
"\n\n\n");
413 if (BBI != BB->
begin()) {
415 while (isa<DbgInfoIntrinsic>(BBI)) {
416 if (BBI == BB->
begin())
420 if (!isa<DbgInfoIntrinsic>(BBI) && !isa<PHINode>(BBI))
429 if (!canMergeBlocks(BB, DestBB))
439 bool CodeGenPrepare::eliminateMostlyEmptyBlocks(
Function &F) {
442 while (!LoopList.empty()) {
443 Loop *
L = LoopList.pop_back_val();
444 LoopList.insert(LoopList.end(), L->
begin(), L->
end());
446 Preheaders.
insert(Preheader);
449 bool MadeChange =
false;
453 BasicBlock *DestBB = findDestBlockOfMergeableEmptyBlock(BB);
455 !isMergingEmptyBlockProfitable(BB, DestBB, Preheaders.
count(BB)))
458 eliminateMostlyEmptyBlock(BB);
464 bool CodeGenPrepare::isMergingEmptyBlockProfitable(
BasicBlock *BB,
504 if (!isa<PHINode>(DestBB->
begin()))
514 if (DestBBPred == BB)
517 bool HasAllSameValue =
true;
519 while (
const PHINode *DestPN = dyn_cast<PHINode>(DestBBI++)) {
520 if (DestPN->getIncomingValueForBlock(BB) !=
521 DestPN->getIncomingValueForBlock(DestBBPred)) {
522 HasAllSameValue =
false;
527 SameIncomingValueBBs.
insert(DestBBPred);
533 if (SameIncomingValueBBs.
count(Pred))
546 for (
auto SameValueBB : SameIncomingValueBBs)
547 if (SameValueBB->getUniquePredecessor() == Pred &&
548 DestBB == findDestBlockOfMergeableEmptyBlock(SameValueBB))
549 BBFreq +=
BFI->getBlockFreq(SameValueBB);
558 bool CodeGenPrepare::canMergeBlocks(
const BasicBlock *BB,
564 while (
const PHINode *PN = dyn_cast<PHINode>(BBI++)) {
567 if (UI->
getParent() != DestBB || !isa<PHINode>(UI))
573 if (
const PHINode *UPN = dyn_cast<PHINode>(UI))
574 for (
unsigned I = 0,
E = UPN->getNumIncomingValues();
I !=
E; ++
I) {
577 Insn->
getParent() != UPN->getIncomingBlock(
I))
588 if (!DestBBPN)
return true;
592 if (
const PHINode *BBPN = dyn_cast<PHINode>(BB->
begin())) {
594 for (
unsigned i = 0, e = BBPN->getNumIncomingValues();
i != e; ++
i)
595 BBPreds.
insert(BBPN->getIncomingBlock(
i));
603 if (BBPreds.
count(Pred)) {
604 BBI = DestBB->
begin();
605 while (
const PHINode *PN = dyn_cast<PHINode>(BBI++)) {
606 const Value *V1 = PN->getIncomingValueForBlock(Pred);
607 const Value *
V2 = PN->getIncomingValueForBlock(BB);
610 if (
const PHINode *V2PN = dyn_cast<PHINode>(V2))
611 if (V2PN->getParent() == BB)
612 V2 = V2PN->getIncomingValueForBlock(Pred);
615 if (V1 != V2)
return false;
626 void CodeGenPrepare::eliminateMostlyEmptyBlock(
BasicBlock *BB) {
630 DEBUG(
dbgs() <<
"MERGING MOSTLY EMPTY BLOCKS - BEFORE:\n" << *BB << *DestBB);
635 if (SinglePred != DestBB) {
644 DEBUG(
dbgs() <<
"AFTER:\n" << *DestBB <<
"\n\n\n");
660 if (InValPhi && InValPhi->
getParent() == BB) {
669 for (
unsigned i = 0, e = BBPN->getNumIncomingValues();
i != e; ++
i)
684 DEBUG(
dbgs() <<
"AFTER:\n" << *DestBB <<
"\n\n\n");
697 for (
auto *ThisRelocate : AllRelocateCalls) {
698 auto K = std::make_pair(ThisRelocate->getBasePtrIndex(),
699 ThisRelocate->getDerivedPtrIndex());
700 RelocateIdxMap.
insert(std::make_pair(K, ThisRelocate));
702 for (
auto &Item : RelocateIdxMap) {
703 std::pair<unsigned, unsigned> Key = Item.first;
704 if (Key.first == Key.second)
709 auto BaseKey = std::make_pair(Key.first, Key.first);
712 auto MaybeBase = RelocateIdxMap.find(BaseKey);
713 if (MaybeBase == RelocateIdxMap.end())
718 RelocateInstMap[MaybeBase->second].push_back(I);
729 if (!
Op ||
Op->getZExtValue() > 20)
743 bool MadeChange =
false;
746 "Not relocating a derived object of the original base object");
747 if (ToReplace->getBasePtrIndex() == ToReplace->getDerivedPtrIndex()) {
760 Value *Base = ToReplace->getBasePtr();
762 if (!Derived || Derived->getPointerOperand() != Base)
771 "Should always have one since it's not a terminator");
798 Value *ActualRelocatedBase = RelocatedBase;
800 ActualRelocatedBase =
801 Builder.CreateBitCast(RelocatedBase, Base->
getType());
803 Value *Replacement = Builder.CreateGEP(
804 Derived->getSourceElementType(), ActualRelocatedBase,
makeArrayRef(OffsetV));
808 Value *ActualReplacement = Replacement;
809 if (Replacement->
getType() != ToReplace->getType()) {
811 Builder.CreateBitCast(Replacement, ToReplace->
getType());
814 ToReplace->eraseFromParent();
838 bool CodeGenPrepare::simplifyOffsetableRelocate(
Instruction &
I) {
839 bool MadeChange =
false;
842 for (
auto *U : I.
users())
849 if (AllRelocateCalls.
size() < 2)
856 if (RelocateInstMap.
empty())
859 for (
auto &Item : RelocateInstMap)
873 bool MadeChange =
false;
876 Use &TheUse = UI.getUse();
882 if (
PHINode *PN = dyn_cast<PHINode>(User)) {
901 if (UserBB == DefBB)
continue;
904 CastInst *&InsertedCast = InsertedCasts[UserBB];
910 CI->
getType(),
"", &*InsertPt);
914 TheUse = InsertedCast;
938 if (
auto *ASC = dyn_cast<AddrSpaceCastInst>(CI)) {
940 ASC->getDestAddressSpace()))
954 if (SrcVT.
bitsLT(DstVT))
return false;
985 if (!isa<IntegerType>(Ty))
1005 auto *InsertPt = AddI->
hasOneUse() ? CI : AddI;
1007 auto *UAddWithOverflow =
1036 bool MadeChange =
false;
1039 Use &TheUse = UI.getUse();
1046 if (isa<PHINode>(User))
1053 if (UserBB == DefBB)
continue;
1056 CmpInst *&InsertedCmp = InsertedCmps[UserBB];
1069 TheUse = InsertedCmp;
1099 if (!isa<TruncInst>(User)) {
1104 const APInt &Cimm = cast<ConstantInt>(User->
getOperand(1))->getValue();
1106 if ((Cimm & (Cimm + 1)).getBoolValue())
1120 bool MadeChange =
false;
1124 TruncUI != TruncE;) {
1126 Use &TruncTheUse = TruncUI.getUse();
1127 Instruction *TruncUser = cast<Instruction>(*TruncUI);
1146 if (isa<PHINode>(TruncUser))
1151 if (UserBB == TruncUserBB)
1155 CastInst *&InsertedTrunc = InsertedTruncs[TruncUserBB];
1157 if (!InsertedShift && !InsertedTrunc) {
1161 if (ShiftI->
getOpcode() == Instruction::AShr)
1162 InsertedShift = BinaryOperator::CreateAShr(ShiftI->
getOperand(0), CI,
1165 InsertedShift = BinaryOperator::CreateLShr(ShiftI->
getOperand(0), CI,
1171 assert(TruncInsertPt != TruncUserBB->
end());
1174 TruncI->
getType(),
"", &*TruncInsertPt);
1178 TruncTheUse = InsertedTrunc;
1211 bool MadeChange =
false;
1214 Use &TheUse = UI.getUse();
1220 if (isa<PHINode>(User))
1228 if (UserBB == DefBB) {
1243 if (isa<TruncInst>(User) && shiftIsLegal
1256 if (!InsertedShift) {
1260 if (ShiftI->
getOpcode() == Instruction::AShr)
1261 InsertedShift = BinaryOperator::CreateAShr(ShiftI->
getOperand(0), CI,
1264 InsertedShift = BinaryOperator::CreateLShr(ShiftI->
getOperand(0), CI,
1271 TheUse = InsertedShift;
1322 unsigned AlignVal = cast<ConstantInt>(Alignment)->getZExtValue();
1324 assert(VecType &&
"Unexpected return type of masked load intrinsic");
1334 Builder.SetInsertPoint(InsertPt);
1335 Builder.SetCurrentDebugLocation(CI->
getDebugLoc());
1338 bool IsAllOnesMask = isa<Constant>(
Mask) &&
1339 cast<Constant>(Mask)->isAllOnesValue();
1341 if (IsAllOnesMask) {
1342 Value *NewI = Builder.CreateAlignedLoad(Ptr, AlignVal);
1353 Value *FirstEltPtr = Builder.CreateBitCast(Ptr, NewPtrType);
1359 Value *VResult = UndefVal;
1361 if (isa<ConstantVector>(Mask)) {
1362 for (
unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
1363 if (cast<ConstantVector>(Mask)->getOperand(Idx)->isNullValue())
1366 Builder.CreateInBoundsGEP(EltTy, FirstEltPtr, Builder.getInt32(Idx));
1367 LoadInst*
Load = Builder.CreateAlignedLoad(Gep, AlignVal);
1368 VResult = Builder.CreateInsertElement(VResult, Load,
1369 Builder.getInt32(Idx));
1371 Value *NewI = Builder.CreateSelect(Mask, VResult, Src0);
1378 Value *PrevPhi = UndefVal;
1380 for (
unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
1390 Phi = Builder.CreatePHI(VecType, 2,
"res.phi.else");
1397 Value *
Predicate = Builder.CreateExtractElement(Mask, Builder.getInt32(Idx));
1407 CondBlock = IfBlock->
splitBasicBlock(InsertPt->getIterator(),
"cond.load");
1408 Builder.SetInsertPoint(InsertPt);
1411 Builder.CreateInBoundsGEP(EltTy, FirstEltPtr, Builder.getInt32(Idx));
1412 LoadInst *
Load = Builder.CreateAlignedLoad(Gep, AlignVal);
1413 VResult = Builder.CreateInsertElement(VResult, Load, Builder.getInt32(Idx));
1418 Builder.SetInsertPoint(InsertPt);
1422 PrevIfBlock = IfBlock;
1423 IfBlock = NewIfBlock;
1426 Phi = Builder.CreatePHI(VecType, 2,
"res.phi.select");
1429 Value *NewI = Builder.CreateSelect(Mask, Phi, Src0);
1468 unsigned AlignVal = cast<ConstantInt>(Alignment)->getZExtValue();
1470 assert(VecType &&
"Unexpected data type in masked store intrinsic");
1477 Builder.SetInsertPoint(InsertPt);
1478 Builder.SetCurrentDebugLocation(CI->
getDebugLoc());
1481 bool IsAllOnesMask = isa<Constant>(
Mask) &&
1482 cast<Constant>(Mask)->isAllOnesValue();
1484 if (IsAllOnesMask) {
1485 Builder.CreateAlignedStore(Src, Ptr, AlignVal);
1495 Value *FirstEltPtr = Builder.CreateBitCast(Ptr, NewPtrType);
1498 if (isa<ConstantVector>(Mask)) {
1499 for (
unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
1500 if (cast<ConstantVector>(Mask)->getOperand(Idx)->isNullValue())
1502 Value *OneElt = Builder.CreateExtractElement(Src, Builder.getInt32(Idx));
1504 Builder.CreateInBoundsGEP(EltTy, FirstEltPtr, Builder.getInt32(Idx));
1505 Builder.CreateAlignedStore(OneElt, Gep, AlignVal);
1511 for (
unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
1519 Value *
Predicate = Builder.CreateExtractElement(Mask, Builder.getInt32(Idx));
1531 Builder.SetInsertPoint(InsertPt);
1533 Value *OneElt = Builder.CreateExtractElement(Src, Builder.getInt32(Idx));
1535 Builder.CreateInBoundsGEP(EltTy, FirstEltPtr, Builder.getInt32(Idx));
1536 Builder.CreateAlignedStore(OneElt, Gep, AlignVal);
1541 Builder.SetInsertPoint(InsertPt);
1545 IfBlock = NewIfBlock;
1589 assert(VecType &&
"Unexpected return type of masked load intrinsic");
1596 Builder.SetInsertPoint(InsertPt);
1597 unsigned AlignVal = cast<ConstantInt>(Alignment)->getZExtValue();
1599 Builder.SetCurrentDebugLocation(CI->
getDebugLoc());
1604 Value *VResult = UndefVal;
1608 bool IsConstMask = isa<ConstantVector>(
Mask);
1611 for (
unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
1612 if (cast<ConstantVector>(Mask)->getOperand(Idx)->isNullValue())
1614 Value *
Ptr = Builder.CreateExtractElement(Ptrs, Builder.getInt32(Idx),
1615 "Ptr" +
Twine(Idx));
1616 LoadInst *
Load = Builder.CreateAlignedLoad(Ptr, AlignVal,
1617 "Load" +
Twine(Idx));
1618 VResult = Builder.CreateInsertElement(VResult, Load,
1619 Builder.getInt32(Idx),
1620 "Res" +
Twine(Idx));
1622 Value *NewI = Builder.CreateSelect(Mask, VResult, Src0);
1629 Value *PrevPhi = UndefVal;
1631 for (
unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
1640 Phi = Builder.CreatePHI(VecType, 2,
"res.phi.else");
1648 Builder.getInt32(Idx),
1649 "Mask" +
Twine(Idx));
1652 "ToLoad" +
Twine(Idx));
1661 Builder.SetInsertPoint(InsertPt);
1663 Value *
Ptr = Builder.CreateExtractElement(Ptrs, Builder.getInt32(Idx),
1664 "Ptr" +
Twine(Idx));
1665 LoadInst *
Load = Builder.CreateAlignedLoad(Ptr, AlignVal,
1666 "Load" +
Twine(Idx));
1667 VResult = Builder.CreateInsertElement(VResult, Load, Builder.getInt32(Idx),
1668 "Res" +
Twine(Idx));
1672 Builder.SetInsertPoint(InsertPt);
1676 PrevIfBlock = IfBlock;
1677 IfBlock = NewIfBlock;
1680 Phi = Builder.CreatePHI(VecType, 2,
"res.phi.select");
1683 Value *NewI = Builder.CreateSelect(Mask, Phi, Src0);
1723 "Unexpected data type in masked scatter intrinsic");
1726 "Vector of pointers is expected in masked scatter intrinsic");
1731 Builder.SetInsertPoint(InsertPt);
1732 Builder.SetCurrentDebugLocation(CI->
getDebugLoc());
1734 unsigned AlignVal = cast<ConstantInt>(Alignment)->getZExtValue();
1738 bool IsConstMask = isa<ConstantVector>(
Mask);
1741 for (
unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
1742 if (cast<ConstantVector>(Mask)->getOperand(Idx)->isNullValue())
1744 Value *OneElt = Builder.CreateExtractElement(Src, Builder.getInt32(Idx),
1745 "Elt" +
Twine(Idx));
1746 Value *
Ptr = Builder.CreateExtractElement(Ptrs, Builder.getInt32(Idx),
1747 "Ptr" +
Twine(Idx));
1748 Builder.CreateAlignedStore(OneElt, Ptr, AlignVal);
1753 for (
unsigned Idx = 0; Idx < VectorWidth; ++Idx) {
1761 Builder.getInt32(Idx),
1762 "Mask" +
Twine(Idx));
1766 "ToStore" +
Twine(Idx));
1775 Builder.SetInsertPoint(InsertPt);
1777 Value *OneElt = Builder.CreateExtractElement(Src, Builder.getInt32(Idx),
1778 "Elt" +
Twine(Idx));
1779 Value *
Ptr = Builder.CreateExtractElement(Ptrs, Builder.getInt32(Idx),
1780 "Ptr" +
Twine(Idx));
1781 Builder.CreateAlignedStore(OneElt, Ptr, AlignVal);
1785 Builder.SetInsertPoint(InsertPt);
1789 IfBlock = NewIfBlock;
1847 Builder.SetCurrentDebugLocation(CountZeros->
getDebugLoc());
1853 Builder.CreateCondBr(Cmp, EndBlock, CallBlock);
1858 Builder.SetInsertPoint(&EndBlock->
front());
1859 PHINode *PN = Builder.CreatePHI(Ty, 2,
"ctz");
1861 Value *BitWidth = Builder.getInt(
APInt(SizeInBits, SizeInBits));
1873 bool CodeGenPrepare::optimizeCallInst(
CallInst *CI,
bool& ModifiedDT) {
1880 if (TLI->ExpandInlineAsm(CI)) {
1882 CurInstIterator = BB->
begin();
1889 if (optimizeInlineAsmInst(CI))
1895 unsigned MinSize, PrefAlign;
1896 if (TLI && TLI->shouldAlignPointerArgs(CI, MinSize, PrefAlign)) {
1902 if (!Arg->getType()->isPointerTy())
1905 cast<PointerType>(Arg->getType())->getAddressSpace()),
1908 uint64_t Offset2 =
Offset.getLimitedValue();
1909 if ((Offset2 & (PrefAlign-1)) != 0)
1912 if ((AI = dyn_cast<AllocaInst>(Val)) && AI->
getAlignment() < PrefAlign &&
1932 if (Align >
MI->getAlignment())
1943 if (!Arg->getType()->isPointerTy())
1945 unsigned AS = Arg->getType()->getPointerAddressSpace();
1946 return optimizeMemoryInst(CI, Arg, Arg->
getType(), AS);
1953 case Intrinsic::objectsize: {
1960 Value *CurValue = &*CurInstIterator;
1961 WeakVH IterHandle(CurValue);
1967 if (IterHandle != CurValue) {
1968 CurInstIterator = BB->
begin();
1973 case Intrinsic::masked_load: {
1975 if (!TTI->isLegalMaskedLoad(CI->
getType())) {
1982 case Intrinsic::masked_store: {
1990 case Intrinsic::masked_gather: {
1991 if (!TTI->isLegalMaskedGather(CI->
getType())) {
1998 case Intrinsic::masked_scatter: {
2006 case Intrinsic::aarch64_stlxr:
2007 case Intrinsic::aarch64_stxr: {
2016 InsertedInsts.insert(ExtVal);
2019 case Intrinsic::invariant_group_barrier:
2024 case Intrinsic::cttz:
2025 case Intrinsic::ctlz:
2034 unsigned AddrSpace = ~0u;
2037 if (TLI->GetAddrModeArguments(II, PtrOps, AccessTy, AddrSpace))
2038 while (!PtrOps.
empty())
2039 if (optimizeMemoryInst(II, PtrOps.
pop_back_val(), AccessTy, AddrSpace))
2052 if (
Value *V = Simplifier.optimizeCall(CI)) {
2090 bool CodeGenPrepare::dupRetToEnableTailCallOpts(
BasicBlock *BB) {
2118 do { ++BI; }
while (isa<DbgInfoIntrinsic>(BI));
2126 while (isa<DbgInfoIntrinsic>(BI)) ++BI;
2140 TLI->mayBeEmittedAsTailCall(CI) &&
2147 if (!VisitedBBs.
insert(*PI).second)
2153 do { ++RI; }
while (RI != RE && isa<DbgInfoIntrinsic>(&*RI));
2158 if (CI && CI->
use_empty() && TLI->mayBeEmittedAsTailCall(CI) &&
2164 bool Changed =
false;
2165 for (
unsigned i = 0, e = TailCalls.
size();
i != e; ++
i) {
2187 ModifiedDT = Changed =
true;
2209 ExtAddrMode() : BaseReg(nullptr), ScaledReg(nullptr) {}
2213 bool operator==(
const ExtAddrMode& O)
const {
2214 return (BaseReg == O.BaseReg) && (ScaledReg == O.ScaledReg) &&
2215 (BaseGV == O.BaseGV) && (BaseOffs == O.BaseOffs) &&
2216 (HasBaseReg == O.HasBaseReg) && (Scale == O.Scale);
2228 bool NeedPlus =
false;
2231 OS << (NeedPlus ?
" + " :
"")
2233 BaseGV->printAsOperand(OS,
false);
2238 OS << (NeedPlus ?
" + " :
"")
2244 OS << (NeedPlus ?
" + " :
"")
2246 BaseReg->printAsOperand(OS,
false);
2250 OS << (NeedPlus ?
" + " :
"")
2252 ScaledReg->printAsOperand(OS,
false);
2258 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2268 class TypePromotionTransaction {
2273 class TypePromotionAction {
2281 TypePromotionAction(
Instruction *Inst) : Inst(Inst) {}
2283 virtual ~TypePromotionAction() {}
2290 virtual void undo() = 0;
2295 virtual void commit() {
2301 class InsertionHandler {
2311 bool HasPrevInstruction;
2318 if (HasPrevInstruction)
2319 Point.PrevInst = &*--It;
2326 if (HasPrevInstruction) {
2331 Instruction *Position = &*Point.BB->getFirstInsertionPt();
2341 class InstructionMoveBefore :
public TypePromotionAction {
2343 InsertionHandler Position;
2348 : TypePromotionAction(Inst), Position(Inst) {
2349 DEBUG(
dbgs() <<
"Do: move: " << *Inst <<
"\nbefore: " << *Before <<
"\n");
2354 void undo()
override {
2355 DEBUG(
dbgs() <<
"Undo: moveBefore: " << *Inst <<
"\n");
2356 Position.insert(Inst);
2361 class OperandSetter :
public TypePromotionAction {
2370 : TypePromotionAction(Inst), Idx(Idx) {
2371 DEBUG(
dbgs() <<
"Do: setOperand: " << Idx <<
"\n"
2372 <<
"for:" << *Inst <<
"\n"
2373 <<
"with:" << *NewVal <<
"\n");
2379 void undo()
override {
2380 DEBUG(
dbgs() <<
"Undo: setOperand:" << Idx <<
"\n"
2381 <<
"for: " << *Inst <<
"\n"
2382 <<
"with: " << *Origin <<
"\n");
2389 class OperandsHider :
public TypePromotionAction {
2395 OperandsHider(
Instruction *Inst) : TypePromotionAction(Inst) {
2396 DEBUG(
dbgs() <<
"Do: OperandsHider: " << *Inst <<
"\n");
2398 OriginalValues.reserve(NumOpnds);
2399 for (
unsigned It = 0; It < NumOpnds; ++It) {
2402 OriginalValues.push_back(Val);
2411 void undo()
override {
2412 DEBUG(
dbgs() <<
"Undo: OperandsHider: " << *Inst <<
"\n");
2413 for (
unsigned It = 0, EndIt = OriginalValues.size(); It != EndIt; ++It)
2419 class TruncBuilder :
public TypePromotionAction {
2425 TruncBuilder(
Instruction *Opnd,
Type *Ty) : TypePromotionAction(Opnd) {
2427 Val = Builder.CreateTrunc(Opnd, Ty,
"promoted");
2428 DEBUG(
dbgs() <<
"Do: TruncBuilder: " << *Val <<
"\n");
2432 Value *getBuiltValue() {
return Val; }
2435 void undo()
override {
2436 DEBUG(
dbgs() <<
"Undo: TruncBuilder: " << *Val <<
"\n");
2437 if (
Instruction *IVal = dyn_cast<Instruction>(Val))
2438 IVal->eraseFromParent();
2443 class SExtBuilder :
public TypePromotionAction {
2450 : TypePromotionAction(InsertPt) {
2452 Val = Builder.CreateSExt(Opnd, Ty,
"promoted");
2453 DEBUG(
dbgs() <<
"Do: SExtBuilder: " << *Val <<
"\n");
2457 Value *getBuiltValue() {
return Val; }
2460 void undo()
override {
2461 DEBUG(
dbgs() <<
"Undo: SExtBuilder: " << *Val <<
"\n");
2462 if (
Instruction *IVal = dyn_cast<Instruction>(Val))
2463 IVal->eraseFromParent();
2468 class ZExtBuilder :
public TypePromotionAction {
2475 : TypePromotionAction(InsertPt) {
2477 Val = Builder.CreateZExt(Opnd, Ty,
"promoted");
2478 DEBUG(
dbgs() <<
"Do: ZExtBuilder: " << *Val <<
"\n");
2482 Value *getBuiltValue() {
return Val; }
2485 void undo()
override {
2486 DEBUG(
dbgs() <<
"Undo: ZExtBuilder: " << *Val <<
"\n");
2487 if (
Instruction *IVal = dyn_cast<Instruction>(Val))
2488 IVal->eraseFromParent();
2493 class TypeMutator :
public TypePromotionAction {
2500 : TypePromotionAction(Inst), OrigTy(Inst->
getType()) {
2501 DEBUG(
dbgs() <<
"Do: MutateType: " << *Inst <<
" with " << *NewTy
2507 void undo()
override {
2508 DEBUG(
dbgs() <<
"Undo: MutateType: " << *Inst <<
" with " << *OrigTy
2515 class UsesReplacer :
public TypePromotionAction {
2517 struct InstructionAndIdx {
2522 InstructionAndIdx(
Instruction *Inst,
unsigned Idx)
2523 : Inst(Inst), Idx(Idx) {}
2533 DEBUG(
dbgs() <<
"Do: UsersReplacer: " << *Inst <<
" with " << *New
2537 Instruction *UserI = cast<Instruction>(U.getUser());
2538 OriginalUses.push_back(InstructionAndIdx(UserI, U.getOperandNo()));
2545 void undo()
override {
2546 DEBUG(
dbgs() <<
"Undo: UsersReplacer: " << *Inst <<
"\n");
2547 for (use_iterator UseIt = OriginalUses.begin(),
2548 EndIt = OriginalUses.end();
2549 UseIt != EndIt; ++UseIt) {
2550 UseIt->Inst->setOperand(UseIt->Idx, Inst);
2556 class InstructionRemover :
public TypePromotionAction {
2561 OperandsHider Hider;
2563 UsesReplacer *Replacer;
2570 : TypePromotionAction(Inst),
Inserter(Inst), Hider(Inst),
2573 Replacer =
new UsesReplacer(Inst, New);
2574 DEBUG(
dbgs() <<
"Do: InstructionRemover: " << *Inst <<
"\n");
2578 ~InstructionRemover()
override {
delete Replacer; }
2581 void commit()
override {
delete Inst; }
2585 void undo()
override {
2586 DEBUG(
dbgs() <<
"Undo: InstructionRemover: " << *Inst <<
"\n");
2598 typedef const TypePromotionAction *ConstRestorationPt;
2602 void rollback(ConstRestorationPt Point);
2604 ConstRestorationPt getRestorationPoint()
const;
2632 void TypePromotionTransaction::setOperand(
Instruction *Inst,
unsigned Idx,
2635 make_unique<TypePromotionTransaction::OperandSetter>(Inst, Idx, NewVal));
2638 void TypePromotionTransaction::eraseInstruction(
Instruction *Inst,
2641 make_unique<TypePromotionTransaction::InstructionRemover>(Inst, NewVal));
2644 void TypePromotionTransaction::replaceAllUsesWith(
Instruction *Inst,
2646 Actions.push_back(make_unique<TypePromotionTransaction::UsesReplacer>(Inst, New));
2649 void TypePromotionTransaction::mutateType(
Instruction *Inst,
Type *NewTy) {
2650 Actions.push_back(make_unique<TypePromotionTransaction::TypeMutator>(Inst, NewTy));
2655 std::unique_ptr<TruncBuilder>
Ptr(
new TruncBuilder(Opnd, Ty));
2657 Actions.push_back(std::move(
Ptr));
2663 std::unique_ptr<SExtBuilder>
Ptr(
new SExtBuilder(Inst, Opnd, Ty));
2665 Actions.push_back(std::move(
Ptr));
2671 std::unique_ptr<ZExtBuilder>
Ptr(
new ZExtBuilder(Inst, Opnd, Ty));
2673 Actions.push_back(std::move(
Ptr));
2677 void TypePromotionTransaction::moveBefore(
Instruction *Inst,
2680 make_unique<TypePromotionTransaction::InstructionMoveBefore>(Inst, Before));
2683 TypePromotionTransaction::ConstRestorationPt
2684 TypePromotionTransaction::getRestorationPoint()
const {
2685 return !Actions.empty() ? Actions.back().get() :
nullptr;
2688 void TypePromotionTransaction::commit() {
2689 for (CommitPt It = Actions.begin(), EndIt = Actions.end(); It != EndIt;
2695 void TypePromotionTransaction::rollback(
2696 TypePromotionTransaction::ConstRestorationPt Point) {
2697 while (!Actions.empty() && Point != Actions.back().get()) {
2698 std::unique_ptr<TypePromotionAction> Curr = Actions.pop_back_val();
2706 class AddressingModeMatcher {
2723 const SetOfInstrs &InsertedInsts;
2725 InstrToOrigTy &PromotedInsts;
2727 TypePromotionTransaction &TPT;
2731 bool IgnoreProfitability;
2736 const SetOfInstrs &InsertedInsts,
2737 InstrToOrigTy &PromotedInsts,
2738 TypePromotionTransaction &TPT)
2739 : AddrModeInsts(AMI), TM(TM),
2741 ->getTargetLowering()),
2742 DL(MI->getModule()->getDataLayout()), AccessTy(AT), AddrSpace(AS),
2743 MemoryInst(MI),
AddrMode(AM), InsertedInsts(InsertedInsts),
2744 PromotedInsts(PromotedInsts), TPT(TPT) {
2745 IgnoreProfitability =
false;
2756 static ExtAddrMode Match(
Value *V,
Type *AccessTy,
unsigned AS,
2760 const SetOfInstrs &InsertedInsts,
2761 InstrToOrigTy &PromotedInsts,
2762 TypePromotionTransaction &TPT) {
2765 bool Success = AddressingModeMatcher(AddrModeInsts, TM, AccessTy, AS,
2766 MemoryInst, Result, InsertedInsts,
2767 PromotedInsts, TPT).matchAddr(V, 0);
2768 (void)Success;
assert(Success &&
"Couldn't select *anything*?");
2772 bool matchScaledValue(
Value *ScaleReg, int64_t Scale,
unsigned Depth);
2774 bool matchOperationAddr(
User *Operation,
unsigned Opcode,
unsigned Depth,
2775 bool *MovedAway =
nullptr);
2776 bool isProfitableToFoldIntoAddressingMode(
Instruction *I,
2777 ExtAddrMode &AMBefore,
2778 ExtAddrMode &AMAfter);
2779 bool valueAlreadyLiveAtInst(
Value *Val,
Value *KnownLive1,
Value *KnownLive2);
2780 bool isPromotionProfitable(
unsigned NewCost,
unsigned OldCost,
2781 Value *PromotedOperand)
const;
2787 bool AddressingModeMatcher::matchScaledValue(
Value *ScaleReg, int64_t Scale,
2792 return matchAddr(ScaleReg, Depth);
2803 ExtAddrMode TestAddrMode =
AddrMode;
2807 TestAddrMode.Scale += Scale;
2808 TestAddrMode.ScaledReg = ScaleReg;
2811 if (!TLI.isLegalAddressingMode(DL, TestAddrMode, AccessTy, AddrSpace))
2821 if (isa<Instruction>(ScaleReg) &&
2823 TestAddrMode.ScaledReg = AddLHS;
2824 TestAddrMode.BaseOffs += CI->
getSExtValue()*TestAddrMode.Scale;
2828 if (TLI.isLegalAddressingMode(DL, TestAddrMode, AccessTy, AddrSpace)) {
2829 AddrModeInsts.push_back(cast<Instruction>(ScaleReg));
2845 case Instruction::BitCast:
2846 case Instruction::AddrSpaceCast:
2851 case Instruction::PtrToInt:
2854 case Instruction::IntToPtr:
2859 case Instruction::Mul:
2860 case Instruction::Shl:
2863 case Instruction::GetElementPtr:
2874 static bool isPromotedInstructionLegal(
const TargetLowering &TLI,
2889 class TypePromotionHelper {
2901 static bool canGetThrough(
const Instruction *Inst,
Type *ConsideredExtType,
2902 const InstrToOrigTy &PromotedInsts,
bool IsSExt);
2906 static bool shouldExtOperand(
const Instruction *Inst,
int OpIdx) {
2907 return !(isa<SelectInst>(Inst) && OpIdx == 0);
2919 static Value *promoteOperandForTruncAndAnyExt(
2921 InstrToOrigTy &PromotedInsts,
unsigned &CreatedInstsCost,
2935 TypePromotionTransaction &TPT,
2936 InstrToOrigTy &PromotedInsts,
2937 unsigned &CreatedInstsCost,
2943 static Value *signExtendOperandForOther(
2945 InstrToOrigTy &PromotedInsts,
unsigned &CreatedInstsCost,
2948 return promoteOperandForOther(Ext, TPT, PromotedInsts, CreatedInstsCost,
2949 Exts, Truncs, TLI,
true);
2953 static Value *zeroExtendOperandForOther(
2955 InstrToOrigTy &PromotedInsts,
unsigned &CreatedInstsCost,
2958 return promoteOperandForOther(Ext, TPT, PromotedInsts, CreatedInstsCost,
2959 Exts, Truncs, TLI,
false);
2965 InstrToOrigTy &PromotedInsts,
2966 unsigned &CreatedInstsCost,
2979 static Action getAction(
Instruction *
Ext,
const SetOfInstrs &InsertedInsts,
2981 const InstrToOrigTy &PromotedInsts);
2984 bool TypePromotionHelper::canGetThrough(
const Instruction *Inst,
2985 Type *ConsideredExtType,
2986 const InstrToOrigTy &PromotedInsts,
2995 if (isa<ZExtInst>(Inst))
2999 if (IsSExt && isa<SExtInst>(Inst))
3005 if (BinOp && isa<OverflowingBinaryOperator>(BinOp) &&
3012 if (!isa<TruncInst>(Inst))
3034 const Type *OpndType;
3035 InstrToOrigTy::const_iterator It = PromotedInsts.find(Opnd);
3036 if (It != PromotedInsts.end() && It->second.getInt() == IsSExt)
3037 OpndType = It->second.getPointer();
3038 else if ((IsSExt && isa<SExtInst>(Opnd)) || (!IsSExt && isa<ZExtInst>(Opnd)))
3048 TypePromotionHelper::Action TypePromotionHelper::getAction(
3051 assert((isa<SExtInst>(Ext) || isa<ZExtInst>(Ext)) &&
3052 "Unexpected instruction type");
3055 bool IsSExt = isa<SExtInst>(
Ext);
3059 if (!ExtOpnd || !canGetThrough(ExtOpnd, ExtTy, PromotedInsts, IsSExt))
3065 if (isa<TruncInst>(ExtOpnd) && InsertedInsts.count(ExtOpnd))
3070 if (isa<SExtInst>(ExtOpnd) || isa<TruncInst>(ExtOpnd) ||
3071 isa<ZExtInst>(ExtOpnd))
3072 return promoteOperandForTruncAndAnyExt;
3078 return IsSExt ? signExtendOperandForOther : zeroExtendOperandForOther;
3081 Value *TypePromotionHelper::promoteOperandForTruncAndAnyExt(
3083 InstrToOrigTy &PromotedInsts,
unsigned &CreatedInstsCost,
3089 Value *ExtVal = SExt;
3090 bool HasMergedNonFreeExt =
false;
3091 if (isa<ZExtInst>(SExtOpnd)) {
3094 HasMergedNonFreeExt = !TLI.
isExtFree(SExtOpnd);
3097 TPT.replaceAllUsesWith(SExt, ZExt);
3098 TPT.eraseInstruction(SExt);
3103 TPT.setOperand(SExt, 0, SExtOpnd->
getOperand(0));
3105 CreatedInstsCost = 0;
3109 TPT.eraseInstruction(SExtOpnd);
3117 CreatedInstsCost = !TLI.
isExtFree(ExtInst) && !HasMergedNonFreeExt;
3125 TPT.eraseInstruction(ExtInst, NextVal);
3129 Value *TypePromotionHelper::promoteOperandForOther(
3131 InstrToOrigTy &PromotedInsts,
unsigned &CreatedInstsCost,
3138 CreatedInstsCost = 0;
3144 Value *Trunc = TPT.createTrunc(Ext, ExtOpnd->
getType());
3145 if (
Instruction *ITrunc = dyn_cast<Instruction>(Trunc)) {
3146 ITrunc->removeFromParent();
3148 ITrunc->insertAfter(ExtOpnd);
3153 TPT.replaceAllUsesWith(ExtOpnd, Trunc);
3156 TPT.setOperand(Ext, 0, ExtOpnd);
3166 PromotedInsts.insert(std::pair<Instruction *, TypeIsSExt>(
3167 ExtOpnd, TypeIsSExt(ExtOpnd->
getType(), IsSExt)));
3169 TPT.mutateType(ExtOpnd, Ext->
getType());
3171 TPT.replaceAllUsesWith(Ext, ExtOpnd);
3175 DEBUG(
dbgs() <<
"Propagate Ext to operands\n");
3176 for (
int OpIdx = 0, EndOpIdx = ExtOpnd->
getNumOperands(); OpIdx != EndOpIdx;
3180 !shouldExtOperand(ExtOpnd, OpIdx)) {
3181 DEBUG(
dbgs() <<
"No need to propagate\n");
3186 if (
const ConstantInt *Cst = dyn_cast<ConstantInt>(Opnd)) {
3189 APInt CstVal = IsSExt ? Cst->getValue().sext(BitWidth)
3190 : Cst->getValue().zext(BitWidth);
3195 if (isa<UndefValue>(Opnd)) {
3205 DEBUG(
dbgs() <<
"More operands to ext\n");
3206 Value *ValForExtOpnd = IsSExt ? TPT.createSExt(Ext, Opnd, Ext->
getType())
3207 : TPT.createZExt(Ext, Opnd, Ext->
getType());
3208 if (!isa<Instruction>(ValForExtOpnd)) {
3209 TPT.setOperand(ExtOpnd, OpIdx, ValForExtOpnd);
3212 ExtForOpnd = cast<Instruction>(ValForExtOpnd);
3216 TPT.setOperand(ExtForOpnd, 0, Opnd);
3219 TPT.moveBefore(ExtForOpnd, ExtOpnd);
3220 TPT.setOperand(ExtOpnd, OpIdx, ExtForOpnd);
3221 CreatedInstsCost += !TLI.
isExtFree(ExtForOpnd);
3223 ExtForOpnd =
nullptr;
3225 if (ExtForOpnd == Ext) {
3226 DEBUG(
dbgs() <<
"Extension is useless now\n");
3227 TPT.eraseInstruction(Ext);
3240 bool AddressingModeMatcher::isPromotionProfitable(
3241 unsigned NewCost,
unsigned OldCost,
Value *PromotedOperand)
const {
3242 DEBUG(
dbgs() <<
"OldCost: " << OldCost <<
"\tNewCost: " << NewCost <<
'\n');
3246 if (NewCost > OldCost)
3248 if (NewCost < OldCost)
3253 return isPromotedInstructionLegal(TLI, DL, PromotedOperand);
3267 bool AddressingModeMatcher::matchOperationAddr(
User *AddrInst,
unsigned Opcode,
3271 if (Depth >= 5)
return false;
3278 case Instruction::PtrToInt:
3281 case Instruction::IntToPtr: {
3289 case Instruction::BitCast:
3300 case Instruction::AddrSpaceCast: {
3310 ExtAddrMode BackupAddrMode =
AddrMode;
3311 unsigned OldSize = AddrModeInsts.size();
3316 TypePromotionTransaction::ConstRestorationPt LastKnownGood =
3317 TPT.getRestorationPoint();
3319 if (matchAddr(AddrInst->
getOperand(1), Depth+1) &&
3325 AddrModeInsts.resize(OldSize);
3326 TPT.rollback(LastKnownGood);
3329 if (matchAddr(AddrInst->
getOperand(0), Depth+1) &&
3335 AddrModeInsts.resize(OldSize);
3336 TPT.rollback(LastKnownGood);
3342 case Instruction::Mul:
3343 case Instruction::Shl: {
3349 if (Opcode == Instruction::Shl)
3350 Scale = 1LL << Scale;
3354 case Instruction::GetElementPtr: {
3357 int VariableOperand = -1;
3358 unsigned VariableScale = 0;
3360 int64_t ConstantOffset = 0;
3366 cast<ConstantInt>(AddrInst->
getOperand(
i))->getZExtValue();
3372 }
else if (TypeSize) {
3374 if (VariableOperand != -1)
3378 VariableOperand =
i;
3379 VariableScale = TypeSize;
3386 if (VariableOperand == -1) {
3387 AddrMode.BaseOffs += ConstantOffset;
3388 if (ConstantOffset == 0 ||
3391 if (matchAddr(AddrInst->
getOperand(0), Depth+1))
3394 AddrMode.BaseOffs -= ConstantOffset;
3399 ExtAddrMode BackupAddrMode =
AddrMode;
3400 unsigned OldSize = AddrModeInsts.size();
3403 AddrMode.BaseOffs += ConstantOffset;
3406 if (!matchAddr(AddrInst->
getOperand(0), Depth+1)) {
3410 AddrModeInsts.resize(OldSize);
3418 if (!matchScaledValue(AddrInst->
getOperand(VariableOperand), VariableScale,
3423 AddrModeInsts.resize(OldSize);
3428 AddrMode.BaseOffs += ConstantOffset;
3429 if (!matchScaledValue(AddrInst->
getOperand(VariableOperand),
3430 VariableScale,
Depth)) {
3433 AddrModeInsts.resize(OldSize);
3440 case Instruction::SExt:
3441 case Instruction::ZExt: {
3448 TypePromotionHelper::Action TPH =
3449 TypePromotionHelper::getAction(Ext, InsertedInsts, TLI, PromotedInsts);
3453 TypePromotionTransaction::ConstRestorationPt LastKnownGood =
3454 TPT.getRestorationPoint();
3455 unsigned CreatedInstsCost = 0;
3457 Value *PromotedOperand =
3458 TPH(Ext, TPT, PromotedInsts, CreatedInstsCost,
nullptr,
nullptr, TLI);
3473 assert(PromotedOperand &&
3474 "TypePromotionHelper should have filtered out those cases");
3476 ExtAddrMode BackupAddrMode =
AddrMode;
3477 unsigned OldSize = AddrModeInsts.size();
3479 if (!matchAddr(PromotedOperand, Depth) ||
3484 !isPromotionProfitable(CreatedInstsCost,
3485 ExtCost + (AddrModeInsts.size() - OldSize),
3488 AddrModeInsts.resize(OldSize);
3489 DEBUG(
dbgs() <<
"Sign extension does not pay off: rollback\n");
3490 TPT.rollback(LastKnownGood);
3504 bool AddressingModeMatcher::matchAddr(
Value *Addr,
unsigned Depth) {
3507 TypePromotionTransaction::ConstRestorationPt LastKnownGood =
3508 TPT.getRestorationPoint();
3509 if (
ConstantInt *CI = dyn_cast<ConstantInt>(Addr)) {
3515 }
else if (
GlobalValue *GV = dyn_cast<GlobalValue>(Addr)) {
3523 }
else if (
Instruction *I = dyn_cast<Instruction>(Addr)) {
3524 ExtAddrMode BackupAddrMode =
AddrMode;
3525 unsigned OldSize = AddrModeInsts.size();
3528 bool MovedAway =
false;
3538 isProfitableToFoldIntoAddressingMode(I, BackupAddrMode,
AddrMode)) {
3539 AddrModeInsts.push_back(I);
3546 AddrModeInsts.resize(OldSize);
3547 TPT.rollback(LastKnownGood);
3549 }
else if (
ConstantExpr *CE = dyn_cast<ConstantExpr>(Addr)) {
3550 if (matchOperationAddr(CE,
CE->getOpcode(),
Depth))
3552 TPT.rollback(LastKnownGood);
3553 }
else if (isa<ConstantPointerNull>(Addr)) {
3579 TPT.rollback(LastKnownGood);
3593 for (
unsigned i = 0, e = TargetConstraints.size();
i != e; ++
i) {
3613 static bool FindAllMemoryUses(
3618 if (!ConsideredInsts.
insert(I).second)
3622 if (!MightBeFoldableInst(I))
3629 Instruction *UserI = cast<Instruction>(U.getUser());
3631 if (
LoadInst *LI = dyn_cast<LoadInst>(UserI)) {
3632 MemoryUses.push_back(std::make_pair(LI, U.getOperandNo()));
3636 if (
StoreInst *SI = dyn_cast<StoreInst>(UserI)) {
3637 unsigned opNo = U.getOperandNo();
3638 if (opNo == 0)
return true;
3639 MemoryUses.push_back(std::make_pair(SI, opNo));
3643 if (
CallInst *CI = dyn_cast<CallInst>(UserI)) {
3650 if (!IA)
return true;
3653 if (!IsOperandAMemoryOperand(CI, IA, I, TM))
3658 if (FindAllMemoryUses(UserI, MemoryUses, ConsideredInsts, TM))
3669 bool AddressingModeMatcher::valueAlreadyLiveAtInst(
Value *Val,
Value *KnownLive1,
3670 Value *KnownLive2) {
3672 if (Val ==
nullptr || Val == KnownLive1 || Val == KnownLive2)
3676 if (!isa<Instruction>(Val) && !isa<Argument>(Val))
return true;
3681 if (
AllocaInst *AI = dyn_cast<AllocaInst>(Val))
3712 bool AddressingModeMatcher::
3713 isProfitableToFoldIntoAddressingMode(
Instruction *I, ExtAddrMode &AMBefore,
3714 ExtAddrMode &AMAfter) {
3715 if (IgnoreProfitability)
return true;
3726 Value *BaseReg = AMAfter.BaseReg, *ScaledReg = AMAfter.ScaledReg;
3730 if (valueAlreadyLiveAtInst(BaseReg, AMBefore.BaseReg, AMBefore.ScaledReg))
3732 if (valueAlreadyLiveAtInst(ScaledReg, AMBefore.BaseReg, AMBefore.ScaledReg))
3733 ScaledReg =
nullptr;
3737 if (!BaseReg && !ScaledReg)
3746 if (FindAllMemoryUses(I, MemoryUses, ConsideredInsts, TM))
3759 for (
unsigned i = 0, e = MemoryUses.
size();
i != e; ++
i) {
3761 unsigned OpNo = MemoryUses[
i].second;
3776 TypePromotionTransaction::ConstRestorationPt LastKnownGood =
3777 TPT.getRestorationPoint();
3778 AddressingModeMatcher Matcher(MatchedAddrModeInsts, TM, AddressAccessTy, AS,
3779 MemoryInst, Result, InsertedInsts,
3780 PromotedInsts, TPT);
3781 Matcher.IgnoreProfitability =
true;
3782 bool Success = Matcher.matchAddr(Address, 0);
3783 (void)Success;
assert(Success &&
"Couldn't select *anything*?");
3788 TPT.rollback(LastKnownGood);
3794 MatchedAddrModeInsts.
clear();
3829 bool CodeGenPrepare::optimizeMemoryInst(
Instruction *MemoryInst,
Value *Addr,
3830 Type *AccessTy,
unsigned AddrSpace) {
3842 Value *Consensus =
nullptr;
3843 unsigned NumUsesConsensus = 0;
3844 bool IsNumUsesConsensusValid =
false;
3847 TypePromotionTransaction TPT;
3848 TypePromotionTransaction::ConstRestorationPt LastKnownGood =
3849 TPT.getRestorationPoint();
3850 while (!worklist.
empty()) {
3855 if (!Visited.
insert(V).second) {
3856 Consensus =
nullptr;
3861 if (
PHINode *
P = dyn_cast<PHINode>(V)) {
3862 for (
Value *IncValue :
P->incoming_values())
3871 ExtAddrMode NewAddrMode = AddressingModeMatcher::Match(
3872 V, AccessTy, AddrSpace, MemoryInst, NewAddrModeInsts, *TM,
3873 InsertedInsts, PromotedInsts, TPT);
3881 AddrMode = NewAddrMode;
3882 AddrModeInsts = NewAddrModeInsts;
3884 }
else if (NewAddrMode == AddrMode) {
3885 if (!IsNumUsesConsensusValid) {
3887 IsNumUsesConsensusValid =
true;
3896 if (NumUses > NumUsesConsensus) {
3898 NumUsesConsensus = NumUses;
3899 AddrModeInsts = NewAddrModeInsts;
3904 Consensus =
nullptr;
3911 TPT.rollback(LastKnownGood);
3920 DEBUG(
dbgs() <<
"CGP: Found local addrmode: " << AddrMode <<
"\n");
3933 Value *&SunkAddr = SunkAddrs[Addr];
3935 DEBUG(
dbgs() <<
"CGP: Reusing nonlocal addrmode: " << AddrMode <<
" for "
3936 << *MemoryInst <<
"\n");
3938 SunkAddr = Builder.CreateBitCast(SunkAddr, Addr->
getType());
3945 DEBUG(
dbgs() <<
"CGP: SINKING nonlocal addrmode: " << AddrMode <<
" for "
3946 << *MemoryInst <<
"\n");
3948 Value *ResultPtr =
nullptr, *ResultIndex =
nullptr;
3951 if (AddrMode.BaseReg && AddrMode.BaseReg->getType()->isPointerTy()) {
3952 ResultPtr = AddrMode.BaseReg;
3953 AddrMode.BaseReg =
nullptr;
3956 if (AddrMode.Scale && AddrMode.ScaledReg->getType()->isPointerTy()) {
3959 if (ResultPtr || AddrMode.Scale != 1)
3962 ResultPtr = AddrMode.ScaledReg;
3966 if (AddrMode.BaseGV) {
3970 ResultPtr = AddrMode.BaseGV;
3976 if (!ResultPtr && AddrMode.BaseReg) {
3978 Builder.CreateIntToPtr(AddrMode.BaseReg, Addr->
getType(),
"sunkaddr");
3979 AddrMode.BaseReg =
nullptr;
3980 }
else if (!ResultPtr && AddrMode.Scale == 1) {
3982 Builder.CreateIntToPtr(AddrMode.ScaledReg, Addr->
getType(),
"sunkaddr");
3987 !AddrMode.BaseReg && !AddrMode.Scale && !AddrMode.BaseOffs) {
3989 }
else if (!ResultPtr) {
4001 if (AddrMode.BaseReg) {
4002 Value *V = AddrMode.BaseReg;
4004 V = Builder.CreateIntCast(V, IntPtrTy,
true,
"sunkaddr");
4010 if (AddrMode.Scale) {
4011 Value *V = AddrMode.ScaledReg;
4012 if (V->
getType() == IntPtrTy) {
4014 }
else if (cast<IntegerType>(IntPtrTy)->getBitWidth() <
4016 V = Builder.CreateTrunc(V, IntPtrTy,
"sunkaddr");
4023 Instruction *I = dyn_cast_or_null<Instruction>(ResultIndex);
4024 if (I && (ResultIndex != AddrMode.BaseReg))
4029 if (AddrMode.Scale != 1)
4033 ResultIndex = Builder.CreateAdd(ResultIndex, V,
"sunkaddr");
4039 if (AddrMode.BaseOffs) {
4044 if (ResultPtr->
getType() != I8PtrTy)
4045 ResultPtr = Builder.CreateBitCast(ResultPtr, I8PtrTy);
4046 ResultPtr = Builder.CreateGEP(I8Ty, ResultPtr, ResultIndex,
"sunkaddr");
4053 SunkAddr = ResultPtr;
4055 if (ResultPtr->
getType() != I8PtrTy)
4056 ResultPtr = Builder.CreateBitCast(ResultPtr, I8PtrTy);
4057 SunkAddr = Builder.CreateGEP(I8Ty, ResultPtr, ResultIndex,
"sunkaddr");
4061 SunkAddr = Builder.CreateBitCast(SunkAddr, Addr->
getType());
4064 DEBUG(
dbgs() <<
"CGP: SINKING nonlocal addrmode: " << AddrMode <<
" for "
4065 << *MemoryInst <<
"\n");
4067 Value *Result =
nullptr;
4074 if (AddrMode.BaseReg) {
4075 Value *V = AddrMode.BaseReg;
4077 V = Builder.CreatePtrToInt(V, IntPtrTy,
"sunkaddr");
4079 V = Builder.CreateIntCast(V, IntPtrTy,
true,
"sunkaddr");
4084 if (AddrMode.Scale) {
4085 Value *V = AddrMode.ScaledReg;
4086 if (V->
getType() == IntPtrTy) {
4089 V = Builder.CreatePtrToInt(V, IntPtrTy,
"sunkaddr");
4090 }
else if (cast<IntegerType>(IntPtrTy)->
getBitWidth() <
4092 V = Builder.CreateTrunc(V, IntPtrTy,
"sunkaddr");
4099 Instruction *I = dyn_cast_or_null<Instruction>(Result);
4100 if (I && (Result != AddrMode.BaseReg))
4104 if (AddrMode.Scale != 1)
4108 Result = Builder.CreateAdd(Result, V,
"sunkaddr");
4114 if (AddrMode.BaseGV) {
4115 Value *V = Builder.CreatePtrToInt(AddrMode.BaseGV, IntPtrTy,
"sunkaddr");
4117 Result = Builder.CreateAdd(Result, V,
"sunkaddr");
4123 if (AddrMode.BaseOffs) {
4126 Result = Builder.CreateAdd(Result, V,
"sunkaddr");
4134 SunkAddr = Builder.CreateIntToPtr(Result, Addr->
getType(),
"sunkaddr");
4144 Value *CurValue = &*CurInstIterator;
4145 WeakVH IterHandle(CurValue);
4150 if (IterHandle != CurValue) {
4153 CurInstIterator = BB->
begin();
4163 bool CodeGenPrepare::optimizeInlineAsmInst(
CallInst *CS) {
4164 bool MadeChange =
false;
4171 for (
unsigned i = 0, e = TargetConstraints.size();
i != e; ++
i) {
4180 MadeChange |= optimizeMemoryInst(CS, OpVal, OpVal->
getType(), ~0u);
4193 bool IsSExt = isa<SExtInst>(FirstUser);
4194 Type *ExtTy = FirstUser->getType();
4195 for (
const User *U : Inst->
users()) {
4197 if ((IsSExt && !isa<SExtInst>(UI)) || (!IsSExt && !isa<ZExtInst>(UI)))
4220 if (ExtTy->getScalarType()->getIntegerBitWidth() >
4260 bool CodeGenPrepare::extLdPromotion(TypePromotionTransaction &TPT,
4263 unsigned CreatedInstsCost = 0) {
4265 for (
auto I : Exts) {
4267 if ((LI = dyn_cast<LoadInst>(I->
getOperand(0)))) {
4276 TypePromotionHelper::Action TPH = TypePromotionHelper::getAction(
4277 I, InsertedInsts, *TLI, PromotedInsts);
4282 TypePromotionTransaction::ConstRestorationPt LastKnownGood =
4283 TPT.getRestorationPoint();
4285 unsigned NewCreatedInstsCost = 0;
4288 Value *PromotedVal = TPH(I, TPT, PromotedInsts, NewCreatedInstsCost,
4289 &NewExts,
nullptr, *TLI);
4291 "TypePromotionHelper should have filtered out those cases");
4299 long long TotalCreatedInstsCost = CreatedInstsCost + NewCreatedInstsCost;
4300 TotalCreatedInstsCost -= ExtCost;
4302 (TotalCreatedInstsCost > 1 ||
4303 !isPromotedInstructionLegal(*TLI, *DL, PromotedVal))) {
4305 TPT.rollback(LastKnownGood);
4310 (void)extLdPromotion(TPT, LI, Inst, NewExts, TotalCreatedInstsCost);
4319 TPT.rollback(LastKnownGood);
4333 bool CodeGenPrepare::moveExtToFormExtLoad(
Instruction *&I) {
4340 TypePromotionTransaction TPT;
4341 TypePromotionTransaction::ConstRestorationPt LastKnownGood =
4342 TPT.getRestorationPoint();
4348 bool HasPromoted = extLdPromotion(TPT, LI, I, Exts);
4350 assert(!HasPromoted && !LI &&
"If we did not match any load instruction "
4351 "the code must remain the same");
4371 TPT.rollback(LastKnownGood);
4377 if (isa<ZExtInst>(I))
4380 assert(isa<SExtInst>(I) &&
"Unexpected ext type!");
4385 TPT.rollback(LastKnownGood);
4406 bool CodeGenPrepare::optimizeExtUses(
Instruction *I) {
4421 if (!isa<Instruction>(Src) || DefBB != cast<Instruction>(Src)->
getParent())
4424 bool DefIsLiveOut =
false;
4425 for (User *U : I->
users()) {
4430 if (UserBB == DefBB)
continue;
4431 DefIsLiveOut =
true;
4438 for (User *U : Src->
users()) {
4441 if (UserBB == DefBB)
continue;
4444 if (isa<PHINode>(UI) || isa<LoadInst>(UI) || isa<StoreInst>(UI))
4451 bool MadeChange =
false;
4453 Instruction *User = cast<Instruction>(U.getUser());
4457 if (UserBB == DefBB)
continue;
4460 Instruction *&InsertedTrunc = InsertedTruncs[UserBB];
4462 if (!InsertedTrunc) {
4464 assert(InsertPt != UserBB->end());
4466 InsertedInsts.insert(InsertedTrunc);
4550 for (
auto *U : Load->
users())
4551 WorkList.
push_back(cast<Instruction>(U));
4555 APInt DemandBits(BitWidth, 0);
4556 APInt WidestAndBits(BitWidth, 0);
4558 while (!WorkList.
empty()) {
4563 if (!Visited.
insert(I).second)
4567 if (
auto *Phi = dyn_cast<PHINode>(I)) {
4568 for (
auto *U : Phi->users())
4569 WorkList.
push_back(cast<Instruction>(U));
4578 APInt AndBits = AndC->getValue();
4579 DemandBits |= AndBits;
4581 if (AndBits.
ugt(WidestAndBits))
4582 WidestAndBits = AndBits;
4588 case llvm::Instruction::Shl: {
4594 DemandBits |= ShlDemandBits;
4598 case llvm::Instruction::Trunc: {
4602 DemandBits |= TruncBits;
4611 uint32_t ActiveBits = DemandBits.getActiveBits();
4624 WidestAndBits != DemandBits)
4643 NewAnd->setOperand(0, Load);
4646 for (
auto *
And : AndsToMaybeRemove)
4649 if (cast<ConstantInt>(
And->getOperand(1))->getValue() == DemandBits) {
4650 And->replaceAllUsesWith(NewAnd);
4651 if (&*CurInstIterator ==
And)
4652 CurInstIterator = std::next(
And->getIterator());
4653 And->eraseFromParent();
4684 uint64_t TrueWeight, FalseWeight;
4686 uint64_t
Max = std::max(TrueWeight, FalseWeight);
4687 uint64_t Sum = TrueWeight + FalseWeight;
4716 static Value *getTrueOrFalseValue(
4721 for (
SelectInst *DefSI = SI; DefSI !=
nullptr && Selects.
count(DefSI);
4724 "The condition of DefSI does not match with SI");
4725 V = (isTrue ? DefSI->getTrueValue() : DefSI->getFalseValue());
4732 bool CodeGenPrepare::optimizeSelectInst(
SelectInst *SI) {
4749 CurInstIterator = std::next(LastSI->
getIterator());
4767 !isFormingBranchFromSelectProfitable(TTI, TLI, SI))
4814 if (TrueBlock ==
nullptr) {
4819 auto *TrueInst = cast<Instruction>(SI->
getTrueValue());
4820 TrueInst->moveBefore(TrueBranch);
4823 if (FalseBlock ==
nullptr) {
4829 FalseInst->moveBefore(FalseBranch);
4835 if (TrueBlock == FalseBlock) {
4836 assert(TrueBlock ==
nullptr &&
4837 "Unexpected basic block transform while optimizing select");
4850 if (TrueBlock ==
nullptr) {
4853 TrueBlock = StartBlock;
4854 }
else if (FalseBlock ==
nullptr) {
4857 FalseBlock = StartBlock;
4865 INS.insert(ASI.begin(), ASI.end());
4869 for (
auto It = ASI.rbegin(); It != ASI.rend(); ++It) {
4874 PN->
addIncoming(getTrueOrFalseValue(SI,
true, INS), TrueBlock);
4875 PN->
addIncoming(getTrueOrFalseValue(SI,
false, INS), FalseBlock);
4880 ++NumSelectsExpanded;
4884 CurInstIterator = StartBlock->
end();
4891 for (
unsigned i = 0;
i <
Mask.size(); ++
i) {
4892 if (SplatElem != -1 &&
Mask[
i] != -1 &&
Mask[
i] != SplatElem)
4894 SplatElem =
Mask[
i];
4913 if (!isBroadcastShuffle(SVI))
4919 bool MadeChange =
false;
4920 for (User *U : SVI->
users()) {
4925 if (UserBB == DefBB)
continue;
4932 Instruction *&InsertedShuffle = InsertedShuffles[UserBB];
4934 if (!InsertedShuffle) {
4936 assert(InsertPt != UserBB->end());
4955 bool CodeGenPrepare::optimizeSwitchInst(
SwitchInst *SI) {
4965 if (RegWidth <= cast<IntegerType>(OldType)->
getBitWidth())
4981 if (
auto *Arg = dyn_cast<Argument>(Cond))
4982 if (Arg->hasSExtAttr())
4983 ExtType = Instruction::SExt;
4989 APInt NarrowConst = Case.getCaseValue()->getValue();
4990 APInt WideConst = (ExtType == Instruction::ZExt) ?
4991 NarrowConst.
zext(RegWidth) : NarrowConst.
sext(RegWidth);
5014 class VectorPromoteHelper {
5029 unsigned StoreExtractCombineCost;
5037 if (InstsToBePromoted.empty())
5039 return InstsToBePromoted.back();
5045 unsigned getTransitionOriginalValueIdx()
const {
5046 assert(isa<ExtractElementInst>(Transition) &&
5047 "Other kind of transitions are not supported yet");
5054 unsigned getTransitionIdx()
const {
5055 assert(isa<ExtractElementInst>(Transition) &&
5056 "Other kind of transitions are not supported yet");
5064 Type *getTransitionType()
const {
5065 return Transition->getOperand(getTransitionOriginalValueIdx())->getType();
5079 bool isProfitableToPromote() {
5080 Value *ValIdx = Transition->getOperand(getTransitionOriginalValueIdx());
5081 unsigned Index = isa<ConstantInt>(ValIdx)
5082 ? cast<ConstantInt>(ValIdx)->getZExtValue()
5084 Type *PromotedType = getTransitionType();
5101 uint64_t ScalarCost =
5103 uint64_t VectorCost = StoreExtractCombineCost;
5104 for (
const auto &Inst : InstsToBePromoted) {
5110 bool IsArg0Constant = isa<UndefValue>(Arg0) || isa<ConstantInt>(Arg0) ||
5111 isa<ConstantFP>(Arg0);
5123 DEBUG(
dbgs() <<
"Estimated cost of computation to be promoted:\nScalar: "
5124 << ScalarCost <<
"\nVector: " << VectorCost <<
'\n');
5125 return ScalarCost > VectorCost;
5137 unsigned ExtractIdx = UINT_MAX;
5141 Value *ValExtractIdx = Transition->getOperand(getTransitionIdx());
5142 if (
ConstantInt *CstVal = dyn_cast<ConstantInt>(ValExtractIdx))
5148 unsigned End = getTransitionType()->getVectorNumElements();
5154 for (
unsigned Idx = 0; Idx !=
End; ++Idx) {
5155 if (Idx == ExtractIdx)
5166 unsigned OperandIdx) {
5169 if (OperandIdx != 1)
5174 case Instruction::SDiv:
5175 case Instruction::UDiv:
5176 case Instruction::SRem:
5177 case Instruction::URem:
5179 case Instruction::FDiv:
5180 case Instruction::FRem:
5189 unsigned CombineCost)
5190 : DL(DL), TLI(TLI), TTI(TTI), Transition(Transition),
5191 StoreExtractCombineCost(CombineCost), CombineInst(nullptr) {
5192 assert(Transition &&
"Do not know how to promote null");
5196 bool canPromote(
const Instruction *ToBePromoted)
const {
5198 return isa<BinaryOperator>(ToBePromoted);
5203 bool shouldPromote(
const Instruction *ToBePromoted)
const {
5206 for (
const Use &U : ToBePromoted->
operands()) {
5207 const Value *Val = U.get();
5208 if (Val == getEndOfTransition()) {
5212 if (canCauseUndefinedBehavior(ToBePromoted, U.getOperandNo()))
5216 if (!isa<ConstantInt>(Val) && !isa<UndefValue>(Val) &&
5217 !isa<ConstantFP>(Val))
5226 ISDOpcode, TLI.
getValueType(DL, getTransitionType(),
true));
5235 void enqueueForPromotion(
Instruction *ToBePromoted) {
5236 InstsToBePromoted.push_back(ToBePromoted);
5240 void recordCombineInstruction(
Instruction *ToBeCombined) {
5242 CombineInst = ToBeCombined;
5252 if (InstsToBePromoted.empty() || !CombineInst)
5260 for (
auto &ToBePromoted : InstsToBePromoted)
5261 promoteImpl(ToBePromoted);
5262 InstsToBePromoted.clear();
5268 void VectorPromoteHelper::promoteImpl(
Instruction *ToBePromoted) {
5278 "The type of the result of the transition does not match "
5283 Type *TransitionTy = getTransitionType();
5288 for (Use &U : ToBePromoted->
operands()) {
5289 Value *Val = U.get();
5290 Value *NewVal =
nullptr;
5291 if (Val == Transition)
5292 NewVal = Transition->getOperand(getTransitionOriginalValueIdx());
5293 else if (isa<UndefValue>(Val) || isa<ConstantInt>(Val) ||
5294 isa<ConstantFP>(Val)) {
5297 cast<Constant>(Val),
5298 isa<UndefValue>(Val) ||
5299 canCauseUndefinedBehavior(ToBePromoted, U.getOperandNo()));
5303 ToBePromoted->
setOperand(U.getOperandNo(), NewVal);
5305 Transition->removeFromParent();
5306 Transition->insertAfter(ToBePromoted);
5307 Transition->setOperand(getTransitionOriginalValueIdx(), ToBePromoted);
5313 bool CodeGenPrepare::optimizeExtractElementInst(
Instruction *Inst) {
5314 unsigned CombineCost = UINT_MAX;
5329 DEBUG(
dbgs() <<
"Found an interesting transition: " << *Inst <<
'\n');
5330 VectorPromoteHelper VPH(*DL, *TLI, *TTI, Inst, CombineCost);
5335 DEBUG(
dbgs() <<
"Use: " << *ToBePromoted <<
'\n');
5337 if (ToBePromoted->
getParent() != Parent) {
5338 DEBUG(
dbgs() <<
"Instruction to promote is in a different block ("
5340 <<
") than the transition (" << Parent->
getName() <<
").\n");
5344 if (VPH.canCombine(ToBePromoted)) {
5345 DEBUG(
dbgs() <<
"Assume " << *Inst <<
'\n'
5346 <<
"will be combined with: " << *ToBePromoted <<
'\n');
5347 VPH.recordCombineInstruction(ToBePromoted);
5348 bool Changed = VPH.promote();
5349 NumStoreExtractExposed += Changed;
5354 if (!VPH.canPromote(ToBePromoted) || !VPH.shouldPromote(ToBePromoted))
5357 DEBUG(
dbgs() <<
"Promoting is possible... Enqueue for promotion!\n");
5359 VPH.enqueueForPromotion(ToBePromoted);
5360 Inst = ToBePromoted;
5419 Value *LValue, *HValue;
5450 if (LBC && LBC->getParent() != SI.
getParent())
5451 LValue = Builder.CreateBitCast(LBC->getOperand(0), LBC->getType());
5452 if (HBC && HBC->getParent() != SI.
getParent())
5453 HValue = Builder.CreateBitCast(HBC->getOperand(0), HBC->getType());
5455 auto CreateSplitStore = [&](
Value *V,
bool Upper) {
5456 V = Builder.CreateZExtOrBitCast(V, SplitStoreType);
5457 Value *Addr = Builder.CreateBitCast(
5461 Addr = Builder.CreateGEP(
5462 SplitStoreType, Addr,
5464 Builder.CreateAlignedStore(
5468 CreateSplitStore(LValue,
false);
5469 CreateSplitStore(HValue,
true);
5476 bool CodeGenPrepare::optimizeInst(
Instruction *I,
bool& ModifiedDT) {
5479 if (InsertedInsts.count(I))
5482 if (
PHINode *
P = dyn_cast<PHINode>(I)) {
5487 P->replaceAllUsesWith(V);
5488 P->eraseFromParent();
5495 if (
CastInst *CI = dyn_cast<CastInst>(I)) {
5508 if (isa<ZExtInst>(I) || isa<SExtInst>(
I)) {
5517 bool MadeChange = moveExtToFormExtLoad(I);
5518 return MadeChange | optimizeExtUses(I);
5524 if (
CmpInst *CI = dyn_cast<CmpInst>(I))
5528 if (
LoadInst *LI = dyn_cast<LoadInst>(I)) {
5531 bool Modified = optimizeLoadExt(LI);
5539 if (
StoreInst *SI = dyn_cast<StoreInst>(I)) {
5540 if (TLI && splitMergedValStore(*SI, *DL, *TLI))
5545 return optimizeMemoryInst(I, SI->
getOperand(1),
5553 if (BinOp && (BinOp->
getOpcode() == Instruction::AShr ||
5554 BinOp->
getOpcode() == Instruction::LShr)) {
5563 if (GEPI->hasAllZeroIndices()) {
5566 GEPI->getName(), GEPI);
5568 GEPI->eraseFromParent();
5570 optimizeInst(NC, ModifiedDT);
5576 if (
CallInst *CI = dyn_cast<CallInst>(I))
5577 return optimizeCallInst(CI, ModifiedDT);
5579 if (
SelectInst *SI = dyn_cast<SelectInst>(I))
5580 return optimizeSelectInst(SI);
5583 return optimizeShuffleVectorInst(SVI);
5585 if (
auto *
Switch = dyn_cast<SwitchInst>(I))
5586 return optimizeSwitchInst(
Switch);
5588 if (isa<ExtractElementInst>(I))
5589 return optimizeExtractElementInst(I);
5615 bool CodeGenPrepare::optimizeBlock(
BasicBlock &BB,
bool& ModifiedDT) {
5617 bool MadeChange =
false;
5619 CurInstIterator = BB.
begin();
5620 while (CurInstIterator != BB.
end()) {
5621 MadeChange |= optimizeInst(&*CurInstIterator++, ModifiedDT);
5626 bool MadeBitReverse =
true;
5627 while (TLI && MadeBitReverse) {
5628 MadeBitReverse =
false;
5630 if (makeBitReverse(I, *DL, *TLI)) {
5631 MadeBitReverse = MadeChange =
true;
5637 MadeChange |= dupRetToEnableTailCallOpts(&BB);
5645 bool CodeGenPrepare::placeDbgValues(
Function &F) {
5646 bool MadeChange =
false;
5658 PrevNonDbgInst = Insn;
5663 if (VI && VI != PrevNonDbgInst && !VI->
isTerminator()) {
5668 DEBUG(
dbgs() <<
"Moving Debug Value before :\n" << *DVI <<
' ' << *VI);
5670 if (isa<PHINode>(VI))
5688 bool CodeGenPrepare::sinkAndCmp(
Function &F) {
5693 bool MadeChange =
false;
5706 if (!Zero || !Zero->
isZero())
5719 for (Use &TheUse : Cmp->
uses()) {
5725 if (UserBB == &BB)
continue;
5745 static void scaleWeights(uint64_t &NewTrue, uint64_t &NewFalse) {
5746 uint64_t NewMax = (NewTrue > NewFalse) ? NewTrue : NewFalse;
5747 uint32_t Scale = (NewMax / UINT32_MAX) + 1;
5748 NewTrue = NewTrue / Scale;
5749 NewFalse = NewFalse / Scale;
5774 bool CodeGenPrepare::splitBranchCondition(
Function &F) {
5778 bool MadeChange =
false;
5779 for (
auto &BB : F) {
5795 Value *Cond1, *Cond2;
5809 DEBUG(
dbgs() <<
"Before branch condition splitting\n"; BB.
dump());
5818 Br1->setCondition(Cond1);
5824 Br1->setSuccessor(0, TmpBB);
5826 Br1->setSuccessor(1, TmpBB);
5829 auto *Br2 =
IRBuilder<>(TmpBB).CreateCondBr(Cond2, TBB, FBB);
5830 if (
auto *I = dyn_cast<Instruction>(Cond2)) {
5848 for (
auto &I : *TBB) {
5858 for (
auto &I : *FBB) {
5888 uint64_t TrueWeight, FalseWeight;
5889 if (Br1->extractProfMetadata(TrueWeight, FalseWeight)) {
5890 uint64_t NewTrueWeight = TrueWeight;
5891 uint64_t NewFalseWeight = TrueWeight + 2 * FalseWeight;
5892 scaleWeights(NewTrueWeight, NewFalseWeight);
5894 .createBranchWeights(TrueWeight, FalseWeight));
5896 NewTrueWeight = TrueWeight;
5897 NewFalseWeight = 2 * FalseWeight;
5898 scaleWeights(NewTrueWeight, NewFalseWeight);
5900 .createBranchWeights(TrueWeight, FalseWeight));
5921 uint64_t TrueWeight, FalseWeight;
5922 if (Br1->extractProfMetadata(TrueWeight, FalseWeight)) {
5923 uint64_t NewTrueWeight = 2 * TrueWeight + FalseWeight;
5924 uint64_t NewFalseWeight = FalseWeight;
5925 scaleWeights(NewTrueWeight, NewFalseWeight);
5927 .createBranchWeights(TrueWeight, FalseWeight));
5929 NewTrueWeight = 2 * TrueWeight;
5930 NewFalseWeight = FalseWeight;
5931 scaleWeights(NewTrueWeight, NewFalseWeight);
5933 .createBranchWeights(TrueWeight, FalseWeight));
5943 DEBUG(
dbgs() <<
"After branch condition splitting\n"; BB.
dump();
static unsigned getBitWidth(Type *Ty, const DataLayout &DL)
Returns the bitwidth of the given scalar or pointer type (if unknown returns 0).
BinaryOp_match< LHS, RHS, Instruction::And > m_And(const LHS &L, const RHS &R)
bool hasFnAttr(Attribute::AttrKind Kind) const
Determine whether this call has the given attribute.
Return a value (possibly void), from a function.
Value * getValueOperand()
const Value * getCalledValue() const
Get a pointer to the function that is invoked by this instruction.
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
static MVT getIntegerVT(unsigned BitWidth)
void push_back(const T &Elt)
A parsed version of the target data layout string in and methods for querying it. ...
FunctionPass * createCodeGenPreparePass(const TargetMachine *TM=nullptr)
createCodeGenPreparePass - Transform the code to expose more pattern matching during instruction sele...
Type * getIndexedType() const
Instruction::CastOps getOpcode() const
Return the opcode of this CastInst.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
This class is the base class for the comparison instructions.
iterator_range< use_iterator > uses()
BasicBlock * getUniquePredecessor()
Return the predecessor of this block if it has a unique predecessor block.
class_match< CmpInst > m_Cmp()
Matches any compare instruction and ignore it.
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
static bool SinkCmpExpression(CmpInst *CI, const TargetLowering *TLI)
Sink the given CmpInst into user blocks to reduce the number of virtual registers that must be create...
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
iterator_range< CaseIt > cases()
Iteration adapter for range-for loops.
static APInt getAllOnesValue(unsigned numBits)
Get the all-ones value.
static cl::opt< bool > ForceSplitStore("force-split-store", cl::Hidden, cl::init(false), cl::desc("Force store splitting no matter what the target query says."))
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
STATISTIC(NumFunctions,"Total number of functions")
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds...
unsigned EnableFastISel
EnableFastISel - This flag enables fast-path instruction selection which trades away generated code q...
A Module instance is used to store all the information related to an LLVM module. ...
void setAlignment(unsigned Align)
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
virtual bool isCheapToSpeculateCttz() const
Return true if it is cheap to speculate a call to intrinsic cttz.
This class represents zero extension of integer types.
unsigned getNumOperands() const
void DeleteDeadBlock(BasicBlock *BB)
Delete the specified block, which must have no predecessors.
virtual bool isMultiStoresCheaperThanBitsMerge(EVT LTy, EVT HTy) const
Return true if it is cheaper to split the store of a merged int val from a pair of smaller values int...
Analysis providing profile information.
Type * getValueType() const
This class represents a function call, abstracting a target machine's calling convention.
size_type count(PtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
void MergeBasicBlockIntoOnlyPred(BasicBlock *BB, DominatorTree *DT=nullptr)
BB is a block with one predecessor and its predecessor is known to have one successor (BB!)...
unsigned getSizeInBits() const
This instruction constructs a fixed permutation of two input vectors.
bool isMask(unsigned numBits, const APInt &APIVal)
void setSectionPrefix(StringRef Prefix)
Set the section prefix for this function.
static void dump(StringRef Title, SpillInfo const &Spills)
const Function * getParent() const
Return the enclosing method, or null if none.
const Instruction & front() const
The two locations do not alias at all.
const APInt & getUniqueInteger() const
If C is a constant integer then return its value, otherwise C must be a vector of constant integers...
virtual bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace=0, unsigned Align=1, bool *=nullptr) const
Determine if the target supports unaligned memory accesses.
unsigned getAddressSpace() const
Return the address space of the Pointer type.
An instruction for reading from memory.
FunctionType * getType(LLVMContext &Context, ID id, ArrayRef< Type * > Tys=None)
Return the function type for an intrinsic.
uint64_t getFrequency() const
Returns the frequency as a fixpoint number scaled by the entry frequency.
Type * getElementType() const
Value * CallOperandVal
If this is the result output operand or a clobber, this is null, otherwise it is the incoming operand...
void setAlignment(unsigned Align)
bool canIncreaseAlignment() const
bool optForSize() const
Optimize this function for size (-Os) or minimum size (-Oz).
bool isFunctionEntryCold(const Function *F)
Returns true if F has cold function entry.
bool bitsLT(EVT VT) const
bitsLT - Return true if this has less bits than VT.
static bool despeculateCountZeros(IntrinsicInst *CountZeros, const TargetLowering *TLI, const DataLayout *DL, bool &ModifiedDT)
If counting leading or trailing zeros is an expensive operation and a zero input is defined...
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
StringRef getName() const
Return a constant reference to the value's name.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
iterator begin()
Instruction iterator methods.
bool enableExtLdPromotion() const
Return true if the target wants to use the optimization that turns ext(promotableInst1(...(promotableInstN(load)))) into promotedInst1(...(promotedInstN(ext(load)))).
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
bool match(Val *V, const Pattern &P)
match_combine_or< BinaryOp_match< LHS, RHS, Instruction::Or >, BinaryOp_match< RHS, LHS, Instruction::Or > > m_c_Or(const LHS &L, const RHS &R)
Matches an Or with LHS and RHS in either order.
AnalysisUsage & addRequired()
StructType * getStructTypeOrNull() const
Used to lazily calculate structure layout information for a target machine, based on the DataLayout s...
#define INITIALIZE_PASS_DEPENDENCY(depName)
bool hasNoNaNs() const
Determine whether the no-NaNs flag is set.
bool isUnconditional() const
Value * removeIncomingValue(unsigned Idx, bool DeletePHIIfEmpty=true)
Remove an incoming value.
bool isRound() const
isRound - Return true if the size is a power-of-two number of bytes.
This class represents the LLVM 'select' instruction.
bool hasMultipleConditionRegisters() const
Return true if multiple condition registers are available.
unsigned getPointerAlignment(const DataLayout &DL) const
Returns an alignment of the pointer value.
Value * getReturnValue() const
Convenience accessor. Returns null if there is no return value.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
const StructLayout * getStructLayout(StructType *Ty) const
Returns a StructLayout object, indicating the alignment of the struct, its size, and the offsets of i...
An analysis pass based on legacy pass manager to deliver ProfileSummaryInfo.
This is the base class for all instructions that perform data casts.
static cl::opt< bool > DisableExtLdPromotion("disable-cgp-ext-ld-promotion", cl::Hidden, cl::init(false), cl::desc("Disable ext(promotable(ld)) -> promoted(ext(ld)) optimization in ""CodeGenPrepare"))
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
'undef' values are things that do not have specified contents.
bool bypassSlowDivision(BasicBlock *BB, const DenseMap< unsigned int, unsigned int > &BypassWidth)
This optimization identifies DIV instructions in a BB that can be profitably bypassed and carried out...
Class to represent struct types.
A Use represents the edge between a Value definition and its users.
bool ConstantFoldTerminator(BasicBlock *BB, bool DeleteDeadConditions=false, const TargetLibraryInfo *TLI=nullptr)
If a terminator instruction is predicated on a constant value, convert it into an unconditional branc...
TargetLowering::ConstraintType ConstraintType
Information about the constraint code, e.g.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Instruction * getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
virtual bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const
Returns true if a cast between SrcAS and DestAS is a noop.
static Constant * get(ArrayRef< Constant * > V)
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
static cl::opt< bool > DisableBranchOpts("disable-cgp-branch-opts", cl::Hidden, cl::init(false), cl::desc("Disable branch optimizations in CodeGenPrepare"))
bool hasAddressTaken() const
Returns true if there are any uses of this basic block other than direct branches, switches, etc.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly...
Windows NT (Windows on ARM)
static void scalarizeMaskedStore(CallInst *CI)
Type * getVectorElementType() const
Value * stripAndAccumulateInBoundsConstantOffsets(const DataLayout &DL, APInt &Offset)
Accumulate offsets from stripInBoundsConstantOffsets().
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
bool isInteger() const
isInteger - Return true if this is an integer, or a vector integer type.
int InstructionOpcodeToISD(unsigned Opcode) const
Get the ISD node that corresponds to the Instruction class opcode.
static cl::opt< bool > DisableStoreExtract("disable-cgp-store-extract", cl::Hidden, cl::init(false), cl::desc("Disable store(extract) optimizations in CodeGenPrepare"))
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
bool isExtFree(const Instruction *I) const
Return true if the extension represented by I is free.
LLVM_NODISCARD bool empty() const
static cl::opt< bool > DisablePreheaderProtect("disable-preheader-prot", cl::Hidden, cl::init(false), cl::desc("Disable protection against removing loop preheaders"))
Interval::succ_iterator succ_begin(Interval *I)
succ_begin/succ_end - define methods so that Intervals may be used just like BasicBlocks can with the...
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
auto reverse(ContainerTy &&C, typename std::enable_if< has_rbegin< ContainerTy >::value >::type *=nullptr) -> decltype(make_range(C.rbegin(), C.rend()))
static bool OptimizeNoopCopyExpression(CastInst *CI, const TargetLowering &TLI, const DataLayout &DL)
If the specified cast instruction is a noop copy (e.g.
user_iterator_impl< User > user_iterator
A constant value that is initialized with an expression using other constant values.
static cl::opt< bool > AddrSinkUsingGEPs("addr-sink-using-gep", cl::Hidden, cl::init(false), cl::desc("Address sinking in CGP using GEPs."))
static bool simplifyRelocatesOffABase(GCRelocateInst *RelocatedBase, const SmallVectorImpl< GCRelocateInst * > &Targets)
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
match_combine_or< LTy, RTy > m_CombineOr(const LTy &L, const RTy &R)
Combine two pattern matchers matching L || R.
CastClass_match< OpTy, Instruction::ZExt > m_ZExt(const OpTy &Op)
Matches ZExt.
Value handle that is nullable, but tries to track the Value.
This contains information for each constraint that we are lowering.
uint64_t getLimitedValue(uint64_t Limit=~0ULL) const
getLimitedValue - If the value is smaller than the specified limit, return it, otherwise return the l...
BasicBlock * getSuccessor(unsigned i) const
This class represents a no-op cast from one type to another.
static CmpInst * Create(OtherOps Op, Predicate predicate, Value *S1, Value *S2, const Twine &Name="", Instruction *InsertBefore=nullptr)
Construct a compare instruction, given the opcode, the predicate and the two operands.
class_match< ConstantInt > m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=None)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
static GCRegistry::Add< OcamlGC > B("ocaml","ocaml 3.10-compatible GC")
const Function * getFunction() const
Return the function this instruction belongs to.
An instruction for storing to memory.
static bool canCombine(MachineBasicBlock &MBB, MachineOperand &MO, unsigned CombineOpc, unsigned ZeroReg=0, bool CheckZeroReg=false)
void SetCurrentDebugLocation(DebugLoc L)
Set location information used by debugging information.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
ConstraintPrefix Type
Type - The basic type of the constraint: input/output/clobber.
virtual bool isSelectSupported(SelectSupportKind) const
Optimize for code generation
void takeName(Value *V)
Transfer the name from V to this value.
This class implements simplifications for calls to fortified library functions (__st*cpy_chk, __memcpy_chk, __memmove_chk, __memset_chk), to, when possible, replace them with their non-checking counterparts.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
Type * getScalarType() const LLVM_READONLY
If this is a vector type, return the element type, otherwise return 'this'.
Type * getElementType() const
This class represents a truncation of integer types.
static unsigned getKnownAlignment(Value *V, const DataLayout &DL, const Instruction *CxtI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr)
Try to infer an alignment for the specified pointer.
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block...
Maximum length of the test input libFuzzer tries to guess a good value based on the corpus and reports it always prefer smaller inputs during the corpus shuffle When libFuzzer itself reports a bug this exit code will be used If indicates the maximal total time in seconds to run the fuzzer minimizes the provided crash input Use with etc Experimental Use value profile to guide fuzzing Number of simultaneous worker processes to run the jobs If min(jobs, NumberOfCpuCores()/2)\" is used.") FUZZER_FLAG_INT(reload
Class to represent pointers.
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
unsigned getNumIncomingValues() const
Return the number of incoming edges.
Interval::succ_iterator succ_end(Interval *I)
void replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
static cl::opt< bool > StressStoreExtract("stress-cgp-store-extract", cl::Hidden, cl::init(false), cl::desc("Stress test store(extract) optimizations in CodeGenPrepare"))
static bool isExtractBitsCandidateUse(Instruction *User)
Check if the candidates could be combined with a shift instruction, which includes: ...
uint64_t getElementOffset(unsigned Idx) const
static bool OptimizeCmpExpression(CmpInst *CI, const TargetLowering *TLI)
an instruction for type-safe pointer arithmetic to access elements of arrays and structs ...
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
OneUse_match< T > m_OneUse(const T &SubPattern)
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
initializer< Ty > init(const Ty &Val)
static cl::opt< bool > ProfileGuidedSectionPrefix("profile-guided-section-prefix", cl::Hidden, cl::init(true), cl::desc("Use profile info to add section prefix for hot/cold functions"))
static bool CombineUAddWithOverflow(CmpInst *CI)
Try to combine CI into a call to the llvm.uadd.with.overflow intrinsic if possible.
unsigned getAlignment() const
Return the alignment of the access that is being performed.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
static void scalarizeMaskedGather(CallInst *CI)
void insertBefore(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified instruction...
MVT - Machine Value Type.
ConstantInt * lowerObjectSizeCall(IntrinsicInst *ObjectSize, const DataLayout &DL, const TargetLibraryInfo *TLI, bool MustSucceed)
Try to turn a call to .objectsize into an integer value of the given Type.
LLVM Basic Block Representation.
PointerIntPair - This class implements a pair of a pointer and small integer.
The instances of the Type class are immutable: once they are created, they are never changed...
BinaryOp_match< LHS, RHS, Instruction::Or > m_Or(const LHS &L, const RHS &R)
This is an important class for using LLVM in a threaded context.
uint64_t getTypeStoreSizeInBits(Type *Ty) const
Returns the maximum number of bits that may be overwritten by storing the specified type; always a mu...
virtual bool isCheapAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const
Returns true if a cast from SrcAS to DestAS is "cheap", such that e.g.
Conditional or Unconditional Branch instruction.
bool isVectorTy() const
True if this is an instance of VectorType.
bool isIndirect
isIndirect - True if this operand is an indirect operand.
bool isOperationLegalOrCustom(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
This is an important base class in LLVM.
const Value * getCondition() const
int64_t getSExtValue() const
Get sign extended value.
This file contains the declarations for the subclasses of Constant, which represent the different fla...
bool isMaskAndBranchFoldingLegal() const
Return if the target supports combining a chain like:
APInt Or(const APInt &LHS, const APInt &RHS)
Bitwise OR function for APInt.
bool hasNoSignedWrap() const
Determine whether the no signed wrap flag is set.
unsigned getAlignment() const
Return the alignment of the memory that is being allocated by the instruction.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
#define INITIALIZE_TM_PASS_END(passName, arg, name, cfg, analysis)
Target machine pass initializer for passes with dependencies.
Interval::pred_iterator pred_begin(Interval *I)
pred_begin/pred_end - define methods so that Intervals may be used just like BasicBlocks can with the...
static void computeBaseDerivedRelocateMap(const SmallVectorImpl< GCRelocateInst * > &AllRelocateCalls, DenseMap< GCRelocateInst *, SmallVector< GCRelocateInst *, 2 >> &RelocateInstMap)
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
APInt sext(unsigned width) const
Sign extend to a new width.
bool extractProfMetadata(uint64_t &TrueVal, uint64_t &FalseVal) const
Retrieve the raw weight values of a conditional branch or select.
brc_match< Cond_t > m_Br(const Cond_t &C, BasicBlock *&T, BasicBlock *&F)
Represent the analysis usage information of a pass.
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
BinaryOp_match< LHS, RHS, Instruction::Shl > m_Shl(const LHS &L, const RHS &R)
static void scalarizeMaskedLoad(CallInst *CI)
This instruction compares its operands according to the predicate given to the constructor.
static const unsigned End
static bool SinkCast(CastInst *CI)
SinkCast - Sink the specified cast instruction into its user blocks.
virtual bool useSoftFloat() const
uint64_t getNumElements() const
for(unsigned i=0, e=MI->getNumOperands();i!=e;++i)
FunctionPass class - This class is used to implement most global optimizations.
Value * getOperand(unsigned i) const
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
Interval::pred_iterator pred_end(Interval *I)
bool isPredictableSelectExpensive() const
Return true if selects are only cheaper than branches if the branch is unlikely to be predicted right...
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
self_iterator getIterator()
class_match< BinaryOperator > m_BinOp()
Match an arbitrary binary operation and ignore it.
unsigned getIntegerBitWidth() const
Predicate getPredicate() const
Return the predicate for this instruction.
INITIALIZE_TM_PASS_BEGIN(CodeGenPrepare,"codegenprepare","Optimize for code generation", false, false) INITIALIZE_TM_PASS_END(CodeGenPrepare
static cl::opt< bool > DisableGCOpts("disable-cgp-gc-opts", cl::Hidden, cl::init(false), cl::desc("Disable GC optimizations in CodeGenPrepare"))
LLVM_NODISCARD bool empty() const
EVT - Extended Value Type.
bool isPointerTy() const
True if this is an instance of PointerType.
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
bool RecursivelyDeleteTriviallyDeadInstructions(Value *V, const TargetLibraryInfo *TLI=nullptr)
If the specified value is a trivially dead instruction, delete it.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
LLVMContext & getContext() const
All values hold a context through their type.
bool isLoadExtLegal(unsigned ExtType, EVT ValVT, EVT MemVT) const
Return true if the specified load with extension is legal on this target.
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
bool isUsedInBasicBlock(const BasicBlock *BB) const
Check if this value is used in the specified basic block.
const Value * getTrueValue() const
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
virtual void ComputeConstraintToUse(AsmOperandInfo &OpInfo, SDValue Op, SelectionDAG *DAG=nullptr) const
Determines the constraint code and constraint type to use for the specific AsmOperandInfo, setting OpInfo.ConstraintCode and OpInfo.ConstraintType.
void dump() const
Support for debugging, callable in GDB: V->dump()
bool isTerminator() const
bool isConditional() const
bool bitsGT(EVT VT) const
bitsGT - Return true if this has more bits than VT.
virtual const TargetSubtargetInfo * getSubtargetImpl(const Function &) const
Virtual method implemented by subclasses that returns a reference to that target's TargetSubtargetInf...
bool ugt(const APInt &RHS) const
Unsigned greather than comparison.
IntegerType * getIntPtrType(LLVMContext &C, unsigned AddressSpace=0) const
Returns an integer type with size at least as big as that of a pointer in the given address space...
std::vector< AsmOperandInfo > AsmOperandInfoVector
BinaryOps getOpcode() const
static Constant * getSplat(unsigned NumElts, Constant *Elt)
Return a ConstantVector with the specified constant in each element.
unsigned getBasePtrIndex() const
The index into the associate statepoint's argument list which contains the base pointer of the pointe...
void initializeCodeGenPreparePass(PassRegistry &)
This is the common base class for memset/memcpy/memmove.
Iterator for intrusive lists based on ilist_node.
static bool IsNonLocalValue(Value *V, BasicBlock *BB)
Return true if the specified values are defined in a different basic block than BB.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
This is the shared class of boolean and integer constants.
void setIncomingBlock(unsigned i, BasicBlock *BB)
bool hasNoUnsignedWrap() const
Determine whether the no unsigned wrap flag is set.
bool erase(PtrType Ptr)
erase - If the set contains the specified pointer, remove it and return true, otherwise return false...
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
uint64_t getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type...
static bool SinkShiftAndTruncate(BinaryOperator *ShiftI, Instruction *User, ConstantInt *CI, DenseMap< BasicBlock *, BinaryOperator * > &InsertedShifts, const TargetLowering &TLI, const DataLayout &DL)
Sink both shift and truncate instruction to the use of truncate's BB.
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
static CallInst * Create(Value *Func, ArrayRef< Value * > Args, ArrayRef< OperandBundleDef > Bundles=None, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
bool isStaticAlloca() const
Return true if this alloca is in the entry block of the function and is a constant size...
Type * getType() const
All values are typed, get the type of this value.
Provides information about what library functions are available for the current target.
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
virtual const TargetLowering * getTargetLowering() const
static Constant * getConstantVector(MVT VT, APInt SplatValue, unsigned SplatBitSize, LLVMContext &C)
static BranchProbability getBranchProbability(uint64_t Numerator, uint64_t Denominator)
static void scalarizeMaskedScatter(CallInst *CI)
static cl::opt< bool > DisableSelectToBranch("disable-cgp-select2branch", cl::Hidden, cl::init(false), cl::desc("Disable select to branch conversion."))
LLVM_NODISCARD T pop_back_val()
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Function * getCalledFunction() const
Return the function called, or null if this is an indirect function invocation.
static BranchInst * Create(BasicBlock *IfTrue, Instruction *InsertBefore=nullptr)
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
const BasicBlock & getEntryBlock() const
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
bool hasExtractBitsInsn() const
Return true if the target has BitExtract instructions.
void setOperand(unsigned i, Value *Val)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Value * getArgOperand(unsigned i) const
getArgOperand/setArgOperand - Return/set the i-th call argument.
Class to represent vector types.
Class for arbitrary precision integers.
BasicBlock * getSingleSuccessor()
Return the successor of this block if it has a single successor.
Value * getIncomingValueForBlock(const BasicBlock *BB) const
bool isIntegerTy() const
True if this is an instance of IntegerType.
iterator_range< user_iterator > users()
BasicBlock * getSinglePredecessor()
Return the predecessor of this block if it has a single predecessor block.
AddrMode
ARM Addressing Modes.
unsigned getVectorNumElements() const
static cl::opt< bool > StressExtLdPromotion("stress-cgp-ext-ld-promotion", cl::Hidden, cl::init(false), cl::desc("Stress test ext(promotable(ld)) -> promoted(ext(ld)) ""optimization in CodeGenPrepare"))
Value * getCondition() const
static cl::opt< unsigned > FreqRatioToSkipMerge("cgp-freq-ratio-to-skip-merge", cl::Hidden, cl::init(2), cl::desc("Skip merging empty blocks if (frequency of empty block) / ""(frequency of destination block) is greater than this ratio"))
APInt And(const APInt &LHS, const APInt &RHS)
Bitwise AND function for APInt.
void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
static bool getGEPSmallConstantIntOffsetV(GetElementPtrInst *GEP, SmallVectorImpl< Value * > &OffsetV)
bool isEHPad() const
Return true if the instruction is a variety of EH-block.
static CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name="", Instruction *InsertBefore=nullptr)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
This class wraps the llvm.memcpy/memmove intrinsics.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Analysis providing branch probability information.
bool isFunctionEntryHot(const Function *F)
Returns true if F has hot function entry.
virtual bool isVectorShiftByScalarCheap(Type *Ty) const
Return true if it's significantly cheaper to shift a vector by a uniform scalar than by an amount whi...
SelectSupportKind
Enum that describes what type of support for selects the target has.
static IntegerType * getInt32Ty(LLVMContext &C)
void setCondition(Value *V)
This represents the llvm.dbg.value instruction.
Represents a single loop in the control flow graph.
virtual bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AddrSpace) const
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
ImmutableCallSite - establish a view to a call site for examination.
unsigned getSizeInBits() const
getSizeInBits - Return the size of the specified value type in bits.
void insertAfter(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately after the specified instruction...
SymbolTableList< BasicBlock >::iterator eraseFromParent()
Unlink 'this' from the containing function and delete it.
TerminatorInst * getTerminator()
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
bool hasOneUse() const
Return true if there is exactly one user of this value.
void setArgOperand(unsigned i, Value *v)
unsigned getPointerSizeInBits(unsigned AS=0) const
Layout pointer size, in bits FIXME: The defaults need to be removed once all of the backends/clients ...
bool recognizeBSwapOrBitReverseIdiom(Instruction *I, bool MatchBSwaps, bool MatchBitReversals, SmallVectorImpl< Instruction * > &InsertedInsts)
Try and match a bswap or bitreverse idiom.
static bool OptimizeExtractBits(BinaryOperator *ShiftI, ConstantInt *CI, const TargetLowering &TLI, const DataLayout &DL)
Sink the shift right instruction into user blocks if the uses could potentially be combined with this...
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
ReturnInst * FoldReturnIntoUncondBranch(ReturnInst *RI, BasicBlock *BB, BasicBlock *Pred)
This method duplicates the specified return instruction into a predecessor which ends in an unconditi...
VectorType * getType() const
Overload to return most specific vector type.
BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="")
Split the basic block into two basic blocks at the specified instruction.
static EVT getEVT(Type *Ty, bool HandleUnknown=false)
getEVT - Return the value type corresponding to the specified type.
raw_ostream & operator<<(raw_ostream &OS, const APInt &I)
void mutateType(Type *Ty)
Mutate the type of this Value to be of the specified type.
LLVM_NODISCARD bool empty() const
OtherOps getOpcode() const
Get the opcode casted to the right type.
bool isStatepoint(ImmutableCallSite CS)
virtual bool isTruncateFree(Type *FromTy, Type *ToTy) const
Return true if it's free to truncate a value of type FromTy to type ToTy.
iterator_range< op_iterator > arg_operands()
Iteration adapter for range-for loops.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
virtual BranchProbability getPredictableBranchThreshold() const
If a branch or a select condition is skewed in one direction by more than this factor, it is very likely to be predicted correctly.
user_iterator user_begin()
LLVMContext & getContext() const
Get the context in which this basic block lives.
bool isSafeToSpeculativelyExecute(const Value *V, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr)
Return true if the instruction does not have any effects besides calculating the result and does not ...
Represents calls to the gc.relocate intrinsic.
unsigned getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
Module * getParent()
Get the module that this global value is contained inside of...
LLVM Value Representation.
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
cst_pred_ty< is_one > m_One()
Match an integer 1 or a vector with all elements equal to 1.
static cl::opt< bool > EnableAndCmpSinking("enable-andcmp-sinking", cl::Hidden, cl::init(true), cl::desc("Enable sinkinig and/cmp into branches."))
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
static const Function * getParent(const Value *V)
void moveBefore(Instruction *MovePos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
uint64_t getTypeSizeInBits(Type *Ty) const
Size examples:
This class implements an extremely fast bulk output stream that can only output to a stream...
virtual bool canCombineStoreAndExtract(Type *VectorTy, Value *Idx, unsigned &Cost) const
Return true if the target can combine store(extractelement VectorTy, Idx).
Primary interface to the complete machine description for the target machine.
The legacy pass manager's analysis pass to compute loop information.
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
const Value * getFalseValue() const
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
StringRef - Represent a constant reference to a string, i.e.
APInt zext(unsigned width) const
Zero extend to a new width.
unsigned getLargestLegalIntTypeSizeInBits() const
Returns the size of largest legal integer type size, or 0 if none are set.
Value * SimplifyInstruction(Instruction *I, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr, const DominatorTree *DT=nullptr, AssumptionCache *AC=nullptr)
See if we can compute a simplified version of this instruction.
static bool hasSameExtUse(Instruction *Inst, const TargetLowering &TLI)
Check if all the uses of Inst are equivalent (or free) zero or sign extensions.
bool operator==(uint64_t V1, const APInt &V2)
iterator getFirstInsertionPt()
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
specific_intval m_SpecificInt(uint64_t V)
Match a specific integer value or vector with all elements equal to the value.
unsigned getNumUses() const
This method computes the number of uses of this Value.
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
bool replaceAndRecursivelySimplify(Instruction *I, Value *SimpleV, const TargetLibraryInfo *TLI=nullptr, const DominatorTree *DT=nullptr, AssumptionCache *AC=nullptr)
Replace all uses of 'I' with 'SimpleV' and simplify the uses recursively.
bool attributesPermitTailCall(const Function *F, const Instruction *I, const ReturnInst *Ret, const TargetLoweringBase &TLI, bool *AllowDifferingSizes=nullptr)
Test if given that the input instruction is in the tail call position, if there is an attribute misma...
static void getShuffleMask(Constant *Mask, SmallVectorImpl< int > &Result)
Convert the input shuffle mask operand to a vector of integers.
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
static GCRegistry::Add< ErlangGC > A("erlang","erlang-compatible garbage collector")
bool isJumpExpensive() const
Return true if Flow Control is an expensive operation that should be avoided.
int getBasicBlockIndex(const BasicBlock *BB) const
Return the first index of the specified basic block in the value list for this PHI.
base_list_type::reverse_iterator reverse_iterator
static IntegerType * getInt8Ty(LLVMContext &C)
void moveBefore(BasicBlock *MovePos)
Unlink this basic block from its current function and insert it into the function that MovePos lives ...
const BasicBlock * getParent() const
InstListType::iterator iterator
Instruction iterators...
virtual AsmOperandInfoVector ParseConstraints(const DataLayout &DL, const TargetRegisterInfo *TRI, ImmutableCallSite CS) const
Split up the constraint string from the inline assembly value into the specific constraints and their...
bind_ty< Instruction > m_Instruction(Instruction *&I)
Match an instruction, capturing it if we match.
UAddWithOverflow_match< LHS_t, RHS_t, Sum_t > m_UAddWithOverflow(const LHS_t &L, const RHS_t &R, const Sum_t &S)
Match an icmp instruction checking for unsigned overflow on addition.
A wrapper class for inspecting calls to intrinsic functions.
This file describes how to lower LLVM code to machine code.
an instruction to allocate memory on the stack
virtual bool isCheapToSpeculateCtlz() const
Return true if it is cheap to speculate a call to intrinsic ctlz.
gep_type_iterator gep_type_begin(const User *GEP)
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.