47#include "llvm/Config/llvm-config.h"
70#include "llvm/IR/IntrinsicsAArch64.h"
114#define DEBUG_TYPE "codegenprepare"
117STATISTIC(NumPHIsElim,
"Number of trivial PHIs eliminated");
118STATISTIC(NumGEPsElim,
"Number of GEPs converted to casts");
119STATISTIC(NumCmpUses,
"Number of uses of Cmp expressions replaced with uses of "
121STATISTIC(NumCastUses,
"Number of uses of Cast expressions replaced with uses "
123STATISTIC(NumMemoryInsts,
"Number of memory instructions whose address "
124 "computations were sunk");
126 "Number of phis created when address "
127 "computations were sunk to memory instructions");
129 "Number of select created when address "
130 "computations were sunk to memory instructions");
131STATISTIC(NumExtsMoved,
"Number of [s|z]ext instructions combined with loads");
132STATISTIC(NumExtUses,
"Number of uses of [s|z]ext instructions optimized");
134 "Number of and mask instructions added to form ext loads");
135STATISTIC(NumAndUses,
"Number of uses of and mask instructions optimized");
136STATISTIC(NumRetsDup,
"Number of return instructions duplicated");
137STATISTIC(NumDbgValueMoved,
"Number of debug value instructions moved");
138STATISTIC(NumSelectsExpanded,
"Number of selects turned into branches");
139STATISTIC(NumStoreExtractExposed,
"Number of store(extractelement) exposed");
167class TypePromotionTransaction;
169class CodeGenPrepare {
170 friend class CodeGenPrepareLegacyPass;
171 const CodeGenOptions &Opts;
172 const TargetMachine *TM =
nullptr;
173 const TargetSubtargetInfo *SubtargetInfo =
nullptr;
174 const TargetLowering *TLI =
nullptr;
175 const TargetRegisterInfo *TRI =
nullptr;
176 const TargetTransformInfo *TTI =
nullptr;
177 const BasicBlockSectionsProfileReader *BBSectionsProfileReader =
nullptr;
178 const TargetLibraryInfo *TLInfo =
nullptr;
179 DomTreeUpdater *DTU =
nullptr;
180 LoopInfo *LI =
nullptr;
181 BlockFrequencyInfo *BFI;
182 BranchProbabilityInfo *BPI;
183 ProfileSummaryInfo *PSI =
nullptr;
194 ValueMap<Value *, WeakTrackingVH> SunkAddrs;
197 SetOfInstrs InsertedInsts;
201 InstrToOrigTy PromotedInsts;
204 SetOfInstrs RemovedInsts;
207 DenseMap<Value *, Instruction *> SeenChainsForSExt;
212 MapVector<AssertingVH<Value>,
217 SmallSet<AssertingVH<Value>, 2> NewGEPBases;
220 DenseMap<AssertingVH<GetElementPtrInst>,
int> LargeOffsetGEPID;
223 ValueToSExts ValToSExtendedUses;
229 const DataLayout *DL =
nullptr;
232 CodeGenPrepare(
const TargetMachine *TM)
233 : Opts(cl::LibraryOptionsContext::
get<CodeGenOptions>(
234 TM->
Options.MCOptions.LibraryOptions)),
237 bool IsHugeFunc =
false;
243 SmallPtrSet<BasicBlock *, 32> FreshBBs;
245 void releaseMemory() {
247 InsertedInsts.clear();
248 PromotedInsts.clear();
255 template <
typename F>
256 void resetIteratorIfInvalidatedWhileCalling(BasicBlock *BB,
F f) {
260 Value *CurValue = &*CurInstIterator;
261 WeakTrackingVH IterHandle(CurValue);
267 if (IterHandle != CurValue) {
268 CurInstIterator = BB->
begin();
274 DominatorTree &getDT() {
return DTU->getDomTree(); }
276 void removeAllAssertingVHReferences(
Value *
V);
279 bool eliminateMostlyEmptyBlocks(
Function &
F,
bool &ResetLI);
280 BasicBlock *findDestBlockOfMergeableEmptyBlock(BasicBlock *BB);
281 bool canMergeBlocks(
const BasicBlock *BB,
const BasicBlock *DestBB)
const;
282 bool eliminateMostlyEmptyBlock(BasicBlock *BB);
283 bool isMergingEmptyBlockProfitable(BasicBlock *BB, BasicBlock *DestBB,
285 bool makeBitReverse(Instruction &
I);
287 bool optimizeInst(Instruction *
I, ModifyDT &ModifiedDT);
288 bool optimizeMemoryInst(Instruction *MemoryInst,
Value *Addr,
Type *AccessTy,
290 bool optimizeGatherScatterInst(Instruction *MemoryInst,
Value *Ptr);
291 bool optimizeMulWithOverflow(Instruction *
I,
bool IsSigned,
292 ModifyDT &ModifiedDT);
293 bool optimizeInlineAsmInst(CallInst *CS);
295 bool optimizeExt(Instruction *&
I);
296 bool optimizeExtUses(Instruction *
I);
297 bool optimizeLoadExt(LoadInst *
Load);
298 bool optimizeShiftInst(BinaryOperator *BO);
299 bool optimizeFunnelShift(IntrinsicInst *Fsh);
300 bool optimizeSelectInst(SelectInst *SI);
301 bool optimizeShuffleVectorInst(ShuffleVectorInst *SVI);
302 bool optimizeSwitchType(SwitchInst *SI);
303 bool optimizeSwitchPhiConstants(SwitchInst *SI);
304 bool optimizeSwitchInst(SwitchInst *SI);
305 bool optimizeExtractElementInst(Instruction *Inst);
306 bool dupRetToEnableTailCallOpts(BasicBlock *BB, ModifyDT &ModifiedDT);
307 bool fixupDbgVariableRecord(DbgVariableRecord &
I);
308 bool fixupDbgVariableRecordsOnInst(Instruction &
I);
311 bool canFormExtLd(
const SmallVectorImpl<Instruction *> &MovedExts,
312 LoadInst *&LI, Instruction *&Inst,
bool HasPromoted);
313 bool tryToPromoteExts(TypePromotionTransaction &TPT,
314 const SmallVectorImpl<Instruction *> &Exts,
315 SmallVectorImpl<Instruction *> &ProfitablyMovedExts,
316 unsigned CreatedInstsCost = 0);
318 bool splitLargeGEPOffsets();
319 bool optimizePhiType(PHINode *Inst, SmallPtrSetImpl<PHINode *> &Visited,
320 SmallPtrSetImpl<Instruction *> &DeletedInstrs);
322 bool performAddressTypePromotion(
323 Instruction *&Inst,
bool AllowPromotionWithoutCommonHeader,
324 bool HasPromoted, TypePromotionTransaction &TPT,
325 SmallVectorImpl<Instruction *> &SpeculativelyMovedExts);
327 bool simplifyOffsetableRelocate(GCStatepointInst &
I);
329 bool tryToSinkFreeOperands(Instruction *
I);
330 bool replaceMathCmpWithIntrinsic(BinaryOperator *BO,
Value *Arg0,
Value *Arg1,
332 bool optimizeCmp(CmpInst *Cmp, ModifyDT &ModifiedDT);
333 bool optimizeURem(Instruction *Rem);
334 bool combineToUSubWithOverflow(CmpInst *Cmp, ModifyDT &ModifiedDT);
335 bool combineToUAddWithOverflow(CmpInst *Cmp, ModifyDT &ModifiedDT);
336 bool unfoldPowerOf2Test(CmpInst *Cmp);
345 CodeGenPrepareLegacyPass() : FunctionPass(ID) {}
349 StringRef getPassName()
const override {
return "CodeGen Prepare"; }
351 void getAnalysisUsage(AnalysisUsage &AU)
const override {
359 AU.
addRequired<BranchProbabilityInfoWrapperPass>();
367char CodeGenPrepareLegacyPass::ID = 0;
369bool CodeGenPrepareLegacyPass::runOnFunction(
Function &
F) {
372 auto TM = &getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
373 CodeGenPrepare CGP(TM);
374 CGP.DL = &
F.getDataLayout();
377 CGP.TRI = CGP.SubtargetInfo->getRegisterInfo();
378 CGP.TLInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(
F);
379 CGP.TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(
F);
380 CGP.LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
381 CGP.BPI = &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
382 CGP.BFI = &getAnalysis<BlockFrequencyInfoWrapperPass>().getBFI();
383 CGP.PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
385 getAnalysisIfAvailable<BasicBlockSectionsProfileReaderWrapperPass>();
386 CGP.BBSectionsProfileReader = BBSPRWP ? &BBSPRWP->getBBSPR() :
nullptr;
387 DomTreeUpdater DTUpdater(
388 &getAnalysis<DominatorTreeWrapperPass>().
getDomTree(),
389 DomTreeUpdater::UpdateStrategy::Lazy);
390 CGP.DTU = &DTUpdater;
396 "Optimize for code generation",
false,
false)
408 return new CodeGenPrepareLegacyPass();
413 CodeGenPrepare CGP(TM);
426 DL = &
F.getDataLayout();
439 "analysis to be available");
440 BBSectionsProfileReader =
443 DomTreeUpdater::UpdateStrategy::Lazy);
449 bool EverMadeChange =
false;
451 OptSize =
F.hasOptSize();
454 if (Opts.bbsections_guided_section_prefix && BBSectionsProfileReader &&
456 (void)
F.setSectionPrefix(
"hot");
457 }
else if (Opts.profile_guided_section_prefix) {
461 if (
F.hasFnAttribute(Attribute::Hot) ||
462 PSI->isFunctionHotInCallGraph(&
F, *BFI))
463 (void)
F.setSectionPrefix(
"hot");
467 else if (PSI->isFunctionColdInCallGraph(&
F, *BFI) ||
468 F.hasFnAttribute(Attribute::Cold))
469 (void)
F.setSectionPrefix(
"unlikely");
470 else if (Opts.profile_unknown_in_special_section &&
471 PSI->hasPartialSampleProfile() && PSI->isFunctionHotnessUnknown(
F))
472 (void)
F.setSectionPrefix(
"unknown");
478 const DenseMap<unsigned int, unsigned int> &BypassWidths =
481 while (BB !=
nullptr) {
494 EverMadeChange |= eliminateAssumptions(
F);
496 auto resetLoopInfo = [
this]() {
503 bool ResetLI =
false;
504 EverMadeChange |= eliminateMostlyEmptyBlocks(
F, ResetLI);
508 if (Opts.cgp_branch_opts)
509 EverMadeChange |= splitBranchCondition(
F);
515 EverMadeChange |=
Split;
521 assert(getDT().
verify(DominatorTree::VerificationLevel::Fast) &&
522 "Incorrect DominatorTree updates in CGP");
530 IsHugeFunc =
F.size() > Opts.cgp_huge_func;
532 bool MadeChange =
true;
533 bool FuncIterated =
false;
543 if (FuncIterated && !FreshBBs.
contains(&BB))
546 ModifyDT ModifiedDTOnIteration = ModifyDT::NotModifyDT;
562 else if (FuncIterated)
567 if (ModifiedDTOnIteration != ModifyDT::NotModifyDT)
572 FuncIterated = IsHugeFunc;
574 if (Opts.cgp_type_promotion_merge && !ValToSExtendedUses.empty())
575 MadeChange |= mergeSExts(
F);
576 if (!LargeOffsetGEPMap.
empty())
577 MadeChange |= splitLargeGEPOffsets();
578 MadeChange |= optimizePhiTypes(
F);
581 eliminateFallThrough(
F);
585 assert(getDT().
verify(DominatorTree::VerificationLevel::Fast) &&
586 "Incorrect DominatorTree updates in CGP");
593 for (Instruction *
I : RemovedInsts)
596 EverMadeChange |= MadeChange;
597 SeenChainsForSExt.
clear();
598 ValToSExtendedUses.clear();
599 RemovedInsts.clear();
600 LargeOffsetGEPMap.
clear();
601 LargeOffsetGEPID.
clear();
610 if (Opts.cgp_branch_opts) {
615 SmallSetVector<BasicBlock *, 8> WorkList;
616 for (BasicBlock &BB :
F) {
622 for (BasicBlock *Succ : Successors)
628 MadeChange |= !WorkList.
empty();
629 while (!WorkList.
empty()) {
635 for (BasicBlock *Succ : Successors)
645 if (EverMadeChange || MadeChange)
646 MadeChange |= eliminateFallThrough(
F);
648 EverMadeChange |= MadeChange;
651 if (Opts.cgp_gc_opts) {
653 for (BasicBlock &BB :
F)
654 for (Instruction &
I : BB)
657 for (
auto &
I : Statepoints)
658 EverMadeChange |= simplifyOffsetableRelocate(*
I);
663 EverMadeChange |= placeDbgValues(
F);
664 EverMadeChange |= placePseudoProbes(
F);
667 if (Opts.cgp_verify_bfi_updates)
671 return EverMadeChange;
674bool CodeGenPrepare::eliminateAssumptions(
Function &
F) {
675 bool MadeChange =
false;
676 for (BasicBlock &BB :
F) {
677 CurInstIterator = BB.begin();
678 while (CurInstIterator != BB.end()) {
683 Assume->eraseFromParent();
685 resetIteratorIfInvalidatedWhileCalling(&BB, [&]() {
696void CodeGenPrepare::removeAllAssertingVHReferences(
Value *
V) {
697 LargeOffsetGEPMap.
erase(
V);
706 auto VecI = LargeOffsetGEPMap.
find(
GEP->getPointerOperand());
707 if (VecI == LargeOffsetGEPMap.
end())
710 auto &GEPVector = VecI->second;
713 if (GEPVector.empty())
714 LargeOffsetGEPMap.
erase(VecI);
718[[maybe_unused]]
void CodeGenPrepare::verifyBFIUpdates(
Function &
F) {
721 BranchProbabilityInfo NewBPI(
F, NewCI, TLInfo);
722 BlockFrequencyInfo NewBFI(
F, NewBPI, NewCI);
723 NewBFI.verifyMatch(*BFI);
729bool CodeGenPrepare::eliminateFallThrough(
Function &
F) {
731 SmallPtrSet<BasicBlock *, 8> Preds;
739 BasicBlock *SinglePred = BB->getSinglePredecessor();
742 if (!SinglePred || SinglePred == BB || BB->hasAddressTaken())
755 FreshBBs.
insert(SinglePred);
763 for (
auto *
Pred : Preds)
771BasicBlock *CodeGenPrepare::findDestBlockOfMergeableEmptyBlock(BasicBlock *BB) {
780 if (BBI != BB->
begin()) {
791 if (!canMergeBlocks(BB, DestBB))
801bool CodeGenPrepare::eliminateMostlyEmptyBlocks(
Function &
F,
bool &ResetLI) {
802 SmallPtrSet<BasicBlock *, 16> Preheaders;
804 while (!LoopList.empty()) {
805 Loop *
L = LoopList.pop_back_val();
807 if (BasicBlock *Preheader =
L->getLoopPreheader())
808 Preheaders.
insert(Preheader);
812 bool MadeChange =
false;
813 SmallPtrSet<PHINode *, 32> KnownNonDeadPHIs;
817 if (Opts.cgp_delete_phis)
825 BasicBlock *DestBB = findDestBlockOfMergeableEmptyBlock(BB);
827 !isMergingEmptyBlockProfitable(BB, DestBB, Preheaders.
count(BB)))
830 ResetLI |= eliminateMostlyEmptyBlock(BB);
836bool CodeGenPrepare::isMergingEmptyBlockProfitable(BasicBlock *BB,
843 if (Opts.cgp_preheader_prot && isPreheader &&
887 SmallPtrSet<BasicBlock *, 16> SameIncomingValueBBs;
892 if (DestBBPred == BB)
896 return DestPN.getIncomingValueForBlock(BB) ==
897 DestPN.getIncomingValueForBlock(DestBBPred);
899 SameIncomingValueBBs.
insert(DestBBPred);
908 BlockFrequency PredFreq = BFI->getBlockFreq(
Pred);
909 BlockFrequency
BBFreq = BFI->getBlockFreq(BB);
911 for (
auto *SameValueBB : SameIncomingValueBBs)
912 if (SameValueBB->getUniquePredecessor() ==
Pred &&
913 DestBB == findDestBlockOfMergeableEmptyBlock(SameValueBB))
914 BBFreq += BFI->getBlockFreq(SameValueBB);
916 std::optional<BlockFrequency> Limit =
917 BBFreq.mul(Opts.cgp_freq_ratio_to_skip_merge);
918 return !Limit || PredFreq <= *Limit;
924bool CodeGenPrepare::canMergeBlocks(
const BasicBlock *BB,
925 const BasicBlock *DestBB)
const {
929 for (
const PHINode &PN : BB->
phis()) {
930 for (
const User *U : PN.users()) {
939 for (
unsigned I = 0,
E = UPN->getNumIncomingValues();
I !=
E; ++
I) {
942 Insn->
getParent() != UPN->getIncomingBlock(
I))
957 SmallPtrSet<const BasicBlock *, 16> BBPreds;
960 for (
unsigned i = 0, e = BBPN->getNumIncomingValues(); i != e; ++i)
961 BBPreds.
insert(BBPN->getIncomingBlock(i));
970 for (
const PHINode &PN : DestBB->
phis()) {
971 const Value *
V1 = PN.getIncomingValueForBlock(
Pred);
972 const Value *V2 = PN.getIncomingValueForBlock(BB);
976 if (V2PN->getParent() == BB)
977 V2 = V2PN->getIncomingValueForBlock(
Pred);
996 E = OldI->user_end();
1009bool CodeGenPrepare::eliminateMostlyEmptyBlock(BasicBlock *BB) {
1019 if (SinglePred != DestBB) {
1020 assert(SinglePred == BB &&
1021 "Single predecessor not the same as predecessor");
1030 FreshBBs.
insert(SinglePred);
1031 FreshBBs.
erase(DestBB);
1039 for (PHINode &PN : DestBB->
phis()) {
1041 Value *InVal = PN.removeIncomingValue(BB,
false);
1046 if (InValPhi && InValPhi->
getParent() == BB) {
1055 for (
unsigned i = 0, e = BBPN->getNumIncomingValues(); i != e; ++i)
1056 PN.addIncoming(InVal, BBPN->getIncomingBlock(i));
1059 PN.addIncoming(InVal,
Pred);
1067 Pred->getTerminator()->copyMetadata(*BI, LLVMContext::MD_loop);
1073 SmallPtrSet<BasicBlock *, 8> SeenPreds;
1077 if (!PredOfDestBB.contains(
Pred)) {
1087 DTUpdates.
push_back({DominatorTree::Delete, BB, DestBB});
1107 for (
auto *ThisRelocate : AllRelocateCalls) {
1108 auto K = std::make_pair(ThisRelocate->getBasePtrIndex(),
1109 ThisRelocate->getDerivedPtrIndex());
1110 RelocateIdxMap.
insert(std::make_pair(
K, ThisRelocate));
1112 for (
auto &Item : RelocateIdxMap) {
1113 std::pair<unsigned, unsigned>
Key = Item.first;
1114 if (
Key.first ==
Key.second)
1119 auto BaseKey = std::make_pair(
Key.first,
Key.first);
1122 auto MaybeBase = RelocateIdxMap.
find(BaseKey);
1123 if (MaybeBase == RelocateIdxMap.
end())
1128 RelocateInstMap[MaybeBase->second].push_back(
I);
1136 for (
unsigned i = 1; i <
GEP->getNumOperands(); i++) {
1139 if (!
Op ||
Op->getZExtValue() > 20)
1143 for (
unsigned i = 1; i <
GEP->getNumOperands(); i++)
1153 bool MadeChange =
false;
1160 for (
auto R = RelocatedBase->
getParent()->getFirstInsertionPt();
1161 &*R != RelocatedBase; ++R)
1165 RelocatedBase->
moveBefore(RI->getIterator());
1172 "Not relocating a derived object of the original base object");
1173 if (ToReplace->getBasePtrIndex() == ToReplace->getDerivedPtrIndex()) {
1178 if (RelocatedBase->
getParent() != ToReplace->getParent()) {
1188 if (!Derived || Derived->getPointerOperand() !=
Base)
1197 "Should always have one since it's not a terminator");
1201 Builder.disableGEPCanonicalization();
1202 Builder.SetCurrentDebugLocation(ToReplace->getDebugLoc());
1204 Value *Replacement = Builder.CreateGEP(Derived->getSourceElementType(),
1207 ToReplace->replaceAllUsesWith(Replacement);
1208 ToReplace->eraseFromParent();
1232bool CodeGenPrepare::simplifyOffsetableRelocate(GCStatepointInst &
I) {
1233 bool MadeChange =
false;
1235 for (
auto *U :
I.users())
1242 if (AllRelocateCalls.
size() < 2)
1247 MapVector<GCRelocateInst *, SmallVector<GCRelocateInst *, 0>> RelocateInstMap;
1249 if (RelocateInstMap.
empty())
1252 for (
auto &Item : RelocateInstMap)
1266 bool MadeChange =
false;
1269 Use &TheUse = UI.getUse();
1276 UserBB = PN->getIncomingBlock(TheUse);
1284 if (
User->isEHPad())
1294 if (UserBB == DefBB)
1298 CastInst *&InsertedCast = InsertedCasts[UserBB];
1300 if (!InsertedCast) {
1308 TheUse = InsertedCast;
1327 if (!SrcInst || SrcInst->getParent() == BCI->
getParent() ||
1328 SrcInst->isTerminator())
1332 Type *SrcTy = SrcInst->getType();
1333 EVT SrcVT =
TLI.getValueType(
DL, SrcTy);
1334 EVT DestVT =
TLI.getValueType(
DL, DestTy);
1346 bool IsCrossDomain = DestTy->
isFPOrFPVectorTy() != SrcTy->isFPOrFPVectorTy();
1349 unsigned NativeWidth =
DL.getPointerSizeInBits();
1350 bool IsLargeScalar =
1352 DL.getTypeSizeInBits(DestTy).getFixedValue() > NativeWidth;
1354 if (IsCrossDomain || IsLargeScalar)
1360 : std::next(SrcInst->getIterator());
1376 if (!
TLI.isFreeAddrSpaceCast(
DL, ASC->getSrcAddressSpace(),
1377 ASC->getDestAddressSpace()))
1432static std::optional<std::pair<Instruction *, Constant *>>
1435 if (!L || L->getHeader() != PN->
getParent() || !L->getLoopLatch())
1436 return std::nullopt;
1439 if (!IVInc || LI->
getLoopFor(IVInc->getParent()) != L)
1440 return std::nullopt;
1444 return std::make_pair(IVInc, Step);
1445 return std::nullopt;
1458 return IVInc->first ==
I;
1462bool CodeGenPrepare::replaceMathCmpWithIntrinsic(BinaryOperator *BO,
1466 auto IsReplacableIVIncrement = [
this, &
Cmp](BinaryOperator *BO) {
1470 assert(L &&
"L should not be null after isIVIncrement()");
1472 if (LI->getLoopFor(
Cmp->getParent()) != L)
1485 return BO->
hasOneUse() && DT.dominates(
Cmp->getParent(),
L->getLoopLatch());
1487 if (BO->
getParent() !=
Cmp->getParent() && !IsReplacableIVIncrement(BO)) {
1510 if (BO->
getOpcode() == Instruction::Add &&
1511 IID == Intrinsic::usub_with_overflow) {
1518 for (Instruction &Iter : *
Cmp->getParent()) {
1521 if ((BO->
getOpcode() != Instruction::Xor && &Iter == BO) || &Iter == Cmp) {
1526 assert(InsertPt !=
nullptr &&
"Parent block did not contain cmp or binop");
1529 Value *MathOV = Builder.CreateBinaryIntrinsic(IID, Arg0, Arg1);
1530 if (BO->
getOpcode() != Instruction::Xor) {
1531 Value *Math = Builder.CreateExtractValue(MathOV, 0,
"math");
1535 "Patterns with XOr should use the BO only in the compare");
1536 Value *OV = Builder.CreateExtractValue(MathOV, 1,
"ov");
1538 Cmp->eraseFromParent();
1548 Value *
A = Cmp->getOperand(0), *
B = Cmp->getOperand(1);
1556 B = ConstantInt::get(
B->getType(), 1);
1564 for (
User *U :
A->users()) {
1575bool CodeGenPrepare::combineToUAddWithOverflow(CmpInst *Cmp,
1576 ModifyDT &ModifiedDT) {
1577 bool EdgeCase =
false;
1579 BinaryOperator *
Add;
1584 A =
Add->getOperand(0);
1585 B =
Add->getOperand(1);
1591 Add->hasNUsesOrMore(EdgeCase ? 1 : 2)))
1597 if (
Add->getParent() !=
Cmp->getParent() && !
Add->hasOneUse())
1600 if (!replaceMathCmpWithIntrinsic(
Add,
A,
B, Cmp,
1601 Intrinsic::uadd_with_overflow))
1605 ModifiedDT = ModifyDT::ModifyInstDT;
1609bool CodeGenPrepare::combineToUSubWithOverflow(CmpInst *Cmp,
1610 ModifyDT &ModifiedDT) {
1617 ICmpInst::Predicate
Pred =
Cmp->getPredicate();
1618 if (
Pred == ICmpInst::ICMP_UGT) {
1620 Pred = ICmpInst::ICMP_ULT;
1624 B = ConstantInt::get(
B->getType(), 1);
1625 Pred = ICmpInst::ICMP_ULT;
1630 Pred = ICmpInst::ICMP_ULT;
1632 if (
Pred != ICmpInst::ICMP_ULT)
1639 BinaryOperator *
Sub =
nullptr;
1640 for (User *U : CmpVariableOperand->
users()) {
1648 const APInt *CmpC, *AddC;
1660 Sub->hasNUsesOrMore(1)))
1666 if (
Sub->getParent() !=
Cmp->getParent() && !
Sub->hasOneUse())
1669 if (!replaceMathCmpWithIntrinsic(
Sub,
Sub->getOperand(0),
Sub->getOperand(1),
1670 Cmp, Intrinsic::usub_with_overflow))
1674 ModifiedDT = ModifyDT::ModifyInstDT;
1681bool CodeGenPrepare::unfoldPowerOf2Test(CmpInst *Cmp) {
1694 if (!IsStrictlyPowerOf2Test && !IsPowerOf2OrZeroTest)
1700 Type *OpTy =
X->getType();
1708 if (
Pred == ICmpInst::ICMP_EQ) {
1709 Cmp->setOperand(1, ConstantInt::get(OpTy, 2));
1710 Cmp->setPredicate(ICmpInst::ICMP_ULT);
1712 Cmp->setPredicate(ICmpInst::ICMP_UGT);
1718 if (IsPowerOf2OrZeroTest ||
1729 NewCmp = Builder.CreateICmp(NewPred,
And, ConstantInt::getNullValue(OpTy));
1738 NewCmp = Builder.CreateICmp(NewPred,
Xor,
Sub);
1741 Cmp->replaceAllUsesWith(NewCmp);
1754 if (
TLI.hasMultipleConditionRegisters(
EVT::getEVT(Cmp->getType())))
1761 bool UsedInPhiOrCurrentBlock =
any_of(Cmp->users(), [Cmp](
User *U) {
1762 return isa<PHINode>(U) ||
1763 cast<Instruction>(U)->getParent() == Cmp->getParent();
1768 if (UsedInPhiOrCurrentBlock && Cmp->getOperand(0)->getType()->isIntegerTy() &&
1769 Cmp->getOperand(0)->getType()->getScalarSizeInBits() >
1770 DL.getLargestLegalIntTypeSizeInBits())
1776 bool MadeChange =
false;
1779 Use &TheUse = UI.getUse();
1794 if (UserBB == DefBB)
1798 CmpInst *&InsertedCmp = InsertedCmps[UserBB];
1804 Cmp->getOperand(0), Cmp->getOperand(1),
"");
1811 TheUse = InsertedCmp;
1817 if (Cmp->use_empty()) {
1818 Cmp->eraseFromParent();
1845 bool EnableICmpEqToICmpSt) {
1846 if (!EnableICmpEqToICmpSt &&
TLI.isEqualityCmpFoldedWithSignedCmp())
1855 for (
User *U : Cmp->users()) {
1877 if (CmpBB != FalseBB)
1880 Value *CmpOp0 = Cmp->getOperand(0), *CmpOp1 = Cmp->getOperand(1);
1894 for (
User *U : Cmp->users()) {
1896 BI->swapSuccessors();
1902 SI->swapProfMetadata();
1914 Value *Op0 = Cmp->getOperand(0);
1915 Value *Op1 = Cmp->getOperand(1);
1924 unsigned NumInspected = 0;
1927 if (++NumInspected > 128)
1935 if (GoodToSwap > 0) {
1936 Cmp->swapOperands();
1949 EVT VT =
TLI.getValueType(
DL, Cmp->getOperand(0)->getType());
1950 if (
TLI.isFAbsFree(VT) &&
1956 auto ShouldReverseTransform = [](
FPClassTest ClassTest) {
1959 auto [ClassVal, ClassTest] =
1965 if (!ShouldReverseTransform(ClassTest) && !ShouldReverseTransform(~ClassTest))
1969 Value *IsFPClass = Builder.createIsFPClass(ClassVal, ClassTest);
1970 Cmp->replaceAllUsesWith(IsFPClass);
1978 Value *Incr, *RemAmt;
1983 Value *AddInst, *AddOffset;
1986 if (PN !=
nullptr) {
1988 AddOffset =
nullptr;
2006 if (!L || !L->getLoopPreheader() || !L->getLoopLatch())
2010 if (!L->contains(Rem))
2014 if (!L->isLoopInvariant(RemAmt))
2018 if (AddOffset && !L->isLoopInvariant(AddOffset))
2028 Constant *Step = LoopIncrInfo->second;
2032 if (!StepC || !RemC || StepC->isZero() ||
2033 StepC->getValue().uge(RemC->getValue()))
2037 (void)(RemC->getValue() - 1).uadd_ov(StepC->getValue(), Overflow);
2049 AddInstOut = AddInst;
2050 AddOffsetOut = AddOffset;
2072 Value *AddOffset, *RemAmt, *AddInst;
2076 AddOffset, LoopIncrPN, Step))
2105 IRBuilder<> PHBuilder(L->getLoopPreheader()->getTerminator());
2107 assert(AddOffset &&
"We found an add but missing values");
2113 if (!NewStart && !IsConstRem)
2115 Start = NewStart ? NewStart : PHBuilder.
CreateNUWAdd(Start, AddOffset);
2123 StartRem = PHBuilder.
CreateURem(Start, RemAmt);
2128 PHINode *NewRem = Builder.CreatePHI(Ty, 2);
2134 Value *RemAdd = Builder.CreateNUWAdd(NewRem, Step);
2141 Value *RemSub = Builder.CreateNUWSub(RemAdd, RemAmt);
2142 RemSel = Builder.CreateSelect(RemCmp, RemSub, RemAdd);
2147 NewRem->
addIncoming(StartRem, L->getLoopPreheader());
2151 FreshBBs.
insert(L->getLoopPreheader());
2153 FreshBBs.
insert(L->getLoopLatch());
2164bool CodeGenPrepare::optimizeURem(Instruction *Rem) {
2170bool CodeGenPrepare::optimizeCmp(CmpInst *Cmp, ModifyDT &ModifiedDT) {
2174 if (combineToUAddWithOverflow(Cmp, ModifiedDT))
2177 if (combineToUSubWithOverflow(Cmp, ModifiedDT))
2180 if (unfoldPowerOf2Test(Cmp))
2201 SetOfInstrs &InsertedInsts) {
2204 assert(!InsertedInsts.count(AndI) &&
2205 "Attempting to optimize already optimized and instruction");
2206 (void)InsertedInsts;
2220 for (
auto *U : AndI->
users()) {
2228 if (!CmpC || !CmpC->
isZero())
2232 if (!
TLI.isMaskAndCmp0FoldingBeneficial(*AndI))
2243 Use &TheUse = UI.getUse();
2261 TheUse = InsertedAnd;
2278 if (
User->getOpcode() != Instruction::And ||
2284 if ((Cimm & (Cimm + 1)).getBoolValue())
2298 bool MadeChange =
false;
2301 TruncE = TruncI->user_end();
2302 TruncUI != TruncE;) {
2304 Use &TruncTheUse = TruncUI.getUse();
2310 int ISDOpcode =
TLI.InstructionOpcodeToISD(TruncUser->
getOpcode());
2319 if (
TLI.isOperationLegalOrCustom(
2320 ISDOpcode,
TLI.getValueType(
DL, TruncUser->
getType(),
true)))
2329 if (UserBB == TruncUserBB)
2333 CastInst *&InsertedTrunc = InsertedTruncs[TruncUserBB];
2335 if (!InsertedShift && !InsertedTrunc) {
2339 if (ShiftI->
getOpcode() == Instruction::AShr)
2341 BinaryOperator::CreateAShr(ShiftI->
getOperand(0), CI,
"");
2344 BinaryOperator::CreateLShr(ShiftI->
getOperand(0), CI,
"");
2352 TruncInsertPt.setHeadBit(
true);
2353 assert(TruncInsertPt != TruncUserBB->
end());
2357 InsertedTrunc->
insertBefore(*TruncUserBB, TruncInsertPt);
2358 InsertedTrunc->
setDebugLoc(TruncI->getDebugLoc());
2362 TruncTheUse = InsertedTrunc;
2393 bool shiftIsLegal =
TLI.isTypeLegal(
TLI.getValueType(
DL, ShiftI->
getType()));
2395 bool MadeChange =
false;
2399 Use &TheUse = UI.getUse();
2413 if (UserBB == DefBB) {
2441 if (!InsertedShift) {
2445 if (ShiftI->
getOpcode() == Instruction::AShr)
2447 BinaryOperator::CreateAShr(ShiftI->
getOperand(0), CI,
"");
2450 BinaryOperator::CreateLShr(ShiftI->
getOperand(0), CI,
"");
2458 TheUse = InsertedShift;
2501 if ((IntrinsicID == Intrinsic::cttz &&
TLI->isCheapToSpeculateCttz(Ty)) ||
2502 (IntrinsicID == Intrinsic::ctlz &&
TLI->isCheapToSpeculateCtlz(Ty)))
2506 unsigned SizeInBits = Ty->getScalarSizeInBits();
2507 if (Ty->isVectorTy())
2518 nullptr,
"cond.false");
2520 FreshBBs.
insert(CallBlock);
2527 SplitPt.setHeadBit(
true);
2529 nullptr,
"cond.end");
2531 FreshBBs.
insert(EndBlock);
2535 Builder.SetCurrentDebugLocation(CountZeros->
getDebugLoc());
2542 Op = Builder.CreateFreeze(
Op,
Op->getName() +
".fr");
2543 Value *Cmp = Builder.CreateICmpEQ(
Op,
Zero,
"cmpz");
2544 Builder.CreateCondBr(Cmp, EndBlock, CallBlock);
2550 Builder.SetInsertPoint(EndBlock->
begin());
2551 PHINode *PN = Builder.CreatePHI(Ty, 2,
"ctz");
2561 ModifiedDT = ModifyDT::ModifyBBDT;
2565bool CodeGenPrepare::optimizeCallInst(CallInst *CI, ModifyDT &ModifiedDT) {
2569 if (CI->
isInlineAsm() && optimizeInlineAsmInst(CI))
2577 for (
auto &Arg : CI->
args()) {
2582 if (!Arg->getType()->isPointerTy())
2584 APInt
Offset(
DL->getIndexSizeInBits(
2587 Value *Val = Arg->stripAndAccumulateInBoundsConstantOffsets(*
DL,
Offset);
2594 if (AllocaSize && AllocaSize->getKnownMinValue() >= MinSize + Offset2)
2612 MaybeAlign MIDestAlign =
MI->getDestAlign();
2613 if (!MIDestAlign || DestAlign > *MIDestAlign)
2614 MI->setDestAlignment(DestAlign);
2616 MaybeAlign MTISrcAlign = MTI->getSourceAlign();
2618 if (!MTISrcAlign || SrcAlign > *MTISrcAlign)
2619 MTI->setSourceAlignment(SrcAlign);
2629 for (
auto &Arg : CI->
args()) {
2630 if (!Arg->getType()->isPointerTy())
2632 unsigned AS = Arg->getType()->getPointerAddressSpace();
2633 if (optimizeMemoryInst(CI, Arg, Arg->getType(), AS))
2639 switch (
II->getIntrinsicID()) {
2642 case Intrinsic::assume:
2644 case Intrinsic::allow_runtime_check:
2645 case Intrinsic::allow_ubsan_check:
2646 case Intrinsic::experimental_widenable_condition: {
2650 if (
II->use_empty()) {
2651 II->eraseFromParent();
2655 resetIteratorIfInvalidatedWhileCalling(BB, [&]() {
2660 case Intrinsic::objectsize:
2662 case Intrinsic::is_constant:
2664 case Intrinsic::aarch64_stlxr:
2665 case Intrinsic::aarch64_stxr: {
2674 InsertedInsts.insert(ExtVal);
2678 case Intrinsic::launder_invariant_group: {
2679 Value *ArgVal =
II->getArgOperand(0);
2680 auto it = LargeOffsetGEPMap.
find(
II);
2681 if (it != LargeOffsetGEPMap.
end()) {
2685 auto GEPs = std::move(it->second);
2686 LargeOffsetGEPMap[ArgVal].append(GEPs.begin(), GEPs.end());
2691 II->eraseFromParent();
2694 case Intrinsic::cttz:
2695 case Intrinsic::ctlz:
2699 case Intrinsic::fshl:
2700 case Intrinsic::fshr:
2701 return optimizeFunnelShift(
II);
2702 case Intrinsic::masked_gather:
2703 return optimizeGatherScatterInst(
II,
II->getArgOperand(0));
2704 case Intrinsic::masked_scatter:
2705 return optimizeGatherScatterInst(
II,
II->getArgOperand(1));
2706 case Intrinsic::masked_load:
2709 if (VT->getNumElements() == 1) {
2710 Value *PtrVal =
II->getArgOperand(0);
2712 if (optimizeMemoryInst(
II, PtrVal, VT->getElementType(), AS))
2717 case Intrinsic::masked_store:
2721 if (VT->getNumElements() == 1) {
2722 Value *PtrVal =
II->getArgOperand(1);
2724 if (optimizeMemoryInst(
II, PtrVal, VT->getElementType(), AS))
2729 case Intrinsic::umul_with_overflow:
2730 return optimizeMulWithOverflow(
II,
false, ModifiedDT);
2731 case Intrinsic::smul_with_overflow:
2732 return optimizeMulWithOverflow(
II,
true, ModifiedDT);
2735 SmallVector<Value *, 2> PtrOps;
2738 while (!PtrOps.
empty()) {
2741 if (optimizeMemoryInst(
II, PtrVal, AccessTy, AS))
2755 FortifiedLibCallSimplifier Simplifier(TLInfo,
true);
2757 if (
Value *
V = Simplifier.optimizeCall(CI, Builder)) {
2767 auto GetUniformReturnValue = [](
const Function *
F) -> GlobalVariable * {
2768 if (!
F->getReturnType()->isPointerTy())
2771 GlobalVariable *UniformValue =
nullptr;
2772 for (
auto &BB : *
F) {
2777 else if (
V != UniformValue)
2785 return UniformValue;
2788 if (
Callee->hasExactDefinition()) {
2789 if (GlobalVariable *RV = GetUniformReturnValue(Callee)) {
2790 bool MadeChange =
false;
2816 switch (
II->getIntrinsicID()) {
2817 case Intrinsic::memset:
2818 case Intrinsic::memcpy:
2819 case Intrinsic::memmove:
2826 if (Callee && TLInfo)
2828 case LibFunc_strcpy:
2829 case LibFunc_strncpy:
2830 case LibFunc_strcat:
2831 case LibFunc_strncat:
2872bool CodeGenPrepare::dupRetToEnableTailCallOpts(BasicBlock *BB,
2873 ModifyDT &ModifiedDT) {
2878 assert(LI->getLoopFor(BB) ==
nullptr &&
"A return block cannot be in a loop");
2880 PHINode *PN =
nullptr;
2881 ExtractValueInst *EVI =
nullptr;
2882 BitCastInst *BCI =
nullptr;
2904 auto isFakeUse = [&FakeUses](
const Instruction *Inst) {
2906 II &&
II->getIntrinsicID() == Intrinsic::fake_use) {
2935 auto MayBePermittedAsTailCall = [&](
const auto *CI) {
2952 MayBePermittedAsTailCall(CI)) {
2973 MayBePermittedAsTailCall(CI)) {
2980 SmallPtrSet<BasicBlock *, 4> VisitedBBs;
2984 if (Instruction *
I =
Pred->rbegin()->getPrevNode()) {
2986 if (CI && CI->
use_empty() && MayBePermittedAsTailCall(CI)) {
3001 for (
auto const &TailCallBB : TailCallBBs) {
3010 assert(!Opts.cgp_verify_bfi_updates ||
3011 BFI->getBlockFreq(BB) >= BFI->getBlockFreq(TailCallBB));
3012 BFI->setBlockFreq(BB,
3013 (BFI->getBlockFreq(BB) - BFI->getBlockFreq(TailCallBB)));
3014 ModifiedDT = ModifyDT::ModifyBBDT;
3023 for (
auto *CI : CallInsts) {
3024 for (
auto const *FakeUse : FakeUses) {
3025 auto *ClonedInst = FakeUse->clone();
3043struct ExtAddrMode :
public TargetLowering::AddrMode {
3044 Value *BaseReg =
nullptr;
3045 Value *ScaledReg =
nullptr;
3046 Value *OriginalValue =
nullptr;
3047 bool InBounds =
true;
3051 BaseRegField = 0x01,
3053 BaseOffsField = 0x04,
3054 ScaledRegField = 0x08,
3056 MultipleFields = 0xff
3059 ExtAddrMode() =
default;
3061 void print(raw_ostream &OS)
const;
3068 if (ScaledReg == From)
3072 FieldName
compare(
const ExtAddrMode &other) {
3075 if (BaseReg && other.
BaseReg &&
3077 return MultipleFields;
3078 if (BaseGV && other.BaseGV && BaseGV->getType() != other.BaseGV->getType())
3079 return MultipleFields;
3082 return MultipleFields;
3085 if (InBounds != other.InBounds)
3086 return MultipleFields;
3089 unsigned Result = NoField;
3092 if (BaseGV != other.BaseGV)
3094 if (BaseOffs != other.BaseOffs)
3097 Result |= ScaledRegField;
3100 if (Scale && other.
Scale && Scale != other.
Scale)
3104 return MultipleFields;
3106 return static_cast<FieldName
>(
Result);
3116 return !BaseOffs && !Scale && !(BaseGV &&
BaseReg);
3127 case ScaledRegField:
3135 const SmallVectorImpl<ExtAddrMode> &AddrModes) {
3140 case ExtAddrMode::BaseRegField:
3143 case ExtAddrMode::BaseGVField:
3146 assert(BaseReg ==
nullptr);
3150 case ExtAddrMode::ScaledRegField:
3155 for (
const ExtAddrMode &AM : AddrModes)
3161 case ExtAddrMode::BaseOffsField:
3164 assert(ScaledReg ==
nullptr);
3174static inline raw_ostream &
operator<<(raw_ostream &OS,
const ExtAddrMode &AM) {
3180#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3181void ExtAddrMode::print(raw_ostream &OS)
const {
3182 bool NeedPlus =
false;
3188 BaseGV->printAsOperand(OS,
false);
3193 OS << (NeedPlus ?
" + " :
"") << BaseOffs;
3198 OS << (NeedPlus ?
" + " :
"") <<
"Base:";
3199 BaseReg->printAsOperand(OS,
false);
3203 OS << (NeedPlus ?
" + " :
"") << Scale <<
"*";
3226class TypePromotionTransaction {
3230 class TypePromotionAction {
3238 TypePromotionAction(Instruction *Inst) : Inst(Inst) {}
3240 virtual ~TypePromotionAction() =
default;
3247 virtual void undo() = 0;
3252 virtual void commit() {
3258 class InsertionHandler {
3267 std::optional<DbgRecord::self_iterator> BeforeDbgRecord = std::nullopt;
3270 bool HasPrevInstruction;
3274 InsertionHandler(Instruction *Inst) {
3282 if (HasPrevInstruction) {
3290 void insert(Instruction *Inst) {
3291 if (HasPrevInstruction) {
3303 Inst->
getParent()->reinsertInstInDbgRecords(Inst, BeforeDbgRecord);
3308 class OperandSetter :
public TypePromotionAction {
3317 OperandSetter(Instruction *Inst,
unsigned Idx,
Value *NewVal)
3318 : TypePromotionAction(Inst),
Idx(
Idx) {
3320 <<
"for:" << *Inst <<
"\n"
3321 <<
"with:" << *NewVal <<
"\n");
3327 void undo()
override {
3329 <<
"for: " << *Inst <<
"\n"
3330 <<
"with: " << *Origin <<
"\n");
3337 class OperandsHider :
public TypePromotionAction {
3339 SmallVector<Value *, 4> OriginalValues;
3343 OperandsHider(Instruction *Inst) : TypePromotionAction(Inst) {
3346 OriginalValues.
reserve(NumOpnds);
3347 for (
unsigned It = 0; It < NumOpnds; ++It) {
3359 void undo()
override {
3361 for (
unsigned It = 0, EndIt = OriginalValues.
size(); It != EndIt; ++It)
3367 class TruncBuilder :
public TypePromotionAction {
3374 TruncBuilder(Instruction *Opnd,
Type *Ty) : TypePromotionAction(Opnd) {
3376 Builder.SetCurrentDebugLocation(
DebugLoc());
3377 Val = Builder.CreateTrunc(Opnd, Ty,
"promoted");
3382 Value *getBuiltValue() {
return Val; }
3385 void undo()
override {
3388 IVal->eraseFromParent();
3393 class SExtBuilder :
public TypePromotionAction {
3400 SExtBuilder(Instruction *InsertPt,
Value *Opnd,
Type *Ty)
3401 : TypePromotionAction(InsertPt) {
3403 Val = Builder.CreateSExt(Opnd, Ty,
"promoted");
3408 Value *getBuiltValue() {
return Val; }
3411 void undo()
override {
3414 IVal->eraseFromParent();
3419 class ZExtBuilder :
public TypePromotionAction {
3426 ZExtBuilder(Instruction *InsertPt,
Value *Opnd,
Type *Ty)
3427 : TypePromotionAction(InsertPt) {
3429 Builder.SetCurrentDebugLocation(
DebugLoc());
3430 Val = Builder.CreateZExt(Opnd, Ty,
"promoted");
3435 Value *getBuiltValue() {
return Val; }
3438 void undo()
override {
3441 IVal->eraseFromParent();
3446 class TypeMutator :
public TypePromotionAction {
3452 TypeMutator(Instruction *Inst,
Type *NewTy)
3453 : TypePromotionAction(Inst), OrigTy(Inst->
getType()) {
3454 LLVM_DEBUG(
dbgs() <<
"Do: MutateType: " << *Inst <<
" with " << *NewTy
3460 void undo()
override {
3461 LLVM_DEBUG(
dbgs() <<
"Undo: MutateType: " << *Inst <<
" with " << *OrigTy
3468 class UsesReplacer :
public TypePromotionAction {
3470 struct InstructionAndIdx {
3477 InstructionAndIdx(Instruction *Inst,
unsigned Idx)
3478 : Inst(Inst),
Idx(
Idx) {}
3484 SmallVector<DbgVariableRecord *, 1> DbgVariableRecords;
3492 UsesReplacer(Instruction *Inst,
Value *New)
3493 : TypePromotionAction(Inst),
New(
New) {
3494 LLVM_DEBUG(
dbgs() <<
"Do: UsersReplacer: " << *Inst <<
" with " << *New
3497 for (Use &U : Inst->
uses()) {
3499 OriginalUses.
push_back(InstructionAndIdx(UserI,
U.getOperandNo()));
3510 void undo()
override {
3512 for (InstructionAndIdx &Use : OriginalUses)
3513 Use.Inst->setOperand(
Use.Idx, Inst);
3518 for (DbgVariableRecord *DVR : DbgVariableRecords)
3519 DVR->replaceVariableLocationOp(New, Inst);
3524 class InstructionRemover :
public TypePromotionAction {
3526 InsertionHandler Inserter;
3530 OperandsHider Hider;
3533 UsesReplacer *Replacer =
nullptr;
3536 SetOfInstrs &RemovedInsts;
3543 InstructionRemover(Instruction *Inst, SetOfInstrs &RemovedInsts,
3544 Value *New =
nullptr)
3545 : TypePromotionAction(Inst), Inserter(Inst), Hider(Inst),
3546 RemovedInsts(RemovedInsts) {
3548 Replacer =
new UsesReplacer(Inst, New);
3549 LLVM_DEBUG(
dbgs() <<
"Do: InstructionRemover: " << *Inst <<
"\n");
3550 RemovedInsts.insert(Inst);
3557 ~InstructionRemover()
override {
delete Replacer; }
3559 InstructionRemover &operator=(
const InstructionRemover &other) =
delete;
3560 InstructionRemover(
const InstructionRemover &other) =
delete;
3564 void undo()
override {
3565 LLVM_DEBUG(
dbgs() <<
"Undo: InstructionRemover: " << *Inst <<
"\n");
3566 Inserter.insert(Inst);
3570 RemovedInsts.erase(Inst);
3578 using ConstRestorationPt =
const TypePromotionAction *;
3580 TypePromotionTransaction(SetOfInstrs &RemovedInsts)
3581 : RemovedInsts(RemovedInsts) {}
3588 void rollback(ConstRestorationPt Point);
3591 ConstRestorationPt getRestorationPoint()
const;
3596 void setOperand(Instruction *Inst,
unsigned Idx,
Value *NewVal);
3605 void mutateType(Instruction *Inst,
Type *NewTy);
3608 Value *createTrunc(Instruction *Opnd,
Type *Ty);
3620 SetOfInstrs &RemovedInsts;
3625void TypePromotionTransaction::setOperand(Instruction *Inst,
unsigned Idx,
3627 Actions.push_back(std::make_unique<TypePromotionTransaction::OperandSetter>(
3628 Inst, Idx, NewVal));
3631void TypePromotionTransaction::eraseInstruction(Instruction *Inst,
3634 std::make_unique<TypePromotionTransaction::InstructionRemover>(
3635 Inst, RemovedInsts, NewVal));
3638void TypePromotionTransaction::replaceAllUsesWith(Instruction *Inst,
3641 std::make_unique<TypePromotionTransaction::UsesReplacer>(Inst, New));
3644void TypePromotionTransaction::mutateType(Instruction *Inst,
Type *NewTy) {
3646 std::make_unique<TypePromotionTransaction::TypeMutator>(Inst, NewTy));
3649Value *TypePromotionTransaction::createTrunc(Instruction *Opnd,
Type *Ty) {
3650 std::unique_ptr<TruncBuilder> Ptr(
new TruncBuilder(Opnd, Ty));
3651 Value *Val = Ptr->getBuiltValue();
3652 Actions.push_back(std::move(Ptr));
3656Value *TypePromotionTransaction::createSExt(Instruction *Inst,
Value *Opnd,
3658 std::unique_ptr<SExtBuilder> Ptr(
new SExtBuilder(Inst, Opnd, Ty));
3659 Value *Val = Ptr->getBuiltValue();
3660 Actions.push_back(std::move(Ptr));
3664Value *TypePromotionTransaction::createZExt(Instruction *Inst,
Value *Opnd,
3666 std::unique_ptr<ZExtBuilder> Ptr(
new ZExtBuilder(Inst, Opnd, Ty));
3667 Value *Val = Ptr->getBuiltValue();
3668 Actions.push_back(std::move(Ptr));
3672TypePromotionTransaction::ConstRestorationPt
3673TypePromotionTransaction::getRestorationPoint()
const {
3674 return !Actions.empty() ? Actions.back().get() :
nullptr;
3677bool TypePromotionTransaction::commit() {
3678 for (std::unique_ptr<TypePromotionAction> &Action : Actions)
3685void TypePromotionTransaction::rollback(
3686 TypePromotionTransaction::ConstRestorationPt Point) {
3687 while (!Actions.empty() && Point != Actions.back().get()) {
3688 std::unique_ptr<TypePromotionAction> Curr = Actions.pop_back_val();
3698class AddressingModeMatcher {
3699 SmallVectorImpl<Instruction *> &AddrModeInsts;
3700 const TargetLowering &
TLI;
3701 const TargetRegisterInfo &
TRI;
3702 const DataLayout &
DL;
3704 const std::function<
const DominatorTree &()> getDTFn;
3717 const SetOfInstrs &InsertedInsts;
3720 InstrToOrigTy &PromotedInsts;
3723 TypePromotionTransaction &TPT;
3726 std::pair<AssertingVH<GetElementPtrInst>, int64_t> &LargeOffsetGEP;
3730 bool IgnoreProfitability;
3733 bool OptSize =
false;
3735 ProfileSummaryInfo *PSI;
3736 BlockFrequencyInfo *BFI;
3737 const CodeGenOptions &Opts;
3739 AddressingModeMatcher(
3740 SmallVectorImpl<Instruction *> &AMI,
const TargetLowering &
TLI,
3741 const TargetRegisterInfo &
TRI,
const LoopInfo &LI,
3742 const std::function<
const DominatorTree &()> getDTFn,
Type *AT,
3743 unsigned AS, Instruction *
MI, ExtAddrMode &AM,
3744 const SetOfInstrs &InsertedInsts, InstrToOrigTy &PromotedInsts,
3745 TypePromotionTransaction &TPT,
3746 std::pair<AssertingVH<GetElementPtrInst>, int64_t> &LargeOffsetGEP,
3747 bool OptSize, ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI,
3748 const CodeGenOptions &Opts)
3750 getDTFn(getDTFn), AccessTy(AT), AddrSpace(AS), MemoryInst(
MI),
3751 AddrMode(AM), InsertedInsts(InsertedInsts),
3752 PromotedInsts(PromotedInsts), TPT(TPT), LargeOffsetGEP(LargeOffsetGEP),
3753 OptSize(OptSize), PSI(PSI), BFI(BFI), Opts(Opts) {
3754 IgnoreProfitability =
false;
3766 Match(
Value *
V,
Type *AccessTy,
unsigned AS, Instruction *MemoryInst,
3767 SmallVectorImpl<Instruction *> &AddrModeInsts,
3768 const TargetLowering &
TLI,
const LoopInfo &LI,
3769 const std::function<
const DominatorTree &()> getDTFn,
3770 const TargetRegisterInfo &
TRI,
const SetOfInstrs &InsertedInsts,
3771 InstrToOrigTy &PromotedInsts, TypePromotionTransaction &TPT,
3772 std::pair<AssertingVH<GetElementPtrInst>, int64_t> &LargeOffsetGEP,
3773 bool OptSize, ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI,
3774 const CodeGenOptions &Opts) {
3777 bool Success = AddressingModeMatcher(
3778 AddrModeInsts,
TLI,
TRI, LI, getDTFn, AccessTy, AS,
3779 MemoryInst, Result, InsertedInsts, PromotedInsts, TPT,
3780 LargeOffsetGEP, OptSize, PSI, BFI, Opts)
3788 bool matchScaledValue(
Value *ScaleReg, int64_t Scale,
unsigned Depth);
3790 bool matchOperationAddr(User *AddrInst,
unsigned Opcode,
unsigned Depth,
3791 bool *MovedAway =
nullptr);
3792 bool isProfitableToFoldIntoAddressingMode(Instruction *
I,
3793 ExtAddrMode &AMBefore,
3794 ExtAddrMode &AMAfter);
3795 bool valueAlreadyLiveAtInst(
Value *Val,
Value *KnownLive1,
Value *KnownLive2);
3796 bool isPromotionProfitable(
unsigned NewCost,
unsigned OldCost,
3797 Value *PromotedOperand)
const;
3803class PhiNodeSetIterator {
3804 PhiNodeSet *
const Set;
3805 size_t CurrentIndex = 0;
3810 PhiNodeSetIterator(PhiNodeSet *
const Set,
size_t Start);
3812 PhiNodeSetIterator &operator++();
3828 friend class PhiNodeSetIterator;
3830 using MapType = SmallDenseMap<PHINode *, size_t, 32>;
3831 using iterator = PhiNodeSetIterator;
3846 size_t FirstValidElement = 0;
3852 bool insert(PHINode *Ptr) {
3853 if (NodeMap.insert(std::make_pair(Ptr,
NodeList.
size())).second) {
3863 bool erase(PHINode *Ptr) {
3864 if (NodeMap.erase(Ptr)) {
3865 SkipRemovedElements(FirstValidElement);
3875 FirstValidElement = 0;
3881 if (FirstValidElement == 0)
3882 SkipRemovedElements(FirstValidElement);
3883 return PhiNodeSetIterator(
this, FirstValidElement);
3890 size_t size()
const {
return NodeMap.size(); }
3893 size_t count(PHINode *Ptr)
const {
return NodeMap.count(Ptr); }
3901 void SkipRemovedElements(
size_t &CurrentIndex) {
3903 auto it = NodeMap.find(NodeList[CurrentIndex]);
3906 if (it != NodeMap.end() && it->second == CurrentIndex)
3913PhiNodeSetIterator::PhiNodeSetIterator(PhiNodeSet *
const Set,
size_t Start)
3916PHINode *PhiNodeSetIterator::operator*()
const {
3918 "PhiNodeSet access out of range");
3919 return Set->NodeList[CurrentIndex];
3922PhiNodeSetIterator &PhiNodeSetIterator::operator++() {
3924 "PhiNodeSet access out of range");
3926 Set->SkipRemovedElements(CurrentIndex);
3930bool PhiNodeSetIterator::operator==(
const PhiNodeSetIterator &
RHS)
const {
3931 return CurrentIndex ==
RHS.CurrentIndex;
3934bool PhiNodeSetIterator::operator!=(
const PhiNodeSetIterator &
RHS)
const {
3935 return !((*this) ==
RHS);
3941class SimplificationTracker {
3942 DenseMap<Value *, Value *> Storage;
3945 PhiNodeSet AllPhiNodes;
3947 SmallPtrSet<SelectInst *, 32> AllSelectNodes;
3953 if (SV == Storage.
end())
3961 void ReplacePhi(PHINode *From, PHINode *To) {
3962 Value *OldReplacement = Get(From);
3963 while (OldReplacement != From) {
3966 OldReplacement = Get(From);
3968 assert(To && Get(To) == To &&
"Replacement PHI node is already replaced.");
3971 AllPhiNodes.erase(From);
3975 PhiNodeSet &newPhiNodes() {
return AllPhiNodes; }
3977 void insertNewPhi(PHINode *PN) { AllPhiNodes.insert(PN); }
3979 void insertNewSelect(SelectInst *SI) { AllSelectNodes.
insert(SI); }
3981 unsigned countNewPhiNodes()
const {
return AllPhiNodes.size(); }
3983 unsigned countNewSelectNodes()
const {
return AllSelectNodes.
size(); }
3985 void destroyNewNodes(
Type *CommonType) {
3988 for (
auto *
I : AllPhiNodes) {
3989 I->replaceAllUsesWith(Dummy);
3990 I->eraseFromParent();
3992 AllPhiNodes.clear();
3993 for (
auto *
I : AllSelectNodes) {
3994 I->replaceAllUsesWith(Dummy);
3995 I->eraseFromParent();
3997 AllSelectNodes.clear();
4002class AddressingModeCombiner {
4003 typedef DenseMap<Value *, Value *> FoldAddrToValueMapping;
4004 typedef std::pair<PHINode *, PHINode *> PHIPair;
4011 ExtAddrMode::FieldName DifferentField = ExtAddrMode::NoField;
4014 bool AllAddrModesTrivial =
true;
4017 Type *CommonType =
nullptr;
4019 const DataLayout &
DL;
4020 const CodeGenOptions &Opts;
4026 Value *CommonValue =
nullptr;
4029 AddressingModeCombiner(
const DataLayout &
DL,
const CodeGenOptions &Opts,
4030 Value *OriginalValue)
4031 :
DL(
DL), Opts(Opts), Original(OriginalValue) {}
4033 ~AddressingModeCombiner() { eraseCommonValueIfDead(); }
4036 const ExtAddrMode &
getAddrMode()
const {
return AddrModes[0]; }
4041 bool addNewAddrMode(ExtAddrMode &NewAddrMode) {
4045 AllAddrModesTrivial = AllAddrModesTrivial && NewAddrMode.isTrivial();
4048 if (AddrModes.
empty()) {
4056 ExtAddrMode::FieldName ThisDifferentField =
4057 AddrModes[0].compare(NewAddrMode);
4058 if (DifferentField == ExtAddrMode::NoField)
4059 DifferentField = ThisDifferentField;
4060 else if (DifferentField != ThisDifferentField)
4061 DifferentField = ExtAddrMode::MultipleFields;
4064 bool CanHandle = DifferentField != ExtAddrMode::MultipleFields;
4067 CanHandle = CanHandle && DifferentField != ExtAddrMode::ScaleField;
4072 CanHandle = CanHandle && (DifferentField != ExtAddrMode::BaseOffsField ||
4077 CanHandle = CanHandle && (DifferentField != ExtAddrMode::BaseGVField ||
4078 !NewAddrMode.HasBaseReg);
4095 bool combineAddrModes() {
4097 if (AddrModes.
size() == 0)
4101 if (AddrModes.
size() == 1 || DifferentField == ExtAddrMode::NoField)
4106 if (AllAddrModesTrivial)
4109 if (!addrModeCombiningAllowed())
4115 FoldAddrToValueMapping
Map;
4116 if (!initializeMap(Map))
4119 CommonValue = findCommon(Map);
4121 AddrModes[0].SetCombinedField(DifferentField, CommonValue, AddrModes);
4122 return CommonValue !=
nullptr;
4128 void eraseCommonValueIfDead() {
4129 if (CommonValue && CommonValue->
use_empty())
4131 CommonInst->eraseFromParent();
4139 bool initializeMap(FoldAddrToValueMapping &Map) {
4142 SmallVector<Value *, 2> NullValue;
4144 for (
auto &AM : AddrModes) {
4148 if (CommonType && CommonType !=
Type)
4151 Map[AM.OriginalValue] = DV;
4156 assert(CommonType &&
"At least one non-null value must be!");
4157 for (
auto *
V : NullValue)
4185 Value *findCommon(FoldAddrToValueMapping &Map) {
4193 SimplificationTracker
ST;
4198 InsertPlaceholders(Map, TraverseOrder, ST);
4201 FillPlaceholders(Map, TraverseOrder, ST);
4203 if (!Opts.cgp_addr_sink_new_select &&
ST.countNewSelectNodes() > 0) {
4204 ST.destroyNewNodes(CommonType);
4209 unsigned PhiNotMatchedCount = 0;
4210 if (!MatchPhiSet(ST, Opts.cgp_addr_sink_new_phis, PhiNotMatchedCount)) {
4211 ST.destroyNewNodes(CommonType);
4215 auto *
Result =
ST.Get(
Map.find(Original)->second);
4217 NumMemoryInstsPhiCreated +=
ST.countNewPhiNodes() + PhiNotMatchedCount;
4218 NumMemoryInstsSelectCreated +=
ST.countNewSelectNodes();
4225 bool MatchPhiNode(PHINode *
PHI, PHINode *Candidate,
4226 SmallSetVector<PHIPair, 8> &Matcher,
4227 PhiNodeSet &PhiNodesToMatch) {
4230 SmallPtrSet<PHINode *, 8> MatchedPHIs;
4233 SmallSet<PHIPair, 8> Visited;
4234 while (!WorkList.
empty()) {
4236 if (!Visited.
insert(Item).second)
4243 for (
auto *
B : Item.first->blocks()) {
4244 Value *FirstValue = Item.first->getIncomingValueForBlock(
B);
4245 Value *SecondValue = Item.second->getIncomingValueForBlock(
B);
4246 if (FirstValue == SecondValue)
4256 if (!FirstPhi || !SecondPhi || !PhiNodesToMatch.count(FirstPhi) ||
4261 if (Matcher.
count({FirstPhi, SecondPhi}))
4266 if (MatchedPHIs.
insert(FirstPhi).second)
4267 Matcher.
insert({FirstPhi, SecondPhi});
4269 WorkList.
push_back({FirstPhi, SecondPhi});
4278 bool MatchPhiSet(SimplificationTracker &ST,
bool AllowNewPhiNodes,
4279 unsigned &PhiNotMatchedCount) {
4283 SmallSetVector<PHIPair, 8> Matched;
4284 SmallPtrSet<PHINode *, 8> WillNotMatch;
4285 PhiNodeSet &PhiNodesToMatch =
ST.newPhiNodes();
4286 while (PhiNodesToMatch.size()) {
4287 PHINode *
PHI = *PhiNodesToMatch.begin();
4290 WillNotMatch.
clear();
4294 bool IsMatched =
false;
4295 for (
auto &
P :
PHI->getParent()->phis()) {
4297 if (PhiNodesToMatch.count(&
P))
4299 if ((IsMatched = MatchPhiNode(
PHI, &
P, Matched, PhiNodesToMatch)))
4309 for (
auto MV : Matched)
4310 ST.ReplacePhi(MV.first, MV.second);
4315 if (!AllowNewPhiNodes)
4318 PhiNotMatchedCount += WillNotMatch.
size();
4319 for (
auto *
P : WillNotMatch)
4320 PhiNodesToMatch.erase(
P);
4325 void FillPlaceholders(FoldAddrToValueMapping &Map,
4326 SmallVectorImpl<Value *> &TraverseOrder,
4327 SimplificationTracker &ST) {
4328 while (!TraverseOrder.
empty()) {
4330 assert(
Map.contains(Current) &&
"No node to fill!!!");
4336 auto *TrueValue = CurrentSelect->getTrueValue();
4337 assert(
Map.contains(TrueValue) &&
"No True Value!");
4338 Select->setTrueValue(
ST.Get(Map[TrueValue]));
4339 auto *FalseValue = CurrentSelect->getFalseValue();
4340 assert(
Map.contains(FalseValue) &&
"No False Value!");
4341 Select->setFalseValue(
ST.Get(Map[FalseValue]));
4348 assert(
Map.contains(PV) &&
"No predecessor Value!");
4349 PHI->addIncoming(
ST.Get(Map[PV]),
B);
4360 void InsertPlaceholders(FoldAddrToValueMapping &Map,
4361 SmallVectorImpl<Value *> &TraverseOrder,
4362 SimplificationTracker &ST) {
4365 "Address must be a Phi or Select node");
4368 while (!Worklist.
empty()) {
4371 if (
Map.contains(Current))
4382 CurrentSelect->getName(),
4383 CurrentSelect->getIterator(), CurrentSelect);
4387 Worklist.
push_back(CurrentSelect->getTrueValue());
4388 Worklist.
push_back(CurrentSelect->getFalseValue());
4396 ST.insertNewPhi(
PHI);
4402 bool addrModeCombiningAllowed() {
4403 if (!Opts.cgp_complex_addr_modes)
4405 switch (DifferentField) {
4408 case ExtAddrMode::BaseRegField:
4409 return Opts.cgp_addr_sink_combine_base_reg;
4410 case ExtAddrMode::BaseGVField:
4411 return Opts.cgp_addr_sink_combine_base_gv;
4412 case ExtAddrMode::BaseOffsField:
4413 return Opts.cgp_addr_sink_combine_base_offs;
4414 case ExtAddrMode::ScaledRegField:
4415 return Opts.cgp_addr_sink_combine_scaled_reg;
4424bool AddressingModeMatcher::matchScaledValue(
Value *ScaleReg, int64_t Scale,
4429 return matchAddr(ScaleReg,
Depth);
4440 ExtAddrMode TestAddrMode =
AddrMode;
4444 TestAddrMode.
Scale += Scale;
4458 ConstantInt *CI =
nullptr;
4459 Value *AddLHS =
nullptr;
4463 TestAddrMode.InBounds =
false;
4480 auto GetConstantStep =
4481 [
this](
const Value *
V) -> std::optional<std::pair<Instruction *, APInt>> {
4484 return std::nullopt;
4487 return std::nullopt;
4495 if (OIVInc->hasNoSignedWrap() || OIVInc->hasNoUnsignedWrap())
4496 return std::nullopt;
4498 return std::make_pair(IVInc->first, ConstantStep->getValue());
4499 return std::nullopt;
4514 if (
auto IVStep = GetConstantStep(ScaleReg)) {
4521 APInt Step = IVStep->second;
4523 if (
Offset.isSignedIntN(64)) {
4524 TestAddrMode.InBounds =
false;
4526 TestAddrMode.BaseOffs -=
Offset.getLimitedValue();
4531 getDTFn().
dominates(IVInc, MemoryInst)) {
4551 switch (
I->getOpcode()) {
4552 case Instruction::BitCast:
4553 case Instruction::AddrSpaceCast:
4555 if (
I->getType() ==
I->getOperand(0)->getType())
4557 return I->getType()->isIntOrPtrTy();
4558 case Instruction::PtrToInt:
4561 case Instruction::IntToPtr:
4564 case Instruction::Add:
4566 case Instruction::Or:
4568 case Instruction::Mul:
4569 case Instruction::Shl:
4572 case Instruction::GetElementPtr:
4588 int ISDOpcode =
TLI.InstructionOpcodeToISD(PromotedInst->
getOpcode());
4593 return TLI.isOperationLegalOrCustom(
4594 ISDOpcode,
TLI.getValueType(
DL, PromotedInst->
getType()));
4600class TypePromotionHelper {
4603 static void addPromotedInst(InstrToOrigTy &PromotedInsts,
4604 Instruction *ExtOpnd,
bool IsSExt) {
4605 ExtType ExtTy = IsSExt ? SignExtension : ZeroExtension;
4606 auto [It,
Inserted] = PromotedInsts.try_emplace(ExtOpnd);
4610 if (It->second.getInt() == ExtTy)
4616 ExtTy = BothExtension;
4618 It->second = TypeIsSExt(ExtOpnd->
getType(), ExtTy);
4625 static const Type *getOrigType(
const InstrToOrigTy &PromotedInsts,
4626 Instruction *Opnd,
bool IsSExt) {
4627 ExtType ExtTy = IsSExt ? SignExtension : ZeroExtension;
4628 InstrToOrigTy::const_iterator It = PromotedInsts.find(Opnd);
4629 if (It != PromotedInsts.end() && It->second.getInt() == ExtTy)
4630 return It->second.getPointer();
4645 static bool canGetThrough(
const Instruction *Inst,
Type *ConsideredExtType,
4646 const InstrToOrigTy &PromotedInsts,
bool IsSExt);
4650 static bool shouldExtOperand(
const Instruction *Inst,
int OpIdx) {
4663 static Value *promoteOperandForTruncAndAnyExt(
4664 Instruction *Ext, TypePromotionTransaction &TPT,
4665 InstrToOrigTy &PromotedInsts,
unsigned &CreatedInstsCost,
4666 SmallVectorImpl<Instruction *> *Exts,
4667 SmallVectorImpl<Instruction *> *Truncs,
const TargetLowering &
TLI);
4678 static Value *promoteOperandForOther(Instruction *Ext,
4679 TypePromotionTransaction &TPT,
4680 InstrToOrigTy &PromotedInsts,
4681 unsigned &CreatedInstsCost,
4682 SmallVectorImpl<Instruction *> *Exts,
4683 SmallVectorImpl<Instruction *> *Truncs,
4684 const TargetLowering &
TLI,
bool IsSExt);
4687 static Value *signExtendOperandForOther(
4688 Instruction *Ext, TypePromotionTransaction &TPT,
4689 InstrToOrigTy &PromotedInsts,
unsigned &CreatedInstsCost,
4690 SmallVectorImpl<Instruction *> *Exts,
4691 SmallVectorImpl<Instruction *> *Truncs,
const TargetLowering &
TLI) {
4692 return promoteOperandForOther(Ext, TPT, PromotedInsts, CreatedInstsCost,
4693 Exts, Truncs,
TLI,
true);
4697 static Value *zeroExtendOperandForOther(
4698 Instruction *Ext, TypePromotionTransaction &TPT,
4699 InstrToOrigTy &PromotedInsts,
unsigned &CreatedInstsCost,
4700 SmallVectorImpl<Instruction *> *Exts,
4701 SmallVectorImpl<Instruction *> *Truncs,
const TargetLowering &
TLI) {
4702 return promoteOperandForOther(Ext, TPT, PromotedInsts, CreatedInstsCost,
4703 Exts, Truncs,
TLI,
false);
4708 using Action =
Value *(*)(Instruction *Ext, TypePromotionTransaction &TPT,
4709 InstrToOrigTy &PromotedInsts,
4710 unsigned &CreatedInstsCost,
4711 SmallVectorImpl<Instruction *> *Exts,
4712 SmallVectorImpl<Instruction *> *Truncs,
4713 const TargetLowering &
TLI);
4724 static Action getAction(Instruction *Ext,
const SetOfInstrs &InsertedInsts,
4725 const TargetLowering &
TLI,
4726 const InstrToOrigTy &PromotedInsts);
4731bool TypePromotionHelper::canGetThrough(
const Instruction *Inst,
4732 Type *ConsideredExtType,
4733 const InstrToOrigTy &PromotedInsts,
4753 ((!IsSExt && BinOp->hasNoUnsignedWrap()) ||
4754 (IsSExt && BinOp->hasNoSignedWrap())))
4758 if ((Inst->
getOpcode() == Instruction::And ||
4763 if (Inst->
getOpcode() == Instruction::Xor) {
4766 if (!Cst->getValue().isAllOnes())
4775 if (Inst->
getOpcode() == Instruction::LShr && !IsSExt)
4785 if (ExtInst->hasOneUse()) {
4787 if (AndInst && AndInst->getOpcode() == Instruction::And) {
4820 const Type *OpndType = getOrigType(PromotedInsts, Opnd, IsSExt);
4833TypePromotionHelper::Action TypePromotionHelper::getAction(
4834 Instruction *Ext,
const SetOfInstrs &InsertedInsts,
4835 const TargetLowering &
TLI,
const InstrToOrigTy &PromotedInsts) {
4837 "Unexpected instruction type");
4844 if (!ExtOpnd || !canGetThrough(ExtOpnd, ExtTy, PromotedInsts, IsSExt))
4857 return promoteOperandForTruncAndAnyExt;
4863 return IsSExt ? signExtendOperandForOther : zeroExtendOperandForOther;
4866Value *TypePromotionHelper::promoteOperandForTruncAndAnyExt(
4867 Instruction *SExt, TypePromotionTransaction &TPT,
4868 InstrToOrigTy &PromotedInsts,
unsigned &CreatedInstsCost,
4869 SmallVectorImpl<Instruction *> *Exts,
4870 SmallVectorImpl<Instruction *> *Truncs,
const TargetLowering &
TLI) {
4874 Value *ExtVal = SExt;
4875 bool HasMergedNonFreeExt =
false;
4882 TPT.replaceAllUsesWith(SExt, ZExt);
4883 TPT.eraseInstruction(SExt);
4888 TPT.setOperand(SExt, 0, SExtOpnd->
getOperand(0));
4890 CreatedInstsCost = 0;
4894 TPT.eraseInstruction(SExtOpnd);
4902 CreatedInstsCost = !
TLI.
isExtFree(ExtInst) && !HasMergedNonFreeExt;
4910 TPT.eraseInstruction(ExtInst, NextVal);
4914Value *TypePromotionHelper::promoteOperandForOther(
4915 Instruction *Ext, TypePromotionTransaction &TPT,
4916 InstrToOrigTy &PromotedInsts,
unsigned &CreatedInstsCost,
4917 SmallVectorImpl<Instruction *> *Exts,
4918 SmallVectorImpl<Instruction *> *Truncs,
const TargetLowering &
TLI,
4923 CreatedInstsCost = 0;
4929 Value *Trunc = TPT.createTrunc(Ext, ExtOpnd->
getType());
4932 ITrunc->moveAfter(ExtOpnd);
4937 TPT.replaceAllUsesWith(ExtOpnd, Trunc);
4940 TPT.setOperand(Ext, 0, ExtOpnd);
4950 addPromotedInst(PromotedInsts, ExtOpnd, IsSExt);
4952 TPT.mutateType(ExtOpnd, Ext->
getType());
4954 TPT.replaceAllUsesWith(Ext, ExtOpnd);
4957 for (
int OpIdx = 0, EndOpIdx = ExtOpnd->
getNumOperands(); OpIdx != EndOpIdx;
4961 !shouldExtOperand(ExtOpnd, OpIdx)) {
4970 APInt CstVal = IsSExt ? Cst->getValue().sext(
BitWidth)
4972 TPT.setOperand(ExtOpnd, OpIdx, ConstantInt::get(Ext->
getType(), CstVal));
4983 Value *ValForExtOpnd = IsSExt
4984 ? TPT.createSExt(ExtOpnd, Opnd, Ext->
getType())
4985 : TPT.createZExt(ExtOpnd, Opnd, Ext->
getType());
4986 TPT.setOperand(ExtOpnd, OpIdx, ValForExtOpnd);
4988 if (!InstForExtOpnd)
4997 TPT.eraseInstruction(Ext);
5009bool AddressingModeMatcher::isPromotionProfitable(
5010 unsigned NewCost,
unsigned OldCost,
Value *PromotedOperand)
const {
5011 LLVM_DEBUG(
dbgs() <<
"OldCost: " << OldCost <<
"\tNewCost: " << NewCost
5016 if (NewCost > OldCost)
5018 if (NewCost < OldCost)
5037bool AddressingModeMatcher::matchOperationAddr(User *AddrInst,
unsigned Opcode,
5049 case Instruction::PtrToInt:
5052 case Instruction::IntToPtr: {
5060 case Instruction::BitCast:
5070 case Instruction::AddrSpaceCast: {
5078 case Instruction::Or:
5082 case Instruction::Add: {
5085 ExtAddrMode BackupAddrMode =
AddrMode;
5086 unsigned OldSize = AddrModeInsts.
size();
5091 TypePromotionTransaction::ConstRestorationPt LastKnownGood =
5092 TPT.getRestorationPoint();
5096 int First = 0, Second = 1;
5107 AddrModeInsts.
resize(OldSize);
5108 TPT.rollback(LastKnownGood);
5118 AddrModeInsts.
resize(OldSize);
5119 TPT.rollback(LastKnownGood);
5122 case Instruction::Mul:
5123 case Instruction::Shl: {
5127 if (!
RHS ||
RHS->getBitWidth() > 64)
5129 int64_t Scale = Opcode == Instruction::Shl
5130 ? 1LL <<
RHS->getLimitedValue(
RHS->getBitWidth() - 1)
5131 :
RHS->getSExtValue();
5135 case Instruction::GetElementPtr: {
5138 int VariableOperand = -1;
5139 unsigned VariableScale = 0;
5141 int64_t ConstantOffset = 0;
5143 for (
unsigned i = 1, e = AddrInst->
getNumOperands(); i != e; ++i, ++GTI) {
5145 const StructLayout *SL =
DL.getStructLayout(STy);
5156 if (ConstantInt *CI =
5158 const APInt &CVal = CI->
getValue();
5165 if (VariableOperand != -1)
5169 VariableOperand = i;
5170 VariableScale = TypeSize;
5177 if (VariableOperand == -1) {
5178 AddrMode.BaseOffs += ConstantOffset;
5184 AddrMode.BaseOffs -= ConstantOffset;
5188 ConstantOffset != 0) {
5202 BaseI ? BaseI->getParent() : &
GEP->getFunction()->getEntryBlock();
5204 LargeOffsetGEP = std::make_pair(
GEP, ConstantOffset);
5212 ExtAddrMode BackupAddrMode =
AddrMode;
5213 unsigned OldSize = AddrModeInsts.
size();
5216 AddrMode.BaseOffs += ConstantOffset;
5225 AddrModeInsts.
resize(OldSize);
5233 if (!matchScaledValue(AddrInst->
getOperand(VariableOperand), VariableScale,
5238 AddrModeInsts.
resize(OldSize);
5243 AddrMode.BaseOffs += ConstantOffset;
5244 if (!matchScaledValue(AddrInst->
getOperand(VariableOperand),
5245 VariableScale,
Depth)) {
5248 AddrModeInsts.
resize(OldSize);
5255 case Instruction::SExt:
5256 case Instruction::ZExt: {
5263 TypePromotionHelper::Action TPH =
5264 TypePromotionHelper::getAction(Ext, InsertedInsts,
TLI, PromotedInsts);
5268 TypePromotionTransaction::ConstRestorationPt LastKnownGood =
5269 TPT.getRestorationPoint();
5270 unsigned CreatedInstsCost = 0;
5272 Value *PromotedOperand =
5273 TPH(Ext, TPT, PromotedInsts, CreatedInstsCost,
nullptr,
nullptr,
TLI);
5288 assert(PromotedOperand &&
5289 "TypePromotionHelper should have filtered out those cases");
5291 ExtAddrMode BackupAddrMode =
AddrMode;
5292 unsigned OldSize = AddrModeInsts.
size();
5294 if (!matchAddr(PromotedOperand,
Depth) ||
5299 !isPromotionProfitable(CreatedInstsCost,
5300 ExtCost + (AddrModeInsts.
size() - OldSize),
5303 AddrModeInsts.
resize(OldSize);
5304 LLVM_DEBUG(
dbgs() <<
"Sign extension does not pay off: rollback\n");
5305 TPT.rollback(LastKnownGood);
5310 AddrMode.replaceWith(Ext, PromotedOperand);
5313 case Instruction::Call:
5315 if (
II->getIntrinsicID() == Intrinsic::threadlocal_address) {
5331bool AddressingModeMatcher::matchAddr(
Value *Addr,
unsigned Depth) {
5334 TypePromotionTransaction::ConstRestorationPt LastKnownGood =
5335 TPT.getRestorationPoint();
5359 ExtAddrMode BackupAddrMode =
AddrMode;
5360 unsigned OldSize = AddrModeInsts.
size();
5363 bool MovedAway =
false;
5364 if (matchOperationAddr(
I,
I->getOpcode(),
Depth, &MovedAway)) {
5372 if (
I->hasOneUse() ||
5373 isProfitableToFoldIntoAddressingMode(
I, BackupAddrMode,
AddrMode)) {
5380 AddrModeInsts.
resize(OldSize);
5381 TPT.rollback(LastKnownGood);
5384 if (matchOperationAddr(CE,
CE->getOpcode(),
Depth))
5386 TPT.rollback(LastKnownGood);
5413 TPT.rollback(LastKnownGood);
5424 TLI.ParseConstraints(
F->getDataLayout(), &
TRI, *CI);
5428 TLI.ComputeConstraintToUse(OpInfo,
SDValue());
5432 if (OpInfo.CallOperandVal == OpVal &&
5434 !OpInfo.isIndirect))
5450 if (!ConsideredInsts.
insert(
I).second)
5458 for (
Use &U :
I->uses()) {
5461 if (SeenInsts++ >= MaxUsersToScan)
5466 MemoryUses.push_back({&U, LI->getType()});
5473 MemoryUses.push_back({&U,
SI->getValueOperand()->getType()});
5480 MemoryUses.push_back({&U, RMW->getValOperand()->getType()});
5487 MemoryUses.push_back({&U, CmpX->getCompareOperand()->getType()});
5494 if (!
TLI.getAddrModeArguments(
II, PtrOps, AccessTy))
5497 if (!
find(PtrOps, U.get()))
5500 MemoryUses.push_back({&U, AccessTy});
5505 if (CI->hasFnAttr(Attribute::Cold)) {
5523 PSI, BFI, SeenInsts, MaxUsersToScan))
5534 unsigned SeenInsts = 0;
5537 PSI, BFI, SeenInsts, MaxUsersToScan);
5544bool AddressingModeMatcher::valueAlreadyLiveAtInst(
Value *Val,
5546 Value *KnownLive2) {
5548 if (Val ==
nullptr || Val == KnownLive1 || Val == KnownLive2)
5589bool AddressingModeMatcher::isProfitableToFoldIntoAddressingMode(
5590 Instruction *
I, ExtAddrMode &AMBefore, ExtAddrMode &AMAfter) {
5591 if (IgnoreProfitability)
5609 if (valueAlreadyLiveAtInst(ScaledReg, AMBefore.
BaseReg, AMBefore.
ScaledReg))
5610 ScaledReg =
nullptr;
5614 if (!BaseReg && !ScaledReg)
5623 Opts.cgp_max_address_users_to_scan))
5636 for (
const std::pair<Use *, Type *> &Pair : MemoryUses) {
5639 Type *AddressAccessTy = Pair.second;
5640 unsigned AS =
Address->getType()->getPointerAddressSpace();
5646 std::pair<AssertingVH<GetElementPtrInst>, int64_t> LargeOffsetGEP(
nullptr,
5648 TypePromotionTransaction::ConstRestorationPt LastKnownGood =
5649 TPT.getRestorationPoint();
5650 AddressingModeMatcher Matcher(MatchedAddrModeInsts,
TLI,
TRI, LI, getDTFn,
5651 AddressAccessTy, AS, UserI, Result,
5652 InsertedInsts, PromotedInsts, TPT,
5653 LargeOffsetGEP, OptSize, PSI, BFI, Opts);
5654 Matcher.IgnoreProfitability =
true;
5662 TPT.rollback(LastKnownGood);
5668 MatchedAddrModeInsts.
clear();
5678 return I->getParent() != BB;
5694 return std::next(AddrInst->getIterator());
5705 Earliest = UserInst;
5730bool CodeGenPrepare::optimizeMemoryInst(Instruction *MemoryInst,
Value *Addr,
5731 Type *AccessTy,
unsigned AddrSpace) {
5736 SmallVector<Value *, 8> worklist;
5737 SmallPtrSet<Value *, 16> Visited;
5743 bool PhiOrSelectSeen =
false;
5744 SmallVector<Instruction *, 16> AddrModeInsts;
5745 AddressingModeCombiner AddrModes(*
DL, Opts, Addr);
5746 TypePromotionTransaction TPT(RemovedInsts);
5747 TypePromotionTransaction::ConstRestorationPt LastKnownGood =
5748 TPT.getRestorationPoint();
5749 while (!worklist.
empty()) {
5761 if (!Visited.
insert(
V).second)
5767 PhiOrSelectSeen =
true;
5774 PhiOrSelectSeen =
true;
5781 AddrModeInsts.
clear();
5782 std::pair<AssertingVH<GetElementPtrInst>, int64_t> LargeOffsetGEP(
nullptr,
5787 auto getDTFn = [
this]() ->
const DominatorTree & {
return getDT(); };
5788 ExtAddrMode NewAddrMode = AddressingModeMatcher::Match(
5789 V, AccessTy, AddrSpace, MemoryInst, AddrModeInsts, *
TLI, *LI, getDTFn,
5790 *
TRI, InsertedInsts, PromotedInsts, TPT, LargeOffsetGEP, OptSize, PSI,
5793 GetElementPtrInst *
GEP = LargeOffsetGEP.first;
5798 LargeOffsetGEPMap[
GEP->getPointerOperand()].push_back(LargeOffsetGEP);
5799 LargeOffsetGEPID.
insert(std::make_pair(
GEP, LargeOffsetGEPID.
size()));
5802 NewAddrMode.OriginalValue =
V;
5803 if (!AddrModes.addNewAddrMode(NewAddrMode))
5810 if (!AddrModes.combineAddrModes()) {
5811 TPT.rollback(LastKnownGood);
5817 ExtAddrMode
AddrMode = AddrModes.getAddrMode();
5823 if (!PhiOrSelectSeen &&
none_of(AddrModeInsts, [&](
Value *
V) {
5837 WeakTrackingVH SunkAddrVH = SunkAddrs[Addr];
5860 <<
" for " << *MemoryInst <<
"\n");
5864 !
DL->isNonIntegralPointerType(Addr->
getType())) {
5870 SunkAddr = Builder.CreatePtrToInt(SunkAddr,
IntPtrTy,
"sunkaddr");
5872 Builder.CreateIntToPtr(SunkAddr, Addr->
getType(),
"sunkaddr");
5874 SunkAddr = Builder.CreatePointerCast(SunkAddr, Addr->
getType());
5878 <<
" for " << *MemoryInst <<
"\n");
5879 Value *ResultPtr =
nullptr, *ResultIndex =
nullptr;
5890 if (ResultPtr ||
AddrMode.Scale != 1)
5911 GlobalValue *BaseGV =
AddrMode.BaseGV;
5912 if (BaseGV !=
nullptr) {
5917 ResultPtr = Builder.CreateThreadLocalAddress(BaseGV);
5926 if (!
DL->isNonIntegralPointerType(Addr->
getType())) {
5927 if (!ResultPtr &&
AddrMode.BaseReg) {
5931 }
else if (!ResultPtr &&
AddrMode.Scale == 1) {
5932 ResultPtr = Builder.CreateIntToPtr(
AddrMode.ScaledReg, Addr->
getType(),
5941 }
else if (!ResultPtr) {
5954 if (
V->getType() != IdxTy)
5955 V = Builder.CreateIntCast(
V, IdxTy,
true,
"sunkaddr");
5963 if (
V->getType() == IdxTy) {
5968 "We can't transform if ScaledReg is too narrow");
5969 V = Builder.CreateTrunc(
V, IdxTy,
"sunkaddr");
5973 V = Builder.CreateMul(
5976 ResultIndex = Builder.CreateAdd(ResultIndex,
V,
"sunkaddr");
5987 if (ResultPtr->
getType() != I8PtrTy)
5988 ResultPtr = Builder.CreatePointerCast(ResultPtr, I8PtrTy);
5989 ResultPtr = Builder.CreatePtrAdd(ResultPtr, ResultIndex,
"sunkaddr",
6002 if (PtrInst && PtrInst->getParent() != MemoryInst->
getParent())
6004 SunkAddr = ResultPtr;
6006 if (ResultPtr->
getType() != I8PtrTy)
6007 ResultPtr = Builder.CreatePointerCast(ResultPtr, I8PtrTy);
6008 SunkAddr = Builder.CreatePtrAdd(ResultPtr, ResultIndex,
"sunkaddr",
6015 !
DL->isNonIntegralPointerType(Addr->
getType())) {
6021 SunkAddr = Builder.CreatePtrToInt(SunkAddr,
IntPtrTy,
"sunkaddr");
6023 Builder.CreateIntToPtr(SunkAddr, Addr->
getType(),
"sunkaddr");
6025 SunkAddr = Builder.CreatePointerCast(SunkAddr, Addr->
getType());
6033 SunkAddrs[Addr] = WeakTrackingVH(SunkAddr);
6038 resetIteratorIfInvalidatedWhileCalling(CurInstIterator->getParent(), [&]() {
6039 RecursivelyDeleteTriviallyDeadInstructions(
6040 Repl, TLInfo, nullptr,
6041 [&](Value *V) { removeAllAssertingVHReferences(V); });
6065bool CodeGenPrepare::optimizeGatherScatterInst(Instruction *MemoryInst,
6071 if (!
GEP->hasIndices())
6079 SmallVector<Value *, 2>
Ops(
GEP->operands());
6081 bool RewriteGEP =
false;
6090 unsigned FinalIndex =
Ops.size() - 1;
6095 for (
unsigned i = 1; i < FinalIndex; ++i) {
6100 C =
C->getSplatValue();
6102 if (!CI || !CI->
isZero())
6109 if (
Ops[FinalIndex]->
getType()->isVectorTy()) {
6113 if (!
C || !
C->isZero()) {
6114 Ops[FinalIndex] =
V;
6122 if (!RewriteGEP &&
Ops.size() == 2)
6128 Builder.disableGEPCanonicalization();
6130 Type *SourceTy =
GEP->getSourceElementType();
6131 Type *ScalarIndexTy =
DL->getIndexType(
Ops[0]->
getType()->getScalarType());
6135 if (!
Ops[FinalIndex]->
getType()->isVectorTy()) {
6136 NewAddr = Builder.CreateGEP(SourceTy,
Ops[0],
ArrayRef(
Ops).drop_front());
6137 auto *IndexTy = VectorType::get(ScalarIndexTy, NumElts);
6147 if (
Ops.size() != 2) {
6157 NewAddr = Builder.CreateGEP(SourceTy,
Base, Index);
6169 Builder.disableGEPCanonicalization();
6172 Type *ScalarIndexTy =
DL->getIndexType(
V->getType()->getScalarType());
6173 auto *IndexTy = VectorType::get(ScalarIndexTy, NumElts);
6176 Intrinsic::masked_gather) {
6180 Intrinsic::masked_scatter);
6195 Ptr, TLInfo,
nullptr,
6196 [&](
Value *
V) { removeAllAssertingVHReferences(
V); });
6207 if (
I->hasNUsesOrMore(3))
6210 for (
User *U :
I->users()) {
6212 if (!Extract || Extract->getNumIndices() != 1)
6215 unsigned Index = Extract->getIndices()[0];
6217 MulExtract = Extract;
6218 else if (Index == 1)
6219 OverflowExtract = Extract;
6246bool CodeGenPrepare::optimizeMulWithOverflow(Instruction *
I,
bool IsSigned,
6247 ModifyDT &ModifiedDT) {
6254 ExtractValueInst *MulExtract =
nullptr, *OverflowExtract =
nullptr;
6259 InsertedInsts.insert(
I);
6270 OverflowEntryBB->
takeName(
I->getParent());
6276 NoOverflowBB->
moveAfter(OverflowEntryBB);
6284 Value *LoLHS = Builder.CreateTrunc(
LHS, LegalTy,
"lo.lhs");
6285 Value *HiLHS = Builder.CreateLShr(
LHS, VTHalfBitWidth,
"lhs.lsr");
6286 HiLHS = Builder.CreateTrunc(HiLHS, LegalTy,
"hi.lhs");
6289 Value *LoRHS = Builder.CreateTrunc(
RHS, LegalTy,
"lo.rhs");
6290 Value *HiRHS = Builder.CreateLShr(
RHS, VTHalfBitWidth,
"rhs.lsr");
6291 HiRHS = Builder.CreateTrunc(HiRHS, LegalTy,
"hi.rhs");
6293 Value *IsAnyBitTrue;
6296 Builder.CreateAShr(LoLHS, VTHalfBitWidth - 1,
"sign.lo.lhs");
6298 Builder.CreateAShr(LoRHS, VTHalfBitWidth - 1,
"sign.lo.rhs");
6299 Value *XorLHS = Builder.CreateXor(HiLHS, SignLoLHS);
6300 Value *XorRHS = Builder.CreateXor(HiRHS, SignLoRHS);
6301 Value *
Or = Builder.CreateOr(XorLHS, XorRHS,
"or.lhs.rhs");
6302 IsAnyBitTrue = Builder.CreateCmp(ICmpInst::ICMP_NE,
Or,
6303 ConstantInt::getNullValue(
Or->getType()));
6305 Value *CmpLHS = Builder.CreateCmp(ICmpInst::ICMP_NE, HiLHS,
6306 ConstantInt::getNullValue(LegalTy));
6307 Value *CmpRHS = Builder.CreateCmp(ICmpInst::ICMP_NE, HiRHS,
6308 ConstantInt::getNullValue(LegalTy));
6309 IsAnyBitTrue = Builder.CreateOr(CmpLHS, CmpRHS,
"or.lhs.rhs");
6311 Builder.CreateCondBr(IsAnyBitTrue, OverflowBB, NoOverflowBB);
6314 Builder.SetInsertPoint(NoOverflowBB);
6315 Value *ExtLoLHS, *ExtLoRHS;
6317 ExtLoLHS = Builder.CreateSExt(LoLHS, Ty,
"lo.lhs.ext");
6318 ExtLoRHS = Builder.CreateSExt(LoRHS, Ty,
"lo.rhs.ext");
6320 ExtLoLHS = Builder.CreateZExt(LoLHS, Ty,
"lo.lhs.ext");
6321 ExtLoRHS = Builder.CreateZExt(LoRHS, Ty,
"lo.rhs.ext");
6324 Value *
Mul = Builder.CreateMul(ExtLoLHS, ExtLoRHS,
"mul.overflow.no");
6329 OverflowResBB->
setName(
"overflow.res");
6332 Builder.CreateBr(OverflowResBB);
6340 PHINode *OverflowResPHI = Builder.CreatePHI(Ty, 2),
6342 Builder.CreatePHI(IntegerType::getInt1Ty(
I->getContext()), 2);
6354 if (OverflowExtract) {
6355 OverflowExtract->replaceAllUsesWith(OverflowFlagPHI);
6356 OverflowExtract->eraseFromParent();
6361 I->removeFromParent();
6363 I->insertInto(OverflowBB, OverflowBB->
end());
6364 Builder.SetInsertPoint(OverflowBB->
end());
6366 Value *OverflowFlag = Builder.CreateExtractValue(
I, {1},
"overflow.flag");
6367 Builder.CreateBr(OverflowResBB);
6371 OverflowFlagPHI->addIncoming(OverflowFlag, OverflowBB);
6373 DTU->
applyUpdates({{DominatorTree::Insert, OverflowEntryBB, OverflowBB},
6374 {DominatorTree::Insert, OverflowEntryBB, NoOverflowBB},
6375 {DominatorTree::Insert, NoOverflowBB, OverflowResBB},
6376 {DominatorTree::Delete, OverflowEntryBB, OverflowResBB},
6377 {DominatorTree::Insert, OverflowBB, OverflowResBB}});
6379 ModifiedDT = ModifyDT::ModifyBBDT;
6385bool CodeGenPrepare::optimizeInlineAsmInst(CallInst *CS) {
6386 bool MadeChange =
false;
6388 const TargetRegisterInfo *
TRI =
6393 for (TargetLowering::AsmOperandInfo &OpInfo : TargetConstraints) {
6399 OpInfo.isIndirect) {
6401 MadeChange |= optimizeMemoryInst(CS, OpVal, OpVal->
getType(), ~0u);
6450 if (!
TLI.isZExtFree(NarrowTy, LargeTy))
6464bool CodeGenPrepare::tryToPromoteExts(
6465 TypePromotionTransaction &TPT,
const SmallVectorImpl<Instruction *> &Exts,
6466 SmallVectorImpl<Instruction *> &ProfitablyMovedExts,
6467 unsigned CreatedInstsCost) {
6468 bool Promoted =
false;
6471 for (
auto *
I : Exts) {
6486 TypePromotionHelper::Action TPH =
6487 TypePromotionHelper::getAction(
I, InsertedInsts, *
TLI, PromotedInsts);
6496 TypePromotionTransaction::ConstRestorationPt LastKnownGood =
6497 TPT.getRestorationPoint();
6498 SmallVector<Instruction *, 4> NewExts;
6499 unsigned NewCreatedInstsCost = 0;
6502 Value *PromotedVal = TPH(
I, TPT, PromotedInsts, NewCreatedInstsCost,
6503 &NewExts,
nullptr, *
TLI);
6505 "TypePromotionHelper should have filtered out those cases");
6515 long long TotalCreatedInstsCost = CreatedInstsCost + NewCreatedInstsCost;
6518 TotalCreatedInstsCost =
6519 std::max((
long long)0, (TotalCreatedInstsCost - ExtCost));
6520 if (!Opts.cgp_stress_ext_ld_promotion &&
6521 (TotalCreatedInstsCost > 1 ||
6523 (ExtCost == 0 && NewExts.
size() > 1))) {
6527 TPT.rollback(LastKnownGood);
6532 SmallVector<Instruction *, 2> NewlyMovedExts;
6533 (void)tryToPromoteExts(TPT, NewExts, NewlyMovedExts, TotalCreatedInstsCost);
6534 bool NewPromoted =
false;
6535 for (
auto *ExtInst : NewlyMovedExts) {
6541 !(Opts.cgp_stress_ext_ld_promotion ||
6542 NewCreatedInstsCost <= ExtCost ||
6546 ProfitablyMovedExts.
push_back(MovedExt);
6553 TPT.rollback(LastKnownGood);
6564bool CodeGenPrepare::mergeSExts(
Function &
F) {
6566 for (
auto &Entry : ValToSExtendedUses) {
6567 SExts &Insts =
Entry.second;
6569 for (Instruction *Inst : Insts) {
6573 bool inserted =
false;
6574 for (
auto &Pt : CurPts) {
6577 RemovedInsts.insert(Pt);
6578 Pt->removeFromParent();
6589 RemovedInsts.insert(Inst);
6596 CurPts.push_back(Inst);
6638bool CodeGenPrepare::splitLargeGEPOffsets() {
6640 for (
auto &Entry : LargeOffsetGEPMap) {
6642 SmallVectorImpl<std::pair<AssertingVH<GetElementPtrInst>, int64_t>>
6643 &LargeOffsetGEPs =
Entry.second;
6644 auto compareGEPOffset =
6645 [&](
const std::pair<GetElementPtrInst *, int64_t> &
LHS,
6646 const std::pair<GetElementPtrInst *, int64_t> &
RHS) {
6647 if (
LHS.first ==
RHS.first)
6649 if (
LHS.second !=
RHS.second)
6650 return LHS.second <
RHS.second;
6651 return LargeOffsetGEPID[
LHS.first] < LargeOffsetGEPID[
RHS.first];
6654 llvm::sort(LargeOffsetGEPs, compareGEPOffset);
6657 if (LargeOffsetGEPs.
front().second == LargeOffsetGEPs.
back().second)
6659 GetElementPtrInst *BaseGEP = LargeOffsetGEPs.
begin()->first;
6660 int64_t BaseOffset = LargeOffsetGEPs.
begin()->second;
6661 Value *NewBaseGEP =
nullptr;
6663 auto createNewBase = [&](int64_t BaseOffset,
Value *OldBase,
6664 GetElementPtrInst *
GEP) {
6665 LLVMContext &Ctx =
GEP->getContext();
6666 Type *PtrIdxTy =
DL->getIndexType(
GEP->getType());
6668 PointerType::get(Ctx,
GEP->getType()->getPointerAddressSpace());
6680 SplitEdge(NewBaseInsertBB, Invoke->getNormalDest(), &getDT(), LI);
6683 NewBaseInsertPt = std::next(BaseI->getIterator());
6696 NewBaseGEP = OldBase;
6697 if (NewBaseGEP->
getType() != I8PtrTy)
6698 NewBaseGEP = NewBaseBuilder.CreatePointerCast(NewBaseGEP, I8PtrTy);
6700 NewBaseBuilder.CreatePtrAdd(NewBaseGEP, BaseIndex,
"splitgep");
6701 NewGEPBases.
insert(NewBaseGEP);
6707 LargeOffsetGEPs.
front().second, LargeOffsetGEPs.
back().second)) {
6708 BaseOffset = PreferBase;
6711 createNewBase(BaseOffset, OldBase, BaseGEP);
6714 auto *LargeOffsetGEP = LargeOffsetGEPs.
begin();
6715 while (LargeOffsetGEP != LargeOffsetGEPs.
end()) {
6716 GetElementPtrInst *
GEP = LargeOffsetGEP->first;
6717 int64_t
Offset = LargeOffsetGEP->second;
6718 if (
Offset != BaseOffset) {
6725 GEP->getResultElementType(),
6726 GEP->getAddressSpace())) {
6732 NewBaseGEP =
nullptr;
6737 Type *PtrIdxTy =
DL->getIndexType(
GEP->getType());
6742 createNewBase(BaseOffset, OldBase,
GEP);
6746 Value *NewGEP = NewBaseGEP;
6747 if (
Offset != BaseOffset) {
6750 NewGEP = Builder.CreatePtrAdd(NewBaseGEP, Index);
6754 LargeOffsetGEP = LargeOffsetGEPs.
erase(LargeOffsetGEP);
6755 GEP->eraseFromParent();
6762bool CodeGenPrepare::optimizePhiType(
6763 PHINode *
I, SmallPtrSetImpl<PHINode *> &Visited,
6764 SmallPtrSetImpl<Instruction *> &DeletedInstrs) {
6769 Type *PhiTy =
I->getType();
6770 Type *ConvertTy =
nullptr;
6772 (!
I->getType()->isIntegerTy() && !
I->getType()->isFloatingPointTy()))
6775 SmallVector<Instruction *, 4> Worklist;
6777 SmallPtrSet<PHINode *, 4> PhiNodes;
6778 SmallPtrSet<ConstantData *, 4>
Constants;
6781 SmallPtrSet<Instruction *, 4> Defs;
6782 SmallPtrSet<Instruction *, 4>
Uses;
6788 bool AnyAnchored =
false;
6790 while (!Worklist.
empty()) {
6795 for (
Value *
V :
Phi->incoming_values()) {
6797 if (!PhiNodes.
count(OpPhi)) {
6798 if (!Visited.
insert(OpPhi).second)
6804 if (!OpLoad->isSimple())
6806 if (Defs.
insert(OpLoad).second)
6809 if (Defs.
insert(OpEx).second)
6813 ConvertTy = OpBC->getOperand(0)->getType();
6814 if (OpBC->getOperand(0)->getType() != ConvertTy)
6816 if (Defs.
insert(OpBC).second) {
6829 for (User *
V :
II->users()) {
6831 if (!PhiNodes.
count(OpPhi)) {
6832 if (Visited.
count(OpPhi))
6839 if (!OpStore->isSimple() || OpStore->getOperand(0) !=
II)
6841 Uses.insert(OpStore);
6844 ConvertTy = OpBC->getType();
6845 if (OpBC->getType() != ConvertTy)
6849 any_of(OpBC->users(), [](User *U) { return !isa<StoreInst>(U); });
6856 if (!ConvertTy || !AnyAnchored || PhiTy == ConvertTy ||
6860 LLVM_DEBUG(
dbgs() <<
"Converting " << *
I <<
"\n and connected nodes to "
6861 << *ConvertTy <<
"\n");
6866 for (ConstantData *
C : Constants)
6868 for (Instruction *
D : Defs) {
6870 ValMap[
D] =
D->getOperand(0);
6874 ValMap[
D] =
new BitCastInst(
D, ConvertTy,
D->getName() +
".bc", insertPt);
6877 for (PHINode *Phi : PhiNodes)
6879 Phi->getName() +
".tc",
Phi->getIterator());
6881 for (PHINode *Phi : PhiNodes) {
6883 for (
int i = 0, e =
Phi->getNumIncomingValues(); i < e; i++)
6885 Phi->getIncomingBlock(i));
6889 for (Instruction *U :
Uses) {
6894 U->setOperand(0,
new BitCastInst(ValMap[
U->getOperand(0)], PhiTy,
"bc",
6904bool CodeGenPrepare::optimizePhiTypes(
Function &
F) {
6905 if (!Opts.cgp_optimize_phi_types)
6909 SmallPtrSet<PHINode *, 4> Visited;
6910 SmallPtrSet<Instruction *, 4> DeletedInstrs;
6914 for (
auto &Phi : BB.
phis())
6915 Changed |= optimizePhiType(&Phi, Visited, DeletedInstrs);
6918 for (
auto *
I : DeletedInstrs) {
6920 I->eraseFromParent();
6928bool CodeGenPrepare::canFormExtLd(
6929 const SmallVectorImpl<Instruction *> &MovedExts, LoadInst *&LI,
6930 Instruction *&Inst,
bool HasPromoted) {
6931 for (
auto *MovedExtInst : MovedExts) {
6934 Inst = MovedExtInst;
6986bool CodeGenPrepare::optimizeExt(Instruction *&Inst) {
6987 bool AllowPromotionWithoutCommonHeader =
false;
6992 *Inst, AllowPromotionWithoutCommonHeader);
6993 TypePromotionTransaction TPT(RemovedInsts);
6994 TypePromotionTransaction::ConstRestorationPt LastKnownGood =
6995 TPT.getRestorationPoint();
6997 SmallVector<Instruction *, 2> SpeculativelyMovedExts;
7000 bool HasPromoted = tryToPromoteExts(TPT, Exts, SpeculativelyMovedExts);
7003 LoadInst *LI =
nullptr;
7008 if (canFormExtLd(SpeculativelyMovedExts, LI, ExtFedByLoad, HasPromoted)) {
7009 assert(LI && ExtFedByLoad &&
"Expect a valid load and extension");
7014 Inst = ExtFedByLoad;
7019 if (ATPConsiderable &&
7020 performAddressTypePromotion(Inst, AllowPromotionWithoutCommonHeader,
7021 HasPromoted, TPT, SpeculativelyMovedExts))
7024 TPT.rollback(LastKnownGood);
7033bool CodeGenPrepare::performAddressTypePromotion(
7034 Instruction *&Inst,
bool AllowPromotionWithoutCommonHeader,
7035 bool HasPromoted, TypePromotionTransaction &TPT,
7036 SmallVectorImpl<Instruction *> &SpeculativelyMovedExts) {
7037 bool Promoted =
false;
7038 SmallPtrSet<Instruction *, 1> UnhandledExts;
7039 bool AllSeenFirst =
true;
7040 for (
auto *
I : SpeculativelyMovedExts) {
7041 Value *HeadOfChain =
I->getOperand(0);
7042 auto AlreadySeen = SeenChainsForSExt.
find(HeadOfChain);
7045 if (AlreadySeen != SeenChainsForSExt.
end()) {
7046 if (AlreadySeen->second !=
nullptr)
7047 UnhandledExts.
insert(AlreadySeen->second);
7048 AllSeenFirst =
false;
7052 if (!AllSeenFirst || (AllowPromotionWithoutCommonHeader &&
7053 SpeculativelyMovedExts.size() == 1)) {
7057 for (
auto *
I : SpeculativelyMovedExts) {
7058 Value *HeadOfChain =
I->getOperand(0);
7059 SeenChainsForSExt[HeadOfChain] =
nullptr;
7060 ValToSExtendedUses[HeadOfChain].push_back(
I);
7063 Inst = SpeculativelyMovedExts.pop_back_val();
7068 for (
auto *
I : SpeculativelyMovedExts) {
7069 Value *HeadOfChain =
I->getOperand(0);
7070 SeenChainsForSExt[HeadOfChain] = Inst;
7075 if (!AllSeenFirst && !UnhandledExts.
empty())
7076 for (
auto *VisitedSExt : UnhandledExts) {
7077 if (RemovedInsts.count(VisitedSExt))
7079 TypePromotionTransaction TPT(RemovedInsts);
7081 SmallVector<Instruction *, 2> Chains;
7083 bool HasPromoted = tryToPromoteExts(TPT, Exts, Chains);
7087 for (
auto *
I : Chains) {
7088 Value *HeadOfChain =
I->getOperand(0);
7090 SeenChainsForSExt[HeadOfChain] =
nullptr;
7091 ValToSExtendedUses[HeadOfChain].push_back(
I);
7097bool CodeGenPrepare::optimizeExtUses(Instruction *
I) {
7102 Value *Src =
I->getOperand(0);
7103 if (Src->hasOneUse())
7115 bool DefIsLiveOut =
false;
7116 for (User *U :
I->users()) {
7121 if (UserBB == DefBB)
7123 DefIsLiveOut =
true;
7130 for (User *U : Src->users()) {
7133 if (UserBB == DefBB)
7142 DenseMap<BasicBlock *, Instruction *> InsertedTruncs;
7144 bool MadeChange =
false;
7150 if (UserBB == DefBB)
7154 Instruction *&InsertedTrunc = InsertedTruncs[UserBB];
7156 if (!InsertedTrunc) {
7159 InsertedTrunc =
new TruncInst(
I, Src->getType(),
"");
7161 InsertedInsts.insert(InsertedTrunc);
7174 I->dropPoisonGeneratingFlags();
7230bool CodeGenPrepare::optimizeLoadExt(LoadInst *
Load) {
7231 if (!
Load->isSimple() || !
Load->getType()->isIntOrPtrTy())
7235 if (
Load->hasOneUse() &&
7241 SmallVector<Instruction *, 8> WorkList;
7242 SmallPtrSet<Instruction *, 16> Visited;
7243 SmallVector<Instruction *, 8> AndsToMaybeRemove;
7244 SmallVector<Instruction *, 8> DropFlags;
7245 for (
auto *U :
Load->users())
7257 while (!WorkList.
empty()) {
7261 if (!Visited.
insert(
I).second)
7266 for (
auto *U :
Phi->users())
7271 switch (
I->getOpcode()) {
7272 case Instruction::And: {
7276 APInt AndBits = AndC->getValue();
7277 DemandBits |= AndBits;
7279 if (AndBits.
ugt(WidestAndBits))
7280 WidestAndBits = AndBits;
7281 if (AndBits == WidestAndBits &&
I->getOperand(0) ==
Load)
7286 case Instruction::Shl: {
7291 DemandBits.setLowBits(
BitWidth - ShiftAmt);
7296 case Instruction::Trunc: {
7299 DemandBits.setLowBits(TruncBitWidth);
7309 uint32_t ActiveBits = DemandBits.getActiveBits();
7321 if (ActiveBits <= 1 || !DemandBits.isMask(ActiveBits) ||
7322 WidestAndBits != DemandBits)
7325 LLVMContext &Ctx =
Load->getType()->getContext();
7326 Type *TruncTy = Type::getIntNTy(Ctx, ActiveBits);
7337 Builder.CreateAnd(
Load, ConstantInt::get(Ctx, DemandBits)));
7340 InsertedInsts.insert(NewAnd);
7345 NewAnd->setOperand(0,
Load);
7348 for (
auto *
And : AndsToMaybeRemove)
7353 if (&*CurInstIterator ==
And)
7354 CurInstIterator = std::next(
And->getIterator());
7355 And->eraseFromParent();
7360 for (
auto *Inst : DropFlags)
7374 TTI->isExpensiveToSpeculativelyExecute(
I);
7382 if (!
TLI->isPredictableSelectExpensive())
7392 uint64_t Max = std::max(TrueWeight, FalseWeight);
7393 uint64_t Sum = TrueWeight + FalseWeight;
7396 if (Probability >
TTI->getPredictableBranchThreshold())
7406 if (!Cmp || !Cmp->hasOneUse())
7429 assert(DefSI->getCondition() ==
SI->getCondition() &&
7430 "The condition of DefSI does not match with SI");
7431 V = (isTrue ? DefSI->getTrueValue() : DefSI->getFalseValue());
7434 assert(
V &&
"Failed to get select true/false value");
7438bool CodeGenPrepare::optimizeShiftInst(BinaryOperator *Shift) {
7462 BinaryOperator::BinaryOps Opcode = Shift->
getOpcode();
7463 Value *NewTVal = Builder.CreateBinOp(Opcode, Shift->
getOperand(0), TVal);
7464 Value *NewFVal = Builder.CreateBinOp(Opcode, Shift->
getOperand(0), FVal);
7465 Value *NewSel = Builder.CreateSelect(
Cond, NewTVal, NewFVal);
7471bool CodeGenPrepare::optimizeFunnelShift(IntrinsicInst *Fsh) {
7473 assert((Opcode == Intrinsic::fshl || Opcode == Intrinsic::fshr) &&
7474 "Expected a funnel shift");
7498 Value *NewTVal = Builder.CreateIntrinsic(Opcode, Ty, {
X,
Y, TVal});
7499 Value *NewFVal = Builder.CreateIntrinsic(Opcode, Ty, {
X,
Y, FVal});
7500 Value *NewSel = Builder.CreateSelect(
Cond, NewTVal, NewFVal);
7508bool CodeGenPrepare::optimizeSelectInst(SelectInst *SI) {
7509 if (!Opts.cgp_select2branch)
7513 if (!Opts.disable_select_optimize)
7520 It !=
SI->getParent()->
end(); ++It) {
7522 if (
I &&
SI->getCondition() ==
I->getCondition()) {
7529 SelectInst *LastSI = ASI.
back();
7532 CurInstIterator = std::next(LastSI->
getIterator());
7536 for (SelectInst *SI :
ArrayRef(ASI).drop_front())
7537 fixupDbgVariableRecordsOnInst(*SI);
7539 bool VectorCond = !
SI->getCondition()->getType()->isIntegerTy(1);
7542 if (VectorCond ||
SI->getMetadata(LLVMContext::MD_unpredictable))
7545 TargetLowering::SelectSupportKind SelectKind;
7546 if (
SI->getType()->isVectorTy())
7547 SelectKind = TargetLowering::ScalarCondVectorVal;
7549 SelectKind = TargetLowering::ScalarValSelect;
7586 SmallVector<Instruction *> TrueInstrs, FalseInstrs;
7587 for (SelectInst *SI : ASI) {
7599 SplitPt.setHeadBit(
true);
7602 auto *CondFr =
IB.CreateFreeze(
SI->getCondition(),
SI->getName() +
".frozen");
7607 UncondBrInst *TrueBranch =
nullptr;
7608 UncondBrInst *FalseBranch =
nullptr;
7609 if (TrueInstrs.
size() == 0) {
7614 }
else if (FalseInstrs.
size() == 0) {
7631 EndBlock->
setName(
"select.end");
7633 TrueBlock->
setName(
"select.true.sink");
7635 FalseBlock->
setName(FalseInstrs.
size() == 0 ?
"select.false"
7636 :
"select.false.sink");
7640 FreshBBs.
insert(TrueBlock);
7642 FreshBBs.
insert(FalseBlock);
7643 FreshBBs.
insert(EndBlock);
7648 static const unsigned MD[] = {
7649 LLVMContext::MD_prof, LLVMContext::MD_unpredictable,
7650 LLVMContext::MD_make_implicit, LLVMContext::MD_dbg};
7655 for (Instruction *
I : TrueInstrs)
7657 for (Instruction *
I : FalseInstrs)
7664 if (TrueBlock ==
nullptr)
7665 TrueBlock = StartBlock;
7666 else if (FalseBlock ==
nullptr)
7667 FalseBlock = StartBlock;
7683 SI->eraseFromParent();
7685 ++NumSelectsExpanded;
7689 CurInstIterator = StartBlock->
end();
7696bool CodeGenPrepare::optimizeShuffleVectorInst(ShuffleVectorInst *SVI) {
7708 "Expected a type of the same size!");
7714 Value *BC1 = Builder.CreateBitCast(
7716 Value *Shuffle = Builder.CreateVectorSplat(NewVecType->getNumElements(), BC1);
7717 Value *BC2 = Builder.CreateBitCast(Shuffle, SVIVecType);
7721 SVI, TLInfo,
nullptr,
7722 [&](
Value *
V) { removeAllAssertingVHReferences(
V); });
7729 !
Op->isTerminator() && !
Op->isEHPad())
7735bool CodeGenPrepare::tryToSinkFreeOperands(Instruction *
I) {
7750 for (Use *U :
reverse(OpsToSink)) {
7762 SetVector<Instruction *> MaybeDead;
7763 DenseMap<Instruction *, Instruction *> NewInstructions;
7764 for (Use *U : ToReplace) {
7773 FreshBBs.
insert(OpDef->getParent());
7776 NewInstructions[UI] = NI;
7781 InsertedInsts.insert(NI);
7787 if (
auto It = NewInstructions.
find(OldI); It != NewInstructions.
end())
7788 It->second->setOperand(
U->getOperandNo(), NI);
7795 for (
auto *
I : MaybeDead) {
7796 if (!
I->hasNUsesOrMore(1)) {
7798 I->eraseFromParent();
7805bool CodeGenPrepare::optimizeSwitchType(SwitchInst *SI) {
7811 unsigned RegWidth =
RegType.getSizeInBits();
7822 auto *NewType = Type::getIntNTy(
Context, RegWidth);
7831 ExtType = Instruction::SExt;
7834 if (Arg->hasSExtAttr())
7835 ExtType = Instruction::SExt;
7836 if (Arg->hasZExtAttr())
7837 ExtType = Instruction::ZExt;
7843 SI->setCondition(ExtInst);
7844 for (
auto Case :
SI->cases()) {
7845 const APInt &NarrowConst = Case.getCaseValue()->getValue();
7846 APInt WideConst = (ExtType == Instruction::ZExt)
7847 ? NarrowConst.
zext(RegWidth)
7848 : NarrowConst.
sext(RegWidth);
7849 Case.setValue(ConstantInt::get(
Context, WideConst));
7855bool CodeGenPrepare::optimizeSwitchPhiConstants(SwitchInst *SI) {
7862 Value *Condition =
SI->getCondition();
7871 for (
const SwitchInst::CaseHandle &Case :
SI->cases()) {
7872 ConstantInt *CaseValue = Case.getCaseValue();
7873 BasicBlock *CaseBB = Case.getCaseSuccessor();
7876 bool CheckedForSinglePred =
false;
7877 for (PHINode &
PHI : CaseBB->
phis()) {
7878 Type *PHIType =
PHI.getType();
7886 if (PHIType == ConditionType || TryZExt) {
7888 bool SkipCase =
false;
7889 Value *Replacement =
nullptr;
7890 for (
unsigned I = 0,
E =
PHI.getNumIncomingValues();
I !=
E;
I++) {
7891 Value *PHIValue =
PHI.getIncomingValue(
I);
7892 if (PHIValue != CaseValue) {
7901 if (
PHI.getIncomingBlock(
I) != SwitchBB)
7906 if (!CheckedForSinglePred) {
7907 CheckedForSinglePred =
true;
7908 if (
SI->findCaseDest(CaseBB) ==
nullptr) {
7914 if (Replacement ==
nullptr) {
7915 if (PHIValue == CaseValue) {
7916 Replacement = Condition;
7919 Replacement = Builder.CreateZExt(Condition, PHIType);
7922 PHI.setIncomingValue(
I, Replacement);
7933bool CodeGenPrepare::optimizeSwitchInst(SwitchInst *SI) {
7934 bool Changed = optimizeSwitchType(SI);
7935 Changed |= optimizeSwitchPhiConstants(SI);
7956class VectorPromoteHelper {
7958 const DataLayout &
DL;
7961 const TargetLowering &
TLI;
7964 const TargetTransformInfo &
TTI;
7970 SmallVector<Instruction *, 4> InstsToBePromoted;
7973 unsigned StoreExtractCombineCost;
7975 bool StressStoreExtract;
7984 if (InstsToBePromoted.
empty())
7986 return InstsToBePromoted.
back();
7992 unsigned getTransitionOriginalValueIdx()
const {
7994 "Other kind of transitions are not supported yet");
8001 unsigned getTransitionIdx()
const {
8003 "Other kind of transitions are not supported yet");
8011 Type *getTransitionType()
const {
8022 void promoteImpl(Instruction *ToBePromoted);
8026 bool isProfitableToPromote() {
8027 Value *ValIdx = Transition->
getOperand(getTransitionOriginalValueIdx());
8031 Type *PromotedType = getTransitionType();
8034 unsigned AS =
ST->getPointerAddressSpace();
8052 for (
const auto &Inst : InstsToBePromoted) {
8060 TargetTransformInfo::OperandValueInfo Arg0Info, Arg1Info;
8069 CostKind, Arg0Info, Arg1Info);
8072 dbgs() <<
"Estimated cost of computation to be promoted:\nScalar: "
8073 << ScalarCost <<
"\nVector: " << VectorCost <<
'\n');
8074 return ScalarCost > VectorCost;
8086 unsigned ExtractIdx = std::numeric_limits<unsigned>::max();
8101 if (!
EC.isScalable()) {
8102 SmallVector<Constant *, 4> ConstVec;
8104 for (
unsigned Idx = 0;
Idx !=
EC.getKnownMinValue(); ++
Idx) {
8105 if (Idx == ExtractIdx)
8113 "Generate scalable vector for non-splat is unimplemented");
8118 static bool canCauseUndefinedBehavior(
const Instruction *Use,
8119 unsigned OperandIdx) {
8122 if (OperandIdx != 1)
8124 switch (
Use->getOpcode()) {
8127 case Instruction::SDiv:
8128 case Instruction::UDiv:
8129 case Instruction::SRem:
8130 case Instruction::URem:
8132 case Instruction::FDiv:
8133 case Instruction::FRem:
8134 return !
Use->hasNoNaNs();
8140 VectorPromoteHelper(
const DataLayout &
DL,
const TargetLowering &
TLI,
8141 const TargetTransformInfo &
TTI, Instruction *Transition,
8142 unsigned CombineCost,
bool StressStoreExtract)
8144 StoreExtractCombineCost(CombineCost),
8145 StressStoreExtract(StressStoreExtract) {
8146 assert(Transition &&
"Do not know how to promote null");
8150 bool canPromote(
const Instruction *ToBePromoted)
const {
8157 bool shouldPromote(
const Instruction *ToBePromoted)
const {
8162 for (
const Use &U : ToBePromoted->
operands()) {
8163 const Value *Val =
U.get();
8164 if (Val == getEndOfTransition()) {
8175 return StressStoreExtract ||
8186 void enqueueForPromotion(Instruction *ToBePromoted) {
8187 InstsToBePromoted.push_back(ToBePromoted);
8191 void recordCombineInstruction(Instruction *ToBeCombined) {
8193 CombineInst = ToBeCombined;
8203 if (InstsToBePromoted.empty() || !CombineInst)
8207 if (!StressStoreExtract && !isProfitableToPromote())
8211 for (
auto &ToBePromoted : InstsToBePromoted)
8212 promoteImpl(ToBePromoted);
8213 InstsToBePromoted.clear();
8220void VectorPromoteHelper::promoteImpl(Instruction *ToBePromoted) {
8230 "The type of the result of the transition does not match "
8235 Type *TransitionTy = getTransitionType();
8240 for (Use &U : ToBePromoted->
operands()) {
8242 Value *NewVal =
nullptr;
8243 if (Val == Transition)
8244 NewVal = Transition->
getOperand(getTransitionOriginalValueIdx());
8251 canCauseUndefinedBehavior(ToBePromoted,
U.getOperandNo()));
8255 ToBePromoted->
setOperand(
U.getOperandNo(), NewVal);
8258 Transition->
setOperand(getTransitionOriginalValueIdx(), ToBePromoted);
8264bool CodeGenPrepare::optimizeExtractElementInst(Instruction *Inst) {
8265 unsigned CombineCost = std::numeric_limits<unsigned>::max();
8266 if (!Opts.cgp_store_extract ||
8267 (!Opts.cgp_stress_store_extract &&
8280 LLVM_DEBUG(
dbgs() <<
"Found an interesting transition: " << *Inst <<
'\n');
8281 VectorPromoteHelper VPH(*
DL, *
TLI, *
TTI, Inst, CombineCost,
8282 Opts.cgp_stress_store_extract);
8289 if (ToBePromoted->
getParent() != Parent) {
8290 LLVM_DEBUG(
dbgs() <<
"Instruction to promote is in a different block ("
8292 <<
") than the transition (" << Parent->
getName()
8297 if (VPH.canCombine(ToBePromoted)) {
8299 <<
"will be combined with: " << *ToBePromoted <<
'\n');
8300 VPH.recordCombineInstruction(ToBePromoted);
8302 NumStoreExtractExposed +=
Changed;
8307 if (!VPH.canPromote(ToBePromoted) || !VPH.shouldPromote(ToBePromoted))
8310 LLVM_DEBUG(
dbgs() <<
"Promoting is possible... Enqueue for promotion!\n");
8312 VPH.enqueueForPromotion(ToBePromoted);
8313 Inst = ToBePromoted;
8352 bool ForceSplitStore) {
8354 Type *StoreType =
SI.getValueOperand()->getType();
8363 if (!
DL.typeSizeEqualsStoreSize(StoreType) ||
8364 DL.getTypeSizeInBits(StoreType) == 0)
8367 unsigned HalfValBitSize =
DL.getTypeSizeInBits(StoreType) / 2;
8369 if (!
DL.typeSizeEqualsStoreSize(SplitStoreType))
8385 if (!
match(
SI.getValueOperand(),
8392 if (!
LValue->getType()->isIntegerTy() ||
8393 DL.getTypeSizeInBits(
LValue->getType()) > HalfValBitSize ||
8395 DL.getTypeSizeInBits(HValue->
getType()) > HalfValBitSize)
8406 if (!ForceSplitStore && !
TLI.isMultiStoresCheaperThanBitsMerge(LowTy, HighTy))
8411 Builder.disableGEPCanonicalization();
8415 if (LBC && LBC->getParent() !=
SI.getParent())
8416 LValue = Builder.CreateBitCast(LBC->getOperand(0), LBC->getType());
8417 if (HBC && HBC->getParent() !=
SI.getParent())
8418 HValue = Builder.CreateBitCast(HBC->getOperand(0), HBC->getType());
8420 bool IsLE =
SI.getDataLayout().isLittleEndian();
8421 auto CreateSplitStore = [&](
Value *
V,
bool Upper) {
8422 V = Builder.CreateZExtOrBitCast(
V, SplitStoreType);
8423 Value *Addr =
SI.getPointerOperand();
8424 Align Alignment =
SI.getAlign();
8425 const bool IsOffsetStore = (IsLE &&
Upper) || (!IsLE && !
Upper);
8426 if (IsOffsetStore) {
8427 Addr = Builder.CreateGEP(
8428 SplitStoreType, Addr,
8436 Builder.CreateAlignedStore(
V, Addr, Alignment);
8439 CreateSplitStore(
LValue,
false);
8440 CreateSplitStore(HValue,
true);
8443 SI.eraseFromParent();
8528 if (auto *I = dyn_cast<Instruction>(Usr)) {
8529 if (I->getParent() != SrcBlock) {
8537 std::vector<GetElementPtrInst *> UGEPIs;
8540 for (User *Usr : GEPIOp->
users()) {
8552 ConstantInt *UGEPIIdx;
8555 if (GEPIIdx->getType() != UGEPIIdx->
getType())
8563 if (UGEPIs.size() == 0)
8566 for (GetElementPtrInst *UGEPI : UGEPIs) {
8568 APInt NewIdx = UGEPIIdx->
getValue() - GEPIIdx->getValue();
8575 for (GetElementPtrInst *UGEPI : UGEPIs) {
8576 UGEPI->setOperand(0, GEPI);
8578 auto NewIdx = UGEPIIdx->
getValue() - GEPIIdx->getValue();
8579 Constant *NewUGEPIIdx = ConstantInt::get(GEPIIdx->getType(), NewIdx);
8580 UGEPI->setOperand(1, NewUGEPIIdx);
8582 auto SourceFlags = GEPI->getNoWrapFlags();
8585 UGEPI->getNoWrapFlags().intersectForOffsetAdd(SourceFlags);
8587 if (NewIdx.
isNegative() && TargetFlags.hasNoUnsignedWrap())
8588 TargetFlags = TargetFlags.withoutNoUnsignedWrap();
8589 UGEPI->setNoWrapFlags(TargetFlags);
8595 return cast<Instruction>(Usr)->getParent() != SrcBlock;
8597 "GEPIOp is used outside SrcBlock");
8614 if (!
TLI.preferZeroCompareBranch())
8621 Value *
X = Cmp->getOperand(0);
8622 if (!
X->hasUseList())
8627 for (
auto *U :
X->users()) {
8631 (UI->
getParent() != Branch->getParent() &&
8632 UI->
getParent() != Branch->getSuccessor(0) &&
8633 UI->
getParent() != Branch->getSuccessor(1)) ||
8634 (UI->
getParent() != Branch->getParent() &&
8635 !UI->
getParent()->getSinglePredecessor()))
8641 if (UI->
getParent() != Branch->getParent())
8645 ConstantInt::get(UI->
getType(), 0));
8647 LLVM_DEBUG(
dbgs() <<
" to compare on zero: " << *NewCmp <<
"\n");
8651 if (Cmp->isEquality() &&
8656 if (UI->
getParent() != Branch->getParent())
8659 Value *NewCmp = Builder.CreateCmp(Cmp->getPredicate(), UI,
8660 ConstantInt::get(UI->
getType(), 0));
8662 LLVM_DEBUG(
dbgs() <<
" to compare on zero: " << *NewCmp <<
"\n");
8702 if (!L || !L->isInnermost())
8706 BasicBlock *PreHeader = L->getLoopPreheader();
8708 if (!PreHeader || !Latch || Phi->getParent() != Header ||
8709 Phi->getNumIncomingValues() != 2)
8713 Value *Start = Phi->getIncomingValueForBlock(PreHeader);
8714 Value *Step = Phi->getIncomingValueForBlock(Latch);
8721 Type *ITy = VTy->getElementType();
8722 if (
DL.getTypeStoreSize(ITy) !=
DL.getPointerSize())
8729 Value *LoopStride =
nullptr;
8735 const APInt *ShiftAmt =
nullptr;
8736 Value *PtrStride =
nullptr;
8740 (1ULL << ShiftAmt->
getZExtValue()) != EltCnt.getKnownMinValue())
8752 for (
User *U : Phi->users()) {
8764 if (!
GEP ||
GEP->getNumOperands() != 2)
8769 if (!L->isLoopInvariant(
Base))
8778 unsigned Size =
DL.getTypeAllocSize(
GEP->getSourceElementType());
8780 Value *StructSize = ConstantInt::get(ITy,
Size);
8782 Builder.CreateVectorSplat(VTy->getElementCount(), StructSize);
8783 Value *NewStart = Builder.CreateMul(Start, SizeSplat);
8784 NewStart = Builder.CreatePtrAdd(
Base, NewStart);
8788 Value *StructStride = Builder.CreateMul(PtrStride, StructSize);
8790 Builder.CreateVectorSplat(VTy->getElementCount(), StructStride);
8792 Builder.SetInsertPoint(Phi);
8793 PHINode *NewPhi = Builder.CreatePHI(
GEP->getType(), 2,
"vec.lsr.phi");
8795 StructStride = Builder.CreatePtrAdd(NewPhi, StructStride);
8802 Builder.SetInsertPoint(
GEP);
8803 GEP->replaceAllUsesWith(NewPhi);
8812bool CodeGenPrepare::optimizeInst(Instruction *
I, ModifyDT &ModifiedDT) {
8813 bool AnyChange =
false;
8814 AnyChange = fixupDbgVariableRecordsOnInst(*
I);
8818 if (InsertedInsts.count(
I))
8827 LargeOffsetGEPMap.erase(
P);
8829 P->eraseFromParent();
8837 if (
P->getType()->isScalableTy())
8866 I, LI->getLoopFor(
I->getParent()), *
TTI))
8874 TargetLowering::TypeExpandInteger) {
8878 I, LI->getLoopFor(
I->getParent()), *
TTI))
8881 bool MadeChange = optimizeExt(
I);
8882 return MadeChange | optimizeExtUses(
I);
8889 if (optimizeCmp(Cmp, ModifiedDT))
8893 if (optimizeURem(
I))
8897 LI->
setMetadata(LLVMContext::MD_invariant_group,
nullptr);
8898 bool Modified = optimizeLoadExt(LI);
8907 bool Modified = tryToSinkFreeOperands(
I);
8908 SI->setMetadata(LLVMContext::MD_invariant_group,
nullptr);
8909 unsigned AS =
SI->getPointerAddressSpace();
8910 Modified |= optimizeMemoryInst(
I,
SI->getOperand(1),
8911 SI->getOperand(0)->getType(), AS);
8916 unsigned AS = RMW->getPointerAddressSpace();
8917 return optimizeMemoryInst(
I, RMW->getPointerOperand(), RMW->getType(), AS);
8921 unsigned AS = CmpX->getPointerAddressSpace();
8922 return optimizeMemoryInst(
I, CmpX->getPointerOperand(),
8923 CmpX->getCompareOperand()->getType(), AS);
8928 if (BinOp && BinOp->
getOpcode() == Instruction::And &&
8929 Opts.cgp_andcmp_sinking &&
8934 if (BinOp && (BinOp->
getOpcode() == Instruction::AShr ||
8935 BinOp->
getOpcode() == Instruction::LShr)) {
8943 if (GEPI->hasAllZeroIndices()) {
8945 Instruction *
NC =
new BitCastInst(GEPI->getOperand(0), GEPI->getType(),
8946 GEPI->getName(), GEPI->getIterator());
8947 NC->setDebugLoc(GEPI->getDebugLoc());
8950 GEPI, TLInfo,
nullptr,
8951 [&](
Value *
V) { removeAllAssertingVHReferences(
V); });
8953 optimizeInst(
NC, ModifiedDT);
8971 if (Const0 || Const1) {
8972 if (!Const0 || !Const1) {
8973 auto *
F =
new FreezeInst(Const0 ? Op1 : Op0,
"", CmpI->
getIterator());
8979 FI->eraseFromParent();
8987 if (
I->getType()->isVectorTy() &&
8991 if (tryToSinkFreeOperands(
I))
8994 switch (
I->getOpcode()) {
8995 case Instruction::Shl:
8996 case Instruction::LShr:
8997 case Instruction::AShr:
8999 case Instruction::Call:
9001 case Instruction::Select:
9003 case Instruction::ShuffleVector:
9005 case Instruction::Switch:
9007 case Instruction::ExtractElement:
9009 case Instruction::CondBr:
9018bool CodeGenPrepare::makeBitReverse(Instruction &
I) {
9019 if (!
I.getType()->isIntegerTy() ||
9024 SmallVector<Instruction *, 4> Insts;
9030 &
I, TLInfo,
nullptr,
9031 [&](
Value *
V) { removeAllAssertingVHReferences(
V); });
9038bool CodeGenPrepare::optimizeBlock(BasicBlock &BB, ModifyDT &ModifiedDT) {
9040 bool MadeChange =
false;
9043 CurInstIterator = BB.
begin();
9044 ModifiedDT = ModifyDT::NotModifyDT;
9045 while (CurInstIterator != BB.
end()) {
9046 MadeChange |= optimizeInst(&*CurInstIterator++, ModifiedDT);
9047 if (ModifiedDT != ModifyDT::NotModifyDT) {
9056 }
while (ModifiedDT == ModifyDT::ModifyInstDT);
9058 bool MadeBitReverse =
true;
9059 while (MadeBitReverse) {
9060 MadeBitReverse =
false;
9062 if (makeBitReverse(
I)) {
9063 MadeBitReverse = MadeChange =
true;
9068 MadeChange |= dupRetToEnableTailCallOpts(&BB, ModifiedDT);
9073bool CodeGenPrepare::fixupDbgVariableRecordsOnInst(Instruction &
I) {
9074 bool AnyChange =
false;
9075 for (DbgVariableRecord &DVR :
filterDbgVars(
I.getDbgRecordRange()))
9076 AnyChange |= fixupDbgVariableRecord(DVR);
9082bool CodeGenPrepare::fixupDbgVariableRecord(DbgVariableRecord &DVR) {
9083 if (DVR.
Type != DbgVariableRecord::LocationType::Value &&
9084 DVR.
Type != DbgVariableRecord::LocationType::Assign)
9088 bool AnyChange =
false;
9089 SmallDenseSet<Value *> LocationOps(DVR.
location_ops().begin(),
9091 for (
Value *Location : LocationOps) {
9092 WeakTrackingVH SunkAddrVH = SunkAddrs[
Location];
9121bool CodeGenPrepare::placeDbgValues(
Function &
F) {
9122 bool MadeChange =
false;
9123 DominatorTree &DT = getDT();
9125 auto DbgProcessor = [&](
auto *DbgItem,
Instruction *Position) {
9126 SmallVector<Instruction *, 4> VIs;
9127 for (
Value *
V : DbgItem->location_ops())
9135 for (Instruction *VI : VIs) {
9136 if (
VI->isTerminator())
9141 if (
isa<PHINode>(VI) &&
VI->getParent()->getTerminator()->isEHPad())
9152 if (VIs.size() > 1) {
9155 <<
"Unable to find valid location for Debug Value, undefing:\n"
9157 DbgItem->setKillLocation();
9162 << *DbgItem <<
' ' << *VI);
9169 for (BasicBlock &BB :
F) {
9175 if (DVR.
Type != DbgVariableRecord::LocationType::Value)
9177 DbgProcessor(&DVR, &Insn);
9188bool CodeGenPrepare::placePseudoProbes(
Function &
F) {
9189 bool MadeChange =
false;
9192 auto FirstInst =
Block.getFirstInsertionPt();
9197 while (
I !=
Block.end()) {
9199 II->moveBefore(FirstInst);
9229bool CodeGenPrepare::splitBranchCondition(
Function &
F) {
9233 bool MadeChange =
false;
9234 for (
auto &BB :
F) {
9247 if (Br1->getMetadata(LLVMContext::MD_unpredictable))
9255 Value *Cond1, *Cond2;
9258 Opc = Instruction::And;
9261 Opc = Instruction::Or;
9271 if (!IsGoodCond(Cond1) || !IsGoodCond(Cond2))
9285 Br1->setCondition(Cond1);
9290 if (
Opc == Instruction::And)
9291 Br1->setSuccessor(0, TmpBB);
9293 Br1->setSuccessor(1, TmpBB);
9298 I->removeFromParent();
9299 I->insertBefore(Br2->getIterator());
9311 if (
Opc == Instruction::Or)
9315 TBB->replacePhiUsesWith(&BB, TmpBB);
9318 for (PHINode &PN : FBB->
phis()) {
9323 if (
Loop *L = LI->getLoopFor(&BB))
9324 L->addBasicBlockToLoop(TmpBB, *LI);
9328 DTU->
applyUpdates({{DominatorTree::Insert, &BB, TmpBB},
9329 {DominatorTree::Insert, TmpBB,
TBB},
9330 {DominatorTree::Insert, TmpBB, FBB},
9331 {DominatorTree::Delete, &BB,
TBB}});
9335 if (
Opc == Instruction::Or) {
9357 uint64_t NewTrueWeight = TrueWeight;
9358 uint64_t NewFalseWeight = TrueWeight + 2 * FalseWeight;
9362 NewTrueWeight = TrueWeight;
9363 NewFalseWeight = 2 * FalseWeight;
9388 uint64_t NewTrueWeight = 2 * TrueWeight + FalseWeight;
9389 uint64_t NewFalseWeight = FalseWeight;
9393 NewTrueWeight = 2 * TrueWeight;
9394 NewFalseWeight = FalseWeight;
static unsigned getIntrinsicID(const SDNode *N)
const TargetLowering & TLI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPU Register Bank Select
This file implements a class to represent arbitrary precision integral constant values and operations...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static void print(raw_ostream &Out, object::Archive::Kind Kind, T Val)
This file contains the simple types necessary to represent the attributes associated with functions a...
static const Function * getParent(const Value *V)
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static bool sinkAndCmp0Expression(Instruction *AndI, const TargetLowering &TLI, SetOfInstrs &InsertedInsts)
Duplicate and sink the given 'and' instruction into user blocks where it is used in a compare to allo...
static bool SinkShiftAndTruncate(BinaryOperator *ShiftI, Instruction *User, ConstantInt *CI, DenseMap< BasicBlock *, BinaryOperator * > &InsertedShifts, const TargetLowering &TLI, const DataLayout &DL)
Sink both shift and truncate instruction to the use of truncate's BB.
static bool getGEPSmallConstantIntOffsetV(GetElementPtrInst *GEP, SmallVectorImpl< Value * > &OffsetV)
static bool sinkSelectOperand(const TargetTransformInfo *TTI, Value *V)
Check if V (an operand of a select instruction) is an expensive instruction that is only used once.
static bool isExtractBitsCandidateUse(Instruction *User)
Check if the candidates could be combined with a shift instruction, which includes:
static bool optimizeBitCast(BitCastInst *BCI, const TargetLowering &TLI, const DataLayout &DL)
Hoists bitcasts to the source block to reduce register pressure.
static bool foldFCmpToFPClassTest(CmpInst *Cmp, const TargetLowering &TLI, const DataLayout &DL)
static bool OptimizeExtractBits(BinaryOperator *ShiftI, ConstantInt *CI, const TargetLowering &TLI, const DataLayout &DL)
Sink the shift right instruction into user blocks if the uses could potentially be combined with this...
static bool OptimizeNoopCopyExpression(CastInst *CI, const TargetLowering &TLI, const DataLayout &DL)
If the specified cast instruction is a noop copy (e.g.
static bool SinkCast(CastInst *CI)
Sink the specified cast instruction into its user blocks.
static bool splitMergedValStore(StoreInst &SI, const DataLayout &DL, const TargetLowering &TLI, bool ForceSplitStore)
For the instruction sequence of store below, F and I values are bundled together as an i64 value befo...
static bool swapICmpOperandsToExposeCSEOpportunities(CmpInst *Cmp)
Many architectures use the same instruction for both subtract and cmp.
static bool isFormingBranchFromSelectProfitable(const TargetTransformInfo *TTI, const TargetLowering *TLI, SelectInst *SI)
Returns true if a SelectInst should be turned into an explicit branch.
static bool FindAllMemoryUses(Instruction *I, SmallVectorImpl< std::pair< Use *, Type * > > &MemoryUses, SmallPtrSetImpl< Instruction * > &ConsideredInsts, const TargetLowering &TLI, const TargetRegisterInfo &TRI, bool OptSize, ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI, unsigned &SeenInsts, unsigned MaxUsersToScan)
Recursively walk all the uses of I until we find a memory use.
static std::optional< std::pair< Instruction *, Constant * > > getIVIncrement(const PHINode *PN, const LoopInfo *LI)
If given PN is an inductive variable with value IVInc coming from the backedge, and on each iteration...
static Value * getTrueOrFalseValue(SelectInst *SI, bool isTrue, const SmallPtrSet< const Instruction *, 2 > &Selects)
If isTrue is true, return the true value of SI, otherwise return false value of SI.
static bool foldICmpWithDominatingICmp(CmpInst *Cmp, const TargetLowering &TLI, bool EnableICmpEqToICmpSt)
For pattern like:
static bool strengthReduceVectorPhiUsers(PHINode *Phi, LoopInfo *LI)
static bool optimizeBranch(CondBrInst *Branch, const TargetLowering &TLI, SmallPtrSet< BasicBlock *, 32 > &FreshBBs, bool IsHugeFunc)
static bool tryUnmergingGEPsAcrossIndirectBr(GetElementPtrInst *GEPI, const TargetTransformInfo *TTI)
static bool IsOperandAMemoryOperand(CallInst *CI, InlineAsm *IA, Value *OpVal, const TargetLowering &TLI, const TargetRegisterInfo &TRI)
Check to see if all uses of OpVal by the specified inline asm call are due to memory operands.
static bool isIntrinsicOrLFToBeTailCalled(const TargetLibraryInfo *TLInfo, const CallInst *CI)
static void replaceAllUsesWith(Value *Old, Value *New, SmallPtrSet< BasicBlock *, 32 > &FreshBBs, bool IsHuge)
Replace all old uses with new ones, and push the updated BBs into FreshBBs.
static bool matchOverflowPattern(Instruction *&I, ExtractValueInst *&MulExtract, ExtractValueInst *&OverflowExtract)
static void computeBaseDerivedRelocateMap(const SmallVectorImpl< GCRelocateInst * > &AllRelocateCalls, MapVector< GCRelocateInst *, SmallVector< GCRelocateInst *, 0 > > &RelocateInstMap)
static bool simplifyRelocatesOffABase(GCRelocateInst *RelocatedBase, const SmallVectorImpl< GCRelocateInst * > &Targets)
static bool MightBeFoldableInst(Instruction *I)
This is a little filter, which returns true if an addressing computation involving I might be folded ...
static bool matchIncrement(const Instruction *IVInc, Instruction *&LHS, Constant *&Step)
static bool isIVIncrement(const Value *V, const LoopInfo *LI)
static void DbgInserterHelper(DbgVariableRecord *DVR, BasicBlock::iterator VI)
static bool isPromotedInstructionLegal(const TargetLowering &TLI, const DataLayout &DL, Value *Val)
Check whether or not Val is a legal instruction for TLI.
static BasicBlock::iterator findInsertPos(Value *Addr, Instruction *MemoryInst, Value *SunkAddr)
static bool isRemOfLoopIncrementWithLoopInvariant(Instruction *Rem, const LoopInfo *LI, Value *&RemAmtOut, Value *&AddInstOut, Value *&AddOffsetOut, PHINode *&LoopIncrPNOut, Constant *&StepOut)
static bool IsNonLocalValue(Value *V, BasicBlock *BB)
Return true if the specified values are defined in a different basic block than BB.
static bool despeculateCountZeros(IntrinsicInst *CountZeros, DomTreeUpdater *DTU, LoopInfo *LI, const TargetLowering *TLI, const DataLayout *DL, ModifyDT &ModifiedDT, SmallPtrSet< BasicBlock *, 32 > &FreshBBs, bool IsHugeFunc)
If counting leading or trailing zeros is an expensive operation and a zero input is defined,...
static bool sinkCmpExpression(CmpInst *Cmp, const TargetLowering &TLI, const DataLayout &DL)
Sink the given CmpInst into user blocks to reduce the number of virtual registers that must be create...
static bool hasSameExtUse(Value *Val, const TargetLowering &TLI)
Check if all the uses of Val are equivalent (or free) zero or sign extensions.
static bool foldURemOfLoopIncrement(Instruction *Rem, const DataLayout *DL, const LoopInfo *LI, DominatorTree &DT, SmallPtrSet< BasicBlock *, 32 > &FreshBBs, bool IsHuge)
static bool matchUAddWithOverflowConstantEdgeCases(CmpInst *Cmp, BinaryOperator *&Add)
Match special-case patterns that check for unsigned add overflow.
Defines an IR pass for CodeGen Prepare.
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
This file contains the declarations for the subclasses of Constant, which represent the different fla...
This file declares the LLVM IR specialization of the GenericCycle templates.
This file defines the DenseMap class.
static bool runOnFunction(Function &F, bool PostInlining)
static Value * getCondition(Instruction *I)
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
Module.h This file contains the declarations for the Module class.
This defines the Use class.
static constexpr Value * getValue(Ty &ValueOrUse)
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static void eraseInstruction(Instruction &I, const LICMDependencies &Deps)
Register const TargetRegisterInfo * TRI
This file implements a map that provides insertion order iteration.
uint64_t IntrinsicInst * II
OptimizedStructLayoutField Field
#define INITIALIZE_PASS_DEPENDENCY(depName)
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
This file defines the PointerIntPair class.
This file contains the declarations for profiling metadata utility functions.
SmallVector< MachineOperand, 4 > Cond
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
static DominatorTree getDomTree(Function &F)
static bool dominates(InstrPosIndexes &PosIndexes, const MachineInstr &A, const MachineInstr &B)
Remove Loads Into Fake Uses
static bool optimizeBlock(BasicBlock &BB, bool &ModifiedDT, const TargetTransformInfo &TTI, const DataLayout &DL, bool HasBranchDivergence, DomTreeUpdater *DTU)
static bool optimizeCallInst(CallInst *CI, bool &ModifiedDT, const TargetTransformInfo &TTI, const DataLayout &DL, bool HasBranchDivergence, DomTreeUpdater *DTU)
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static SymbolRef::Type getType(const Symbol *Sym)
static bool canCombine(MachineBasicBlock &MBB, MachineOperand &MO, unsigned CombineOpc=0)
This file describes how to lower LLVM code to machine code.
Target-Independent Code Generator Pass Configuration Options pass.
static unsigned getBitWidth(Type *Ty, const DataLayout &DL)
Returns the bitwidth of the given scalar or pointer type.
static Constant * getConstantVector(MVT VT, ArrayRef< APInt > Bits, const APInt &Undefs, LLVMContext &C)
Class for arbitrary precision integers.
LLVM_ABI APInt zext(unsigned width) const
Zero extend to a new width.
uint64_t getZExtValue() const
Get zero extended value.
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
bool isNegative() const
Determine sign of this APInt.
bool isSignedIntN(unsigned N) const
Check if this APInt has an N-bits signed integer value.
unsigned getSignificantBits() const
Get the minimum bit size for this signed APInt.
unsigned logBase2() const
LLVM_ABI APInt sext(unsigned width) const
Sign extend to a new width.
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
int64_t getSExtValue() const
Get sign extended value.
LLVM_ABI bool isStaticAlloca() const
Return true if this alloca is in the entry block of the function and is a constant size.
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
LLVM_ABI std::optional< TypeSize > getAllocationSize(const DataLayout &DL) const
Get allocation size in bytes.
void setAlignment(Align Align)
PassT::Result * getCachedResult(IRUnitT &IR) const
Get the cached result of an analysis pass for a given IR unit.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
AnalysisUsage & addUsedIfAvailable()
Add the specified Pass class to the set of analyses used by this pass.
AnalysisUsage & addRequired()
Represent a constant reference to an array (0 or more elements consecutively in memory),...
An instruction that atomically checks whether a specified value is in a memory location,...
static unsigned getPointerOperandIndex()
an instruction that atomically reads a memory location, combines it with another value,...
static unsigned getPointerOperandIndex()
Analysis pass providing the BasicBlockSectionsProfileReader.
LLVM_ABI bool isFunctionHot(StringRef FuncName) const
LLVM Basic Block Representation.
iterator begin()
Instruction iterator methods.
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
LLVM_ABI const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const Function * getParent() const
Return the enclosing method, or null if none.
bool hasAddressTaken() const
Returns true if there are any uses of this basic block other than direct branches,...
LLVM_ABI InstListType::const_iterator getFirstNonPHIIt() const
Returns an iterator to the first instruction in this block that is not a PHINode instruction.
LLVM_ABI void insertDbgRecordBefore(DbgRecord *DR, InstListType::iterator Here)
Insert a DbgRecord into a block at the position given by Here.
InstListType::const_iterator const_iterator
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
LLVM_ABI void moveAfter(BasicBlock *MovePos)
Unlink this basic block from its current function and insert it right after MovePos in the function M...
LLVM_ABI const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
LLVM_ABI const BasicBlock * getUniquePredecessor() const
Return the predecessor of this block if it has a unique predecessor block.
LLVM_ABI const BasicBlock * getSingleSuccessor() const
Return the successor of this block if it has a single successor.
LLVM_ABI InstListType::const_iterator getFirstNonPHIOrPseudoProbe() const
Returns an iterator to the first instruction in this block that is not a PHINode or a pseudo operatio...
LLVM_ABI void insertDbgRecordAfter(DbgRecord *DR, Instruction *I)
Insert a DbgRecord into a block at the position given by I.
InstListType::iterator iterator
Instruction iterators...
LLVM_ABI LLVMContext & getContext() const
Get the context in which this basic block lives.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction; assumes that the block is well-formed.
BinaryOps getOpcode() const
static LLVM_ABI BinaryOperator * Create(BinaryOps Op, Value *S1, Value *S2, const Twine &Name=Twine(), InsertPosition InsertBefore=nullptr)
Construct a binary instruction, given the opcode and the two operands.
This class represents a no-op cast from one type to another.
Analysis pass which computes BlockFrequencyInfo.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
LLVM_ABI void setBlockFreq(const BasicBlock *BB, BlockFrequency Freq)
LLVM_ABI BlockFrequency getBlockFreq(const BasicBlock *BB) const
getblockFreq - Return block frequency.
Analysis pass which computes BranchProbabilityInfo.
static LLVM_ABI BranchProbability getBranchProbability(uint64_t Numerator, uint64_t Denominator)
bool isInlineAsm() const
Check if this call is an inline asm statement.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
bool hasFnAttr(Attribute::AttrKind Kind) const
Determine whether this call has the given attribute.
Value * getArgOperand(unsigned i) const
void setArgOperand(unsigned i, Value *v)
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
This class represents a function call, abstracting a target machine's calling convention.
This is the base class for all instructions that perform data casts.
static LLVM_ABI CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
This class is the base class for the comparison instructions.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ ICMP_SLT
signed less than
@ ICMP_UGE
unsigned greater or equal
@ ICMP_UGT
unsigned greater than
@ ICMP_SGT
signed greater than
@ ICMP_ULT
unsigned less than
@ ICMP_ULE
unsigned less or equal
Predicate getSwappedPredicate() const
For example, EQ->EQ, SLE->SGE, ULT->UGT, OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
static LLVM_ABI CmpInst * Create(OtherOps Op, Predicate Pred, Value *S1, Value *S2, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Construct a compare instruction, given the opcode, the predicate and the two operands.
Predicate getPredicate() const
Return the predicate for this instruction.
An abstraction over a floating-point predicate, and a pack of an integer predicate with samesign info...
LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
Conditional Branch instruction.
static LLVM_ABI Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static LLVM_ABI Constant * getNeg(Constant *C, bool HasNSW=false)
This is the shared class of boolean and integer constants.
static LLVM_ABI ConstantInt * getTrue(LLVMContext &Context)
static ConstantInt * getSigned(IntegerType *Ty, int64_t V, bool ImplicitTrunc=false)
Return a ConstantInt with the specified value for the specified type.
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
static LLVM_ABI ConstantInt * getFalse(LLVMContext &Context)
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
const APInt & getValue() const
Return the constant as an APInt value reference.
static LLVM_ABI Constant * getSplat(ElementCount EC, Constant *Elt)
Return a ConstantVector with the specified constant in each element.
static LLVM_ABI Constant * get(ArrayRef< Constant * > V)
This is an important base class in LLVM.
static LLVM_ABI Constant * getAllOnesValue(Type *Ty)
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
A parsed version of the target data layout string in and methods for querying it.
LLVM_ABI void removeFromParent()
Record of a variable value-assignment, aka a non instruction representation of the dbg....
LocationType Type
Classification of the debug-info record that this DbgVariableRecord represents.
LLVM_ABI void replaceVariableLocationOp(Value *OldValue, Value *NewValue, bool AllowEmpty=false)
LLVM_ABI iterator_range< location_op_iterator > location_ops() const
Get the locations corresponding to the variable referenced by the debug info intrinsic.
iterator find(const_arg_type_t< KeyT > Val)
bool erase(const KeyT &Val)
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
LLVM_ABI void deleteBB(BasicBlock *DelBB)
Delete DelBB.
Analysis pass which computes a DominatorTree.
static constexpr UpdateKind Insert
Legacy analysis pass which computes a DominatorTree.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
LLVM_ABI bool dominates(const BasicBlock *BB, const Use &U) const
Return true if the (end of the) basic block BB dominates the use U.
This instruction compares its operands according to the predicate given to the constructor.
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
FunctionPass class - This class is used to implement most global optimizations.
const BasicBlock & getEntryBlock() const
LLVM_ABI const Value * getStatepoint() const
The statepoint with which this gc.relocate is associated.
Represents calls to the gc.relocate intrinsic.
unsigned getBasePtrIndex() const
The index into the associate statepoint's argument list which contains the base pointer of the pointe...
void compute(FunctionT &F)
Compute the cycle info for a function.
DomTreeT & getDomTree()
Flush DomTree updates and return DomTree.
void applyUpdates(ArrayRef< UpdateT > Updates)
Submit updates to all available trees.
void flush()
Apply all pending updates to available trees and flush all BasicBlocks awaiting deletion.
bool isBBPendingDeletion(BasicBlockT *DelBB) const
Returns true if DelBB is awaiting deletion.
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
static LLVM_ABI Type * getIndexedType(Type *Ty, ArrayRef< Value * > IdxList)
Returns the result type of a getelementptr with the given source element type and indexes.
LLVM_ABI bool canIncreaseAlignment() const
Returns true if the alignment of the value can be unilaterally increased.
bool isThreadLocal() const
If the value is "Thread Local", its value isn't shared by the threads.
LLVM_ABI uint64_t getGlobalSize(const DataLayout &DL) const
Get the size of this global variable in bytes.
void setAlignment(Align Align)
Sets the alignment attribute of the GlobalVariable.
This instruction compares its operands according to the predicate given to the constructor.
bool isEquality() const
Return true if this predicate is either EQ or NE.
Value * CreateNUWAdd(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateURem(Value *LHS, Value *RHS, const Twine &Name="")
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
LLVM_ABI Instruction * clone() const
Create a copy of 'this' instruction that is identical in all ways except the following:
LLVM_ABI void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
LLVM_ABI void setHasNoSignedWrap(bool b=true)
Set or clear the nsw flag on this instruction, which must be an operator which supports this flag.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
user_iterator_impl< Instruction > user_iterator
Specialize the methods defined in Value, as we know that an instruction can only be used by other ins...
LLVM_ABI void moveAfter(Instruction *MovePos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
bool hasMetadata() const
Return true if this instruction has any metadata attached to it.
LLVM_ABI void moveBefore(InstListType::iterator InsertPos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
LLVM_ABI void insertBefore(InstListType::iterator InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified position.
bool isEHPad() const
Return true if the instruction is a variety of EH-block.
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
LLVM_ABI const Function * getFunction() const
Return the function this instruction belongs to.
LLVM_ABI bool mayHaveSideEffects() const LLVM_READONLY
Return true if the instruction may have side effects.
LLVM_ABI bool comesBefore(const Instruction *Other) const
Given an instruction Other in the same basic block as this instruction, return true if this instructi...
LLVM_ABI bool mayReadFromMemory() const LLVM_READONLY
Return true if this instruction may read memory.
iterator_range< user_iterator > users()
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
user_iterator user_begin()
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
LLVM_ABI void dropPoisonGeneratingFlags()
Drops flags that may cause this instruction to evaluate to poison despite having non-poison inputs.
LLVM_ABI std::optional< simple_ilist< DbgRecord >::iterator > getDbgReinsertionPosition()
Return an iterator to the position of the "Next" DbgRecord after this instruction,...
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
LLVM_ABI void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
LLVM_ABI void insertAfter(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately after the specified instruction.
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
A wrapper class for inspecting calls to intrinsic functions.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
An instruction for reading from memory.
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
Analysis pass that exposes the LoopInfo for a function.
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
The legacy pass manager's analysis pass to compute loop information.
Represents a single loop in the control flow graph.
static MVT getIntegerVT(unsigned BitWidth)
This class implements a map that also provides access to all stored values in a deterministic order.
iterator find(const KeyT &Key)
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
VectorType::iterator erase(typename VectorType::iterator Iterator)
Remove the element given by Iterator.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
op_range incoming_values()
Value * getIncomingValueForBlock(const BasicBlock *BB) const
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
unsigned getNumIncomingValues() const
Return the number of incoming edges.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
PointerIntPair - This class implements a pair of a pointer and small integer.
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
PreservedAnalyses & preserve()
Mark an analysis as preserved.
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
An analysis pass based on legacy pass manager to deliver ProfileSummaryInfo.
Analysis providing profile information.
Value * getReturnValue() const
Convenience accessor. Returns null if there is no return value.
This class represents the LLVM 'select' instruction.
static SelectInst * Create(Value *C, Value *S1, Value *S2, const Twine &NameStr="", InsertPosition InsertBefore=nullptr, const Instruction *MDFrom=nullptr)
size_type count(const_arg_type key) const
Count the number of elements of a given key in the SetVector.
void clear()
Completely clear the SetVector.
bool empty() const
Determine if the SetVector is empty or not.
bool insert(const value_type &X)
Insert a new element into the SetVector.
value_type pop_back_val()
VectorType * getType() const
Overload to return most specific vector type.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
bool erase(PtrType Ptr)
Remove pointer from the set.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
void insert_range(Range &&R)
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
iterator erase(const_iterator CI)
void push_back(const T &Elt)
LLVM_NO_SANITIZE_CFI_UNRELATED_CAST iterator begin()
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
static unsigned getPointerOperandIndex()
TypeSize getElementOffset(unsigned Idx) const
Analysis pass providing the TargetTransformInfo.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
LibFunc getLibFunc(StringRef funcName) const
Searches for a particular function name.
int InstructionOpcodeToISD(unsigned Opcode) const
Get the ISD node that corresponds to the Instruction class opcode.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
virtual bool isSelectSupported(SelectSupportKind) const
virtual bool shouldFormOverflowOp(unsigned Opcode, EVT VT, bool MathUsed) const
Try to convert math with an overflow comparison into the corresponding DAG node operation.
virtual bool shouldOptimizeMulOverflowWithZeroHighBits(LLVMContext &Context, EVT VT) const
bool isExtLoad(const LoadInst *Load, const Instruction *Ext, const DataLayout &DL) const
Return true if Load and Ext can form an ExtLoad.
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
const TargetMachine & getTargetMachine() const
virtual bool isCtpopFast(EVT VT) const
Return true if ctpop instruction is fast.
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
bool enableExtLdPromotion() const
Return true if the target wants to use the optimization that turns ext(promotableInst1(....
bool isJumpExpensive() const
Return true if Flow Control is an expensive operation that should be avoided.
bool hasExtractBitsInsn() const
Return true if the target has BitExtract instructions.
virtual bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *=nullptr) const
Determine if the target supports unaligned memory accesses.
bool isSlowDivBypassed() const
Returns true if target has indicated at least one type should be bypassed.
virtual bool isTruncateFree(Type *FromTy, Type *ToTy) const
Return true if it's free to truncate a value of type FromTy to type ToTy.
virtual MVT getPreferredSwitchConditionType(LLVMContext &Context, EVT ConditionVT) const
Returns preferred type for switch condition.
virtual bool canCombineStoreAndExtract(Type *VectorTy, Value *Idx, unsigned &Cost) const
Return true if the target can combine store(extractelement VectorTy,Idx).
virtual bool shouldConsiderGEPOffsetSplit() const
bool isExtFree(const Instruction *I) const
Return true if the extension represented by I is free.
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual bool getAddrModeArguments(const IntrinsicInst *, SmallVectorImpl< Value * > &, Type *&) const
CodeGenPrepare sinks address calculations into the same BB as Load/Store instructions reading the add...
const DenseMap< unsigned int, unsigned int > & getBypassSlowDivWidths() const
Returns map of slow types for division or remainder with corresponding fast types.
virtual int64_t getPreferredLargeGEPBaseOffset(int64_t MinOffset, int64_t MaxOffset) const
Return the prefered common base offset.
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
virtual bool optimizeVectorCrossLaneOperation(Instruction *I, Loop *L) const
virtual bool shouldAlignPointerArgs(CallInst *, unsigned &, Align &) const
Return true if the pointer arguments to CI should be aligned by aligning the object whose address is ...
virtual Type * shouldConvertSplatType(ShuffleVectorInst *SVI) const
Given a shuffle vector SVI representing a vector splat, return a new scalar type of size equal to SVI...
bool isLoadLegal(EVT ValVT, EVT MemVT, Align Alignment, unsigned AddrSpace, unsigned ExtType, bool Atomic) const
Return true if the specified load with extension is legal on this target.
virtual bool addressingModeSupportsTLS(const GlobalValue &) const
Returns true if the targets addressing mode can target thread local storage (TLS).
virtual bool shouldConvertPhiType(Type *From, Type *To) const
Given a set in interconnected phis of type 'From' that are loaded/stored or bitcast to type 'To',...
virtual bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AddrSpace, Instruction *I=nullptr) const
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
virtual bool optimizeExtendOrTruncateConversion(Instruction *I, Loop *L, const TargetTransformInfo &TTI) const
Try to optimize extending or truncating conversion instructions (like zext, trunc,...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
std::vector< AsmOperandInfo > AsmOperandInfoVector
virtual AsmOperandInfoVector ParseConstraints(const DataLayout &DL, const TargetRegisterInfo *TRI, const CallBase &Call) const
Split up the constraint string from the inline assembly value into the specific constraints and their...
virtual void ComputeConstraintToUse(AsmOperandInfo &OpInfo, SDValue Op, SelectionDAG *DAG=nullptr) const
Determines the constraint code and constraint type to use for the specific AsmOperandInfo,...
virtual bool mayBeEmittedAsTailCall(const CallInst *) const
Return true if the target may be able emit the call instruction as a tail call.
virtual bool isNoopAddrSpaceCast(const DataLayout &DL, unsigned SrcAS, unsigned DestAS) const
Returns true if a cast between SrcAS and DestAS is a noop.
virtual const TargetSubtargetInfo * getSubtargetImpl(const Function &) const
Virtual method implemented by subclasses that returns a reference to that target's TargetSubtargetInf...
unsigned EnableFastISel
EnableFastISel - This flag enables fast-path instruction selection which trades away generated code q...
Target-Independent Code Generator Pass Configuration Options.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const =0
Return the target's register information.
virtual const TargetLowering * getTargetLowering() const
The instances of the Type class are immutable: once they are created, they are never changed.
LLVM_ABI unsigned getIntegerBitWidth() const
bool isVectorTy() const
True if this is an instance of VectorType.
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
LLVM_ABI unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
LLVM_ABI Type * getWithNewBitWidth(unsigned NewBitWidth) const
Given an integer or vector type, change the lane bitwidth to NewBitwidth, whilst keeping the old numb...
LLVM_ABI bool isScalableTy() const
Return true if this is a type whose size is a known multiple of vscale.
bool isIntOrPtrTy() const
Return true if this is an integer type or a pointer type.
bool isIntegerTy() const
True if this is an instance of IntegerType.
static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
BasicBlock * getSuccessor(unsigned i=0) const
static LLVM_ABI UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
A Use represents the edge between a Value definition and its users.
const Use & getOperandUse(unsigned i) const
void setOperand(unsigned i, Value *Val)
LLVM_ABI bool replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
user_iterator user_begin()
LLVM_ABI void setName(const Twine &Name)
Change the name of the value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
LLVMContext & getContext() const
All values hold a context through their type.
iterator_range< user_iterator > users()
LLVM_ABI Align getPointerAlignment(const DataLayout &DL) const
Returns an alignment of the pointer value.
LLVM_ABI bool isUsedInBasicBlock(const BasicBlock *BB) const
Check if this value is used in the specified basic block.
LLVM_ABI void printAsOperand(raw_ostream &O, bool PrintType=true, const Module *M=nullptr) const
Print the name of this Value out to the specified raw_ostream.
LLVM_ABI const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
iterator_range< use_iterator > uses()
void mutateType(Type *Ty)
Mutate the type of this Value to be of the specified type.
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
LLVM_ABI void takeName(Value *V)
Transfer the name from V to this value.
LLVM_ABI void dump() const
Support for debugging, callable in GDB: V->dump()
bool pointsToAliveValue() const
constexpr ScalarTy getFixedValue() const
constexpr bool isNonZero() const
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
StructType * getStructTypeOrNull() const
TypeSize getSequentialElementStride(const DataLayout &DL) const
const ParentTy * getParent() const
self_iterator getIterator()
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
unsigned getAddrMode(MCInstrInfo const &MCII, MCInst const &MCI)
@ BasicBlock
Various leaf nodes.
SpecificConstantMatch m_ZeroInt()
Convenience matchers for specific integer values.
AllOnesConstantMatch m_AllOnes()
OneUse_match< SubPat > m_OneUse(const SubPat &SP)
match_combine_or< Ty... > m_CombineOr(const Ty &...Ps)
Combine pattern matchers matching any of Ps patterns.
match_bind< PHINode > m_Phi(PHINode *&PN)
Match a PHI node, capturing it if we match.
auto m_Cmp()
Matches any compare instruction and ignore it.
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::URem > m_URem(const LHS &L, const RHS &R)
ap_match< APInt > m_APInt(const APInt *&Res)
Match a ConstantInt or splatted ConstantVector, binding the specified pointer to the contained APInt.
BinaryOp_match< LHS, RHS, Instruction::Xor > m_Xor(const LHS &L, const RHS &R)
ap_match< APInt > m_APIntAllowPoison(const APInt *&Res)
Match APInt while allowing poison in splat vector constants.
specific_intval< false > m_SpecificInt(const APInt &V)
Match a specific integer value or vector with all elements equal to the value.
bool match(Val *V, const Pattern &P)
match_bind< Instruction > m_Instruction(Instruction *&I)
Match an instruction, capturing it if we match.
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
BinOpPred_match< LHS, RHS, is_right_shift_op > m_Shr(const LHS &L, const RHS &R)
Matches logical shift operations.
OverflowingBinaryOp_match< LHS, RHS, Instruction::Add, OverflowingBinaryOperator::NoUnsignedWrap, true > m_c_NUWAdd(const LHS &L, const RHS &R)
cst_pred_ty< is_one > m_One()
Match an integer 1 or a vector with all elements equal to 1.
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
auto m_BinOp()
Match an arbitrary binary operation and ignore it.
ExtractValue_match< Ind, Val_t > m_ExtractValue(const Val_t &V)
Match a single index ExtractValue instruction.
auto m_Value()
Match an arbitrary value and ignore it.
auto m_Ctpop(const Opnd0 &Op0)
auto m_Constant()
Match an arbitrary Constant and ignore it.
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
auto m_VScale()
Matches a call to llvm.vscale().
CastInst_match< OpTy, ZExtInst > m_ZExt(const OpTy &Op)
Matches ZExt.
Splat_match< T > m_Splat(const T &SubPattern)
Match a vector splat.
PtrAdd_match< PointerOpTy, OffsetOpTy > m_PtrAdd(const DataLayout &DL, const PointerOpTy &PointerOp, const OffsetOpTy &OffsetOp)
Matches GEP with i8 source element type and canonical index type.
match_combine_or< OverflowingBinaryOp_match< LHS, RHS, Instruction::Add, OverflowingBinaryOperator::NoUnsignedWrap, true >, DisjointOr_match< LHS, RHS, true > > m_c_NUWAddLike(const LHS &L, const RHS &R)
Match either "add nuw" or "or disjoint", with LHS and RHS commutable.
match_immconstant_ty m_ImmConstant()
Match an arbitrary immediate Constant and ignore it.
BinaryOp_match< LHS, RHS, Instruction::Add, true > m_c_Add(const LHS &L, const RHS &R)
Matches a Add with LHS and RHS in either order.
auto m_Intrinsic(const Ts &...Ops)
Match intrinsic calls like this: m_Intrinsic<Intrinsic::fabs>(m_Value(X))
OverflowingBinaryOp_match< LHS, RHS, Instruction::Add, OverflowingBinaryOperator::NoSignedWrap > m_NSWAdd(const LHS &L, const RHS &R)
CmpClass_match< LHS, RHS, ICmpInst > m_ICmp(CmpPredicate &Pred, const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::Shl > m_Shl(const LHS &L, const RHS &R)
UAddWithOverflow_match< LHS_t, RHS_t, Sum_t > m_UAddWithOverflow(const LHS_t &L, const RHS_t &R, const Sum_t &S)
Match an icmp instruction checking for unsigned overflow on addition.
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
brc_match< Cond_t, match_bind< BasicBlock >, match_bind< BasicBlock > > m_Br(const Cond_t &C, BasicBlock *&T, BasicBlock *&F)
auto m_Undef()
Match an arbitrary undef constant.
BinaryOp_match< LHS, RHS, Instruction::Or, true > m_c_Or(const LHS &L, const RHS &R)
Matches an Or with LHS and RHS in either order.
ThreeOps_match< Val_t, Elt_t, Idx_t, Instruction::InsertElement > m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx)
Matches InsertElementInst.
BinaryOp_match< LHS, RHS, Instruction::Sub > m_Sub(const LHS &L, const RHS &R)
auto m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
int compare(DigitsT LDigits, int16_t LScale, DigitsT RDigits, int16_t RScale)
Compare two scaled numbers.
@ CE
Windows NT (Windows on ARM)
PointerTypeMap run(const Module &M)
Compute the PointerTypeMap for the module M.
@ User
could "use" a pointer
NodeAddr< PhiNode * > Phi
NodeAddr< UseNode * > Use
SmallVector< Node, 4 > NodeList
friend class Instruction
Iterator for Instructions in a `BasicBlock.
LLVM_ABI iterator begin() const
BaseReg
Stack frame base register. Bit 0 of FREInfo.Info.
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
LLVM_ABI bool RemoveRedundantDbgInstrs(BasicBlock *BB)
Try to remove redundant dbg.value instructions from given basic block.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
LLVM_ABI bool RecursivelyDeleteTriviallyDeadInstructions(Value *V, const TargetLibraryInfo *TLI=nullptr, MemorySSAUpdater *MSSAU=nullptr, std::function< void(Value *)> AboutToDeleteCallback=std::function< void(Value *)>())
If the specified value is a trivially dead instruction, delete it.
LLVM_ABI bool ConstantFoldTerminator(BasicBlock *BB, bool DeleteDeadConditions=false, const TargetLibraryInfo *TLI=nullptr, DomTreeUpdater *DTU=nullptr)
If a terminator instruction is predicated on a constant value, convert it into an unconditional branc...
LLVM_ABI bool bypassSlowDivision(BasicBlock *BB, const DenseMap< unsigned int, unsigned int > &BypassWidth, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BranchProbabilityInfo *BPI=nullptr)
This optimization identifies DIV instructions in a BB that can be profitably bypassed and carried out...
LLVM_ABI void findDbgValues(Value *V, SmallVectorImpl< DbgVariableRecord * > &DbgVariableRecords)
Finds the dbg.values describing a value.
LLVM_ABI void setExplicitlyUnknownBranchWeightsIfProfiled(Instruction &I, StringRef PassName, const Function *F=nullptr)
Like setExplicitlyUnknownBranchWeights(...), but only sets unknown branch weights in the new instruct...
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
APInt operator*(APInt a, uint64_t RHS)
bool isAligned(Align Lhs, uint64_t SizeInBytes)
Checks that SizeInBytes is a multiple of the alignment.
LLVM_ABI void salvageDebugInfo(const MachineRegisterInfo &MRI, MachineInstr &MI)
Assuming the instruction MI is going to be deleted, attempt to salvage debug users of MI by writing t...
auto successors(const MachineBasicBlock *BB)
@ Load
The value being inserted comes from a load (InsertElement only).
OuterAnalysisManagerProxy< ModuleAnalysisManager, Function > ModuleAnalysisManagerFunctionProxy
Provide the ModuleAnalysisManager to Function proxy.
LLVM_ABI ReturnInst * FoldReturnIntoUncondBranch(ReturnInst *RI, BasicBlock *BB, BasicBlock *Pred, DomTreeUpdater *DTU=nullptr)
This method duplicates the specified return instruction into a predecessor which ends in an unconditi...
bool operator!=(uint64_t V1, const APInt &V2)
constexpr from_range_t from_range
LLVM_ABI BasicBlock * splitBlockBefore(BasicBlock *Old, BasicBlock::iterator SplitPt, DomTreeUpdater *DTU, LoopInfo *LI, MemorySSAUpdater *MSSAU, const Twine &BBName="")
Split the specified block at the specified instruction SplitPt.
LLVM_ABI Instruction * SplitBlockAndInsertIfElse(Value *Cond, BasicBlock::iterator SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BasicBlock *ElseBlock=nullptr)
Similar to SplitBlockAndInsertIfThen, but the inserted block is on the false path of the branch.
LLVM_ABI bool SplitIndirectBrCriticalEdges(Function &F, bool IgnoreBlocksWithoutPHI, BranchProbabilityInfo *BPI=nullptr, BlockFrequencyInfo *BFI=nullptr, DomTreeUpdater *DTU=nullptr)
LLVM_ABI bool DeleteDeadPHIs(BasicBlock *BB, const TargetLibraryInfo *TLI=nullptr, MemorySSAUpdater *MSSAU=nullptr, SmallPtrSetImpl< PHINode * > *KnownNonDeadPHIs=nullptr)
Examine each PHI in the given block and delete it if it is dead.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
LLVM_ABI bool shouldOptimizeForSize(const MachineFunction *MF, ProfileSummaryInfo *PSI, const MachineBlockFrequencyInfo *BFI, PGSOQueryType QueryType=PGSOQueryType::Other)
Returns true if machine function MF is suggested to be size-optimized based on the profile.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
constexpr std::enable_if_t< std::is_signed_v< T >, std::pair< T, bool > > AddOverflow(T X, T Y)
Add two signed integers, computing the two's complement truncated result, returning a pair {result,...
LLVM_ABI void DeleteDeadBlock(BasicBlock *BB, DomTreeUpdater *DTU=nullptr, bool KeepOneInputPHIs=false)
Delete the specified block, which must have no predecessors.
LLVM_ABI bool isSafeToSpeculativelyExecute(const Instruction *I, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr, bool UseVariableInfo=true, bool IgnoreUBImplyingAttrs=true)
Return true if the instruction does not have any effects besides calculating the result and does not ...
auto unique(Range &&R, Predicate P)
LLVM_ABI Value * getSplatValue(const Value *V)
Get splat value if the input is a splat vector or return nullptr.
LLVM_ABI bool hasBranchWeightOrigin(const Instruction &I)
Check if Branch Weight Metadata has an "expected" field from an llvm.expect* intrinsic.
constexpr auto equal_to(T &&Arg)
Functor variant of std::equal_to that can be used as a UnaryPredicate in functional algorithms like a...
bool operator==(const AddressRangeValuePair &LHS, const AddressRangeValuePair &RHS)
constexpr int popcount(T Value) noexcept
Count the number of set bits in a value.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
LLVM_ABI Value * simplifyInstruction(Instruction *I, const SimplifyQuery &Q)
See if we can compute a simplified version of this instruction.
LLVM_ABI Value * simplifyAddInst(Value *LHS, Value *RHS, bool IsNSW, bool IsNUW, const SimplifyQuery &Q)
Given operands for an Add, fold the result or return null.
auto dyn_cast_or_null(const Y &Val)
void erase(Container &C, ValueType V)
Wrapper function to remove a value from a container:
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
LLVM_ABI bool isSplatValue(const Value *V, int Index=-1, unsigned Depth=0)
Return true if each element of the vector value V is poisoned or equal to every other non-poisoned el...
LLVM_ABI bool replaceAndRecursivelySimplify(Instruction *I, Value *SimpleV, const TargetLibraryInfo *TLI=nullptr, const DominatorTree *DT=nullptr, AssumptionCache *AC=nullptr, SmallSetVector< Instruction *, 8 > *UnsimplifiedUsers=nullptr)
Replace all uses of 'I' with 'SimpleV' and simplify the uses recursively.
auto reverse(ContainerTy &&C)
LLVM_ABI bool recognizeBSwapOrBitReverseIdiom(Instruction *I, bool MatchBSwaps, bool MatchBitReversals, SmallVectorImpl< Instruction * > &InsertedInsts)
Try to match a bswap or bitreverse idiom.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
void sort(IteratorTy Start, IteratorTy End)
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
LLVM_ABI void SplitBlockAndInsertIfThenElse(Value *Cond, BasicBlock::iterator SplitBefore, Instruction **ThenTerm, Instruction **ElseTerm, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr)
SplitBlockAndInsertIfThenElse is similar to SplitBlockAndInsertIfThen, but also creates the ElseBlock...
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
IRBuilder(LLVMContext &, FolderTy, InserterTy) -> IRBuilder< FolderTy, InserterTy >
auto make_first_range(ContainerTy &&c)
Given a container of pairs, return a range over the first elements.
generic_gep_type_iterator<> gep_type_iterator
LLVM_ABI FunctionPass * createCodeGenPrepareLegacyPass()
createCodeGenPrepareLegacyPass - Transform the code to expose more pattern matching during instructio...
LLVM_ABI ISD::CondCode getFCmpCondCode(FCmpInst::Predicate Pred)
getFCmpCondCode - Return the ISD condition code corresponding to the given LLVM IR floating-point con...
LLVM_ABI bool VerifyLoopInfo
Enable verification of loop info.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
LLVM_ABI bool isKnownNonZero(const Value *V, const SimplifyQuery &Q, unsigned Depth=0)
Return true if the given value is known to be non-zero when defined.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
LLVM_ABI bool attributesPermitTailCall(const Function *F, const Instruction *I, const ReturnInst *Ret, const TargetLoweringBase &TLI, bool *AllowDifferingSizes=nullptr)
Test if given that the input instruction is in the tail call position, if there is an attribute misma...
LLVM_ABI bool MergeBlockIntoPredecessor(BasicBlock *BB, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, MemoryDependenceResults *MemDep=nullptr, bool PredecessorWithTwoSuccessors=false, DominatorTree *DT=nullptr)
Attempts to merge a block into its predecessor, if possible.
@ Or
Bitwise or logical OR of integers.
@ Xor
Bitwise or logical XOR of integers.
@ And
Bitwise or logical AND of integers.
@ Sub
Subtraction of integers.
LLVM_ABI BasicBlock * SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="")
Split the specified block at the specified instruction.
auto count(R &&Range, const E &Element)
Wrapper function around std::count to count the number of times an element Element occurs in the give...
DWARFExpression::Operation Op
bool isSafeToSpeculativelyExecuteWithVariableReplaced(const Instruction *I, bool IgnoreUBImplyingAttrs=true)
Don't use information from its non-constant operands.
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
LLVM_ABI bool isGuaranteedNotToBeUndefOrPoison(const Value *V, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Return true if this function can prove that V does not have undef bits and is never poison.
ArrayRef(const T &OneElt) -> ArrayRef< T >
LLVM_ABI bool VerifyDomInfo
Enables verification of dominator trees.
constexpr unsigned BitWidth
LLVM_ABI bool extractBranchWeights(const MDNode *ProfileData, SmallVectorImpl< uint32_t > &Weights)
Extract branch weights from MD_prof metadata.
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
gep_type_iterator gep_type_begin(const User *GEP)
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
auto predecessors(const MachineBasicBlock *BB)
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Align getKnownAlignment(Value *V, const DataLayout &DL, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr)
Try to infer an alignment for the specified pointer.
constexpr std::enable_if_t< std::is_signed_v< T >, std::pair< T, bool > > MulOverflow(T X, T Y)
Multiply two signed integers, computing the two's complement truncated result, returning a pair {resu...
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Next
bool pred_empty(const BasicBlock *BB)
LLVM_ABI Instruction * SplitBlockAndInsertIfThen(Value *Cond, BasicBlock::iterator SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...
AnalysisManager< Function > FunctionAnalysisManager
Convenience typedef for the Function analysis manager.
LLVM_ABI BasicBlock * SplitEdge(BasicBlock *From, BasicBlock *To, DominatorTree *DT=nullptr, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="")
Split the edge connecting the specified blocks, and return the newly created basic block between From...
LLVM_ABI void setFittedBranchWeights(Instruction &I, ArrayRef< uint64_t > Weights, bool IsExpected, bool ElideAllZero=false)
Variant of setBranchWeights where the Weights will be fit first to uint32_t by shifting right.
std::pair< Value *, FPClassTest > fcmpToClassTest(FCmpInst::Predicate Pred, const Function &F, Value *LHS, Value *RHS, bool LookThroughSrc=true)
Returns a pair of values, which if passed to llvm.is.fpclass, returns the same result as an fcmp with...
static auto filterDbgVars(iterator_range< simple_ilist< DbgRecord >::iterator > R)
Filter the DbgRecord range to DbgVariableRecord types only and downcast.
LLVM_ABI Value * simplifyURemInst(Value *LHS, Value *RHS, const SimplifyQuery &Q)
Given operands for a URem, fold the result or return null.
DenseMap< const Value *, Value * > ValueToValueMap
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
static LLVM_ABI EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool isRound() const
Return true if the size is a power-of-two number of bytes.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
bool isInteger() const
Return true if this is an integer or a vector integer type.
This contains information for each constraint that we are lowering.