46 #define DEBUG_TYPE "rewrite-statepoints-for-gc"
65 #ifdef EXPENSIVE_CHECKS
79 struct RewriteStatepointsForGC :
public ModulePass {
86 bool runOnModule(
Module &M)
override {
89 Changed |= runOnFunction(
F);
95 stripNonValidAttributes(M);
117 void stripNonValidAttributes(
Module &M);
120 void stripNonValidAttributesFromBody(
Function &
F);
121 void stripNonValidAttributesFromPrototype(
Function &
F);
128 return new RewriteStatepointsForGC();
132 "Make relocations explicit at statepoints",
false,
false)
198 "Found non-leaf call without deopt info!");
207 GCPtrLivenessData &Data);
218 if (
auto *PT = dyn_cast<PointerType>(T))
222 return PT->getAddressSpace() == 1;
236 if (
auto VT = dyn_cast<VectorType>(T))
248 if (
VectorType *VT = dyn_cast<VectorType>(Ty))
250 if (
ArrayType *AT = dyn_cast<ArrayType>(Ty))
278 GCPtrLivenessData &OriginalLivenessData,
CallSite CS,
279 PartiallyConstructedSafepointRecord &Result) {
286 dbgs() <<
"Live Variables:\n";
287 for (
Value *V : LiveSet)
288 dbgs() <<
" " << V->getName() <<
" " << *V <<
"\n";
292 dbgs() <<
"Number live values: " << LiveSet.size() <<
"\n";
294 Result.LiveSet = LiveSet;
307 struct BaseDefiningValueResult {
312 const bool IsKnownBase;
313 BaseDefiningValueResult(
Value *BDV,
bool IsKnownBase)
314 : BDV(BDV), IsKnownBase(IsKnownBase) {
319 assert(!MustBeBase || MustBeBase == IsKnownBase);
336 static BaseDefiningValueResult
341 if (isa<Argument>(I))
343 return BaseDefiningValueResult(I,
true);
345 if (isa<Constant>(I))
351 if (isa<LoadInst>(I))
352 return BaseDefiningValueResult(I,
true);
354 if (isa<InsertElementInst>(I))
358 return BaseDefiningValueResult(I,
false);
360 if (isa<ShuffleVectorInst>(I))
366 return BaseDefiningValueResult(I,
false);
370 assert((isa<SelectInst>(I) || isa<PHINode>(I)) &&
371 "unknown vector instruction - no base found for vector element");
372 return BaseDefiningValueResult(I,
false);
381 "Illegal to ask for the base pointer of a non-pointer type");
386 if (isa<Argument>(I))
389 return BaseDefiningValueResult(I,
true);
391 if (isa<Constant>(I)) {
402 return BaseDefiningValueResult(
406 if (
CastInst *CI = dyn_cast<CastInst>(I)) {
407 Value *
Def = CI->stripPointerCasts();
410 assert(cast<PointerType>(Def->
getType())->getAddressSpace() ==
411 cast<PointerType>(CI->getType())->getAddressSpace() &&
412 "unsupported addrspacecast");
416 assert(!isa<CastInst>(Def) &&
"shouldn't find another cast here");
420 if (isa<LoadInst>(I))
422 return BaseDefiningValueResult(I,
true);
430 switch (II->getIntrinsicID()) {
434 case Intrinsic::experimental_gc_statepoint:
436 case Intrinsic::experimental_gc_relocate: {
442 case Intrinsic::gcroot:
447 "interaction with the gcroot mechanism is not supported");
453 if (isa<CallInst>(I) || isa<InvokeInst>(I))
454 return BaseDefiningValueResult(I,
true);
458 assert(!isa<LandingPadInst>(I) &&
"Landing Pad is unimplemented");
460 if (isa<AtomicCmpXchgInst>(I))
464 return BaseDefiningValueResult(I,
true);
466 assert(!isa<AtomicRMWInst>(I) &&
"Xchg handled above, all others are "
467 "binary ops which don't apply to pointers");
472 if (isa<ExtractValueInst>(I))
473 return BaseDefiningValueResult(I,
true);
477 assert(!isa<InsertValueInst>(I) &&
478 "Base pointer for a struct is meaningless");
484 if (isa<ExtractElementInst>(I))
488 return BaseDefiningValueResult(I,
false);
494 assert((isa<SelectInst>(I) || isa<PHINode>(I)) &&
495 "missing instruction case in findBaseDefiningValing");
496 return BaseDefiningValueResult(I,
false);
501 Value *&Cached = Cache[
I];
507 assert(Cache[I] !=
nullptr);
515 auto Found = Cache.find(Def);
516 if (Found != Cache.end()) {
518 return Found->second;
527 if (!isa<PHINode>(V) && !isa<SelectInst>(V) &&
528 !isa<ExtractElementInst>(V) && !isa<InsertElementInst>(V) &&
529 !isa<ShuffleVectorInst>(V)) {
533 if (isa<Instruction>(V) &&
534 cast<Instruction>(V)->getMetadata(
"is_base_value")) {
555 : Status(Status), BaseValue(BaseValue) {
556 assert(Status != Base || BaseValue);
559 explicit BDVState(
Value *BaseValue) :
Status(Base), BaseValue(BaseValue) {}
562 Value *getBaseValue()
const {
return BaseValue; }
564 bool isBase()
const {
return getStatus() == Base; }
565 bool isUnknown()
const {
return getStatus() ==
Unknown; }
566 bool isConflict()
const {
return getStatus() == Conflict; }
568 bool operator==(
const BDVState &Other)
const {
569 return BaseValue == Other.BaseValue &&
Status == Other.Status;
572 bool operator!=(
const BDVState &other)
const {
return !(*
this == other); }
581 switch (getStatus()) {
592 OS <<
" (" << getBaseValue() <<
" - "
593 << (getBaseValue() ? getBaseValue()->getName() :
"nullptr") <<
"): ";
610 switch (LHS.getStatus()) {
615 assert(LHS.getBaseValue() &&
"can't be null");
620 if (LHS.getBaseValue() == RHS.getBaseValue()) {
621 assert(LHS == RHS &&
"equality broken!");
624 return BDVState(BDVState::Conflict);
626 assert(RHS.isConflict() &&
"only three states!");
627 return BDVState(BDVState::Conflict);
629 case BDVState::Conflict:
640 "Math is wrong: meet does not commute!");
677 auto isExpectedBDVType = [](
Value *BDV) {
678 return isa<PHINode>(BDV) || isa<SelectInst>(BDV) ||
679 isa<ExtractElementInst>(BDV) || isa<InsertElementInst>(BDV) ||
680 isa<ShuffleVectorInst>(BDV);
697 while (!Worklist.
empty()) {
701 auto visitIncomingValue = [&](
Value *InVal) {
707 assert(isExpectedBDVType(Base) &&
"the only non-base values "
708 "we see should be base defining values");
709 if (States.
insert(std::make_pair(Base, BDVState())).second)
712 if (
PHINode *PN = dyn_cast<PHINode>(Current)) {
713 for (
Value *InVal : PN->incoming_values())
714 visitIncomingValue(InVal);
715 }
else if (
SelectInst *
SI = dyn_cast<SelectInst>(Current)) {
716 visitIncomingValue(
SI->getTrueValue());
717 visitIncomingValue(
SI->getFalseValue());
718 }
else if (
auto *EE = dyn_cast<ExtractElementInst>(Current)) {
719 visitIncomingValue(EE->getVectorOperand());
720 }
else if (
auto *
IE = dyn_cast<InsertElementInst>(Current)) {
721 visitIncomingValue(
IE->getOperand(0));
722 visitIncomingValue(
IE->getOperand(1));
723 }
else if (
auto *SV = dyn_cast<ShuffleVectorInst>(Current)) {
724 visitIncomingValue(SV->getOperand(0));
725 visitIncomingValue(SV->getOperand(1));
734 DEBUG(
dbgs() <<
"States after initialization:\n");
735 for (
auto Pair : States) {
736 DEBUG(
dbgs() <<
" " << Pair.second <<
" for " << *Pair.first <<
"\n");
742 auto getStateForBDV = [&](
Value *baseValue) {
744 return BDVState(baseValue);
745 auto I = States.find(baseValue);
746 assert(I != States.end() &&
"lookup failed!");
750 bool Progress =
true;
753 const size_t OldSize = States.size();
760 for (
auto Pair : States) {
761 Value *BDV = Pair.first;
766 auto getStateForInput = [&](
Value *V)
mutable {
768 return getStateForBDV(BDV);
773 NewState =
meetBDVState(NewState, getStateForInput(
SI->getTrueValue()));
776 }
else if (
PHINode *PN = dyn_cast<PHINode>(BDV)) {
777 for (
Value *Val : PN->incoming_values())
778 NewState =
meetBDVState(NewState, getStateForInput(Val));
779 }
else if (
auto *EE = dyn_cast<ExtractElementInst>(BDV)) {
783 meetBDVState(NewState, getStateForInput(EE->getVectorOperand()));
784 }
else if (
auto *
IE = dyn_cast<InsertElementInst>(BDV)){
787 NewState =
meetBDVState(NewState, getStateForInput(
IE->getOperand(0)));
788 NewState =
meetBDVState(NewState, getStateForInput(
IE->getOperand(1)));
792 auto *SV = cast<ShuffleVectorInst>(BDV);
793 NewState =
meetBDVState(NewState, getStateForInput(SV->getOperand(0)));
794 NewState =
meetBDVState(NewState, getStateForInput(SV->getOperand(1)));
797 BDVState OldState = States[BDV];
798 if (OldState != NewState) {
800 States[BDV] = NewState;
804 assert(OldSize == States.size() &&
805 "fixed point shouldn't be adding any new nodes to state");
809 DEBUG(
dbgs() <<
"States after meet iteration:\n");
810 for (
auto Pair : States) {
811 DEBUG(
dbgs() <<
" " << Pair.second <<
" for " << *Pair.first <<
"\n");
817 for (
auto Pair : States) {
819 BDVState State = Pair.second;
821 assert(!State.isUnknown() &&
"Optimistic algorithm didn't complete!");
827 if (State.isBase() && isa<ExtractElementInst>(
I) &&
828 isa<VectorType>(State.getBaseValue()->getType())) {
829 auto *EE = cast<ExtractElementInst>(
I);
834 State.getBaseValue(), EE->getIndexOperand(),
"base_ee", EE);
836 States[
I] = BDVState(BDVState::Base, BaseInst);
842 assert(!isa<InsertElementInst>(I) || State.isConflict());
844 if (!State.isConflict())
850 if (isa<PHINode>(I)) {
853 assert(NumPreds > 0 &&
"how did we reach here");
861 }
else if (
auto *EE = dyn_cast<ExtractElementInst>(I)) {
866 }
else if (
auto *
IE = dyn_cast<InsertElementInst>(I)) {
873 auto *SV = cast<ShuffleVectorInst>(
I);
880 Instruction *BaseInst = MakeBaseInstPlaceholder(I);
883 States[
I] = BDVState(BDVState::Conflict, BaseInst);
896 Value *Base =
nullptr;
901 assert(States.count(BDV));
902 Base = States[BDV].getBaseValue();
904 assert(Base &&
"Can't be null");
914 for (
auto Pair : States) {
916 BDVState State = Pair.second;
919 assert(!State.isUnknown() &&
"Optimistic algorithm didn't complete!");
920 if (!State.isConflict())
923 if (
PHINode *BasePHI = dyn_cast<PHINode>(State.getBaseValue())) {
924 PHINode *PN = cast<PHINode>(BDV);
926 for (
unsigned i = 0;
i < NumPHIValues;
i++) {
939 int BlockIndex = BasePHI->getBasicBlockIndex(InBB);
940 if (BlockIndex != -1) {
941 Value *OldBase = BasePHI->getIncomingValue(BlockIndex);
942 BasePHI->addIncoming(OldBase, InBB);
945 Value *Base = getBaseForInput(InVal,
nullptr);
953 "Sanity -- findBaseOrBDV should be pure!");
962 BasePHI->addIncoming(Base, InBB);
964 assert(BasePHI->getNumIncomingValues() == NumPHIValues);
966 dyn_cast<SelectInst>(State.getBaseValue())) {
971 BaseSI->setTrueValue(getBaseForInput(SI->
getTrueValue(), BaseSI));
972 BaseSI->setFalseValue(getBaseForInput(SI->
getFalseValue(), BaseSI));
973 }
else if (
auto *BaseEE =
974 dyn_cast<ExtractElementInst>(State.getBaseValue())) {
975 Value *InVal = cast<ExtractElementInst>(BDV)->getVectorOperand();
978 BaseEE->setOperand(0, getBaseForInput(InVal, BaseEE));
979 }
else if (
auto *BaseIE = dyn_cast<InsertElementInst>(State.getBaseValue())){
980 auto *BdvIE = cast<InsertElementInst>(BDV);
981 auto UpdateOperand = [&](
int OperandIdx) {
982 Value *InVal = BdvIE->getOperand(OperandIdx);
983 Value *Base = getBaseForInput(InVal, BaseIE);
984 BaseIE->setOperand(OperandIdx, Base);
989 auto *BaseSV = cast<ShuffleVectorInst>(State.getBaseValue());
990 auto *BdvSV = cast<ShuffleVectorInst>(BDV);
991 auto UpdateOperand = [&](
int OperandIdx) {
992 Value *InVal = BdvSV->getOperand(OperandIdx);
993 Value *Base = getBaseForInput(InVal, BaseSV);
994 BaseSV->setOperand(OperandIdx, Base);
1004 for (
auto Pair : States) {
1005 auto *BDV = Pair.first;
1006 Value *Base = Pair.second.getBaseValue();
1010 DEBUG(
dbgs() <<
"Updating base value cache"
1011 <<
" for: " << BDV->getName() <<
" from: "
1012 << (Cache.count(BDV) ? Cache[BDV]->getName().str() :
"none")
1013 <<
" to: " << Base->
getName() <<
"\n");
1015 if (Cache.count(BDV)) {
1017 "must be something we 'know' is a base pointer");
1021 "base relation should be stable");
1025 assert(Cache.count(Def));
1048 for (
Value *ptr : live) {
1050 assert(base &&
"failed to find base pointer");
1051 PointerToBase[ptr] = base;
1052 assert((!isa<Instruction>(base) || !isa<Instruction>(ptr) ||
1053 DT->
dominates(cast<Instruction>(base)->getParent(),
1054 cast<Instruction>(ptr)->
getParent())) &&
1055 "The base we found better dominate the derived pointer");
1063 PartiallyConstructedSafepointRecord &result) {
1068 errs() <<
"Base Pairs (w/o Relocation):\n";
1069 for (
auto &Pair : PointerToBase) {
1070 errs() <<
" derived ";
1071 Pair.first->printAsOperand(
errs(),
false);
1073 Pair.second->printAsOperand(
errs(),
false);
1078 result.PointerToBase = PointerToBase;
1085 PartiallyConstructedSafepointRecord &result);
1092 GCPtrLivenessData RevisedLivenessData;
1094 for (
size_t i = 0;
i < records.
size();
i++) {
1095 struct PartiallyConstructedSafepointRecord &
info = records[
i];
1117 "All PHI nodes should have been removed!");
1129 for (
unsigned Slot = 0; Slot < AS.
getNumSlots(); Slot++) {
1139 if (Attr.hasAttribute(Attribute::ReadNone) ||
1171 const int LiveStart,
1175 if (LiveVariables.
empty())
1179 auto ValIt =
find(LiveVec, Val);
1180 assert(ValIt != LiveVec.
end() &&
"Val not found in LiveVec!");
1181 size_t Index = std::distance(LiveVec.
begin(), ValIt);
1182 assert(Index < LiveVec.
size() &&
"Bug in std::find?");
1194 auto getGCRelocateDecl = [&] (
Type *Ty) {
1196 auto AS = Ty->getScalarType()->getPointerAddressSpace();
1198 if (
auto *VT = dyn_cast<VectorType>(Ty))
1209 for (
unsigned i = 0;
i < LiveVariables.
size();
i++) {
1212 Builder.
getInt32(LiveStart + FindIndex(LiveVariables, BasePtrs[
i]));
1215 Type *Ty = LiveVariables[
i]->getType();
1216 if (!TypeToDeclMap.
count(Ty))
1217 TypeToDeclMap[Ty] = getGCRelocateDecl(Ty);
1218 Value *GCRelocateDecl = TypeToDeclMap[Ty];
1222 GCRelocateDecl, {StatepointToken, BaseIdx, LiveIdx},
1234 class DeferredReplacement {
1237 bool IsDeoptimize =
false;
1239 DeferredReplacement() {}
1243 assert(Old != New && Old && New &&
1244 "Cannot RAUW equal values or to / from null!");
1246 DeferredReplacement
D;
1252 static DeferredReplacement createDelete(
Instruction *ToErase) {
1253 DeferredReplacement
D;
1258 static DeferredReplacement createDeoptimizeReplacement(
Instruction *Old) {
1261 assert(
F &&
F->getIntrinsicID() == Intrinsic::experimental_deoptimize &&
1262 "Only way to construct a deoptimize deferred replacement");
1264 DeferredReplacement
D;
1266 D.IsDeoptimize =
true;
1271 void doReplacement() {
1275 assert(OldI != NewI &&
"Disallowed at construction?!");
1276 assert((!IsDeoptimize || !New) &&
1277 "Deoptimize instrinsics are not replaced!");
1299 const char *DeoptLowering =
"deopt-lowering";
1312 return "live-through";
1320 PartiallyConstructedSafepointRecord &Result,
1321 std::vector<DeferredReplacement> &Replacements) {
1339 if (
auto TransitionBundle =
1342 TransitionArgs = TransitionBundle->Inputs;
1348 bool IsDeoptimize =
false;
1359 if (DeoptLowering.
equals(
"live-in"))
1362 assert(DeoptLowering.
equals(
"live-through") &&
"Unsupported value!");
1366 if (
Function *
F = dyn_cast<Function>(CallTarget)) {
1367 if (
F->getIntrinsicID() == Intrinsic::experimental_deoptimize) {
1373 for (
Value *Arg : CallArgs)
1383 F->getParent()->getOrInsertFunction(
"__llvm_deoptimize", FTy);
1385 IsDeoptimize =
true;
1395 StatepointID, NumPatchBytes, CallTarget, Flags, CallArgs,
1396 TransitionArgs, DeoptArgs, GCArgs,
"safepoint_token");
1423 StatepointID, NumPatchBytes, CallTarget, ToReplace->
getNormalDest(),
1425 GCArgs,
"statepoint_token");
1443 "can't safely insert in this block!");
1450 Result.UnwindToken = ExceptionalToken;
1460 "can't safely insert in this block!");
1467 assert(Token &&
"Should be set in one of the above branches!");
1473 Replacements.push_back(
1474 DeferredReplacement::createDeoptimizeReplacement(CS.
getInstruction()));
1476 Token->
setName(
"statepoint_token");
1489 Replacements.emplace_back(
1492 Replacements.emplace_back(
1497 Result.StatepointToken =
Token;
1511 PartiallyConstructedSafepointRecord &Result,
1512 std::vector<DeferredReplacement> &Replacements) {
1513 const auto &LiveSet = Result.LiveSet;
1514 const auto &PointerToBase = Result.PointerToBase;
1518 LiveVec.
reserve(LiveSet.size());
1519 BaseVec.
reserve(LiveSet.size());
1520 for (
Value *
L : LiveSet) {
1522 assert(PointerToBase.count(
L));
1523 Value *Base = PointerToBase.find(
L)->second;
1543 for (
User *U : GCRelocs) {
1550 Value *Alloca = AllocaMap[OriginalValue];
1556 "Should always have one since it's not a terminator");
1558 Value *CastedRelocatedValue =
1559 Builder.CreateBitCast(Relocate,
1560 cast<AllocaInst>(Alloca)->getAllocatedType(),
1564 Store->
insertAfter(cast<Instruction>(CastedRelocatedValue));
1567 VisitedLiveValues.
insert(OriginalValue);
1579 for (
auto RematerializedValuePair: RematerializedValues) {
1580 Instruction *RematerializedValue = RematerializedValuePair.first;
1581 Value *OriginalValue = RematerializedValuePair.second;
1584 "Can not find alloca for rematerialized value");
1585 Value *Alloca = AllocaMap[OriginalValue];
1591 VisitedLiveValues.
insert(OriginalValue);
1603 int InitialAllocaNum = 0;
1605 if (isa<AllocaInst>(
I))
1613 std::size_t NumRematerializedValues = 0;
1618 auto emitAllocaFor = [&](
Value *LiveValue) {
1621 AllocaMap[LiveValue] = Alloca;
1626 for (
Value *V : Live)
1630 for (
const auto &Info : Records)
1631 for (
auto RematerializedValuePair : Info.RematerializedValues) {
1632 Value *OriginalValue = RematerializedValuePair.second;
1633 if (AllocaMap.
count(OriginalValue) != 0)
1636 emitAllocaFor(OriginalValue);
1637 ++NumRematerializedValues;
1649 for (
const auto &Info : Records) {
1660 if (isa<InvokeInst>(Statepoint)) {
1676 for (
auto Pair : AllocaMap) {
1678 AllocaInst *Alloca = cast<AllocaInst>(Pair.second);
1681 if (VisitedLiveValues.
count(Def)) {
1688 for (
auto *AI : ToClobber) {
1689 auto PT = cast<PointerType>(AI->getAllocatedType());
1698 if (
auto II = dyn_cast<InvokeInst>(Statepoint)) {
1699 InsertClobbersAt(&*II->getNormalDest()->getFirstInsertionPt());
1700 InsertClobbersAt(&*II->getUnwindDest()->getFirstInsertionPt());
1702 InsertClobbersAt(cast<Instruction>(Statepoint)->getNextNode());
1708 for (
auto Pair : AllocaMap) {
1710 Value *Alloca = Pair.second;
1719 if (!isa<ConstantExpr>(U)) {
1729 std::sort(Uses.
begin(), Uses.
end());
1730 auto Last = std::unique(Uses.
begin(), Uses.
end());
1734 if (isa<PHINode>(
Use)) {
1745 Use->replaceUsesOfWith(Def, Load);
1753 if (
Instruction *Inst = dyn_cast<Instruction>(Def)) {
1754 if (
InvokeInst *Invoke = dyn_cast<InvokeInst>(Inst)) {
1757 BasicBlock *NormalDest = Invoke->getNormalDest();
1760 assert(!Inst->isTerminator() &&
1761 "The only TerminatorInst that can produce a value is "
1762 "InvokeInst which is handled above.");
1766 assert(isa<Argument>(Def));
1771 assert(PromotableAllocas.
size() == Live.size() + NumRematerializedValues &&
1772 "we must have the same allocas with lives");
1773 if (!PromotableAllocas.
empty()) {
1780 if (isa<AllocaInst>(
I))
1782 assert(InitialAllocaNum == 0 &&
"We must not introduce any extra allocas");
1817 Func, Values,
"", &*II->getNormalDest()->getFirstInsertionPt()));
1819 Func, Values,
"", &*II->getUnwindDest()->getFirstInsertionPt()));
1825 GCPtrLivenessData OriginalLivenessData;
1827 for (
size_t i = 0;
i < records.
size();
i++) {
1828 struct PartiallyConstructedSafepointRecord &
info = records[
i];
1841 Value *CurrentValue) {
1846 GEP->getPointerOperand());
1849 if (
CastInst *CI = dyn_cast<CastInst>(CurrentValue)) {
1850 if (!CI->isNoopCast(CI->getModule()->getDataLayout()))
1860 return CurrentValue;
1871 if (
CastInst *CI = dyn_cast<CastInst>(Instr)) {
1872 assert(CI->isNoopCast(CI->getModule()->getDataLayout()) &&
1873 "non noop cast is found during rematerialization");
1875 Type *SrcTy = CI->getOperand(0)->getType();
1880 Type *ValTy =
GEP->getSourceElementType();
1886 if (!
GEP->hasAllConstantIndices())
1906 for (
unsigned i = 0;
i < PhiNum;
i++)
1912 for (
unsigned i = 0;
i < PhiNum;
i++) {
1915 if (CIVI == CurrentIncomingValues.
end())
1917 BasicBlock *CurrentIncomingBB = CIVI->second;
1930 PartiallyConstructedSafepointRecord &Info,
1932 const unsigned int ChainLengthThreshold = 10;
1938 for (
Value *LiveValue: Info.LiveSet) {
1941 assert(Info.PointerToBase.count(LiveValue));
1942 Value *RootOfChain =
1947 if ( ChainToBase.
size() == 0 ||
1948 ChainToBase.
size() > ChainLengthThreshold)
1953 if (RootOfChain != Info.PointerToBase[LiveValue]) {
1956 if (!OrigRootPhi || !AlternateRootPhi)
1972 assert(Info.LiveSet.count(AlternateRootPhi));
1991 LiveValuesToBeDeleted.
push_back(LiveValue);
2001 auto rematerializeChain = [&ChainToBase](
2010 assert(isa<GetElementPtrInst>(Instr) || isa<CastInst>(Instr));
2014 ClonedValue->
setName(Instr->getName() +
".remat");
2018 if (LastClonedValue) {
2026 "incorrect use in rematerialization chain");
2029 assert(OpValue != RootOfChain && OpValue != AlternateLiveBase);
2038 if (RootOfChain != AlternateLiveBase)
2042 LastClonedValue = ClonedValue;
2046 return LastClonedValue;
2054 Instruction *RematerializedValue = rematerializeChain(
2055 InsertBefore, RootOfChain, Info.PointerToBase[LiveValue]);
2056 Info.RematerializedValues[RematerializedValue] = LiveValue;
2065 Instruction *NormalRematerializedValue = rematerializeChain(
2066 NormalInsertBefore, RootOfChain, Info.PointerToBase[LiveValue]);
2067 Instruction *UnwindRematerializedValue = rematerializeChain(
2068 UnwindInsertBefore, RootOfChain, Info.PointerToBase[LiveValue]);
2070 Info.RematerializedValues[NormalRematerializedValue] = LiveValue;
2071 Info.RematerializedValues[UnwindRematerializedValue] = LiveValue;
2076 for (
auto LiveValue: LiveValuesToBeDeleted) {
2077 Info.LiveSet.remove(LiveValue);
2086 std::set<CallSite> Uniqued;
2087 Uniqued.insert(ToUpdate.
begin(), ToUpdate.
end());
2088 assert(Uniqued.size() == ToUpdate.
size() &&
"no duplicates please!");
2091 assert(CS.getInstruction()->getFunction() == &
F);
2101 auto *II = cast<InvokeInst>(CS.getInstruction());
2119 "support for FCA unimplemented");
2140 for (
size_t i = 0;
i < Records.size();
i++) {
2141 PartiallyConstructedSafepointRecord &
info = Records[
i];
2160 for (
size_t i = 0;
i < Records.size();
i++) {
2161 PartiallyConstructedSafepointRecord &Info = Records[
i];
2164 for (
auto Pair : Info.PointerToBase)
2176 for (
auto &Info : Records) {
2177 errs() <<
"Base Pairs: (w/Relocation)\n";
2178 for (
auto Pair : Info.PointerToBase) {
2179 errs() <<
" derived ";
2180 Pair.first->printAsOperand(
errs(),
false);
2182 Pair.second->printAsOperand(
errs(),
false);
2196 for (
auto &Info : Records)
2197 for (
auto &BasePair : Info.PointerToBase)
2198 if (isa<Constant>(BasePair.second))
2199 Info.LiveSet.remove(BasePair.first);
2202 CI->eraseFromParent();
2209 for (
size_t i = 0;
i < Records.size();
i++)
2215 std::vector<DeferredReplacement> Replacements;
2223 for (
size_t i = 0; i < Records.size(); i++)
2228 for (
auto &PR : Replacements)
2231 Replacements.clear();
2233 for (
auto &Info : Records) {
2242 Info.LiveSet.clear();
2243 Info.PointerToBase.clear();
2248 for (
size_t i = 0; i < Records.size(); i++) {
2249 PartiallyConstructedSafepointRecord &Info = Records[
i];
2265 "statepoint must be reachable or liveness is meaningless");
2267 if (!isa<Instruction>(V))
2270 auto *LiveInst = cast<Instruction>(V);
2272 "unreachable values should never be live");
2274 "basic SSA liveness expectation violated by liveness analysis");
2282 for (
auto *
Ptr : Live)
2284 "must be a gc pointer type");
2288 return !Records.empty();
2292 template <
typename AttrHolder>
2296 if (AH.getDereferenceableBytes(Index))
2298 AH.getDereferenceableBytes(Index)));
2299 if (AH.getDereferenceableOrNullBytes(Index))
2301 AH.getDereferenceableOrNullBytes(Index)));
2302 if (AH.doesNotAlias(Index))
2306 AH.setAttributes(AH.getAttributes().removeAttributes(
2307 Ctx, Index, AttributeSet::get(Ctx, Index, R)));
2311 RewriteStatepointsForGC::stripNonValidAttributesFromPrototype(
Function &
F) {
2315 if (isa<PointerType>(
A.getType()))
2322 void RewriteStatepointsForGC::stripNonValidAttributesFromBody(
Function &F) {
2331 assert(MD->getNumOperands() < 5 &&
"unrecognized metadata shape!");
2332 bool IsImmutableTBAA =
2333 MD->getNumOperands() == 4 &&
2334 mdconst::extract<ConstantInt>(MD->getOperand(3))->getValue() == 1;
2336 if (!IsImmutableTBAA)
2339 MDNode *Base = cast<MDNode>(MD->getOperand(0));
2340 MDNode *Access = cast<MDNode>(MD->getOperand(1));
2342 mdconst::extract<ConstantInt>(MD->getOperand(2))->getZExtValue();
2345 Builder.createTBAAStructTagNode(Base, Access, Offset);
2350 for (
int i = 0, e =
CS.arg_size();
i != e;
i++)
2351 if (isa<PointerType>(
CS.getArgument(
i)->getType()))
2353 if (isa<PointerType>(
CS.getType()))
2364 const auto &FunctionGCName = F.
getGC();
2365 const StringRef StatepointExampleName(
"statepoint-example");
2367 return (StatepointExampleName == FunctionGCName) ||
2368 (CoreCLRName == FunctionGCName);
2373 void RewriteStatepointsForGC::stripNonValidAttributes(
Module &M) {
2379 stripNonValidAttributesFromPrototype(F);
2382 stripNonValidAttributesFromBody(F);
2385 bool RewriteStatepointsForGC::runOnFunction(
Function &F) {
2395 DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>(
F).getDomTree();
2397 getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
2409 bool HasUnreachableStatepoint =
false;
2412 if (NeedsRewrite(
I)) {
2416 HasUnreachableStatepoint =
true;
2420 bool MadeChange =
false;
2426 if (HasUnreachableStatepoint)
2430 if (ParsePointNeeded.
empty())
2438 if (BB.getUniquePredecessor()) {
2457 if (
auto *BI = dyn_cast<BranchInst>(TI))
2458 if (BI->isConditional())
2459 return dyn_cast<Instruction>(BI->getCondition());
2465 if (
auto *Cond = getConditionInst(TI))
2468 if (isa<ICmpInst>(Cond) && Cond->hasOneUse()) {
2470 Cond->moveBefore(TI);
2495 if (isa<PHINode>(
I))
2499 for (
Value *V :
I.operands()) {
2501 "support for FCA unimplemented");
2521 for (
auto &
I : *Succ) {
2528 "support for FCA unimplemented");
2548 for (
Value *V : Live) {
2549 if (
auto *
I = dyn_cast<Instruction>(V)) {
2553 if (TermOkay && TI ==
I)
2556 "basic SSA liveness expectation violated by liveness analysis");
2573 GCPtrLivenessData &Data) {
2579 Data.LiveSet[&BB].clear();
2584 assert(!Data.LiveSet[&BB].count(
Kill) &&
"live set contains kill");
2589 Data.LiveIn[&BB] = Data.LiveSet[&BB];
2590 Data.LiveIn[&BB].set_union(Data.LiveOut[&BB]);
2591 Data.LiveIn[&BB].set_subtract(Data.KillSet[&BB]);
2592 if (!Data.LiveIn[&BB].empty())
2597 while (!Worklist.
empty()) {
2603 const auto OldLiveOutSize = LiveOut.
size();
2605 assert(Data.LiveIn.count(Succ));
2609 if (OldLiveOutSize == LiveOut.
size()) {
2615 Data.LiveOut[BB] = LiveOut;
2622 assert(Data.LiveIn.count(BB));
2625 if (OldLiveIn.size() != LiveTmp.
size()) {
2626 Data.LiveIn[BB] = LiveTmp;
2645 assert(Data.LiveOut.count(BB));
2654 LiveOut.remove(Inst);
2655 Out.insert(LiveOut.begin(), LiveOut.end());
2660 PartiallyConstructedSafepointRecord &Info) {
2667 for (
auto KVPair : Info.PointerToBase)
2668 Bases.
insert(KVPair.second);
2673 for (
auto V : Updated)
2674 if (Info.PointerToBase.insert({V, V}).second) {
2675 assert(Bases.count(V) &&
"Can't find base for unexpected live value!");
2680 for (
auto V : Updated)
2681 assert(Info.PointerToBase.count(V) &&
2682 "Must be able to find base for live value!");
2688 for (
auto KVPair : Info.PointerToBase)
2689 if (!Updated.count(KVPair.first))
2690 ToErase.
insert(KVPair.first);
2692 for (
auto *V : ToErase)
2693 Info.PointerToBase.erase(V);
2696 for (
auto KVPair : Info.PointerToBase)
2697 assert(Updated.count(KVPair.first) &&
"record for non-live value");
2700 Info.LiveSet = Updated;
AttributeSet getAttributes() const
Return the parameter attributes for this call.
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
static void findLiveReferences(Function &F, DominatorTree &DT, ArrayRef< CallSite > toUpdate, MutableArrayRef< struct PartiallyConstructedSafepointRecord > records)
void push_back(const T &Elt)
static void unique_unsorted(SmallVectorImpl< T > &Vec)
Implement a unique function which doesn't require we sort the input vector.
static bool isHandledGCPointerType(Type *T)
BasicBlock * getUniquePredecessor()
Return the predecessor of this block if it has a unique predecessor block.
static cl::opt< bool, true > ClobberNonLiveOverride("rs4gc-clobber-non-live", cl::location(ClobberNonLive), cl::Hidden)
raw_ostream & errs()
This returns a reference to a raw_ostream for standard error.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
LLVM Argument representation.
static Value * findBaseDefiningValueCached(Value *I, DefiningValueMapTy &Cache)
Returns the base defining value for this value.
static bool AreEquivalentPhiNodes(PHINode &OrigRootPhi, PHINode &AlternateRootPhi)
static BDVState meetBDVStateImpl(const BDVState &LHS, const BDVState &RHS)
static bool insertParsePoints(Function &F, DominatorTree &DT, TargetTransformInfo &TTI, SmallVectorImpl< CallSite > &ToUpdate)
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
LLVM_NODISCARD T pop_back_val()
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds...
Instruction * StatepointToken
The new gc.statepoint instruction itself.
AttributeSet getAttributes() const
Return the parameter attributes for this invoke.
Instruction * UnwindToken
Instruction to which exceptional gc relocates are attached Makes it easier to iterate through them du...
A Module instance is used to store all the information related to an LLVM module. ...
auto remove_if(R &&Range, UnaryPredicate P) -> decltype(std::begin(Range))
Provide wrappers to std::remove_if which take ranges instead of having to pass begin/end explicitly...
static ConstantAggregateZero * get(Type *Ty)
static void rematerializeLiveValues(CallSite CS, PartiallyConstructedSafepointRecord &Info, TargetTransformInfo &TTI)
static void computeLiveOutSeed(BasicBlock *BB, SetVector< Value * > &LiveTmp)
static StringRef getDeoptLowering(CallSite CS)
ilist_iterator< OptionsT,!IsReverse, IsConst > getReverse() const
Get a reverse iterator to the same node.
This class represents a function call, abstracting a target machine's calling convention.
MapVector< BasicBlock *, SetVector< Value * > > KillSet
Values defined in this block.
unsigned gcArgsStartIdx() const
InvokeInst * CreateGCStatepointInvoke(uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef< Value * > InvokeArgs, ArrayRef< Value * > DeoptArgs, ArrayRef< Value * > GCArgs, const Twine &Name="")
brief Create an invoke to the experimental.gc.statepoint intrinsic to start a new statepoint sequence...
This instruction constructs a fixed permutation of two input vectors.
static SelectInst * Create(Value *C, Value *S1, Value *S2, const Twine &NameStr="", Instruction *InsertBefore=nullptr, Instruction *MDFrom=nullptr)
MapVector< BasicBlock *, SetVector< Value * > > LiveSet
Values used in this block (and thus live); does not included values killed within this block...
iterator begin(unsigned Slot) const
MapVector< BasicBlock *, SetVector< Value * > > LiveIn
Values live into this basic block (i.e.
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
Type * getReturnType() const
Returns the type of the ret val.
This class implements a map that also provides access to all stored values in a deterministic order...
The two locations do not alias at all.
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
An instruction for reading from memory.
reverse_iterator rbegin()
static BDVState meetBDVState(BDVState LHS, BDVState RHS)
AttrBuilder & addAttribute(Attribute::AttrKind Val)
Add an attribute to the builder.
static BaseDefiningValueResult findBaseDefiningValueOfVector(Value *I)
Return a base defining value for the 'Index' element of the given vector instruction 'I'...
void reserve(size_type N)
bool hasAttribute(unsigned Index, Attribute::AttrKind Kind) const
Return true if the attribute exists at the given index.
size_type size() const
Determine the number of elements in the SetVector.
static void computeLiveInValues(DominatorTree &DT, Function &F, GCPtrLivenessData &Data)
Compute the live-in set for every basic block in the function.
AttributeSet getRetAttributes() const
The attributes for the ret value are returned.
StringRef getName() const
Return a constant reference to the value's name.
iterator begin()
Instruction iterator methods.
void setCallingConv(CallingConv::ID CC)
static void relocationViaAlloca(Function &F, DominatorTree &DT, ArrayRef< Value * > Live, ArrayRef< PartiallyConstructedSafepointRecord > Records)
Do all the relocation update via allocas and mem2reg.
Value * getDerivedPtr() const
AnalysisUsage & addRequired()
static InsertElementInst * Create(Value *Vec, Value *NewElt, Value *Idx, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
rewrite statepoints for Make relocations at false
#define INITIALIZE_PASS_DEPENDENCY(depName)
static unsigned chainToBasePointerCost(SmallVectorImpl< Instruction * > &Chain, TargetTransformInfo &TTI)
bool hasGC() const
hasGC/getGC/setGC/clearGC - The name of the garbage collection algorithm to use during code generatio...
This class represents the LLVM 'select' instruction.
struct fuzzer::@269 Flags
This is the base class for all instructions that perform data casts.
'undef' values are things that do not have specified contents.
static bool isKnownBaseResult(Value *V)
Given the result of a call to findBaseDefiningValue, or findBaseOrBDV, is it known to be a base point...
Class to represent struct types.
static void CreateGCRelocates(ArrayRef< Value * > LiveVariables, const int LiveStart, ArrayRef< Value * > BasePtrs, Instruction *StatepointToken, IRBuilder<> Builder)
Helper function to place all gc relocates necessary for the given statepoint.
static Value * findBasePointer(Value *I, DefiningValueMapTy &Cache)
For a given value or instruction, figure out what base ptr its derived from.
A Use represents the edge between a Value definition and its users.
ValTy * getCalledValue() const
getCalledValue - Return the pointer to function that is being called.
static GCRegistry::Add< StatepointGC > D("statepoint-example","an example strategy for statepoint")
Instruction * getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
static bool isGCPointerType(Type *T)
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
bool isCall() const
isCall - true if a CallInst is enclosed.
static void makeStatepointExplicit(DominatorTree &DT, CallSite CS, PartiallyConstructedSafepointRecord &Result, std::vector< DeferredReplacement > &Replacements)
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE bool equals(StringRef RHS) const
equals - Check for string equality, this is more efficient than compare() when the relative ordering ...
void setName(const Twine &Name)
Change the name of the value.
Instruction * clone() const
Create a copy of 'this' instruction that is identical in all ways except the following: ...
bool remove(const value_type &X)
Remove an item from the set vector.
LLVM_NODISCARD bool empty() const
auto reverse(ContainerTy &&C, typename std::enable_if< has_rbegin< ContainerTy >::value >::type *=nullptr) -> decltype(make_range(C.rbegin(), C.rend()))
static cl::opt< bool > PrintLiveSet("spp-print-liveset", cl::Hidden, cl::init(false))
void PromoteMemToReg(ArrayRef< AllocaInst * > Allocas, DominatorTree &DT, AliasSetTracker *AST=nullptr, AssumptionCache *AC=nullptr)
Promote the specified list of alloca instructions into scalar registers, inserting PHI nodes as appro...
bool insert(const value_type &X)
Insert a new element into the SetVector.
static void insertUseHolderAfter(CallSite &CS, const ArrayRef< Value * > Values, SmallVectorImpl< CallInst * > &Holders)
Insert holders so that each Value is obviously live through the entire lifetime of the call...
const T & getValue() const LLVM_LVALUE_FUNCTION
Class to represent array types.
bool isReachableFromEntry(const Use &U) const
Provide an overload for a Use.
This class represents a no-op cast from one type to another.
bool empty() const
Determine if the SetVector is empty or not.
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=None)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
An instruction for storing to memory.
void SetCurrentDebugLocation(DebugLoc L)
Set location information used by debugging information.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
static Value * findRematerializableChainToBasePointer(SmallVectorImpl< Instruction * > &ChainToBase, Value *CurrentValue)
StatepointLiveSetTy LiveSet
The set of values known to be live across this safepoint.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
size_t size() const
size - Get the array size.
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block...
BasicBlock * getNormalDest() const
void setAttributes(AttributeSet Attrs)
Set the parameter attributes for this call.
unsigned getNumIncomingValues() const
Return the number of incoming edges.
void replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
static SetVector< Value * > computeKillSet(BasicBlock *BB)
bool isInvoke() const
isInvoke - true if a InvokeInst is enclosed.
CallSiteTy::arg_iterator gc_args_begin() const
an instruction for type-safe pointer arithmetic to access elements of arrays and structs ...
unsigned getNumSlots() const
Return the number of slots used in this attribute list.
initializer< Ty > init(const Ty &Val)
static void RemoveNonValidAttrAtIndex(LLVMContext &Ctx, AttrHolder &AH, unsigned Index)
Optional< OperandBundleUse > getOperandBundle(StringRef Name) const
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
Subclasses of this class are all able to terminate a basic block.
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
static ArrayRef< Use > GetDeoptBundleOperands(ImmutableCallSite CS)
Constant * getOrInsertFunction(StringRef Name, FunctionType *T, AttributeSet AttributeList)
Look up the specified function in the module symbol table.
void insertBefore(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified instruction...
MapVector< Value *, Value * > DefiningValueMapTy
LLVM Basic Block Representation.
The instances of the Type class are immutable: once they are created, they are never changed...
This is an important class for using LLVM in a threaded context.
bool isVectorTy() const
True if this is an instance of VectorType.
This function has undefined behavior.
This is an important base class in LLVM.
bool set_union(const STy &S)
Compute This := This u S, return whether 'This' changed.
static void analyzeParsePointLiveness(DominatorTree &DT, GCPtrLivenessData &OriginalLivenessData, CallSite CS, PartiallyConstructedSafepointRecord &Result)
CallSiteTy::arg_iterator gc_args_end() const
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin()
static cl::opt< unsigned > RematerializationThreshold("spp-rematerialization-threshold", cl::Hidden, cl::init(6))
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
std::pair< iterator, bool > insert(const ValueT &V)
LandingPadInst * getLandingPadInst()
Return the landingpad instruction associated with the landing pad.
iterator_range< arg_iterator > gc_args() const
range adapter for gc arguments
CallInst * CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee, ArrayRef< Value * > CallArgs, ArrayRef< Value * > DeoptArgs, ArrayRef< Value * > GCArgs, const Twine &Name="")
Create a call to the experimental.gc.statepoint intrinsic to start a new statepoint sequence...
Interval::pred_iterator pred_begin(Interval *I)
pred_begin/pred_end - define methods so that Intervals may be used just like BasicBlocks can with the...
Optional< uint32_t > NumPatchBytes
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Represent the analysis usage information of a pass.
static Type * getVoidTy(LLVMContext &C)
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
static void insertRelocationStores(iterator_range< Value::user_iterator > GCRelocs, DenseMap< Value *, Value * > &AllocaMap, DenseSet< Value * > &VisitedLiveValues)
bool any_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly...
RematerializedValueMapTy RematerializedValues
Record live values we are rematerialized instead of relocating.
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE,"Assign register bank of generic virtual registers", false, false) RegBankSelect
static const unsigned End
for(unsigned i=0, e=MI->getNumOperands();i!=e;++i)
A specialization of it's base class for read-write access to a gc.statepoint.
bool hasFnAttr(Attribute::AttrKind Kind) const
Return true if this function has the given attribute.
Interval::pred_iterator pred_end(Interval *I)
self_iterator getIterator()
SetVector< Value * > StatepointLiveSetTy
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
bool callsGCLeafFunction(ImmutableCallSite CS)
Return true if the CallSite CS calls a gc leaf function.
rewrite statepoints for gc
bool empty() const
empty - Check if the array is empty.
void setTailCallKind(TailCallKind TCK)
static bool containsGCPtrType(Type *Ty)
Returns true if this type contains a gc pointer whether we know how to handle that type or not...
LLVM_NODISCARD std::string str() const
str - Get the contents as an std::string.
Type * getType() const
getType - Return the type of the instruction that generated this call site
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
iterator erase(const_iterator CI)
StatepointDirectives parseStatepointDirectivesFromAttrs(AttributeSet AS)
Parse out statepoint directives from the function attributes present in AS.
static void findLiveSetAtInst(Instruction *inst, GCPtrLivenessData &Data, StatepointLiveSetTy &out)
Given results from the dataflow liveness computation, find the set of live Values at a particular ins...
LLVMContext & getContext() const
All values hold a context through their type.
INITIALIZE_PASS_BEGIN(RewriteStatepointsForGC,"rewrite-statepoints-for-gc","Make relocations explicit at statepoints", false, false) INITIALIZE_PASS_END(RewriteStatepointsForGC
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
const Value * getTrueValue() const
static Value * findBaseOrBDV(Value *I, DefiningValueMapTy &Cache)
Return a base pointer for this value if known.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
static bool shouldRewriteStatepointsIn(Function &F)
Returns true if this function should be rewritten by this pass.
CallingConv::ID getCallingConv() const
getCallingConv/setCallingConv - Get or set the calling convention of this function call...
bool dominates(const Instruction *Def, const Use &U) const
Return true if Def dominates a use in User.
AttributeSet getAttributes() const
getAttributes/setAttributes - get or set the parameter attributes of the call.
static void findBasePointers(const StatepointLiveSetTy &live, MapVector< Value *, Value * > &PointerToBase, DominatorTree *DT, DefiningValueMapTy &DVCache)
LLVMContext & getContext() const
Retrieve the LLVM context.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
static cl::opt< bool > PrintBasePointers("spp-print-base-pointers", cl::Hidden, cl::init(false))
static cl::opt< bool > AllowStatepointWithNoDeoptInfo("rs4gc-allow-statepoint-with-no-deopt-info", cl::Hidden, cl::init(true))
rewrite statepoints for Make relocations at statepoints
A SetVector that performs no allocations if smaller than a certain size.
Iterator for intrusive lists based on ilist_node.
BasicBlock * getUnwindDest() const
MapVector< AssertingVH< Instruction >, AssertingVH< Value > > RematerializedValueMapTy
auto find(R &&Range, const T &Val) -> decltype(std::begin(Range))
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly...
InstrTy * getInstruction() const
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
bool removeUnreachableBlocks(Function &F, LazyValueInfo *LVI=nullptr)
Remove all blocks that can not be reached from the function's entry.
static AttributeSet legalizeCallAttributes(AttributeSet AS)
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
static CallInst * Create(Value *Func, ArrayRef< Value * > Args, ArrayRef< OperandBundleDef > Bundles=None, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Module.h This file contains the declarations for the Module class.
Type * getType() const
All values are typed, get the type of this value.
void initializeRewriteStatepointsForGCPass(PassRegistry &)
iterator end(unsigned Slot) const
TailCallKind getTailCallKind() const
Indicates that this statepoint is a transition from GC-aware code to code that is not GC-aware...
bool empty() const
Return true if the builder contains no target-independent attributes.
LLVM_NODISCARD T pop_back_val()
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Value * stripPointerCasts()
Strip off pointer casts, all-zero GEPs, and aliases.
static BasicBlock * normalizeForInvokeSafepoint(BasicBlock *BB, BasicBlock *InvokeParent, DominatorTree &DT)
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
const BasicBlock & getEntryBlock() const
void set_subtract(const STy &S)
Compute This := This - B TODO: We should be able to use set_subtract from SetOperations.h, but SetVector interface is inconsistent with DenseSet.
Value handle that asserts if the Value is deleted.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
size_type count(const KeyT &Val) const
Return 1 if the specified key is in the map, 0 otherwise.
static bool ClobberNonLive
static void checkBasicSSA(DominatorTree &DT, SetVector< Value * > &Live, TerminatorInst *TI, bool TermOkay=false)
Check that the items in 'Live' dominate 'TI'.
A range adaptor for a pair of iterators.
Class to represent vector types.
static std::string suffixed_name_or(Value *V, StringRef Suffix, StringRef DefaultName)
bool isStatepointDirectiveAttr(Attribute Attr)
Return true if the the Attr is an attribute that is a statepoint directive.
Optional< uint64_t > StatepointID
Value * getIncomingValueForBlock(const BasicBlock *BB) const
iterator_range< user_iterator > users()
static const uint64_t DefaultStatepointID
void FoldSingleEntryPHINodes(BasicBlock *BB, MemoryDependenceResults *MemDep=nullptr)
We know that BB has one predecessor.
iterator insert(iterator I, T &&Elt)
Call sites that get wrapped by a gc.statepoint (currently only in RewriteStatepointsForGC and potenti...
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
bool operator!=(uint64_t V1, const APInt &V2)
static void insertRematerializationStores(const RematerializedValueMapTy &RematerializedValues, DenseMap< Value *, Value * > &AllocaMap, DenseSet< Value * > &VisitedLiveValues)
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
static cl::opt< bool > PrintLiveSetSize("spp-print-liveset-size", cl::Hidden, cl::init(false))
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
size_type count(const ValueT &V) const
Return 1 if the specified key is in the set, 0 otherwise.
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
unsigned getSlotIndex(unsigned Slot) const
Return the index for the given slot.
static bool isUnhandledGCPointerType(Type *Ty)
ImmutableCallSite - establish a view to a call site for examination.
void insertAfter(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately after the specified instruction...
const std::string & getGC() const
TerminatorInst * getTerminator()
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
CallInst * CreateGCResult(Instruction *Statepoint, Type *ResultType, const Twine &Name="")
Create a call to the experimental.gc.result intrinsic to extract the result from a call wrapped in a ...
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
CallInst * CreateCall(Value *Callee, ArrayRef< Value * > Args=None, const Twine &Name="", MDNode *FPMathTag=nullptr)
iterator_range< value_op_iterator > operand_values()
iterator find(const KeyT &Val)
static void makeStatepointExplicitImpl(const CallSite CS, const SmallVectorImpl< Value * > &BasePtrs, const SmallVectorImpl< Value * > &LiveVariables, PartiallyConstructedSafepointRecord &Result, std::vector< DeferredReplacement > &Replacements)
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
raw_ostream & operator<<(raw_ostream &OS, const APInt &I)
static Function * getCalledFunction(const Value *V, bool LookThroughBitCast, bool &IsNoBuiltin)
Attribute getAttribute(unsigned Index, Attribute::AttrKind Kind) const
Return the attribute object that exists at the given index.
BasicBlock * SplitBlockPredecessors(BasicBlock *BB, ArrayRef< BasicBlock * > Preds, const char *Suffix, DominatorTree *DT=nullptr, LoopInfo *LI=nullptr, bool PreserveLCSSA=false)
This method introduces at least one new basic block into the function and moves some of the predecess...
bool isStatepoint(ImmutableCallSite CS)
StringRef getValueAsString() const
Return the attribute's value as a string.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
user_iterator user_begin()
FunTy * getCalledFunction() const
getCalledFunction - Return the function being called if this is a direct call, otherwise return null ...
Represents calls to the gc.relocate intrinsic.
Mark the deopt arguments associated with the statepoint as only being "live-in".
LLVM Value Representation.
succ_range successors(BasicBlock *BB)
ModulePass * createRewriteStatepointsForGCPass()
static VectorType * get(Type *ElementType, unsigned NumElements)
This static method is the primary way to construct an VectorType.
static void recomputeLiveInValues(GCPtrLivenessData &RevisedLivenessData, CallSite CS, PartiallyConstructedSafepointRecord &result)
Given an updated version of the dataflow liveness results, update the liveset and base pointer maps f...
static const Function * getParent(const Value *V)
void setCallingConv(CallingConv::ID CC)
MapVector< BasicBlock *, SetVector< Value * > > LiveOut
Values live out of this basic block (i.e.
This class implements an extremely fast bulk output stream that can only output to a stream...
const Value * getFalseValue() const
StringRef - Represent a constant reference to a string, i.e.
inst_range instructions(Function *F)
CallingConv::ID getCallingConv() const
getCallingConv/setCallingConv - Get or set the calling convention of this function call...
Legacy analysis pass which computes a DominatorTree.
bool operator==(uint64_t V1, const APInt &V2)
iterator getFirstInsertionPt()
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
MapVector< Value *, Value * > PointerToBase
Mapping from live pointers to a base-defining-value.
void setIncomingValue(unsigned i, Value *V)
AttributeSet addAttributes(LLVMContext &C, unsigned Index, AttributeSet Attrs) const
Add attributes to the attribute set at the given index.
static GCRegistry::Add< ErlangGC > A("erlang","erlang-compatible garbage collector")
LocationClass< Ty > location(Ty &L)
const BasicBlock * getParent() const
iterator_range< arg_iterator > args()
A wrapper class for inspecting calls to intrinsic functions.
LLVMContext & getContext() const
Get the global data context.
bool isVoidTy() const
Return true if this is 'void'.
an instruction to allocate memory on the stack
static BaseDefiningValueResult findBaseDefiningValue(Value *I)
Helper function for findBasePointer - Will return a value which either a) defines the base pointer fo...
AttributeSet getFnAttributes() const
The function attributes are returned.
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.