LLVM 20.0.0git
|
#include "llvm/Transforms/Scalar/RewriteStatepointsForGC.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/Sequence.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/iterator_range.h"
#include "llvm/Analysis/DomTreeUpdater.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/IR/Argument.h"
#include "llvm/IR/AttributeMask.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GCStrategy.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstIterator.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/Metadata.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Statepoint.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
#include "llvm/IR/ValueHandle.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Transforms/Utils/PromoteMemToReg.h"
#include <algorithm>
#include <cassert>
#include <cstddef>
#include <cstdint>
#include <iterator>
#include <optional>
#include <set>
#include <string>
#include <utility>
#include <vector>
Go to the source code of this file.
Macros | |
#define | DEBUG_TYPE "rewrite-statepoints-for-gc" |
Functions | |
static void | stripNonValidData (Module &M) |
The IR fed into RewriteStatepointsForGC may have had attributes and metadata implying dereferenceability that are no longer valid/correct after RewriteStatepointsForGC has run. | |
static std::unique_ptr< GCStrategy > | findGCStrategy (Function &F) |
Looks up the GC strategy for a given function, returning null if the function doesn't have a GC tag. | |
static bool | shouldRewriteStatepointsIn (Function &F) |
Returns true if this function should be rewritten by this pass. | |
static ArrayRef< Use > | GetDeoptBundleOperands (const CallBase *Call) |
static void | computeLiveInValues (DominatorTree &DT, Function &F, GCPtrLivenessData &Data, GCStrategy *GC) |
Compute the live-in set for every basic block in the function. | |
static void | findLiveSetAtInst (Instruction *inst, GCPtrLivenessData &Data, StatepointLiveSetTy &out, GCStrategy *GC) |
Given results from the dataflow liveness computation, find the set of live Values at a particular instruction. | |
static bool | isGCPointerType (Type *T, GCStrategy *GC) |
static bool | isHandledGCPointerType (Type *T, GCStrategy *GC) |
static bool | containsGCPtrType (Type *Ty, GCStrategy *GC) |
Returns true if this type contains a gc pointer whether we know how to handle that type or not. | |
static bool | isUnhandledGCPointerType (Type *Ty, GCStrategy *GC) |
static std::string | suffixed_name_or (Value *V, StringRef Suffix, StringRef DefaultName) |
static void | analyzeParsePointLiveness (DominatorTree &DT, GCPtrLivenessData &OriginalLivenessData, CallBase *Call, PartiallyConstructedSafepointRecord &Result, GCStrategy *GC) |
static bool | isKnownBase (Value *V, const IsKnownBaseMapTy &KnownBases) |
Returns true if V is a known base. | |
static void | setKnownBase (Value *V, bool IsKnownBase, IsKnownBaseMapTy &KnownBases) |
Caches the IsKnownBase flag for a value and asserts that it wasn't present in the cache before. | |
static Value * | findBaseDefiningValue (Value *I, DefiningValueMapTy &Cache, IsKnownBaseMapTy &KnownBases) |
Helper function for findBasePointer - Will return a value which either a) defines the base pointer for the input, b) blocks the simple search (i.e. | |
static Value * | findBaseDefiningValueOfVector (Value *I, DefiningValueMapTy &Cache, IsKnownBaseMapTy &KnownBases) |
Return a base defining value for the 'Index' element of the given vector instruction 'I'. | |
static Value * | findBaseDefiningValueCached (Value *I, DefiningValueMapTy &Cache, IsKnownBaseMapTy &KnownBases) |
Returns the base defining value for this value. | |
static Value * | findBaseOrBDV (Value *I, DefiningValueMapTy &Cache, IsKnownBaseMapTy &KnownBases) |
Return a base pointer for this value if known. | |
static bool | isOriginalBaseResult (Value *V) |
This value is a base pointer that is not generated by RS4GC, i.e. | |
static bool | areBothVectorOrScalar (Value *First, Value *Second) |
static raw_ostream & | operator<< (raw_ostream &OS, const BDVState &State) |
static Value * | findBasePointer (Value *I, DefiningValueMapTy &Cache, IsKnownBaseMapTy &KnownBases) |
For a given value or instruction, figure out what base ptr its derived from. | |
static void | findBasePointers (const StatepointLiveSetTy &live, PointerToBaseTy &PointerToBase, DominatorTree *DT, DefiningValueMapTy &DVCache, IsKnownBaseMapTy &KnownBases) |
static void | findBasePointers (DominatorTree &DT, DefiningValueMapTy &DVCache, CallBase *Call, PartiallyConstructedSafepointRecord &result, PointerToBaseTy &PointerToBase, IsKnownBaseMapTy &KnownBases) |
Find the required based pointers (and adjust the live set) for the given parse point. | |
static void | recomputeLiveInValues (GCPtrLivenessData &RevisedLivenessData, CallBase *Call, PartiallyConstructedSafepointRecord &result, PointerToBaseTy &PointerToBase, GCStrategy *GC) |
Given an updated version of the dataflow liveness results, update the liveset and base pointer maps for the call site CS. | |
static void | recomputeLiveInValues (Function &F, DominatorTree &DT, ArrayRef< CallBase * > toUpdate, MutableArrayRef< struct PartiallyConstructedSafepointRecord > records, PointerToBaseTy &PointerToBase, GCStrategy *GC) |
static Instruction * | rematerializeChain (ArrayRef< Instruction * > ChainToBase, Instruction *InsertBefore, Value *RootOfChain, Value *AlternateLiveBase) |
static BasicBlock * | normalizeForInvokeSafepoint (BasicBlock *BB, BasicBlock *InvokeParent, DominatorTree &DT) |
static AttributeList | legalizeCallAttributes (CallBase *Call, bool IsMemIntrinsic, AttributeList StatepointAL) |
static void | CreateGCRelocates (ArrayRef< Value * > LiveVariables, ArrayRef< Value * > BasePtrs, Instruction *StatepointToken, IRBuilder<> &Builder, GCStrategy *GC) |
Helper function to place all gc relocates necessary for the given statepoint. | |
static StringRef | getDeoptLowering (CallBase *Call) |
static void | makeStatepointExplicitImpl (CallBase *Call, const SmallVectorImpl< Value * > &BasePtrs, const SmallVectorImpl< Value * > &LiveVariables, PartiallyConstructedSafepointRecord &Result, std::vector< DeferredReplacement > &Replacements, const PointerToBaseTy &PointerToBase, GCStrategy *GC) |
static void | makeStatepointExplicit (DominatorTree &DT, CallBase *Call, PartiallyConstructedSafepointRecord &Result, std::vector< DeferredReplacement > &Replacements, const PointerToBaseTy &PointerToBase, GCStrategy *GC) |
static void | insertRelocationStores (iterator_range< Value::user_iterator > GCRelocs, DenseMap< Value *, AllocaInst * > &AllocaMap, DenseSet< Value * > &VisitedLiveValues) |
static void | insertRematerializationStores (const RematerializedValueMapTy &RematerializedValues, DenseMap< Value *, AllocaInst * > &AllocaMap, DenseSet< Value * > &VisitedLiveValues) |
static void | relocationViaAlloca (Function &F, DominatorTree &DT, ArrayRef< Value * > Live, ArrayRef< PartiallyConstructedSafepointRecord > Records) |
Do all the relocation update via allocas and mem2reg. | |
template<typename T > | |
static void | unique_unsorted (SmallVectorImpl< T > &Vec) |
Implement a unique function which doesn't require we sort the input vector. | |
static void | insertUseHolderAfter (CallBase *Call, const ArrayRef< Value * > Values, SmallVectorImpl< CallInst * > &Holders) |
Insert holders so that each Value is obviously live through the entire lifetime of the call. | |
static void | findLiveReferences (Function &F, DominatorTree &DT, ArrayRef< CallBase * > toUpdate, MutableArrayRef< struct PartiallyConstructedSafepointRecord > records, GCStrategy *GC) |
static Value * | findRematerializableChainToBasePointer (SmallVectorImpl< Instruction * > &ChainToBase, Value *CurrentValue) |
static InstructionCost | chainToBasePointerCost (SmallVectorImpl< Instruction * > &Chain, TargetTransformInfo &TTI) |
static bool | AreEquivalentPhiNodes (PHINode &OrigRootPhi, PHINode &AlternateRootPhi) |
static void | findRematerializationCandidates (PointerToBaseTy PointerToBase, RematCandTy &RematerizationCandidates, TargetTransformInfo &TTI) |
static void | rematerializeLiveValuesAtUses (RematCandTy &RematerizationCandidates, MutableArrayRef< PartiallyConstructedSafepointRecord > Records, PointerToBaseTy &PointerToBase) |
static void | rematerializeLiveValues (CallBase *Call, PartiallyConstructedSafepointRecord &Info, PointerToBaseTy &PointerToBase, RematCandTy &RematerizationCandidates, TargetTransformInfo &TTI) |
static bool | inlineGetBaseAndOffset (Function &F, SmallVectorImpl< CallInst * > &Intrinsics, DefiningValueMapTy &DVCache, IsKnownBaseMapTy &KnownBases) |
static bool | insertParsePoints (Function &F, DominatorTree &DT, TargetTransformInfo &TTI, SmallVectorImpl< CallBase * > &ToUpdate, DefiningValueMapTy &DVCache, IsKnownBaseMapTy &KnownBases) |
static AttributeMask | getParamAndReturnAttributesToRemove () |
static void | stripNonValidAttributesFromPrototype (Function &F) |
static void | stripInvalidMetadataFromInstruction (Instruction &I) |
Certain metadata on instructions are invalid after running RS4GC. | |
static void | stripNonValidDataFromBody (Function &F) |
static void | computeLiveInValues (BasicBlock::reverse_iterator Begin, BasicBlock::reverse_iterator End, SetVector< Value * > &LiveTmp, GCStrategy *GC) |
Compute the live-in set for the location rbegin starting from the live-out set of the basic block. | |
static void | computeLiveOutSeed (BasicBlock *BB, SetVector< Value * > &LiveTmp, GCStrategy *GC) |
static SetVector< Value * > | computeKillSet (BasicBlock *BB, GCStrategy *GC) |
static void | checkBasicSSA (DominatorTree &DT, SetVector< Value * > &Live, Instruction *TI, bool TermOkay=false) |
Check that the items in 'Live' dominate 'TI'. | |
static void | checkBasicSSA (DominatorTree &DT, GCPtrLivenessData &Data, BasicBlock &BB) |
Check that all the liveness sets used during the computation of liveness obey basic SSA properties. | |
Variables | |
static cl::opt< bool > | PrintLiveSet ("spp-print-liveset", cl::Hidden, cl::init(false)) |
static cl::opt< bool > | PrintLiveSetSize ("spp-print-liveset-size", cl::Hidden, cl::init(false)) |
static cl::opt< bool > | PrintBasePointers ("spp-print-base-pointers", cl::Hidden, cl::init(false)) |
static cl::opt< unsigned > | RematerializationThreshold ("spp-rematerialization-threshold", cl::Hidden, cl::init(6)) |
static bool | ClobberNonLive = false |
static cl::opt< bool, true > | ClobberNonLiveOverride ("rs4gc-clobber-non-live", cl::location(ClobberNonLive), cl::Hidden) |
static cl::opt< bool > | AllowStatepointWithNoDeoptInfo ("rs4gc-allow-statepoint-with-no-deopt-info", cl::Hidden, cl::init(true)) |
static cl::opt< bool > | RematDerivedAtUses ("rs4gc-remat-derived-at-uses", cl::Hidden, cl::init(true)) |
static constexpr Attribute::AttrKind | FnAttrsToStrip [] |
#define DEBUG_TYPE "rewrite-statepoints-for-gc" |
Definition at line 78 of file RewriteStatepointsForGC.cpp.
|
static |
Definition at line 315 of file RewriteStatepointsForGC.cpp.
References llvm::dbgs(), findLiveSetAtInst(), PrintLiveSet, and PrintLiveSetSize.
Referenced by findLiveReferences().
Definition at line 682 of file RewriteStatepointsForGC.cpp.
References llvm::First, and llvm::Value::getType().
Referenced by findBasePointer().
Definition at line 2323 of file RewriteStatepointsForGC.cpp.
References llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::end(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::find(), llvm::PHINode::getIncomingBlock(), llvm::PHINode::getIncomingValue(), llvm::PHINode::getNumIncomingValues(), and llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent().
Referenced by findRematerializationCandidates().
|
static |
Definition at line 2290 of file RewriteStatepointsForGC.cpp.
References assert(), GEP, llvm::TargetTransformInfo::getAddressComputationCost(), llvm::TargetTransformInfo::getCastContextHint(), llvm::TargetTransformInfo::getCastInstrCost(), llvm_unreachable, and llvm::TargetTransformInfo::TCK_SizeAndLatency.
Referenced by findRematerializationCandidates().
|
static |
Check that all the liveness sets used during the computation of liveness obey basic SSA properties.
This is useful for finding cases where we miss a def.
Definition at line 3268 of file RewriteStatepointsForGC.cpp.
References checkBasicSSA(), llvm::Data, and llvm::BasicBlock::getTerminator().
|
static |
Check that the items in 'Live' dominate 'TI'.
This is used as a basic validation check for the liveness computation.
Definition at line 3250 of file RewriteStatepointsForGC.cpp.
References assert(), llvm::DominatorTree::dominates(), I, and Live.
Referenced by checkBasicSSA(), and computeLiveInValues().
|
static |
Definition at line 3239 of file RewriteStatepointsForGC.cpp.
References I, llvm::SetVector< T, Vector, Set, N >::insert(), and isHandledGCPointerType().
Referenced by computeLiveInValues().
|
static |
Compute the live-in set for the location rbegin starting from the live-out set of the basic block.
Definition at line 3189 of file RewriteStatepointsForGC.cpp.
References assert(), End, I, llvm::SetVector< T, Vector, Set, N >::insert(), isHandledGCPointerType(), isUnhandledGCPointerType(), llvm::make_range(), and llvm::SetVector< T, Vector, Set, N >::remove().
|
static |
Compute the live-in set for every basic block in the function.
Definition at line 3276 of file RewriteStatepointsForGC.cpp.
References assert(), checkBasicSSA(), computeKillSet(), computeLiveInValues(), computeLiveOutSeed(), llvm::Data, llvm::SetVector< T, Vector, Set, N >::empty(), F, llvm::SetVector< T, Vector, Set, N >::insert(), llvm::SetVector< T, Vector, Set, N >::pop_back_val(), llvm::pred_begin(), llvm::pred_end(), llvm::SetVector< T, Vector, Set, N >::set_subtract(), llvm::SetVector< T, Vector, Set, N >::set_union(), llvm::SetVector< T, Vector, Set, N >::size(), and llvm::successors().
Referenced by computeLiveInValues(), findLiveReferences(), findLiveSetAtInst(), and recomputeLiveInValues().
|
static |
Definition at line 3222 of file RewriteStatepointsForGC.cpp.
References assert(), llvm::PHINode::getIncomingValueForBlock(), I, llvm::SetVector< T, Vector, Set, N >::insert(), isHandledGCPointerType(), isUnhandledGCPointerType(), and llvm::successors().
Referenced by computeLiveInValues().
|
static |
Returns true if this type contains a gc pointer whether we know how to handle that type or not.
Definition at line 283 of file RewriteStatepointsForGC.cpp.
References llvm::any_of(), containsGCPtrType(), and isGCPointerType().
|
static |
Helper function to place all gc relocates necessary for the given statepoint.
Inputs: liveVariables - list of variables to be relocated. basePtrs - base pointers. statepointToken - statepoint instruction to which relocates should be bound. Builder - Llvm IR builder to be used to construct new calls.
Definition at line 1498 of file RewriteStatepointsForGC.cpp.
References assert(), llvm::ArrayRef< T >::begin(), llvm::CallingConv::Cold, llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::count(), llvm::IRBuilderBase::CreateCall(), llvm::ArrayRef< T >::end(), llvm::find(), llvm::PointerType::get(), llvm::FixedVectorType::get(), llvm::Intrinsic::getDeclaration(), llvm::IRBuilderBase::getInt32(), llvm::Instruction::getModule(), getNumElements(), isHandledGCPointerType(), llvm::CallBase::setCallingConv(), llvm::ArrayRef< T >::size(), and suffixed_name_or().
Referenced by makeStatepointExplicitImpl().
|
static |
Helper function for findBasePointer - Will return a value which either a) defines the base pointer for the input, b) blocks the simple search (i.e.
a PHI or Select of two derived pointers), or c) involves a change from pointer to vector type or back.
Definition at line 451 of file RewriteStatepointsForGC.cpp.
References assert(), findBaseDefiningValue(), findBaseDefiningValueOfVector(), GEP, llvm::ConstantPointerNull::get(), I, II, llvm_unreachable, and setKnownBase().
Referenced by findBaseDefiningValue(), findBaseDefiningValueCached(), and findBaseDefiningValueOfVector().
|
static |
Returns the base defining value for this value.
Definition at line 625 of file RewriteStatepointsForGC.cpp.
References assert(), llvm::dbgs(), findBaseDefiningValue(), I, and LLVM_DEBUG.
Referenced by findBaseOrBDV().
|
static |
Return a base defining value for the 'Index' element of the given vector instruction 'I'.
If Index is null, returns a BDV for the entire vector 'I'. As an optimization, this method will try to determine when the element is known to already be a base pointer. If this can be established, the second value in the returned pair will be true. Note that either a vector or a pointer typed value can be returned. For the former, the vector returned is a BDV (and possibly a base) of the entire vector 'I'. If the later, the return pointer is a BDV (or possibly a base) for the particular element in 'I'.
Definition at line 353 of file RewriteStatepointsForGC.cpp.
References assert(), findBaseDefiningValue(), GEP, llvm::ConstantAggregateZero::get(), I, and setKnownBase().
Referenced by findBaseDefiningValue().
|
static |
Return a base pointer for this value if known.
Otherwise, return it's base defining value.
Definition at line 642 of file RewriteStatepointsForGC.cpp.
References findBaseDefiningValueCached(), and I.
Referenced by findBasePointer().
|
static |
For a given value or instruction, figure out what base ptr its derived from.
For gc objects, this is simply itself. On success, returns a value which is the base pointer. (This is reliable and can be used for relocation.) On failure, returns nullptr.
Definition at line 806 of file RewriteStatepointsForGC.cpp.
References areBothVectorOrScalar(), assert(), llvm::sampleprof::Base, llvm::MapVector< KeyT, ValueT, MapType, VectorType >::count(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::count(), llvm::dbgs(), DL, llvm::SmallVectorBase< Size_T >::empty(), llvm::MapVector< KeyT, ValueT, MapType, VectorType >::end(), F, llvm::MapVector< KeyT, ValueT, MapType, VectorType >::find(), findBaseOrBDV(), llvm::MDNode::get(), llvm::PoisonValue::get(), llvm::PHINode::getIncomingBlock(), llvm::PHINode::getIncomingValue(), llvm::Value::getName(), llvm::PHINode::getNumIncomingValues(), llvm::BasicBlock::getTerminator(), llvm::Value::getType(), I, llvm::MapVector< KeyT, ValueT, MapType, VectorType >::insert(), llvm::Instruction::insertBefore(), isKnownBase(), isOriginalBaseResult(), LLVM_DEBUG, llvm_unreachable, llvm::SmallVectorImpl< T >::pop_back_val(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), setKnownBase(), llvm::Instruction::setMetadata(), llvm::Value::setName(), llvm::MapVector< KeyT, ValueT, MapType, VectorType >::size(), suffixed_name_or(), and ToRemove.
Referenced by findBasePointers(), and inlineGetBaseAndOffset().
|
static |
Definition at line 1302 of file RewriteStatepointsForGC.cpp.
References assert(), llvm::DominatorTree::dominates(), findBasePointer(), and live.
Referenced by findBasePointers(), and insertParsePoints().
|
static |
Find the required based pointers (and adjust the live set) for the given parse point.
Definition at line 1319 of file RewriteStatepointsForGC.cpp.
References findBasePointers(), and llvm::LLVMContext::OB_deopt.
|
static |
Looks up the GC strategy for a given function, returning null if the function doesn't have a GC tag.
The strategy is stored in the cache.
Definition at line 2996 of file RewriteStatepointsForGC.cpp.
References F, and llvm::getGCStrategy().
Referenced by insertParsePoints(), and shouldRewriteStatepointsIn().
|
static |
Definition at line 2246 of file RewriteStatepointsForGC.cpp.
References analyzeParsePointLiveness(), computeLiveInValues(), F, info, and llvm::ArrayRef< T >::size().
Referenced by insertParsePoints().
|
static |
Given results from the dataflow liveness computation, find the set of live Values at a particular instruction.
Definition at line 3343 of file RewriteStatepointsForGC.cpp.
References assert(), llvm::SetVector< T, Vector, Set, N >::begin(), computeLiveInValues(), llvm::Data, llvm::SetVector< T, Vector, Set, N >::end(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::BasicBlock::rbegin(), and llvm::SetVector< T, Vector, Set, N >::remove().
Referenced by analyzeParsePointLiveness(), and recomputeLiveInValues().
|
static |
Definition at line 2264 of file RewriteStatepointsForGC.cpp.
References findRematerializableChainToBasePointer(), GEP, and llvm::SmallVectorTemplateBase< T, bool >::push_back().
Referenced by findRematerializableChainToBasePointer(), findRematerializationCandidates(), and rematerializeLiveValuesAtUses().
|
static |
Definition at line 2352 of file RewriteStatepointsForGC.cpp.
References AreEquivalentPhiNodes(), llvm::sampleprof::Base, chainToBasePointerCost(), findRematerializableChainToBasePointer(), and llvm::SmallVectorBase< Size_T >::size().
Referenced by insertParsePoints().
Definition at line 232 of file RewriteStatepointsForGC.cpp.
References AllowStatepointWithNoDeoptInfo, assert(), and llvm::LLVMContext::OB_deopt.
Referenced by insertParsePoints().
Definition at line 1626 of file RewriteStatepointsForGC.cpp.
References assert(), F, llvm::AttributeList::getFnAttr(), llvm::Attribute::getValueAsString(), and llvm::AttributeList::hasFnAttr().
Referenced by makeStatepointExplicitImpl().
|
static |
Definition at line 2878 of file RewriteStatepointsForGC.cpp.
Referenced by stripNonValidAttributesFromPrototype(), and stripNonValidDataFromBody().
|
static |
Definition at line 2599 of file RewriteStatepointsForGC.cpp.
References assert(), llvm::sampleprof::Base, llvm::IRBuilderBase::CreatePtrToInt(), llvm::IRBuilderBase::CreateSub(), DL, F, findBasePointer(), llvm::Type::getIntNTy(), llvm::Type::getPointerAddressSpace(), llvm::Value::getType(), llvm_unreachable, llvm::Offset, llvm::Value::replaceAllUsesWith(), and suffixed_name_or().
Referenced by llvm::RewriteStatepointsForGC::runOnFunction().
|
static |
Global mapping from live pointers to a base-defining-value.
Definition at line 2647 of file RewriteStatepointsForGC.cpp.
References llvm::append_range(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::SmallVectorImpl< T >::clear(), llvm::DominatorTree::dominates(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::errs(), F, findBasePointers(), findGCStrategy(), findLiveReferences(), findRematerializationCandidates(), GetDeoptBundleOperands(), II, info, Info, insertUseHolderAfter(), isHandledGCPointerType(), llvm::DominatorTree::isReachableFromEntry(), isUnhandledGCPointerType(), Live, makeStatepointExplicit(), normalizeForInvokeSafepoint(), PrintBasePointers, Ptr, llvm::SmallVectorTemplateBase< T, bool >::push_back(), recomputeLiveInValues(), relocationViaAlloca(), rematerializeLiveValues(), rematerializeLiveValuesAtUses(), llvm::SmallVectorImpl< T >::reserve(), llvm::SmallVectorBase< Size_T >::size(), and unique_unsorted().
Referenced by llvm::RewriteStatepointsForGC::runOnFunction().
|
static |
Definition at line 1965 of file RewriteStatepointsForGC.cpp.
References assert(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::count(), llvm::GCRelocateInst::getDerivedPtr(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::ilist_node_with_parent< NodeTy, ParentTy, Options >::getNextNode(), and llvm::detail::DenseSetImpl< ValueT, MapTy, ValueInfoT >::insert().
Referenced by relocationViaAlloca().
|
static |
Definition at line 1990 of file RewriteStatepointsForGC.cpp.
References assert(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::count(), llvm::ilist_node_impl< OptionsT >::getIterator(), and llvm::detail::DenseSetImpl< ValueT, MapTy, ValueInfoT >::insert().
Referenced by relocationViaAlloca().
|
static |
Insert holders so that each Value is obviously live through the entire lifetime of the call.
Definition at line 2221 of file RewriteStatepointsForGC.cpp.
References llvm::CallInst::Create(), llvm::ArrayRef< T >::empty(), llvm::FunctionType::get(), llvm::Type::getVoidTy(), II, and llvm::SmallVectorTemplateBase< T, bool >::push_back().
Referenced by insertParsePoints().
|
static |
Definition at line 254 of file RewriteStatepointsForGC.cpp.
References assert().
|
static |
Definition at line 268 of file RewriteStatepointsForGC.cpp.
References isGCPointerType().
Referenced by computeKillSet(), computeLiveInValues(), computeLiveOutSeed(), CreateGCRelocates(), insertParsePoints(), and isUnhandledGCPointerType().
Returns true if V is a known base.
Definition at line 665 of file RewriteStatepointsForGC.cpp.
References assert().
Referenced by findBasePointer().
This value is a base pointer that is not generated by RS4GC, i.e.
it already exists in the code.
Definition at line 657 of file RewriteStatepointsForGC.cpp.
Referenced by findBasePointer().
|
static |
Definition at line 299 of file RewriteStatepointsForGC.cpp.
References containsGCPtrType(), and isHandledGCPointerType().
Referenced by computeLiveInValues(), computeLiveOutSeed(), and insertParsePoints().
|
static |
Definition at line 1453 of file RewriteStatepointsForGC.cpp.
References A, llvm::AttributeList::addFnAttributes(), llvm::AttributeList::addParamAttributes(), llvm::GCStatepointInst::CallArgsBeginPos, FnAttrsToStrip, llvm::AttributeList::getFnAttrs(), llvm::AttributeList::getParamAttrs(), I, llvm::AttributeList::isEmpty(), llvm::isStatepointDirectiveAttr(), llvm::AttrBuilder::removeAttribute(), and llvm::seq().
Referenced by makeStatepointExplicitImpl().
|
static |
Definition at line 1935 of file RewriteStatepointsForGC.cpp.
References assert(), llvm::sampleprof::Base, makeStatepointExplicitImpl(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SmallVectorImpl< T >::reserve(), and llvm::SmallVectorBase< Size_T >::size().
Referenced by insertParsePoints().
|
static |
Definition at line 1642 of file RewriteStatepointsForGC.cpp.
References assert(), llvm::sampleprof::Base, llvm::BasicBlock::begin(), llvm::SmallVectorImpl< T >::clear(), CreateGCRelocates(), llvm::IRBuilderBase::CreateGCResult(), llvm::IRBuilderBase::CreateGCStatepointCall(), llvm::IRBuilderBase::CreateGCStatepointInvoke(), llvm::IRBuilderBase::CreatePtrToInt(), llvm::IRBuilderBase::CreateSub(), llvm::StatepointDirectives::DefaultStatepointID, llvm::DeoptLiveIn, DL, F, llvm::GCTransition, llvm::AttributeList::get(), llvm::ConstantPointerNull::get(), llvm::FunctionType::get(), llvm::CallBase::getAttributes(), llvm::FunctionCallee::getCallee(), llvm::Value::getContext(), getDeoptLowering(), llvm::BasicBlock::getFirstInsertionPt(), llvm::Type::getIntNTy(), llvm::BasicBlock::getLandingPadInst(), llvm::BasicBlock::getUniquePredecessor(), llvm::Type::getVoidTy(), II, legalizeCallAttributes(), llvm_unreachable, Name, llvm::None, llvm::StatepointDirectives::NumPatchBytes, llvm::LLVMContext::OB_deopt, llvm::LLVMContext::OB_gc_transition, llvm::parseStatepointDirectivesFromAttrs(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::AttributeList::ReturnIndex, llvm::CallBase::setAttributes(), llvm::CallBase::setCallingConv(), llvm::IRBuilderBase::SetCurrentDebugLocation(), llvm::IRBuilderBase::SetInsertPoint(), llvm::Value::setName(), llvm::CallInst::setTailCallKind(), llvm::SmallVectorBase< Size_T >::size(), and llvm::StatepointDirectives::StatepointID.
Referenced by makeStatepointExplicit().
|
static |
Definition at line 1426 of file RewriteStatepointsForGC.cpp.
References assert(), llvm::FoldSingleEntryPHINodes(), llvm::BasicBlock::getUniquePredecessor(), and llvm::SplitBlockPredecessors().
Referenced by insertParsePoints().
|
static |
Definition at line 796 of file RewriteStatepointsForGC.cpp.
References OS.
|
static |
Definition at line 1349 of file RewriteStatepointsForGC.cpp.
References computeLiveInValues(), F, info, recomputeLiveInValues(), and llvm::ArrayRef< T >::size().
|
static |
Given an updated version of the dataflow liveness results, update the liveset and base pointer maps for the call site CS.
Definition at line 3361 of file RewriteStatepointsForGC.cpp.
References findLiveSetAtInst(), and Info.
Referenced by insertParsePoints(), and recomputeLiveInValues().
|
static |
Do all the relocation update via allocas and mem2reg.
Definition at line 2012 of file RewriteStatepointsForGC.cpp.
References assert(), ClobberNonLive, llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::contains(), llvm::detail::DenseSetImpl< ValueT, MapTy, ValueInfoT >::count(), DL, llvm::SmallVectorBase< Size_T >::empty(), F, llvm::ConstantPointerNull::get(), llvm::ConstantAggregateZero::get(), llvm::AllocaInst::getAllocatedType(), llvm::BasicBlock::getFirstNonPHI(), I, II, Info, insertRelocationStores(), insertRematerializationStores(), llvm::Last, Live, llvm::PromoteMemToReg(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SmallVectorImpl< T >::reserve(), llvm::SmallVectorBase< Size_T >::size(), llvm::sort(), llvm::unique(), llvm::Value::users(), and Uses.
Referenced by insertParsePoints().
|
static |
Definition at line 1367 of file RewriteStatepointsForGC.cpp.
References assert(), llvm::Instruction::insertBefore(), llvm::is_contained(), llvm::make_range(), llvm::User::operand_values(), llvm::ArrayRef< T >::rbegin(), llvm::ArrayRef< T >::rend(), llvm::User::replaceUsesOfWith(), and llvm::Value::setName().
Referenced by rematerializeLiveValues(), and rematerializeLiveValuesAtUses().
|
static |
Definition at line 2533 of file RewriteStatepointsForGC.cpp.
References assert(), Info, llvm::SmallVectorTemplateBase< T, bool >::push_back(), RematerializationThreshold, and rematerializeChain().
Referenced by insertParsePoints().
|
static |
Definition at line 2413 of file RewriteStatepointsForGC.cpp.
References llvm::any_of(), assert(), llvm::count_if(), llvm::dbgs(), llvm::SmallVectorBase< Size_T >::empty(), findRematerializableChainToBasePointer(), llvm::Value::getNumUses(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::Value::getUniqueUndroppableUser(), llvm::Value::hasOneUse(), LLVM_DEBUG, P, llvm::SmallVectorTemplateBase< T, bool >::push_back(), RematDerivedAtUses, RematerializationThreshold, rematerializeChain(), llvm::User::replaceUsesOfWith(), llvm::SmallVectorBase< Size_T >::size(), llvm::Value::user_begin(), llvm::Value::user_empty(), and llvm::Value::users().
Referenced by insertParsePoints().
Caches the IsKnownBase flag for a value and asserts that it wasn't present in the cache before.
Definition at line 671 of file RewriteStatepointsForGC.cpp.
References assert().
Referenced by findBaseDefiningValue(), findBaseDefiningValueOfVector(), and findBasePointer().
Returns true if this function should be rewritten by this pass.
The main point of this function is as an extension point for custom logic.
Definition at line 3005 of file RewriteStatepointsForGC.cpp.
References assert(), F, and findGCStrategy().
Referenced by llvm::RewriteStatepointsForGC::run(), llvm::RewriteStatepointsForGC::runOnFunction(), and stripNonValidData().
|
static |
Certain metadata on instructions are invalid after running RS4GC.
Optimizations that run after RS4GC can incorrectly use this metadata to optimize functions. We drop such metadata on the instruction.
Definition at line 2918 of file RewriteStatepointsForGC.cpp.
References I.
Referenced by stripNonValidDataFromBody().
|
static |
Definition at line 2890 of file RewriteStatepointsForGC.cpp.
References A, F, FnAttrsToStrip, llvm::Intrinsic::getAttributes(), and getParamAndReturnAttributesToRemove().
Referenced by stripNonValidData().
|
static |
The IR fed into RewriteStatepointsForGC may have had attributes and metadata implying dereferenceability that are no longer valid/correct after RewriteStatepointsForGC has run.
This is because semantically, after RewriteStatepointsForGC runs, all calls to gc.statepoint "free" the entire heap. stripNonValidData (conservatively) restores correctness by erasing all attributes in the module that externally imply dereferenceability. Similar reasoning also applies to the noalias attributes and metadata. gc.statepoint can touch the entire heap including noalias objects. Apart from attributes and metadata, we also remove instructions that imply constant physical memory: llvm.invariant.start.
Definition at line 3016 of file RewriteStatepointsForGC.cpp.
References llvm::any_of(), assert(), F, shouldRewriteStatepointsIn(), stripNonValidAttributesFromPrototype(), and stripNonValidDataFromBody().
Referenced by llvm::RewriteStatepointsForGC::run().
|
static |
Definition at line 2946 of file RewriteStatepointsForGC.cpp.
References llvm::MDBuilder::createMutableTBAAAccessTag(), F, llvm::PoisonValue::get(), getParamAndReturnAttributesToRemove(), I, II, instructions, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and stripInvalidMetadataFromInstruction().
Referenced by stripNonValidData().
Definition at line 306 of file RewriteStatepointsForGC.cpp.
References llvm::StringRef::str().
Referenced by CreateGCRelocates(), findBasePointer(), and inlineGetBaseAndOffset().
|
static |
Implement a unique function which doesn't require we sort the input vector.
Doing so has the effect of changing the output of a couple of tests in ways which make them less useful in testing fused safepoints.
Definition at line 2214 of file RewriteStatepointsForGC.cpp.
References llvm::erase_if(), and llvm::SmallSet< T, N, C >::insert().
Referenced by insertParsePoints().
|
static |
Referenced by GetDeoptBundleOperands(), and llvm::RewriteStatepointsForGC::runOnFunction().
|
static |
Definition at line 101 of file RewriteStatepointsForGC.cpp.
Referenced by relocationViaAlloca().
|
static |
|
staticconstexpr |
Definition at line 1448 of file RewriteStatepointsForGC.cpp.
Referenced by legalizeCallAttributes(), and stripNonValidAttributesFromPrototype().
|
static |
Referenced by insertParsePoints().
|
static |
Referenced by analyzeParsePointLiveness().
|
static |
Referenced by analyzeParsePointLiveness().
|
static |
Referenced by rematerializeLiveValuesAtUses().
|
static |
Referenced by rematerializeLiveValues(), and rematerializeLiveValuesAtUses().