44 "mem-intrinsic-expand-size",
50struct PreISelIntrinsicLowering {
58 const bool UseMemIntrinsicLibFunc;
60 explicit PreISelIntrinsicLowering(
64 bool UseMemIntrinsicLibFunc_ =
true)
65 : TM(TM_), LookupTTI(LookupTTI_), LookupTLI(LookupTLI_),
66 UseMemIntrinsicLibFunc(UseMemIntrinsicLibFunc_) {}
68 static bool shouldExpandMemIntrinsicWithSize(
Value *
Size,
70 bool expandMemIntrinsicUses(
Function &
F)
const;
71 bool lowerIntrinsics(
Module &M)
const;
82 Use *LastUse =
nullptr;
84 while (!Intrin.
use_empty() && (!LastUse || LastUse->getNext())) {
85 Use *U = LastUse ? LastUse->getNext() : &*Intrin.
use_begin();
89 if (
auto CI = dyn_cast<CallInst>(U->getUser()))
90 Changed |= Removed = Callback(CI);
101 bool Changed =
false;
105 auto CI = dyn_cast<CallInst>(U.getUser());
106 if (!CI || CI->getCalledOperand() != &
F)
111 B.CreatePtrAdd(CI->getArgOperand(0), CI->getArgOperand(1));
112 Value *OffsetI32 =
B.CreateAlignedLoad(Int32Ty, OffsetPtr,
Align(4));
114 Value *ResultPtr =
B.CreatePtrAdd(CI->getArgOperand(0), OffsetI32);
116 CI->replaceAllUsesWith(ResultPtr);
117 CI->eraseFromParent();
138 "Pre-ISel intrinsics do lower into regular function calls");
145 FunctionCallee FCache = M->getOrInsertFunction(NewFn,
F.getFunctionType());
148 Fn->setLinkage(
F.getLinkage());
152 Fn->addFnAttr(Attribute::NonLazyBind);
159 auto *CB = cast<CallBase>(U.getUser());
161 if (CB->getCalledFunction() != &
F) {
164 assert((Kind == objcarc::ARCInstKind::RetainRV ||
165 Kind == objcarc::ARCInstKind::UnsafeClaimRV) &&
166 "use expected to be the argument of operand bundle "
167 "\"clang.arc.attachedcall\"");
172 auto *CI = cast<CallInst>(CB);
173 assert(CI->getCalledFunction() &&
"Cannot lower an indirect call!");
175 IRBuilder<> Builder(CI->getParent(), CI->getIterator());
178 CI->getOperandBundlesAsDefs(BundleList);
197 if (
F.getAttributes().hasAttrSomewhere(Attribute::Returned, &Index) &&
200 Attribute::Returned);
202 if (!CI->use_empty())
203 CI->replaceAllUsesWith(NewCI);
204 CI->eraseFromParent();
212bool PreISelIntrinsicLowering::shouldExpandMemIntrinsicWithSize(
224 return SizeVal > Threshold || Threshold == 0;
232 const TargetLowering *TLI = TM->getSubtargetImpl(*F)->getTargetLowering();
238bool PreISelIntrinsicLowering::expandMemIntrinsicUses(
Function &
F)
const {
240 bool Changed =
false;
246 case Intrinsic::memcpy: {
247 auto *Memcpy = cast<MemCpyInst>(Inst);
250 if (shouldExpandMemIntrinsicWithSize(Memcpy->getLength(),
TTI)) {
251 if (UseMemIntrinsicLibFunc &&
258 Memcpy->eraseFromParent();
263 case Intrinsic::memcpy_inline: {
267 auto *Memcpy = cast<MemCpyInlineInst>(Inst);
268 if (isa<ConstantInt>(Memcpy->getLength()))
275 Memcpy->eraseFromParent();
278 case Intrinsic::memmove: {
279 auto *Memmove = cast<MemMoveInst>(Inst);
282 if (shouldExpandMemIntrinsicWithSize(Memmove->getLength(),
TTI)) {
283 if (UseMemIntrinsicLibFunc &&
289 Memmove->eraseFromParent();
295 case Intrinsic::memset: {
296 auto *Memset = cast<MemSetInst>(Inst);
299 if (shouldExpandMemIntrinsicWithSize(Memset->getLength(),
TTI)) {
300 if (UseMemIntrinsicLibFunc &&
306 Memset->eraseFromParent();
311 case Intrinsic::memset_inline: {
315 auto *Memset = cast<MemSetInlineInst>(Inst);
316 if (isa<ConstantInt>(Memset->getLength()))
321 Memset->eraseFromParent();
324 case Intrinsic::experimental_memset_pattern: {
325 auto *Memset = cast<MemSetPatternInst>(Inst);
328 Memset->eraseFromParent();
339bool PreISelIntrinsicLowering::lowerIntrinsics(
Module &M)
const {
340 bool Changed =
false;
342 switch (
F.getIntrinsicID()) {
345 case Intrinsic::memcpy:
346 case Intrinsic::memcpy_inline:
347 case Intrinsic::memmove:
348 case Intrinsic::memset:
349 case Intrinsic::memset_inline:
350 case Intrinsic::experimental_memset_pattern:
351 Changed |= expandMemIntrinsicUses(
F);
353 case Intrinsic::load_relative:
356 case Intrinsic::is_constant:
357 case Intrinsic::objectsize:
366#define BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, VLENPOS) \
367 case Intrinsic::VPID:
368#include "llvm/IR/VPIntrinsics.def"
372 auto *VPI = cast<VPIntrinsic>(CI);
377 Changed |= ED != VPExpansionDetails::IntrinsicUnchanged;
378 bool Removed = ED == VPExpansionDetails::IntrinsicReplaced;
382 case Intrinsic::objc_autorelease:
385 case Intrinsic::objc_autoreleasePoolPop:
388 case Intrinsic::objc_autoreleasePoolPush:
391 case Intrinsic::objc_autoreleaseReturnValue:
394 case Intrinsic::objc_copyWeak:
397 case Intrinsic::objc_destroyWeak:
400 case Intrinsic::objc_initWeak:
403 case Intrinsic::objc_loadWeak:
406 case Intrinsic::objc_loadWeakRetained:
409 case Intrinsic::objc_moveWeak:
412 case Intrinsic::objc_release:
415 case Intrinsic::objc_retain:
418 case Intrinsic::objc_retainAutorelease:
421 case Intrinsic::objc_retainAutoreleaseReturnValue:
424 case Intrinsic::objc_retainAutoreleasedReturnValue:
425 Changed |=
lowerObjCCall(
F,
"objc_retainAutoreleasedReturnValue");
427 case Intrinsic::objc_retainBlock:
430 case Intrinsic::objc_storeStrong:
433 case Intrinsic::objc_storeWeak:
436 case Intrinsic::objc_unsafeClaimAutoreleasedReturnValue:
437 Changed |=
lowerObjCCall(
F,
"objc_unsafeClaimAutoreleasedReturnValue");
439 case Intrinsic::objc_retainedObject:
442 case Intrinsic::objc_unretainedObject:
445 case Intrinsic::objc_unretainedPointer:
448 case Intrinsic::objc_retain_autorelease:
451 case Intrinsic::objc_sync_enter:
454 case Intrinsic::objc_sync_exit:
458 case Intrinsic::exp2:
461 if (!isa<ScalableVectorType>(Ty))
477class PreISelIntrinsicLoweringLegacyPass :
public ModulePass {
481 PreISelIntrinsicLoweringLegacyPass() :
ModulePass(
ID) {}
491 return this->getAnalysis<TargetTransformInfoWrapperPass>().getTTI(
F);
494 return this->getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(
F);
497 const auto *
TM = &getAnalysis<TargetPassConfig>().getTM<
TargetMachine>();
498 PreISelIntrinsicLowering
Lowering(TM, LookupTTI, LookupTLI);
505char PreISelIntrinsicLoweringLegacyPass::ID;
508 "pre-isel-intrinsic-lowering",
509 "Pre-ISel Intrinsic Lowering",
false,
false)
518 return new PreISelIntrinsicLoweringLegacyPass();
532 PreISelIntrinsicLowering
Lowering(
TM, LookupTTI, LookupTLI);
static const Function * getParent(const Value *V)
static bool setNonLazyBind(Function &F)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Module.h This file contains the declarations for the Module class.
This defines the Use class.
The header file for the LowerConstantIntrinsics pass as used by the new pass manager.
This file defines ARC utility functions which are used by various parts of the compiler.
FunctionAnalysisManager FAM
#define INITIALIZE_PASS_DEPENDENCY(depName)
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
static cl::opt< int64_t > MemIntrinsicExpandSizeThresholdOpt("mem-intrinsic-expand-size", cl::desc("Set minimum mem intrinsic size to expand in IR"), cl::init(-1), cl::Hidden)
Threshold to leave statically sized memory intrinsic calls.
pre isel intrinsic lowering
static bool forEachCall(Function &Intrin, T Callback)
pre isel intrinsic Pre ISel Intrinsic Lowering
static bool lowerObjCCall(Function &F, const char *NewFn, bool setNonLazyBind=false)
static bool canEmitLibcall(const TargetMachine *TM, Function *F, RTLIB::Libcall LC)
static CallInst::TailCallKind getOverridingTailCallKind(const Function &F)
static bool lowerLoadRelative(Function &F)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static SymbolRef::Type getType(const Symbol *Sym)
This file describes how to lower LLVM code to machine code.
Target-Independent Code Generator Pass Configuration Options pass.
A container for analyses that lazily runs them and caches their results.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
Value * getArgOperand(unsigned i) const
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
This class represents a function call, abstracting a target machine's calling convention.
void setTailCallKind(TailCallKind TCK)
This is the shared class of boolean and integer constants.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
This class represents an Operation in the Expression.
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
const Function & getFunction() const
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
static bool mayLowerToFunctionCall(Intrinsic::ID IID)
Check if the intrinsic might lower into a regular function call in the course of IR transformations.
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
virtual bool runOnModule(Module &M)=0
runOnModule - Virtual method overriden by subclasses to process the module being operated on.
A Module instance is used to store all the information related to an LLVM module.
virtual void getAnalysisUsage(AnalysisUsage &) const
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Analysis pass providing the TargetTransformInfo.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
bool isOperationExpand(unsigned Op, EVT VT) const
Return true if the specified operation is illegal on this target or unlikely to be made legal with cu...
int IntrinsicIDToISD(Intrinsic::ID ID) const
Get the ISD node that corresponds to the Intrinsic ID.
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Primary interface to the complete machine description for the target machine.
Target-Independent Code Generator Pass Configuration Options.
The instances of the Type class are immutable: once they are created, they are never changed.
static IntegerType * getInt32Ty(LLVMContext &C)
A Use represents the edge between a Value definition and its users.
LLVM Value Representation.
void setName(const Twine &Name)
Change the name of the value.
An efficient, type-erasing, non-owning reference to a callable.
const ParentTy * getParent() const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
initializer< Ty > init(const Ty &Val)
ARCInstKind getAttachedARCFunctionKind(const CallBase *CB)
This function returns the ARCInstKind of the function attached to operand bundle clang_arc_attachedca...
bool IsNeverTail(ARCInstKind Class)
Test if the given class represents instructions which are never safe to mark with the "tail" keyword.
bool IsAlwaysTail(ARCInstKind Class)
Test if the given class represents instructions which are always safe to mark with the "tail" keyword...
ARCInstKind
Equivalence classes of instructions in the ARC Model.
ARCInstKind GetFunctionClass(const Function *F)
Determine if F is one of the special known Functions.
This is an optimization pass for GlobalISel generic memory operations.
bool lowerUnaryVectorIntrinsicAsLoop(Module &M, CallInst *CI)
Lower CI as a loop.
bool lowerConstantIntrinsics(Function &F, const TargetLibraryInfo &TLI, DominatorTree *DT)
void expandMemSetPatternAsLoop(MemSetPatternInst *MemSet)
Expand MemSetPattern as a loop. MemSet is not deleted.
bool expandMemMoveAsLoop(MemMoveInst *MemMove, const TargetTransformInfo &TTI)
Expand MemMove as a loop.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
ModulePass * createPreISelIntrinsicLoweringPass()
This pass lowers the @llvm.load.relative and @llvm.objc.
VPExpansionDetails expandVectorPredicationIntrinsic(VPIntrinsic &VPI, const TargetTransformInfo &TTI)
Expand a vector predication intrinsic.
void expandMemCpyAsLoop(MemCpyInst *MemCpy, const TargetTransformInfo &TTI, ScalarEvolution *SE=nullptr)
Expand MemCpy as a loop. MemCpy is not deleted.
VPExpansionDetails
Represents the details the expansion of a VP intrinsic.
void expandMemSetAsLoop(MemSetInst *MemSet)
Expand MemSet as a loop. MemSet is not deleted.
This struct is a compact representation of a valid (non-zero power of two) alignment.
static EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)