LLVM 19.0.0git
PreISelIntrinsicLowering.cpp
Go to the documentation of this file.
1//===- PreISelIntrinsicLowering.cpp - Pre-ISel intrinsic lowering pass ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass implements IR lowering for the llvm.memcpy, llvm.memmove,
10// llvm.memset, llvm.load.relative and llvm.objc.* intrinsics.
11//
12//===----------------------------------------------------------------------===//
13
18#include "llvm/CodeGen/Passes.h"
21#include "llvm/IR/Function.h"
22#include "llvm/IR/IRBuilder.h"
25#include "llvm/IR/Module.h"
26#include "llvm/IR/Type.h"
28#include "llvm/Pass.h"
32
33using namespace llvm;
34
35/// Threshold to leave statically sized memory intrinsic calls. Calls of known
36/// size larger than this will be expanded by the pass. Calls of unknown or
37/// lower size will be left for expansion in codegen.
39 "mem-intrinsic-expand-size",
40 cl::desc("Set minimum mem intrinsic size to expand in IR"), cl::init(-1),
42
43namespace {
44
45struct PreISelIntrinsicLowering {
46 const TargetMachine &TM;
47 const function_ref<TargetTransformInfo &(Function &)> LookupTTI;
48
49 /// If this is true, assume it's preferably to leave memory intrinsic calls
50 /// for replacement with a library call later. Otherwise this depends on
51 /// TargetLoweringInfo availability of the corresponding function.
52 const bool UseMemIntrinsicLibFunc;
53
54 explicit PreISelIntrinsicLowering(
55 const TargetMachine &TM_,
57 bool UseMemIntrinsicLibFunc_ = true)
58 : TM(TM_), LookupTTI(LookupTTI_),
59 UseMemIntrinsicLibFunc(UseMemIntrinsicLibFunc_) {}
60
61 static bool shouldExpandMemIntrinsicWithSize(Value *Size,
63 bool expandMemIntrinsicUses(Function &F) const;
64 bool lowerIntrinsics(Module &M) const;
65};
66
67} // namespace
68
70 if (F.use_empty())
71 return false;
72
73 bool Changed = false;
74 Type *Int32Ty = Type::getInt32Ty(F.getContext());
75
76 for (Use &U : llvm::make_early_inc_range(F.uses())) {
77 auto CI = dyn_cast<CallInst>(U.getUser());
78 if (!CI || CI->getCalledOperand() != &F)
79 continue;
80
81 IRBuilder<> B(CI);
82 Value *OffsetPtr =
83 B.CreatePtrAdd(CI->getArgOperand(0), CI->getArgOperand(1));
84 Value *OffsetI32 = B.CreateAlignedLoad(Int32Ty, OffsetPtr, Align(4));
85
86 Value *ResultPtr = B.CreatePtrAdd(CI->getArgOperand(0), OffsetI32);
87
88 CI->replaceAllUsesWith(ResultPtr);
89 CI->eraseFromParent();
90 Changed = true;
91 }
92
93 return Changed;
94}
95
96// ObjCARC has knowledge about whether an obj-c runtime function needs to be
97// always tail-called or never tail-called.
100 if (objcarc::IsAlwaysTail(Kind))
101 return CallInst::TCK_Tail;
102 else if (objcarc::IsNeverTail(Kind))
104 return CallInst::TCK_None;
105}
106
107static bool lowerObjCCall(Function &F, const char *NewFn,
108 bool setNonLazyBind = false) {
110 "Pre-ISel intrinsics do lower into regular function calls");
111 if (F.use_empty())
112 return false;
113
114 // If we haven't already looked up this function, check to see if the
115 // program already contains a function with this name.
116 Module *M = F.getParent();
117 FunctionCallee FCache = M->getOrInsertFunction(NewFn, F.getFunctionType());
118
119 if (Function *Fn = dyn_cast<Function>(FCache.getCallee())) {
120 Fn->setLinkage(F.getLinkage());
121 if (setNonLazyBind && !Fn->isWeakForLinker()) {
122 // If we have Native ARC, set nonlazybind attribute for these APIs for
123 // performance.
124 Fn->addFnAttr(Attribute::NonLazyBind);
125 }
126 }
127
129
130 for (Use &U : llvm::make_early_inc_range(F.uses())) {
131 auto *CB = cast<CallBase>(U.getUser());
132
133 if (CB->getCalledFunction() != &F) {
135 (void)Kind;
136 assert((Kind == objcarc::ARCInstKind::RetainRV ||
137 Kind == objcarc::ARCInstKind::UnsafeClaimRV) &&
138 "use expected to be the argument of operand bundle "
139 "\"clang.arc.attachedcall\"");
140 U.set(FCache.getCallee());
141 continue;
142 }
143
144 auto *CI = cast<CallInst>(CB);
145 assert(CI->getCalledFunction() && "Cannot lower an indirect call!");
146
147 IRBuilder<> Builder(CI->getParent(), CI->getIterator());
148 SmallVector<Value *, 8> Args(CI->args());
150 CI->getOperandBundlesAsDefs(BundleList);
151 CallInst *NewCI = Builder.CreateCall(FCache, Args, BundleList);
152 NewCI->setName(CI->getName());
153
154 // Try to set the most appropriate TailCallKind based on both the current
155 // attributes and the ones that we could get from ObjCARC's special
156 // knowledge of the runtime functions.
157 //
158 // std::max respects both requirements of notail and tail here:
159 // * notail on either the call or from ObjCARC becomes notail
160 // * tail on either side is stronger than none, but not notail
161 CallInst::TailCallKind TCK = CI->getTailCallKind();
162 NewCI->setTailCallKind(std::max(TCK, OverridingTCK));
163
164 // Transfer the 'returned' attribute from the intrinsic to the call site.
165 // By applying this only to intrinsic call sites, we avoid applying it to
166 // non-ARC explicit calls to things like objc_retain which have not been
167 // auto-upgraded to use the intrinsics.
168 unsigned Index;
169 if (F.getAttributes().hasAttrSomewhere(Attribute::Returned, &Index) &&
170 Index)
172 Attribute::Returned);
173
174 if (!CI->use_empty())
175 CI->replaceAllUsesWith(NewCI);
176 CI->eraseFromParent();
177 }
178
179 return true;
180}
181
182// TODO: Should refine based on estimated number of accesses (e.g. does it
183// require splitting based on alignment)
184bool PreISelIntrinsicLowering::shouldExpandMemIntrinsicWithSize(
186 ConstantInt *CI = dyn_cast<ConstantInt>(Size);
187 if (!CI)
188 return true;
189 uint64_t Threshold = MemIntrinsicExpandSizeThresholdOpt.getNumOccurrences()
192 uint64_t SizeVal = CI->getZExtValue();
193
194 // Treat a threshold of 0 as a special case to force expansion of all
195 // intrinsics, including size 0.
196 return SizeVal > Threshold || Threshold == 0;
197}
198
200 RTLIB::Libcall LC) {
201 // TODO: Should this consider the address space of the memcpy?
202 const TargetLowering *TLI = TM.getSubtargetImpl(*F)->getTargetLowering();
203 return TLI->getLibcallName(LC) != nullptr;
204}
205
206// TODO: Handle atomic memcpy and memcpy.inline
207// TODO: Pass ScalarEvolution
208bool PreISelIntrinsicLowering::expandMemIntrinsicUses(Function &F) const {
209 Intrinsic::ID ID = F.getIntrinsicID();
210 bool Changed = false;
211
212 for (User *U : llvm::make_early_inc_range(F.users())) {
213 Instruction *Inst = cast<Instruction>(U);
214
215 switch (ID) {
216 case Intrinsic::memcpy: {
217 auto *Memcpy = cast<MemCpyInst>(Inst);
218 Function *ParentFunc = Memcpy->getFunction();
219 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
220 if (shouldExpandMemIntrinsicWithSize(Memcpy->getLength(), TTI)) {
221 if (UseMemIntrinsicLibFunc &&
222 canEmitLibcall(TM, ParentFunc, RTLIB::MEMCPY))
223 break;
224
225 // TODO: For optsize, emit the loop into a separate function
226 expandMemCpyAsLoop(Memcpy, TTI);
227 Changed = true;
228 Memcpy->eraseFromParent();
229 }
230
231 break;
232 }
233 case Intrinsic::memmove: {
234 auto *Memmove = cast<MemMoveInst>(Inst);
235 Function *ParentFunc = Memmove->getFunction();
236 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
237 if (shouldExpandMemIntrinsicWithSize(Memmove->getLength(), TTI)) {
238 if (UseMemIntrinsicLibFunc &&
239 canEmitLibcall(TM, ParentFunc, RTLIB::MEMMOVE))
240 break;
241
242 if (expandMemMoveAsLoop(Memmove, TTI)) {
243 Changed = true;
244 Memmove->eraseFromParent();
245 }
246 }
247
248 break;
249 }
250 case Intrinsic::memset: {
251 auto *Memset = cast<MemSetInst>(Inst);
252 Function *ParentFunc = Memset->getFunction();
253 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
254 if (shouldExpandMemIntrinsicWithSize(Memset->getLength(), TTI)) {
255 if (UseMemIntrinsicLibFunc &&
256 canEmitLibcall(TM, ParentFunc, RTLIB::MEMSET))
257 break;
258
259 expandMemSetAsLoop(Memset);
260 Changed = true;
261 Memset->eraseFromParent();
262 }
263
264 break;
265 }
266 default:
267 llvm_unreachable("unhandled intrinsic");
268 }
269 }
270
271 return Changed;
272}
273
274bool PreISelIntrinsicLowering::lowerIntrinsics(Module &M) const {
275 bool Changed = false;
276 for (Function &F : M) {
277 switch (F.getIntrinsicID()) {
278 default:
279 break;
280 case Intrinsic::memcpy:
281 case Intrinsic::memmove:
282 case Intrinsic::memset:
283 Changed |= expandMemIntrinsicUses(F);
284 break;
285 case Intrinsic::load_relative:
286 Changed |= lowerLoadRelative(F);
287 break;
288 case Intrinsic::objc_autorelease:
289 Changed |= lowerObjCCall(F, "objc_autorelease");
290 break;
291 case Intrinsic::objc_autoreleasePoolPop:
292 Changed |= lowerObjCCall(F, "objc_autoreleasePoolPop");
293 break;
294 case Intrinsic::objc_autoreleasePoolPush:
295 Changed |= lowerObjCCall(F, "objc_autoreleasePoolPush");
296 break;
297 case Intrinsic::objc_autoreleaseReturnValue:
298 Changed |= lowerObjCCall(F, "objc_autoreleaseReturnValue");
299 break;
300 case Intrinsic::objc_copyWeak:
301 Changed |= lowerObjCCall(F, "objc_copyWeak");
302 break;
303 case Intrinsic::objc_destroyWeak:
304 Changed |= lowerObjCCall(F, "objc_destroyWeak");
305 break;
306 case Intrinsic::objc_initWeak:
307 Changed |= lowerObjCCall(F, "objc_initWeak");
308 break;
309 case Intrinsic::objc_loadWeak:
310 Changed |= lowerObjCCall(F, "objc_loadWeak");
311 break;
312 case Intrinsic::objc_loadWeakRetained:
313 Changed |= lowerObjCCall(F, "objc_loadWeakRetained");
314 break;
315 case Intrinsic::objc_moveWeak:
316 Changed |= lowerObjCCall(F, "objc_moveWeak");
317 break;
318 case Intrinsic::objc_release:
319 Changed |= lowerObjCCall(F, "objc_release", true);
320 break;
321 case Intrinsic::objc_retain:
322 Changed |= lowerObjCCall(F, "objc_retain", true);
323 break;
324 case Intrinsic::objc_retainAutorelease:
325 Changed |= lowerObjCCall(F, "objc_retainAutorelease");
326 break;
327 case Intrinsic::objc_retainAutoreleaseReturnValue:
328 Changed |= lowerObjCCall(F, "objc_retainAutoreleaseReturnValue");
329 break;
330 case Intrinsic::objc_retainAutoreleasedReturnValue:
331 Changed |= lowerObjCCall(F, "objc_retainAutoreleasedReturnValue");
332 break;
333 case Intrinsic::objc_retainBlock:
334 Changed |= lowerObjCCall(F, "objc_retainBlock");
335 break;
336 case Intrinsic::objc_storeStrong:
337 Changed |= lowerObjCCall(F, "objc_storeStrong");
338 break;
339 case Intrinsic::objc_storeWeak:
340 Changed |= lowerObjCCall(F, "objc_storeWeak");
341 break;
342 case Intrinsic::objc_unsafeClaimAutoreleasedReturnValue:
343 Changed |= lowerObjCCall(F, "objc_unsafeClaimAutoreleasedReturnValue");
344 break;
345 case Intrinsic::objc_retainedObject:
346 Changed |= lowerObjCCall(F, "objc_retainedObject");
347 break;
348 case Intrinsic::objc_unretainedObject:
349 Changed |= lowerObjCCall(F, "objc_unretainedObject");
350 break;
351 case Intrinsic::objc_unretainedPointer:
352 Changed |= lowerObjCCall(F, "objc_unretainedPointer");
353 break;
354 case Intrinsic::objc_retain_autorelease:
355 Changed |= lowerObjCCall(F, "objc_retain_autorelease");
356 break;
357 case Intrinsic::objc_sync_enter:
358 Changed |= lowerObjCCall(F, "objc_sync_enter");
359 break;
360 case Intrinsic::objc_sync_exit:
361 Changed |= lowerObjCCall(F, "objc_sync_exit");
362 break;
363 }
364 }
365 return Changed;
366}
367
368namespace {
369
370class PreISelIntrinsicLoweringLegacyPass : public ModulePass {
371public:
372 static char ID;
373
374 PreISelIntrinsicLoweringLegacyPass() : ModulePass(ID) {}
375
376 void getAnalysisUsage(AnalysisUsage &AU) const override {
379 }
380
381 bool runOnModule(Module &M) override {
382 auto LookupTTI = [this](Function &F) -> TargetTransformInfo & {
383 return this->getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
384 };
385
386 const auto &TM = getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
387 PreISelIntrinsicLowering Lowering(TM, LookupTTI);
388 return Lowering.lowerIntrinsics(M);
389 }
390};
391
392} // end anonymous namespace
393
394char PreISelIntrinsicLoweringLegacyPass::ID;
395
396INITIALIZE_PASS_BEGIN(PreISelIntrinsicLoweringLegacyPass,
397 "pre-isel-intrinsic-lowering",
398 "Pre-ISel Intrinsic Lowering", false, false)
401INITIALIZE_PASS_END(PreISelIntrinsicLoweringLegacyPass,
402 "pre-isel-intrinsic-lowering",
403 "Pre-ISel Intrinsic Lowering", false, false)
404
406 return new PreISelIntrinsicLoweringLegacyPass();
407}
408
411 auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
412
413 auto LookupTTI = [&FAM](Function &F) -> TargetTransformInfo & {
415 };
416
417 PreISelIntrinsicLowering Lowering(TM, LookupTTI);
418 if (!Lowering.lowerIntrinsics(M))
419 return PreservedAnalyses::all();
420 else
422}
amdgpu isel
static bool setNonLazyBind(Function &F)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static bool lowerIntrinsics(Module &M)
uint64_t Size
#define F(x, y, z)
Definition: MD5.cpp:55
Module.h This file contains the declarations for the Module class.
IntegerType * Int32Ty
This file defines ARC utility functions which are used by various parts of the compiler.
FunctionAnalysisManager FAM
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:55
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:59
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:52
static cl::opt< int64_t > MemIntrinsicExpandSizeThresholdOpt("mem-intrinsic-expand-size", cl::desc("Set minimum mem intrinsic size to expand in IR"), cl::init(-1), cl::Hidden)
Threshold to leave statically sized memory intrinsic calls.
pre isel intrinsic lowering
pre isel intrinsic Pre ISel Intrinsic Lowering
static bool lowerObjCCall(Function &F, const char *NewFn, bool setNonLazyBind=false)
static CallInst::TailCallKind getOverridingTailCallKind(const Function &F)
static bool lowerLoadRelative(Function &F)
static bool canEmitLibcall(const TargetMachine &TM, Function *F, RTLIB::Libcall LC)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file describes how to lower LLVM code to machine code.
Target-Independent Code Generator Pass Configuration Options pass.
This pass exposes codegen information to IR-level passes.
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:348
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:500
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
Definition: InstrTypes.h:1825
This class represents a function call, abstracting a target machine's calling convention.
void setTailCallKind(TailCallKind TCK)
This is the shared class of boolean and integer constants.
Definition: Constants.h:79
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:153
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
Definition: DerivedTypes.h:168
const Function & getFunction() const
Definition: Function.h:160
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2395
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2649
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
Definition: PassManager.h:658
static bool mayLowerToFunctionCall(Intrinsic::ID IID)
Check if the intrinsic might lower into a regular function call in the course of IR transformations.
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
Definition: Pass.h:251
virtual bool runOnModule(Module &M)=0
runOnModule - Virtual method overriden by subclasses to process the module being operated on.
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
virtual void getAnalysisUsage(AnalysisUsage &) const
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
Definition: Pass.cpp:98
A set of analyses that are preserved following a run of a transformation pass.
Definition: Analysis.h:109
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition: Analysis.h:112
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: Analysis.h:115
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
Analysis pass providing the TargetTransformInfo.
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:76
Target-Independent Code Generator Pass Configuration Options.
Wrapper pass for TargetTransformInfo.
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
uint64_t getMaxMemIntrinsicInlineSizeThreshold() const
Returns the maximum memset / memcpy size in bytes that still makes it profitable to inline the call.
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
static IntegerType * getInt32Ty(LLVMContext &C)
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
LLVM Value Representation.
Definition: Value.h:74
void setName(const Twine &Name)
Change the name of the value.
Definition: Value.cpp:377
An efficient, type-erasing, non-owning reference to a callable.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:450
ARCInstKind getAttachedARCFunctionKind(const CallBase *CB)
This function returns the ARCInstKind of the function attached to operand bundle clang_arc_attachedca...
Definition: ObjCARCUtil.h:60
bool IsNeverTail(ARCInstKind Class)
Test if the given class represents instructions which are never safe to mark with the "tail" keyword.
bool IsAlwaysTail(ARCInstKind Class)
Test if the given class represents instructions which are always safe to mark with the "tail" keyword...
ARCInstKind
Equivalence classes of instructions in the ARC Model.
ARCInstKind GetFunctionClass(const Function *F)
Determine if F is one of the special known Functions.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
bool expandMemMoveAsLoop(MemMoveInst *MemMove, const TargetTransformInfo &TTI)
Expand MemMove as a loop.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:665
ModulePass * createPreISelIntrinsicLoweringPass()
This pass lowers the @llvm.load.relative and @llvm.objc.
void expandMemCpyAsLoop(MemCpyInst *MemCpy, const TargetTransformInfo &TTI, ScalarEvolution *SE=nullptr)
Expand MemCpy as a loop. MemCpy is not deleted.
void expandMemSetAsLoop(MemSetInst *MemSet)
Expand MemSet as a loop. MemSet is not deleted.
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)