LLVM 19.0.0git
NVPTXCtorDtorLowering.cpp
Go to the documentation of this file.
1//===-- NVPTXCtorDtorLowering.cpp - Handle global ctors and dtors --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This pass creates a unified init and fini kernel with the required metadata
11//===----------------------------------------------------------------------===//
12
15#include "NVPTX.h"
17#include "llvm/IR/Constants.h"
18#include "llvm/IR/Function.h"
20#include "llvm/IR/IRBuilder.h"
21#include "llvm/IR/Module.h"
22#include "llvm/IR/Value.h"
23#include "llvm/Pass.h"
25#include "llvm/Support/MD5.h"
27
28using namespace llvm;
29
30#define DEBUG_TYPE "nvptx-lower-ctor-dtor"
31
33 GlobalStr("nvptx-lower-global-ctor-dtor-id",
34 cl::desc("Override unique ID of ctor/dtor globals."),
35 cl::init(""), cl::Hidden);
36
37static cl::opt<bool>
38 CreateKernels("nvptx-emit-init-fini-kernel",
39 cl::desc("Emit kernels to call ctor/dtor globals."),
40 cl::init(true), cl::Hidden);
41
42namespace {
43
44static std::string getHash(StringRef Str) {
45 llvm::MD5 Hasher;
47 Hasher.update(Str);
48 Hasher.final(Hash);
49 return llvm::utohexstr(Hash.low(), /*LowerCase=*/true);
50}
51
52static void addKernelMetadata(Module &M, GlobalValue *GV) {
53 llvm::LLVMContext &Ctx = M.getContext();
54
55 // Get "nvvm.annotations" metadata node.
56 llvm::NamedMDNode *MD = M.getOrInsertNamedMetadata("nvvm.annotations");
57
58 llvm::Metadata *KernelMDVals[] = {
61 llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), 1))};
62
63 // This kernel is only to be called single-threaded.
64 llvm::Metadata *ThreadXMDVals[] = {
67 llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), 1))};
68 llvm::Metadata *ThreadYMDVals[] = {
71 llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), 1))};
72 llvm::Metadata *ThreadZMDVals[] = {
75 llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), 1))};
76
77 llvm::Metadata *BlockMDVals[] = {
79 llvm::MDString::get(Ctx, "maxclusterrank"),
81 llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), 1))};
82
83 // Append metadata to nvvm.annotations.
84 MD->addOperand(llvm::MDNode::get(Ctx, KernelMDVals));
85 MD->addOperand(llvm::MDNode::get(Ctx, ThreadXMDVals));
86 MD->addOperand(llvm::MDNode::get(Ctx, ThreadYMDVals));
87 MD->addOperand(llvm::MDNode::get(Ctx, ThreadZMDVals));
88 MD->addOperand(llvm::MDNode::get(Ctx, BlockMDVals));
89}
90
91static Function *createInitOrFiniKernelFunction(Module &M, bool IsCtor) {
92 StringRef InitOrFiniKernelName =
93 IsCtor ? "nvptx$device$init" : "nvptx$device$fini";
94 if (M.getFunction(InitOrFiniKernelName))
95 return nullptr;
96
97 Function *InitOrFiniKernel = Function::createWithDefaultAttr(
98 FunctionType::get(Type::getVoidTy(M.getContext()), false),
99 GlobalValue::WeakODRLinkage, 0, InitOrFiniKernelName, &M);
100 addKernelMetadata(M, InitOrFiniKernel);
101
102 return InitOrFiniKernel;
103}
104
105// We create the IR required to call each callback in this section. This is
106// equivalent to the following code. Normally, the linker would provide us with
107// the definitions of the init and fini array sections. The 'nvlink' linker does
108// not do this so initializing these values is done by the runtime.
109//
110// extern "C" void **__init_array_start = nullptr;
111// extern "C" void **__init_array_end = nullptr;
112// extern "C" void **__fini_array_start = nullptr;
113// extern "C" void **__fini_array_end = nullptr;
114//
115// using InitCallback = void();
116// using FiniCallback = void();
117//
118// void call_init_array_callbacks() {
119// for (auto start = __init_array_start; start != __init_array_end; ++start)
120// reinterpret_cast<InitCallback *>(*start)();
121// }
122//
123// void call_init_array_callbacks() {
124// size_t fini_array_size = __fini_array_end - __fini_array_start;
125// for (size_t i = fini_array_size; i > 0; --i)
126// reinterpret_cast<FiniCallback *>(__fini_array_start[i - 1])();
127// }
128static void createInitOrFiniCalls(Function &F, bool IsCtor) {
129 Module &M = *F.getParent();
130 LLVMContext &C = M.getContext();
131
132 IRBuilder<> IRB(BasicBlock::Create(C, "entry", &F));
133 auto *LoopBB = BasicBlock::Create(C, "while.entry", &F);
134 auto *ExitBB = BasicBlock::Create(C, "while.end", &F);
135 Type *PtrTy = IRB.getPtrTy(llvm::ADDRESS_SPACE_GLOBAL);
136
137 auto *Begin = M.getOrInsertGlobal(
138 IsCtor ? "__init_array_start" : "__fini_array_start",
139 PointerType::get(C, 0), [&]() {
140 auto *GV = new GlobalVariable(
141 M, PointerType::get(C, 0),
142 /*isConstant=*/false, GlobalValue::WeakAnyLinkage,
143 Constant::getNullValue(PointerType::get(C, 0)),
144 IsCtor ? "__init_array_start" : "__fini_array_start",
145 /*InsertBefore=*/nullptr, GlobalVariable::NotThreadLocal,
146 /*AddressSpace=*/llvm::ADDRESS_SPACE_GLOBAL);
147 GV->setVisibility(GlobalVariable::ProtectedVisibility);
148 return GV;
149 });
150 auto *End = M.getOrInsertGlobal(
151 IsCtor ? "__init_array_end" : "__fini_array_end", PointerType::get(C, 0),
152 [&]() {
153 auto *GV = new GlobalVariable(
154 M, PointerType::get(C, 0),
155 /*isConstant=*/false, GlobalValue::WeakAnyLinkage,
156 Constant::getNullValue(PointerType::get(C, 0)),
157 IsCtor ? "__init_array_end" : "__fini_array_end",
158 /*InsertBefore=*/nullptr, GlobalVariable::NotThreadLocal,
159 /*AddressSpace=*/llvm::ADDRESS_SPACE_GLOBAL);
160 GV->setVisibility(GlobalVariable::ProtectedVisibility);
161 return GV;
162 });
163
164 // The constructor type is suppoed to allow using the argument vectors, but
165 // for now we just call them with no arguments.
166 auto *CallBackTy = FunctionType::get(IRB.getVoidTy(), {});
167
168 // The destructor array must be called in reverse order. Get an expression to
169 // the end of the array and iterate backwards in that case.
170 Value *BeginVal = IRB.CreateLoad(Begin->getType(), Begin, "begin");
171 Value *EndVal = IRB.CreateLoad(Begin->getType(), End, "stop");
172 if (!IsCtor) {
173 auto *BeginInt = IRB.CreatePtrToInt(BeginVal, IntegerType::getInt64Ty(C));
174 auto *EndInt = IRB.CreatePtrToInt(EndVal, IntegerType::getInt64Ty(C));
175 auto *SubInst = IRB.CreateSub(EndInt, BeginInt);
176 auto *Offset = IRB.CreateAShr(
177 SubInst, ConstantInt::get(IntegerType::getInt64Ty(C), 3), "offset",
178 /*IsExact=*/true);
179 auto *ValuePtr = IRB.CreateGEP(PointerType::get(C, 0), BeginVal,
181 EndVal = BeginVal;
182 BeginVal = IRB.CreateInBoundsGEP(
183 PointerType::get(C, 0), ValuePtr,
184 ArrayRef<Value *>(ConstantInt::get(IntegerType::getInt64Ty(C), -1)),
185 "start");
186 }
187 IRB.CreateCondBr(
188 IRB.CreateCmp(IsCtor ? ICmpInst::ICMP_NE : ICmpInst::ICMP_UGT, BeginVal,
189 EndVal),
190 LoopBB, ExitBB);
191 IRB.SetInsertPoint(LoopBB);
192 auto *CallBackPHI = IRB.CreatePHI(PtrTy, 2, "ptr");
193 auto *CallBack = IRB.CreateLoad(IRB.getPtrTy(F.getAddressSpace()),
194 CallBackPHI, "callback");
195 IRB.CreateCall(CallBackTy, CallBack);
196 auto *NewCallBack =
197 IRB.CreateConstGEP1_64(PtrTy, CallBackPHI, IsCtor ? 1 : -1, "next");
198 auto *EndCmp = IRB.CreateCmp(IsCtor ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_ULT,
199 NewCallBack, EndVal, "end");
200 CallBackPHI->addIncoming(BeginVal, &F.getEntryBlock());
201 CallBackPHI->addIncoming(NewCallBack, LoopBB);
202 IRB.CreateCondBr(EndCmp, ExitBB, LoopBB);
203 IRB.SetInsertPoint(ExitBB);
204 IRB.CreateRetVoid();
205}
206
207static bool createInitOrFiniGlobals(Module &M, GlobalVariable *GV,
208 bool IsCtor) {
209 ConstantArray *GA = dyn_cast<ConstantArray>(GV->getInitializer());
210 if (!GA || GA->getNumOperands() == 0)
211 return false;
212
213 // NVPTX has no way to emit variables at specific sections or support for
214 // the traditional constructor sections. Instead, we emit mangled global
215 // names so the runtime can build the list manually.
216 for (Value *V : GA->operands()) {
217 auto *CS = cast<ConstantStruct>(V);
218 auto *F = cast<Constant>(CS->getOperand(1));
219 uint64_t Priority = cast<ConstantInt>(CS->getOperand(0))->getSExtValue();
220 std::string PriorityStr = "." + std::to_string(Priority);
221 // We append a semi-unique hash and the priority to the global name.
222 std::string GlobalID =
223 !GlobalStr.empty() ? GlobalStr : getHash(M.getSourceFileName());
224 std::string NameStr =
225 ((IsCtor ? "__init_array_object_" : "__fini_array_object_") +
226 F->getName() + "_" + GlobalID + "_" + std::to_string(Priority))
227 .str();
228 // PTX does not support exported names with '.' in them.
229 llvm::transform(NameStr, NameStr.begin(),
230 [](char c) { return c == '.' ? '_' : c; });
231
232 auto *GV = new GlobalVariable(M, F->getType(), /*IsConstant=*/true,
235 /*AddressSpace=*/4);
236 // This isn't respected by Nvidia, simply put here for clarity.
237 GV->setSection(IsCtor ? ".init_array" + PriorityStr
238 : ".fini_array" + PriorityStr);
239 GV->setVisibility(GlobalVariable::ProtectedVisibility);
240 appendToUsed(M, {GV});
241 }
242
243 return true;
244}
245
246static bool createInitOrFiniKernel(Module &M, StringRef GlobalName,
247 bool IsCtor) {
248 GlobalVariable *GV = M.getGlobalVariable(GlobalName);
249 if (!GV || !GV->hasInitializer())
250 return false;
251
252 if (!createInitOrFiniGlobals(M, GV, IsCtor))
253 return false;
254
255 if (!CreateKernels)
256 return true;
257
258 Function *InitOrFiniKernel = createInitOrFiniKernelFunction(M, IsCtor);
259 if (!InitOrFiniKernel)
260 return false;
261
262 createInitOrFiniCalls(*InitOrFiniKernel, IsCtor);
263
264 GV->eraseFromParent();
265 return true;
266}
267
268static bool lowerCtorsAndDtors(Module &M) {
269 bool Modified = false;
270 Modified |= createInitOrFiniKernel(M, "llvm.global_ctors", /*IsCtor =*/true);
271 Modified |= createInitOrFiniKernel(M, "llvm.global_dtors", /*IsCtor =*/false);
272 return Modified;
273}
274
275class NVPTXCtorDtorLoweringLegacy final : public ModulePass {
276public:
277 static char ID;
278 NVPTXCtorDtorLoweringLegacy() : ModulePass(ID) {}
279 bool runOnModule(Module &M) override { return lowerCtorsAndDtors(M); }
280};
281
282} // End anonymous namespace
283
286 return lowerCtorsAndDtors(M) ? PreservedAnalyses::none()
288}
289
290char NVPTXCtorDtorLoweringLegacy::ID = 0;
291char &llvm::NVPTXCtorDtorLoweringLegacyPassID = NVPTXCtorDtorLoweringLegacy::ID;
292INITIALIZE_PASS(NVPTXCtorDtorLoweringLegacy, DEBUG_TYPE,
293 "Lower ctors and dtors for NVPTX", false, false)
294
296 return new NVPTXCtorDtorLoweringLegacy();
297}
This file contains the declarations for the subclasses of Constant, which represent the different fla...
bool End
Definition: ELF_riscv.cpp:480
#define DEBUG_TYPE
#define F(x, y, z)
Definition: MD5.cpp:55
Module.h This file contains the declarations for the Module class.
static cl::opt< bool > CreateKernels("nvptx-emit-init-fini-kernel", cl::desc("Emit kernels to call ctor/dtor globals."), cl::init(true), cl::Hidden)
static cl::opt< std::string > GlobalStr("nvptx-lower-global-ctor-dtor-id", cl::desc("Override unique ID of ctor/dtor globals."), cl::init(""), cl::Hidden)
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:38
This file contains some functions that are useful when dealing with strings.
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:321
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:199
ConstantArray - Constant Array Declarations.
Definition: Constants.h:424
static ConstantAsMetadata * get(Constant *C)
Definition: Metadata.h:528
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition: Constants.cpp:370
static Function * createWithDefaultAttr(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Creates a function with some attributes recorded in llvm.module.flags applied.
Definition: Function.cpp:373
void setSection(StringRef S)
Change the section for this global.
Definition: Globals.cpp:263
void setVisibility(VisibilityTypes V)
Definition: GlobalValue.h:253
@ WeakODRLinkage
Same, but only replaced by something equivalent.
Definition: GlobalValue.h:56
@ ExternalLinkage
Externally visible function.
Definition: GlobalValue.h:51
@ WeakAnyLinkage
Keep one copy of named function when linking (weak)
Definition: GlobalValue.h:55
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
bool hasInitializer() const
Definitions have initializers, declarations don't.
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
Definition: Globals.cpp:467
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2666
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
Definition: MD5.h:41
void update(ArrayRef< uint8_t > Data)
Updates the hash for the byte stream provided.
Definition: MD5.cpp:189
void final(MD5Result &Result)
Finishes off the hash and puts the result in result.
Definition: MD5.cpp:234
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition: Metadata.h:1541
static MDString * get(LLVMContext &Context, StringRef Str)
Definition: Metadata.cpp:600
Root of the metadata hierarchy.
Definition: Metadata.h:62
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
Definition: Pass.h:251
virtual bool runOnModule(Module &M)=0
runOnModule - Virtual method overriden by subclasses to process the module being operated on.
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
A tuple of MDNodes.
Definition: Metadata.h:1729
void addOperand(MDNode *M)
Definition: Metadata.cpp:1387
A set of analyses that are preserved following a run of a transformation pass.
Definition: Analysis.h:109
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition: Analysis.h:112
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: Analysis.h:115
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt32Ty(LLVMContext &C)
op_range operands()
Definition: User.h:242
unsigned getNumOperands() const
Definition: User.h:191
LLVM Value Representation.
Definition: Value.h:74
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:450
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:456
@ ADDRESS_SPACE_GLOBAL
Definition: NVPTXBaseInfo.h:23
OutputIt transform(R &&Range, OutputIt d_first, UnaryFunction F)
Wrapper function around std::transform to apply a function to a range and store the result elsewhere.
Definition: STLExtras.h:1928
char & NVPTXCtorDtorLoweringLegacyPassID
ModulePass * createNVPTXCtorDtorLoweringLegacyPass()
void appendToUsed(Module &M, ArrayRef< GlobalValue * > Values)
Adds global values to the llvm.used list.
uint64_t low() const
Definition: MD5.h:46