LLVM 20.0.0git
NVPTXUtilities.cpp
Go to the documentation of this file.
1//===- NVPTXUtilities.cpp - Utility Functions -----------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains miscellaneous utility functions
10//
11//===----------------------------------------------------------------------===//
12
13#include "NVPTXUtilities.h"
14#include "NVPTX.h"
15#include "NVPTXTargetMachine.h"
16#include "llvm/ADT/StringRef.h"
17#include "llvm/IR/Constants.h"
18#include "llvm/IR/Function.h"
20#include "llvm/IR/Module.h"
22#include "llvm/Support/Mutex.h"
23#include <cstring>
24#include <map>
25#include <mutex>
26#include <optional>
27#include <string>
28#include <vector>
29
30namespace llvm {
31
32namespace {
33typedef std::map<std::string, std::vector<unsigned>> key_val_pair_t;
34typedef std::map<const GlobalValue *, key_val_pair_t> global_val_annot_t;
35
36struct AnnotationCache {
37 sys::Mutex Lock;
38 std::map<const Module *, global_val_annot_t> Cache;
39};
40
41AnnotationCache &getAnnotationCache() {
42 static AnnotationCache AC;
43 return AC;
44}
45} // anonymous namespace
46
48 auto &AC = getAnnotationCache();
49 std::lock_guard<sys::Mutex> Guard(AC.Lock);
50 AC.Cache.erase(Mod);
51}
52
53static void readIntVecFromMDNode(const MDNode *MetadataNode,
54 std::vector<unsigned> &Vec) {
55 for (unsigned i = 0, e = MetadataNode->getNumOperands(); i != e; ++i) {
56 ConstantInt *Val =
57 mdconst::extract<ConstantInt>(MetadataNode->getOperand(i));
58 Vec.push_back(Val->getZExtValue());
59 }
60}
61
62static void cacheAnnotationFromMD(const MDNode *MetadataNode,
63 key_val_pair_t &retval) {
64 auto &AC = getAnnotationCache();
65 std::lock_guard<sys::Mutex> Guard(AC.Lock);
66 assert(MetadataNode && "Invalid mdnode for annotation");
67 assert((MetadataNode->getNumOperands() % 2) == 1 &&
68 "Invalid number of operands");
69 // start index = 1, to skip the global variable key
70 // increment = 2, to skip the value for each property-value pairs
71 for (unsigned i = 1, e = MetadataNode->getNumOperands(); i != e; i += 2) {
72 // property
73 const MDString *prop = dyn_cast<MDString>(MetadataNode->getOperand(i));
74 assert(prop && "Annotation property not a string");
75 std::string Key = prop->getString().str();
76
77 // value
78 if (ConstantInt *Val = mdconst::dyn_extract<ConstantInt>(
79 MetadataNode->getOperand(i + 1))) {
80 retval[Key].push_back(Val->getZExtValue());
81 } else if (MDNode *VecMd =
82 dyn_cast<MDNode>(MetadataNode->getOperand(i + 1))) {
83 // note: only "grid_constant" annotations support vector MDNodes.
84 // assert: there can only exist one unique key value pair of
85 // the form (string key, MDNode node). Operands of such a node
86 // shall always be unsigned ints.
87 auto [It, Inserted] = retval.try_emplace(Key);
88 if (Inserted) {
89 readIntVecFromMDNode(VecMd, It->second);
90 continue;
91 }
92 } else {
93 llvm_unreachable("Value operand not a constant int or an mdnode");
94 }
95 }
96}
97
98static void cacheAnnotationFromMD(const Module *m, const GlobalValue *gv) {
99 auto &AC = getAnnotationCache();
100 std::lock_guard<sys::Mutex> Guard(AC.Lock);
101 NamedMDNode *NMD = m->getNamedMetadata("nvvm.annotations");
102 if (!NMD)
103 return;
104 key_val_pair_t tmp;
105 for (unsigned i = 0, e = NMD->getNumOperands(); i != e; ++i) {
106 const MDNode *elem = NMD->getOperand(i);
107
108 GlobalValue *entity =
109 mdconst::dyn_extract_or_null<GlobalValue>(elem->getOperand(0));
110 // entity may be null due to DCE
111 if (!entity)
112 continue;
113 if (entity != gv)
114 continue;
115
116 // accumulate annotations for entity in tmp
117 cacheAnnotationFromMD(elem, tmp);
118 }
119
120 if (tmp.empty()) // no annotations for this gv
121 return;
122
123 AC.Cache[m][gv] = std::move(tmp);
124}
125
126static std::optional<unsigned> findOneNVVMAnnotation(const GlobalValue *gv,
127 const std::string &prop) {
128 auto &AC = getAnnotationCache();
129 std::lock_guard<sys::Mutex> Guard(AC.Lock);
130 const Module *m = gv->getParent();
131 if (AC.Cache.find(m) == AC.Cache.end())
133 else if (AC.Cache[m].find(gv) == AC.Cache[m].end())
135 if (AC.Cache[m][gv].find(prop) == AC.Cache[m][gv].end())
136 return std::nullopt;
137 return AC.Cache[m][gv][prop][0];
138}
139
140static bool findAllNVVMAnnotation(const GlobalValue *gv,
141 const std::string &prop,
142 std::vector<unsigned> &retval) {
143 auto &AC = getAnnotationCache();
144 std::lock_guard<sys::Mutex> Guard(AC.Lock);
145 const Module *m = gv->getParent();
146 if (AC.Cache.find(m) == AC.Cache.end())
148 else if (AC.Cache[m].find(gv) == AC.Cache[m].end())
150 if (AC.Cache[m][gv].find(prop) == AC.Cache[m][gv].end())
151 return false;
152 retval = AC.Cache[m][gv][prop];
153 return true;
154}
155
156static bool globalHasNVVMAnnotation(const Value &V, const std::string &Prop) {
157 if (const auto *GV = dyn_cast<GlobalValue>(&V))
158 if (const auto Annot = findOneNVVMAnnotation(GV, Prop)) {
159 assert((*Annot == 1) && "Unexpected annotation on a symbol");
160 return true;
161 }
162
163 return false;
164}
165
166static bool argHasNVVMAnnotation(const Value &Val,
167 const std::string &Annotation,
168 const bool StartArgIndexAtOne = false) {
169 if (const Argument *Arg = dyn_cast<Argument>(&Val)) {
170 const Function *Func = Arg->getParent();
171 std::vector<unsigned> Annot;
172 if (findAllNVVMAnnotation(Func, Annotation, Annot)) {
173 const unsigned BaseOffset = StartArgIndexAtOne ? 1 : 0;
174 if (is_contained(Annot, BaseOffset + Arg->getArgNo())) {
175 return true;
176 }
177 }
178 }
179 return false;
180}
181
183 if (const Argument *Arg = dyn_cast<Argument>(&V)) {
184 // "grid_constant" counts argument indices starting from 1
185 if (Arg->hasByValAttr() &&
186 argHasNVVMAnnotation(*Arg, "grid_constant",
187 /*StartArgIndexAtOne*/ true)) {
188 assert(isKernelFunction(*Arg->getParent()) &&
189 "only kernel arguments can be grid_constant");
190 return true;
191 }
192 }
193 return false;
194}
195
196bool isTexture(const Value &V) { return globalHasNVVMAnnotation(V, "texture"); }
197
198bool isSurface(const Value &V) { return globalHasNVVMAnnotation(V, "surface"); }
199
200bool isSampler(const Value &V) {
201 const char *AnnotationName = "sampler";
202
203 return globalHasNVVMAnnotation(V, AnnotationName) ||
204 argHasNVVMAnnotation(V, AnnotationName);
205}
206
207bool isImageReadOnly(const Value &V) {
208 return argHasNVVMAnnotation(V, "rdoimage");
209}
210
211bool isImageWriteOnly(const Value &V) {
212 return argHasNVVMAnnotation(V, "wroimage");
213}
214
215bool isImageReadWrite(const Value &V) {
216 return argHasNVVMAnnotation(V, "rdwrimage");
217}
218
219bool isImage(const Value &V) {
221}
222
223bool isManaged(const Value &V) { return globalHasNVVMAnnotation(V, "managed"); }
224
226 assert(V.hasName() && "Found texture variable with no name");
227 return V.getName();
228}
229
231 assert(V.hasName() && "Found surface variable with no name");
232 return V.getName();
233}
234
236 assert(V.hasName() && "Found sampler variable with no name");
237 return V.getName();
238}
239
240std::optional<unsigned> getMaxNTIDx(const Function &F) {
241 return findOneNVVMAnnotation(&F, "maxntidx");
242}
243
244std::optional<unsigned> getMaxNTIDy(const Function &F) {
245 return findOneNVVMAnnotation(&F, "maxntidy");
246}
247
248std::optional<unsigned> getMaxNTIDz(const Function &F) {
249 return findOneNVVMAnnotation(&F, "maxntidz");
250}
251
252std::optional<unsigned> getMaxNTID(const Function &F) {
253 // Note: The semantics here are a bit strange. The PTX ISA states the
254 // following (11.4.2. Performance-Tuning Directives: .maxntid):
255 //
256 // Note that this directive guarantees that the total number of threads does
257 // not exceed the maximum, but does not guarantee that the limit in any
258 // particular dimension is not exceeded.
259 std::optional<unsigned> MaxNTIDx = getMaxNTIDx(F);
260 std::optional<unsigned> MaxNTIDy = getMaxNTIDy(F);
261 std::optional<unsigned> MaxNTIDz = getMaxNTIDz(F);
262 if (MaxNTIDx || MaxNTIDy || MaxNTIDz)
263 return MaxNTIDx.value_or(1) * MaxNTIDy.value_or(1) * MaxNTIDz.value_or(1);
264 return std::nullopt;
265}
266
267std::optional<unsigned> getClusterDimx(const Function &F) {
268 return findOneNVVMAnnotation(&F, "cluster_dim_x");
269}
270
271std::optional<unsigned> getClusterDimy(const Function &F) {
272 return findOneNVVMAnnotation(&F, "cluster_dim_y");
273}
274
275std::optional<unsigned> getClusterDimz(const Function &F) {
276 return findOneNVVMAnnotation(&F, "cluster_dim_z");
277}
278
279std::optional<unsigned> getMaxClusterRank(const Function &F) {
280 return findOneNVVMAnnotation(&F, "maxclusterrank");
281}
282
283std::optional<unsigned> getReqNTIDx(const Function &F) {
284 return findOneNVVMAnnotation(&F, "reqntidx");
285}
286
287std::optional<unsigned> getReqNTIDy(const Function &F) {
288 return findOneNVVMAnnotation(&F, "reqntidy");
289}
290
291std::optional<unsigned> getReqNTIDz(const Function &F) {
292 return findOneNVVMAnnotation(&F, "reqntidz");
293}
294
295std::optional<unsigned> getReqNTID(const Function &F) {
296 // Note: The semantics here are a bit strange. See getMaxNTID.
297 std::optional<unsigned> ReqNTIDx = getReqNTIDx(F);
298 std::optional<unsigned> ReqNTIDy = getReqNTIDy(F);
299 std::optional<unsigned> ReqNTIDz = getReqNTIDz(F);
300 if (ReqNTIDx || ReqNTIDy || ReqNTIDz)
301 return ReqNTIDx.value_or(1) * ReqNTIDy.value_or(1) * ReqNTIDz.value_or(1);
302 return std::nullopt;
303}
304
305std::optional<unsigned> getMinCTASm(const Function &F) {
306 return findOneNVVMAnnotation(&F, "minctasm");
307}
308
309std::optional<unsigned> getMaxNReg(const Function &F) {
310 return findOneNVVMAnnotation(&F, "maxnreg");
311}
312
314 if (const auto X = findOneNVVMAnnotation(&F, "kernel"))
315 return (*X == 1);
316
317 // There is no NVVM metadata, check the calling convention
318 return F.getCallingConv() == CallingConv::PTX_Kernel;
319}
320
321MaybeAlign getAlign(const Function &F, unsigned Index) {
322 // First check the alignstack metadata
323 if (MaybeAlign StackAlign =
324 F.getAttributes().getAttributes(Index).getStackAlignment())
325 return StackAlign;
326
327 // If that is missing, check the legacy nvvm metadata
328 std::vector<unsigned> Vs;
329 bool retval = findAllNVVMAnnotation(&F, "align", Vs);
330 if (!retval)
331 return std::nullopt;
332 for (unsigned V : Vs)
333 if ((V >> 16) == Index)
334 return Align(V & 0xFFFF);
335
336 return std::nullopt;
337}
338
339MaybeAlign getAlign(const CallInst &I, unsigned Index) {
340 // First check the alignstack metadata
341 if (MaybeAlign StackAlign =
342 I.getAttributes().getAttributes(Index).getStackAlignment())
343 return StackAlign;
344
345 // If that is missing, check the legacy nvvm metadata
346 if (MDNode *alignNode = I.getMetadata("callalign")) {
347 for (int i = 0, n = alignNode->getNumOperands(); i < n; i++) {
348 if (const ConstantInt *CI =
349 mdconst::dyn_extract<ConstantInt>(alignNode->getOperand(i))) {
350 unsigned V = CI->getZExtValue();
351 if ((V >> 16) == Index)
352 return Align(V & 0xFFFF);
353 if ((V >> 16) > Index)
354 return std::nullopt;
355 }
356 }
357 }
358 return std::nullopt;
359}
360
362 return dyn_cast<Function>(CB->getCalledOperand()->stripPointerCasts());
363}
364
365bool shouldEmitPTXNoReturn(const Value *V, const TargetMachine &TM) {
366 const auto &ST =
367 *static_cast<const NVPTXTargetMachine &>(TM).getSubtargetImpl();
368 if (!ST.hasNoReturn())
369 return false;
370
371 assert((isa<Function>(V) || isa<CallInst>(V)) &&
372 "Expect either a call instruction or a function");
373
374 if (const CallInst *CallI = dyn_cast<CallInst>(V))
375 return CallI->doesNotReturn() &&
376 CallI->getFunctionType()->getReturnType()->isVoidTy();
377
378 const Function *F = cast<Function>(V);
379 return F->doesNotReturn() &&
380 F->getFunctionType()->getReturnType()->isVoidTy() &&
382}
383
384bool Isv2x16VT(EVT VT) {
385 return (VT == MVT::v2f16 || VT == MVT::v2bf16 || VT == MVT::v2i16);
386}
387
388} // namespace llvm
This file contains the declarations for the subclasses of Constant, which represent the different fla...
uint32_t Index
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
Module.h This file contains the declarations for the Module class.
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This class represents an incoming formal argument to a Function.
Definition: Argument.h:31
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1120
Value * getCalledOperand() const
Definition: InstrTypes.h:1342
This class represents a function call, abstracting a target machine's calling convention.
This is the shared class of boolean and integer constants.
Definition: Constants.h:83
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:157
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:656
Metadata node.
Definition: Metadata.h:1069
const MDOperand & getOperand(unsigned I) const
Definition: Metadata.h:1430
unsigned getNumOperands() const
Return number of MDNode operands.
Definition: Metadata.h:1436
A single uniqued string.
Definition: Metadata.h:720
StringRef getString() const
Definition: Metadata.cpp:616
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
NamedMDNode * getNamedMetadata(StringRef Name) const
Return the first NamedMDNode in the module with the specified name.
Definition: Module.cpp:297
A tuple of MDNodes.
Definition: Metadata.h:1731
MDNode * getOperand(unsigned i) const
Definition: Metadata.cpp:1425
unsigned getNumOperands() const
Definition: Metadata.cpp:1421
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:51
std::string str() const
str - Get the contents as an std::string.
Definition: StringRef.h:229
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:77
LLVM Value Representation.
Definition: Value.h:74
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition: Value.cpp:694
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ PTX_Kernel
Call to a PTX kernel. Passes all arguments in parameter space.
Definition: CallingConv.h:125
SmartMutex< false > Mutex
Mutex - A standard, always enforced mutex.
Definition: Mutex.h:66
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
bool isManaged(const Value &V)
bool shouldEmitPTXNoReturn(const Value *V, const TargetMachine &TM)
static bool globalHasNVVMAnnotation(const Value &V, const std::string &Prop)
std::optional< unsigned > getMaxNReg(const Function &F)
bool Isv2x16VT(EVT VT)
std::optional< unsigned > getMaxNTIDy(const Function &F)
static void readIntVecFromMDNode(const MDNode *MetadataNode, std::vector< unsigned > &Vec)
bool isParamGridConstant(const Value &V)
StringRef getSamplerName(const Value &V)
bool isImageReadWrite(const Value &V)
bool isImageReadOnly(const Value &V)
std::optional< unsigned > getMaxNTIDz(const Function &F)
MaybeAlign getAlign(const Function &F, unsigned Index)
std::optional< unsigned > getMaxNTIDx(const Function &F)
std::optional< unsigned > getMinCTASm(const Function &F)
bool isImage(const Value &V)
bool isSampler(const Value &V)
static void cacheAnnotationFromMD(const MDNode *MetadataNode, key_val_pair_t &retval)
void clearAnnotationCache(const Module *Mod)
std::optional< unsigned > getReqNTIDy(const Function &F)
bool isSurface(const Value &V)
static bool findAllNVVMAnnotation(const GlobalValue *gv, const std::string &prop, std::vector< unsigned > &retval)
std::optional< unsigned > getMaxClusterRank(const Function &F)
StringRef getTextureName(const Value &V)
std::optional< unsigned > getClusterDimx(const Function &F)
@ Mod
The access may modify the value stored in memory.
static bool argHasNVVMAnnotation(const Value &Val, const std::string &Annotation, const bool StartArgIndexAtOne=false)
StringRef getSurfaceName(const Value &V)
std::optional< unsigned > getClusterDimy(const Function &F)
static std::optional< unsigned > findOneNVVMAnnotation(const GlobalValue *gv, const std::string &prop)
bool isKernelFunction(const Function &F)
bool isTexture(const Value &V)
Function * getMaybeBitcastedCallee(const CallBase *CB)
bool isImageWriteOnly(const Value &V)
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition: STLExtras.h:1903
std::optional< unsigned > getReqNTIDz(const Function &F)
std::optional< unsigned > getReqNTIDx(const Function &F)
std::optional< unsigned > getClusterDimz(const Function &F)
std::optional< unsigned > getReqNTID(const Function &F)
std::optional< unsigned > getMaxNTID(const Function &F)
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Extended Value Type.
Definition: ValueTypes.h:35
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition: Alignment.h:117