LLVM 24.0.0git
NVVMIntrinsicUtils.h
Go to the documentation of this file.
1//===--- NVVMIntrinsicUtils.h -----------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This file contains the definitions of the enumerations and flags
11/// associated with NVVM Intrinsics, along with some helper functions.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_IR_NVVMINTRINSICUTILS_H
16#define LLVM_IR_NVVMINTRINSICUTILS_H
17
18#include <stdint.h>
19
20#include "llvm/ADT/APFloat.h"
21#include "llvm/ADT/APInt.h"
22#include "llvm/ADT/StringRef.h"
23#include "llvm/IR/Constants.h"
24#include "llvm/IR/Intrinsics.h"
25#include "llvm/IR/IntrinsicsNVPTX.h"
26#include "llvm/Support/raw_ostream.h"
27
28namespace llvm {
29namespace nvvm {
30
31// Reduction Ops supported with TMA Copy from Shared
32// to Global Memory for the "cp.reduce.async.bulk.tensor.*"
33// family of PTX instructions.
34enum class TMAReductionOp : uint8_t {
35 ADD = 0,
36 MIN = 1,
37 MAX = 2,
38 INC = 3,
39 DEC = 4,
40 AND = 5,
41 OR = 6,
42 XOR = 7,
43};
44
45inline StringRef getTMATensorReductionOpName(TMAReductionOp Op) {
46 switch (Op) {
47 case TMAReductionOp::ADD:
48 return "add";
49 case TMAReductionOp::MIN:
50 return "min";
51 case TMAReductionOp::MAX:
52 return "max";
53 case TMAReductionOp::INC:
54 return "inc";
55 case TMAReductionOp::DEC:
56 return "dec";
57 case TMAReductionOp::AND:
58 return "and";
59 case TMAReductionOp::OR:
60 return "or";
61 case TMAReductionOp::XOR:
62 return "xor";
63 }
64 llvm_unreachable("invalid TMA tensorreduction operation");
65}
66
67// Enum to represent the cta_group::1 and
68// cta_group::2 variants in TMA/TCGEN05 family of
69// PTX instructions.
70enum class CTAGroupKind : uint8_t {
71 CG_NONE = 0, // default with no cta_group modifier
72 CG_1 = 1, // cta_group::1 modifier
73 CG_2 = 2, // cta_group::2 modifier
74};
75
76// Eviction priorities applicable for prefetch and applypriority intrinsics.
77enum class EvictPolicyType : uint8_t {
78 EVICT_NORMAL = 0, // default
79 EVICT_LAST = 1,
80};
81
82inline StringRef getEvictPolicyName(EvictPolicyType Policy) {
83 switch (Policy) {
84 case EvictPolicyType::EVICT_NORMAL:
85 return "L2::evict_normal";
86 case EvictPolicyType::EVICT_LAST:
87 return "L2::evict_last";
88 }
89 llvm_unreachable("invalid evict policy");
90}
91
92enum class Tcgen05MMAKind : uint8_t {
93 F16 = 0,
94 TF32 = 1,
95 F8F6F4 = 2,
96 I8 = 3,
97 TI16 = 4,
98};
99
106
107enum class Tcgen05MMACollectorBBuffer : uint8_t {
108 B0 = 0,
109 B1 = 1,
110 B2 = 2,
111 B3 = 3,
112};
113
114enum class TensormapElemType : uint8_t {
115 U8 = 0,
116 U16 = 1,
117 U32 = 2,
118 S32 = 3,
119 U64 = 4,
120 S64 = 5,
121 F16 = 6,
122 F32 = 7,
123 F32_FTZ = 8,
124 F64 = 9,
125 BF16 = 10,
126 TF32 = 11,
127 TF32_FTZ = 12,
128 B4x16 = 13,
129 B4x16_p64 = 14,
130 B6x16_p32 = 15,
131};
132
138
146
153
158
159LLVM_ABI void printTcgen05MMAKind(raw_ostream &OS, const Constant *ImmArgVal);
160
161LLVM_ABI void printEvictPolicyType(raw_ostream &OS, const Constant *ImmArgVal);
162
163LLVM_ABI void printTMAReductionOp(raw_ostream &OS, const Constant *ImmArgVal);
164
165LLVM_ABI void printTcgen05CollectorUsageOp(raw_ostream &OS,
166 const Constant *ImmArgVal);
167
168LLVM_ABI void printTcgen05MMACollectorBBuffer(raw_ostream &OS,
169 const Constant *ImmArgVal);
170
171LLVM_ABI void printTensormapElemType(raw_ostream &OS,
172 const Constant *ImmArgVal);
173LLVM_ABI void printTensormapInterleaveLayout(raw_ostream &OS,
174 const Constant *ImmArgVal);
175LLVM_ABI void printTensormapSwizzleMode(raw_ostream &OS,
176 const Constant *ImmArgVal);
177LLVM_ABI void printTensormapSwizzleAtomicity(raw_ostream &OS,
178 const Constant *ImmArgVal);
179LLVM_ABI void printTensormapFillMode(raw_ostream &OS,
180 const Constant *ImmArgVal);
181
182inline bool FPToIntegerIntrinsicShouldFTZ(Intrinsic::ID IntrinsicID) {
183 switch (IntrinsicID) {
184 case Intrinsic::nvvm_f2i_rm_ftz:
185 case Intrinsic::nvvm_f2i_rn_ftz:
186 case Intrinsic::nvvm_f2i_rp_ftz:
187 case Intrinsic::nvvm_f2i_rz_ftz:
188
189 case Intrinsic::nvvm_f2ui_rm_ftz:
190 case Intrinsic::nvvm_f2ui_rn_ftz:
191 case Intrinsic::nvvm_f2ui_rp_ftz:
192 case Intrinsic::nvvm_f2ui_rz_ftz:
193
194 case Intrinsic::nvvm_f2ll_rm_ftz:
195 case Intrinsic::nvvm_f2ll_rn_ftz:
196 case Intrinsic::nvvm_f2ll_rp_ftz:
197 case Intrinsic::nvvm_f2ll_rz_ftz:
198
199 case Intrinsic::nvvm_f2ull_rm_ftz:
200 case Intrinsic::nvvm_f2ull_rn_ftz:
201 case Intrinsic::nvvm_f2ull_rp_ftz:
202 case Intrinsic::nvvm_f2ull_rz_ftz:
203 return true;
204
205 case Intrinsic::nvvm_f2i_rm:
206 case Intrinsic::nvvm_f2i_rn:
207 case Intrinsic::nvvm_f2i_rp:
208 case Intrinsic::nvvm_f2i_rz:
209
210 case Intrinsic::nvvm_f2ui_rm:
211 case Intrinsic::nvvm_f2ui_rn:
212 case Intrinsic::nvvm_f2ui_rp:
213 case Intrinsic::nvvm_f2ui_rz:
214
215 case Intrinsic::nvvm_d2i_rm:
216 case Intrinsic::nvvm_d2i_rn:
217 case Intrinsic::nvvm_d2i_rp:
218 case Intrinsic::nvvm_d2i_rz:
219
220 case Intrinsic::nvvm_d2ui_rm:
221 case Intrinsic::nvvm_d2ui_rn:
222 case Intrinsic::nvvm_d2ui_rp:
223 case Intrinsic::nvvm_d2ui_rz:
224
225 case Intrinsic::nvvm_f2ll_rm:
226 case Intrinsic::nvvm_f2ll_rn:
227 case Intrinsic::nvvm_f2ll_rp:
228 case Intrinsic::nvvm_f2ll_rz:
229
230 case Intrinsic::nvvm_f2ull_rm:
231 case Intrinsic::nvvm_f2ull_rn:
232 case Intrinsic::nvvm_f2ull_rp:
233 case Intrinsic::nvvm_f2ull_rz:
234
235 case Intrinsic::nvvm_d2ll_rm:
236 case Intrinsic::nvvm_d2ll_rn:
237 case Intrinsic::nvvm_d2ll_rp:
238 case Intrinsic::nvvm_d2ll_rz:
239
240 case Intrinsic::nvvm_d2ull_rm:
241 case Intrinsic::nvvm_d2ull_rn:
242 case Intrinsic::nvvm_d2ull_rp:
243 case Intrinsic::nvvm_d2ull_rz:
244 return false;
245 }
246 llvm_unreachable("Checking FTZ flag for invalid f2i/d2i intrinsic");
247}
248
249inline bool FPToIntegerIntrinsicResultIsSigned(Intrinsic::ID IntrinsicID) {
250 switch (IntrinsicID) {
251 // f2i
252 case Intrinsic::nvvm_f2i_rm:
253 case Intrinsic::nvvm_f2i_rm_ftz:
254 case Intrinsic::nvvm_f2i_rn:
255 case Intrinsic::nvvm_f2i_rn_ftz:
256 case Intrinsic::nvvm_f2i_rp:
257 case Intrinsic::nvvm_f2i_rp_ftz:
258 case Intrinsic::nvvm_f2i_rz:
259 case Intrinsic::nvvm_f2i_rz_ftz:
260 // d2i
261 case Intrinsic::nvvm_d2i_rm:
262 case Intrinsic::nvvm_d2i_rn:
263 case Intrinsic::nvvm_d2i_rp:
264 case Intrinsic::nvvm_d2i_rz:
265 // f2ll
266 case Intrinsic::nvvm_f2ll_rm:
267 case Intrinsic::nvvm_f2ll_rm_ftz:
268 case Intrinsic::nvvm_f2ll_rn:
269 case Intrinsic::nvvm_f2ll_rn_ftz:
270 case Intrinsic::nvvm_f2ll_rp:
271 case Intrinsic::nvvm_f2ll_rp_ftz:
272 case Intrinsic::nvvm_f2ll_rz:
273 case Intrinsic::nvvm_f2ll_rz_ftz:
274 // d2ll
275 case Intrinsic::nvvm_d2ll_rm:
276 case Intrinsic::nvvm_d2ll_rn:
277 case Intrinsic::nvvm_d2ll_rp:
278 case Intrinsic::nvvm_d2ll_rz:
279 return true;
280
281 // f2ui
282 case Intrinsic::nvvm_f2ui_rm:
283 case Intrinsic::nvvm_f2ui_rm_ftz:
284 case Intrinsic::nvvm_f2ui_rn:
285 case Intrinsic::nvvm_f2ui_rn_ftz:
286 case Intrinsic::nvvm_f2ui_rp:
287 case Intrinsic::nvvm_f2ui_rp_ftz:
288 case Intrinsic::nvvm_f2ui_rz:
289 case Intrinsic::nvvm_f2ui_rz_ftz:
290 // d2ui
291 case Intrinsic::nvvm_d2ui_rm:
292 case Intrinsic::nvvm_d2ui_rn:
293 case Intrinsic::nvvm_d2ui_rp:
294 case Intrinsic::nvvm_d2ui_rz:
295 // f2ull
296 case Intrinsic::nvvm_f2ull_rm:
297 case Intrinsic::nvvm_f2ull_rm_ftz:
298 case Intrinsic::nvvm_f2ull_rn:
299 case Intrinsic::nvvm_f2ull_rn_ftz:
300 case Intrinsic::nvvm_f2ull_rp:
301 case Intrinsic::nvvm_f2ull_rp_ftz:
302 case Intrinsic::nvvm_f2ull_rz:
303 case Intrinsic::nvvm_f2ull_rz_ftz:
304 // d2ull
305 case Intrinsic::nvvm_d2ull_rm:
306 case Intrinsic::nvvm_d2ull_rn:
307 case Intrinsic::nvvm_d2ull_rp:
308 case Intrinsic::nvvm_d2ull_rz:
309 return false;
310 }
311 llvm_unreachable(
312 "Checking invalid f2i/d2i intrinsic for signed int conversion");
313}
314
315inline bool FPToIntegerIntrinsicNaNZero(Intrinsic::ID IntrinsicID) {
316 switch (IntrinsicID) {
317 // f2i
318 case Intrinsic::nvvm_f2i_rm:
319 case Intrinsic::nvvm_f2i_rn:
320 case Intrinsic::nvvm_f2i_rp:
321 case Intrinsic::nvvm_f2i_rz:
322 case Intrinsic::nvvm_f2i_rm_ftz:
323 case Intrinsic::nvvm_f2i_rn_ftz:
324 case Intrinsic::nvvm_f2i_rp_ftz:
325 case Intrinsic::nvvm_f2i_rz_ftz:
326 // f2ui
327 case Intrinsic::nvvm_f2ui_rm:
328 case Intrinsic::nvvm_f2ui_rn:
329 case Intrinsic::nvvm_f2ui_rp:
330 case Intrinsic::nvvm_f2ui_rz:
331 case Intrinsic::nvvm_f2ui_rm_ftz:
332 case Intrinsic::nvvm_f2ui_rn_ftz:
333 case Intrinsic::nvvm_f2ui_rp_ftz:
334 case Intrinsic::nvvm_f2ui_rz_ftz:
335 return true;
336 // d2i
337 case Intrinsic::nvvm_d2i_rm:
338 case Intrinsic::nvvm_d2i_rn:
339 case Intrinsic::nvvm_d2i_rp:
340 case Intrinsic::nvvm_d2i_rz:
341 // d2ui
342 case Intrinsic::nvvm_d2ui_rm:
343 case Intrinsic::nvvm_d2ui_rn:
344 case Intrinsic::nvvm_d2ui_rp:
345 case Intrinsic::nvvm_d2ui_rz:
346 // f2ll
347 case Intrinsic::nvvm_f2ll_rm:
348 case Intrinsic::nvvm_f2ll_rn:
349 case Intrinsic::nvvm_f2ll_rp:
350 case Intrinsic::nvvm_f2ll_rz:
351 case Intrinsic::nvvm_f2ll_rm_ftz:
352 case Intrinsic::nvvm_f2ll_rn_ftz:
353 case Intrinsic::nvvm_f2ll_rp_ftz:
354 case Intrinsic::nvvm_f2ll_rz_ftz:
355 // f2ull
356 case Intrinsic::nvvm_f2ull_rm:
357 case Intrinsic::nvvm_f2ull_rn:
358 case Intrinsic::nvvm_f2ull_rp:
359 case Intrinsic::nvvm_f2ull_rz:
360 case Intrinsic::nvvm_f2ull_rm_ftz:
361 case Intrinsic::nvvm_f2ull_rn_ftz:
362 case Intrinsic::nvvm_f2ull_rp_ftz:
363 case Intrinsic::nvvm_f2ull_rz_ftz:
364 // d2ll
365 case Intrinsic::nvvm_d2ll_rm:
366 case Intrinsic::nvvm_d2ll_rn:
367 case Intrinsic::nvvm_d2ll_rp:
368 case Intrinsic::nvvm_d2ll_rz:
369 // d2ull
370 case Intrinsic::nvvm_d2ull_rm:
371 case Intrinsic::nvvm_d2ull_rn:
372 case Intrinsic::nvvm_d2ull_rp:
373 case Intrinsic::nvvm_d2ull_rz:
374 return false;
375 }
376 llvm_unreachable("Checking NaN result for invalid f2i/d2i intrinsic");
377}
378
379inline APFloat::roundingMode
380GetFPToIntegerRoundingMode(Intrinsic::ID IntrinsicID) {
381 switch (IntrinsicID) {
382 // RM:
383 case Intrinsic::nvvm_f2i_rm:
384 case Intrinsic::nvvm_f2ui_rm:
385 case Intrinsic::nvvm_f2i_rm_ftz:
386 case Intrinsic::nvvm_f2ui_rm_ftz:
387 case Intrinsic::nvvm_d2i_rm:
388 case Intrinsic::nvvm_d2ui_rm:
389
390 case Intrinsic::nvvm_f2ll_rm:
391 case Intrinsic::nvvm_f2ull_rm:
392 case Intrinsic::nvvm_f2ll_rm_ftz:
393 case Intrinsic::nvvm_f2ull_rm_ftz:
394 case Intrinsic::nvvm_d2ll_rm:
395 case Intrinsic::nvvm_d2ull_rm:
396 return APFloat::rmTowardNegative;
397
398 // RN:
399 case Intrinsic::nvvm_f2i_rn:
400 case Intrinsic::nvvm_f2ui_rn:
401 case Intrinsic::nvvm_f2i_rn_ftz:
402 case Intrinsic::nvvm_f2ui_rn_ftz:
403 case Intrinsic::nvvm_d2i_rn:
404 case Intrinsic::nvvm_d2ui_rn:
405
406 case Intrinsic::nvvm_f2ll_rn:
407 case Intrinsic::nvvm_f2ull_rn:
408 case Intrinsic::nvvm_f2ll_rn_ftz:
409 case Intrinsic::nvvm_f2ull_rn_ftz:
410 case Intrinsic::nvvm_d2ll_rn:
411 case Intrinsic::nvvm_d2ull_rn:
412 return APFloat::rmNearestTiesToEven;
413
414 // RP:
415 case Intrinsic::nvvm_f2i_rp:
416 case Intrinsic::nvvm_f2ui_rp:
417 case Intrinsic::nvvm_f2i_rp_ftz:
418 case Intrinsic::nvvm_f2ui_rp_ftz:
419 case Intrinsic::nvvm_d2i_rp:
420 case Intrinsic::nvvm_d2ui_rp:
421
422 case Intrinsic::nvvm_f2ll_rp:
423 case Intrinsic::nvvm_f2ull_rp:
424 case Intrinsic::nvvm_f2ll_rp_ftz:
425 case Intrinsic::nvvm_f2ull_rp_ftz:
426 case Intrinsic::nvvm_d2ll_rp:
427 case Intrinsic::nvvm_d2ull_rp:
428 return APFloat::rmTowardPositive;
429
430 // RZ:
431 case Intrinsic::nvvm_f2i_rz:
432 case Intrinsic::nvvm_f2ui_rz:
433 case Intrinsic::nvvm_f2i_rz_ftz:
434 case Intrinsic::nvvm_f2ui_rz_ftz:
435 case Intrinsic::nvvm_d2i_rz:
436 case Intrinsic::nvvm_d2ui_rz:
437
438 case Intrinsic::nvvm_f2ll_rz:
439 case Intrinsic::nvvm_f2ull_rz:
440 case Intrinsic::nvvm_f2ll_rz_ftz:
441 case Intrinsic::nvvm_f2ull_rz_ftz:
442 case Intrinsic::nvvm_d2ll_rz:
443 case Intrinsic::nvvm_d2ull_rz:
444 return APFloat::rmTowardZero;
445 }
446 llvm_unreachable("Checking rounding mode for invalid f2i/d2i intrinsic");
447}
448
449inline bool FMinFMaxShouldFTZ(Intrinsic::ID IntrinsicID) {
450 switch (IntrinsicID) {
451 case Intrinsic::nvvm_fmax_ftz_f:
452 case Intrinsic::nvvm_fmax_ftz_nan_f:
453 case Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f:
454 case Intrinsic::nvvm_fmax_ftz_xorsign_abs_f:
455
456 case Intrinsic::nvvm_fmin_ftz_f:
457 case Intrinsic::nvvm_fmin_ftz_nan_f:
458 case Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f:
459 case Intrinsic::nvvm_fmin_ftz_xorsign_abs_f:
460 return true;
461
462 case Intrinsic::nvvm_fmax_d:
463 case Intrinsic::nvvm_fmax_f:
464 case Intrinsic::nvvm_fmax_nan_f:
465 case Intrinsic::nvvm_fmax_nan_xorsign_abs_f:
466 case Intrinsic::nvvm_fmax_xorsign_abs_f:
467
468 case Intrinsic::nvvm_fmin_d:
469 case Intrinsic::nvvm_fmin_f:
470 case Intrinsic::nvvm_fmin_nan_f:
471 case Intrinsic::nvvm_fmin_nan_xorsign_abs_f:
472 case Intrinsic::nvvm_fmin_xorsign_abs_f:
473 return false;
474 }
475 llvm_unreachable("Checking FTZ flag for invalid fmin/fmax intrinsic");
476}
477
478inline bool FMinFMaxPropagatesNaNs(Intrinsic::ID IntrinsicID) {
479 switch (IntrinsicID) {
480 case Intrinsic::nvvm_fmax_ftz_nan_f:
481 case Intrinsic::nvvm_fmax_nan_f:
482 case Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f:
483 case Intrinsic::nvvm_fmax_nan_xorsign_abs_f:
484
485 case Intrinsic::nvvm_fmin_ftz_nan_f:
486 case Intrinsic::nvvm_fmin_nan_f:
487 case Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f:
488 case Intrinsic::nvvm_fmin_nan_xorsign_abs_f:
489 return true;
490
491 case Intrinsic::nvvm_fmax_d:
492 case Intrinsic::nvvm_fmax_f:
493 case Intrinsic::nvvm_fmax_ftz_f:
494 case Intrinsic::nvvm_fmax_ftz_xorsign_abs_f:
495 case Intrinsic::nvvm_fmax_xorsign_abs_f:
496
497 case Intrinsic::nvvm_fmin_d:
498 case Intrinsic::nvvm_fmin_f:
499 case Intrinsic::nvvm_fmin_ftz_f:
500 case Intrinsic::nvvm_fmin_ftz_xorsign_abs_f:
501 case Intrinsic::nvvm_fmin_xorsign_abs_f:
502 return false;
503 }
504 llvm_unreachable("Checking NaN flag for invalid fmin/fmax intrinsic");
505}
506
507inline bool FMinFMaxIsXorSignAbs(Intrinsic::ID IntrinsicID) {
508 switch (IntrinsicID) {
509 case Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f:
510 case Intrinsic::nvvm_fmax_ftz_xorsign_abs_f:
511 case Intrinsic::nvvm_fmax_nan_xorsign_abs_f:
512 case Intrinsic::nvvm_fmax_xorsign_abs_f:
513
514 case Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f:
515 case Intrinsic::nvvm_fmin_ftz_xorsign_abs_f:
516 case Intrinsic::nvvm_fmin_nan_xorsign_abs_f:
517 case Intrinsic::nvvm_fmin_xorsign_abs_f:
518 return true;
519
520 case Intrinsic::nvvm_fmax_d:
521 case Intrinsic::nvvm_fmax_f:
522 case Intrinsic::nvvm_fmax_ftz_f:
523 case Intrinsic::nvvm_fmax_ftz_nan_f:
524 case Intrinsic::nvvm_fmax_nan_f:
525
526 case Intrinsic::nvvm_fmin_d:
527 case Intrinsic::nvvm_fmin_f:
528 case Intrinsic::nvvm_fmin_ftz_f:
529 case Intrinsic::nvvm_fmin_ftz_nan_f:
530 case Intrinsic::nvvm_fmin_nan_f:
531 return false;
532 }
533 llvm_unreachable("Checking XorSignAbs flag for invalid fmin/fmax intrinsic");
534}
535
536inline bool UnaryMathIntrinsicShouldFTZ(Intrinsic::ID IntrinsicID) {
537 switch (IntrinsicID) {
538 case Intrinsic::nvvm_ceil_ftz_f:
539 case Intrinsic::nvvm_fabs_ftz:
540 case Intrinsic::nvvm_floor_ftz_f:
541 case Intrinsic::nvvm_round_ftz_f:
542 case Intrinsic::nvvm_saturate_ftz_f:
543 case Intrinsic::nvvm_sqrt_rn_ftz_f:
544 return true;
545 case Intrinsic::nvvm_ceil_f:
546 case Intrinsic::nvvm_ceil_d:
547 case Intrinsic::nvvm_fabs:
548 case Intrinsic::nvvm_floor_f:
549 case Intrinsic::nvvm_floor_d:
550 case Intrinsic::nvvm_round_f:
551 case Intrinsic::nvvm_round_d:
552 case Intrinsic::nvvm_saturate_d:
553 case Intrinsic::nvvm_saturate_f:
554 case Intrinsic::nvvm_sqrt_f:
555 case Intrinsic::nvvm_sqrt_rn_d:
556 case Intrinsic::nvvm_sqrt_rn_f:
557 return false;
558 }
559 llvm_unreachable("Checking FTZ flag for invalid unary intrinsic");
560}
561
562inline bool RCPShouldFTZ(Intrinsic::ID IntrinsicID) {
563 switch (IntrinsicID) {
564 case Intrinsic::nvvm_rcp_rm_ftz_f:
565 case Intrinsic::nvvm_rcp_rn_ftz_f:
566 case Intrinsic::nvvm_rcp_rp_ftz_f:
567 case Intrinsic::nvvm_rcp_rz_ftz_f:
568 return true;
569 case Intrinsic::nvvm_rcp_rm_d:
570 case Intrinsic::nvvm_rcp_rm_f:
571 case Intrinsic::nvvm_rcp_rn_d:
572 case Intrinsic::nvvm_rcp_rn_f:
573 case Intrinsic::nvvm_rcp_rp_d:
574 case Intrinsic::nvvm_rcp_rp_f:
575 case Intrinsic::nvvm_rcp_rz_d:
576 case Intrinsic::nvvm_rcp_rz_f:
577 return false;
578 }
579 llvm_unreachable("Checking FTZ flag for invalid rcp intrinsic");
580}
581
582inline APFloat::roundingMode GetRCPRoundingMode(Intrinsic::ID IntrinsicID) {
583 switch (IntrinsicID) {
584 case Intrinsic::nvvm_rcp_rm_f:
585 case Intrinsic::nvvm_rcp_rm_d:
586 case Intrinsic::nvvm_rcp_rm_ftz_f:
587 return APFloat::rmTowardNegative;
588
589 case Intrinsic::nvvm_rcp_rn_f:
590 case Intrinsic::nvvm_rcp_rn_d:
591 case Intrinsic::nvvm_rcp_rn_ftz_f:
592 return APFloat::rmNearestTiesToEven;
593
594 case Intrinsic::nvvm_rcp_rp_f:
595 case Intrinsic::nvvm_rcp_rp_d:
596 case Intrinsic::nvvm_rcp_rp_ftz_f:
597 return APFloat::rmTowardPositive;
598
599 case Intrinsic::nvvm_rcp_rz_f:
600 case Intrinsic::nvvm_rcp_rz_d:
601 case Intrinsic::nvvm_rcp_rz_ftz_f:
602 return APFloat::rmTowardZero;
603 }
604 llvm_unreachable("Checking rounding mode for invalid rcp intrinsic");
605}
606
607inline DenormalMode GetNVVMDenormMode(bool ShouldFTZ) {
608 if (ShouldFTZ)
609 return DenormalMode::getPreserveSign();
610 return DenormalMode::getIEEE();
611}
612
613inline bool FAddShouldFTZ(Intrinsic::ID IntrinsicID) {
614 switch (IntrinsicID) {
615 case Intrinsic::nvvm_add_rm_ftz_f:
616 case Intrinsic::nvvm_add_rn_ftz_f:
617 case Intrinsic::nvvm_add_rp_ftz_f:
618 case Intrinsic::nvvm_add_rz_ftz_f:
619 return true;
620
621 case Intrinsic::nvvm_add_rm_f:
622 case Intrinsic::nvvm_add_rn_f:
623 case Intrinsic::nvvm_add_rp_f:
624 case Intrinsic::nvvm_add_rz_f:
625 case Intrinsic::nvvm_add_rm_d:
626 case Intrinsic::nvvm_add_rn_d:
627 case Intrinsic::nvvm_add_rp_d:
628 case Intrinsic::nvvm_add_rz_d:
629 return false;
630 }
631 llvm_unreachable("Checking FTZ flag for invalid NVVM add intrinsic");
632}
633
634inline APFloat::roundingMode GetFAddRoundingMode(Intrinsic::ID IntrinsicID) {
635 switch (IntrinsicID) {
636 case Intrinsic::nvvm_add_rm_f:
637 case Intrinsic::nvvm_add_rm_d:
638 case Intrinsic::nvvm_add_rm_ftz_f:
639 return APFloat::rmTowardNegative;
640 case Intrinsic::nvvm_add_rn_f:
641 case Intrinsic::nvvm_add_rn_d:
642 case Intrinsic::nvvm_add_rn_ftz_f:
643 return APFloat::rmNearestTiesToEven;
644 case Intrinsic::nvvm_add_rp_f:
645 case Intrinsic::nvvm_add_rp_d:
646 case Intrinsic::nvvm_add_rp_ftz_f:
647 return APFloat::rmTowardPositive;
648 case Intrinsic::nvvm_add_rz_f:
649 case Intrinsic::nvvm_add_rz_d:
650 case Intrinsic::nvvm_add_rz_ftz_f:
651 return APFloat::rmTowardZero;
652 }
653 llvm_unreachable("Invalid FP instrinsic rounding mode for NVVM add");
654}
655
656inline bool FMulShouldFTZ(Intrinsic::ID IntrinsicID) {
657 switch (IntrinsicID) {
658 case Intrinsic::nvvm_mul_rm_ftz_f:
659 case Intrinsic::nvvm_mul_rn_ftz_f:
660 case Intrinsic::nvvm_mul_rp_ftz_f:
661 case Intrinsic::nvvm_mul_rz_ftz_f:
662 return true;
663
664 case Intrinsic::nvvm_mul_rm_f:
665 case Intrinsic::nvvm_mul_rn_f:
666 case Intrinsic::nvvm_mul_rp_f:
667 case Intrinsic::nvvm_mul_rz_f:
668 case Intrinsic::nvvm_mul_rm_d:
669 case Intrinsic::nvvm_mul_rn_d:
670 case Intrinsic::nvvm_mul_rp_d:
671 case Intrinsic::nvvm_mul_rz_d:
672 return false;
673 }
674 llvm_unreachable("Checking FTZ flag for invalid NVVM mul intrinsic");
675}
676
677inline APFloat::roundingMode GetFMulRoundingMode(Intrinsic::ID IntrinsicID) {
678 switch (IntrinsicID) {
679 case Intrinsic::nvvm_mul_rm_f:
680 case Intrinsic::nvvm_mul_rm_d:
681 case Intrinsic::nvvm_mul_rm_ftz_f:
682 return APFloat::rmTowardNegative;
683 case Intrinsic::nvvm_mul_rn_f:
684 case Intrinsic::nvvm_mul_rn_d:
685 case Intrinsic::nvvm_mul_rn_ftz_f:
686 return APFloat::rmNearestTiesToEven;
687 case Intrinsic::nvvm_mul_rp_f:
688 case Intrinsic::nvvm_mul_rp_d:
689 case Intrinsic::nvvm_mul_rp_ftz_f:
690 return APFloat::rmTowardPositive;
691 case Intrinsic::nvvm_mul_rz_f:
692 case Intrinsic::nvvm_mul_rz_d:
693 case Intrinsic::nvvm_mul_rz_ftz_f:
694 return APFloat::rmTowardZero;
695 }
696 llvm_unreachable("Invalid FP instrinsic rounding mode for NVVM mul");
697}
698
699inline bool FDivShouldFTZ(Intrinsic::ID IntrinsicID) {
700 switch (IntrinsicID) {
701 case Intrinsic::nvvm_div_rm_ftz_f:
702 case Intrinsic::nvvm_div_rn_ftz_f:
703 case Intrinsic::nvvm_div_rp_ftz_f:
704 case Intrinsic::nvvm_div_rz_ftz_f:
705 return true;
706
707 case Intrinsic::nvvm_div_rm_f:
708 case Intrinsic::nvvm_div_rn_f:
709 case Intrinsic::nvvm_div_rp_f:
710 case Intrinsic::nvvm_div_rz_f:
711 case Intrinsic::nvvm_div_rm_d:
712 case Intrinsic::nvvm_div_rn_d:
713 case Intrinsic::nvvm_div_rp_d:
714 case Intrinsic::nvvm_div_rz_d:
715 return false;
716 }
717 llvm_unreachable("Checking FTZ flag for invalid NVVM div intrinsic");
718}
719
720inline APFloat::roundingMode GetFDivRoundingMode(Intrinsic::ID IntrinsicID) {
721 switch (IntrinsicID) {
722 case Intrinsic::nvvm_div_rm_f:
723 case Intrinsic::nvvm_div_rm_d:
724 case Intrinsic::nvvm_div_rm_ftz_f:
725 return APFloat::rmTowardNegative;
726 case Intrinsic::nvvm_div_rn_f:
727 case Intrinsic::nvvm_div_rn_d:
728 case Intrinsic::nvvm_div_rn_ftz_f:
729 return APFloat::rmNearestTiesToEven;
730 case Intrinsic::nvvm_div_rp_f:
731 case Intrinsic::nvvm_div_rp_d:
732 case Intrinsic::nvvm_div_rp_ftz_f:
733 return APFloat::rmTowardPositive;
734 case Intrinsic::nvvm_div_rz_f:
735 case Intrinsic::nvvm_div_rz_d:
736 case Intrinsic::nvvm_div_rz_ftz_f:
737 return APFloat::rmTowardZero;
738 }
739 llvm_unreachable("Invalid FP instrinsic rounding mode for NVVM div");
740}
741
742inline bool FMAShouldFTZ(Intrinsic::ID IntrinsicID) {
743 switch (IntrinsicID) {
744 case Intrinsic::nvvm_fma_rm_ftz_f:
745 case Intrinsic::nvvm_fma_rn_ftz_f:
746 case Intrinsic::nvvm_fma_rp_ftz_f:
747 case Intrinsic::nvvm_fma_rz_ftz_f:
748 return true;
749
750 case Intrinsic::nvvm_fma_rm_f:
751 case Intrinsic::nvvm_fma_rn_f:
752 case Intrinsic::nvvm_fma_rp_f:
753 case Intrinsic::nvvm_fma_rz_f:
754 case Intrinsic::nvvm_fma_rm_d:
755 case Intrinsic::nvvm_fma_rn_d:
756 case Intrinsic::nvvm_fma_rp_d:
757 case Intrinsic::nvvm_fma_rz_d:
758 return false;
759 }
760 llvm_unreachable("Checking FTZ flag for invalid NVVM fma intrinsic");
761}
762
763inline APFloat::roundingMode GetFMARoundingMode(Intrinsic::ID IntrinsicID) {
764 switch (IntrinsicID) {
765 case Intrinsic::nvvm_fma_rm_f:
766 case Intrinsic::nvvm_fma_rm_d:
767 case Intrinsic::nvvm_fma_rm_ftz_f:
768 return APFloat::rmTowardNegative;
769 case Intrinsic::nvvm_fma_rn_f:
770 case Intrinsic::nvvm_fma_rn_d:
771 case Intrinsic::nvvm_fma_rn_ftz_f:
772 return APFloat::rmNearestTiesToEven;
773 case Intrinsic::nvvm_fma_rp_f:
774 case Intrinsic::nvvm_fma_rp_d:
775 case Intrinsic::nvvm_fma_rp_ftz_f:
776 return APFloat::rmTowardPositive;
777 case Intrinsic::nvvm_fma_rz_f:
778 case Intrinsic::nvvm_fma_rz_d:
779 case Intrinsic::nvvm_fma_rz_ftz_f:
780 return APFloat::rmTowardZero;
781 }
782 llvm_unreachable("Invalid FP instrinsic rounding mode for NVVM fma");
783}
784
785} // namespace nvvm
786} // namespace llvm
787#endif // LLVM_IR_NVVMINTRINSICUTILS_H
APFloat.h
This file declares a class to represent arbitrary precision floating point values and provide a varie...
APInt.h
This file implements a class to represent arbitrary precision integral constant values and operations...
LLVM_ABI
#define LLVM_ABI
Definition Compiler.h:215
Constants.h
This file contains the declarations for the subclasses of Constant, which represent the different fla...
llvm::APFloatBase::rmTowardZero
static constexpr roundingMode rmTowardZero
Definition APFloat.h:365
llvm::APFloatBase::roundingMode
llvm::RoundingMode roundingMode
IEEE-754R 4.3: Rounding-direction attributes.
Definition APFloat.h:359
llvm::APFloatBase::rmTowardNegative
static constexpr roundingMode rmTowardNegative
Definition APFloat.h:364
llvm::APFloatBase::rmNearestTiesToEven
static constexpr roundingMode rmNearestTiesToEven
Definition APFloat.h:361
llvm::APFloatBase::rmTowardPositive
static constexpr roundingMode rmTowardPositive
Definition APFloat.h:363
llvm::Constant
This is an important base class in LLVM.
Definition Constant.h:43
llvm::StringRef
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
llvm::raw_ostream
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition raw_ostream.h:53
uint8_t
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition ErrorHandling.h:164
llvm::nvvm::GetFMARoundingMode
APFloat::roundingMode GetFMARoundingMode(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:763
llvm::nvvm::printTcgen05MMACollectorBBuffer
LLVM_ABI void printTcgen05MMACollectorBBuffer(raw_ostream &OS, const Constant *ImmArgVal)
Definition NVVMIntrinsicUtils.cpp:83
llvm::nvvm::printTensormapSwizzleMode
LLVM_ABI void printTensormapSwizzleMode(raw_ostream &OS, const Constant *ImmArgVal)
Definition NVVMIntrinsicUtils.cpp:136
llvm::nvvm::GetNVVMDenormMode
DenormalMode GetNVVMDenormMode(bool ShouldFTZ)
Definition NVVMIntrinsicUtils.h:607
llvm::nvvm::getEvictPolicyName
StringRef getEvictPolicyName(EvictPolicyType Policy)
Definition NVVMIntrinsicUtils.h:82
llvm::nvvm::printTMAReductionOp
LLVM_ABI void printTMAReductionOp(raw_ostream &OS, const Constant *ImmArgVal)
Definition NVVMIntrinsicUtils.cpp:29
llvm::nvvm::FPToIntegerIntrinsicNaNZero
bool FPToIntegerIntrinsicNaNZero(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:315
llvm::nvvm::printTensormapInterleaveLayout
LLVM_ABI void printTensormapInterleaveLayout(raw_ostream &OS, const Constant *ImmArgVal)
Definition NVVMIntrinsicUtils.cpp:118
llvm::nvvm::GetFDivRoundingMode
APFloat::roundingMode GetFDivRoundingMode(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:720
llvm::nvvm::printTensormapSwizzleAtomicity
LLVM_ABI void printTensormapSwizzleAtomicity(raw_ostream &OS, const Constant *ImmArgVal)
Definition NVVMIntrinsicUtils.cpp:150
llvm::nvvm::FPToIntegerIntrinsicResultIsSigned
bool FPToIntegerIntrinsicResultIsSigned(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:249
llvm::nvvm::GetFPToIntegerRoundingMode
APFloat::roundingMode GetFPToIntegerRoundingMode(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:380
llvm::nvvm::RCPShouldFTZ
bool RCPShouldFTZ(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:562
llvm::nvvm::getTMATensorReductionOpName
StringRef getTMATensorReductionOpName(TMAReductionOp Op)
Definition NVVMIntrinsicUtils.h:45
llvm::nvvm::FPToIntegerIntrinsicShouldFTZ
bool FPToIntegerIntrinsicShouldFTZ(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:182
llvm::nvvm::FDivShouldFTZ
bool FDivShouldFTZ(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:699
llvm::nvvm::FAddShouldFTZ
bool FAddShouldFTZ(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:613
llvm::nvvm::FMinFMaxIsXorSignAbs
bool FMinFMaxIsXorSignAbs(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:507
llvm::nvvm::GetFMulRoundingMode
APFloat::roundingMode GetFMulRoundingMode(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:677
llvm::nvvm::printTcgen05MMAKind
LLVM_ABI void printTcgen05MMAKind(raw_ostream &OS, const Constant *ImmArgVal)
Definition NVVMIntrinsicUtils.cpp:39
llvm::nvvm::printTcgen05CollectorUsageOp
LLVM_ABI void printTcgen05CollectorUsageOp(raw_ostream &OS, const Constant *ImmArgVal)
Definition NVVMIntrinsicUtils.cpp:62
llvm::nvvm::UnaryMathIntrinsicShouldFTZ
bool UnaryMathIntrinsicShouldFTZ(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:536
llvm::nvvm::FMinFMaxShouldFTZ
bool FMinFMaxShouldFTZ(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:449
llvm::nvvm::GetFAddRoundingMode
APFloat::roundingMode GetFAddRoundingMode(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:634
llvm::nvvm::printEvictPolicyType
LLVM_ABI void printEvictPolicyType(raw_ostream &OS, const Constant *ImmArgVal)
Definition NVVMIntrinsicUtils.cpp:19
llvm::nvvm::FMAShouldFTZ
bool FMAShouldFTZ(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:742
llvm::nvvm::FMulShouldFTZ
bool FMulShouldFTZ(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:656
llvm::nvvm::printTensormapFillMode
LLVM_ABI void printTensormapFillMode(raw_ostream &OS, const Constant *ImmArgVal)
Definition NVVMIntrinsicUtils.cpp:164
llvm::nvvm::GetRCPRoundingMode
APFloat::roundingMode GetRCPRoundingMode(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:582
llvm::nvvm::FMinFMaxPropagatesNaNs
bool FMinFMaxPropagatesNaNs(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:478
llvm::nvvm::printTensormapElemType
LLVM_ABI void printTensormapElemType(raw_ostream &OS, const Constant *ImmArgVal)
Definition NVVMIntrinsicUtils.cpp:104
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition FunctionInfo.h:25
llvm::Op
DWARFExpression::Operation Op
Definition DWARFExpressionPrinter.cpp:24
raw_ostream.h
llvm::DenormalMode
Represent subnormal handling kind for floating point instruction inputs and outputs.
Definition FloatingPointMode.h:71
llvm::DenormalMode::getPreserveSign
static constexpr DenormalMode getPreserveSign()
Definition FloatingPointMode.h:119
llvm::DenormalMode::getIEEE
static constexpr DenormalMode getIEEE()
Definition FloatingPointMode.h:115
Generated on for LLVM by doxygen 1.14.0