LLVM 24.0.0git
NVVMIntrinsicUtils.h
Go to the documentation of this file.
1//===--- NVVMIntrinsicUtils.h -----------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This file contains the definitions of the enumerations and flags
11/// associated with NVVM Intrinsics, along with some helper functions.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_IR_NVVMINTRINSICUTILS_H
16#define LLVM_IR_NVVMINTRINSICUTILS_H
17
18#include <stdint.h>
19
20#include "llvm/ADT/APFloat.h"
21#include "llvm/ADT/APInt.h"
22#include "llvm/ADT/StringRef.h"
23#include "llvm/IR/Constants.h"
24#include "llvm/IR/Intrinsics.h"
25#include "llvm/IR/IntrinsicsNVPTX.h"
27
28namespace llvm {
29namespace nvvm {
30
31// Reduction Ops supported with TMA Copy from Shared
32// to Global Memory for the "cp.reduce.async.bulk.tensor.*"
33// family of PTX instructions.
34enum class TMAReductionOp : uint8_t {
35 ADD = 0,
36 MIN = 1,
37 MAX = 2,
38 INC = 3,
39 DEC = 4,
40 AND = 5,
41 OR = 6,
42 XOR = 7,
43};
44
46 switch (Op) {
48 return "add";
50 return "min";
52 return "max";
54 return "inc";
56 return "dec";
58 return "and";
60 return "or";
62 return "xor";
63 }
64 llvm_unreachable("invalid TMA tensorreduction operation");
65}
66
67// Enum to represent the cta_group::1 and
68// cta_group::2 variants in TMA/TCGEN05 family of
69// PTX instructions.
70enum class CTAGroupKind : uint8_t {
71 CG_NONE = 0, // default with no cta_group modifier
72 CG_1 = 1, // cta_group::1 modifier
73 CG_2 = 2, // cta_group::2 modifier
74};
75
76// Validate data patterns supported with the TMA Copy from Global to Shared
77// memory (CTA and Cluster) intrinsics.
79 DISABLED = 0, // Default; no validate_data qualifier is emitted.
85};
86
88 using VDTy = TMAValidateDataPattern;
89 switch (Pattern) {
90 case VDTy::DISABLED:
91 return "disabled";
92 case VDTy::PER_16BYTES_80000000:
93 return "per_16bytes::80000000";
94 case VDTy::PER_16BYTES_8000:
95 return "per_16bytes::8000";
96 case VDTy::PER_16BYTES_80:
97 return "per_16bytes::80";
98 case VDTy::PER_16BYTES_8:
99 return "per_16bytes::8";
100 case VDTy::PER_ELEMENT_FF:
101 return "per_element::ff";
102 }
103 llvm_unreachable("invalid TMA validate data pattern");
104}
105
106// Scope of the memory ordering semantics.
107enum class MemScope : uint8_t {
108 CTA = 0,
110 GPU = 2,
111 SYS = 3,
112};
113
115 switch (Scope) {
116 case MemScope::CTA:
117 return "cta";
119 return "cluster";
120 case MemScope::GPU:
121 return "gpu";
122 case MemScope::SYS:
123 return "sys";
124 }
125 llvm_unreachable("invalid memory scope");
126}
127
128// Eviction priorities applicable for prefetch and applypriority intrinsics.
130 EVICT_NORMAL = 0, // default
132};
133
135 switch (Policy) {
137 return "L2::evict_normal";
139 return "L2::evict_last";
140 }
141 llvm_unreachable("invalid evict policy");
142}
143
145 F16 = 0,
146 TF32 = 1,
148 I8 = 3,
149 TI16 = 4,
150};
151
158
160 B0 = 0,
161 B1 = 1,
162 B2 = 2,
163 B3 = 3,
164};
165
167 U8 = 0,
168 U16 = 1,
169 U32 = 2,
170 S32 = 3,
171 U64 = 4,
172 S64 = 5,
173 F16 = 6,
174 F32 = 7,
176 F64 = 9,
177 BF16 = 10,
178 TF32 = 11,
180 B4x16 = 13,
183};
184
190
198
205
210
212 SMALL = 0,
213 LARGE = 1,
214 SAME = 2,
215};
216
217// In-memory layout of an mbarrier object, as selected by the layout operand of
218// the llvm.nvvm.mbarrier.init and llvm.nvvm.mbarrier.check_layout intrinsics.
220 V0 = 0,
221 V1 = 1,
222};
223
224LLVM_ABI void printTcgen05MMAKind(raw_ostream &OS, const Constant *ImmArgVal);
225
226LLVM_ABI void printMBarrierLayout(raw_ostream &OS, const Constant *ImmArgVal);
227
228LLVM_ABI void printEvictPolicyType(raw_ostream &OS, const Constant *ImmArgVal);
229
230LLVM_ABI void printTMAReductionOp(raw_ostream &OS, const Constant *ImmArgVal);
231
233 const Constant *ImmArgVal);
234
235LLVM_ABI void printMemScope(raw_ostream &OS, const Constant *ImmArgVal);
236
238 const Constant *ImmArgVal);
239
241 const Constant *ImmArgVal);
242
244 const Constant *ImmArgVal);
246 const Constant *ImmArgVal);
248 const Constant *ImmArgVal);
250 const Constant *ImmArgVal);
252 const Constant *ImmArgVal);
254 const Constant *ImmArgVal);
255LLVM_ABI void printFPRoundingMode(raw_ostream &OS, const Constant *ImmArgVal);
256
258 switch (IntrinsicID) {
259 case Intrinsic::nvvm_f2i_rm_ftz:
260 case Intrinsic::nvvm_f2i_rn_ftz:
261 case Intrinsic::nvvm_f2i_rp_ftz:
262 case Intrinsic::nvvm_f2i_rz_ftz:
263
264 case Intrinsic::nvvm_f2ui_rm_ftz:
265 case Intrinsic::nvvm_f2ui_rn_ftz:
266 case Intrinsic::nvvm_f2ui_rp_ftz:
267 case Intrinsic::nvvm_f2ui_rz_ftz:
268
269 case Intrinsic::nvvm_f2ll_rm_ftz:
270 case Intrinsic::nvvm_f2ll_rn_ftz:
271 case Intrinsic::nvvm_f2ll_rp_ftz:
272 case Intrinsic::nvvm_f2ll_rz_ftz:
273
274 case Intrinsic::nvvm_f2ull_rm_ftz:
275 case Intrinsic::nvvm_f2ull_rn_ftz:
276 case Intrinsic::nvvm_f2ull_rp_ftz:
277 case Intrinsic::nvvm_f2ull_rz_ftz:
278 return true;
279
280 case Intrinsic::nvvm_f2i_rm:
281 case Intrinsic::nvvm_f2i_rn:
282 case Intrinsic::nvvm_f2i_rp:
283 case Intrinsic::nvvm_f2i_rz:
284
285 case Intrinsic::nvvm_f2ui_rm:
286 case Intrinsic::nvvm_f2ui_rn:
287 case Intrinsic::nvvm_f2ui_rp:
288 case Intrinsic::nvvm_f2ui_rz:
289
290 case Intrinsic::nvvm_d2i_rm:
291 case Intrinsic::nvvm_d2i_rn:
292 case Intrinsic::nvvm_d2i_rp:
293 case Intrinsic::nvvm_d2i_rz:
294
295 case Intrinsic::nvvm_d2ui_rm:
296 case Intrinsic::nvvm_d2ui_rn:
297 case Intrinsic::nvvm_d2ui_rp:
298 case Intrinsic::nvvm_d2ui_rz:
299
300 case Intrinsic::nvvm_f2ll_rm:
301 case Intrinsic::nvvm_f2ll_rn:
302 case Intrinsic::nvvm_f2ll_rp:
303 case Intrinsic::nvvm_f2ll_rz:
304
305 case Intrinsic::nvvm_f2ull_rm:
306 case Intrinsic::nvvm_f2ull_rn:
307 case Intrinsic::nvvm_f2ull_rp:
308 case Intrinsic::nvvm_f2ull_rz:
309
310 case Intrinsic::nvvm_d2ll_rm:
311 case Intrinsic::nvvm_d2ll_rn:
312 case Intrinsic::nvvm_d2ll_rp:
313 case Intrinsic::nvvm_d2ll_rz:
314
315 case Intrinsic::nvvm_d2ull_rm:
316 case Intrinsic::nvvm_d2ull_rn:
317 case Intrinsic::nvvm_d2ull_rp:
318 case Intrinsic::nvvm_d2ull_rz:
319 return false;
320 }
321 llvm_unreachable("Checking FTZ flag for invalid f2i/d2i intrinsic");
322}
323
325 switch (IntrinsicID) {
326 // f2i
327 case Intrinsic::nvvm_f2i_rm:
328 case Intrinsic::nvvm_f2i_rm_ftz:
329 case Intrinsic::nvvm_f2i_rn:
330 case Intrinsic::nvvm_f2i_rn_ftz:
331 case Intrinsic::nvvm_f2i_rp:
332 case Intrinsic::nvvm_f2i_rp_ftz:
333 case Intrinsic::nvvm_f2i_rz:
334 case Intrinsic::nvvm_f2i_rz_ftz:
335 // d2i
336 case Intrinsic::nvvm_d2i_rm:
337 case Intrinsic::nvvm_d2i_rn:
338 case Intrinsic::nvvm_d2i_rp:
339 case Intrinsic::nvvm_d2i_rz:
340 // f2ll
341 case Intrinsic::nvvm_f2ll_rm:
342 case Intrinsic::nvvm_f2ll_rm_ftz:
343 case Intrinsic::nvvm_f2ll_rn:
344 case Intrinsic::nvvm_f2ll_rn_ftz:
345 case Intrinsic::nvvm_f2ll_rp:
346 case Intrinsic::nvvm_f2ll_rp_ftz:
347 case Intrinsic::nvvm_f2ll_rz:
348 case Intrinsic::nvvm_f2ll_rz_ftz:
349 // d2ll
350 case Intrinsic::nvvm_d2ll_rm:
351 case Intrinsic::nvvm_d2ll_rn:
352 case Intrinsic::nvvm_d2ll_rp:
353 case Intrinsic::nvvm_d2ll_rz:
354 return true;
355
356 // f2ui
357 case Intrinsic::nvvm_f2ui_rm:
358 case Intrinsic::nvvm_f2ui_rm_ftz:
359 case Intrinsic::nvvm_f2ui_rn:
360 case Intrinsic::nvvm_f2ui_rn_ftz:
361 case Intrinsic::nvvm_f2ui_rp:
362 case Intrinsic::nvvm_f2ui_rp_ftz:
363 case Intrinsic::nvvm_f2ui_rz:
364 case Intrinsic::nvvm_f2ui_rz_ftz:
365 // d2ui
366 case Intrinsic::nvvm_d2ui_rm:
367 case Intrinsic::nvvm_d2ui_rn:
368 case Intrinsic::nvvm_d2ui_rp:
369 case Intrinsic::nvvm_d2ui_rz:
370 // f2ull
371 case Intrinsic::nvvm_f2ull_rm:
372 case Intrinsic::nvvm_f2ull_rm_ftz:
373 case Intrinsic::nvvm_f2ull_rn:
374 case Intrinsic::nvvm_f2ull_rn_ftz:
375 case Intrinsic::nvvm_f2ull_rp:
376 case Intrinsic::nvvm_f2ull_rp_ftz:
377 case Intrinsic::nvvm_f2ull_rz:
378 case Intrinsic::nvvm_f2ull_rz_ftz:
379 // d2ull
380 case Intrinsic::nvvm_d2ull_rm:
381 case Intrinsic::nvvm_d2ull_rn:
382 case Intrinsic::nvvm_d2ull_rp:
383 case Intrinsic::nvvm_d2ull_rz:
384 return false;
385 }
387 "Checking invalid f2i/d2i intrinsic for signed int conversion");
388}
389
391 switch (IntrinsicID) {
392 // f2i
393 case Intrinsic::nvvm_f2i_rm:
394 case Intrinsic::nvvm_f2i_rn:
395 case Intrinsic::nvvm_f2i_rp:
396 case Intrinsic::nvvm_f2i_rz:
397 case Intrinsic::nvvm_f2i_rm_ftz:
398 case Intrinsic::nvvm_f2i_rn_ftz:
399 case Intrinsic::nvvm_f2i_rp_ftz:
400 case Intrinsic::nvvm_f2i_rz_ftz:
401 // f2ui
402 case Intrinsic::nvvm_f2ui_rm:
403 case Intrinsic::nvvm_f2ui_rn:
404 case Intrinsic::nvvm_f2ui_rp:
405 case Intrinsic::nvvm_f2ui_rz:
406 case Intrinsic::nvvm_f2ui_rm_ftz:
407 case Intrinsic::nvvm_f2ui_rn_ftz:
408 case Intrinsic::nvvm_f2ui_rp_ftz:
409 case Intrinsic::nvvm_f2ui_rz_ftz:
410 return true;
411 // d2i
412 case Intrinsic::nvvm_d2i_rm:
413 case Intrinsic::nvvm_d2i_rn:
414 case Intrinsic::nvvm_d2i_rp:
415 case Intrinsic::nvvm_d2i_rz:
416 // d2ui
417 case Intrinsic::nvvm_d2ui_rm:
418 case Intrinsic::nvvm_d2ui_rn:
419 case Intrinsic::nvvm_d2ui_rp:
420 case Intrinsic::nvvm_d2ui_rz:
421 // f2ll
422 case Intrinsic::nvvm_f2ll_rm:
423 case Intrinsic::nvvm_f2ll_rn:
424 case Intrinsic::nvvm_f2ll_rp:
425 case Intrinsic::nvvm_f2ll_rz:
426 case Intrinsic::nvvm_f2ll_rm_ftz:
427 case Intrinsic::nvvm_f2ll_rn_ftz:
428 case Intrinsic::nvvm_f2ll_rp_ftz:
429 case Intrinsic::nvvm_f2ll_rz_ftz:
430 // f2ull
431 case Intrinsic::nvvm_f2ull_rm:
432 case Intrinsic::nvvm_f2ull_rn:
433 case Intrinsic::nvvm_f2ull_rp:
434 case Intrinsic::nvvm_f2ull_rz:
435 case Intrinsic::nvvm_f2ull_rm_ftz:
436 case Intrinsic::nvvm_f2ull_rn_ftz:
437 case Intrinsic::nvvm_f2ull_rp_ftz:
438 case Intrinsic::nvvm_f2ull_rz_ftz:
439 // d2ll
440 case Intrinsic::nvvm_d2ll_rm:
441 case Intrinsic::nvvm_d2ll_rn:
442 case Intrinsic::nvvm_d2ll_rp:
443 case Intrinsic::nvvm_d2ll_rz:
444 // d2ull
445 case Intrinsic::nvvm_d2ull_rm:
446 case Intrinsic::nvvm_d2ull_rn:
447 case Intrinsic::nvvm_d2ull_rp:
448 case Intrinsic::nvvm_d2ull_rz:
449 return false;
450 }
451 llvm_unreachable("Checking NaN result for invalid f2i/d2i intrinsic");
452}
453
456 switch (IntrinsicID) {
457 // RM:
458 case Intrinsic::nvvm_f2i_rm:
459 case Intrinsic::nvvm_f2ui_rm:
460 case Intrinsic::nvvm_f2i_rm_ftz:
461 case Intrinsic::nvvm_f2ui_rm_ftz:
462 case Intrinsic::nvvm_d2i_rm:
463 case Intrinsic::nvvm_d2ui_rm:
464
465 case Intrinsic::nvvm_f2ll_rm:
466 case Intrinsic::nvvm_f2ull_rm:
467 case Intrinsic::nvvm_f2ll_rm_ftz:
468 case Intrinsic::nvvm_f2ull_rm_ftz:
469 case Intrinsic::nvvm_d2ll_rm:
470 case Intrinsic::nvvm_d2ull_rm:
472
473 // RN:
474 case Intrinsic::nvvm_f2i_rn:
475 case Intrinsic::nvvm_f2ui_rn:
476 case Intrinsic::nvvm_f2i_rn_ftz:
477 case Intrinsic::nvvm_f2ui_rn_ftz:
478 case Intrinsic::nvvm_d2i_rn:
479 case Intrinsic::nvvm_d2ui_rn:
480
481 case Intrinsic::nvvm_f2ll_rn:
482 case Intrinsic::nvvm_f2ull_rn:
483 case Intrinsic::nvvm_f2ll_rn_ftz:
484 case Intrinsic::nvvm_f2ull_rn_ftz:
485 case Intrinsic::nvvm_d2ll_rn:
486 case Intrinsic::nvvm_d2ull_rn:
488
489 // RP:
490 case Intrinsic::nvvm_f2i_rp:
491 case Intrinsic::nvvm_f2ui_rp:
492 case Intrinsic::nvvm_f2i_rp_ftz:
493 case Intrinsic::nvvm_f2ui_rp_ftz:
494 case Intrinsic::nvvm_d2i_rp:
495 case Intrinsic::nvvm_d2ui_rp:
496
497 case Intrinsic::nvvm_f2ll_rp:
498 case Intrinsic::nvvm_f2ull_rp:
499 case Intrinsic::nvvm_f2ll_rp_ftz:
500 case Intrinsic::nvvm_f2ull_rp_ftz:
501 case Intrinsic::nvvm_d2ll_rp:
502 case Intrinsic::nvvm_d2ull_rp:
504
505 // RZ:
506 case Intrinsic::nvvm_f2i_rz:
507 case Intrinsic::nvvm_f2ui_rz:
508 case Intrinsic::nvvm_f2i_rz_ftz:
509 case Intrinsic::nvvm_f2ui_rz_ftz:
510 case Intrinsic::nvvm_d2i_rz:
511 case Intrinsic::nvvm_d2ui_rz:
512
513 case Intrinsic::nvvm_f2ll_rz:
514 case Intrinsic::nvvm_f2ull_rz:
515 case Intrinsic::nvvm_f2ll_rz_ftz:
516 case Intrinsic::nvvm_f2ull_rz_ftz:
517 case Intrinsic::nvvm_d2ll_rz:
518 case Intrinsic::nvvm_d2ull_rz:
520 }
521 llvm_unreachable("Checking rounding mode for invalid f2i/d2i intrinsic");
522}
523
524inline bool FMinFMaxShouldFTZ(Intrinsic::ID IntrinsicID) {
525 switch (IntrinsicID) {
526 case Intrinsic::nvvm_fmax_ftz_f:
527 case Intrinsic::nvvm_fmax_ftz_nan_f:
528 case Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f:
529 case Intrinsic::nvvm_fmax_ftz_xorsign_abs_f:
530
531 case Intrinsic::nvvm_fmin_ftz_f:
532 case Intrinsic::nvvm_fmin_ftz_nan_f:
533 case Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f:
534 case Intrinsic::nvvm_fmin_ftz_xorsign_abs_f:
535 return true;
536
537 case Intrinsic::nvvm_fmax_d:
538 case Intrinsic::nvvm_fmax_f:
539 case Intrinsic::nvvm_fmax_nan_f:
540 case Intrinsic::nvvm_fmax_nan_xorsign_abs_f:
541 case Intrinsic::nvvm_fmax_xorsign_abs_f:
542
543 case Intrinsic::nvvm_fmin_d:
544 case Intrinsic::nvvm_fmin_f:
545 case Intrinsic::nvvm_fmin_nan_f:
546 case Intrinsic::nvvm_fmin_nan_xorsign_abs_f:
547 case Intrinsic::nvvm_fmin_xorsign_abs_f:
548 return false;
549 }
550 llvm_unreachable("Checking FTZ flag for invalid fmin/fmax intrinsic");
551}
552
553inline bool FMinFMaxPropagatesNaNs(Intrinsic::ID IntrinsicID) {
554 switch (IntrinsicID) {
555 case Intrinsic::nvvm_fmax_ftz_nan_f:
556 case Intrinsic::nvvm_fmax_nan_f:
557 case Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f:
558 case Intrinsic::nvvm_fmax_nan_xorsign_abs_f:
559
560 case Intrinsic::nvvm_fmin_ftz_nan_f:
561 case Intrinsic::nvvm_fmin_nan_f:
562 case Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f:
563 case Intrinsic::nvvm_fmin_nan_xorsign_abs_f:
564 return true;
565
566 case Intrinsic::nvvm_fmax_d:
567 case Intrinsic::nvvm_fmax_f:
568 case Intrinsic::nvvm_fmax_ftz_f:
569 case Intrinsic::nvvm_fmax_ftz_xorsign_abs_f:
570 case Intrinsic::nvvm_fmax_xorsign_abs_f:
571
572 case Intrinsic::nvvm_fmin_d:
573 case Intrinsic::nvvm_fmin_f:
574 case Intrinsic::nvvm_fmin_ftz_f:
575 case Intrinsic::nvvm_fmin_ftz_xorsign_abs_f:
576 case Intrinsic::nvvm_fmin_xorsign_abs_f:
577 return false;
578 }
579 llvm_unreachable("Checking NaN flag for invalid fmin/fmax intrinsic");
580}
581
582inline bool FMinFMaxIsXorSignAbs(Intrinsic::ID IntrinsicID) {
583 switch (IntrinsicID) {
584 case Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f:
585 case Intrinsic::nvvm_fmax_ftz_xorsign_abs_f:
586 case Intrinsic::nvvm_fmax_nan_xorsign_abs_f:
587 case Intrinsic::nvvm_fmax_xorsign_abs_f:
588
589 case Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f:
590 case Intrinsic::nvvm_fmin_ftz_xorsign_abs_f:
591 case Intrinsic::nvvm_fmin_nan_xorsign_abs_f:
592 case Intrinsic::nvvm_fmin_xorsign_abs_f:
593 return true;
594
595 case Intrinsic::nvvm_fmax_d:
596 case Intrinsic::nvvm_fmax_f:
597 case Intrinsic::nvvm_fmax_ftz_f:
598 case Intrinsic::nvvm_fmax_ftz_nan_f:
599 case Intrinsic::nvvm_fmax_nan_f:
600
601 case Intrinsic::nvvm_fmin_d:
602 case Intrinsic::nvvm_fmin_f:
603 case Intrinsic::nvvm_fmin_ftz_f:
604 case Intrinsic::nvvm_fmin_ftz_nan_f:
605 case Intrinsic::nvvm_fmin_nan_f:
606 return false;
607 }
608 llvm_unreachable("Checking XorSignAbs flag for invalid fmin/fmax intrinsic");
609}
610
612 switch (IntrinsicID) {
613 case Intrinsic::nvvm_ceil_ftz_f:
614 case Intrinsic::nvvm_fabs_ftz:
615 case Intrinsic::nvvm_floor_ftz_f:
616 case Intrinsic::nvvm_round_ftz_f:
617 case Intrinsic::nvvm_saturate_ftz_f:
618 case Intrinsic::nvvm_sqrt_rn_ftz_f:
619 return true;
620 case Intrinsic::nvvm_ceil_f:
621 case Intrinsic::nvvm_ceil_d:
622 case Intrinsic::nvvm_fabs:
623 case Intrinsic::nvvm_floor_f:
624 case Intrinsic::nvvm_floor_d:
625 case Intrinsic::nvvm_round_f:
626 case Intrinsic::nvvm_round_d:
627 case Intrinsic::nvvm_saturate_d:
628 case Intrinsic::nvvm_saturate_f:
629 case Intrinsic::nvvm_sqrt_f:
630 case Intrinsic::nvvm_sqrt_rn_d:
631 case Intrinsic::nvvm_sqrt_rn_f:
632 return false;
633 }
634 llvm_unreachable("Checking FTZ flag for invalid unary intrinsic");
635}
636
637inline bool RCPShouldFTZ(Intrinsic::ID IntrinsicID) {
638 switch (IntrinsicID) {
639 case Intrinsic::nvvm_rcp_rm_ftz_f:
640 case Intrinsic::nvvm_rcp_rn_ftz_f:
641 case Intrinsic::nvvm_rcp_rp_ftz_f:
642 case Intrinsic::nvvm_rcp_rz_ftz_f:
643 return true;
644 case Intrinsic::nvvm_rcp_rm_d:
645 case Intrinsic::nvvm_rcp_rm_f:
646 case Intrinsic::nvvm_rcp_rn_d:
647 case Intrinsic::nvvm_rcp_rn_f:
648 case Intrinsic::nvvm_rcp_rp_d:
649 case Intrinsic::nvvm_rcp_rp_f:
650 case Intrinsic::nvvm_rcp_rz_d:
651 case Intrinsic::nvvm_rcp_rz_f:
652 return false;
653 }
654 llvm_unreachable("Checking FTZ flag for invalid rcp intrinsic");
655}
656
658 switch (IntrinsicID) {
659 case Intrinsic::nvvm_rcp_rm_f:
660 case Intrinsic::nvvm_rcp_rm_d:
661 case Intrinsic::nvvm_rcp_rm_ftz_f:
663
664 case Intrinsic::nvvm_rcp_rn_f:
665 case Intrinsic::nvvm_rcp_rn_d:
666 case Intrinsic::nvvm_rcp_rn_ftz_f:
668
669 case Intrinsic::nvvm_rcp_rp_f:
670 case Intrinsic::nvvm_rcp_rp_d:
671 case Intrinsic::nvvm_rcp_rp_ftz_f:
673
674 case Intrinsic::nvvm_rcp_rz_f:
675 case Intrinsic::nvvm_rcp_rz_d:
676 case Intrinsic::nvvm_rcp_rz_ftz_f:
678 }
679 llvm_unreachable("Checking rounding mode for invalid rcp intrinsic");
680}
681
682inline DenormalMode GetNVVMDenormMode(bool ShouldFTZ) {
683 if (ShouldFTZ)
685 return DenormalMode::getIEEE();
686}
687
689 return static_cast<APFloat::roundingMode>(
690 cast<ConstantInt>(ImmArgVal)->getSExtValue());
691}
692
694 switch (RM) {
696 return "rn";
698 return "rz";
700 return "rm";
702 return "rp";
703 default:
704 return "";
705 }
706}
707
708inline bool FPArithShouldFTZ(Intrinsic::ID IntrinsicID) {
709 switch (IntrinsicID) {
710 case Intrinsic::nvvm_fadd_ftz:
711 case Intrinsic::nvvm_fadd_ftz_sat:
712 case Intrinsic::nvvm_fma_ftz:
713 case Intrinsic::nvvm_fma_ftz_sat:
714 case Intrinsic::nvvm_fma_ftz_relu:
715 case Intrinsic::nvvm_fmul_ftz:
716 case Intrinsic::nvvm_fmul_ftz_sat:
717 return true;
718
719 case Intrinsic::nvvm_fadd:
720 case Intrinsic::nvvm_fadd_sat:
721 case Intrinsic::nvvm_fma:
722 case Intrinsic::nvvm_fma_sat:
723 case Intrinsic::nvvm_fma_relu:
724 case Intrinsic::nvvm_fmul:
725 case Intrinsic::nvvm_fmul_sat:
726 return false;
727 }
729 "Checking FTZ flag for invalid NVVM fadd/fmul/fma intrinsic");
730}
731
732inline bool FPArithIsSaturating(Intrinsic::ID IntrinsicID) {
733 switch (IntrinsicID) {
734 case Intrinsic::nvvm_fadd_sat:
735 case Intrinsic::nvvm_fadd_ftz_sat:
736 case Intrinsic::nvvm_fma_sat:
737 case Intrinsic::nvvm_fma_ftz_sat:
738 case Intrinsic::nvvm_fmul_sat:
739 case Intrinsic::nvvm_fmul_ftz_sat:
740 return true;
741
742 case Intrinsic::nvvm_fadd:
743 case Intrinsic::nvvm_fadd_ftz:
744 case Intrinsic::nvvm_fma:
745 case Intrinsic::nvvm_fma_ftz:
746 case Intrinsic::nvvm_fma_relu:
747 case Intrinsic::nvvm_fma_ftz_relu:
748 case Intrinsic::nvvm_fmul:
749 case Intrinsic::nvvm_fmul_ftz:
750 return false;
751 }
753 "Checking sat flag for invalid NVVM fadd/fmul/fma intrinsic");
754}
755
756inline bool FPArithIsRelu(Intrinsic::ID IntrinsicID) {
757 switch (IntrinsicID) {
758 case Intrinsic::nvvm_fma_relu:
759 case Intrinsic::nvvm_fma_ftz_relu:
760 return true;
761
762 case Intrinsic::nvvm_fadd:
763 case Intrinsic::nvvm_fadd_ftz:
764 case Intrinsic::nvvm_fadd_sat:
765 case Intrinsic::nvvm_fadd_ftz_sat:
766 case Intrinsic::nvvm_fma:
767 case Intrinsic::nvvm_fma_ftz:
768 case Intrinsic::nvvm_fma_sat:
769 case Intrinsic::nvvm_fma_ftz_sat:
770 case Intrinsic::nvvm_fmul:
771 case Intrinsic::nvvm_fmul_ftz:
772 case Intrinsic::nvvm_fmul_sat:
773 case Intrinsic::nvvm_fmul_ftz_sat:
774 return false;
775 }
777 "Checking relu flag for invalid NVVM fadd/fmul/fma intrinsic");
778}
779
780inline bool FDivShouldFTZ(Intrinsic::ID IntrinsicID) {
781 switch (IntrinsicID) {
782 case Intrinsic::nvvm_div_rm_ftz_f:
783 case Intrinsic::nvvm_div_rn_ftz_f:
784 case Intrinsic::nvvm_div_rp_ftz_f:
785 case Intrinsic::nvvm_div_rz_ftz_f:
786 return true;
787
788 case Intrinsic::nvvm_div_rm_f:
789 case Intrinsic::nvvm_div_rn_f:
790 case Intrinsic::nvvm_div_rp_f:
791 case Intrinsic::nvvm_div_rz_f:
792 case Intrinsic::nvvm_div_rm_d:
793 case Intrinsic::nvvm_div_rn_d:
794 case Intrinsic::nvvm_div_rp_d:
795 case Intrinsic::nvvm_div_rz_d:
796 return false;
797 }
798 llvm_unreachable("Checking FTZ flag for invalid NVVM div intrinsic");
799}
800
802 switch (IntrinsicID) {
803 case Intrinsic::nvvm_div_rm_f:
804 case Intrinsic::nvvm_div_rm_d:
805 case Intrinsic::nvvm_div_rm_ftz_f:
807 case Intrinsic::nvvm_div_rn_f:
808 case Intrinsic::nvvm_div_rn_d:
809 case Intrinsic::nvvm_div_rn_ftz_f:
811 case Intrinsic::nvvm_div_rp_f:
812 case Intrinsic::nvvm_div_rp_d:
813 case Intrinsic::nvvm_div_rp_ftz_f:
815 case Intrinsic::nvvm_div_rz_f:
816 case Intrinsic::nvvm_div_rz_d:
817 case Intrinsic::nvvm_div_rz_ftz_f:
819 }
820 llvm_unreachable("Invalid FP instrinsic rounding mode for NVVM div");
821}
822
823} // namespace nvvm
824} // namespace llvm
825#endif // LLVM_IR_NVVMINTRINSICUTILS_H
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
#define LLVM_ABI
Definition Compiler.h:211
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static constexpr roundingMode rmTowardZero
Definition APFloat.h:365
llvm::RoundingMode roundingMode
IEEE-754R 4.3: Rounding-direction attributes.
Definition APFloat.h:359
static constexpr roundingMode rmTowardNegative
Definition APFloat.h:364
static constexpr roundingMode rmNearestTiesToEven
Definition APFloat.h:361
static constexpr roundingMode rmTowardPositive
Definition APFloat.h:363
This is an important base class in LLVM.
Definition Constant.h:44
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
LLVM Value Representation.
Definition Value.h:75
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition raw_ostream.h:53
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
APFloat::roundingMode GetRoundingModeFromImmArg(const Value *ImmArgVal)
LLVM_ABI void printTcgen05MMACollectorBBuffer(raw_ostream &OS, const Constant *ImmArgVal)
StringRef GetRoundingModeName(APFloat::roundingMode RM)
LLVM_ABI void printTensormapSwizzleMode(raw_ostream &OS, const Constant *ImmArgVal)
DenormalMode GetNVVMDenormMode(bool ShouldFTZ)
StringRef getEvictPolicyName(EvictPolicyType Policy)
LLVM_ABI void printFPRoundingMode(raw_ostream &OS, const Constant *ImmArgVal)
LLVM_ABI void printTMAReductionOp(raw_ostream &OS, const Constant *ImmArgVal)
LLVM_ABI void printTMAValidateDataPattern(raw_ostream &OS, const Constant *ImmArgVal)
LLVM_ABI void printTensormapAllocClass(raw_ostream &OS, const Constant *ImmArgVal)
bool FPToIntegerIntrinsicNaNZero(Intrinsic::ID IntrinsicID)
LLVM_ABI void printTensormapInterleaveLayout(raw_ostream &OS, const Constant *ImmArgVal)
APFloat::roundingMode GetFDivRoundingMode(Intrinsic::ID IntrinsicID)
LLVM_ABI void printTensormapSwizzleAtomicity(raw_ostream &OS, const Constant *ImmArgVal)
bool FPToIntegerIntrinsicResultIsSigned(Intrinsic::ID IntrinsicID)
APFloat::roundingMode GetFPToIntegerRoundingMode(Intrinsic::ID IntrinsicID)
bool RCPShouldFTZ(Intrinsic::ID IntrinsicID)
StringRef getTMATensorReductionOpName(TMAReductionOp Op)
bool FPToIntegerIntrinsicShouldFTZ(Intrinsic::ID IntrinsicID)
LLVM_ABI void printMemScope(raw_ostream &OS, const Constant *ImmArgVal)
StringRef getTMAValidateDataPatternName(TMAValidateDataPattern Pattern)
bool FDivShouldFTZ(Intrinsic::ID IntrinsicID)
bool FMinFMaxIsXorSignAbs(Intrinsic::ID IntrinsicID)
bool FPArithIsSaturating(Intrinsic::ID IntrinsicID)
bool FPArithShouldFTZ(Intrinsic::ID IntrinsicID)
bool FPArithIsRelu(Intrinsic::ID IntrinsicID)
LLVM_ABI void printTcgen05MMAKind(raw_ostream &OS, const Constant *ImmArgVal)
LLVM_ABI void printTcgen05CollectorUsageOp(raw_ostream &OS, const Constant *ImmArgVal)
bool UnaryMathIntrinsicShouldFTZ(Intrinsic::ID IntrinsicID)
bool FMinFMaxShouldFTZ(Intrinsic::ID IntrinsicID)
LLVM_ABI void printEvictPolicyType(raw_ostream &OS, const Constant *ImmArgVal)
LLVM_ABI void printMBarrierLayout(raw_ostream &OS, const Constant *ImmArgVal)
StringRef getMemScopeName(MemScope Scope)
LLVM_ABI void printTensormapFillMode(raw_ostream &OS, const Constant *ImmArgVal)
APFloat::roundingMode GetRCPRoundingMode(Intrinsic::ID IntrinsicID)
bool FMinFMaxPropagatesNaNs(Intrinsic::ID IntrinsicID)
LLVM_ABI void printTensormapElemType(raw_ostream &OS, const Constant *ImmArgVal)
This is an optimization pass for GlobalISel generic memory operations.
DWARFExpression::Operation Op
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
Represent subnormal handling kind for floating point instruction inputs and outputs.
static constexpr DenormalMode getPreserveSign()
static constexpr DenormalMode getIEEE()