LLVM 24.0.0git
NVVMIntrinsicUtils.h
Go to the documentation of this file.
1//===--- NVVMIntrinsicUtils.h -----------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This file contains the definitions of the enumerations and flags
11/// associated with NVVM Intrinsics, along with some helper functions.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_IR_NVVMINTRINSICUTILS_H
16#define LLVM_IR_NVVMINTRINSICUTILS_H
17
18#include <stdint.h>
19
20#include "llvm/ADT/APFloat.h"
21#include "llvm/ADT/APInt.h"
22#include "llvm/ADT/StringRef.h"
23#include "llvm/IR/Constants.h"
24#include "llvm/IR/Intrinsics.h"
25#include "llvm/IR/IntrinsicsNVPTX.h"
26#include "llvm/Support/raw_ostream.h"
27
28namespace llvm {
29namespace nvvm {
30
31// Reduction Ops supported with TMA Copy from Shared
32// to Global Memory for the "cp.reduce.async.bulk.tensor.*"
33// family of PTX instructions.
34enum class TMAReductionOp : uint8_t {
35 ADD = 0,
36 MIN = 1,
37 MAX = 2,
38 INC = 3,
39 DEC = 4,
40 AND = 5,
41 OR = 6,
42 XOR = 7,
43};
44
45inline StringRef getTMATensorReductionOpName(TMAReductionOp Op) {
46 switch (Op) {
47 case TMAReductionOp::ADD:
48 return "add";
49 case TMAReductionOp::MIN:
50 return "min";
51 case TMAReductionOp::MAX:
52 return "max";
53 case TMAReductionOp::INC:
54 return "inc";
55 case TMAReductionOp::DEC:
56 return "dec";
57 case TMAReductionOp::AND:
58 return "and";
59 case TMAReductionOp::OR:
60 return "or";
61 case TMAReductionOp::XOR:
62 return "xor";
63 }
64 llvm_unreachable("invalid TMA tensorreduction operation");
65}
66
67// Enum to represent the cta_group::1 and
68// cta_group::2 variants in TMA/TCGEN05 family of
69// PTX instructions.
70enum class CTAGroupKind : uint8_t {
71 CG_NONE = 0, // default with no cta_group modifier
72 CG_1 = 1, // cta_group::1 modifier
73 CG_2 = 2, // cta_group::2 modifier
74};
75
76enum class Tcgen05MMAKind : uint8_t { F16 = 0, TF32 = 1, F8F6F4 = 2, I8 = 3 };
77
78enum class Tcgen05CollectorUsageOp : uint8_t {
79 DISCARD = 0,
80 LASTUSE = 1,
81 FILL = 2,
82 USE = 3,
83};
84
85enum class TensormapElemType : uint8_t {
86 U8 = 0,
87 U16 = 1,
88 U32 = 2,
89 S32 = 3,
90 U64 = 4,
91 S64 = 5,
92 F16 = 6,
93 F32 = 7,
94 F32_FTZ = 8,
95 F64 = 9,
96 BF16 = 10,
97 TF32 = 11,
98 TF32_FTZ = 12,
99 B4x16 = 13,
100 B4x16_p64 = 14,
101 B6x16_p32 = 15,
102};
103
109
117
124
129
130LLVM_ABI void printTcgen05MMAKind(raw_ostream &OS, const Constant *ImmArgVal);
131
132LLVM_ABI void printTMAReductionOp(raw_ostream &OS, const Constant *ImmArgVal);
133
134LLVM_ABI void printTcgen05CollectorUsageOp(raw_ostream &OS,
135 const Constant *ImmArgVal);
136
137LLVM_ABI void printTensormapElemType(raw_ostream &OS,
138 const Constant *ImmArgVal);
139LLVM_ABI void printTensormapInterleaveLayout(raw_ostream &OS,
140 const Constant *ImmArgVal);
141LLVM_ABI void printTensormapSwizzleMode(raw_ostream &OS,
142 const Constant *ImmArgVal);
143LLVM_ABI void printTensormapSwizzleAtomicity(raw_ostream &OS,
144 const Constant *ImmArgVal);
145LLVM_ABI void printTensormapFillMode(raw_ostream &OS,
146 const Constant *ImmArgVal);
147
148inline bool FPToIntegerIntrinsicShouldFTZ(Intrinsic::ID IntrinsicID) {
149 switch (IntrinsicID) {
150 case Intrinsic::nvvm_f2i_rm_ftz:
151 case Intrinsic::nvvm_f2i_rn_ftz:
152 case Intrinsic::nvvm_f2i_rp_ftz:
153 case Intrinsic::nvvm_f2i_rz_ftz:
154
155 case Intrinsic::nvvm_f2ui_rm_ftz:
156 case Intrinsic::nvvm_f2ui_rn_ftz:
157 case Intrinsic::nvvm_f2ui_rp_ftz:
158 case Intrinsic::nvvm_f2ui_rz_ftz:
159
160 case Intrinsic::nvvm_f2ll_rm_ftz:
161 case Intrinsic::nvvm_f2ll_rn_ftz:
162 case Intrinsic::nvvm_f2ll_rp_ftz:
163 case Intrinsic::nvvm_f2ll_rz_ftz:
164
165 case Intrinsic::nvvm_f2ull_rm_ftz:
166 case Intrinsic::nvvm_f2ull_rn_ftz:
167 case Intrinsic::nvvm_f2ull_rp_ftz:
168 case Intrinsic::nvvm_f2ull_rz_ftz:
169 return true;
170
171 case Intrinsic::nvvm_f2i_rm:
172 case Intrinsic::nvvm_f2i_rn:
173 case Intrinsic::nvvm_f2i_rp:
174 case Intrinsic::nvvm_f2i_rz:
175
176 case Intrinsic::nvvm_f2ui_rm:
177 case Intrinsic::nvvm_f2ui_rn:
178 case Intrinsic::nvvm_f2ui_rp:
179 case Intrinsic::nvvm_f2ui_rz:
180
181 case Intrinsic::nvvm_d2i_rm:
182 case Intrinsic::nvvm_d2i_rn:
183 case Intrinsic::nvvm_d2i_rp:
184 case Intrinsic::nvvm_d2i_rz:
185
186 case Intrinsic::nvvm_d2ui_rm:
187 case Intrinsic::nvvm_d2ui_rn:
188 case Intrinsic::nvvm_d2ui_rp:
189 case Intrinsic::nvvm_d2ui_rz:
190
191 case Intrinsic::nvvm_f2ll_rm:
192 case Intrinsic::nvvm_f2ll_rn:
193 case Intrinsic::nvvm_f2ll_rp:
194 case Intrinsic::nvvm_f2ll_rz:
195
196 case Intrinsic::nvvm_f2ull_rm:
197 case Intrinsic::nvvm_f2ull_rn:
198 case Intrinsic::nvvm_f2ull_rp:
199 case Intrinsic::nvvm_f2ull_rz:
200
201 case Intrinsic::nvvm_d2ll_rm:
202 case Intrinsic::nvvm_d2ll_rn:
203 case Intrinsic::nvvm_d2ll_rp:
204 case Intrinsic::nvvm_d2ll_rz:
205
206 case Intrinsic::nvvm_d2ull_rm:
207 case Intrinsic::nvvm_d2ull_rn:
208 case Intrinsic::nvvm_d2ull_rp:
209 case Intrinsic::nvvm_d2ull_rz:
210 return false;
211 }
212 llvm_unreachable("Checking FTZ flag for invalid f2i/d2i intrinsic");
213}
214
215inline bool FPToIntegerIntrinsicResultIsSigned(Intrinsic::ID IntrinsicID) {
216 switch (IntrinsicID) {
217 // f2i
218 case Intrinsic::nvvm_f2i_rm:
219 case Intrinsic::nvvm_f2i_rm_ftz:
220 case Intrinsic::nvvm_f2i_rn:
221 case Intrinsic::nvvm_f2i_rn_ftz:
222 case Intrinsic::nvvm_f2i_rp:
223 case Intrinsic::nvvm_f2i_rp_ftz:
224 case Intrinsic::nvvm_f2i_rz:
225 case Intrinsic::nvvm_f2i_rz_ftz:
226 // d2i
227 case Intrinsic::nvvm_d2i_rm:
228 case Intrinsic::nvvm_d2i_rn:
229 case Intrinsic::nvvm_d2i_rp:
230 case Intrinsic::nvvm_d2i_rz:
231 // f2ll
232 case Intrinsic::nvvm_f2ll_rm:
233 case Intrinsic::nvvm_f2ll_rm_ftz:
234 case Intrinsic::nvvm_f2ll_rn:
235 case Intrinsic::nvvm_f2ll_rn_ftz:
236 case Intrinsic::nvvm_f2ll_rp:
237 case Intrinsic::nvvm_f2ll_rp_ftz:
238 case Intrinsic::nvvm_f2ll_rz:
239 case Intrinsic::nvvm_f2ll_rz_ftz:
240 // d2ll
241 case Intrinsic::nvvm_d2ll_rm:
242 case Intrinsic::nvvm_d2ll_rn:
243 case Intrinsic::nvvm_d2ll_rp:
244 case Intrinsic::nvvm_d2ll_rz:
245 return true;
246
247 // f2ui
248 case Intrinsic::nvvm_f2ui_rm:
249 case Intrinsic::nvvm_f2ui_rm_ftz:
250 case Intrinsic::nvvm_f2ui_rn:
251 case Intrinsic::nvvm_f2ui_rn_ftz:
252 case Intrinsic::nvvm_f2ui_rp:
253 case Intrinsic::nvvm_f2ui_rp_ftz:
254 case Intrinsic::nvvm_f2ui_rz:
255 case Intrinsic::nvvm_f2ui_rz_ftz:
256 // d2ui
257 case Intrinsic::nvvm_d2ui_rm:
258 case Intrinsic::nvvm_d2ui_rn:
259 case Intrinsic::nvvm_d2ui_rp:
260 case Intrinsic::nvvm_d2ui_rz:
261 // f2ull
262 case Intrinsic::nvvm_f2ull_rm:
263 case Intrinsic::nvvm_f2ull_rm_ftz:
264 case Intrinsic::nvvm_f2ull_rn:
265 case Intrinsic::nvvm_f2ull_rn_ftz:
266 case Intrinsic::nvvm_f2ull_rp:
267 case Intrinsic::nvvm_f2ull_rp_ftz:
268 case Intrinsic::nvvm_f2ull_rz:
269 case Intrinsic::nvvm_f2ull_rz_ftz:
270 // d2ull
271 case Intrinsic::nvvm_d2ull_rm:
272 case Intrinsic::nvvm_d2ull_rn:
273 case Intrinsic::nvvm_d2ull_rp:
274 case Intrinsic::nvvm_d2ull_rz:
275 return false;
276 }
277 llvm_unreachable(
278 "Checking invalid f2i/d2i intrinsic for signed int conversion");
279}
280
281inline bool FPToIntegerIntrinsicNaNZero(Intrinsic::ID IntrinsicID) {
282 switch (IntrinsicID) {
283 // f2i
284 case Intrinsic::nvvm_f2i_rm:
285 case Intrinsic::nvvm_f2i_rn:
286 case Intrinsic::nvvm_f2i_rp:
287 case Intrinsic::nvvm_f2i_rz:
288 case Intrinsic::nvvm_f2i_rm_ftz:
289 case Intrinsic::nvvm_f2i_rn_ftz:
290 case Intrinsic::nvvm_f2i_rp_ftz:
291 case Intrinsic::nvvm_f2i_rz_ftz:
292 // f2ui
293 case Intrinsic::nvvm_f2ui_rm:
294 case Intrinsic::nvvm_f2ui_rn:
295 case Intrinsic::nvvm_f2ui_rp:
296 case Intrinsic::nvvm_f2ui_rz:
297 case Intrinsic::nvvm_f2ui_rm_ftz:
298 case Intrinsic::nvvm_f2ui_rn_ftz:
299 case Intrinsic::nvvm_f2ui_rp_ftz:
300 case Intrinsic::nvvm_f2ui_rz_ftz:
301 return true;
302 // d2i
303 case Intrinsic::nvvm_d2i_rm:
304 case Intrinsic::nvvm_d2i_rn:
305 case Intrinsic::nvvm_d2i_rp:
306 case Intrinsic::nvvm_d2i_rz:
307 // d2ui
308 case Intrinsic::nvvm_d2ui_rm:
309 case Intrinsic::nvvm_d2ui_rn:
310 case Intrinsic::nvvm_d2ui_rp:
311 case Intrinsic::nvvm_d2ui_rz:
312 // f2ll
313 case Intrinsic::nvvm_f2ll_rm:
314 case Intrinsic::nvvm_f2ll_rn:
315 case Intrinsic::nvvm_f2ll_rp:
316 case Intrinsic::nvvm_f2ll_rz:
317 case Intrinsic::nvvm_f2ll_rm_ftz:
318 case Intrinsic::nvvm_f2ll_rn_ftz:
319 case Intrinsic::nvvm_f2ll_rp_ftz:
320 case Intrinsic::nvvm_f2ll_rz_ftz:
321 // f2ull
322 case Intrinsic::nvvm_f2ull_rm:
323 case Intrinsic::nvvm_f2ull_rn:
324 case Intrinsic::nvvm_f2ull_rp:
325 case Intrinsic::nvvm_f2ull_rz:
326 case Intrinsic::nvvm_f2ull_rm_ftz:
327 case Intrinsic::nvvm_f2ull_rn_ftz:
328 case Intrinsic::nvvm_f2ull_rp_ftz:
329 case Intrinsic::nvvm_f2ull_rz_ftz:
330 // d2ll
331 case Intrinsic::nvvm_d2ll_rm:
332 case Intrinsic::nvvm_d2ll_rn:
333 case Intrinsic::nvvm_d2ll_rp:
334 case Intrinsic::nvvm_d2ll_rz:
335 // d2ull
336 case Intrinsic::nvvm_d2ull_rm:
337 case Intrinsic::nvvm_d2ull_rn:
338 case Intrinsic::nvvm_d2ull_rp:
339 case Intrinsic::nvvm_d2ull_rz:
340 return false;
341 }
342 llvm_unreachable("Checking NaN result for invalid f2i/d2i intrinsic");
343}
344
345inline APFloat::roundingMode
346GetFPToIntegerRoundingMode(Intrinsic::ID IntrinsicID) {
347 switch (IntrinsicID) {
348 // RM:
349 case Intrinsic::nvvm_f2i_rm:
350 case Intrinsic::nvvm_f2ui_rm:
351 case Intrinsic::nvvm_f2i_rm_ftz:
352 case Intrinsic::nvvm_f2ui_rm_ftz:
353 case Intrinsic::nvvm_d2i_rm:
354 case Intrinsic::nvvm_d2ui_rm:
355
356 case Intrinsic::nvvm_f2ll_rm:
357 case Intrinsic::nvvm_f2ull_rm:
358 case Intrinsic::nvvm_f2ll_rm_ftz:
359 case Intrinsic::nvvm_f2ull_rm_ftz:
360 case Intrinsic::nvvm_d2ll_rm:
361 case Intrinsic::nvvm_d2ull_rm:
362 return APFloat::rmTowardNegative;
363
364 // RN:
365 case Intrinsic::nvvm_f2i_rn:
366 case Intrinsic::nvvm_f2ui_rn:
367 case Intrinsic::nvvm_f2i_rn_ftz:
368 case Intrinsic::nvvm_f2ui_rn_ftz:
369 case Intrinsic::nvvm_d2i_rn:
370 case Intrinsic::nvvm_d2ui_rn:
371
372 case Intrinsic::nvvm_f2ll_rn:
373 case Intrinsic::nvvm_f2ull_rn:
374 case Intrinsic::nvvm_f2ll_rn_ftz:
375 case Intrinsic::nvvm_f2ull_rn_ftz:
376 case Intrinsic::nvvm_d2ll_rn:
377 case Intrinsic::nvvm_d2ull_rn:
378 return APFloat::rmNearestTiesToEven;
379
380 // RP:
381 case Intrinsic::nvvm_f2i_rp:
382 case Intrinsic::nvvm_f2ui_rp:
383 case Intrinsic::nvvm_f2i_rp_ftz:
384 case Intrinsic::nvvm_f2ui_rp_ftz:
385 case Intrinsic::nvvm_d2i_rp:
386 case Intrinsic::nvvm_d2ui_rp:
387
388 case Intrinsic::nvvm_f2ll_rp:
389 case Intrinsic::nvvm_f2ull_rp:
390 case Intrinsic::nvvm_f2ll_rp_ftz:
391 case Intrinsic::nvvm_f2ull_rp_ftz:
392 case Intrinsic::nvvm_d2ll_rp:
393 case Intrinsic::nvvm_d2ull_rp:
394 return APFloat::rmTowardPositive;
395
396 // RZ:
397 case Intrinsic::nvvm_f2i_rz:
398 case Intrinsic::nvvm_f2ui_rz:
399 case Intrinsic::nvvm_f2i_rz_ftz:
400 case Intrinsic::nvvm_f2ui_rz_ftz:
401 case Intrinsic::nvvm_d2i_rz:
402 case Intrinsic::nvvm_d2ui_rz:
403
404 case Intrinsic::nvvm_f2ll_rz:
405 case Intrinsic::nvvm_f2ull_rz:
406 case Intrinsic::nvvm_f2ll_rz_ftz:
407 case Intrinsic::nvvm_f2ull_rz_ftz:
408 case Intrinsic::nvvm_d2ll_rz:
409 case Intrinsic::nvvm_d2ull_rz:
410 return APFloat::rmTowardZero;
411 }
412 llvm_unreachable("Checking rounding mode for invalid f2i/d2i intrinsic");
413}
414
415inline bool FMinFMaxShouldFTZ(Intrinsic::ID IntrinsicID) {
416 switch (IntrinsicID) {
417 case Intrinsic::nvvm_fmax_ftz_f:
418 case Intrinsic::nvvm_fmax_ftz_nan_f:
419 case Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f:
420 case Intrinsic::nvvm_fmax_ftz_xorsign_abs_f:
421
422 case Intrinsic::nvvm_fmin_ftz_f:
423 case Intrinsic::nvvm_fmin_ftz_nan_f:
424 case Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f:
425 case Intrinsic::nvvm_fmin_ftz_xorsign_abs_f:
426 return true;
427
428 case Intrinsic::nvvm_fmax_d:
429 case Intrinsic::nvvm_fmax_f:
430 case Intrinsic::nvvm_fmax_nan_f:
431 case Intrinsic::nvvm_fmax_nan_xorsign_abs_f:
432 case Intrinsic::nvvm_fmax_xorsign_abs_f:
433
434 case Intrinsic::nvvm_fmin_d:
435 case Intrinsic::nvvm_fmin_f:
436 case Intrinsic::nvvm_fmin_nan_f:
437 case Intrinsic::nvvm_fmin_nan_xorsign_abs_f:
438 case Intrinsic::nvvm_fmin_xorsign_abs_f:
439 return false;
440 }
441 llvm_unreachable("Checking FTZ flag for invalid fmin/fmax intrinsic");
442}
443
444inline bool FMinFMaxPropagatesNaNs(Intrinsic::ID IntrinsicID) {
445 switch (IntrinsicID) {
446 case Intrinsic::nvvm_fmax_ftz_nan_f:
447 case Intrinsic::nvvm_fmax_nan_f:
448 case Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f:
449 case Intrinsic::nvvm_fmax_nan_xorsign_abs_f:
450
451 case Intrinsic::nvvm_fmin_ftz_nan_f:
452 case Intrinsic::nvvm_fmin_nan_f:
453 case Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f:
454 case Intrinsic::nvvm_fmin_nan_xorsign_abs_f:
455 return true;
456
457 case Intrinsic::nvvm_fmax_d:
458 case Intrinsic::nvvm_fmax_f:
459 case Intrinsic::nvvm_fmax_ftz_f:
460 case Intrinsic::nvvm_fmax_ftz_xorsign_abs_f:
461 case Intrinsic::nvvm_fmax_xorsign_abs_f:
462
463 case Intrinsic::nvvm_fmin_d:
464 case Intrinsic::nvvm_fmin_f:
465 case Intrinsic::nvvm_fmin_ftz_f:
466 case Intrinsic::nvvm_fmin_ftz_xorsign_abs_f:
467 case Intrinsic::nvvm_fmin_xorsign_abs_f:
468 return false;
469 }
470 llvm_unreachable("Checking NaN flag for invalid fmin/fmax intrinsic");
471}
472
473inline bool FMinFMaxIsXorSignAbs(Intrinsic::ID IntrinsicID) {
474 switch (IntrinsicID) {
475 case Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f:
476 case Intrinsic::nvvm_fmax_ftz_xorsign_abs_f:
477 case Intrinsic::nvvm_fmax_nan_xorsign_abs_f:
478 case Intrinsic::nvvm_fmax_xorsign_abs_f:
479
480 case Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f:
481 case Intrinsic::nvvm_fmin_ftz_xorsign_abs_f:
482 case Intrinsic::nvvm_fmin_nan_xorsign_abs_f:
483 case Intrinsic::nvvm_fmin_xorsign_abs_f:
484 return true;
485
486 case Intrinsic::nvvm_fmax_d:
487 case Intrinsic::nvvm_fmax_f:
488 case Intrinsic::nvvm_fmax_ftz_f:
489 case Intrinsic::nvvm_fmax_ftz_nan_f:
490 case Intrinsic::nvvm_fmax_nan_f:
491
492 case Intrinsic::nvvm_fmin_d:
493 case Intrinsic::nvvm_fmin_f:
494 case Intrinsic::nvvm_fmin_ftz_f:
495 case Intrinsic::nvvm_fmin_ftz_nan_f:
496 case Intrinsic::nvvm_fmin_nan_f:
497 return false;
498 }
499 llvm_unreachable("Checking XorSignAbs flag for invalid fmin/fmax intrinsic");
500}
501
502inline bool UnaryMathIntrinsicShouldFTZ(Intrinsic::ID IntrinsicID) {
503 switch (IntrinsicID) {
504 case Intrinsic::nvvm_ceil_ftz_f:
505 case Intrinsic::nvvm_fabs_ftz:
506 case Intrinsic::nvvm_floor_ftz_f:
507 case Intrinsic::nvvm_round_ftz_f:
508 case Intrinsic::nvvm_saturate_ftz_f:
509 case Intrinsic::nvvm_sqrt_rn_ftz_f:
510 return true;
511 case Intrinsic::nvvm_ceil_f:
512 case Intrinsic::nvvm_ceil_d:
513 case Intrinsic::nvvm_fabs:
514 case Intrinsic::nvvm_floor_f:
515 case Intrinsic::nvvm_floor_d:
516 case Intrinsic::nvvm_round_f:
517 case Intrinsic::nvvm_round_d:
518 case Intrinsic::nvvm_saturate_d:
519 case Intrinsic::nvvm_saturate_f:
520 case Intrinsic::nvvm_sqrt_f:
521 case Intrinsic::nvvm_sqrt_rn_d:
522 case Intrinsic::nvvm_sqrt_rn_f:
523 return false;
524 }
525 llvm_unreachable("Checking FTZ flag for invalid unary intrinsic");
526}
527
528inline bool RCPShouldFTZ(Intrinsic::ID IntrinsicID) {
529 switch (IntrinsicID) {
530 case Intrinsic::nvvm_rcp_rm_ftz_f:
531 case Intrinsic::nvvm_rcp_rn_ftz_f:
532 case Intrinsic::nvvm_rcp_rp_ftz_f:
533 case Intrinsic::nvvm_rcp_rz_ftz_f:
534 return true;
535 case Intrinsic::nvvm_rcp_rm_d:
536 case Intrinsic::nvvm_rcp_rm_f:
537 case Intrinsic::nvvm_rcp_rn_d:
538 case Intrinsic::nvvm_rcp_rn_f:
539 case Intrinsic::nvvm_rcp_rp_d:
540 case Intrinsic::nvvm_rcp_rp_f:
541 case Intrinsic::nvvm_rcp_rz_d:
542 case Intrinsic::nvvm_rcp_rz_f:
543 return false;
544 }
545 llvm_unreachable("Checking FTZ flag for invalid rcp intrinsic");
546}
547
548inline APFloat::roundingMode GetRCPRoundingMode(Intrinsic::ID IntrinsicID) {
549 switch (IntrinsicID) {
550 case Intrinsic::nvvm_rcp_rm_f:
551 case Intrinsic::nvvm_rcp_rm_d:
552 case Intrinsic::nvvm_rcp_rm_ftz_f:
553 return APFloat::rmTowardNegative;
554
555 case Intrinsic::nvvm_rcp_rn_f:
556 case Intrinsic::nvvm_rcp_rn_d:
557 case Intrinsic::nvvm_rcp_rn_ftz_f:
558 return APFloat::rmNearestTiesToEven;
559
560 case Intrinsic::nvvm_rcp_rp_f:
561 case Intrinsic::nvvm_rcp_rp_d:
562 case Intrinsic::nvvm_rcp_rp_ftz_f:
563 return APFloat::rmTowardPositive;
564
565 case Intrinsic::nvvm_rcp_rz_f:
566 case Intrinsic::nvvm_rcp_rz_d:
567 case Intrinsic::nvvm_rcp_rz_ftz_f:
568 return APFloat::rmTowardZero;
569 }
570 llvm_unreachable("Checking rounding mode for invalid rcp intrinsic");
571}
572
573inline DenormalMode GetNVVMDenormMode(bool ShouldFTZ) {
574 if (ShouldFTZ)
575 return DenormalMode::getPreserveSign();
576 return DenormalMode::getIEEE();
577}
578
579inline bool FAddShouldFTZ(Intrinsic::ID IntrinsicID) {
580 switch (IntrinsicID) {
581 case Intrinsic::nvvm_add_rm_ftz_f:
582 case Intrinsic::nvvm_add_rn_ftz_f:
583 case Intrinsic::nvvm_add_rp_ftz_f:
584 case Intrinsic::nvvm_add_rz_ftz_f:
585 return true;
586
587 case Intrinsic::nvvm_add_rm_f:
588 case Intrinsic::nvvm_add_rn_f:
589 case Intrinsic::nvvm_add_rp_f:
590 case Intrinsic::nvvm_add_rz_f:
591 case Intrinsic::nvvm_add_rm_d:
592 case Intrinsic::nvvm_add_rn_d:
593 case Intrinsic::nvvm_add_rp_d:
594 case Intrinsic::nvvm_add_rz_d:
595 return false;
596 }
597 llvm_unreachable("Checking FTZ flag for invalid NVVM add intrinsic");
598}
599
600inline APFloat::roundingMode GetFAddRoundingMode(Intrinsic::ID IntrinsicID) {
601 switch (IntrinsicID) {
602 case Intrinsic::nvvm_add_rm_f:
603 case Intrinsic::nvvm_add_rm_d:
604 case Intrinsic::nvvm_add_rm_ftz_f:
605 return APFloat::rmTowardNegative;
606 case Intrinsic::nvvm_add_rn_f:
607 case Intrinsic::nvvm_add_rn_d:
608 case Intrinsic::nvvm_add_rn_ftz_f:
609 return APFloat::rmNearestTiesToEven;
610 case Intrinsic::nvvm_add_rp_f:
611 case Intrinsic::nvvm_add_rp_d:
612 case Intrinsic::nvvm_add_rp_ftz_f:
613 return APFloat::rmTowardPositive;
614 case Intrinsic::nvvm_add_rz_f:
615 case Intrinsic::nvvm_add_rz_d:
616 case Intrinsic::nvvm_add_rz_ftz_f:
617 return APFloat::rmTowardZero;
618 }
619 llvm_unreachable("Invalid FP instrinsic rounding mode for NVVM add");
620}
621
622inline bool FMulShouldFTZ(Intrinsic::ID IntrinsicID) {
623 switch (IntrinsicID) {
624 case Intrinsic::nvvm_mul_rm_ftz_f:
625 case Intrinsic::nvvm_mul_rn_ftz_f:
626 case Intrinsic::nvvm_mul_rp_ftz_f:
627 case Intrinsic::nvvm_mul_rz_ftz_f:
628 return true;
629
630 case Intrinsic::nvvm_mul_rm_f:
631 case Intrinsic::nvvm_mul_rn_f:
632 case Intrinsic::nvvm_mul_rp_f:
633 case Intrinsic::nvvm_mul_rz_f:
634 case Intrinsic::nvvm_mul_rm_d:
635 case Intrinsic::nvvm_mul_rn_d:
636 case Intrinsic::nvvm_mul_rp_d:
637 case Intrinsic::nvvm_mul_rz_d:
638 return false;
639 }
640 llvm_unreachable("Checking FTZ flag for invalid NVVM mul intrinsic");
641}
642
643inline APFloat::roundingMode GetFMulRoundingMode(Intrinsic::ID IntrinsicID) {
644 switch (IntrinsicID) {
645 case Intrinsic::nvvm_mul_rm_f:
646 case Intrinsic::nvvm_mul_rm_d:
647 case Intrinsic::nvvm_mul_rm_ftz_f:
648 return APFloat::rmTowardNegative;
649 case Intrinsic::nvvm_mul_rn_f:
650 case Intrinsic::nvvm_mul_rn_d:
651 case Intrinsic::nvvm_mul_rn_ftz_f:
652 return APFloat::rmNearestTiesToEven;
653 case Intrinsic::nvvm_mul_rp_f:
654 case Intrinsic::nvvm_mul_rp_d:
655 case Intrinsic::nvvm_mul_rp_ftz_f:
656 return APFloat::rmTowardPositive;
657 case Intrinsic::nvvm_mul_rz_f:
658 case Intrinsic::nvvm_mul_rz_d:
659 case Intrinsic::nvvm_mul_rz_ftz_f:
660 return APFloat::rmTowardZero;
661 }
662 llvm_unreachable("Invalid FP instrinsic rounding mode for NVVM mul");
663}
664
665inline bool FDivShouldFTZ(Intrinsic::ID IntrinsicID) {
666 switch (IntrinsicID) {
667 case Intrinsic::nvvm_div_rm_ftz_f:
668 case Intrinsic::nvvm_div_rn_ftz_f:
669 case Intrinsic::nvvm_div_rp_ftz_f:
670 case Intrinsic::nvvm_div_rz_ftz_f:
671 return true;
672
673 case Intrinsic::nvvm_div_rm_f:
674 case Intrinsic::nvvm_div_rn_f:
675 case Intrinsic::nvvm_div_rp_f:
676 case Intrinsic::nvvm_div_rz_f:
677 case Intrinsic::nvvm_div_rm_d:
678 case Intrinsic::nvvm_div_rn_d:
679 case Intrinsic::nvvm_div_rp_d:
680 case Intrinsic::nvvm_div_rz_d:
681 return false;
682 }
683 llvm_unreachable("Checking FTZ flag for invalid NVVM div intrinsic");
684}
685
686inline APFloat::roundingMode GetFDivRoundingMode(Intrinsic::ID IntrinsicID) {
687 switch (IntrinsicID) {
688 case Intrinsic::nvvm_div_rm_f:
689 case Intrinsic::nvvm_div_rm_d:
690 case Intrinsic::nvvm_div_rm_ftz_f:
691 return APFloat::rmTowardNegative;
692 case Intrinsic::nvvm_div_rn_f:
693 case Intrinsic::nvvm_div_rn_d:
694 case Intrinsic::nvvm_div_rn_ftz_f:
695 return APFloat::rmNearestTiesToEven;
696 case Intrinsic::nvvm_div_rp_f:
697 case Intrinsic::nvvm_div_rp_d:
698 case Intrinsic::nvvm_div_rp_ftz_f:
699 return APFloat::rmTowardPositive;
700 case Intrinsic::nvvm_div_rz_f:
701 case Intrinsic::nvvm_div_rz_d:
702 case Intrinsic::nvvm_div_rz_ftz_f:
703 return APFloat::rmTowardZero;
704 }
705 llvm_unreachable("Invalid FP instrinsic rounding mode for NVVM div");
706}
707
708inline bool FMAShouldFTZ(Intrinsic::ID IntrinsicID) {
709 switch (IntrinsicID) {
710 case Intrinsic::nvvm_fma_rm_ftz_f:
711 case Intrinsic::nvvm_fma_rn_ftz_f:
712 case Intrinsic::nvvm_fma_rp_ftz_f:
713 case Intrinsic::nvvm_fma_rz_ftz_f:
714 return true;
715
716 case Intrinsic::nvvm_fma_rm_f:
717 case Intrinsic::nvvm_fma_rn_f:
718 case Intrinsic::nvvm_fma_rp_f:
719 case Intrinsic::nvvm_fma_rz_f:
720 case Intrinsic::nvvm_fma_rm_d:
721 case Intrinsic::nvvm_fma_rn_d:
722 case Intrinsic::nvvm_fma_rp_d:
723 case Intrinsic::nvvm_fma_rz_d:
724 return false;
725 }
726 llvm_unreachable("Checking FTZ flag for invalid NVVM fma intrinsic");
727}
728
729inline APFloat::roundingMode GetFMARoundingMode(Intrinsic::ID IntrinsicID) {
730 switch (IntrinsicID) {
731 case Intrinsic::nvvm_fma_rm_f:
732 case Intrinsic::nvvm_fma_rm_d:
733 case Intrinsic::nvvm_fma_rm_ftz_f:
734 return APFloat::rmTowardNegative;
735 case Intrinsic::nvvm_fma_rn_f:
736 case Intrinsic::nvvm_fma_rn_d:
737 case Intrinsic::nvvm_fma_rn_ftz_f:
738 return APFloat::rmNearestTiesToEven;
739 case Intrinsic::nvvm_fma_rp_f:
740 case Intrinsic::nvvm_fma_rp_d:
741 case Intrinsic::nvvm_fma_rp_ftz_f:
742 return APFloat::rmTowardPositive;
743 case Intrinsic::nvvm_fma_rz_f:
744 case Intrinsic::nvvm_fma_rz_d:
745 case Intrinsic::nvvm_fma_rz_ftz_f:
746 return APFloat::rmTowardZero;
747 }
748 llvm_unreachable("Invalid FP instrinsic rounding mode for NVVM fma");
749}
750
751} // namespace nvvm
752} // namespace llvm
753#endif // LLVM_IR_NVVMINTRINSICUTILS_H
APFloat.h
This file declares a class to represent arbitrary precision floating point values and provide a varie...
APInt.h
This file implements a class to represent arbitrary precision integral constant values and operations...
LLVM_ABI
#define LLVM_ABI
Definition Compiler.h:215
Constants.h
This file contains the declarations for the subclasses of Constant, which represent the different fla...
llvm::APFloatBase::rmTowardZero
static constexpr roundingMode rmTowardZero
Definition APFloat.h:357
llvm::APFloatBase::roundingMode
llvm::RoundingMode roundingMode
IEEE-754R 4.3: Rounding-direction attributes.
Definition APFloat.h:351
llvm::APFloatBase::rmTowardNegative
static constexpr roundingMode rmTowardNegative
Definition APFloat.h:356
llvm::APFloatBase::rmNearestTiesToEven
static constexpr roundingMode rmNearestTiesToEven
Definition APFloat.h:353
llvm::APFloatBase::rmTowardPositive
static constexpr roundingMode rmTowardPositive
Definition APFloat.h:355
llvm::Constant
This is an important base class in LLVM.
Definition Constant.h:43
llvm::StringRef
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
llvm::raw_ostream
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition raw_ostream.h:53
uint8_t
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition ErrorHandling.h:164
llvm::nvvm::GetFMARoundingMode
APFloat::roundingMode GetFMARoundingMode(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:729
llvm::nvvm::printTensormapSwizzleMode
LLVM_ABI void printTensormapSwizzleMode(raw_ostream &OS, const Constant *ImmArgVal)
Definition NVVMIntrinsicUtils.cpp:102
llvm::nvvm::GetNVVMDenormMode
DenormalMode GetNVVMDenormMode(bool ShouldFTZ)
Definition NVVMIntrinsicUtils.h:573
llvm::nvvm::printTMAReductionOp
LLVM_ABI void printTMAReductionOp(raw_ostream &OS, const Constant *ImmArgVal)
Definition NVVMIntrinsicUtils.cpp:19
llvm::nvvm::FPToIntegerIntrinsicNaNZero
bool FPToIntegerIntrinsicNaNZero(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:281
llvm::nvvm::printTensormapInterleaveLayout
LLVM_ABI void printTensormapInterleaveLayout(raw_ostream &OS, const Constant *ImmArgVal)
Definition NVVMIntrinsicUtils.cpp:84
llvm::nvvm::GetFDivRoundingMode
APFloat::roundingMode GetFDivRoundingMode(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:686
llvm::nvvm::printTensormapSwizzleAtomicity
LLVM_ABI void printTensormapSwizzleAtomicity(raw_ostream &OS, const Constant *ImmArgVal)
Definition NVVMIntrinsicUtils.cpp:116
llvm::nvvm::FPToIntegerIntrinsicResultIsSigned
bool FPToIntegerIntrinsicResultIsSigned(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:215
llvm::nvvm::GetFPToIntegerRoundingMode
APFloat::roundingMode GetFPToIntegerRoundingMode(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:346
llvm::nvvm::RCPShouldFTZ
bool RCPShouldFTZ(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:528
llvm::nvvm::getTMATensorReductionOpName
StringRef getTMATensorReductionOpName(TMAReductionOp Op)
Definition NVVMIntrinsicUtils.h:45
llvm::nvvm::FPToIntegerIntrinsicShouldFTZ
bool FPToIntegerIntrinsicShouldFTZ(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:148
llvm::nvvm::FDivShouldFTZ
bool FDivShouldFTZ(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:665
llvm::nvvm::FAddShouldFTZ
bool FAddShouldFTZ(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:579
llvm::nvvm::FMinFMaxIsXorSignAbs
bool FMinFMaxIsXorSignAbs(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:473
llvm::nvvm::GetFMulRoundingMode
APFloat::roundingMode GetFMulRoundingMode(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:643
llvm::nvvm::printTcgen05MMAKind
LLVM_ABI void printTcgen05MMAKind(raw_ostream &OS, const Constant *ImmArgVal)
Definition NVVMIntrinsicUtils.cpp:29
llvm::nvvm::printTcgen05CollectorUsageOp
LLVM_ABI void printTcgen05CollectorUsageOp(raw_ostream &OS, const Constant *ImmArgVal)
Definition NVVMIntrinsicUtils.cpp:49
llvm::nvvm::UnaryMathIntrinsicShouldFTZ
bool UnaryMathIntrinsicShouldFTZ(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:502
llvm::nvvm::FMinFMaxShouldFTZ
bool FMinFMaxShouldFTZ(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:415
llvm::nvvm::GetFAddRoundingMode
APFloat::roundingMode GetFAddRoundingMode(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:600
llvm::nvvm::FMAShouldFTZ
bool FMAShouldFTZ(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:708
llvm::nvvm::FMulShouldFTZ
bool FMulShouldFTZ(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:622
llvm::nvvm::printTensormapFillMode
LLVM_ABI void printTensormapFillMode(raw_ostream &OS, const Constant *ImmArgVal)
Definition NVVMIntrinsicUtils.cpp:130
llvm::nvvm::GetRCPRoundingMode
APFloat::roundingMode GetRCPRoundingMode(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:548
llvm::nvvm::FMinFMaxPropagatesNaNs
bool FMinFMaxPropagatesNaNs(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:444
llvm::nvvm::printTensormapElemType
LLVM_ABI void printTensormapElemType(raw_ostream &OS, const Constant *ImmArgVal)
Definition NVVMIntrinsicUtils.cpp:70
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition FunctionInfo.h:25
llvm::Op
DWARFExpression::Operation Op
Definition DWARFExpressionPrinter.cpp:25
raw_ostream.h
llvm::DenormalMode
Represent subnormal handling kind for floating point instruction inputs and outputs.
Definition FloatingPointMode.h:71
llvm::DenormalMode::getPreserveSign
static constexpr DenormalMode getPreserveSign()
Definition FloatingPointMode.h:119
llvm::DenormalMode::getIEEE
static constexpr DenormalMode getIEEE()
Definition FloatingPointMode.h:115
Generated on for LLVM by doxygen 1.14.0