LLVM 24.0.0git
AMDGPUISelLowering.cpp
Go to the documentation of this file.
1//===-- AMDGPUISelLowering.cpp - AMDGPU Common DAG lowering functions -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This is the parent TargetLowering class for hardware code gen
11/// targets.
12//
13//===----------------------------------------------------------------------===//
14
15#include "AMDGPUISelLowering.h"
16#include "AMDGPU.h"
17#include "AMDGPUInstrInfo.h"
19#include "AMDGPUMemoryUtils.h"
26#include "llvm/IR/IntrinsicsAMDGPU.h"
30
31using namespace llvm;
32
33#define GET_CALLING_CONV_IMPL
34#include "AMDGPUGenCallingConv.inc"
35
36// Find a larger type to do a load / store of a vector with.
38 unsigned StoreSize = VT.getStoreSizeInBits();
39 if (StoreSize <= 32)
40 return EVT::getIntegerVT(Ctx, StoreSize);
41
42 if (StoreSize % 32 == 0)
43 return EVT::getVectorVT(Ctx, MVT::i32, StoreSize / 32);
44
45 return VT;
46}
47
51
53 // In order for this to be a signed 24-bit value, bit 23, must
54 // be a sign bit.
55 return DAG.ComputeMaxSignificantBits(Op);
56}
57
59 const KnownBits &LHSKnown, const KnownBits &RHSKnown, unsigned BitWidth,
60 bool IsSigned, bool SelfMultiply) {
61 // Sign/Zero extend from 24 bits.
62 KnownBits LHS24 = LHSKnown.trunc(24);
63 KnownBits RHS24 = RHSKnown.trunc(24);
64 if (IsSigned)
65 return KnownBits::mul(LHS24.sext(BitWidth), RHS24.sext(BitWidth),
66 SelfMultiply);
67 return KnownBits::mul(LHS24.zext(BitWidth), RHS24.zext(BitWidth),
68 SelfMultiply);
69}
70
72 const TargetSubtargetInfo &STI,
73 const AMDGPUSubtarget &AMDGPUSTI)
74 : TargetLowering(TM, STI), Subtarget(&AMDGPUSTI) {
75 // Always lower memset, memcpy, and memmove intrinsics to load/store
76 // instructions, rather then generating calls to memset, mempcy or memmove.
80
81 // Enable ganging up loads and stores in the memcpy DAG lowering.
83
84 // Lower floating point store/load to integer store/load to reduce the number
85 // of patterns in tablegen.
87 AddPromotedToType(ISD::LOAD, MVT::f32, MVT::i32);
88
90 AddPromotedToType(ISD::LOAD, MVT::v2f32, MVT::v2i32);
91
93 AddPromotedToType(ISD::LOAD, MVT::v3f32, MVT::v3i32);
94
96 AddPromotedToType(ISD::LOAD, MVT::v4f32, MVT::v4i32);
97
99 AddPromotedToType(ISD::LOAD, MVT::v5f32, MVT::v5i32);
100
102 AddPromotedToType(ISD::LOAD, MVT::v6f32, MVT::v6i32);
103
105 AddPromotedToType(ISD::LOAD, MVT::v7f32, MVT::v7i32);
106
108 AddPromotedToType(ISD::LOAD, MVT::v8f32, MVT::v8i32);
109
111 AddPromotedToType(ISD::LOAD, MVT::v9f32, MVT::v9i32);
112
113 setOperationAction(ISD::LOAD, MVT::v10f32, Promote);
114 AddPromotedToType(ISD::LOAD, MVT::v10f32, MVT::v10i32);
115
116 setOperationAction(ISD::LOAD, MVT::v11f32, Promote);
117 AddPromotedToType(ISD::LOAD, MVT::v11f32, MVT::v11i32);
118
119 setOperationAction(ISD::LOAD, MVT::v12f32, Promote);
120 AddPromotedToType(ISD::LOAD, MVT::v12f32, MVT::v12i32);
121
122 setOperationAction(ISD::LOAD, MVT::v16f32, Promote);
123 AddPromotedToType(ISD::LOAD, MVT::v16f32, MVT::v16i32);
124
125 setOperationAction(ISD::LOAD, MVT::v32f32, Promote);
126 AddPromotedToType(ISD::LOAD, MVT::v32f32, MVT::v32i32);
127
129 AddPromotedToType(ISD::LOAD, MVT::i64, MVT::v2i32);
130
132 AddPromotedToType(ISD::LOAD, MVT::v2i64, MVT::v4i32);
133
135 AddPromotedToType(ISD::LOAD, MVT::f64, MVT::v2i32);
136
138 AddPromotedToType(ISD::LOAD, MVT::v2f64, MVT::v4i32);
139
141 AddPromotedToType(ISD::LOAD, MVT::v3i64, MVT::v6i32);
142
144 AddPromotedToType(ISD::LOAD, MVT::v4i64, MVT::v8i32);
145
147 AddPromotedToType(ISD::LOAD, MVT::v3f64, MVT::v6i32);
148
150 AddPromotedToType(ISD::LOAD, MVT::v4f64, MVT::v8i32);
151
153 AddPromotedToType(ISD::LOAD, MVT::v8i64, MVT::v16i32);
154
156 AddPromotedToType(ISD::LOAD, MVT::v8f64, MVT::v16i32);
157
158 setOperationAction(ISD::LOAD, MVT::v16i64, Promote);
159 AddPromotedToType(ISD::LOAD, MVT::v16i64, MVT::v32i32);
160
161 setOperationAction(ISD::LOAD, MVT::v16f64, Promote);
162 AddPromotedToType(ISD::LOAD, MVT::v16f64, MVT::v32i32);
163
165 AddPromotedToType(ISD::LOAD, MVT::i128, MVT::v4i32);
166
167 // There are no 64-bit extloads. These should be done as a 32-bit extload and
168 // an extension to 64-bit.
169 for (MVT VT : MVT::integer_valuetypes())
171 Expand);
172
173 for (MVT VT : MVT::integer_valuetypes()) {
174 if (VT == MVT::i64)
175 continue;
176
177 for (auto Op : {ISD::SEXTLOAD, ISD::ZEXTLOAD, ISD::EXTLOAD}) {
178 setLoadExtAction(Op, VT, MVT::i1, Promote);
179 setLoadExtAction(Op, VT, MVT::i8, Legal);
180 setLoadExtAction(Op, VT, MVT::i16, Legal);
181 setLoadExtAction(Op, VT, MVT::i32, Expand);
182 }
183 }
184
186 for (auto MemVT :
187 {MVT::v2i8, MVT::v4i8, MVT::v2i16, MVT::v3i16, MVT::v4i16})
189 Expand);
190
191 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
192 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::bf16, Expand);
193 setLoadExtAction(ISD::EXTLOAD, MVT::v2f32, MVT::v2f16, Expand);
194 setLoadExtAction(ISD::EXTLOAD, MVT::v2f32, MVT::v2bf16, Expand);
195 setLoadExtAction(ISD::EXTLOAD, MVT::v3f32, MVT::v3f16, Expand);
196 setLoadExtAction(ISD::EXTLOAD, MVT::v3f32, MVT::v3bf16, Expand);
197 setLoadExtAction(ISD::EXTLOAD, MVT::v4f32, MVT::v4f16, Expand);
198 setLoadExtAction(ISD::EXTLOAD, MVT::v4f32, MVT::v4bf16, Expand);
199 setLoadExtAction(ISD::EXTLOAD, MVT::v5f32, MVT::v5f16, Expand);
200 setLoadExtAction(ISD::EXTLOAD, MVT::v6f32, MVT::v6f16, Expand);
201 setLoadExtAction(ISD::EXTLOAD, MVT::v7f32, MVT::v7f16, Expand);
202 setLoadExtAction(ISD::EXTLOAD, MVT::v8f32, MVT::v8f16, Expand);
203 setLoadExtAction(ISD::EXTLOAD, MVT::v8f32, MVT::v8bf16, Expand);
204 setLoadExtAction(ISD::EXTLOAD, MVT::v16f32, MVT::v16f16, Expand);
205 setLoadExtAction(ISD::EXTLOAD, MVT::v16f32, MVT::v16bf16, Expand);
206 setLoadExtAction(ISD::EXTLOAD, MVT::v32f32, MVT::v32f16, Expand);
207 setLoadExtAction(ISD::EXTLOAD, MVT::v32f32, MVT::v32bf16, Expand);
208
209 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
210 setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f32, Expand);
211 setLoadExtAction(ISD::EXTLOAD, MVT::v3f64, MVT::v3f32, Expand);
212 setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f32, Expand);
213 setLoadExtAction(ISD::EXTLOAD, MVT::v8f64, MVT::v8f32, Expand);
214 setLoadExtAction(ISD::EXTLOAD, MVT::v16f64, MVT::v16f32, Expand);
215
216 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);
217 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::bf16, Expand);
218 setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f16, Expand);
219 setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2bf16, Expand);
220 setLoadExtAction(ISD::EXTLOAD, MVT::v3f64, MVT::v3f16, Expand);
221 setLoadExtAction(ISD::EXTLOAD, MVT::v3f64, MVT::v3bf16, Expand);
222 setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f16, Expand);
223 setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4bf16, Expand);
224 setLoadExtAction(ISD::EXTLOAD, MVT::v8f64, MVT::v8f16, Expand);
225 setLoadExtAction(ISD::EXTLOAD, MVT::v8f64, MVT::v8bf16, Expand);
226 setLoadExtAction(ISD::EXTLOAD, MVT::v16f64, MVT::v16f16, Expand);
227 setLoadExtAction(ISD::EXTLOAD, MVT::v16f64, MVT::v16bf16, Expand);
228
230 AddPromotedToType(ISD::STORE, MVT::f32, MVT::i32);
231
233 AddPromotedToType(ISD::STORE, MVT::v2f32, MVT::v2i32);
234
236 AddPromotedToType(ISD::STORE, MVT::v3f32, MVT::v3i32);
237
239 AddPromotedToType(ISD::STORE, MVT::v4f32, MVT::v4i32);
240
242 AddPromotedToType(ISD::STORE, MVT::v5f32, MVT::v5i32);
243
245 AddPromotedToType(ISD::STORE, MVT::v6f32, MVT::v6i32);
246
248 AddPromotedToType(ISD::STORE, MVT::v7f32, MVT::v7i32);
249
251 AddPromotedToType(ISD::STORE, MVT::v8f32, MVT::v8i32);
252
254 AddPromotedToType(ISD::STORE, MVT::v9f32, MVT::v9i32);
255
257 AddPromotedToType(ISD::STORE, MVT::v10f32, MVT::v10i32);
258
260 AddPromotedToType(ISD::STORE, MVT::v11f32, MVT::v11i32);
261
263 AddPromotedToType(ISD::STORE, MVT::v12f32, MVT::v12i32);
264
266 AddPromotedToType(ISD::STORE, MVT::v16f32, MVT::v16i32);
267
269 AddPromotedToType(ISD::STORE, MVT::v32f32, MVT::v32i32);
270
272 AddPromotedToType(ISD::STORE, MVT::i64, MVT::v2i32);
273
275 AddPromotedToType(ISD::STORE, MVT::v2i64, MVT::v4i32);
276
278 AddPromotedToType(ISD::STORE, MVT::f64, MVT::v2i32);
279
281 AddPromotedToType(ISD::STORE, MVT::v2f64, MVT::v4i32);
282
284 AddPromotedToType(ISD::STORE, MVT::v3i64, MVT::v6i32);
285
287 AddPromotedToType(ISD::STORE, MVT::v3f64, MVT::v6i32);
288
290 AddPromotedToType(ISD::STORE, MVT::v4i64, MVT::v8i32);
291
293 AddPromotedToType(ISD::STORE, MVT::v4f64, MVT::v8i32);
294
296 AddPromotedToType(ISD::STORE, MVT::v8i64, MVT::v16i32);
297
299 AddPromotedToType(ISD::STORE, MVT::v8f64, MVT::v16i32);
300
302 AddPromotedToType(ISD::STORE, MVT::v16i64, MVT::v32i32);
303
305 AddPromotedToType(ISD::STORE, MVT::v16f64, MVT::v32i32);
306
308 AddPromotedToType(ISD::STORE, MVT::i128, MVT::v4i32);
309
310 setTruncStoreAction(MVT::i64, MVT::i1, Expand);
311 setTruncStoreAction(MVT::i64, MVT::i8, Expand);
312 setTruncStoreAction(MVT::i64, MVT::i16, Expand);
313 setTruncStoreAction(MVT::i64, MVT::i32, Expand);
314
315 setTruncStoreAction(MVT::v2i64, MVT::v2i1, Expand);
316 setTruncStoreAction(MVT::v2i64, MVT::v2i8, Expand);
317 setTruncStoreAction(MVT::v2i64, MVT::v2i16, Expand);
318 setTruncStoreAction(MVT::v2i64, MVT::v2i32, Expand);
319
320 setTruncStoreAction(MVT::f32, MVT::bf16, Expand);
321 setTruncStoreAction(MVT::f32, MVT::f16, Expand);
322 setTruncStoreAction(MVT::v2f32, MVT::v2bf16, Expand);
323 setTruncStoreAction(MVT::v2f32, MVT::v2f16, Expand);
324 setTruncStoreAction(MVT::v3f32, MVT::v3bf16, Expand);
325 setTruncStoreAction(MVT::v3f32, MVT::v3f16, Expand);
326 setTruncStoreAction(MVT::v4f32, MVT::v4bf16, Expand);
327 setTruncStoreAction(MVT::v4f32, MVT::v4f16, Expand);
328 setTruncStoreAction(MVT::v5f32, MVT::v5f16, Expand);
329 setTruncStoreAction(MVT::v6f32, MVT::v6f16, Expand);
330 setTruncStoreAction(MVT::v7f32, MVT::v7f16, Expand);
331 setTruncStoreAction(MVT::v8f32, MVT::v8bf16, Expand);
332 setTruncStoreAction(MVT::v8f32, MVT::v8f16, Expand);
333 setTruncStoreAction(MVT::v16f32, MVT::v16bf16, Expand);
334 setTruncStoreAction(MVT::v16f32, MVT::v16f16, Expand);
335 setTruncStoreAction(MVT::v32f32, MVT::v32bf16, Expand);
336 setTruncStoreAction(MVT::v32f32, MVT::v32f16, Expand);
337
338 setTruncStoreAction(MVT::f64, MVT::bf16, Expand);
339 setTruncStoreAction(MVT::f64, MVT::f16, Expand);
340 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
341
342 setTruncStoreAction(MVT::v2f64, MVT::v2f32, Expand);
343 setTruncStoreAction(MVT::v2f64, MVT::v2bf16, Expand);
344 setTruncStoreAction(MVT::v2f64, MVT::v2f16, Expand);
345
346 setTruncStoreAction(MVT::v3i32, MVT::v3i8, Expand);
347
348 setTruncStoreAction(MVT::v3i64, MVT::v3i32, Expand);
349 setTruncStoreAction(MVT::v3i64, MVT::v3i16, Expand);
350 setTruncStoreAction(MVT::v3i64, MVT::v3i8, Expand);
351 setTruncStoreAction(MVT::v3i64, MVT::v3i1, Expand);
352 setTruncStoreAction(MVT::v3f64, MVT::v3f32, Expand);
353 setTruncStoreAction(MVT::v3f64, MVT::v3bf16, Expand);
354 setTruncStoreAction(MVT::v3f64, MVT::v3f16, Expand);
355
356 setTruncStoreAction(MVT::v4i64, MVT::v4i32, Expand);
357 setTruncStoreAction(MVT::v4i64, MVT::v4i16, Expand);
358 setTruncStoreAction(MVT::v4f64, MVT::v4f32, Expand);
359 setTruncStoreAction(MVT::v4f64, MVT::v4bf16, Expand);
360 setTruncStoreAction(MVT::v4f64, MVT::v4f16, Expand);
361
362 setTruncStoreAction(MVT::v5i32, MVT::v5i1, Expand);
363 setTruncStoreAction(MVT::v5i32, MVT::v5i8, Expand);
364 setTruncStoreAction(MVT::v5i32, MVT::v5i16, Expand);
365
366 setTruncStoreAction(MVT::v6i32, MVT::v6i1, Expand);
367 setTruncStoreAction(MVT::v6i32, MVT::v6i8, Expand);
368 setTruncStoreAction(MVT::v6i32, MVT::v6i16, Expand);
369
370 setTruncStoreAction(MVT::v7i32, MVT::v7i1, Expand);
371 setTruncStoreAction(MVT::v7i32, MVT::v7i8, Expand);
372 setTruncStoreAction(MVT::v7i32, MVT::v7i16, Expand);
373
374 setTruncStoreAction(MVT::v8f64, MVT::v8f32, Expand);
375 setTruncStoreAction(MVT::v8f64, MVT::v8bf16, Expand);
376 setTruncStoreAction(MVT::v8f64, MVT::v8f16, Expand);
377
378 setTruncStoreAction(MVT::v16f64, MVT::v16f32, Expand);
379 setTruncStoreAction(MVT::v16f64, MVT::v16bf16, Expand);
380 setTruncStoreAction(MVT::v16f64, MVT::v16f16, Expand);
381 setTruncStoreAction(MVT::v16i64, MVT::v16i16, Expand);
382 setTruncStoreAction(MVT::v16i64, MVT::v16i8, Expand);
383 setTruncStoreAction(MVT::v16i64, MVT::v16i8, Expand);
384 setTruncStoreAction(MVT::v16i64, MVT::v16i1, Expand);
385
386 setOperationAction(ISD::Constant, {MVT::i32, MVT::i64}, Legal);
387 setOperationAction(ISD::ConstantFP, {MVT::f32, MVT::f64}, Legal);
388
390
391 // For R600, this is totally unsupported, just custom lower to produce an
392 // error.
394
395 // Library functions. These default to Expand, but we have instructions
396 // for them.
399 {MVT::f16, MVT::f32}, Legal);
401
403 setOperationAction(ISD::FROUND, {MVT::f32, MVT::f64}, Custom);
405 {MVT::f16, MVT::f32, MVT::f64}, Expand);
406
409 MVT::f32, Custom);
411
412 setOperationAction(ISD::FNEARBYINT, {MVT::f16, MVT::f32, MVT::f64}, Custom);
413
414 setOperationAction(ISD::FRINT, {MVT::f16, MVT::f32, MVT::f64}, Custom);
415
416 setOperationAction({ISD::LRINT, ISD::LLRINT}, {MVT::f16, MVT::f32, MVT::f64},
417 Expand);
418
419 setOperationAction(ISD::FREM, {MVT::f16, MVT::f32, MVT::f64}, Expand);
420 setOperationAction(ISD::IS_FPCLASS, {MVT::f32, MVT::f64}, Legal);
422
424 Custom);
425
426 setOperationAction(ISD::FCANONICALIZE, {MVT::f32, MVT::f64}, Legal);
427
428 // FIXME: These IS_FPCLASS vector fp types are marked custom so it reaches
429 // scalarization code. Can be removed when IS_FPCLASS expand isn't called by
430 // default unless marked custom/legal.
432 {MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32,
433 MVT::v6f32, MVT::v7f32, MVT::v8f32, MVT::v16f32,
434 MVT::v2f64, MVT::v3f64, MVT::v4f64, MVT::v8f64,
435 MVT::v16f64},
436 Custom);
437
438 // Expand to fneg + fadd.
440
442 {MVT::v3i32, MVT::v3f32, MVT::v4i32, MVT::v4f32,
443 MVT::v5i32, MVT::v5f32, MVT::v6i32, MVT::v6f32,
444 MVT::v7i32, MVT::v7f32, MVT::v8i32, MVT::v8f32,
445 MVT::v9i32, MVT::v9f32, MVT::v10i32, MVT::v10f32,
446 MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32},
447 Custom);
448
451 {MVT::v2f32, MVT::v2i32, MVT::v3f32, MVT::v3i32, MVT::v4f32,
452 MVT::v4i32, MVT::v5f32, MVT::v5i32, MVT::v6f32, MVT::v6i32,
453 MVT::v7f32, MVT::v7i32, MVT::v8f32, MVT::v8i32, MVT::v9f32,
454 MVT::v9i32, MVT::v10i32, MVT::v10f32, MVT::v11i32, MVT::v11f32,
455 MVT::v12i32, MVT::v12f32, MVT::v16i32, MVT::v32f32, MVT::v32i32,
456 MVT::v2f64, MVT::v2i64, MVT::v3f64, MVT::v3i64, MVT::v4f64,
457 MVT::v4i64, MVT::v8f64, MVT::v8i64, MVT::v16f64, MVT::v16i64},
458 Custom);
459
461 Expand);
462 setOperationAction(ISD::FP_TO_FP16, {MVT::f64, MVT::f32}, Custom);
463
464 const MVT ScalarIntVTs[] = { MVT::i32, MVT::i64 };
465 for (MVT VT : ScalarIntVTs) {
466 // These should use [SU]DIVREM, so set them to expand
468 Expand);
469
470 // GPU does not have divrem function for signed or unsigned.
472
473 // GPU does not have [S|U]MUL_LOHI functions as a single instruction.
475
477 }
478
479 // The hardware supports 32-bit FSHR, but not FSHL.
481
482 setOperationAction({ISD::ROTL, ISD::ROTR}, {MVT::i32, MVT::i64}, Expand);
483
485
490 MVT::i64, Custom);
492
494 Legal);
495
498 MVT::i64, Custom);
499
500 for (auto VT : {MVT::i8, MVT::i16})
502
503 static const MVT::SimpleValueType VectorIntTypes[] = {
504 MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32, MVT::v6i32, MVT::v7i32,
505 MVT::v9i32, MVT::v10i32, MVT::v11i32, MVT::v12i32};
506
507 for (MVT VT : VectorIntTypes) {
508 // Expand the following operations for the current type by default.
509 // clang-format off
529 VT, Expand);
530 // clang-format on
531 }
532
533 static const MVT::SimpleValueType FloatVectorTypes[] = {
534 MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32, MVT::v6f32, MVT::v7f32,
535 MVT::v9f32, MVT::v10f32, MVT::v11f32, MVT::v12f32};
536
537 for (MVT VT : FloatVectorTypes) {
550 VT, Expand);
551 }
552
553 // This causes using an unrolled select operation rather than expansion with
554 // bit operations. This is in general better, but the alternative using BFI
555 // instructions may be better if the select sources are SGPRs.
557 AddPromotedToType(ISD::SELECT, MVT::v2f32, MVT::v2i32);
558
560 AddPromotedToType(ISD::SELECT, MVT::v3f32, MVT::v3i32);
561
563 AddPromotedToType(ISD::SELECT, MVT::v4f32, MVT::v4i32);
564
566 AddPromotedToType(ISD::SELECT, MVT::v5f32, MVT::v5i32);
567
569 AddPromotedToType(ISD::SELECT, MVT::v6f32, MVT::v6i32);
570
572 AddPromotedToType(ISD::SELECT, MVT::v7f32, MVT::v7i32);
573
575 AddPromotedToType(ISD::SELECT, MVT::v9f32, MVT::v9i32);
576
578 AddPromotedToType(ISD::SELECT, MVT::v10f32, MVT::v10i32);
579
581 AddPromotedToType(ISD::SELECT, MVT::v11f32, MVT::v11i32);
582
584 AddPromotedToType(ISD::SELECT, MVT::v12f32, MVT::v12i32);
585
587 setJumpIsExpensive(true);
588
591
593
594 // We want to find all load dependencies for long chains of stores to enable
595 // merging into very wide vectors. The problem is with vectors with > 4
596 // elements. MergeConsecutiveStores will attempt to merge these because x8/x16
597 // vectors are a legal type, even though we have to split the loads
598 // usually. When we can more precisely specify load legality per address
599 // space, we should be able to make FindBetterChain/MergeConsecutiveStores
600 // smarter so that they can figure out what to do in 2 iterations without all
601 // N > 4 stores on the same chain.
603
604 // memcpy/memmove/memset are expanded in the IR, so we shouldn't need to worry
605 // about these during lowering.
606 MaxStoresPerMemcpy = 0xffffffff;
607 MaxStoresPerMemmove = 0xffffffff;
608 MaxStoresPerMemset = 0xffffffff;
609
610 // The expansion for 64-bit division is enormous.
611 if (static_cast<const AMDGPUTargetMachine &>(TM).getCLOpts().bypass_slow_div)
612 addBypassSlowDiv(64, 32);
613
624
628}
629
630//===----------------------------------------------------------------------===//
631// Target Information
632//===----------------------------------------------------------------------===//
633
635static bool fnegFoldsIntoOpcode(unsigned Opc) {
636 switch (Opc) {
637 case ISD::FADD:
638 case ISD::FSUB:
639 case ISD::FMUL:
640 case ISD::FMA:
641 case ISD::FMAD:
642 case ISD::FMINNUM:
643 case ISD::FMAXNUM:
646 case ISD::FMINIMUM:
647 case ISD::FMAXIMUM:
648 case ISD::FMINIMUMNUM:
649 case ISD::FMAXIMUMNUM:
650 case ISD::SELECT:
651 case ISD::FSIN:
652 case ISD::FTRUNC:
653 case ISD::FRINT:
654 case ISD::FNEARBYINT:
655 case ISD::FROUNDEVEN:
657 case AMDGPUISD::RCP:
658 case AMDGPUISD::RCP_LEGACY:
659 case AMDGPUISD::RCP_IFLAG:
660 case AMDGPUISD::SIN_HW:
661 case AMDGPUISD::FMUL_LEGACY:
662 case AMDGPUISD::FMIN_LEGACY:
663 case AMDGPUISD::FMAX_LEGACY:
664 case AMDGPUISD::FMED3:
665 // TODO: handle llvm.amdgcn.fma.legacy
666 return true;
667 case ISD::BITCAST:
668 llvm_unreachable("bitcast is special cased");
669 default:
670 return false;
671 }
672}
673
674static bool fnegFoldsIntoOp(const SDNode *N) {
675 unsigned Opc = N->getOpcode();
676 if (Opc == ISD::BITCAST) {
677 // TODO: Is there a benefit to checking the conditions performFNegCombine
678 // does? We don't for the other cases.
679 SDValue BCSrc = N->getOperand(0);
680 if (BCSrc.getOpcode() == ISD::BUILD_VECTOR) {
681 return BCSrc.getNumOperands() == 2 &&
682 BCSrc.getOperand(1).getValueSizeInBits() == 32;
683 }
684
685 return BCSrc.getOpcode() == ISD::SELECT && BCSrc.getValueType() == MVT::f32;
686 }
687
688 return fnegFoldsIntoOpcode(Opc);
689}
690
691/// \p returns true if the operation will definitely need to use a 64-bit
692/// encoding, and thus will use a VOP3 encoding regardless of the source
693/// modifiers.
695static bool opMustUseVOP3Encoding(const SDNode *N, MVT VT) {
696 return (N->getNumOperands() > 2 && N->getOpcode() != ISD::SELECT) ||
697 VT == MVT::f64;
698}
699
700/// Return true if v_cndmask_b32 will support fabs/fneg source modifiers for the
701/// type for ISD::SELECT.
703static bool selectSupportsSourceMods(const SDNode *N) {
704 // TODO: Only applies if select will be vector
705 return N->getValueType(0) == MVT::f32;
706}
707
708// Most FP instructions support source modifiers, but this could be refined
709// slightly.
711static bool hasSourceMods(const SDNode *N) {
712 if (isa<MemSDNode>(N))
713 return false;
714
715 switch (N->getOpcode()) {
716 case ISD::CopyToReg:
717 case ISD::FDIV:
718 case ISD::FREM:
719 case ISD::INLINEASM:
721 case AMDGPUISD::DIV_SCALE:
723
724 // TODO: Should really be looking at the users of the bitcast. These are
725 // problematic because bitcasts are used to legalize all stores to integer
726 // types.
727 case ISD::BITCAST:
728 return false;
730 switch (N->getConstantOperandVal(0)) {
731 case Intrinsic::amdgcn_interp_p1:
732 case Intrinsic::amdgcn_interp_p2:
733 case Intrinsic::amdgcn_interp_mov:
734 case Intrinsic::amdgcn_interp_p1_f16:
735 case Intrinsic::amdgcn_interp_p2_f16:
736 return false;
737 default:
738 return true;
739 }
740 }
741 case ISD::SELECT:
743 default:
744 return true;
745 }
746}
747
749 unsigned CostThreshold) {
750 // Some users (such as 3-operand FMA/MAD) must use a VOP3 encoding, and thus
751 // it is truly free to use a source modifier in all cases. If there are
752 // multiple users but for each one will necessitate using VOP3, there will be
753 // a code size increase. Try to avoid increasing code size unless we know it
754 // will save on the instruction count.
755 unsigned NumMayIncreaseSize = 0;
756 MVT VT = N->getValueType(0).getScalarType().getSimpleVT();
757
758 assert(!N->use_empty());
759
760 // XXX - Should this limit number of uses to check?
761 for (const SDNode *U : N->users()) {
762 if (!hasSourceMods(U))
763 return false;
764
765 if (!opMustUseVOP3Encoding(U, VT)) {
766 if (++NumMayIncreaseSize > CostThreshold)
767 return false;
768 }
769 }
770
771 return true;
772}
773
776 assert(!VT.isVector() && "only scalar expected");
777
778 // Round to the next multiple of 32-bits.
779 unsigned Size = VT.getSizeInBits();
780 if (Size <= 32)
781 return MVT::i32;
782 return EVT::getIntegerVT(Context, 32 * ((Size + 31) / 32));
783}
784
786 return 32;
787}
788
790 return true;
791}
792
793// The backend supports 32 and 64 bit floating point immediates.
794// FIXME: Why are we reporting vectors of FP immediates as legal?
797 bool ForCodeSize) const {
800}
801
802// We don't want to shrink f64 / f32 constants.
804 EVT ScalarVT = VT.getScalarType();
805 return (ScalarVT != MVT::f32 && ScalarVT != MVT::f64);
806}
807
809 SDNode *N, ISD::LoadExtType ExtTy, EVT NewVT,
810 std::optional<unsigned> ByteOffset) const {
811 // TODO: This may be worth removing. Check regression tests for diffs.
812 if (!TargetLoweringBase::shouldReduceLoadWidth(N, ExtTy, NewVT, ByteOffset))
813 return false;
814
815 unsigned NewSize = NewVT.getStoreSizeInBits();
816
817 // If we are reducing to a 32-bit load or a smaller multi-dword load,
818 // this is always better.
819 if (NewSize >= 32)
820 return true;
821
822 EVT OldVT = N->getValueType(0);
823 unsigned OldSize = OldVT.getStoreSizeInBits();
824
826 unsigned AS = MN->getAddressSpace();
827 // Do not shrink an aligned scalar load to sub-dword.
828 // Scalar engine cannot do sub-dword loads.
829 // Do not enable for gfx1250+ even though it has sub-dword loads because
830 // this will convert:
831 // i16 = trunc (zextload i16->i32)
832 // to:
833 // i16 = (load i16)
834 // This transformation will be reversed by LowerLOAD resulting in an infinite
835 // loop. Also, tablegen already has a pattern to match zextload i16->i32, but
836 // load i16 will not be matched since there is no instruction that does it.
837 if (OldSize >= 32 && NewSize < 32 && MN->getAlign() >= Align(4) &&
841 MN->isInvariant())) &&
843 return false;
844
845 // Don't produce extloads from sub 32-bit types. SI doesn't have scalar
846 // extloads, so doing one requires using a buffer_load. In cases where we
847 // still couldn't use a scalar load, using the wider load shouldn't really
848 // hurt anything.
849
850 // If the old size already had to be an extload, there's no harm in continuing
851 // to reduce the width.
852 return (OldSize < 32);
853}
854
856 const SelectionDAG &DAG,
857 const MachineMemOperand &MMO) const {
858
859 assert(LoadTy.getSizeInBits() == CastTy.getSizeInBits());
860
861 if (LoadTy.getScalarType() == MVT::i32)
862 return false;
863
864 unsigned LScalarSize = LoadTy.getScalarSizeInBits();
865 unsigned CastScalarSize = CastTy.getScalarSizeInBits();
866
867 if ((LScalarSize >= CastScalarSize) && (CastScalarSize < 32))
868 return false;
869
870 unsigned Fast = 0;
872 CastTy, MMO, &Fast) &&
873 Fast;
874}
875
876// SI+ has instructions for cttz / ctlz for 32-bit values. This is probably also
877// profitable with the expansion for 64-bit since it's generally good to
878// speculate things.
880 return true;
881}
882
884 return true;
885}
886
888 switch (N->getOpcode()) {
889 case ISD::EntryToken:
890 case ISD::TokenFactor:
891 return true;
893 unsigned IntrID = N->getConstantOperandVal(0);
895 }
897 unsigned IntrID = N->getConstantOperandVal(1);
899 }
900 case ISD::LOAD:
901 if (cast<LoadSDNode>(N)->getMemOperand()->getAddrSpace() ==
903 return true;
904 return false;
905 case AMDGPUISD::SETCC: // ballot-style instruction
906 return true;
907 }
908 return false;
909}
910
912 SDValue Op, SelectionDAG &DAG, bool LegalOperations, bool ForCodeSize,
913 NegatibleCost &Cost, unsigned Depth) const {
914
915 switch (Op.getOpcode()) {
916 case ISD::FMA:
917 case ISD::FMAD: {
918 // Negating a fma is not free if it has users without source mods.
919 if (!allUsesHaveSourceMods(Op.getNode()))
920 return SDValue();
921 break;
922 }
923 case AMDGPUISD::RCP: {
924 SDValue Src = Op.getOperand(0);
925 EVT VT = Op.getValueType();
926 SDLoc SL(Op);
927
928 SDValue NegSrc = getNegatedExpression(Src, DAG, LegalOperations,
929 ForCodeSize, Cost, Depth + 1);
930 if (NegSrc)
931 return DAG.getNode(AMDGPUISD::RCP, SL, VT, NegSrc, Op->getFlags());
932 return SDValue();
933 }
934 default:
935 break;
936 }
937
938 return TargetLowering::getNegatedExpression(Op, DAG, LegalOperations,
939 ForCodeSize, Cost, Depth);
940}
941
942//===---------------------------------------------------------------------===//
943// Target Properties
944//===---------------------------------------------------------------------===//
945
948
949 // Packed operations do not have a fabs modifier.
950 // Report this based on the end legalized type.
951 return VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f16 || VT == MVT::bf16;
952}
953
956 // Report this based on the end legalized type.
957 VT = VT.getScalarType();
958 return VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f16 || VT == MVT::bf16;
959}
960
962 unsigned NumElem,
963 unsigned AS) const {
964 return true;
965}
966
968 // There are few operations which truly have vector input operands. Any vector
969 // operation is going to involve operations on each component, and a
970 // build_vector will be a copy per element, so it always makes sense to use a
971 // build_vector input in place of the extracted element to avoid a copy into a
972 // super register.
973 //
974 // We should probably only do this if all users are extracts only, but this
975 // should be the common case.
976 return true;
977}
978
980 // Truncate is just accessing a subregister.
981
982 unsigned SrcSize = Source.getSizeInBits();
983 unsigned DestSize = Dest.getSizeInBits();
984
985 return DestSize < SrcSize && DestSize % 32 == 0 ;
986}
987
989 // Truncate is just accessing a subregister.
990
991 unsigned SrcSize = Source->getScalarSizeInBits();
992 unsigned DestSize = Dest->getScalarSizeInBits();
993
994 if (DestSize== 16 && Subtarget->has16BitInsts())
995 return SrcSize >= 32;
996
997 return DestSize < SrcSize && DestSize % 32 == 0;
998}
999
1001 unsigned SrcSize = Src->getScalarSizeInBits();
1002 unsigned DestSize = Dest->getScalarSizeInBits();
1003
1004 if (SrcSize == 16 && Subtarget->has16BitInsts())
1005 return DestSize >= 32;
1006
1007 return SrcSize == 32 && DestSize == 64;
1008}
1009
1011 // Any register load of a 64-bit value really requires 2 32-bit moves. For all
1012 // practical purposes, the extra mov 0 to load a 64-bit is free. As used,
1013 // this will enable reducing 64-bit operations the 32-bit, which is always
1014 // good.
1015
1016 if (Src == MVT::i16)
1017 return Dest == MVT::i32 ||Dest == MVT::i64 ;
1018
1019 return Src == MVT::i32 && Dest == MVT::i64;
1020}
1021
1023 EVT DestVT) const {
1024 switch (N->getOpcode()) {
1025 case ISD::ABS:
1026 case ISD::ADD:
1027 case ISD::SUB:
1028 case ISD::SHL:
1029 case ISD::SRL:
1030 case ISD::SRA:
1031 case ISD::AND:
1032 case ISD::OR:
1033 case ISD::XOR:
1034 case ISD::MUL:
1035 case ISD::SETCC:
1036 case ISD::SELECT:
1037 case ISD::SMIN:
1038 case ISD::SMAX:
1039 case ISD::UMIN:
1040 case ISD::UMAX:
1041 case ISD::USUBSAT:
1042 case ISD::UADDSAT:
1043 if (isTypeLegal(MVT::i16) &&
1044 (!DestVT.isVector() ||
1045 !isOperationLegal(ISD::ADD, MVT::v2i16))) { // Check if VOP3P
1046 // Don't narrow back down to i16 if promoted to i32 already.
1047 if (!N->isDivergent() && DestVT.isInteger() &&
1048 DestVT.getScalarSizeInBits() > 1 &&
1049 DestVT.getScalarSizeInBits() <= 16 &&
1050 SrcVT.getScalarSizeInBits() > 16) {
1051 return false;
1052 }
1053 }
1054 return true;
1055 default:
1056 break;
1057 }
1058
1059 // There aren't really 64-bit registers, but pairs of 32-bit ones and only a
1060 // limited number of native 64-bit operations. Shrinking an operation to fit
1061 // in a single 32-bit register should always be helpful. As currently used,
1062 // this is much less general than the name suggests, and is only used in
1063 // places trying to reduce the sizes of loads. Shrinking loads to < 32-bits is
1064 // not profitable, and may actually be harmful.
1065 if (isa<LoadSDNode>(N))
1066 return SrcVT.getSizeInBits() > 32 && DestVT.getSizeInBits() == 32;
1067
1068 return true;
1069}
1070
1072 const SDNode* N, CombineLevel Level) const {
1073 assert((N->getOpcode() == ISD::SHL || N->getOpcode() == ISD::SRA ||
1074 N->getOpcode() == ISD::SRL) &&
1075 "Expected shift op");
1076
1077 SDValue ShiftLHS = N->getOperand(0);
1078 if (!ShiftLHS->hasOneUse())
1079 return false;
1080
1081 if (ShiftLHS.getOpcode() == ISD::SIGN_EXTEND &&
1082 !ShiftLHS.getOperand(0)->hasOneUse())
1083 return false;
1084
1085 // Always commute pre-type legalization and right shifts.
1086 // We're looking for shl(or(x,y),z) patterns.
1088 N->getOpcode() != ISD::SHL || N->getOperand(0).getOpcode() != ISD::OR)
1089 return true;
1090
1091 // If only user is a i32 right-shift, then don't destroy a BFE pattern.
1092 if (N->getValueType(0) == MVT::i32 && N->hasOneUse() &&
1093 (N->user_begin()->getOpcode() == ISD::SRA ||
1094 N->user_begin()->getOpcode() == ISD::SRL))
1095 return false;
1096
1097 // Don't destroy or(shl(load_zext(),c), load_zext()) patterns.
1098 auto IsShiftAndLoad = [](SDValue LHS, SDValue RHS) {
1099 if (LHS.getOpcode() != ISD::SHL)
1100 return false;
1101 auto *RHSLd = dyn_cast<LoadSDNode>(RHS);
1102 auto *LHS0 = dyn_cast<LoadSDNode>(LHS.getOperand(0));
1103 auto *LHS1 = dyn_cast<ConstantSDNode>(LHS.getOperand(1));
1104 return LHS0 && LHS1 && RHSLd && LHS0->getExtensionType() == ISD::ZEXTLOAD &&
1105 LHS1->getAPIntValue() == LHS0->getMemoryVT().getScalarSizeInBits() &&
1106 RHSLd->getExtensionType() == ISD::ZEXTLOAD;
1107 };
1108 SDValue LHS = N->getOperand(0).getOperand(0);
1109 SDValue RHS = N->getOperand(0).getOperand(1);
1110 return !(IsShiftAndLoad(LHS, RHS) || IsShiftAndLoad(RHS, LHS));
1111}
1112
1113//===---------------------------------------------------------------------===//
1114// TargetLowering Callbacks
1115//===---------------------------------------------------------------------===//
1116
1118 bool IsVarArg) {
1119 switch (CC) {
1127 return CC_AMDGPU;
1130 return CC_AMDGPU_CS_CHAIN;
1131 case CallingConv::C:
1132 case CallingConv::Fast:
1133 case CallingConv::Cold:
1134 return CC_AMDGPU_Func;
1137 return CC_SI_Gfx;
1140 default:
1141 reportFatalUsageError("unsupported calling convention for call");
1142 }
1143}
1144
1146 bool IsVarArg) {
1147 switch (CC) {
1150 llvm_unreachable("kernels should not be handled here");
1160 return RetCC_SI_Shader;
1163 return RetCC_SI_Gfx;
1164 case CallingConv::C:
1165 case CallingConv::Fast:
1166 case CallingConv::Cold:
1167 return RetCC_AMDGPU_Func;
1168 default:
1169 reportFatalUsageError("unsupported calling convention");
1170 }
1171}
1172
1173/// The SelectionDAGBuilder will automatically promote function arguments
1174/// with illegal types. However, this does not work for the AMDGPU targets
1175/// since the function arguments are stored in memory as these illegal types.
1176/// In order to handle this properly we need to get the original types sizes
1177/// from the LLVM IR Function and fixup the ISD:InputArg values before
1178/// passing them to AnalyzeFormalArguments()
1179
1180/// When the SelectionDAGBuilder computes the Ins, it takes care of splitting
1181/// input values across multiple registers. Each item in the Ins array
1182/// represents a single value that will be stored in registers. Ins[x].VT is
1183/// the value type of the value that will be stored in the register, so
1184/// whatever SDNode we lower the argument to needs to be this type.
1185///
1186/// In order to correctly lower the arguments we need to know the size of each
1187/// argument. Since Ins[x].VT gives us the size of the register that will
1188/// hold the value, we need to look at Ins[x].ArgVT to see the 'real' type
1189/// for the original function argument so that we can deduce the correct memory
1190/// type to use for Ins[x]. In most cases the correct memory type will be
1191/// Ins[x].ArgVT. However, this will not always be the case. If, for example,
1192/// we have a kernel argument of type v8i8, this argument will be split into
1193/// 8 parts and each part will be represented by its own item in the Ins array.
1194/// For each part the Ins[x].ArgVT will be the v8i8, which is the full type of
1195/// the argument before it was split. From this, we deduce that the memory type
1196/// for each individual part is i8. We pass the memory type as LocVT to the
1197/// calling convention analysis function and the register type (Ins[x].VT) as
1198/// the ValVT.
1200 CCState &State,
1201 const SmallVectorImpl<ISD::InputArg> &Ins) const {
1202 const MachineFunction &MF = State.getMachineFunction();
1203 const Function &Fn = MF.getFunction();
1204 LLVMContext &Ctx = Fn.getContext();
1205 const unsigned ExplicitOffset = Subtarget->getExplicitKernelArgOffset();
1207
1208 Align MaxAlign = Align(1);
1209 uint64_t ExplicitArgOffset = 0;
1210 const DataLayout &DL = Fn.getDataLayout();
1211
1212 unsigned InIndex = 0;
1213
1214 for (const Argument &Arg : Fn.args()) {
1215 const bool IsByRef = Arg.hasByRefAttr();
1216 Type *BaseArgTy = Arg.getType();
1217 Type *MemArgTy = IsByRef ? Arg.getParamByRefType() : BaseArgTy;
1218 Align Alignment = DL.getValueOrABITypeAlignment(
1219 IsByRef ? Arg.getParamAlign() : std::nullopt, MemArgTy);
1220 MaxAlign = std::max(Alignment, MaxAlign);
1221 uint64_t AllocSize = DL.getTypeAllocSize(MemArgTy);
1222
1223 uint64_t ArgOffset = alignTo(ExplicitArgOffset, Alignment) + ExplicitOffset;
1224 ExplicitArgOffset = alignTo(ExplicitArgOffset, Alignment) + AllocSize;
1225
1226 // We're basically throwing away everything passed into us and starting over
1227 // to get accurate in-memory offsets. The "PartOffset" is completely useless
1228 // to us as computed in Ins.
1229 //
1230 // We also need to figure out what type legalization is trying to do to get
1231 // the correct memory offsets.
1232
1233 SmallVector<EVT, 16> ValueVTs;
1235 ComputeValueVTs(*this, DL, BaseArgTy, ValueVTs, /*MemVTs=*/nullptr,
1236 &Offsets, ArgOffset);
1237
1238 for (unsigned Value = 0, NumValues = ValueVTs.size();
1239 Value != NumValues; ++Value) {
1240 uint64_t BasePartOffset = Offsets[Value];
1241
1242 EVT ArgVT = ValueVTs[Value];
1243 EVT MemVT = ArgVT;
1244 MVT RegisterVT = getRegisterTypeForCallingConv(Ctx, CC, ArgVT);
1245 unsigned NumRegs = getNumRegistersForCallingConv(Ctx, CC, ArgVT);
1246
1247 if (NumRegs == 1) {
1248 // This argument is not split, so the IR type is the memory type.
1249 if (ArgVT.isExtended()) {
1250 // We have an extended type, like i24, so we should just use the
1251 // register type.
1252 MemVT = RegisterVT;
1253 } else {
1254 MemVT = ArgVT;
1255 }
1256 } else if (ArgVT.isVector() && RegisterVT.isVector() &&
1257 ArgVT.getScalarType() == RegisterVT.getScalarType()) {
1258 assert(ArgVT.getVectorNumElements() > RegisterVT.getVectorNumElements());
1259 // We have a vector value which has been split into a vector with
1260 // the same scalar type, but fewer elements. This should handle
1261 // all the floating-point vector types.
1262 MemVT = RegisterVT;
1263 } else if (ArgVT.isVector() &&
1264 ArgVT.getVectorNumElements() == NumRegs) {
1265 // This arg has been split so that each element is stored in a separate
1266 // register.
1267 MemVT = ArgVT.getScalarType();
1268 } else if (ArgVT.isExtended()) {
1269 // We have an extended type, like i65.
1270 MemVT = RegisterVT;
1271 } else {
1272 unsigned MemoryBits = ArgVT.getStoreSizeInBits() / NumRegs;
1273 assert(ArgVT.getStoreSizeInBits() % NumRegs == 0);
1274 if (RegisterVT.isInteger()) {
1275 MemVT = EVT::getIntegerVT(State.getContext(), MemoryBits);
1276 } else if (RegisterVT.isVector()) {
1277 assert(!RegisterVT.getScalarType().isFloatingPoint());
1278 unsigned NumElements = RegisterVT.getVectorNumElements();
1279 assert(MemoryBits % NumElements == 0);
1280 // This vector type has been split into another vector type with
1281 // a different elements size.
1282 EVT ScalarVT = EVT::getIntegerVT(State.getContext(),
1283 MemoryBits / NumElements);
1284 MemVT = EVT::getVectorVT(State.getContext(), ScalarVT, NumElements);
1285 } else {
1286 llvm_unreachable("cannot deduce memory type.");
1287 }
1288 }
1289
1290 // Convert one element vectors to scalar.
1291 if (MemVT.isVector() && MemVT.getVectorNumElements() == 1)
1292 MemVT = MemVT.getScalarType();
1293
1294 // Round up vec3/vec5 argument.
1295 if (MemVT.isVector() && !MemVT.isPow2VectorType()) {
1296 MemVT = MemVT.getPow2VectorType(State.getContext());
1297 } else if (!MemVT.isSimple() && !MemVT.isVector()) {
1298 MemVT = MemVT.getRoundIntegerType(State.getContext());
1299 }
1300
1301 unsigned PartOffset = 0;
1302 for (unsigned i = 0; i != NumRegs; ++i) {
1303 State.addLoc(CCValAssign::getCustomMem(InIndex++, RegisterVT,
1304 BasePartOffset + PartOffset,
1305 MemVT.getSimpleVT(),
1307 PartOffset += MemVT.getStoreSize();
1308 }
1309 }
1310 }
1311}
1312
1314 SDValue Chain, CallingConv::ID CallConv,
1315 bool isVarArg,
1317 const SmallVectorImpl<SDValue> &OutVals,
1318 const SDLoc &DL, SelectionDAG &DAG) const {
1319 // FIXME: Fails for r600 tests
1320 //assert(!isVarArg && Outs.empty() && OutVals.empty() &&
1321 // "wave terminate should not have return values");
1322 return DAG.getNode(AMDGPUISD::ENDPGM, DL, MVT::Other, Chain);
1323}
1324
1325//===---------------------------------------------------------------------===//
1326// Target specific lowering
1327//===---------------------------------------------------------------------===//
1328
1329/// Selects the correct CCAssignFn for a given CallingConvention value.
1334
1339
1341 SelectionDAG &DAG,
1342 MachineFrameInfo &MFI,
1343 int ClobberedFI) const {
1344 SmallVector<SDValue, 8> ArgChains;
1345 int64_t FirstByte = MFI.getObjectOffset(ClobberedFI);
1346 int64_t LastByte = FirstByte + MFI.getObjectSize(ClobberedFI) - 1;
1347
1348 // Include the original chain at the beginning of the list. When this is
1349 // used by target LowerCall hooks, this helps legalize find the
1350 // CALLSEQ_BEGIN node.
1351 ArgChains.push_back(Chain);
1352
1353 // Add a chain value for each stack argument corresponding
1354 for (SDNode *U : DAG.getEntryNode().getNode()->users()) {
1355 if (LoadSDNode *L = dyn_cast<LoadSDNode>(U)) {
1356 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr())) {
1357 if (FI->getIndex() < 0) {
1358 int64_t InFirstByte = MFI.getObjectOffset(FI->getIndex());
1359 int64_t InLastByte = InFirstByte;
1360 InLastByte += MFI.getObjectSize(FI->getIndex()) - 1;
1361
1362 if ((InFirstByte <= FirstByte && FirstByte <= InLastByte) ||
1363 (FirstByte <= InFirstByte && InFirstByte <= LastByte))
1364 ArgChains.push_back(SDValue(L, 1));
1365 }
1366 }
1367 }
1368 }
1369
1370 // Build a tokenfactor for all the chains.
1371 return DAG.getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
1372}
1373
1376 StringRef Reason) const {
1377 SDValue Callee = CLI.Callee;
1378 SelectionDAG &DAG = CLI.DAG;
1379
1380 const Function &Fn = DAG.getMachineFunction().getFunction();
1381
1382 StringRef FuncName("<unknown>");
1383
1385 FuncName = G->getSymbol();
1386 else if (const GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee))
1387 FuncName = G->getGlobal()->getName();
1388
1389 DAG.getContext()->diagnose(
1390 DiagnosticInfoUnsupported(Fn, Reason + FuncName, CLI.DL.getDebugLoc()));
1391
1392 if (!CLI.IsTailCall) {
1393 for (ISD::InputArg &Arg : CLI.Ins)
1394 InVals.push_back(DAG.getPOISON(Arg.VT));
1395 }
1396
1397 // FIXME: Hack because R600 doesn't handle callseq pseudos yet.
1398 if (getTargetMachine().getTargetTriple().getArch() == Triple::r600)
1399 return CLI.Chain;
1400
1401 SDValue Chain = DAG.getCALLSEQ_START(CLI.Chain, 0, 0, CLI.DL);
1402 return DAG.getCALLSEQ_END(Chain, 0, 0, /*InGlue=*/SDValue(), CLI.DL);
1403}
1404
1406 SmallVectorImpl<SDValue> &InVals) const {
1407 return lowerUnhandledCall(CLI, InVals, "unsupported call to function ");
1408}
1409
1411 SelectionDAG &DAG) const {
1412 const Function &Fn = DAG.getMachineFunction().getFunction();
1413
1415 Fn, "unsupported dynamic alloca", SDLoc(Op).getDebugLoc()));
1416 auto Ops = {DAG.getConstant(0, SDLoc(), Op.getValueType()), Op.getOperand(0)};
1417 return DAG.getMergeValues(Ops, SDLoc());
1418}
1419
1421 SelectionDAG &DAG) const {
1422 switch (Op.getOpcode()) {
1423 default:
1424 Op->print(errs(), &DAG);
1425 llvm_unreachable("Custom lowering code for this "
1426 "instruction is not implemented yet!");
1427 break;
1429 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG);
1431 case ISD::UDIVREM: return LowerUDIVREM(Op, DAG);
1432 case ISD::SDIVREM:
1433 return LowerSDIVREM(Op, DAG);
1434 case ISD::FCEIL: return LowerFCEIL(Op, DAG);
1435 case ISD::FTRUNC: return LowerFTRUNC(Op, DAG);
1436 case ISD::FRINT: return LowerFRINT(Op, DAG);
1437 case ISD::FNEARBYINT: return LowerFNEARBYINT(Op, DAG);
1438 case ISD::FROUNDEVEN:
1439 return LowerFROUNDEVEN(Op, DAG);
1440 case ISD::FROUND: return LowerFROUND(Op, DAG);
1441 case ISD::FFLOOR: return LowerFFLOOR(Op, DAG);
1442 case ISD::FLOG2:
1443 return LowerFLOG2(Op, DAG);
1444 case ISD::FLOG:
1445 case ISD::FLOG10:
1446 return LowerFLOGCommon(Op, DAG);
1447 case ISD::FEXP:
1448 case ISD::FEXP10:
1449 return lowerFEXP(Op, DAG);
1450 case ISD::FEXP2:
1451 return lowerFEXP2(Op, DAG);
1452 case ISD::FPOW:
1453 return lowerFPOW(Op, DAG);
1454 case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG);
1455 case ISD::UINT_TO_FP: return LowerUINT_TO_FP(Op, DAG);
1456 case ISD::FP_TO_FP16: return LowerFP_TO_FP16(Op, DAG);
1457 case ISD::FP_TO_SINT:
1458 case ISD::FP_TO_UINT:
1459 return LowerFP_TO_INT(Op, DAG);
1462 return LowerFP_TO_INT_SAT(Op, DAG);
1463 case ISD::CTTZ:
1465 case ISD::CTLZ:
1467 return LowerCTLZ_CTTZ(Op, DAG);
1468 case ISD::CTLS:
1469 return LowerCTLS(Op, DAG);
1471 }
1472 return Op;
1473}
1474
1477 SelectionDAG &DAG) const {
1478 switch (N->getOpcode()) {
1480 // Different parts of legalization seem to interpret which type of
1481 // sign_extend_inreg is the one to check for custom lowering. The extended
1482 // from type is what really matters, but some places check for custom
1483 // lowering of the result type. This results in trying to use
1484 // ReplaceNodeResults to sext_in_reg to an illegal type, so we'll just do
1485 // nothing here and let the illegal result integer be handled normally.
1486 return;
1487 case ISD::FLOG2:
1488 if (SDValue Lowered = LowerFLOG2(SDValue(N, 0), DAG))
1489 Results.push_back(Lowered);
1490 return;
1491 case ISD::FLOG:
1492 case ISD::FLOG10:
1493 if (SDValue Lowered = LowerFLOGCommon(SDValue(N, 0), DAG))
1494 Results.push_back(Lowered);
1495 return;
1496 case ISD::FEXP2:
1497 if (SDValue Lowered = lowerFEXP2(SDValue(N, 0), DAG))
1498 Results.push_back(Lowered);
1499 return;
1500 case ISD::FEXP:
1501 case ISD::FEXP10:
1502 if (SDValue Lowered = lowerFEXP(SDValue(N, 0), DAG))
1503 Results.push_back(Lowered);
1504 return;
1505 case ISD::CTLZ:
1507 if (auto Lowered = lowerCTLZResults(SDValue(N, 0u), DAG))
1508 Results.push_back(Lowered);
1509 return;
1510 default:
1511 return;
1512 }
1513}
1514
1516 SelectionDAG &DAG) const {
1518 SDLoc SL(Op);
1519 EVT VT = Op.getValueType();
1520 return DAG.getTargetBlockAddress(BA->getBlockAddress(), VT, BA->getOffset(),
1521 BA->getTargetFlags());
1522}
1523
1525 SDValue Op,
1526 SelectionDAG &DAG) const {
1527
1528 const DataLayout &DL = DAG.getDataLayout();
1530 const GlobalValue *GV = G->getGlobal();
1531
1532 if (G->getAddressSpace() == AMDGPUAS::BARRIER) {
1533 const GlobalVariable *GVar = cast<GlobalVariable>(GV);
1534
1535 if (!AMDGPU::isNamedBarrier(*GVar)) {
1536 const Function &Fn = DAG.getMachineFunction().getFunction();
1538 Fn, "unsupported use of BARRIER address space",
1540 return DAG.getPOISON(Op.getValueType());
1541 }
1542
1543 unsigned Offset = MFI->allocateBarrierGlobal(DL, *cast<GlobalVariable>(GV));
1544 return DAG.getConstant(Offset, SDLoc(Op), Op.getValueType());
1545 }
1546
1547 if (!MFI->isModuleEntryFunction()) {
1548 if (std::optional<uint32_t> Address =
1551 return DAG.getConstant(*Address, SDLoc(Op), Op.getValueType());
1552 }
1553 }
1554
1555 if (G->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS ||
1556 G->getAddressSpace() == AMDGPUAS::REGION_ADDRESS) {
1557 if (!MFI->isModuleEntryFunction() &&
1558 GV->getName() != "llvm.amdgcn.module.lds") {
1559 SDLoc DL(Op);
1560 const Function &Fn = DAG.getMachineFunction().getFunction();
1562 Fn, "local memory global used by non-kernel function",
1563 DL.getDebugLoc(), DS_Warning));
1564
1565 // We currently don't have a way to correctly allocate LDS objects that
1566 // aren't directly associated with a kernel. We do force inlining of
1567 // functions that use local objects. However, if these dead functions are
1568 // not eliminated, we don't want a compile time error. Just emit a warning
1569 // and a trap, since there should be no callable path here.
1570 SDValue Trap = DAG.getNode(ISD::TRAP, DL, MVT::Other, DAG.getEntryNode());
1571 SDValue OutputChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
1572 Trap, DAG.getRoot());
1573 DAG.setRoot(OutputChain);
1574 return DAG.getPOISON(Op.getValueType());
1575 }
1576
1577 // TODO: We could emit code to handle the initialization somewhere.
1578 // We ignore the initializer for now and legalize it to allow selection.
1579 // The initializer will anyway get errored out during assembly emission.
1580 unsigned Offset = MFI->allocateLDSGlobal(DL, *cast<GlobalVariable>(GV));
1581 // A constant byte offset (e.g. from a GEP into an array of named barriers)
1582 // folds directly into the allocated LDS address.
1583 return DAG.getConstant(Offset + G->getOffset(), SDLoc(Op),
1584 Op.getValueType());
1585 }
1586 return SDValue();
1587}
1588
1590 SelectionDAG &DAG) const {
1592 SDLoc SL(Op);
1593
1594 EVT VT = Op.getValueType();
1595 if (VT.getVectorElementType().getSizeInBits() < 32) {
1596 unsigned OpBitSize = Op.getOperand(0).getValueType().getSizeInBits();
1597 if (OpBitSize >= 32 && OpBitSize % 32 == 0) {
1598 unsigned NewNumElt = OpBitSize / 32;
1599 EVT NewEltVT = (NewNumElt == 1) ? MVT::i32
1601 MVT::i32, NewNumElt);
1602 for (const SDUse &U : Op->ops()) {
1603 SDValue In = U.get();
1604 SDValue NewIn = DAG.getNode(ISD::BITCAST, SL, NewEltVT, In);
1605 if (NewNumElt > 1)
1606 DAG.ExtractVectorElements(NewIn, Args);
1607 else
1608 Args.push_back(NewIn);
1609 }
1610
1611 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32,
1612 NewNumElt * Op.getNumOperands());
1613 SDValue BV = DAG.getBuildVector(NewVT, SL, Args);
1614 return DAG.getNode(ISD::BITCAST, SL, VT, BV);
1615 }
1616 }
1617
1618 for (const SDUse &U : Op->ops())
1619 DAG.ExtractVectorElements(U.get(), Args);
1620
1621 return DAG.getBuildVector(Op.getValueType(), SL, Args);
1622}
1623
1625 SelectionDAG &DAG) const {
1626 SDLoc SL(Op);
1628 unsigned Start = Op.getConstantOperandVal(1);
1629 EVT VT = Op.getValueType();
1630 EVT SrcVT = Op.getOperand(0).getValueType();
1631
1632 if (VT.getScalarSizeInBits() == 16 && Start % 2 == 0) {
1633 unsigned NumElt = VT.getVectorNumElements();
1634 unsigned NumSrcElt = SrcVT.getVectorNumElements();
1635 assert(NumElt % 2 == 0 && NumSrcElt % 2 == 0 && "expect legal types");
1636
1637 // Extract 32-bit registers at a time.
1638 EVT NewSrcVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumSrcElt / 2);
1639 EVT NewVT = NumElt == 2
1640 ? MVT::i32
1641 : EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumElt / 2);
1642 SDValue Tmp = DAG.getNode(ISD::BITCAST, SL, NewSrcVT, Op.getOperand(0));
1643
1644 DAG.ExtractVectorElements(Tmp, Args, Start / 2, NumElt / 2);
1645 if (NumElt == 2)
1646 Tmp = Args[0];
1647 else
1648 Tmp = DAG.getBuildVector(NewVT, SL, Args);
1649
1650 return DAG.getNode(ISD::BITCAST, SL, VT, Tmp);
1651 }
1652
1653 DAG.ExtractVectorElements(Op.getOperand(0), Args, Start,
1655
1656 return DAG.getBuildVector(Op.getValueType(), SL, Args);
1657}
1658
1659// TODO: Handle fabs too
1661 if (Val.getOpcode() == ISD::FNEG)
1662 return Val.getOperand(0);
1663
1664 return Val;
1665}
1666
1667// SelectionDAG twin of AMDGPUCombinerHelper::canIgnoreLegacyMinMaxTies.
1669 SDNodeFlags Flags, SDValue LHS,
1670 SDValue RHS) {
1671 return Flags.hasNoSignedZeros() || DAG.isKnownNeverLogicalZero(LHS) ||
1673}
1674
1676 const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, SDValue True,
1677 SDValue False, SDValue CC, SDNodeFlags Flags, DAGCombinerInfo &DCI) const {
1678 SelectionDAG &DAG = DCI.DAG;
1679 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get();
1680 assert(CCOpcode != ISD::SETCC_INVALID && "Invalid setcc condcode!");
1681
1682 switch (CCOpcode) {
1683 case ISD::SETOLE:
1684 case ISD::SETOLT:
1685 case ISD::SETLE:
1686 case ISD::SETLT:
1687 case ISD::SETOGE:
1688 case ISD::SETOGT:
1689 case ISD::SETGE:
1690 case ISD::SETGT:
1691 // Only do this after legalization to avoid interfering with other combines
1692 // which might occur.
1694 !DCI.isCalledByLegalizer())
1695 return SDValue();
1696 break;
1697 default:
1698 break;
1699 }
1700
1701 // Canonicalize so the select returns the compare's LHS on a true predicate.
1702 if (LHS != True)
1703 CCOpcode = ISD::getSetCCInverse(CCOpcode, VT);
1704
1705 unsigned Opc;
1706 bool Swap; // Emit (rhs, lhs) instead of (lhs, rhs).
1707 switch (CCOpcode) {
1708 case ISD::SETOLT:
1709 case ISD::SETLT:
1710 case ISD::SETOLE:
1711 Opc = AMDGPUISD::FMIN_LEGACY;
1712 Swap = false;
1713 break;
1714 case ISD::SETULE:
1715 case ISD::SETLE:
1716 case ISD::SETULT:
1717 Opc = AMDGPUISD::FMIN_LEGACY;
1718 Swap = true;
1719 break;
1720 case ISD::SETOGE:
1721 case ISD::SETGE:
1722 case ISD::SETOGT:
1723 Opc = AMDGPUISD::FMAX_LEGACY;
1724 Swap = false;
1725 break;
1726 case ISD::SETUGT:
1727 case ISD::SETGT:
1728 case ISD::SETUGE:
1729 Opc = AMDGPUISD::FMAX_LEGACY;
1730 Swap = true;
1731 break;
1732 default:
1733 return SDValue();
1734 }
1735
1736 // For these predicates the NaN-correct operand order is the signed zero
1737 // tie-incorrect one, so the fold needs the tie to be unobservable.
1738 if ((CCOpcode == ISD::SETOLE || CCOpcode == ISD::SETULT ||
1739 CCOpcode == ISD::SETOGT || CCOpcode == ISD::SETUGE) &&
1740 !canIgnoreLegacyMinMaxTies(DAG, Flags, LHS, RHS))
1741 return SDValue();
1742
1743 if (Swap)
1744 std::swap(LHS, RHS);
1745 return DAG.getNode(Opc, DL, VT, LHS, RHS, Flags);
1746}
1747
1748/// Generate Min/Max node
1750 const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, SDValue True,
1751 SDValue False, SDValue CC, SDNodeFlags Flags, DAGCombinerInfo &DCI) const {
1752 if ((LHS == True && RHS == False) || (LHS == False && RHS == True))
1753 return combineFMinMaxLegacyImpl(DL, VT, LHS, RHS, True, False, CC, Flags,
1754 DCI);
1755
1756 SelectionDAG &DAG = DCI.DAG;
1757
1758 // If we can't directly match this, try to see if we can fold an fneg to
1759 // match.
1760
1763 SDValue NegTrue = peekFNeg(True);
1764
1765 // Undo the combine foldFreeOpFromSelect does if it helps us match the
1766 // fmin/fmax.
1767 //
1768 // select (fcmp olt (lhs, K)), (fneg lhs), -K
1769 // -> fneg (fmin_legacy lhs, K)
1770 //
1771 // TODO: Use getNegatedExpression
1772 if (LHS == NegTrue && CFalse && CRHS) {
1773 APFloat NegRHS = neg(CRHS->getValueAPF());
1774 if (NegRHS == CFalse->getValueAPF()) {
1775 SDValue Combined = combineFMinMaxLegacyImpl(DL, VT, LHS, RHS, NegTrue,
1776 False, CC, Flags, DCI);
1777 if (Combined)
1778 return DAG.getNode(ISD::FNEG, DL, VT, Combined);
1779 return SDValue();
1780 }
1781 }
1782
1783 return SDValue();
1784}
1785
1786std::pair<SDValue, SDValue>
1788 SDLoc SL(Op);
1789
1790 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op);
1791
1792 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
1793 const SDValue One = DAG.getConstant(1, SL, MVT::i32);
1794
1795 SDValue Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, Zero);
1796 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, One);
1797
1798 return std::pair(Lo, Hi);
1799}
1800
1802 SDLoc SL(Op);
1803
1804 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op);
1805 const SDValue One = DAG.getConstant(1, SL, MVT::i32);
1806 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, One);
1807}
1808
1809// Split a vector type into two parts. The first part is a power of two vector.
1810// The second part is whatever is left over, and is a scalar if it would
1811// otherwise be a 1-vector.
1812std::pair<EVT, EVT>
1814 EVT LoVT, HiVT;
1815 EVT EltVT = VT.getVectorElementType();
1816 unsigned NumElts = VT.getVectorNumElements();
1817 unsigned LoNumElts = PowerOf2Ceil((NumElts + 1) / 2);
1818 LoVT = EVT::getVectorVT(*DAG.getContext(), EltVT, LoNumElts);
1819 HiVT = NumElts - LoNumElts == 1
1820 ? EltVT
1821 : EVT::getVectorVT(*DAG.getContext(), EltVT, NumElts - LoNumElts);
1822 return std::pair(LoVT, HiVT);
1823}
1824
1825// Split a vector value into two parts of types LoVT and HiVT. HiVT could be
1826// scalar.
1827std::pair<SDValue, SDValue>
1829 const EVT &LoVT, const EVT &HiVT,
1830 SelectionDAG &DAG) const {
1831 EVT VT = N.getValueType();
1833 (HiVT.isVector() ? HiVT.getVectorNumElements() : 1) <=
1834 VT.getVectorNumElements() &&
1835 "More vector elements requested than available!");
1837 DAG.getVectorIdxConstant(0, DL));
1838
1839 unsigned LoNumElts = LoVT.getVectorNumElements();
1840
1841 if (HiVT.isVector()) {
1842 unsigned HiNumElts = HiVT.getVectorNumElements();
1843 if ((VT.getVectorNumElements() % HiNumElts) == 0) {
1844 // Avoid creating an extract_subvector with an index that isn't a multiple
1845 // of the result type.
1847 DAG.getConstant(LoNumElts, DL, MVT::i32));
1848 return {Lo, Hi};
1849 }
1850
1852 DAG.ExtractVectorElements(N, Elts, /*Start=*/LoNumElts,
1853 /*Count=*/HiNumElts);
1854 SDValue Hi = DAG.getBuildVector(HiVT, DL, Elts);
1855 return {Lo, Hi};
1856 }
1857
1859 DAG.getVectorIdxConstant(LoNumElts, DL));
1860 return {Lo, Hi};
1861}
1862
1864 SelectionDAG &DAG) const {
1866 EVT VT = Op.getValueType();
1867 SDLoc SL(Op);
1868
1869
1870 // If this is a 2 element vector, we really want to scalarize and not create
1871 // weird 1 element vectors.
1872 if (VT.getVectorNumElements() == 2) {
1873 SDValue Ops[2];
1874 std::tie(Ops[0], Ops[1]) = scalarizeVectorLoad(Load, DAG);
1875 return DAG.getMergeValues(Ops, SL);
1876 }
1877
1878 SDValue BasePtr = Load->getBasePtr();
1879 EVT MemVT = Load->getMemoryVT();
1880
1881 const MachinePointerInfo &SrcValue = Load->getMemOperand()->getPointerInfo();
1882
1883 EVT LoVT, HiVT;
1884 EVT LoMemVT, HiMemVT;
1885 SDValue Lo, Hi;
1886
1887 std::tie(LoVT, HiVT) = getSplitDestVTs(VT, DAG);
1888 std::tie(LoMemVT, HiMemVT) = getSplitDestVTs(MemVT, DAG);
1889 std::tie(Lo, Hi) = splitVector(Op, SL, LoVT, HiVT, DAG);
1890
1891 unsigned Size = LoMemVT.getStoreSize();
1892 Align BaseAlign = Load->getAlign();
1893 Align HiAlign = commonAlignment(BaseAlign, Size);
1894
1895 SDValue LoLoad = DAG.getExtLoad(
1896 Load->getExtensionType(), SL, LoVT, Load->getChain(), BasePtr, SrcValue,
1897 LoMemVT, BaseAlign, Load->getMemOperand()->getFlags(), Load->getAAInfo());
1898 SDValue HiPtr = DAG.getObjectPtrOffset(SL, BasePtr, TypeSize::getFixed(Size));
1899 SDValue HiLoad = DAG.getExtLoad(
1900 Load->getExtensionType(), SL, HiVT, Load->getChain(), HiPtr,
1901 SrcValue.getWithOffset(LoMemVT.getStoreSize()), HiMemVT, HiAlign,
1902 Load->getMemOperand()->getFlags(), Load->getAAInfo());
1903
1904 SDValue Join;
1905 if (LoVT == HiVT) {
1906 // This is the case that the vector is power of two so was evenly split.
1907 Join = DAG.getNode(ISD::CONCAT_VECTORS, SL, VT, LoLoad, HiLoad);
1908 } else {
1909 Join = DAG.getNode(ISD::INSERT_SUBVECTOR, SL, VT, DAG.getPOISON(VT), LoLoad,
1910 DAG.getVectorIdxConstant(0, SL));
1911 Join = DAG.getNode(
1913 VT, Join, HiLoad,
1915 }
1916
1917 SDValue Ops[] = {Join, DAG.getNode(ISD::TokenFactor, SL, MVT::Other,
1918 LoLoad.getValue(1), HiLoad.getValue(1))};
1919
1920 return DAG.getMergeValues(Ops, SL);
1921}
1922
1924 SelectionDAG &DAG) const {
1926 EVT VT = Op.getValueType();
1927 SDValue BasePtr = Load->getBasePtr();
1928 EVT MemVT = Load->getMemoryVT();
1929 SDLoc SL(Op);
1930 const MachinePointerInfo &SrcValue = Load->getMemOperand()->getPointerInfo();
1931 Align BaseAlign = Load->getAlign();
1932 unsigned NumElements = MemVT.getVectorNumElements();
1933
1934 // Widen from vec3 to vec4 when the load is at least 8-byte aligned
1935 // or 16-byte fully dereferenceable. Otherwise, split the vector load.
1936 if (NumElements != 3 ||
1937 (BaseAlign < Align(8) &&
1938 !SrcValue.isDereferenceable(16, *DAG.getContext(), DAG.getDataLayout())))
1939 return SplitVectorLoad(Op, DAG);
1940
1941 assert(NumElements == 3);
1942
1943 EVT WideVT =
1945 EVT WideMemVT =
1947 SDValue WideLoad = DAG.getExtLoad(
1948 Load->getExtensionType(), SL, WideVT, Load->getChain(), BasePtr, SrcValue,
1949 WideMemVT, BaseAlign, Load->getMemOperand()->getFlags());
1950 return DAG.getMergeValues(
1951 {DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, VT, WideLoad,
1952 DAG.getVectorIdxConstant(0, SL)),
1953 WideLoad.getValue(1)},
1954 SL);
1955}
1956
1958 SelectionDAG &DAG) const {
1960 SDValue Val = Store->getValue();
1961 EVT VT = Val.getValueType();
1962
1963 // If this is a 2 element vector, we really want to scalarize and not create
1964 // weird 1 element vectors.
1965 if (VT.getVectorNumElements() == 2)
1966 return scalarizeVectorStore(Store, DAG);
1967
1968 EVT MemVT = Store->getMemoryVT();
1969 SDValue Chain = Store->getChain();
1970 SDValue BasePtr = Store->getBasePtr();
1971 SDLoc SL(Op);
1972
1973 EVT LoVT, HiVT;
1974 EVT LoMemVT, HiMemVT;
1975 SDValue Lo, Hi;
1976
1977 std::tie(LoVT, HiVT) = getSplitDestVTs(VT, DAG);
1978 std::tie(LoMemVT, HiMemVT) = getSplitDestVTs(MemVT, DAG);
1979 std::tie(Lo, Hi) = splitVector(Val, SL, LoVT, HiVT, DAG);
1980
1981 SDValue HiPtr = DAG.getObjectPtrOffset(SL, BasePtr, LoMemVT.getStoreSize());
1982
1983 const MachinePointerInfo &SrcValue = Store->getMemOperand()->getPointerInfo();
1984 Align BaseAlign = Store->getAlign();
1985 unsigned Size = LoMemVT.getStoreSize();
1986 Align HiAlign = commonAlignment(BaseAlign, Size);
1987
1988 SDValue LoStore =
1989 DAG.getTruncStore(Chain, SL, Lo, BasePtr, SrcValue, LoMemVT, BaseAlign,
1990 Store->getMemOperand()->getFlags(), Store->getAAInfo());
1991 SDValue HiStore = DAG.getTruncStore(
1992 Chain, SL, Hi, HiPtr, SrcValue.getWithOffset(Size), HiMemVT, HiAlign,
1993 Store->getMemOperand()->getFlags(), Store->getAAInfo());
1994
1995 return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, LoStore, HiStore);
1996}
1997
1998// This is a shortcut for integer division because we have fast i32<->f32
1999// conversions, and fast f32 reciprocal instructions.
2001 bool Sign) const {
2002 SDLoc DL(Op);
2003 EVT VT = Op.getValueType();
2004 assert(VT == MVT::i32 && "LowerDIVREMToFloat expects an i32");
2005
2006 SDValue LHS = Op.getOperand(0);
2007 SDValue RHS = Op.getOperand(1);
2008 MVT IntVT = MVT::i32;
2009 MVT FltVT = MVT::f32;
2010
2011 unsigned LHSSignBits;
2012 unsigned RHSSignBits;
2013 if (Sign) {
2014 LHSSignBits = DAG.ComputeNumSignBits(LHS);
2015 RHSSignBits = DAG.ComputeNumSignBits(RHS);
2016 if (LHSSignBits < 9 || RHSSignBits < 9)
2017 return SDValue();
2018 } else {
2019 KnownBits LHSKnown = DAG.computeKnownBits(LHS);
2020 KnownBits RHSKnown = DAG.computeKnownBits(RHS);
2021
2022 LHSSignBits = LHSKnown.countMinLeadingZeros();
2023 RHSSignBits = RHSKnown.countMinLeadingZeros();
2024 }
2025
2026 unsigned BitSize = VT.getSizeInBits();
2027 unsigned SignBits = std::min(LHSSignBits, RHSSignBits);
2028 unsigned DivBits = BitSize - SignBits;
2029 if (Sign)
2030 ++DivBits;
2031
2032 // In order to avoid problems due to 1 ulp accuracy issues with v_rcp_f32,
2033 // limit LowerDIVREMToFloat to:
2034 // [-0x400000,0x3FFFFF] for Sign
2035 // [ 0x000000,0x3FFFFF] for !Sign
2036 // This matches what is done in expandDivRemToFloatImpl.
2037 if (DivBits > (Sign ? 23 : 22))
2038 return SDValue();
2039
2042
2043 // int ia = (int)LHS;
2044 SDValue ia = LHS;
2045
2046 // int ib, (int)RHS;
2047 SDValue ib = RHS;
2048
2049 // The calculation:
2050 // fq = fa*recip(fb)
2051 // may be too small due to the 1ulp accuracy in the recip
2052 // operation and rounding issues. Since fq is truncated to produce
2053 // an integer value it may be too small by one. This is
2054 // dealt with by incrementing fa by 1ulp:
2055 // fq = (fa+1ulp)*recip(fb)
2056 // This will increase fa's magnitude by at most 0.5
2057 // (i.e. when fabs(fa)==0x400000 the LSB of the mantissa represents 0.5).
2058 // Thus, this method is safe since fa must be incremented by at least 1.0
2059 // for the quotient to increase by one.
2060 SDValue fa = DAG.getNode(ToFp, DL, FltVT, ia);
2061 SDValue faAsInt = DAG.getNode(ISD::BITCAST, DL, MVT::i32, fa);
2062 SDValue faIncremented = DAG.getNode(ISD::ADD, DL, MVT::i32, faAsInt,
2063 DAG.getConstant(1, DL, MVT::i32));
2064 fa = DAG.getNode(ISD::BITCAST, DL, FltVT, faIncremented);
2065
2066 // float fb = (float)ib;
2067 SDValue fb = DAG.getNode(ToFp, DL, FltVT, ib);
2068
2069 SDValue fq = DAG.getNode(ISD::FMUL, DL, FltVT,
2070 fa, DAG.getNode(AMDGPUISD::RCP, DL, FltVT, fb));
2071
2072 // fq = trunc(fq);
2073 fq = DAG.getNode(ISD::FTRUNC, DL, FltVT, fq);
2074
2075 // int iq = (int)fq;
2076 SDValue Div = DAG.getNode(ToInt, DL, IntVT, fq);
2077
2078 // Rem needs compensation, it's easier to recompute it
2079 SDValue Rem = DAG.getNode(ISD::MUL, DL, VT, Div, RHS);
2080 Rem = DAG.getNode(ISD::SUB, DL, VT, LHS, Rem);
2081
2082 return DAG.getMergeValues({ Div, Rem }, DL);
2083}
2084
2086 SelectionDAG &DAG,
2088 SDLoc DL(Op);
2089 EVT VT = Op.getValueType();
2090
2091 assert(VT == MVT::i64 && "LowerUDIVREM64 expects an i64");
2092
2093 EVT HalfVT = VT.getHalfSizedIntegerVT(*DAG.getContext());
2094
2095 SDValue One = DAG.getConstant(1, DL, HalfVT);
2096 SDValue Zero = DAG.getConstant(0, DL, HalfVT);
2097
2098 //HiLo split
2099 SDValue LHS_Lo, LHS_Hi;
2100 SDValue LHS = Op.getOperand(0);
2101 std::tie(LHS_Lo, LHS_Hi) = DAG.SplitScalar(LHS, DL, HalfVT, HalfVT);
2102
2103 SDValue RHS_Lo, RHS_Hi;
2104 SDValue RHS = Op.getOperand(1);
2105 std::tie(RHS_Lo, RHS_Hi) = DAG.SplitScalar(RHS, DL, HalfVT, HalfVT);
2106
2107 if (DAG.MaskedValueIsZero(RHS, APInt::getHighBitsSet(64, 32)) &&
2108 DAG.MaskedValueIsZero(LHS, APInt::getHighBitsSet(64, 32))) {
2109
2110 SDValue Res = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(HalfVT, HalfVT),
2111 LHS_Lo, RHS_Lo);
2112
2113 SDValue DIV = DAG.getBuildVector(MVT::v2i32, DL, {Res.getValue(0), Zero});
2114 SDValue REM = DAG.getBuildVector(MVT::v2i32, DL, {Res.getValue(1), Zero});
2115
2116 Results.push_back(DAG.getNode(ISD::BITCAST, DL, MVT::i64, DIV));
2117 Results.push_back(DAG.getNode(ISD::BITCAST, DL, MVT::i64, REM));
2118 return;
2119 }
2120
2121 if (isTypeLegal(MVT::i64)) {
2122 // The algorithm here is based on ideas from "Software Integer Division",
2123 // Tom Rodeheffer, August 2008.
2124
2127
2128 // Compute denominator reciprocal.
2129 unsigned FMAD =
2130 !Subtarget->hasMadMacF32Insts() ? (unsigned)ISD::FMA
2133 : (unsigned)AMDGPUISD::FMAD_FTZ;
2134
2135 SDValue Cvt_Lo = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, RHS_Lo);
2136 SDValue Cvt_Hi = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, RHS_Hi);
2137 SDValue Mad1 = DAG.getNode(FMAD, DL, MVT::f32, Cvt_Hi,
2138 DAG.getConstantFP(APInt(32, 0x4f800000).bitsToFloat(), DL, MVT::f32),
2139 Cvt_Lo);
2140 SDValue Rcp = DAG.getNode(AMDGPUISD::RCP, DL, MVT::f32, Mad1);
2141 SDValue Mul1 = DAG.getNode(ISD::FMUL, DL, MVT::f32, Rcp,
2142 DAG.getConstantFP(APInt(32, 0x5f7ffffc).bitsToFloat(), DL, MVT::f32));
2143 SDValue Mul2 = DAG.getNode(ISD::FMUL, DL, MVT::f32, Mul1,
2144 DAG.getConstantFP(APInt(32, 0x2f800000).bitsToFloat(), DL, MVT::f32));
2145 SDValue Trunc = DAG.getNode(ISD::FTRUNC, DL, MVT::f32, Mul2);
2146 SDValue Mad2 = DAG.getNode(FMAD, DL, MVT::f32, Trunc,
2147 DAG.getConstantFP(APInt(32, 0xcf800000).bitsToFloat(), DL, MVT::f32),
2148 Mul1);
2149 SDValue Rcp_Lo = DAG.getNode(ISD::FP_TO_UINT, DL, HalfVT, Mad2);
2150 SDValue Rcp_Hi = DAG.getNode(ISD::FP_TO_UINT, DL, HalfVT, Trunc);
2151 SDValue Rcp64 = DAG.getBitcast(VT,
2152 DAG.getBuildVector(MVT::v2i32, DL, {Rcp_Lo, Rcp_Hi}));
2153
2154 SDValue Zero64 = DAG.getConstant(0, DL, VT);
2155 SDValue One64 = DAG.getConstant(1, DL, VT);
2156 SDValue Zero1 = DAG.getConstant(0, DL, MVT::i1);
2157 SDVTList HalfCarryVT = DAG.getVTList(HalfVT, MVT::i1);
2158
2159 // First round of UNR (Unsigned integer Newton-Raphson).
2160 SDValue Neg_RHS = DAG.getNode(ISD::SUB, DL, VT, Zero64, RHS);
2161 SDValue Mullo1 = DAG.getNode(ISD::MUL, DL, VT, Neg_RHS, Rcp64);
2162 SDValue Mulhi1 = DAG.getNode(ISD::MULHU, DL, VT, Rcp64, Mullo1);
2163 SDValue Mulhi1_Lo, Mulhi1_Hi;
2164 std::tie(Mulhi1_Lo, Mulhi1_Hi) =
2165 DAG.SplitScalar(Mulhi1, DL, HalfVT, HalfVT);
2166 SDValue Add1_Lo = DAG.getNode(ISD::UADDO_CARRY, DL, HalfCarryVT, Rcp_Lo,
2167 Mulhi1_Lo, Zero1);
2168 SDValue Add1_Hi = DAG.getNode(ISD::UADDO_CARRY, DL, HalfCarryVT, Rcp_Hi,
2169 Mulhi1_Hi, Add1_Lo.getValue(1));
2170 SDValue Add1 = DAG.getBitcast(VT,
2171 DAG.getBuildVector(MVT::v2i32, DL, {Add1_Lo, Add1_Hi}));
2172
2173 // Second round of UNR.
2174 SDValue Mullo2 = DAG.getNode(ISD::MUL, DL, VT, Neg_RHS, Add1);
2175 SDValue Mulhi2 = DAG.getNode(ISD::MULHU, DL, VT, Add1, Mullo2);
2176 SDValue Mulhi2_Lo, Mulhi2_Hi;
2177 std::tie(Mulhi2_Lo, Mulhi2_Hi) =
2178 DAG.SplitScalar(Mulhi2, DL, HalfVT, HalfVT);
2179 SDValue Add2_Lo = DAG.getNode(ISD::UADDO_CARRY, DL, HalfCarryVT, Add1_Lo,
2180 Mulhi2_Lo, Zero1);
2181 SDValue Add2_Hi = DAG.getNode(ISD::UADDO_CARRY, DL, HalfCarryVT, Add1_Hi,
2182 Mulhi2_Hi, Add2_Lo.getValue(1));
2183 SDValue Add2 = DAG.getBitcast(VT,
2184 DAG.getBuildVector(MVT::v2i32, DL, {Add2_Lo, Add2_Hi}));
2185
2186 SDValue Mulhi3 = DAG.getNode(ISD::MULHU, DL, VT, LHS, Add2);
2187
2188 SDValue Mul3 = DAG.getNode(ISD::MUL, DL, VT, RHS, Mulhi3);
2189
2190 SDValue Mul3_Lo, Mul3_Hi;
2191 std::tie(Mul3_Lo, Mul3_Hi) = DAG.SplitScalar(Mul3, DL, HalfVT, HalfVT);
2192 SDValue Sub1_Lo = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, LHS_Lo,
2193 Mul3_Lo, Zero1);
2194 SDValue Sub1_Hi = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, LHS_Hi,
2195 Mul3_Hi, Sub1_Lo.getValue(1));
2196 SDValue Sub1_Mi = DAG.getNode(ISD::SUB, DL, HalfVT, LHS_Hi, Mul3_Hi);
2197 SDValue Sub1 = DAG.getBitcast(VT,
2198 DAG.getBuildVector(MVT::v2i32, DL, {Sub1_Lo, Sub1_Hi}));
2199
2200 SDValue MinusOne = DAG.getConstant(0xffffffffu, DL, HalfVT);
2201 SDValue C1 = DAG.getSelectCC(DL, Sub1_Hi, RHS_Hi, MinusOne, Zero,
2202 ISD::SETUGE);
2203 SDValue C2 = DAG.getSelectCC(DL, Sub1_Lo, RHS_Lo, MinusOne, Zero,
2204 ISD::SETUGE);
2205 SDValue C3 = DAG.getSelectCC(DL, Sub1_Hi, RHS_Hi, C2, C1, ISD::SETEQ);
2206
2207 // TODO: Here and below portions of the code can be enclosed into if/endif.
2208 // Currently control flow is unconditional and we have 4 selects after
2209 // potential endif to substitute PHIs.
2210
2211 // if C3 != 0 ...
2212 SDValue Sub2_Lo = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub1_Lo,
2213 RHS_Lo, Zero1);
2214 SDValue Sub2_Mi = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub1_Mi,
2215 RHS_Hi, Sub1_Lo.getValue(1));
2216 SDValue Sub2_Hi = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub2_Mi,
2217 Zero, Sub2_Lo.getValue(1));
2218 SDValue Sub2 = DAG.getBitcast(VT,
2219 DAG.getBuildVector(MVT::v2i32, DL, {Sub2_Lo, Sub2_Hi}));
2220
2221 SDValue Add3 = DAG.getNode(ISD::ADD, DL, VT, Mulhi3, One64);
2222
2223 SDValue C4 = DAG.getSelectCC(DL, Sub2_Hi, RHS_Hi, MinusOne, Zero,
2224 ISD::SETUGE);
2225 SDValue C5 = DAG.getSelectCC(DL, Sub2_Lo, RHS_Lo, MinusOne, Zero,
2226 ISD::SETUGE);
2227 SDValue C6 = DAG.getSelectCC(DL, Sub2_Hi, RHS_Hi, C5, C4, ISD::SETEQ);
2228
2229 // if (C6 != 0)
2230 SDValue Add4 = DAG.getNode(ISD::ADD, DL, VT, Add3, One64);
2231
2232 SDValue Sub3_Lo = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub2_Lo,
2233 RHS_Lo, Zero1);
2234 SDValue Sub3_Mi = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub2_Mi,
2235 RHS_Hi, Sub2_Lo.getValue(1));
2236 SDValue Sub3_Hi = DAG.getNode(ISD::USUBO_CARRY, DL, HalfCarryVT, Sub3_Mi,
2237 Zero, Sub3_Lo.getValue(1));
2238 SDValue Sub3 = DAG.getBitcast(VT,
2239 DAG.getBuildVector(MVT::v2i32, DL, {Sub3_Lo, Sub3_Hi}));
2240
2241 // endif C6
2242 // endif C3
2243
2244 SDValue Sel1 = DAG.getSelectCC(DL, C6, Zero, Add4, Add3, ISD::SETNE);
2245 SDValue Div = DAG.getSelectCC(DL, C3, Zero, Sel1, Mulhi3, ISD::SETNE);
2246
2247 SDValue Sel2 = DAG.getSelectCC(DL, C6, Zero, Sub3, Sub2, ISD::SETNE);
2248 SDValue Rem = DAG.getSelectCC(DL, C3, Zero, Sel2, Sub1, ISD::SETNE);
2249
2250 Results.push_back(Div);
2251 Results.push_back(Rem);
2252
2253 return;
2254 }
2255
2256 // r600 expandion.
2257 // Get Speculative values
2258 SDValue DIV_Part = DAG.getNode(ISD::UDIV, DL, HalfVT, LHS_Hi, RHS_Lo);
2259 SDValue REM_Part = DAG.getNode(ISD::UREM, DL, HalfVT, LHS_Hi, RHS_Lo);
2260
2261 SDValue REM_Lo = DAG.getSelectCC(DL, RHS_Hi, Zero, REM_Part, LHS_Hi, ISD::SETEQ);
2262 SDValue REM = DAG.getBuildVector(MVT::v2i32, DL, {REM_Lo, Zero});
2263 REM = DAG.getNode(ISD::BITCAST, DL, MVT::i64, REM);
2264
2265 SDValue DIV_Hi = DAG.getSelectCC(DL, RHS_Hi, Zero, DIV_Part, Zero, ISD::SETEQ);
2266 SDValue DIV_Lo = Zero;
2267
2268 const unsigned halfBitWidth = HalfVT.getSizeInBits();
2269
2270 for (unsigned i = 0; i < halfBitWidth; ++i) {
2271 const unsigned bitPos = halfBitWidth - i - 1;
2272 SDValue POS = DAG.getConstant(bitPos, DL, HalfVT);
2273 // Get value of high bit
2274 SDValue HBit = DAG.getNode(ISD::SRL, DL, HalfVT, LHS_Lo, POS);
2275 HBit = DAG.getNode(ISD::AND, DL, HalfVT, HBit, One);
2276 HBit = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, HBit);
2277
2278 // Shift
2279 REM = DAG.getNode(ISD::SHL, DL, VT, REM, DAG.getConstant(1, DL, VT));
2280 // Add LHS high bit
2281 REM = DAG.getNode(ISD::OR, DL, VT, REM, HBit);
2282
2283 SDValue BIT = DAG.getConstant(1ULL << bitPos, DL, HalfVT);
2284 SDValue realBIT = DAG.getSelectCC(DL, REM, RHS, BIT, Zero, ISD::SETUGE);
2285
2286 DIV_Lo = DAG.getNode(ISD::OR, DL, HalfVT, DIV_Lo, realBIT);
2287
2288 // Update REM
2289 SDValue REM_sub = DAG.getNode(ISD::SUB, DL, VT, REM, RHS);
2290 REM = DAG.getSelectCC(DL, REM, RHS, REM_sub, REM, ISD::SETUGE);
2291 }
2292
2293 SDValue DIV = DAG.getBuildVector(MVT::v2i32, DL, {DIV_Lo, DIV_Hi});
2294 DIV = DAG.getNode(ISD::BITCAST, DL, MVT::i64, DIV);
2295 Results.push_back(DIV);
2296 Results.push_back(REM);
2297}
2298
2300 SelectionDAG &DAG) const {
2301 SDLoc DL(Op);
2302 EVT VT = Op.getValueType();
2303
2304 if (VT == MVT::i64) {
2306 LowerUDIVREM64(Op, DAG, Results);
2307 return DAG.getMergeValues(Results, DL);
2308 }
2309
2310 if (VT == MVT::i32) {
2311 if (SDValue Res = LowerDIVREMToFloat(Op, DAG, false))
2312 return Res;
2313 }
2314
2315 SDValue X = Op.getOperand(0);
2316 SDValue Y = Op.getOperand(1);
2317
2318 // See AMDGPUCodeGenPrepare::expandDivRem32 for a description of the
2319 // algorithm used here.
2320
2321 // Initial estimate of inv(y).
2322 SDValue Z = DAG.getNode(AMDGPUISD::URECIP, DL, VT, Y);
2323
2324 // One round of UNR.
2325 SDValue NegY = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Y);
2326 SDValue NegYZ = DAG.getNode(ISD::MUL, DL, VT, NegY, Z);
2327 Z = DAG.getNode(ISD::ADD, DL, VT, Z,
2328 DAG.getNode(ISD::MULHU, DL, VT, Z, NegYZ));
2329
2330 // Quotient/remainder estimate.
2331 SDValue Q = DAG.getNode(ISD::MULHU, DL, VT, X, Z);
2332 SDValue R =
2333 DAG.getNode(ISD::SUB, DL, VT, X, DAG.getNode(ISD::MUL, DL, VT, Q, Y));
2334
2335 // First quotient/remainder refinement.
2336 EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
2337 SDValue One = DAG.getConstant(1, DL, VT);
2338 SDValue Cond = DAG.getSetCC(DL, CCVT, R, Y, ISD::SETUGE);
2339 Q = DAG.getNode(ISD::SELECT, DL, VT, Cond,
2340 DAG.getNode(ISD::ADD, DL, VT, Q, One), Q);
2341 R = DAG.getNode(ISD::SELECT, DL, VT, Cond,
2342 DAG.getNode(ISD::SUB, DL, VT, R, Y), R);
2343
2344 // Second quotient/remainder refinement.
2345 Cond = DAG.getSetCC(DL, CCVT, R, Y, ISD::SETUGE);
2346 Q = DAG.getNode(ISD::SELECT, DL, VT, Cond,
2347 DAG.getNode(ISD::ADD, DL, VT, Q, One), Q);
2348 R = DAG.getNode(ISD::SELECT, DL, VT, Cond,
2349 DAG.getNode(ISD::SUB, DL, VT, R, Y), R);
2350
2351 return DAG.getMergeValues({Q, R}, DL);
2352}
2353
2355 SelectionDAG &DAG) const {
2356 SDLoc DL(Op);
2357 EVT VT = Op.getValueType();
2358
2359 SDValue LHS = Op.getOperand(0);
2360 SDValue RHS = Op.getOperand(1);
2361
2362 SDValue Zero = DAG.getConstant(0, DL, VT);
2363 SDValue NegOne = DAG.getAllOnesConstant(DL, VT);
2364
2365 if (VT == MVT::i32) {
2366 if (SDValue Res = LowerDIVREMToFloat(Op, DAG, true))
2367 return Res;
2368 }
2369
2370 // LHS must have > 33 sign-bits to ensure that LHS != -2147483648
2371 // Otherwise 32-bit division cannot be used safely.
2372 // -2147483648/1 and -2147483648/-1 are not equal,
2373 // but they produce the same lower 32-bit result.
2374 if (VT == MVT::i64 && DAG.ComputeNumSignBits(LHS) > 33 &&
2375 DAG.ComputeNumSignBits(RHS) > 32) {
2376 EVT HalfVT = VT.getHalfSizedIntegerVT(*DAG.getContext());
2377
2378 //HiLo split
2379 SDValue LHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, LHS, Zero);
2380 SDValue RHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, RHS, Zero);
2381 SDValue DIVREM = DAG.getNode(ISD::SDIVREM, DL, DAG.getVTList(HalfVT, HalfVT),
2382 LHS_Lo, RHS_Lo);
2383 SDValue Res[2] = {
2384 DAG.getNode(ISD::SIGN_EXTEND, DL, VT, DIVREM.getValue(0)),
2385 DAG.getNode(ISD::SIGN_EXTEND, DL, VT, DIVREM.getValue(1))
2386 };
2387 return DAG.getMergeValues(Res, DL);
2388 }
2389
2390 SDValue LHSign = DAG.getSelectCC(DL, LHS, Zero, NegOne, Zero, ISD::SETLT);
2391 SDValue RHSign = DAG.getSelectCC(DL, RHS, Zero, NegOne, Zero, ISD::SETLT);
2392 SDValue DSign = DAG.getNode(ISD::XOR, DL, VT, LHSign, RHSign);
2393 SDValue RSign = LHSign; // Remainder sign is the same as LHS
2394
2395 LHS = DAG.getNode(ISD::ADD, DL, VT, LHS, LHSign);
2396 RHS = DAG.getNode(ISD::ADD, DL, VT, RHS, RHSign);
2397
2398 LHS = DAG.getNode(ISD::XOR, DL, VT, LHS, LHSign);
2399 RHS = DAG.getNode(ISD::XOR, DL, VT, RHS, RHSign);
2400
2401 SDValue Div = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(VT, VT), LHS, RHS);
2402 SDValue Rem = Div.getValue(1);
2403
2404 Div = DAG.getNode(ISD::XOR, DL, VT, Div, DSign);
2405 Rem = DAG.getNode(ISD::XOR, DL, VT, Rem, RSign);
2406
2407 Div = DAG.getNode(ISD::SUB, DL, VT, Div, DSign);
2408 Rem = DAG.getNode(ISD::SUB, DL, VT, Rem, RSign);
2409
2410 SDValue Res[2] = {
2411 Div,
2412 Rem
2413 };
2414 return DAG.getMergeValues(Res, DL);
2415}
2416
2418 SDLoc SL(Op);
2419 SDValue Src = Op.getOperand(0);
2420
2421 // result = trunc(src)
2422 // if (src > 0.0 && src != result)
2423 // result += 1.0
2424
2425 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src);
2426
2427 const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f64);
2428 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f64);
2429
2430 EVT SetCCVT =
2431 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f64);
2432
2433 SDValue Lt0 = DAG.getSetCC(SL, SetCCVT, Src, Zero, ISD::SETOGT);
2434 SDValue NeTrunc = DAG.getSetCC(SL, SetCCVT, Src, Trunc, ISD::SETONE);
2435 SDValue And = DAG.getNode(ISD::AND, SL, SetCCVT, Lt0, NeTrunc);
2436
2437 SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, One, Zero);
2438 // TODO: Should this propagate fast-math-flags?
2439 return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add);
2440}
2441
2443 SelectionDAG &DAG) {
2444 const unsigned FractBits = 52;
2445 const unsigned ExpBits = 11;
2446
2447 SDValue ExpPart = DAG.getNode(AMDGPUISD::BFE_U32, SL, MVT::i32,
2448 Hi,
2449 DAG.getConstant(FractBits - 32, SL, MVT::i32),
2450 DAG.getConstant(ExpBits, SL, MVT::i32));
2451 SDValue Exp = DAG.getNode(ISD::SUB, SL, MVT::i32, ExpPart,
2452 DAG.getConstant(1023, SL, MVT::i32));
2453
2454 return Exp;
2455}
2456
2458 SDLoc SL(Op);
2459 SDValue Src = Op.getOperand(0);
2460
2461 assert(Op.getValueType() == MVT::f64);
2462
2463 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
2464
2465 // Extract the upper half, since this is where we will find the sign and
2466 // exponent.
2467 SDValue Hi = getHiHalf64(Src, DAG);
2468
2469 SDValue Exp = extractF64Exponent(Hi, SL, DAG);
2470
2471 const unsigned FractBits = 52;
2472
2473 // Extract the sign bit.
2474 const SDValue SignBitMask = DAG.getConstant(UINT32_C(1) << 31, SL, MVT::i32);
2475 SDValue SignBit = DAG.getNode(ISD::AND, SL, MVT::i32, Hi, SignBitMask);
2476
2477 // Extend back to 64-bits.
2478 SDValue SignBit64 = DAG.getBuildVector(MVT::v2i32, SL, {Zero, SignBit});
2479 SignBit64 = DAG.getNode(ISD::BITCAST, SL, MVT::i64, SignBit64);
2480
2481 SDValue BcInt = DAG.getNode(ISD::BITCAST, SL, MVT::i64, Src);
2482 const SDValue FractMask
2483 = DAG.getConstant((UINT64_C(1) << FractBits) - 1, SL, MVT::i64);
2484
2485 SDValue Shr = DAG.getNode(ISD::SRA, SL, MVT::i64, FractMask, Exp);
2486 SDValue Not = DAG.getNOT(SL, Shr, MVT::i64);
2487 SDValue Tmp0 = DAG.getNode(ISD::AND, SL, MVT::i64, BcInt, Not);
2488
2489 EVT SetCCVT =
2490 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i32);
2491
2492 const SDValue FiftyOne = DAG.getConstant(FractBits - 1, SL, MVT::i32);
2493
2494 SDValue ExpLt0 = DAG.getSetCC(SL, SetCCVT, Exp, Zero, ISD::SETLT);
2495 SDValue ExpGt51 = DAG.getSetCC(SL, SetCCVT, Exp, FiftyOne, ISD::SETGT);
2496
2497 SDValue Tmp1 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpLt0, SignBit64, Tmp0);
2498 SDValue Tmp2 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpGt51, BcInt, Tmp1);
2499
2500 return DAG.getNode(ISD::BITCAST, SL, MVT::f64, Tmp2);
2501}
2502
2504 SelectionDAG &DAG) const {
2505 SDLoc SL(Op);
2506 SDValue Src = Op.getOperand(0);
2507
2508 assert(Op.getValueType() == MVT::f64);
2509
2510 APFloat C1Val(APFloat::IEEEdouble(), "0x1.0p+52");
2511 SDValue C1 = DAG.getConstantFP(C1Val, SL, MVT::f64);
2512 SDValue CopySign = DAG.getNode(ISD::FCOPYSIGN, SL, MVT::f64, C1, Src);
2513
2514 // TODO: Should this propagate fast-math-flags?
2515
2516 SDValue Tmp1 = DAG.getNode(ISD::FADD, SL, MVT::f64, Src, CopySign);
2517 SDValue Tmp2 = DAG.getNode(ISD::FSUB, SL, MVT::f64, Tmp1, CopySign);
2518
2519 SDValue Fabs = DAG.getNode(ISD::FABS, SL, MVT::f64, Src);
2520
2521 APFloat C2Val(APFloat::IEEEdouble(), "0x1.fffffffffffffp+51");
2522 SDValue C2 = DAG.getConstantFP(C2Val, SL, MVT::f64);
2523
2524 EVT SetCCVT =
2525 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f64);
2526 SDValue Cond = DAG.getSetCC(SL, SetCCVT, Fabs, C2, ISD::SETOGT);
2527
2528 return DAG.getSelect(SL, MVT::f64, Cond, Src, Tmp2);
2529}
2530
2532 SelectionDAG &DAG) const {
2533 // FNEARBYINT and FRINT are the same, except in their handling of FP
2534 // exceptions. Those aren't really meaningful for us, and OpenCL only has
2535 // rint, so just treat them as equivalent.
2536 return DAG.getNode(ISD::FROUNDEVEN, SDLoc(Op), Op.getValueType(),
2537 Op.getOperand(0));
2538}
2539
2541 auto VT = Op.getValueType();
2542 auto Arg = Op.getOperand(0u);
2543 return DAG.getNode(ISD::FROUNDEVEN, SDLoc(Op), VT, Arg);
2544}
2545
2546// XXX - May require not supporting f32 denormals?
2547
2548// Don't handle v2f16. The extra instructions to scalarize and repack around the
2549// compare and vselect end up producing worse code than scalarizing the whole
2550// operation.
2552 SDLoc SL(Op);
2553 SDValue X = Op.getOperand(0);
2554 EVT VT = Op.getValueType();
2555
2556 SDValue T = DAG.getNode(ISD::FTRUNC, SL, VT, X);
2557
2558 // TODO: Should this propagate fast-math-flags?
2559
2560 SDValue Diff = DAG.getNode(ISD::FSUB, SL, VT, X, T);
2561
2562 SDValue AbsDiff = DAG.getNode(ISD::FABS, SL, VT, Diff);
2563
2564 const SDValue Zero = DAG.getConstantFP(0.0, SL, VT);
2565 const SDValue One = DAG.getConstantFP(1.0, SL, VT);
2566
2567 EVT SetCCVT =
2568 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
2569
2570 const SDValue Half = DAG.getConstantFP(0.5, SL, VT);
2571 SDValue Cmp = DAG.getSetCC(SL, SetCCVT, AbsDiff, Half, ISD::SETOGE);
2572 SDValue OneOrZeroFP = DAG.getNode(ISD::SELECT, SL, VT, Cmp, One, Zero);
2573
2574 SDValue SignedOffset = DAG.getNode(ISD::FCOPYSIGN, SL, VT, OneOrZeroFP, X);
2575 return DAG.getNode(ISD::FADD, SL, VT, T, SignedOffset);
2576}
2577
2579 SDLoc SL(Op);
2580 SDValue Src = Op.getOperand(0);
2581
2582 // result = trunc(src);
2583 // if (src < 0.0 && src != result)
2584 // result += -1.0.
2585
2586 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src);
2587
2588 const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f64);
2589 const SDValue NegOne = DAG.getConstantFP(-1.0, SL, MVT::f64);
2590
2591 EVT SetCCVT =
2592 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f64);
2593
2594 SDValue Lt0 = DAG.getSetCC(SL, SetCCVT, Src, Zero, ISD::SETOLT);
2595 SDValue NeTrunc = DAG.getSetCC(SL, SetCCVT, Src, Trunc, ISD::SETONE);
2596 SDValue And = DAG.getNode(ISD::AND, SL, SetCCVT, Lt0, NeTrunc);
2597
2598 SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, NegOne, Zero);
2599 // TODO: Should this propagate fast-math-flags?
2600 return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add);
2601}
2602
2603/// Return true if it's known that \p Src can never be an f32 denormal value.
2605 switch (Src.getOpcode()) {
2606 case ISD::FP_EXTEND:
2607 return Src.getOperand(0).getValueType() == MVT::f16;
2608 case ISD::FP16_TO_FP:
2609 case ISD::FFREXP:
2610 case ISD::FSQRT:
2611 case AMDGPUISD::LOG:
2612 case AMDGPUISD::EXP:
2613 return true;
2615 unsigned IntrinsicID = Src.getConstantOperandVal(0);
2616 switch (IntrinsicID) {
2617 case Intrinsic::amdgcn_frexp_mant:
2618 case Intrinsic::amdgcn_log:
2619 case Intrinsic::amdgcn_log_clamp:
2620 case Intrinsic::amdgcn_exp2:
2621 case Intrinsic::amdgcn_sqrt:
2622 return true;
2623 default:
2624 return false;
2625 }
2626 }
2627 default:
2628 return false;
2629 }
2630
2631 llvm_unreachable("covered opcode switch");
2632}
2633
2635 SDNodeFlags Flags) {
2636 return Flags.hasApproximateFuncs();
2637}
2638
2647
2649 SDNodeFlags Flags) const {
2650 SDLoc SL(Src);
2651 EVT VT = Src.getValueType();
2652 const fltSemantics &Semantics = VT.getFltSemantics();
2653 SDValue Inf = DAG.getConstantFP(APFloat::getInf(Semantics), SL, VT);
2654
2655 SDValue Fabs = DAG.getNode(ISD::FABS, SL, VT, Src, Flags);
2656 SDValue IsFinite = DAG.getSetCC(
2657 SL, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT), Fabs,
2658 Inf, ISD::SETOLT);
2659 return IsFinite;
2660}
2661
2662/// If denormal handling is required return the scaled input to FLOG2, and the
2663/// check for denormal range. Otherwise, return null values.
2664std::pair<SDValue, SDValue>
2666 SDValue Src, SDNodeFlags Flags) const {
2667 if (!needsDenormHandlingF32(DAG, Src, Flags))
2668 return {};
2669
2670 MVT VT = MVT::f32;
2671 const fltSemantics &Semantics = APFloat::IEEEsingle();
2672 SDValue SmallestNormal =
2673 DAG.getConstantFP(APFloat::getSmallestNormalized(Semantics), SL, VT);
2674
2675 SDValue IsLtSmallestNormal = DAG.getSetCC(
2676 SL, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT), Src,
2677 SmallestNormal, ISD::SETOLT);
2678
2679 SDValue Scale32 = DAG.getConstantFP(0x1.0p+32, SL, VT);
2680 SDValue One = DAG.getConstantFP(1.0, SL, VT);
2681 SDValue ScaleFactor =
2682 DAG.getNode(ISD::SELECT, SL, VT, IsLtSmallestNormal, Scale32, One, Flags);
2683
2684 SDValue ScaledInput = DAG.getNode(ISD::FMUL, SL, VT, Src, ScaleFactor, Flags);
2685 return {ScaledInput, IsLtSmallestNormal};
2686}
2687
2689 // v_log_f32 is good enough for OpenCL, except it doesn't handle denormals.
2690 // If we have to handle denormals, scale up the input and adjust the result.
2691
2692 // scaled = x * (is_denormal ? 0x1.0p+32 : 1.0)
2693 // log2 = amdgpu_log2 - (is_denormal ? 32.0 : 0.0)
2694
2695 SDLoc SL(Op);
2696 EVT VT = Op.getValueType();
2697 SDValue Src = Op.getOperand(0);
2698 SDNodeFlags Flags = Op->getFlags();
2699
2700 if (VT == MVT::f16) {
2701 // Nothing in half is a denormal when promoted to f32.
2702 assert(!isTypeLegal(VT));
2703 SDValue Ext = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src, Flags);
2704 SDValue Log = DAG.getNode(AMDGPUISD::LOG, SL, MVT::f32, Ext, Flags);
2705 return DAG.getNode(ISD::FP_ROUND, SL, VT, Log,
2706 DAG.getTargetConstant(0, SL, MVT::i32), Flags);
2707 }
2708
2709 auto [ScaledInput, IsLtSmallestNormal] =
2710 getScaledLogInput(DAG, SL, Src, Flags);
2711 if (!ScaledInput)
2712 return DAG.getNode(AMDGPUISD::LOG, SL, VT, Src, Flags);
2713
2714 SDValue Log2 = DAG.getNode(AMDGPUISD::LOG, SL, VT, ScaledInput, Flags);
2715
2716 SDValue ThirtyTwo = DAG.getConstantFP(32.0, SL, VT);
2717 SDValue Zero = DAG.getConstantFP(0.0, SL, VT);
2718 SDValue ResultOffset =
2719 DAG.getNode(ISD::SELECT, SL, VT, IsLtSmallestNormal, ThirtyTwo, Zero);
2720 return DAG.getNode(ISD::FSUB, SL, VT, Log2, ResultOffset, Flags);
2721}
2722
2723static SDValue getMad(SelectionDAG &DAG, const SDLoc &SL, EVT VT, SDValue X,
2724 SDValue Y, SDValue C, SDNodeFlags Flags = SDNodeFlags()) {
2725 SDValue Mul = DAG.getNode(ISD::FMUL, SL, VT, X, Y, Flags);
2726 return DAG.getNode(ISD::FADD, SL, VT, Mul, C, Flags);
2727}
2728
2730 SelectionDAG &DAG) const {
2731 SDValue X = Op.getOperand(0);
2732 EVT VT = Op.getValueType();
2733 SDNodeFlags Flags = Op->getFlags();
2734 SDLoc DL(Op);
2735 const bool IsLog10 = Op.getOpcode() == ISD::FLOG10;
2736 assert(IsLog10 || Op.getOpcode() == ISD::FLOG);
2737
2738 if (VT == MVT::f16 || Flags.hasApproximateFuncs()) {
2739 // TODO: The direct f16 path is 1.79 ulp for f16. This should be used
2740 // depending on !fpmath metadata.
2741
2742 bool PromoteToF32 = VT == MVT::f16 && (!Flags.hasApproximateFuncs() ||
2743 !isTypeLegal(MVT::f16));
2744
2745 if (PromoteToF32) {
2746 // Log and multiply in f32 is always good enough for f16.
2747 X = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, X, Flags);
2748 }
2749
2750 SDValue Lowered = LowerFLOGUnsafe(X, DL, DAG, IsLog10, Flags);
2751 if (PromoteToF32) {
2752 return DAG.getNode(ISD::FP_ROUND, DL, VT, Lowered,
2753 DAG.getTargetConstant(0, DL, MVT::i32), Flags);
2754 }
2755
2756 return Lowered;
2757 }
2758
2759 SDValue ScaledInput, IsScaled;
2760 if (VT == MVT::f16)
2761 X = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, X, Flags);
2762 else {
2763 std::tie(ScaledInput, IsScaled) = getScaledLogInput(DAG, DL, X, Flags);
2764 if (ScaledInput)
2765 X = ScaledInput;
2766 }
2767
2768 SDValue Y = DAG.getNode(AMDGPUISD::LOG, DL, VT, X, Flags);
2769
2770 SDValue R;
2771 if (Subtarget->hasFastFMAF32()) {
2772 // c+cc are ln(2)/ln(10) to more than 49 bits
2773 const float c_log10 = 0x1.344134p-2f;
2774 const float cc_log10 = 0x1.09f79ep-26f;
2775
2776 // c + cc is ln(2) to more than 49 bits
2777 const float c_log = 0x1.62e42ep-1f;
2778 const float cc_log = 0x1.efa39ep-25f;
2779
2780 SDValue C = DAG.getConstantFP(IsLog10 ? c_log10 : c_log, DL, VT);
2781 SDValue CC = DAG.getConstantFP(IsLog10 ? cc_log10 : cc_log, DL, VT);
2782 // This adds correction terms for which contraction may lead to an increase
2783 // in the error of the approximation, so disable it.
2784 Flags.setAllowContract(false);
2785 R = DAG.getNode(ISD::FMUL, DL, VT, Y, C, Flags);
2786 SDValue NegR = DAG.getNode(ISD::FNEG, DL, VT, R, Flags);
2787 SDValue FMA0 = DAG.getNode(ISD::FMA, DL, VT, Y, C, NegR, Flags);
2788 SDValue FMA1 = DAG.getNode(ISD::FMA, DL, VT, Y, CC, FMA0, Flags);
2789 R = DAG.getNode(ISD::FADD, DL, VT, R, FMA1, Flags);
2790 } else {
2791 // ch+ct is ln(2)/ln(10) to more than 36 bits
2792 const float ch_log10 = 0x1.344000p-2f;
2793 const float ct_log10 = 0x1.3509f6p-18f;
2794
2795 // ch + ct is ln(2) to more than 36 bits
2796 const float ch_log = 0x1.62e000p-1f;
2797 const float ct_log = 0x1.0bfbe8p-15f;
2798
2799 SDValue CH = DAG.getConstantFP(IsLog10 ? ch_log10 : ch_log, DL, VT);
2800 SDValue CT = DAG.getConstantFP(IsLog10 ? ct_log10 : ct_log, DL, VT);
2801
2802 SDValue YAsInt = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Y);
2803 SDValue MaskConst = DAG.getConstant(0xfffff000, DL, MVT::i32);
2804 SDValue YHInt = DAG.getNode(ISD::AND, DL, MVT::i32, YAsInt, MaskConst);
2805 SDValue YH = DAG.getNode(ISD::BITCAST, DL, MVT::f32, YHInt);
2806 SDValue YT = DAG.getNode(ISD::FSUB, DL, VT, Y, YH, Flags);
2807 // This adds correction terms for which contraction may lead to an increase
2808 // in the error of the approximation, so disable it.
2809 Flags.setAllowContract(false);
2810 SDValue YTCT = DAG.getNode(ISD::FMUL, DL, VT, YT, CT, Flags);
2811 SDValue Mad0 = getMad(DAG, DL, VT, YH, CT, YTCT, Flags);
2812 SDValue Mad1 = getMad(DAG, DL, VT, YT, CH, Mad0, Flags);
2813 R = getMad(DAG, DL, VT, YH, CH, Mad1);
2814 }
2815
2816 const bool IsFiniteOnly = Flags.hasNoNaNs() && Flags.hasNoInfs();
2817
2818 // TODO: Check if known finite from source value.
2819 if (!IsFiniteOnly) {
2820 SDValue IsFinite = getIsFinite(DAG, Y, Flags);
2821 R = DAG.getNode(ISD::SELECT, DL, VT, IsFinite, R, Y, Flags);
2822 }
2823
2824 if (IsScaled) {
2825 SDValue Zero = DAG.getConstantFP(0.0f, DL, VT);
2826 SDValue ShiftK =
2827 DAG.getConstantFP(IsLog10 ? 0x1.344136p+3f : 0x1.62e430p+4f, DL, VT);
2828 SDValue Shift =
2829 DAG.getNode(ISD::SELECT, DL, VT, IsScaled, ShiftK, Zero, Flags);
2830 R = DAG.getNode(ISD::FSUB, DL, VT, R, Shift, Flags);
2831 }
2832
2833 return R;
2834}
2835
2836// Do f32 fast math expansion for flog2 or flog10. This is accurate enough for a
2837// promote f16 operation.
2839 SelectionDAG &DAG, bool IsLog10,
2840 SDNodeFlags Flags) const {
2841 EVT VT = Src.getValueType();
2842 unsigned LogOp =
2843 VT == MVT::f32 ? (unsigned)AMDGPUISD::LOG : (unsigned)ISD::FLOG2;
2844
2845 double Log2BaseInverted =
2847
2848 if (VT == MVT::f32) {
2849 auto [ScaledInput, IsScaled] = getScaledLogInput(DAG, SL, Src, Flags);
2850 if (ScaledInput) {
2851 SDValue LogSrc = DAG.getNode(AMDGPUISD::LOG, SL, VT, ScaledInput, Flags);
2852 SDValue ScaledResultOffset =
2853 DAG.getConstantFP(-32.0 * Log2BaseInverted, SL, VT);
2854
2855 SDValue Zero = DAG.getConstantFP(0.0f, SL, VT);
2856
2857 SDValue ResultOffset = DAG.getNode(ISD::SELECT, SL, VT, IsScaled,
2858 ScaledResultOffset, Zero, Flags);
2859
2860 SDValue Log2Inv = DAG.getConstantFP(Log2BaseInverted, SL, VT);
2861
2862 if (Subtarget->hasFastFMAF32())
2863 return DAG.getNode(ISD::FMA, SL, VT, LogSrc, Log2Inv, ResultOffset,
2864 Flags);
2865 SDValue Mul = DAG.getNode(ISD::FMUL, SL, VT, LogSrc, Log2Inv, Flags);
2866 return DAG.getNode(ISD::FADD, SL, VT, Mul, ResultOffset);
2867 }
2868 }
2869
2870 SDValue Log2Operand = DAG.getNode(LogOp, SL, VT, Src, Flags);
2871 SDValue Log2BaseInvertedOperand = DAG.getConstantFP(Log2BaseInverted, SL, VT);
2872
2873 return DAG.getNode(ISD::FMUL, SL, VT, Log2Operand, Log2BaseInvertedOperand,
2874 Flags);
2875}
2876
2877// This expansion gives a result slightly better than 1ulp.
2879 SelectionDAG &DAG) const {
2880 SDLoc DL(Op);
2881 SDValue X = Op.getOperand(0);
2882
2883 // TODO: Check if reassoc is safe. There is an output change in exp2 and
2884 // exp10, which slightly increases ulp.
2885 SDNodeFlags Flags = Op->getFlags() & ~SDNodeFlags::AllowReassociation;
2886
2887 SDValue DN, F, T;
2888
2889 if (Op.getOpcode() == ISD::FEXP2) {
2890 // dn = rint(x)
2891 DN = DAG.getNode(ISD::FRINT, DL, MVT::f64, X, Flags);
2892 // f = x - dn
2893 F = DAG.getNode(ISD::FSUB, DL, MVT::f64, X, DN, Flags);
2894 // t = f*C1 + f*C2
2895 SDValue C1 = DAG.getConstantFP(0x1.62e42fefa39efp-1, DL, MVT::f64);
2896 SDValue C2 = DAG.getConstantFP(0x1.abc9e3b39803fp-56, DL, MVT::f64);
2897 SDValue Mul2 = DAG.getNode(ISD::FMUL, DL, MVT::f64, F, C2, Flags);
2898 T = DAG.getNode(ISD::FMA, DL, MVT::f64, F, C1, Mul2, Flags);
2899 } else if (Op.getOpcode() == ISD::FEXP10) {
2900 // dn = rint(x * C1)
2901 SDValue C1 = DAG.getConstantFP(0x1.a934f0979a371p+1, DL, MVT::f64);
2902 SDValue Mul = DAG.getNode(ISD::FMUL, DL, MVT::f64, X, C1, Flags);
2903 DN = DAG.getNode(ISD::FRINT, DL, MVT::f64, Mul, Flags);
2904
2905 // f = FMA(-dn, C2, FMA(-dn, C3, x))
2906 SDValue NegDN = DAG.getNode(ISD::FNEG, DL, MVT::f64, DN, Flags);
2907 SDValue C2 = DAG.getConstantFP(-0x1.9dc1da994fd21p-59, DL, MVT::f64);
2908 SDValue C3 = DAG.getConstantFP(0x1.34413509f79ffp-2, DL, MVT::f64);
2909 SDValue Inner = DAG.getNode(ISD::FMA, DL, MVT::f64, NegDN, C3, X, Flags);
2910 F = DAG.getNode(ISD::FMA, DL, MVT::f64, NegDN, C2, Inner, Flags);
2911
2912 // t = FMA(f, C4, f*C5)
2913 SDValue C4 = DAG.getConstantFP(0x1.26bb1bbb55516p+1, DL, MVT::f64);
2914 SDValue C5 = DAG.getConstantFP(-0x1.f48ad494ea3e9p-53, DL, MVT::f64);
2915 SDValue MulF = DAG.getNode(ISD::FMUL, DL, MVT::f64, F, C5, Flags);
2916 T = DAG.getNode(ISD::FMA, DL, MVT::f64, F, C4, MulF, Flags);
2917 } else { // ISD::FEXP
2918 // dn = rint(x * C1)
2919 SDValue C1 = DAG.getConstantFP(0x1.71547652b82fep+0, DL, MVT::f64);
2920 SDValue Mul = DAG.getNode(ISD::FMUL, DL, MVT::f64, X, C1, Flags);
2921 DN = DAG.getNode(ISD::FRINT, DL, MVT::f64, Mul, Flags);
2922
2923 // t = FMA(-dn, C2, FMA(-dn, C3, x))
2924 SDValue NegDN = DAG.getNode(ISD::FNEG, DL, MVT::f64, DN, Flags);
2925 SDValue C2 = DAG.getConstantFP(0x1.abc9e3b39803fp-56, DL, MVT::f64);
2926 SDValue C3 = DAG.getConstantFP(0x1.62e42fefa39efp-1, DL, MVT::f64);
2927 SDValue Inner = DAG.getNode(ISD::FMA, DL, MVT::f64, NegDN, C3, X, Flags);
2928 T = DAG.getNode(ISD::FMA, DL, MVT::f64, NegDN, C2, Inner, Flags);
2929 }
2930
2931 // Polynomial expansion for p
2932 SDValue P = DAG.getConstantFP(0x1.ade156a5dcb37p-26, DL, MVT::f64);
2933 P = DAG.getNode(ISD::FMA, DL, MVT::f64, T, P,
2934 DAG.getConstantFP(0x1.28af3fca7ab0cp-22, DL, MVT::f64),
2935 Flags);
2936 P = DAG.getNode(ISD::FMA, DL, MVT::f64, T, P,
2937 DAG.getConstantFP(0x1.71dee623fde64p-19, DL, MVT::f64),
2938 Flags);
2939 P = DAG.getNode(ISD::FMA, DL, MVT::f64, T, P,
2940 DAG.getConstantFP(0x1.a01997c89e6b0p-16, DL, MVT::f64),
2941 Flags);
2942 P = DAG.getNode(ISD::FMA, DL, MVT::f64, T, P,
2943 DAG.getConstantFP(0x1.a01a014761f6ep-13, DL, MVT::f64),
2944 Flags);
2945 P = DAG.getNode(ISD::FMA, DL, MVT::f64, T, P,
2946 DAG.getConstantFP(0x1.6c16c1852b7b0p-10, DL, MVT::f64),
2947 Flags);
2948 P = DAG.getNode(ISD::FMA, DL, MVT::f64, T, P,
2949 DAG.getConstantFP(0x1.1111111122322p-7, DL, MVT::f64), Flags);
2950 P = DAG.getNode(ISD::FMA, DL, MVT::f64, T, P,
2951 DAG.getConstantFP(0x1.55555555502a1p-5, DL, MVT::f64), Flags);
2952 P = DAG.getNode(ISD::FMA, DL, MVT::f64, T, P,
2953 DAG.getConstantFP(0x1.5555555555511p-3, DL, MVT::f64), Flags);
2954 P = DAG.getNode(ISD::FMA, DL, MVT::f64, T, P,
2955 DAG.getConstantFP(0x1.000000000000bp-1, DL, MVT::f64), Flags);
2956
2957 SDValue One = DAG.getConstantFP(1.0, DL, MVT::f64);
2958
2959 P = DAG.getNode(ISD::FMA, DL, MVT::f64, T, P, One, Flags);
2960 P = DAG.getNode(ISD::FMA, DL, MVT::f64, T, P, One, Flags);
2961
2962 // z = ldexp(p, (int)dn)
2963 SDValue DNInt = DAG.getNode(ISD::FP_TO_SINT, DL, MVT::i32, DN);
2964 SDValue Z = DAG.getNode(ISD::FLDEXP, DL, MVT::f64, P, DNInt, Flags);
2965
2966 // Overflow/underflow guards
2967 SDValue CondHi = DAG.getSetCC(
2968 DL, MVT::i1, X, DAG.getConstantFP(1024.0, DL, MVT::f64), ISD::SETULE);
2969
2970 if (!Flags.hasNoInfs()) {
2971 SDValue PInf = DAG.getConstantFP(std::numeric_limits<double>::infinity(),
2972 DL, MVT::f64);
2973 Z = DAG.getSelect(DL, MVT::f64, CondHi, Z, PInf, Flags);
2974 }
2975
2976 SDValue CondLo = DAG.getSetCC(
2977 DL, MVT::i1, X, DAG.getConstantFP(-1075.0, DL, MVT::f64), ISD::SETUGE);
2978 SDValue Zero = DAG.getConstantFP(0.0, DL, MVT::f64);
2979 Z = DAG.getSelect(DL, MVT::f64, CondLo, Z, Zero, Flags);
2980
2981 return Z;
2982}
2983
2985 // v_exp_f32 is good enough for OpenCL, except it doesn't handle denormals.
2986 // If we have to handle denormals, scale up the input and adjust the result.
2987
2988 EVT VT = Op.getValueType();
2989 if (VT == MVT::f64)
2990 return lowerFEXPF64(Op, DAG);
2991
2992 SDLoc SL(Op);
2993 SDValue Src = Op.getOperand(0);
2994 SDNodeFlags Flags = Op->getFlags();
2995
2996 if (VT == MVT::f16) {
2997 // Nothing in half is a denormal when promoted to f32.
2998 assert(!isTypeLegal(MVT::f16));
2999 SDValue Ext = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src, Flags);
3000 SDValue Log = DAG.getNode(AMDGPUISD::EXP, SL, MVT::f32, Ext, Flags);
3001 return DAG.getNode(ISD::FP_ROUND, SL, VT, Log,
3002 DAG.getTargetConstant(0, SL, MVT::i32), Flags);
3003 }
3004
3005 assert(VT == MVT::f32);
3006
3007 if (!needsDenormHandlingF32(DAG, Src, Flags))
3008 return DAG.getNode(AMDGPUISD::EXP, SL, MVT::f32, Src, Flags);
3009
3010 // bool needs_scaling = x < -0x1.f80000p+6f;
3011 // v_exp_f32(x + (s ? 0x1.0p+6f : 0.0f)) * (s ? 0x1.0p-64f : 1.0f);
3012
3013 // -nextafter(128.0, -1)
3014 SDValue RangeCheckConst = DAG.getConstantFP(-0x1.f80000p+6f, SL, VT);
3015
3016 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
3017
3018 SDValue NeedsScaling =
3019 DAG.getSetCC(SL, SetCCVT, Src, RangeCheckConst, ISD::SETOLT);
3020
3021 SDValue SixtyFour = DAG.getConstantFP(0x1.0p+6f, SL, VT);
3022 SDValue Zero = DAG.getConstantFP(0.0, SL, VT);
3023
3024 SDValue AddOffset =
3025 DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, SixtyFour, Zero);
3026
3027 SDValue AddInput = DAG.getNode(ISD::FADD, SL, VT, Src, AddOffset, Flags);
3028 SDValue Exp2 = DAG.getNode(AMDGPUISD::EXP, SL, VT, AddInput, Flags);
3029
3030 SDValue TwoExpNeg64 = DAG.getConstantFP(0x1.0p-64f, SL, VT);
3031 SDValue One = DAG.getConstantFP(1.0, SL, VT);
3032 SDValue ResultScale =
3033 DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, TwoExpNeg64, One);
3034
3035 return DAG.getNode(ISD::FMUL, SL, VT, Exp2, ResultScale, Flags);
3036}
3037
3039 SelectionDAG &DAG,
3040 SDNodeFlags Flags,
3041 bool IsExp10) const {
3042 // exp(x) -> exp2(M_LOG2E_F * x);
3043 // exp10(x) -> exp2(log2(10) * x);
3044 EVT VT = X.getValueType();
3045 SDValue Const =
3046 DAG.getConstantFP(IsExp10 ? 0x1.a934f0p+1f : numbers::log2e, SL, VT);
3047
3048 SDValue Mul = DAG.getNode(ISD::FMUL, SL, VT, X, Const, Flags);
3049 return DAG.getNode(VT == MVT::f32 ? (unsigned)AMDGPUISD::EXP
3050 : (unsigned)ISD::FEXP2,
3051 SL, VT, Mul, Flags);
3052}
3053
3055 SelectionDAG &DAG,
3056 SDNodeFlags Flags) const {
3057 EVT VT = X.getValueType();
3058 if (VT != MVT::f32 || !needsDenormHandlingF32(DAG, X, Flags))
3059 return lowerFEXPUnsafeImpl(X, SL, DAG, Flags, /*IsExp10=*/false);
3060
3061 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
3062
3063 SDValue Threshold = DAG.getConstantFP(-0x1.5d58a0p+6f, SL, VT);
3064 SDValue NeedsScaling = DAG.getSetCC(SL, SetCCVT, X, Threshold, ISD::SETOLT);
3065
3066 SDValue ScaleOffset = DAG.getConstantFP(0x1.0p+6f, SL, VT);
3067
3068 SDValue ScaledX = DAG.getNode(ISD::FADD, SL, VT, X, ScaleOffset, Flags);
3069
3070 SDValue AdjustedX =
3071 DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, ScaledX, X);
3072
3073 const SDValue Log2E = DAG.getConstantFP(numbers::log2e, SL, VT);
3074 SDValue ExpInput = DAG.getNode(ISD::FMUL, SL, VT, AdjustedX, Log2E, Flags);
3075
3076 SDValue Exp2 = DAG.getNode(AMDGPUISD::EXP, SL, VT, ExpInput, Flags);
3077
3078 SDValue ResultScaleFactor = DAG.getConstantFP(0x1.969d48p-93f, SL, VT);
3079 SDValue AdjustedResult =
3080 DAG.getNode(ISD::FMUL, SL, VT, Exp2, ResultScaleFactor, Flags);
3081
3082 return DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, AdjustedResult, Exp2,
3083 Flags);
3084}
3085
3086/// Emit approx-funcs appropriate lowering for exp10. inf/nan should still be
3087/// handled correctly.
3089 SelectionDAG &DAG,
3090 SDNodeFlags Flags) const {
3091 const EVT VT = X.getValueType();
3092
3093 const unsigned Exp2Op = VT == MVT::f32 ? static_cast<unsigned>(AMDGPUISD::EXP)
3094 : static_cast<unsigned>(ISD::FEXP2);
3095
3096 if (VT != MVT::f32 || !needsDenormHandlingF32(DAG, X, Flags)) {
3097 // exp2(x * 0x1.a92000p+1f) * exp2(x * 0x1.4f0978p-11f);
3098 SDValue K0 = DAG.getConstantFP(0x1.a92000p+1f, SL, VT);
3099 SDValue K1 = DAG.getConstantFP(0x1.4f0978p-11f, SL, VT);
3100
3101 SDValue Mul0 = DAG.getNode(ISD::FMUL, SL, VT, X, K0, Flags);
3102 SDValue Exp2_0 = DAG.getNode(Exp2Op, SL, VT, Mul0, Flags);
3103 SDValue Mul1 = DAG.getNode(ISD::FMUL, SL, VT, X, K1, Flags);
3104 SDValue Exp2_1 = DAG.getNode(Exp2Op, SL, VT, Mul1, Flags);
3105 return DAG.getNode(ISD::FMUL, SL, VT, Exp2_0, Exp2_1);
3106 }
3107
3108 // bool s = x < -0x1.2f7030p+5f;
3109 // x += s ? 0x1.0p+5f : 0.0f;
3110 // exp10 = exp2(x * 0x1.a92000p+1f) *
3111 // exp2(x * 0x1.4f0978p-11f) *
3112 // (s ? 0x1.9f623ep-107f : 1.0f);
3113
3114 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
3115
3116 SDValue Threshold = DAG.getConstantFP(-0x1.2f7030p+5f, SL, VT);
3117 SDValue NeedsScaling = DAG.getSetCC(SL, SetCCVT, X, Threshold, ISD::SETOLT);
3118
3119 SDValue ScaleOffset = DAG.getConstantFP(0x1.0p+5f, SL, VT);
3120 SDValue ScaledX = DAG.getNode(ISD::FADD, SL, VT, X, ScaleOffset, Flags);
3121 SDValue AdjustedX =
3122 DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, ScaledX, X);
3123
3124 SDValue K0 = DAG.getConstantFP(0x1.a92000p+1f, SL, VT);
3125 SDValue K1 = DAG.getConstantFP(0x1.4f0978p-11f, SL, VT);
3126
3127 SDValue Mul0 = DAG.getNode(ISD::FMUL, SL, VT, AdjustedX, K0, Flags);
3128 SDValue Exp2_0 = DAG.getNode(Exp2Op, SL, VT, Mul0, Flags);
3129 SDValue Mul1 = DAG.getNode(ISD::FMUL, SL, VT, AdjustedX, K1, Flags);
3130 SDValue Exp2_1 = DAG.getNode(Exp2Op, SL, VT, Mul1, Flags);
3131
3132 SDValue MulExps = DAG.getNode(ISD::FMUL, SL, VT, Exp2_0, Exp2_1, Flags);
3133
3134 SDValue ResultScaleFactor = DAG.getConstantFP(0x1.9f623ep-107f, SL, VT);
3135 SDValue AdjustedResult =
3136 DAG.getNode(ISD::FMUL, SL, VT, MulExps, ResultScaleFactor, Flags);
3137
3138 return DAG.getNode(ISD::SELECT, SL, VT, NeedsScaling, AdjustedResult, MulExps,
3139 Flags);
3140}
3141
3143 EVT VT = Op.getValueType();
3144
3145 if (VT == MVT::f64)
3146 return lowerFEXPF64(Op, DAG);
3147
3148 SDLoc SL(Op);
3149 SDValue X = Op.getOperand(0);
3150 SDNodeFlags Flags = Op->getFlags();
3151 const bool IsExp10 = Op.getOpcode() == ISD::FEXP10;
3152
3153 // TODO: Interpret allowApproxFunc as ignoring DAZ. This is currently copying
3154 // library behavior. Also, is known-not-daz source sufficient?
3155 if (allowApproxFunc(DAG, Flags)) { // TODO: Does this really require fast?
3156 return IsExp10 ? lowerFEXP10Unsafe(X, SL, DAG, Flags)
3157 : lowerFEXPUnsafe(X, SL, DAG, Flags);
3158 }
3159
3160 if (VT.getScalarType() == MVT::f16) {
3161 if (VT.isVector())
3162 return SDValue();
3163
3164 // Nothing in half is a denormal when promoted to f32.
3165 //
3166 // exp(f16 x) ->
3167 // fptrunc (v_exp_f32 (fmul (fpext x), log2e))
3168 //
3169 // exp10(f16 x) ->
3170 // fptrunc (v_exp_f32 (fmul (fpext x), log2(10)))
3171 SDValue Ext = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, X, Flags);
3172 SDValue Lowered = lowerFEXPUnsafeImpl(Ext, SL, DAG, Flags, IsExp10);
3173 return DAG.getNode(ISD::FP_ROUND, SL, VT, Lowered,
3174 DAG.getTargetConstant(0, SL, MVT::i32), Flags);
3175 }
3176
3177 assert(VT == MVT::f32);
3178
3179 // Algorithm:
3180 //
3181 // e^x = 2^(x/ln(2)) = 2^(x*(64/ln(2))/64)
3182 //
3183 // x*(64/ln(2)) = n + f, |f| <= 0.5, n is integer
3184 // n = 64*m + j, 0 <= j < 64
3185 //
3186 // e^x = 2^((64*m + j + f)/64)
3187 // = (2^m) * (2^(j/64)) * 2^(f/64)
3188 // = (2^m) * (2^(j/64)) * e^(f*(ln(2)/64))
3189 //
3190 // f = x*(64/ln(2)) - n
3191 // r = f*(ln(2)/64) = x - n*(ln(2)/64)
3192 //
3193 // e^x = (2^m) * (2^(j/64)) * e^r
3194 //
3195 // (2^(j/64)) is precomputed
3196 //
3197 // e^r = 1 + r + (r^2)/2! + (r^3)/3! + (r^4)/4! + (r^5)/5!
3198 // e^r = 1 + q
3199 //
3200 // q = r + (r^2)/2! + (r^3)/3! + (r^4)/4! + (r^5)/5!
3201 //
3202 // e^x = (2^m) * ( (2^(j/64)) + q*(2^(j/64)) )
3203 SDNodeFlags FlagsNoContract = Flags;
3204 FlagsNoContract.setAllowContract(false);
3205
3206 SDValue PH, PL;
3207 if (Subtarget->hasFastFMAF32()) {
3208 const float c_exp = numbers::log2ef;
3209 const float cc_exp = 0x1.4ae0bep-26f; // c+cc are 49 bits
3210 const float c_exp10 = 0x1.a934f0p+1f;
3211 const float cc_exp10 = 0x1.2f346ep-24f;
3212
3213 SDValue C = DAG.getConstantFP(IsExp10 ? c_exp10 : c_exp, SL, VT);
3214 SDValue CC = DAG.getConstantFP(IsExp10 ? cc_exp10 : cc_exp, SL, VT);
3215
3216 PH = DAG.getNode(ISD::FMUL, SL, VT, X, C, Flags);
3217 SDValue NegPH = DAG.getNode(ISD::FNEG, SL, VT, PH, Flags);
3218 SDValue FMA0 = DAG.getNode(ISD::FMA, SL, VT, X, C, NegPH, Flags);
3219 PL = DAG.getNode(ISD::FMA, SL, VT, X, CC, FMA0, Flags);
3220 } else {
3221 const float ch_exp = 0x1.714000p+0f;
3222 const float cl_exp = 0x1.47652ap-12f; // ch + cl are 36 bits
3223
3224 const float ch_exp10 = 0x1.a92000p+1f;
3225 const float cl_exp10 = 0x1.4f0978p-11f;
3226
3227 SDValue CH = DAG.getConstantFP(IsExp10 ? ch_exp10 : ch_exp, SL, VT);
3228 SDValue CL = DAG.getConstantFP(IsExp10 ? cl_exp10 : cl_exp, SL, VT);
3229
3230 SDValue XAsInt = DAG.getNode(ISD::BITCAST, SL, MVT::i32, X);
3231 SDValue MaskConst = DAG.getConstant(0xfffff000, SL, MVT::i32);
3232 SDValue XHAsInt = DAG.getNode(ISD::AND, SL, MVT::i32, XAsInt, MaskConst);
3233 SDValue XH = DAG.getNode(ISD::BITCAST, SL, VT, XHAsInt);
3234 SDValue XL = DAG.getNode(ISD::FSUB, SL, VT, X, XH, Flags);
3235
3236 PH = DAG.getNode(ISD::FMUL, SL, VT, XH, CH, Flags);
3237
3238 SDValue XLCL = DAG.getNode(ISD::FMUL, SL, VT, XL, CL, Flags);
3239 SDValue Mad0 = getMad(DAG, SL, VT, XL, CH, XLCL, Flags);
3240 PL = getMad(DAG, SL, VT, XH, CL, Mad0, Flags);
3241 }
3242
3243 SDValue E = DAG.getNode(ISD::FROUNDEVEN, SL, VT, PH, Flags);
3244
3245 // It is unsafe to contract this fsub into the PH multiply.
3246 SDValue PHSubE = DAG.getNode(ISD::FSUB, SL, VT, PH, E, FlagsNoContract);
3247
3248 SDValue A = DAG.getNode(ISD::FADD, SL, VT, PHSubE, PL, Flags);
3249 SDValue IntE = DAG.getNode(ISD::FP_TO_SINT, SL, MVT::i32, E);
3250 SDValue Exp2 = DAG.getNode(AMDGPUISD::EXP, SL, VT, A, Flags);
3251
3252 SDValue R = DAG.getNode(ISD::FLDEXP, SL, VT, Exp2, IntE, Flags);
3253
3254 SDValue UnderflowCheckConst =
3255 DAG.getConstantFP(IsExp10 ? -0x1.66d3e8p+5f : -0x1.9d1da0p+6f, SL, VT);
3256
3257 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
3258 SDValue Zero = DAG.getConstantFP(0.0, SL, VT);
3259 SDValue Underflow =
3260 DAG.getSetCC(SL, SetCCVT, X, UnderflowCheckConst, ISD::SETOLT);
3261
3262 R = DAG.getNode(ISD::SELECT, SL, VT, Underflow, Zero, R);
3263
3264 if (!Flags.hasNoInfs()) {
3265 SDValue OverflowCheckConst =
3266 DAG.getConstantFP(IsExp10 ? 0x1.344136p+5f : 0x1.62e430p+6f, SL, VT);
3267 SDValue Overflow =
3268 DAG.getSetCC(SL, SetCCVT, X, OverflowCheckConst, ISD::SETOGT);
3269 SDValue Inf =
3271 R = DAG.getNode(ISD::SELECT, SL, VT, Overflow, Inf, R);
3272 }
3273
3274 return R;
3275}
3276
3277// No pow instruction or libcall to fall back on. fmul_legacy returns 0 for a
3278// zero operand even against an infinity or a NaN, so pow(x, 0) and pow(1, y)
3279// fall out as exp2(0) = 1.
3281 EVT VT = Op.getValueType();
3282 assert(VT == MVT::f32);
3283
3284 SDLoc SL(Op);
3285 SDValue X = Op.getOperand(0);
3286 SDValue Y = Op.getOperand(1);
3287 SDNodeFlags Flags = Op->getFlags();
3288
3289 // log2(0) is -inf, which exp2 turns back into a finite result, so the core
3290 // goes infinite for inputs a ninf fpow still asserts about, like pow(0, 2).
3291 SDNodeFlags CoreFlags = Flags;
3292 CoreFlags.setNoInfs(false);
3293
3294 // Fast expansion: ignores denormals, NaN for a negative base.
3295 if (allowApproxFunc(DAG, Flags)) {
3296 SDValue Log = DAG.getNode(AMDGPUISD::LOG, SL, VT, X, CoreFlags);
3297 SDValue Mul =
3298 DAG.getNode(AMDGPUISD::FMUL_LEGACY, SL, VT, Y, Log, CoreFlags);
3299 return DAG.getNode(AMDGPUISD::EXP, SL, VT, Mul, CoreFlags);
3300 }
3301
3302 SDValue Abs = DAG.getNode(ISD::FABS, SL, VT, X, Flags);
3303 SDValue Log = DAG.getNode(ISD::FLOG2, SL, VT, Abs, CoreFlags);
3304 SDValue Mul = DAG.getNode(AMDGPUISD::FMUL_LEGACY, SL, VT, Y, Log, CoreFlags);
3305 SDValue R = DAG.getNode(ISD::FEXP2, SL, VT, Mul, CoreFlags);
3306
3307 // A base that is never negative needs neither the sign fixup nor the NaN.
3309 return R;
3310
3311 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
3312
3313 // Infinities count as integers, and every f32 >= 2^24 in magnitude is even.
3314 SDValue YTrunc = DAG.getNode(ISD::FTRUNC, SL, VT, Y);
3315 SDValue YIsInt = DAG.getSetCC(SL, SetCCVT, YTrunc, Y, ISD::SETOEQ);
3316 SDValue YHalf =
3317 DAG.getNode(ISD::FMUL, SL, VT, Y, DAG.getConstantFP(0.5, SL, VT));
3318 SDValue YHalfTrunc = DAG.getNode(ISD::FTRUNC, SL, VT, YHalf);
3319 SDValue YIsOdd =
3320 DAG.getNode(ISD::AND, SL, SetCCVT, YIsInt,
3321 DAG.getSetCC(SL, SetCCVT, YHalfTrunc, YHalf, ISD::SETONE));
3322
3323 // pow(-x, odd y) = -pow(x, y). Selecting the copysign lets even y fold it.
3324 R = DAG.getNode(ISD::SELECT, SL, VT, YIsOdd,
3325 DAG.getNode(ISD::FCOPYSIGN, SL, VT, R, X), R);
3326
3327 if (Flags.hasNoNaNs())
3328 return R;
3329
3330 // A negative finite base to a non-integral power is NaN. -inf is excluded:
3331 // the core already gives pow(+inf, y). So are subnormals when flushed.
3332 FPClassTest NegFiniteMask = fcNegNormal;
3333 if (!DAG.getMachineFunction()
3335 .inputsAreZero())
3336 NegFiniteMask |= fcNegSubnormal;
3337 SDValue XNegFinite =
3338 DAG.getNode(ISD::IS_FPCLASS, SL, SetCCVT, X,
3339 DAG.getTargetConstant(NegFiniteMask, SL, MVT::i32));
3340 // Not a SETONE compare: pow(-1, NaN) needs the NaN-true behavior of !SETOEQ.
3341 SDValue NegNonInt = DAG.getNode(ISD::AND, SL, SetCCVT, XNegFinite,
3342 DAG.getNOT(SL, YIsInt, SetCCVT));
3343 SDValue NaN =
3345 return DAG.getNode(ISD::SELECT, SL, VT, NegNonInt, NaN, R);
3346}
3347
3348static bool isCtlzOpc(unsigned Opc) {
3349 return Opc == ISD::CTLZ || Opc == ISD::CTLZ_ZERO_POISON;
3350}
3351
3352static bool isCttzOpc(unsigned Opc) {
3353 return Opc == ISD::CTTZ || Opc == ISD::CTTZ_ZERO_POISON;
3354}
3355
3357 SelectionDAG &DAG) const {
3358 auto SL = SDLoc(Op);
3359 auto Opc = Op.getOpcode();
3360 auto Arg = Op.getOperand(0u);
3361 auto ResultVT = Op.getValueType();
3362
3363 if (ResultVT != MVT::i8 && ResultVT != MVT::i16)
3364 return {};
3365
3367 assert(ResultVT == Arg.getValueType());
3368
3369 const uint64_t NumBits = ResultVT.getFixedSizeInBits();
3370 SDValue NumExtBits = DAG.getConstant(32u - NumBits, SL, MVT::i32);
3371 SDValue NewOp;
3372
3373 if (Opc == ISD::CTLZ_ZERO_POISON) {
3374 NewOp = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, Arg);
3375 NewOp = DAG.getNode(ISD::SHL, SL, MVT::i32, NewOp, NumExtBits);
3376 NewOp = DAG.getNode(Opc, SL, MVT::i32, NewOp);
3377 } else {
3378 NewOp = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Arg);
3379 NewOp = DAG.getNode(Opc, SL, MVT::i32, NewOp);
3380 NewOp = DAG.getNode(ISD::SUB, SL, MVT::i32, NewOp, NumExtBits);
3381 }
3382
3383 return DAG.getNode(ISD::TRUNCATE, SL, ResultVT, NewOp);
3384}
3385
3387 SDLoc SL(Op);
3388 SDValue Src = Op.getOperand(0);
3389
3390 assert(isCtlzOpc(Op.getOpcode()) || isCttzOpc(Op.getOpcode()));
3391 bool Ctlz = isCtlzOpc(Op.getOpcode());
3392 unsigned NewOpc = Ctlz ? AMDGPUISD::FFBH_U32 : AMDGPUISD::FFBL_B32;
3393
3394 bool ZeroUndef = Op.getOpcode() == ISD::CTLZ_ZERO_POISON ||
3395 Op.getOpcode() == ISD::CTTZ_ZERO_POISON;
3396 bool Is64BitScalar = !Src->isDivergent() && Src.getValueType() == MVT::i64;
3397
3398 if (Src.getValueType() == MVT::i32 || Is64BitScalar) {
3399 // (ctlz hi:lo) -> (umin (ffbh src), 32)
3400 // (cttz hi:lo) -> (umin (ffbl src), 32)
3401 // (ctlz_zero_poison src) -> (ffbh src)
3402 // (cttz_zero_poison src) -> (ffbl src)
3403
3404 // 64-bit scalar version produce 32-bit result
3405 // (ctlz hi:lo) -> (umin (S_FLBIT_I32_B64 src), 64)
3406 // (cttz hi:lo) -> (umin (S_FF1_I32_B64 src), 64)
3407 // (ctlz_zero_poison src) -> (S_FLBIT_I32_B64 src)
3408 // (cttz_zero_poison src) -> (S_FF1_I32_B64 src)
3409 SDValue NewOpr = DAG.getNode(NewOpc, SL, MVT::i32, Src);
3410 if (!ZeroUndef) {
3411 const SDValue ConstVal = DAG.getConstant(
3412 Op.getValueType().getScalarSizeInBits(), SL, MVT::i32);
3413 NewOpr = DAG.getNode(ISD::UMIN, SL, MVT::i32, NewOpr, ConstVal);
3414 }
3415 return DAG.getNode(ISD::ZERO_EXTEND, SL, Src.getValueType(), NewOpr);
3416 }
3417
3418 SDValue Lo, Hi;
3419 std::tie(Lo, Hi) = split64BitValue(Src, DAG);
3420
3421 SDValue OprLo = DAG.getNode(NewOpc, SL, MVT::i32, Lo);
3422 SDValue OprHi = DAG.getNode(NewOpc, SL, MVT::i32, Hi);
3423
3424 // (ctlz hi:lo) -> (umin3 (ffbh hi), (uaddsat (ffbh lo), 32), 64)
3425 // (cttz hi:lo) -> (umin3 (uaddsat (ffbl hi), 32), (ffbl lo), 64)
3426 // (ctlz_zero_poison hi:lo) -> (umin (ffbh hi), (add (ffbh lo), 32))
3427 // (cttz_zero_poison hi:lo) -> (umin (add (ffbl hi), 32), (ffbl lo))
3428
3429 unsigned AddOpc = ZeroUndef ? ISD::ADD : ISD::UADDSAT;
3430 const SDValue Const32 = DAG.getConstant(32, SL, MVT::i32);
3431 if (Ctlz)
3432 OprLo = DAG.getNode(AddOpc, SL, MVT::i32, OprLo, Const32);
3433 else
3434 OprHi = DAG.getNode(AddOpc, SL, MVT::i32, OprHi, Const32);
3435
3436 SDValue NewOpr;
3437 NewOpr = DAG.getNode(ISD::UMIN, SL, MVT::i32, OprLo, OprHi);
3438 if (!ZeroUndef) {
3439 const SDValue Const64 = DAG.getConstant(64, SL, MVT::i32);
3440 NewOpr = DAG.getNode(ISD::UMIN, SL, MVT::i32, NewOpr, Const64);
3441 }
3442
3443 return DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i64, NewOpr);
3444}
3445
3447 SDLoc SL(Op);
3448 SDValue Src = Op.getOperand(0);
3449 assert(Src.getValueType() == MVT::i32 && "LowerCTLS only supports i32");
3450 SDValue Ffbh = DAG.getNode(
3451 ISD::INTRINSIC_WO_CHAIN, SL, MVT::i32,
3452 DAG.getTargetConstant(Intrinsic::amdgcn_sffbh, SL, MVT::i32), Src);
3453 SDValue Clamped = DAG.getNode(ISD::UMIN, SL, MVT::i32, Ffbh,
3454 DAG.getConstant(32, SL, MVT::i32));
3455 return DAG.getNode(ISD::ADD, SL, MVT::i32, Clamped,
3456 DAG.getAllOnesConstant(SL, MVT::i32));
3457}
3458
3460 EVT FP16Ty) const {
3461 assert(FP16Ty == MVT::f16 || FP16Ty == MVT::bf16);
3462 SDLoc SL(Op);
3463 SDValue Src = Op.getOperand(0);
3464 SDValue ToF32 = DAG.getNode(Op.getOpcode(), SL, MVT::f32, Src);
3465 SDValue FPRoundFlag = DAG.getIntPtrConstant(0, SL, /*isTarget=*/true);
3466 return DAG.getNode(ISD::FP_ROUND, SL, FP16Ty, ToF32, FPRoundFlag);
3467}
3468
3470 bool Signed) const {
3471 // The regular method converting a 64-bit integer to float roughly consists of
3472 // 2 steps: normalization and rounding. In fact, after normalization, the
3473 // conversion from a 64-bit integer to a float is essentially the same as the
3474 // one from a 32-bit integer. The only difference is that it has more
3475 // trailing bits to be rounded. To leverage the native 32-bit conversion, a
3476 // 64-bit integer could be preprocessed and fit into a 32-bit integer then
3477 // converted into the correct float number. The basic steps for the unsigned
3478 // conversion are illustrated in the following pseudo code:
3479 //
3480 // f32 uitofp(i64 u) {
3481 // i32 hi, lo = split(u);
3482 // // Only count the leading zeros in hi as we have native support of the
3483 // // conversion from i32 to f32. If hi is all 0s, the conversion is
3484 // // reduced to a 32-bit one automatically.
3485 // i32 shamt = clz(hi); // Return 32 if hi is all 0s.
3486 // u <<= shamt;
3487 // hi, lo = split(u);
3488 // hi |= (lo != 0) ? 1 : 0; // Adjust rounding bit in hi based on lo.
3489 // // convert it as a 32-bit integer and scale the result back.
3490 // return uitofp(hi) * 2^(32 - shamt);
3491 // }
3492 //
3493 // The signed one follows the same principle but uses 'ffbh_i32' to count its
3494 // sign bits instead. If 'ffbh_i32' is not available, its absolute value is
3495 // converted instead followed by negation based its sign bit.
3496
3497 SDLoc SL(Op);
3498 SDValue Src = Op.getOperand(0);
3499
3500 SDValue Lo, Hi;
3501 std::tie(Lo, Hi) = split64BitValue(Src, DAG);
3502 SDValue Sign;
3503 SDValue ShAmt;
3504 if (Signed && Subtarget->isGCN()) {
3505 // We also need to consider the sign bit in Lo if Hi has just sign bits,
3506 // i.e. Hi is 0 or -1. However, that only needs to take the MSB into
3507 // account. That is, the maximal shift is
3508 // - 32 if Lo and Hi have opposite signs;
3509 // - 33 if Lo and Hi have the same sign.
3510 //
3511 // Or, MaxShAmt = 33 + OppositeSign, where
3512 //
3513 // OppositeSign is defined as ((Lo ^ Hi) >> 31), which is
3514 // - -1 if Lo and Hi have opposite signs; and
3515 // - 0 otherwise.
3516 //
3517 // All in all, ShAmt is calculated as
3518 //
3519 // umin(sffbh(Hi), 33 + (Lo^Hi)>>31) - 1.
3520 //
3521 // or
3522 //
3523 // umin(sffbh(Hi) - 1, 32 + (Lo^Hi)>>31).
3524 //
3525 // to reduce the critical path.
3526 SDValue OppositeSign = DAG.getNode(
3527 ISD::SRA, SL, MVT::i32, DAG.getNode(ISD::XOR, SL, MVT::i32, Lo, Hi),
3528 DAG.getConstant(31, SL, MVT::i32));
3529 SDValue MaxShAmt =
3530 DAG.getNode(ISD::ADD, SL, MVT::i32, DAG.getConstant(32, SL, MVT::i32),
3531 OppositeSign);
3532 // Count the leading sign bits.
3533 ShAmt = DAG.getNode(
3534 ISD::INTRINSIC_WO_CHAIN, SL, MVT::i32,
3535 DAG.getTargetConstant(Intrinsic::amdgcn_sffbh, SL, MVT::i32), Hi);
3536 // Different from unsigned conversion, the shift should be one bit less to
3537 // preserve the sign bit.
3538 ShAmt = DAG.getNode(ISD::SUB, SL, MVT::i32, ShAmt,
3539 DAG.getConstant(1, SL, MVT::i32));
3540 ShAmt = DAG.getNode(ISD::UMIN, SL, MVT::i32, ShAmt, MaxShAmt);
3541 } else {
3542 if (Signed) {
3543 // Without 'ffbh_i32', only leading zeros could be counted. Take the
3544 // absolute value first.
3545 Sign = DAG.getNode(ISD::SRA, SL, MVT::i64, Src,
3546 DAG.getConstant(63, SL, MVT::i64));
3547 SDValue Abs =
3548 DAG.getNode(ISD::XOR, SL, MVT::i64,
3549 DAG.getNode(ISD::ADD, SL, MVT::i64, Src, Sign), Sign);
3550 std::tie(Lo, Hi) = split64BitValue(Abs, DAG);
3551 }
3552 // Count the leading zeros.
3553 ShAmt = DAG.getNode(ISD::CTLZ, SL, MVT::i32, Hi);
3554 // The shift amount for signed integers is [0, 32].
3555 }
3556 // Normalize the given 64-bit integer.
3557 SDValue Norm = DAG.getNode(ISD::SHL, SL, MVT::i64, Src, ShAmt);
3558 // Split it again.
3559 std::tie(Lo, Hi) = split64BitValue(Norm, DAG);
3560 // Calculate the adjust bit for rounding.
3561 // (lo != 0) ? 1 : 0 => (lo >= 1) ? 1 : 0 => umin(1, lo)
3562 SDValue Adjust = DAG.getNode(ISD::UMIN, SL, MVT::i32,
3563 DAG.getConstant(1, SL, MVT::i32), Lo);
3564 // Get the 32-bit normalized integer.
3565 Norm = DAG.getNode(ISD::OR, SL, MVT::i32, Hi, Adjust);
3566 // Convert the normalized 32-bit integer into f32.
3567
3568 bool UseLDEXP = isOperationLegal(ISD::FLDEXP, MVT::f32);
3569 unsigned Opc = Signed && UseLDEXP ? ISD::SINT_TO_FP : ISD::UINT_TO_FP;
3570 SDValue FVal = DAG.getNode(Opc, SL, MVT::f32, Norm);
3571
3572 // Finally, need to scale back the converted floating number as the original
3573 // 64-bit integer is converted as a 32-bit one.
3574 ShAmt = DAG.getNode(ISD::SUB, SL, MVT::i32, DAG.getConstant(32, SL, MVT::i32),
3575 ShAmt);
3576 // On GCN, use LDEXP directly.
3577 if (UseLDEXP)
3578 return DAG.getNode(ISD::FLDEXP, SL, MVT::f32, FVal, ShAmt);
3579
3580 // Otherwise, align 'ShAmt' to the exponent part and add it into the exponent
3581 // part directly to emulate the multiplication of 2^ShAmt. That 8-bit
3582 // exponent is enough to avoid overflowing into the sign bit.
3583 SDValue Exp = DAG.getNode(ISD::SHL, SL, MVT::i32, ShAmt,
3584 DAG.getConstant(23, SL, MVT::i32));
3585 SDValue IVal =
3586 DAG.getNode(ISD::ADD, SL, MVT::i32,
3587 DAG.getNode(ISD::BITCAST, SL, MVT::i32, FVal), Exp);
3588 if (Signed) {
3589 // Set the sign bit.
3590 Sign = DAG.getNode(ISD::SHL, SL, MVT::i32,
3591 DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Sign),
3592 DAG.getConstant(31, SL, MVT::i32));
3593 IVal = DAG.getNode(ISD::OR, SL, MVT::i32, IVal, Sign);
3594 }
3595 return DAG.getNode(ISD::BITCAST, SL, MVT::f32, IVal);
3596}
3597
3599 bool Signed) const {
3600 SDLoc SL(Op);
3601 SDValue Src = Op.getOperand(0);
3602
3603 SDValue Lo, Hi;
3604 std::tie(Lo, Hi) = split64BitValue(Src, DAG);
3605
3607 SL, MVT::f64, Hi);
3608
3609 SDValue CvtLo = DAG.getNode(ISD::UINT_TO_FP, SL, MVT::f64, Lo);
3610
3611 SDValue LdExp = DAG.getNode(ISD::FLDEXP, SL, MVT::f64, CvtHi,
3612 DAG.getConstant(32, SL, MVT::i32));
3613 // TODO: Should this propagate fast-math-flags?
3614 return DAG.getNode(ISD::FADD, SL, MVT::f64, LdExp, CvtLo);
3615}
3616
3618 bool Signed) const {
3619 EVT DestVT = Op.getValueType();
3620 SDValue Src = Op.getOperand(0);
3621 EVT SrcVT = Src.getValueType();
3622 unsigned ExtOpc = Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
3623 unsigned CvtOpc = Signed ? ISD::SINT_TO_FP : ISD::UINT_TO_FP;
3624
3625 if (SrcVT == MVT::i16) {
3626 if (DestVT == MVT::f16)
3627 return Op;
3628
3629 SDLoc DL(Op);
3630 SDValue Ext = DAG.getNode(ExtOpc, DL, MVT::i32, Src);
3631 return DAG.getNode(CvtOpc, DL, DestVT, Ext);
3632 }
3633
3634 if (DestVT == MVT::bf16 || DestVT == MVT::f16)
3635 return LowerINT_TO_FP16(Op, DAG, DestVT);
3636
3637 if (SrcVT != MVT::i64)
3638 return Op;
3639
3640 if (DestVT == MVT::f32)
3641 return LowerINT_TO_FP32(Op, DAG, Signed);
3642
3643 assert(DestVT == MVT::f64);
3644 return LowerINT_TO_FP64(Op, DAG, Signed);
3645}
3646
3651
3656
3658 bool Signed) const {
3659 SDLoc SL(Op);
3660
3661 SDValue Src = Op.getOperand(0);
3662 EVT SrcVT = Src.getValueType();
3663
3664 assert(SrcVT == MVT::f32 || SrcVT == MVT::f64);
3665
3666 // The basic idea of converting a floating point number into a pair of 32-bit
3667 // integers is illustrated as follows:
3668 //
3669 // tf := trunc(val);
3670 // hif := floor(tf * 2^-32);
3671 // lof := tf - hif * 2^32; // lof is always positive due to floor.
3672 // hi := fptoi(hif);
3673 // lo := fptoi(lof);
3674 //
3675 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, SrcVT, Src);
3676 SDValue Sign;
3677 if (Signed && SrcVT == MVT::f32) {
3678 // However, a 32-bit floating point number has only 23 bits mantissa and
3679 // it's not enough to hold all the significant bits of `lof` if val is
3680 // negative. To avoid the loss of precision, We need to take the absolute
3681 // value after truncating and flip the result back based on the original
3682 // signedness.
3683 Sign = DAG.getNode(ISD::SRA, SL, MVT::i32,
3684 DAG.getNode(ISD::BITCAST, SL, MVT::i32, Trunc),
3685 DAG.getConstant(31, SL, MVT::i32));
3686 Trunc = DAG.getNode(ISD::FABS, SL, SrcVT, Trunc);
3687 }
3688
3689 SDValue K0, K1;
3690 if (SrcVT == MVT::f64) {
3691 K0 = DAG.getConstantFP(
3692 llvm::bit_cast<double>(UINT64_C(/*2^-32*/ 0x3df0000000000000)), SL,
3693 SrcVT);
3694 K1 = DAG.getConstantFP(
3695 llvm::bit_cast<double>(UINT64_C(/*-2^32*/ 0xc1f0000000000000)), SL,
3696 SrcVT);
3697 } else {
3698 K0 = DAG.getConstantFP(
3699 llvm::bit_cast<float>(UINT32_C(/*2^-32*/ 0x2f800000)), SL, SrcVT);
3700 K1 = DAG.getConstantFP(
3701 llvm::bit_cast<float>(UINT32_C(/*-2^32*/ 0xcf800000)), SL, SrcVT);
3702 }
3703 // TODO: Should this propagate fast-math-flags?
3704 SDValue Mul = DAG.getNode(ISD::FMUL, SL, SrcVT, Trunc, K0);
3705
3706 SDValue FloorMul = DAG.getNode(ISD::FFLOOR, SL, SrcVT, Mul);
3707
3708 SDValue Fma = DAG.getNode(ISD::FMA, SL, SrcVT, FloorMul, K1, Trunc);
3709
3710 SDValue Hi = DAG.getNode((Signed && SrcVT == MVT::f64) ? ISD::FP_TO_SINT
3712 SL, MVT::i32, FloorMul);
3713 SDValue Lo = DAG.getNode(ISD::FP_TO_UINT, SL, MVT::i32, Fma);
3714
3715 SDValue Result = DAG.getNode(ISD::BITCAST, SL, MVT::i64,
3716 DAG.getBuildVector(MVT::v2i32, SL, {Lo, Hi}));
3717
3718 if (Signed && SrcVT == MVT::f32) {
3719 assert(Sign);
3720 // Flip the result based on the signedness, which is either all 0s or 1s.
3721 Sign = DAG.getNode(ISD::BITCAST, SL, MVT::i64,
3722 DAG.getBuildVector(MVT::v2i32, SL, {Sign, Sign}));
3723 // r := xor(r, sign) - sign;
3724 Result =
3725 DAG.getNode(ISD::SUB, SL, MVT::i64,
3726 DAG.getNode(ISD::XOR, SL, MVT::i64, Result, Sign), Sign);
3727 }
3728
3729 return Result;
3730}
3731
3733 SDLoc DL(Op);
3734 SDValue N0 = Op.getOperand(0);
3735
3736 // Convert to target node to get known bits
3737 if (N0.getValueType() == MVT::f32)
3738 return DAG.getNode(AMDGPUISD::FP_TO_FP16, DL, Op.getValueType(), N0);
3739
3740 if (Op->getFlags().hasApproximateFuncs()) {
3741 // There is a generic expand for FP_TO_FP16 with unsafe fast math.
3742 return SDValue();
3743 }
3744
3745 return LowerF64ToF16Safe(N0, DL, DAG);
3746}
3747
3748// return node in i32
3750 SelectionDAG &DAG) const {
3751 assert(Src.getSimpleValueType() == MVT::f64);
3752
3753 // f64 -> f16 conversion using round-to-nearest-even rounding mode.
3754 // TODO: We can generate better code for True16.
3755 const unsigned ExpMask = 0x7ff;
3756 const unsigned ExpBiasf64 = 1023;
3757 const unsigned ExpBiasf16 = 15;
3758 SDValue Zero = DAG.getConstant(0, DL, MVT::i32);
3759 SDValue One = DAG.getConstant(1, DL, MVT::i32);
3760 SDValue U = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Src);
3761 SDValue UH = DAG.getNode(ISD::SRL, DL, MVT::i64, U,
3762 DAG.getConstant(32, DL, MVT::i64));
3763 UH = DAG.getZExtOrTrunc(UH, DL, MVT::i32);
3764 U = DAG.getZExtOrTrunc(U, DL, MVT::i32);
3765 SDValue E = DAG.getNode(ISD::SRL, DL, MVT::i32, UH,
3766 DAG.getConstant(20, DL, MVT::i64));
3767 E = DAG.getNode(ISD::AND, DL, MVT::i32, E,
3768 DAG.getConstant(ExpMask, DL, MVT::i32));
3769 // Subtract the fp64 exponent bias (1023) to get the real exponent and
3770 // add the f16 bias (15) to get the biased exponent for the f16 format.
3771 E = DAG.getNode(ISD::ADD, DL, MVT::i32, E,
3772 DAG.getConstant(-ExpBiasf64 + ExpBiasf16, DL, MVT::i32));
3773
3774 SDValue M = DAG.getNode(ISD::SRL, DL, MVT::i32, UH,
3775 DAG.getConstant(8, DL, MVT::i32));
3776 M = DAG.getNode(ISD::AND, DL, MVT::i32, M,
3777 DAG.getConstant(0xffe, DL, MVT::i32));
3778
3779 SDValue MaskedSig = DAG.getNode(ISD::AND, DL, MVT::i32, UH,
3780 DAG.getConstant(0x1ff, DL, MVT::i32));
3781 MaskedSig = DAG.getNode(ISD::OR, DL, MVT::i32, MaskedSig, U);
3782
3783 SDValue Lo40Set = DAG.getSelectCC(DL, MaskedSig, Zero, Zero, One, ISD::SETEQ);
3784 M = DAG.getNode(ISD::OR, DL, MVT::i32, M, Lo40Set);
3785
3786 // (M != 0 ? 0x0200 : 0) | 0x7c00;
3787 SDValue I = DAG.getNode(ISD::OR, DL, MVT::i32,
3788 DAG.getSelectCC(DL, M, Zero, DAG.getConstant(0x0200, DL, MVT::i32),
3789 Zero, ISD::SETNE), DAG.getConstant(0x7c00, DL, MVT::i32));
3790
3791 // N = M | (E << 12);
3792 SDValue N = DAG.getNode(ISD::OR, DL, MVT::i32, M,
3793 DAG.getNode(ISD::SHL, DL, MVT::i32, E,
3794 DAG.getConstant(12, DL, MVT::i32)));
3795
3796 // B = clamp(1-E, 0, 13);
3797 SDValue OneSubExp = DAG.getNode(ISD::SUB, DL, MVT::i32,
3798 One, E);
3799 SDValue B = DAG.getNode(ISD::SMAX, DL, MVT::i32, OneSubExp, Zero);
3800 B = DAG.getNode(ISD::SMIN, DL, MVT::i32, B,
3801 DAG.getConstant(13, DL, MVT::i32));
3802
3803 SDValue SigSetHigh = DAG.getNode(ISD::OR, DL, MVT::i32, M,
3804 DAG.getConstant(0x1000, DL, MVT::i32));
3805
3806 SDValue D = DAG.getNode(ISD::SRL, DL, MVT::i32, SigSetHigh, B);
3807 SDValue D0 = DAG.getNode(ISD::SHL, DL, MVT::i32, D, B);
3808 SDValue D1 = DAG.getSelectCC(DL, D0, SigSetHigh, One, Zero, ISD::SETNE);
3809 D = DAG.getNode(ISD::OR, DL, MVT::i32, D, D1);
3810
3811 SDValue V = DAG.getSelectCC(DL, E, One, D, N, ISD::SETLT);
3812 SDValue VLow3 = DAG.getNode(ISD::AND, DL, MVT::i32, V,
3813 DAG.getConstant(0x7, DL, MVT::i32));
3814 V = DAG.getNode(ISD::SRL, DL, MVT::i32, V,
3815 DAG.getConstant(2, DL, MVT::i32));
3816 SDValue V0 = DAG.getSelectCC(DL, VLow3, DAG.getConstant(3, DL, MVT::i32),
3817 One, Zero, ISD::SETEQ);
3818 SDValue V1 = DAG.getSelectCC(DL, VLow3, DAG.getConstant(5, DL, MVT::i32),
3819 One, Zero, ISD::SETGT);
3820 V1 = DAG.getNode(ISD::OR, DL, MVT::i32, V0, V1);
3821 V = DAG.getNode(ISD::ADD, DL, MVT::i32, V, V1);
3822
3823 V = DAG.getSelectCC(DL, E, DAG.getConstant(30, DL, MVT::i32),
3824 DAG.getConstant(0x7c00, DL, MVT::i32), V, ISD::SETGT);
3825 V = DAG.getSelectCC(DL, E, DAG.getConstant(1039, DL, MVT::i32),
3826 I, V, ISD::SETEQ);
3827
3828 // Extract the sign bit.
3829 SDValue Sign = DAG.getNode(ISD::SRL, DL, MVT::i32, UH,
3830 DAG.getConstant(16, DL, MVT::i32));
3831 Sign = DAG.getNode(ISD::AND, DL, MVT::i32, Sign,
3832 DAG.getConstant(0x8000, DL, MVT::i32));
3833
3834 return DAG.getNode(ISD::OR, DL, MVT::i32, Sign, V);
3835}
3836
3838 SelectionDAG &DAG) const {
3839 SDValue Src = Op.getOperand(0);
3840 unsigned OpOpcode = Op.getOpcode();
3841 EVT SrcVT = Src.getValueType();
3842 EVT DestVT = Op.getValueType();
3843
3844 // Will be selected natively
3845 if (SrcVT == MVT::f16 && DestVT == MVT::i16)
3846 return Op;
3847
3848 if (SrcVT == MVT::bf16 || (SrcVT == MVT::f16 && DestVT == MVT::i32)) {
3849 SDLoc DL(Op);
3850 SDValue PromotedSrc = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Src);
3851 return DAG.getNode(Op.getOpcode(), DL, DestVT, PromotedSrc);
3852 }
3853
3854 // Promote i16 to i32
3855 if (DestVT == MVT::i16 && (SrcVT == MVT::f32 || SrcVT == MVT::f64)) {
3856 SDLoc DL(Op);
3857
3858 SDValue FpToInt32 = DAG.getNode(OpOpcode, DL, MVT::i32, Src);
3859 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FpToInt32);
3860 }
3861
3862 if (DestVT != MVT::i64)
3863 return Op;
3864
3865 if (SrcVT == MVT::f16 ||
3866 (SrcVT == MVT::f32 && Src.getOpcode() == ISD::FP16_TO_FP)) {
3867 SDLoc DL(Op);
3868
3869 SDValue FpToInt32 = DAG.getNode(OpOpcode, DL, MVT::i32, Src);
3870 unsigned Ext =
3872 return DAG.getNode(Ext, DL, MVT::i64, FpToInt32);
3873 }
3874
3875 if (SrcVT == MVT::f32 || SrcVT == MVT::f64)
3876 return LowerFP_TO_INT64(Op, DAG, OpOpcode == ISD::FP_TO_SINT);
3877
3878 return SDValue();
3879}
3880
3882 SelectionDAG &DAG) const {
3883 SDValue Src = Op.getOperand(0);
3884 unsigned OpOpcode = Op.getOpcode();
3885 EVT SrcVT = Src.getValueType();
3886 EVT DstVT = Op.getValueType();
3887 SDValue SatVTOp = Op.getNode()->getOperand(1);
3888 EVT SatVT = cast<VTSDNode>(SatVTOp)->getVT();
3889 SDLoc DL(Op);
3890
3891 uint64_t DstWidth = DstVT.getScalarSizeInBits();
3892 uint64_t SatWidth = SatVT.getScalarSizeInBits();
3893 assert(SatWidth <= DstWidth && "Saturation width cannot exceed result width");
3894
3895 // Scalar cases will be selected natively to v_cvt_/s_cvt_ instructions.
3896 // v2f32 -> v2i16 will be selected natively to v_cvt_pk_[iu]16_f32.
3897 if (SatWidth == DstWidth) {
3898 if ((DstVT == MVT::i32 && (SrcVT == MVT::f32 || SrcVT == MVT::f64)) ||
3899 (DstVT == MVT::i16 && (SrcVT == MVT::f16 || SrcVT == MVT::f32)) ||
3900 (DstVT == MVT::v2i16 && SrcVT == MVT::v2f32))
3901 return Op;
3902 }
3903
3904 // Vectors can only be selected natively.
3905 if (DstVT.isVector())
3906 return SDValue();
3907
3908 // Perform all saturation at selected width (i16 or i32) and truncate
3909 if (SatWidth < DstWidth && SatWidth <= 32) {
3910 // For f16 conversion with sub-i16 saturation perform saturation
3911 // at i16, if available in the target. This removes the need for extra f16
3912 // to f32 conversion. For all the others use i32.
3913 MVT ResultVT =
3914 Subtarget->has16BitInsts() && SrcVT == MVT::f16 && SatWidth < 16
3915 ? MVT::i16
3916 : MVT::i32;
3917
3918 const SDValue ResultVTOp = DAG.getValueType(ResultVT);
3919 const uint64_t ResultWidth = ResultVT.getScalarSizeInBits();
3920
3921 // First, convert input float into selected integer (i16 or i32)
3922 SDValue FpToInt = DAG.getNode(OpOpcode, DL, ResultVT, Src, ResultVTOp);
3923 SDValue IntSatVal;
3924
3925 // Then, clamp at the saturation width using either i16 or i32 instructions
3926 if (OpOpcode == ISD::FP_TO_SINT_SAT) {
3927 SDValue MinConst = DAG.getConstant(
3928 APInt::getSignedMaxValue(SatWidth).sext(ResultWidth), DL, ResultVT);
3929 SDValue MaxConst = DAG.getConstant(
3930 APInt::getSignedMinValue(SatWidth).sext(ResultWidth), DL, ResultVT);
3931 SDValue MinVal = DAG.getNode(ISD::SMIN, DL, ResultVT, FpToInt, MinConst);
3932 IntSatVal = DAG.getNode(ISD::SMAX, DL, ResultVT, MinVal, MaxConst);
3933 } else {
3934 SDValue MinConst = DAG.getConstant(
3935 APInt::getMaxValue(SatWidth).zext(ResultWidth), DL, ResultVT);
3936 IntSatVal = DAG.getNode(ISD::UMIN, DL, ResultVT, FpToInt, MinConst);
3937 }
3938
3939 // Finally, after saturating at i16 or i32 fit into the destination type
3940 return DAG.getExtOrTrunc(OpOpcode == ISD::FP_TO_SINT_SAT, IntSatVal, DL,
3941 DstVT);
3942 }
3943
3944 // SatWidth == DstWidth or SatWidth > 32
3945
3946 // Saturate at i32 for i64 dst and f16/bf16 src (will invoke f16 promotion
3947 // below)
3948 if (DstVT == MVT::i64 &&
3949 (SrcVT == MVT::f16 || SrcVT == MVT::bf16 ||
3950 (SrcVT == MVT::f32 && Src.getOpcode() == ISD::FP16_TO_FP))) {
3951 const SDValue Int32VTOp = DAG.getValueType(MVT::i32);
3952 return DAG.getNode(OpOpcode, DL, DstVT, Src, Int32VTOp);
3953 }
3954
3955 // Promote f16/bf16 src to f32 for i32 conversion
3956 if (DstVT == MVT::i32 && (SrcVT == MVT::f16 || SrcVT == MVT::bf16)) {
3957 SDValue PromotedSrc = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Src);
3958 return DAG.getNode(Op.getOpcode(), DL, DstVT, PromotedSrc, SatVTOp);
3959 }
3960
3961 // For DstWidth < 16, promote i1 and i8 dst to i16 (if legal) with sub-i16
3962 // saturation. For DstWidth == 16, promote i16 dst to i32 with sub-i32
3963 // saturation; this covers i16.f32 and i16.f64
3964 if (DstWidth < 32) {
3965 // Note: this triggers SatWidth < DstWidth above to generate saturated
3966 // truncate by requesting MVT::i16/i32 destination with SatWidth < 16/32.
3967 MVT PromoteVT =
3968 (DstWidth < 16 && Subtarget->has16BitInsts()) ? MVT::i16 : MVT::i32;
3969 SDValue FpToInt = DAG.getNode(OpOpcode, DL, PromoteVT, Src, SatVTOp);
3970 return DAG.getNode(ISD::TRUNCATE, DL, DstVT, FpToInt);
3971 }
3972
3973 // TODO: can we implement i64 dst for f32/f64?
3974
3975 return SDValue();
3976}
3977
3979 SelectionDAG &DAG) const {
3980 EVT ExtraVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
3981 MVT VT = Op.getSimpleValueType();
3982 MVT ScalarVT = VT.getScalarType();
3983
3984 assert(VT.isVector());
3985
3986 SDValue Src = Op.getOperand(0);
3987 SDLoc DL(Op);
3988
3989 // TODO: Don't scalarize on Evergreen?
3990 unsigned NElts = VT.getVectorNumElements();
3992 DAG.ExtractVectorElements(Src, Args, 0, NElts);
3993
3994 SDValue VTOp = DAG.getValueType(ExtraVT.getScalarType());
3995 for (unsigned I = 0; I < NElts; ++I)
3996 Args[I] = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, ScalarVT, Args[I], VTOp);
3997
3998 return DAG.getBuildVector(VT, DL, Args);
3999}
4000
4001//===----------------------------------------------------------------------===//
4002// Custom DAG optimizations
4003//===----------------------------------------------------------------------===//
4004
4005static bool isU24(SDValue Op, SelectionDAG &DAG) {
4006 return AMDGPUTargetLowering::numBitsUnsigned(Op, DAG) <= 24;
4007}
4008
4009static bool isI24(SDValue Op, SelectionDAG &DAG) {
4010 EVT VT = Op.getValueType();
4011 return VT.getSizeInBits() >= 24 && // Types less than 24-bit should be treated
4012 // as unsigned 24-bit values.
4014}
4015
4018 SelectionDAG &DAG = DCI.DAG;
4020 bool IsIntrin = Node24->getOpcode() == ISD::INTRINSIC_WO_CHAIN;
4021
4022 SDValue LHS = IsIntrin ? Node24->getOperand(1) : Node24->getOperand(0);
4023 SDValue RHS = IsIntrin ? Node24->getOperand(2) : Node24->getOperand(1);
4024 unsigned NewOpcode = Node24->getOpcode();
4025 if (IsIntrin) {
4026 unsigned IID = Node24->getConstantOperandVal(0);
4027 switch (IID) {
4028 case Intrinsic::amdgcn_mul_i24:
4029 NewOpcode = AMDGPUISD::MUL_I24;
4030 break;
4031 case Intrinsic::amdgcn_mul_u24:
4032 NewOpcode = AMDGPUISD::MUL_U24;
4033 break;
4034 case Intrinsic::amdgcn_mulhi_i24:
4035 NewOpcode = AMDGPUISD::MULHI_I24;
4036 break;
4037 case Intrinsic::amdgcn_mulhi_u24:
4038 NewOpcode = AMDGPUISD::MULHI_U24;
4039 break;
4040 default:
4041 llvm_unreachable("Expected 24-bit mul intrinsic");
4042 }
4043 }
4044
4045 APInt Demanded = APInt::getLowBitsSet(LHS.getValueSizeInBits(), 24);
4046
4048 return DAG.getConstant(0, SDLoc(Node24), Node24->getValueType(0));
4049
4050 // First try SimplifyDemandedBits which can simplify the nodes used by our
4051 // operands if this node is the only user.
4052 if (LHS.hasOneUse() && TLI.SimplifyDemandedBits(LHS, Demanded, DCI))
4053 return SDValue(Node24, 0);
4054 if (RHS.hasOneUse() && TLI.SimplifyDemandedBits(RHS, Demanded, DCI))
4055 return SDValue(Node24, 0);
4056
4057 // Then try SimplifyMultipleUseDemandedBits which allows the operands to have
4058 // other uses, but will only perform simplifications that involve bypassing
4059 // some nodes for this user.
4060 SDValue DemandedLHS = TLI.SimplifyMultipleUseDemandedBits(LHS, Demanded, DAG);
4061 SDValue DemandedRHS = TLI.SimplifyMultipleUseDemandedBits(RHS, Demanded, DAG);
4062 if (DemandedLHS || DemandedRHS)
4063 return DAG.getNode(NewOpcode, SDLoc(Node24), Node24->getVTList(),
4064 DemandedLHS ? DemandedLHS : LHS,
4065 DemandedRHS ? DemandedRHS : RHS);
4066
4067 return SDValue();
4068}
4069
4070template <typename IntTy>
4072 uint32_t Width, const SDLoc &DL) {
4073 if (Width + Offset < 32) {
4074 uint32_t Shl = static_cast<uint32_t>(Src0) << (32 - Offset - Width);
4075 IntTy Result = static_cast<IntTy>(Shl) >> (32 - Width);
4076 if constexpr (std::is_signed_v<IntTy>) {
4077 return DAG.getSignedConstant(Result, DL, MVT::i32);
4078 } else {
4079 return DAG.getConstant(Result, DL, MVT::i32);
4080 }
4081 }
4082
4083 return DAG.getConstant(Src0 >> Offset, DL, MVT::i32);
4084}
4085
4086static bool hasVolatileUser(SDNode *Val) {
4087 for (SDNode *U : Val->users()) {
4088 if (MemSDNode *M = dyn_cast<MemSDNode>(U)) {
4089 if (M->isVolatile())
4090 return true;
4091 }
4092 }
4093
4094 return false;
4095}
4096
4098 // i32 vectors are the canonical memory type.
4099 if (VT.getScalarType() == MVT::i32 || isTypeLegal(VT))
4100 return false;
4101
4102 if (!VT.isByteSized())
4103 return false;
4104
4105 unsigned Size = VT.getStoreSize();
4106
4107 if ((Size == 1 || Size == 2 || Size == 4) && !VT.isVector())
4108 return false;
4109
4110 if (Size == 3 || (Size > 4 && (Size % 4 != 0)))
4111 return false;
4112
4113 return true;
4114}
4115
4116// Replace load of an illegal type with a bitcast from a load of a friendlier
4117// type.
4119 DAGCombinerInfo &DCI) const {
4120 if (!DCI.isBeforeLegalize())
4121 return SDValue();
4122
4124 if (!LN->isSimple() || !ISD::isNormalLoad(LN) || hasVolatileUser(LN))
4125 return SDValue();
4126
4127 SDLoc SL(N);
4128 SelectionDAG &DAG = DCI.DAG;
4129 EVT VT = LN->getMemoryVT();
4130
4131 unsigned Size = VT.getStoreSize();
4132 Align Alignment = LN->getAlign();
4133 if (Alignment < Size && isTypeLegal(VT)) {
4134 unsigned IsFast;
4135 unsigned AS = LN->getAddressSpace();
4136
4137 // Expand unaligned loads earlier than legalization. Due to visitation order
4138 // problems during legalization, the emitted instructions to pack and unpack
4139 // the bytes again are not eliminated in the case of an unaligned copy.
4141 VT, AS, Alignment, LN->getMemOperand()->getFlags(), &IsFast)) {
4142 if (VT.isVector())
4143 return SplitVectorLoad(SDValue(LN, 0), DAG);
4144
4145 SDValue Ops[2];
4146 std::tie(Ops[0], Ops[1]) = expandUnalignedLoad(LN, DAG);
4147
4148 return DAG.getMergeValues(Ops, SDLoc(N));
4149 }
4150
4151 if (!IsFast)
4152 return SDValue();
4153 }
4154
4155 if (!shouldCombineMemoryType(VT))
4156 return SDValue();
4157
4158 EVT NewVT = getEquivalentMemType(*DAG.getContext(), VT);
4159
4160 SDValue NewLoad
4161 = DAG.getLoad(NewVT, SL, LN->getChain(),
4162 LN->getBasePtr(), LN->getMemOperand());
4163
4164 SDValue BC = DAG.getNode(ISD::BITCAST, SL, VT, NewLoad);
4165 DCI.CombineTo(N, BC, NewLoad.getValue(1));
4166 return SDValue(N, 0);
4167}
4168
4169// Replace store of an illegal type with a store of a bitcast to a friendlier
4170// type.
4172 DAGCombinerInfo &DCI) const {
4173 if (!DCI.isBeforeLegalize())
4174 return SDValue();
4175
4177 if (!SN->isSimple() || !ISD::isNormalStore(SN))
4178 return SDValue();
4179
4180 EVT VT = SN->getMemoryVT();
4181 unsigned Size = VT.getStoreSize();
4182
4183 SDLoc SL(N);
4184 SelectionDAG &DAG = DCI.DAG;
4185 Align Alignment = SN->getAlign();
4186 if (Alignment < Size && isTypeLegal(VT)) {
4187 unsigned IsFast;
4188 unsigned AS = SN->getAddressSpace();
4189
4190 // Expand unaligned stores earlier than legalization. Due to visitation
4191 // order problems during legalization, the emitted instructions to pack and
4192 // unpack the bytes again are not eliminated in the case of an unaligned
4193 // copy.
4195 VT, AS, Alignment, SN->getMemOperand()->getFlags(), &IsFast)) {
4196 if (VT.isVector())
4197 return SplitVectorStore(SDValue(SN, 0), DAG);
4198
4199 return expandUnalignedStore(SN, DAG);
4200 }
4201
4202 if (!IsFast)
4203 return SDValue();
4204 }
4205
4206 if (!shouldCombineMemoryType(VT))
4207 return SDValue();
4208
4209 EVT NewVT = getEquivalentMemType(*DAG.getContext(), VT);
4210 SDValue Val = SN->getValue();
4211
4212 // DCI.AddToWorklist(Val.getNode());
4213
4214 bool OtherUses = !Val.hasOneUse();
4215 SDValue CastVal = DAG.getBitcast(NewVT, Val);
4216 if (OtherUses) {
4217 SDValue CastBack = DAG.getBitcast(VT, CastVal);
4218 DAG.ReplaceAllUsesOfValueWith(Val, CastBack);
4219 }
4220
4221 return DAG.getStore(SN->getChain(), SL, CastVal,
4222 SN->getBasePtr(), SN->getMemOperand());
4223}
4224
4225// FIXME: This should go in generic DAG combiner with an isTruncateFree check,
4226// but isTruncateFree is inaccurate for i16 now because of SALU vs. VALU
4227// issues.
4229 DAGCombinerInfo &DCI) const {
4230 SelectionDAG &DAG = DCI.DAG;
4231 SDValue N0 = N->getOperand(0);
4232
4233 // (vt2 (assertzext (truncate vt0:x), vt1)) ->
4234 // (vt2 (truncate (assertzext vt0:x, vt1)))
4235 if (N0.getOpcode() == ISD::TRUNCATE) {
4236 SDValue N1 = N->getOperand(1);
4237 EVT ExtVT = cast<VTSDNode>(N1)->getVT();
4238 SDLoc SL(N);
4239
4240 SDValue Src = N0.getOperand(0);
4241 EVT SrcVT = Src.getValueType();
4242 if (SrcVT.bitsGE(ExtVT)) {
4243 SDValue NewInReg = DAG.getNode(N->getOpcode(), SL, SrcVT, Src, N1);
4244 return DAG.getNode(ISD::TRUNCATE, SL, N->getValueType(0), NewInReg);
4245 }
4246 }
4247
4248 return SDValue();
4249}
4250
4252 SDNode *N, DAGCombinerInfo &DCI) const {
4253 unsigned IID = N->getConstantOperandVal(0);
4254 switch (IID) {
4255 case Intrinsic::amdgcn_mul_i24:
4256 case Intrinsic::amdgcn_mul_u24:
4257 case Intrinsic::amdgcn_mulhi_i24:
4258 case Intrinsic::amdgcn_mulhi_u24:
4259 return simplifyMul24(N, DCI);
4260 case Intrinsic::amdgcn_fract:
4261 case Intrinsic::amdgcn_rsq:
4262 case Intrinsic::amdgcn_rcp_legacy:
4263 case Intrinsic::amdgcn_rsq_legacy:
4264 case Intrinsic::amdgcn_rsq_clamp:
4265 case Intrinsic::amdgcn_tanh:
4266 case Intrinsic::amdgcn_prng_b32: {
4267 // FIXME: This is probably wrong. If src is an sNaN, it won't be quieted
4268 SDValue Src = N->getOperand(1);
4269 return Src.isUndef() ? Src : SDValue();
4270 }
4271 case Intrinsic::amdgcn_frexp_exp: {
4272 // frexp_exp (fneg x) -> frexp_exp x
4273 // frexp_exp (fabs x) -> frexp_exp x
4274 // frexp_exp (fneg (fabs x)) -> frexp_exp x
4275 SDValue Src = N->getOperand(1);
4276 SDValue PeekSign = peekFPSignOps(Src);
4277 if (PeekSign == Src)
4278 return SDValue();
4279 return SDValue(DCI.DAG.UpdateNodeOperands(N, N->getOperand(0), PeekSign),
4280 0);
4281 }
4282 default:
4283 return SDValue();
4284 }
4285}
4286
4287/// Split the 64-bit value \p LHS into two 32-bit components, and perform the
4288/// binary operation \p Opc to it with the corresponding constant operands.
4290 DAGCombinerInfo &DCI, const SDLoc &SL,
4291 unsigned Opc, SDValue LHS,
4292 uint32_t ValLo, uint32_t ValHi) const {
4293 SelectionDAG &DAG = DCI.DAG;
4294 SDValue Lo, Hi;
4295 std::tie(Lo, Hi) = split64BitValue(LHS, DAG);
4296
4297 SDValue LoRHS = DAG.getConstant(ValLo, SL, MVT::i32);
4298 SDValue HiRHS = DAG.getConstant(ValHi, SL, MVT::i32);
4299
4300 SDValue LoAnd = DAG.getNode(Opc, SL, MVT::i32, Lo, LoRHS);
4301 SDValue HiAnd = DAG.getNode(Opc, SL, MVT::i32, Hi, HiRHS);
4302
4303 // Re-visit the ands. It's possible we eliminated one of them and it could
4304 // simplify the vector.
4305 DCI.AddToWorklist(Lo.getNode());
4306 DCI.AddToWorklist(Hi.getNode());
4307
4308 SDValue Vec = DAG.getBuildVector(MVT::v2i32, SL, {LoAnd, HiAnd});
4309 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec);
4310}
4311
4313 DAGCombinerInfo &DCI) const {
4314 EVT VT = N->getValueType(0);
4315 SDValue LHS = N->getOperand(0);
4316 SDValue RHS = N->getOperand(1);
4318 SDLoc SL(N);
4319 SelectionDAG &DAG = DCI.DAG;
4320
4321 unsigned RHSVal;
4322 if (CRHS) {
4323 RHSVal = CRHS->getZExtValue();
4324 if (!RHSVal)
4325 return LHS;
4326
4327 switch (LHS->getOpcode()) {
4328 default:
4329 break;
4330 case ISD::ZERO_EXTEND:
4331 case ISD::SIGN_EXTEND:
4332 case ISD::ANY_EXTEND: {
4333 SDValue X = LHS->getOperand(0);
4334
4335 if (VT == MVT::i32 && RHSVal == 16 && X.getValueType() == MVT::i16 &&
4336 isOperationLegal(ISD::BUILD_VECTOR, MVT::v2i16)) {
4337 // Prefer build_vector as the canonical form if packed types are legal.
4338 // (shl ([asz]ext i16:x), 16 -> build_vector 0, x
4339 SDValue Vec = DAG.getBuildVector(
4340 MVT::v2i16, SL,
4341 {DAG.getConstant(0, SL, MVT::i16), LHS->getOperand(0)});
4342 return DAG.getNode(ISD::BITCAST, SL, MVT::i32, Vec);
4343 }
4344
4345 // shl (ext x) => zext (shl x), if shift does not overflow int
4346 if (VT != MVT::i64)
4347 break;
4349 unsigned LZ = Known.countMinLeadingZeros();
4350 if (LZ < RHSVal)
4351 break;
4352 EVT XVT = X.getValueType();
4353 SDValue Shl = DAG.getNode(ISD::SHL, SL, XVT, X, SDValue(CRHS, 0));
4354 return DAG.getZExtOrTrunc(Shl, SL, VT);
4355 }
4356 }
4357 }
4358
4359 if (VT.getScalarType() != MVT::i64)
4360 return SDValue();
4361
4362 // On some subtargets, 64-bit shift is a quarter rate instruction. In the
4363 // common case, splitting this into a move and a 32-bit shift is faster and
4364 // the same code size.
4365 KnownBits Known = DAG.computeKnownBits(RHS);
4366
4367 EVT ElementType = VT.getScalarType();
4368 EVT TargetScalarType = ElementType.getHalfSizedIntegerVT(*DAG.getContext());
4369 EVT TargetType = VT.changeElementType(*DAG.getContext(), TargetScalarType);
4370
4371 if (Known.getMinValue().getZExtValue() < TargetScalarType.getSizeInBits())
4372 return SDValue();
4373 SDValue ShiftAmt;
4374
4375 if (CRHS) {
4376 ShiftAmt = DAG.getConstant(RHSVal - TargetScalarType.getSizeInBits(), SL,
4377 TargetType);
4378 } else {
4379 SDValue TruncShiftAmt = DAG.getNode(ISD::TRUNCATE, SL, TargetType, RHS);
4380 const SDValue ShiftMask =
4381 DAG.getConstant(TargetScalarType.getSizeInBits() - 1, SL, TargetType);
4382 // This AND instruction will clamp out of bounds shift values.
4383 // It will also be removed during later instruction selection.
4384 ShiftAmt = DAG.getNode(ISD::AND, SL, TargetType, TruncShiftAmt, ShiftMask);
4385 }
4386
4387 SDValue Lo = DAG.getNode(ISD::TRUNCATE, SL, TargetType, LHS);
4388 SDValue NewShift =
4389 DAG.getNode(ISD::SHL, SL, TargetType, Lo, ShiftAmt, N->getFlags());
4390
4391 const SDValue Zero = DAG.getConstant(0, SL, TargetScalarType);
4392 SDValue Vec;
4393
4394 if (VT.isVector()) {
4395 EVT ConcatType = TargetType.getDoubleNumVectorElementsVT(*DAG.getContext());
4396 unsigned NElts = TargetType.getVectorNumElements();
4398 SmallVector<SDValue, 16> HiAndLoOps(NElts * 2, Zero);
4399
4400 DAG.ExtractVectorElements(NewShift, HiOps, 0, NElts);
4401 for (unsigned I = 0; I != NElts; ++I)
4402 HiAndLoOps[2 * I + 1] = HiOps[I];
4403 Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, ConcatType, HiAndLoOps);
4404 } else {
4405 EVT ConcatType = EVT::getVectorVT(*DAG.getContext(), TargetType, 2);
4406 Vec = DAG.getBuildVector(ConcatType, SL, {Zero, NewShift});
4407 }
4408 return DAG.getNode(ISD::BITCAST, SL, VT, Vec);
4409}
4410
4412 DAGCombinerInfo &DCI) const {
4413 SDValue RHS = N->getOperand(1);
4415 EVT VT = N->getValueType(0);
4416 SDValue LHS = N->getOperand(0);
4417 SelectionDAG &DAG = DCI.DAG;
4418 SDLoc SL(N);
4419
4420 if (VT.getScalarType() != MVT::i64)
4421 return SDValue();
4422
4423 // For C >= 32
4424 // i64 (sra x, C) -> (build_pair (sra hi_32(x), C - 32), sra hi_32(x), 31))
4425
4426 // On some subtargets, 64-bit shift is a quarter rate instruction. In the
4427 // common case, splitting this into a move and a 32-bit shift is faster and
4428 // the same code size.
4429 KnownBits Known = DAG.computeKnownBits(RHS);
4430
4431 EVT ElementType = VT.getScalarType();
4432 EVT TargetScalarType = ElementType.getHalfSizedIntegerVT(*DAG.getContext());
4433 EVT TargetType = VT.changeElementType(*DAG.getContext(), TargetScalarType);
4434
4435 if (Known.getMinValue().getZExtValue() < TargetScalarType.getSizeInBits())
4436 return SDValue();
4437
4438 SDValue ShiftFullAmt =
4439 DAG.getConstant(TargetScalarType.getSizeInBits() - 1, SL, TargetType);
4440 SDValue ShiftAmt;
4441 if (CRHS) {
4442 unsigned RHSVal = CRHS->getZExtValue();
4443 ShiftAmt = DAG.getConstant(RHSVal - TargetScalarType.getSizeInBits(), SL,
4444 TargetType);
4445 } else if (Known.getMinValue().getZExtValue() ==
4446 (ElementType.getSizeInBits() - 1)) {
4447 ShiftAmt = ShiftFullAmt;
4448 } else {
4449 SDValue TruncShiftAmt = DAG.getNode(ISD::TRUNCATE, SL, TargetType, RHS);
4450 const SDValue ShiftMask =
4451 DAG.getConstant(TargetScalarType.getSizeInBits() - 1, SL, TargetType);
4452 // This AND instruction will clamp out of bounds shift values.
4453 // It will also be removed during later instruction selection.
4454 ShiftAmt = DAG.getNode(ISD::AND, SL, TargetType, TruncShiftAmt, ShiftMask);
4455 }
4456
4457 EVT ConcatType;
4458 SDValue Hi;
4459 SDLoc LHSSL(LHS);
4460 // Bitcast LHS into ConcatType so hi-half of source can be extracted into Hi
4461 if (VT.isVector()) {
4462 unsigned NElts = TargetType.getVectorNumElements();
4463 ConcatType = TargetType.getDoubleNumVectorElementsVT(*DAG.getContext());
4464 SDValue SplitLHS = DAG.getNode(ISD::BITCAST, LHSSL, ConcatType, LHS);
4465 SmallVector<SDValue, 8> HiOps(NElts);
4466 SmallVector<SDValue, 16> HiAndLoOps;
4467
4468 DAG.ExtractVectorElements(SplitLHS, HiAndLoOps, 0, NElts * 2);
4469 for (unsigned I = 0; I != NElts; ++I) {
4470 HiOps[I] = HiAndLoOps[2 * I + 1];
4471 }
4472 Hi = DAG.getNode(ISD::BUILD_VECTOR, LHSSL, TargetType, HiOps);
4473 } else {
4474 const SDValue One = DAG.getConstant(1, LHSSL, TargetScalarType);
4475 ConcatType = EVT::getVectorVT(*DAG.getContext(), TargetType, 2);
4476 SDValue SplitLHS = DAG.getNode(ISD::BITCAST, LHSSL, ConcatType, LHS);
4477 Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, LHSSL, TargetType, SplitLHS, One);
4478 }
4479
4480 KnownBits KnownLHS = DAG.computeKnownBits(LHS);
4481 SDValue NewShift, HiShift;
4482 if (KnownLHS.isNegative()) {
4483 HiShift = DAG.getAllOnesConstant(SL, TargetType);
4484 NewShift =
4485 DAG.getNode(ISD::SRA, SL, TargetType, Hi, ShiftAmt, N->getFlags());
4486 } else if (CRHS &&
4487 CRHS->getZExtValue() == (ElementType.getSizeInBits() - 1)) {
4488 NewShift = HiShift =
4489 DAG.getNode(ISD::SRA, SL, TargetType, Hi, ShiftAmt, N->getFlags());
4490 } else {
4491 Hi = DAG.getFreeze(Hi);
4492 HiShift = DAG.getNode(ISD::SRA, SL, TargetType, Hi, ShiftFullAmt);
4493 NewShift =
4494 DAG.getNode(ISD::SRA, SL, TargetType, Hi, ShiftAmt, N->getFlags());
4495 }
4496
4497 SDValue Vec;
4498 if (VT.isVector()) {
4499 unsigned NElts = TargetType.getVectorNumElements();
4502 SmallVector<SDValue, 16> HiAndLoOps(NElts * 2);
4503
4504 DAG.ExtractVectorElements(HiShift, HiOps, 0, NElts);
4505 DAG.ExtractVectorElements(NewShift, LoOps, 0, NElts);
4506 for (unsigned I = 0; I != NElts; ++I) {
4507 HiAndLoOps[2 * I + 1] = HiOps[I];
4508 HiAndLoOps[2 * I] = LoOps[I];
4509 }
4510 Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, ConcatType, HiAndLoOps);
4511 } else {
4512 Vec = DAG.getBuildVector(ConcatType, SL, {NewShift, HiShift});
4513 }
4514 return DAG.getNode(ISD::BITCAST, SL, VT, Vec);
4515}
4516
4518 DAGCombinerInfo &DCI) const {
4519 SDValue RHS = N->getOperand(1);
4521 EVT VT = N->getValueType(0);
4522 SDValue LHS = N->getOperand(0);
4523 SelectionDAG &DAG = DCI.DAG;
4524 SDLoc SL(N);
4525 unsigned RHSVal;
4526
4527 if (CRHS) {
4528 RHSVal = CRHS->getZExtValue();
4529
4530 // fold (srl (and x, c1 << c2), c2) -> (and (srl(x, c2), c1)
4531 // this improves the ability to match BFE patterns in isel.
4532 if (LHS.getOpcode() == ISD::AND) {
4533 if (auto *Mask = dyn_cast<ConstantSDNode>(LHS.getOperand(1))) {
4534 unsigned MaskIdx, MaskLen;
4535 if (Mask->getAPIntValue().isShiftedMask(MaskIdx, MaskLen) &&
4536 MaskIdx == RHSVal) {
4537 return DAG.getNode(ISD::AND, SL, VT,
4538 DAG.getNode(ISD::SRL, SL, VT, LHS.getOperand(0),
4539 N->getOperand(1)),
4540 DAG.getNode(ISD::SRL, SL, VT, LHS.getOperand(1),
4541 N->getOperand(1)));
4542 }
4543 }
4544 }
4545 }
4546
4547 if (VT.getScalarType() != MVT::i64)
4548 return SDValue();
4549
4550 // for C >= 32
4551 // i64 (srl x, C) -> (build_pair (srl hi_32(x), C - 32), 0)
4552
4553 // On some subtargets, 64-bit shift is a quarter rate instruction. In the
4554 // common case, splitting this into a move and a 32-bit shift is faster and
4555 // the same code size.
4556 KnownBits Known = DAG.computeKnownBits(RHS);
4557
4558 EVT ElementType = VT.getScalarType();
4559 EVT TargetScalarType = ElementType.getHalfSizedIntegerVT(*DAG.getContext());
4560 EVT TargetType = VT.changeElementType(*DAG.getContext(), TargetScalarType);
4561
4562 if (Known.getMinValue().getZExtValue() < TargetScalarType.getSizeInBits())
4563 return SDValue();
4564
4565 SDValue ShiftAmt;
4566 if (CRHS) {
4567 ShiftAmt = DAG.getConstant(RHSVal - TargetScalarType.getSizeInBits(), SL,
4568 TargetType);
4569 } else {
4570 SDValue TruncShiftAmt = DAG.getNode(ISD::TRUNCATE, SL, TargetType, RHS);
4571 const SDValue ShiftMask =
4572 DAG.getConstant(TargetScalarType.getSizeInBits() - 1, SL, TargetType);
4573 // This AND instruction will clamp out of bounds shift values.
4574 // It will also be removed during later instruction selection.
4575 ShiftAmt = DAG.getNode(ISD::AND, SL, TargetType, TruncShiftAmt, ShiftMask);
4576 }
4577
4578 const SDValue Zero = DAG.getConstant(0, SL, TargetScalarType);
4579 EVT ConcatType;
4580 SDValue Hi;
4581 SDLoc LHSSL(LHS);
4582 // Bitcast LHS into ConcatType so hi-half of source can be extracted into Hi
4583 if (VT.isVector()) {
4584 unsigned NElts = TargetType.getVectorNumElements();
4585 ConcatType = TargetType.getDoubleNumVectorElementsVT(*DAG.getContext());
4586 SDValue SplitLHS = DAG.getNode(ISD::BITCAST, LHSSL, ConcatType, LHS);
4587 SmallVector<SDValue, 8> HiOps(NElts);
4588 SmallVector<SDValue, 16> HiAndLoOps;
4589
4590 DAG.ExtractVectorElements(SplitLHS, HiAndLoOps, /*Start=*/0, NElts * 2);
4591 for (unsigned I = 0; I != NElts; ++I)
4592 HiOps[I] = HiAndLoOps[2 * I + 1];
4593 Hi = DAG.getNode(ISD::BUILD_VECTOR, LHSSL, TargetType, HiOps);
4594 } else {
4595 const SDValue One = DAG.getConstant(1, LHSSL, TargetScalarType);
4596 ConcatType = EVT::getVectorVT(*DAG.getContext(), TargetType, 2);
4597 SDValue SplitLHS = DAG.getNode(ISD::BITCAST, LHSSL, ConcatType, LHS);
4598 Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, LHSSL, TargetType, SplitLHS, One);
4599 }
4600
4601 SDValue NewShift =
4602 DAG.getNode(ISD::SRL, SL, TargetType, Hi, ShiftAmt, N->getFlags());
4603
4604 SDValue Vec;
4605 if (VT.isVector()) {
4606 unsigned NElts = TargetType.getVectorNumElements();
4608 SmallVector<SDValue, 16> HiAndLoOps(NElts * 2, Zero);
4609
4610 DAG.ExtractVectorElements(NewShift, LoOps, 0, NElts);
4611 for (unsigned I = 0; I != NElts; ++I)
4612 HiAndLoOps[2 * I] = LoOps[I];
4613 Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, ConcatType, HiAndLoOps);
4614 } else {
4615 Vec = DAG.getBuildVector(ConcatType, SL, {NewShift, Zero});
4616 }
4617 return DAG.getNode(ISD::BITCAST, SL, VT, Vec);
4618}
4619
4621 SDNode *N, DAGCombinerInfo &DCI) const {
4622 SDLoc SL(N);
4623 SelectionDAG &DAG = DCI.DAG;
4624 EVT VT = N->getValueType(0);
4625 SDValue Src = N->getOperand(0);
4626
4627 // vt1 (truncate (bitcast (build_vector vt0:x, ...))) -> vt1 (bitcast vt0:x)
4628 if (Src.getOpcode() == ISD::BITCAST && !VT.isVector()) {
4629 SDValue Vec = Src.getOperand(0);
4630 if (Vec.getOpcode() == ISD::BUILD_VECTOR) {
4631 SDValue Elt0 = Vec.getOperand(0);
4632 EVT EltVT = Elt0.getValueType();
4633 if (VT.getFixedSizeInBits() <= EltVT.getFixedSizeInBits()) {
4634 if (EltVT.isFloatingPoint()) {
4635 Elt0 = DAG.getNode(ISD::BITCAST, SL,
4636 EltVT.changeTypeToInteger(), Elt0);
4637 }
4638
4639 return DAG.getNode(ISD::TRUNCATE, SL, VT, Elt0);
4640 }
4641 }
4642 }
4643
4644 // Equivalent of above for accessing the high element of a vector as an
4645 // integer operation.
4646 // trunc (srl (bitcast (build_vector x, y))), 16 -> trunc (bitcast y)
4647 if (Src.getOpcode() == ISD::SRL && !VT.isVector()) {
4648 if (auto *K = isConstOrConstSplat(Src.getOperand(1))) {
4649 SDValue BV = stripBitcast(Src.getOperand(0));
4650 if (BV.getOpcode() == ISD::BUILD_VECTOR) {
4651 EVT SrcEltVT = BV.getOperand(0).getValueType();
4652 unsigned SrcEltSize = SrcEltVT.getSizeInBits();
4653 unsigned BitIndex = K->getZExtValue();
4654 unsigned PartIndex = BitIndex / SrcEltSize;
4655
4656 if (PartIndex * SrcEltSize == BitIndex &&
4657 PartIndex < BV.getNumOperands()) {
4658 if (SrcEltVT.getSizeInBits() == VT.getSizeInBits()) {
4659 SDValue SrcElt =
4660 DAG.getNode(ISD::BITCAST, SL, SrcEltVT.changeTypeToInteger(),
4661 BV.getOperand(PartIndex));
4662 return DAG.getNode(ISD::TRUNCATE, SL, VT, SrcElt);
4663 }
4664 }
4665 }
4666 }
4667 }
4668
4669 // Partially shrink 64-bit shifts to 32-bit if reduced to 16-bit.
4670 //
4671 // i16 (trunc (srl i64:x, K)), K <= 16 ->
4672 // i16 (trunc (srl (i32 (trunc x), K)))
4673 if (VT.getScalarSizeInBits() < 32) {
4674 EVT SrcVT = Src.getValueType();
4675 if (SrcVT.getScalarSizeInBits() > 32 &&
4676 (Src.getOpcode() == ISD::SRL ||
4677 Src.getOpcode() == ISD::SRA ||
4678 Src.getOpcode() == ISD::SHL)) {
4679 SDValue Amt = Src.getOperand(1);
4680 KnownBits Known = DAG.computeKnownBits(Amt);
4681
4682 // - For left shifts, do the transform as long as the shift
4683 // amount is still legal for i32, so when ShiftAmt < 32 (<= 31)
4684 // - For right shift, do it if ShiftAmt <= (32 - Size) to avoid
4685 // losing information stored in the high bits when truncating.
4686 const unsigned MaxCstSize =
4687 (Src.getOpcode() == ISD::SHL) ? 31 : (32 - VT.getScalarSizeInBits());
4688 if (Known.getMaxValue().ule(MaxCstSize)) {
4689 EVT MidVT = VT.isVector() ?
4690 EVT::getVectorVT(*DAG.getContext(), MVT::i32,
4691 VT.getVectorNumElements()) : MVT::i32;
4692
4693 EVT NewShiftVT = getShiftAmountTy(MidVT, DAG.getDataLayout());
4694 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, MidVT,
4695 Src.getOperand(0));
4696 DCI.AddToWorklist(Trunc.getNode());
4697
4698 if (Amt.getValueType() != NewShiftVT) {
4699 Amt = DAG.getZExtOrTrunc(Amt, SL, NewShiftVT);
4700 DCI.AddToWorklist(Amt.getNode());
4701 }
4702
4703 SDValue ShrunkShift = DAG.getNode(Src.getOpcode(), SL, MidVT,
4704 Trunc, Amt);
4705 return DAG.getNode(ISD::TRUNCATE, SL, VT, ShrunkShift);
4706 }
4707 }
4708 }
4709
4710 return SDValue();
4711}
4712
4713// We need to specifically handle i64 mul here to avoid unnecessary conversion
4714// instructions. If we only match on the legalized i64 mul expansion,
4715// SimplifyDemandedBits will be unable to remove them because there will be
4716// multiple uses due to the separate mul + mulh[su].
4717static SDValue getMul24(SelectionDAG &DAG, const SDLoc &SL,
4718 SDValue N0, SDValue N1, unsigned Size, bool Signed) {
4719 if (Size <= 32) {
4720 unsigned MulOpc = Signed ? AMDGPUISD::MUL_I24 : AMDGPUISD::MUL_U24;
4721 return DAG.getNode(MulOpc, SL, MVT::i32, N0, N1);
4722 }
4723
4724 unsigned MulLoOpc = Signed ? AMDGPUISD::MUL_I24 : AMDGPUISD::MUL_U24;
4725 unsigned MulHiOpc = Signed ? AMDGPUISD::MULHI_I24 : AMDGPUISD::MULHI_U24;
4726
4727 SDValue MulLo = DAG.getNode(MulLoOpc, SL, MVT::i32, N0, N1);
4728 SDValue MulHi = DAG.getNode(MulHiOpc, SL, MVT::i32, N0, N1);
4729
4730 return DAG.getNode(ISD::BUILD_PAIR, SL, MVT::i64, MulLo, MulHi);
4731}
4732
4733/// If \p V is an add of a constant 1, returns the other operand. Otherwise
4734/// return SDValue().
4735static SDValue getAddOneOp(const SDNode *V) {
4736 if (V->getOpcode() != ISD::ADD)
4737 return SDValue();
4738
4739 return isOneConstant(V->getOperand(1)) ? V->getOperand(0) : SDValue();
4740}
4741
4743 DAGCombinerInfo &DCI) const {
4744 assert(N->getOpcode() == ISD::MUL);
4745 EVT VT = N->getValueType(0);
4746
4747 // Don't generate 24-bit multiplies on values that are in SGPRs, since
4748 // we only have a 32-bit scalar multiply (avoid values being moved to VGPRs
4749 // unnecessarily). isDivergent() is used as an approximation of whether the
4750 // value is in an SGPR.
4751 if (!N->isDivergent())
4752 return SDValue();
4753
4754 unsigned Size = VT.getSizeInBits();
4755 if (VT.isVector() || Size > 64)
4756 return SDValue();
4757
4758 SelectionDAG &DAG = DCI.DAG;
4759 SDLoc DL(N);
4760
4761 SDValue N0 = N->getOperand(0);
4762 SDValue N1 = N->getOperand(1);
4763
4764 // Undo InstCombine canonicalize X * (Y + 1) -> X * Y + X to enable mad
4765 // matching.
4766
4767 // mul x, (add y, 1) -> add (mul x, y), x
4768 auto IsFoldableAdd = [](SDValue V) -> SDValue {
4769 SDValue AddOp = getAddOneOp(V.getNode());
4770 if (!AddOp)
4771 return SDValue();
4772
4773 if (V.hasOneUse() || all_of(V->users(), [](const SDNode *U) -> bool {
4774 return U->getOpcode() == ISD::MUL;
4775 }))
4776 return AddOp;
4777
4778 return SDValue();
4779 };
4780
4781 // FIXME: The selection pattern is not properly checking for commuted
4782 // operands, so we have to place the mul in the LHS
4783 if (SDValue MulOper = IsFoldableAdd(N0)) {
4784 SDValue MulVal = DAG.getNode(N->getOpcode(), DL, VT, N1, MulOper);
4785 return DAG.getNode(ISD::ADD, DL, VT, MulVal, N1);
4786 }
4787
4788 if (SDValue MulOper = IsFoldableAdd(N1)) {
4789 SDValue MulVal = DAG.getNode(N->getOpcode(), DL, VT, N0, MulOper);
4790 return DAG.getNode(ISD::ADD, DL, VT, MulVal, N0);
4791 }
4792
4793 // There are i16 integer mul/mad.
4794 if (isTypeLegal(MVT::i16) && VT.getScalarType().bitsLE(MVT::i16))
4795 return SDValue();
4796
4797 // SimplifyDemandedBits has the annoying habit of turning useful zero_extends
4798 // in the source into any_extends if the result of the mul is truncated. Since
4799 // we can assume the high bits are whatever we want, use the underlying value
4800 // to avoid the unknown high bits from interfering.
4801 if (N0.getOpcode() == ISD::ANY_EXTEND)
4802 N0 = N0.getOperand(0);
4803
4804 if (N1.getOpcode() == ISD::ANY_EXTEND)
4805 N1 = N1.getOperand(0);
4806
4807 SDValue Mul;
4808
4809 if (Subtarget->hasMulU24() && isU24(N0, DAG) && isU24(N1, DAG)) {
4810 N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32);
4811 N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32);
4812 Mul = getMul24(DAG, DL, N0, N1, Size, false);
4813 } else if (Subtarget->hasMulI24() && isI24(N0, DAG) && isI24(N1, DAG)) {
4814 N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32);
4815 N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32);
4816 Mul = getMul24(DAG, DL, N0, N1, Size, true);
4817 } else {
4818 return SDValue();
4819 }
4820
4821 // We need to use sext even for MUL_U24, because MUL_U24 is used
4822 // for signed multiply of 8 and 16-bit types.
4823 return DAG.getSExtOrTrunc(Mul, DL, VT);
4824}
4825
4826SDValue
4828 DAGCombinerInfo &DCI) const {
4829 if (N->getValueType(0) != MVT::i32)
4830 return SDValue();
4831
4832 SelectionDAG &DAG = DCI.DAG;
4833 SDLoc DL(N);
4834
4835 bool Signed = N->getOpcode() == ISD::SMUL_LOHI;
4836 SDValue N0 = N->getOperand(0);
4837 SDValue N1 = N->getOperand(1);
4838
4839 // SimplifyDemandedBits has the annoying habit of turning useful zero_extends
4840 // in the source into any_extends if the result of the mul is truncated. Since
4841 // we can assume the high bits are whatever we want, use the underlying value
4842 // to avoid the unknown high bits from interfering.
4843 if (N0.getOpcode() == ISD::ANY_EXTEND)
4844 N0 = N0.getOperand(0);
4845 if (N1.getOpcode() == ISD::ANY_EXTEND)
4846 N1 = N1.getOperand(0);
4847
4848 // Try to use two fast 24-bit multiplies (one for each half of the result)
4849 // instead of one slow extending multiply.
4850 unsigned LoOpcode = 0;
4851 unsigned HiOpcode = 0;
4852 if (Signed) {
4853 if (Subtarget->hasMulI24() && isI24(N0, DAG) && isI24(N1, DAG)) {
4854 N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32);
4855 N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32);
4856 LoOpcode = AMDGPUISD::MUL_I24;
4857 HiOpcode = AMDGPUISD::MULHI_I24;
4858 }
4859 } else {
4860 if (Subtarget->hasMulU24() && isU24(N0, DAG) && isU24(N1, DAG)) {
4861 N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32);
4862 N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32);
4863 LoOpcode = AMDGPUISD::MUL_U24;
4864 HiOpcode = AMDGPUISD::MULHI_U24;
4865 }
4866 }
4867 if (!LoOpcode)
4868 return SDValue();
4869
4870 SDValue Lo = DAG.getNode(LoOpcode, DL, MVT::i32, N0, N1);
4871 SDValue Hi = DAG.getNode(HiOpcode, DL, MVT::i32, N0, N1);
4872 DCI.CombineTo(N, Lo, Hi);
4873 return SDValue(N, 0);
4874}
4875
4877 DAGCombinerInfo &DCI) const {
4878 EVT VT = N->getValueType(0);
4879
4880 if (!Subtarget->hasMulI24() || VT.isVector())
4881 return SDValue();
4882
4883 // Don't generate 24-bit multiplies on values that are in SGPRs, since
4884 // we only have a 32-bit scalar multiply (avoid values being moved to VGPRs
4885 // unnecessarily). isDivergent() is used as an approximation of whether the
4886 // value is in an SGPR.
4887 // This doesn't apply if no s_mul_hi is available (since we'll end up with a
4888 // valu op anyway)
4889 if (Subtarget->hasSMulHi() && !N->isDivergent())
4890 return SDValue();
4891
4892 SelectionDAG &DAG = DCI.DAG;
4893 SDLoc DL(N);
4894
4895 SDValue N0 = N->getOperand(0);
4896 SDValue N1 = N->getOperand(1);
4897
4898 if (!isI24(N0, DAG) || !isI24(N1, DAG))
4899 return SDValue();
4900
4901 N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32);
4902 N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32);
4903
4904 SDValue Mulhi = DAG.getNode(AMDGPUISD::MULHI_I24, DL, MVT::i32, N0, N1);
4905 DCI.AddToWorklist(Mulhi.getNode());
4906 return DAG.getSExtOrTrunc(Mulhi, DL, VT);
4907}
4908
4910 DAGCombinerInfo &DCI) const {
4911 EVT VT = N->getValueType(0);
4912
4913 if (VT.isVector() || VT.getSizeInBits() > 32 || !Subtarget->hasMulU24())
4914 return SDValue();
4915
4916 // Don't generate 24-bit multiplies on values that are in SGPRs, since
4917 // we only have a 32-bit scalar multiply (avoid values being moved to VGPRs
4918 // unnecessarily). isDivergent() is used as an approximation of whether the
4919 // value is in an SGPR.
4920 // This doesn't apply if no s_mul_hi is available (since we'll end up with a
4921 // valu op anyway)
4922 if (!N->isDivergent() && Subtarget->hasSMulHi())
4923 return SDValue();
4924
4925 SelectionDAG &DAG = DCI.DAG;
4926 SDLoc DL(N);
4927
4928 SDValue N0 = N->getOperand(0);
4929 SDValue N1 = N->getOperand(1);
4930
4931 if (!isU24(N0, DAG) || !isU24(N1, DAG))
4932 return SDValue();
4933
4934 N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32);
4935 N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32);
4936
4937 SDValue Mulhi = DAG.getNode(AMDGPUISD::MULHI_U24, DL, MVT::i32, N0, N1);
4938 DCI.AddToWorklist(Mulhi.getNode());
4939 return DAG.getZExtOrTrunc(Mulhi, DL, VT);
4940}
4941
4942SDValue AMDGPUTargetLowering::getFFBX_U32(SelectionDAG &DAG,
4943 SDValue Op,
4944 const SDLoc &DL,
4945 unsigned Opc) const {
4946 EVT VT = Op.getValueType();
4947 if (VT.bitsGT(MVT::i32))
4948 return SDValue();
4949
4950 if (VT != MVT::i32)
4951 Op = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, Op);
4952
4953 SDValue FFBX = DAG.getNode(Opc, DL, MVT::i32, Op);
4954 if (VT != MVT::i32)
4955 FFBX = DAG.getNode(ISD::TRUNCATE, DL, VT, FFBX);
4956
4957 return FFBX;
4958}
4959
4960// The native instructions return -1 on 0 input. Optimize out a select that
4961// produces -1 on 0.
4962//
4963// TODO: If zero is not undef, we could also do this if the output is compared
4964// against the bitwidth.
4965//
4966// TODO: Should probably combine against FFBH_U32 instead of ctlz directly.
4968 SDValue LHS, SDValue RHS,
4969 DAGCombinerInfo &DCI) const {
4970 if (!isNullConstant(Cond.getOperand(1)))
4971 return SDValue();
4972
4973 SelectionDAG &DAG = DCI.DAG;
4974 ISD::CondCode CCOpcode = Cond.getOperandCondCode(2);
4975 SDValue CmpLHS = Cond.getOperand(0);
4976
4977 // select (setcc x, 0, eq), -1, (ctlz_zero_poison x) -> ffbh_u32 x
4978 // select (setcc x, 0, eq), -1, (cttz_zero_poison x) -> ffbl_u32 x
4979 if (CCOpcode == ISD::SETEQ &&
4980 (isCtlzOpc(RHS.getOpcode()) || isCttzOpc(RHS.getOpcode())) &&
4981 RHS.getOperand(0) == CmpLHS && isAllOnesConstant(LHS)) {
4982 unsigned Opc =
4983 isCttzOpc(RHS.getOpcode()) ? AMDGPUISD::FFBL_B32 : AMDGPUISD::FFBH_U32;
4984 return getFFBX_U32(DAG, CmpLHS, SL, Opc);
4985 }
4986
4987 // select (setcc x, 0, ne), (ctlz_zero_poison x), -1 -> ffbh_u32 x
4988 // select (setcc x, 0, ne), (cttz_zero_poison x), -1 -> ffbl_u32 x
4989 if (CCOpcode == ISD::SETNE &&
4990 (isCtlzOpc(LHS.getOpcode()) || isCttzOpc(LHS.getOpcode())) &&
4991 LHS.getOperand(0) == CmpLHS && isAllOnesConstant(RHS)) {
4992 unsigned Opc =
4993 isCttzOpc(LHS.getOpcode()) ? AMDGPUISD::FFBL_B32 : AMDGPUISD::FFBH_U32;
4994
4995 return getFFBX_U32(DAG, CmpLHS, SL, Opc);
4996 }
4997
4998 return SDValue();
4999}
5000
5002 unsigned Op,
5003 const SDLoc &SL,
5004 SDValue Cond,
5005 SDValue N1,
5006 SDValue N2) {
5007 SelectionDAG &DAG = DCI.DAG;
5008 EVT VT = N1.getValueType();
5009
5010 SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, VT, Cond,
5011 N1.getOperand(0), N2.getOperand(0));
5012 DCI.AddToWorklist(NewSelect.getNode());
5013 return DAG.getNode(Op, SL, VT, NewSelect);
5014}
5015
5016// Pull a free FP operation out of a select so it may fold into uses.
5017//
5018// select c, (fneg x), (fneg y) -> fneg (select c, x, y)
5019// select c, (fneg x), k -> fneg (select c, x, (fneg k))
5020//
5021// select c, (fabs x), (fabs y) -> fabs (select c, x, y)
5022// select c, (fabs x), +k -> fabs (select c, x, k)
5023SDValue
5025 SDValue N) const {
5026 SelectionDAG &DAG = DCI.DAG;
5027 SDValue Cond = N.getOperand(0);
5028 SDValue LHS = N.getOperand(1);
5029 SDValue RHS = N.getOperand(2);
5030
5031 EVT VT = N.getValueType();
5032 if ((LHS.getOpcode() == ISD::FABS && RHS.getOpcode() == ISD::FABS) ||
5033 (LHS.getOpcode() == ISD::FNEG && RHS.getOpcode() == ISD::FNEG)) {
5035 return SDValue();
5036
5037 return distributeOpThroughSelect(DCI, LHS.getOpcode(),
5038 SDLoc(N), Cond, LHS, RHS);
5039 }
5040
5041 bool Inv = false;
5042 if (RHS.getOpcode() == ISD::FABS || RHS.getOpcode() == ISD::FNEG) {
5043 std::swap(LHS, RHS);
5044 Inv = true;
5045 }
5046
5047 // TODO: Support vector constants.
5049 if ((LHS.getOpcode() == ISD::FNEG || LHS.getOpcode() == ISD::FABS) && CRHS &&
5050 !selectSupportsSourceMods(N.getNode())) {
5051 SDLoc SL(N);
5052 // If one side is an fneg/fabs and the other is a constant, we can push the
5053 // fneg/fabs down. If it's an fabs, the constant needs to be non-negative.
5054 SDValue NewLHS = LHS.getOperand(0);
5055 SDValue NewRHS = RHS;
5056
5057 // Careful: if the neg can be folded up, don't try to pull it back down.
5058 bool ShouldFoldNeg = true;
5059
5060 if (NewLHS.hasOneUse()) {
5061 unsigned Opc = NewLHS.getOpcode();
5062 if (LHS.getOpcode() == ISD::FNEG && fnegFoldsIntoOp(NewLHS.getNode()))
5063 ShouldFoldNeg = false;
5064 if (LHS.getOpcode() == ISD::FABS && Opc == ISD::FMUL)
5065 ShouldFoldNeg = false;
5066 }
5067
5068 if (ShouldFoldNeg) {
5069 if (LHS.getOpcode() == ISD::FABS && CRHS->isNegative())
5070 return SDValue();
5071
5072 // We're going to be forced to use a source modifier anyway, there's no
5073 // point to pulling the negate out unless we can get a size reduction by
5074 // negating the constant.
5075 //
5076 // TODO: Generalize to use getCheaperNegatedExpression which doesn't know
5077 // about cheaper constants.
5078 if (NewLHS.getOpcode() == ISD::FABS &&
5080 return SDValue();
5081
5083 return SDValue();
5084
5085 if (LHS.getOpcode() == ISD::FNEG)
5086 NewRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
5087
5088 if (Inv)
5089 std::swap(NewLHS, NewRHS);
5090
5091 SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, VT,
5092 Cond, NewLHS, NewRHS);
5093 DCI.AddToWorklist(NewSelect.getNode());
5094 return DAG.getNode(LHS.getOpcode(), SL, VT, NewSelect);
5095 }
5096 }
5097
5098 return SDValue();
5099}
5100
5102 DAGCombinerInfo &DCI) const {
5104 return Folded;
5105
5106 SDValue Cond = N->getOperand(0);
5107 if (Cond.getOpcode() != ISD::SETCC)
5108 return SDValue();
5109
5110 EVT VT = N->getValueType(0);
5111 SDValue LHS = Cond.getOperand(0);
5112 SDValue RHS = Cond.getOperand(1);
5113 SDValue CC = Cond.getOperand(2);
5114
5115 SDValue True = N->getOperand(1);
5116 SDValue False = N->getOperand(2);
5117
5118 if (Cond.hasOneUse()) { // TODO: Look for multiple select uses.
5119 SelectionDAG &DAG = DCI.DAG;
5120 if (DAG.isConstantValueOfAnyType(True) &&
5122 // Swap cmp + select pair to move constant to false input.
5123 // This will allow using VOPC cndmasks more often.
5124 // select (setcc x, y), k, x -> select (setccinv x, y), x, k
5125
5126 SDLoc SL(N);
5127 ISD::CondCode NewCC =
5128 getSetCCInverse(cast<CondCodeSDNode>(CC)->get(), LHS.getValueType());
5129
5130 SDValue NewCond = DAG.getSetCC(SL, Cond.getValueType(), LHS, RHS, NewCC);
5131 return DAG.getNode(ISD::SELECT, SL, VT, NewCond, False, True);
5132 }
5133
5134 if (VT == MVT::f32 && Subtarget->hasFminFmaxLegacy()) {
5135 SDValue MinMax = combineFMinMaxLegacy(SDLoc(N), VT, LHS, RHS, True, False,
5136 CC, N->getFlags(), DCI);
5137 // Revisit this node so we can catch min3/max3/med3 patterns.
5138 //DCI.AddToWorklist(MinMax.getNode());
5139 return MinMax;
5140 }
5141 }
5142
5143 // There's no reason to not do this if the condition has other uses.
5144 return performCtlz_CttzCombine(SDLoc(N), Cond, True, False, DCI);
5145}
5146
5147static bool isInv2Pi(const APFloat &APF) {
5148 static const APFloat KF16(APFloat::IEEEhalf(), APInt(16, 0x3118));
5149 static const APFloat KF32(APFloat::IEEEsingle(), APInt(32, 0x3e22f983));
5150 static const APFloat KF64(APFloat::IEEEdouble(), APInt(64, 0x3fc45f306dc9c882));
5151
5152 return APF.bitwiseIsEqual(KF16) ||
5153 APF.bitwiseIsEqual(KF32) ||
5154 APF.bitwiseIsEqual(KF64);
5155}
5156
5157// 0 and 1.0 / (0.5 * pi) do not have inline immmediates, so there is an
5158// additional cost to negate them.
5161 if (C->isZero())
5162 return C->isNegative() ? NegatibleCost::Cheaper : NegatibleCost::Expensive;
5163
5164 if (Subtarget->hasInv2PiInlineImm() && isInv2Pi(C->getValueAPF()))
5165 return C->isNegative() ? NegatibleCost::Cheaper : NegatibleCost::Expensive;
5166
5168}
5169
5175
5176static unsigned inverseMinMax(unsigned Opc) {
5177 switch (Opc) {
5178 case ISD::FMAXNUM:
5179 return ISD::FMINNUM;
5180 case ISD::FMINNUM:
5181 return ISD::FMAXNUM;
5182 case ISD::FMAXNUM_IEEE:
5183 return ISD::FMINNUM_IEEE;
5184 case ISD::FMINNUM_IEEE:
5185 return ISD::FMAXNUM_IEEE;
5186 case ISD::FMAXIMUM:
5187 return ISD::FMINIMUM;
5188 case ISD::FMINIMUM:
5189 return ISD::FMAXIMUM;
5190 case ISD::FMAXIMUMNUM:
5191 return ISD::FMINIMUMNUM;
5192 case ISD::FMINIMUMNUM:
5193 return ISD::FMAXIMUMNUM;
5194 case AMDGPUISD::FMAX_LEGACY:
5195 return AMDGPUISD::FMIN_LEGACY;
5196 case AMDGPUISD::FMIN_LEGACY:
5197 return AMDGPUISD::FMAX_LEGACY;
5198 default:
5199 llvm_unreachable("invalid min/max opcode");
5200 }
5201}
5202
5203/// \return true if it's profitable to try to push an fneg into its source
5204/// instruction.
5206 // If the input has multiple uses and we can either fold the negate down, or
5207 // the other uses cannot, give up. This both prevents unprofitable
5208 // transformations and infinite loops: we won't repeatedly try to fold around
5209 // a negate that has no 'good' form.
5210 if (N0.hasOneUse()) {
5211 // This may be able to fold into the source, but at a code size cost. Don't
5212 // fold if the fold into the user is free.
5213 if (allUsesHaveSourceMods(N, 0))
5214 return false;
5215 } else {
5216 if (fnegFoldsIntoOp(N0.getNode()) &&
5218 return false;
5219 }
5220
5221 return true;
5222}
5223
5225 DAGCombinerInfo &DCI) const {
5226 SelectionDAG &DAG = DCI.DAG;
5227 SDValue N0 = N->getOperand(0);
5228 EVT VT = N->getValueType(0);
5229
5230 unsigned Opc = N0.getOpcode();
5231
5232 if (!shouldFoldFNegIntoSrc(N, N0))
5233 return SDValue();
5234
5235 bool MayIgnoreSignedZeroForAllUses =
5236 N0->getFlags().hasNoSignedZeros() ||
5237 (N0.hasOneUse() && N->getFlags().hasNoSignedZeros());
5238
5239 SDLoc SL(N);
5240 switch (Opc) {
5241 case ISD::FADD: {
5242 if (!MayIgnoreSignedZeroForAllUses)
5243 return SDValue();
5244
5245 // (fneg (fadd x, y)) -> (fadd (fneg x), (fneg y))
5246 SDValue LHS = N0.getOperand(0);
5247 SDValue RHS = N0.getOperand(1);
5248
5249 if (LHS.getOpcode() != ISD::FNEG)
5250 LHS = DAG.getNode(ISD::FNEG, SL, VT, LHS);
5251 else
5252 LHS = LHS.getOperand(0);
5253
5254 if (RHS.getOpcode() != ISD::FNEG)
5255 RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
5256 else
5257 RHS = RHS.getOperand(0);
5258
5259 SDValue Res = DAG.getNode(ISD::FADD, SL, VT, LHS, RHS, N0->getFlags());
5260 if (Res.getOpcode() != ISD::FADD)
5261 return SDValue(); // Op got folded away.
5262 if (!N0.hasOneUse())
5263 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res));
5264 return Res;
5265 }
5266 case ISD::FMUL:
5267 case AMDGPUISD::FMUL_LEGACY: {
5268 // (fneg (fmul x, y)) -> (fmul x, (fneg y))
5269 // (fneg (fmul_legacy x, y)) -> (fmul_legacy x, (fneg y))
5270 SDValue LHS = N0.getOperand(0);
5271 SDValue RHS = N0.getOperand(1);
5272
5273 if (LHS.getOpcode() == ISD::FNEG)
5274 LHS = LHS.getOperand(0);
5275 else if (RHS.getOpcode() == ISD::FNEG)
5276 RHS = RHS.getOperand(0);
5277 else
5278 RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
5279
5280 SDValue Res = DAG.getNode(Opc, SL, VT, LHS, RHS, N0->getFlags());
5281 if (Res.getOpcode() != Opc)
5282 return SDValue(); // Op got folded away.
5283 if (!N0.hasOneUse())
5284 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res));
5285 return Res;
5286 }
5287 case ISD::FMA:
5288 case ISD::FMAD: {
5289 // TODO: handle llvm.amdgcn.fma.legacy
5290 if (!MayIgnoreSignedZeroForAllUses)
5291 return SDValue();
5292
5293 // (fneg (fma x, y, z)) -> (fma x, (fneg y), (fneg z))
5294 SDValue LHS = N0.getOperand(0);
5295 SDValue MHS = N0.getOperand(1);
5296 SDValue RHS = N0.getOperand(2);
5297
5298 if (LHS.getOpcode() == ISD::FNEG)
5299 LHS = LHS.getOperand(0);
5300 else if (MHS.getOpcode() == ISD::FNEG)
5301 MHS = MHS.getOperand(0);
5302 else
5303 MHS = DAG.getNode(ISD::FNEG, SL, VT, MHS);
5304
5305 if (RHS.getOpcode() != ISD::FNEG)
5306 RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
5307 else
5308 RHS = RHS.getOperand(0);
5309
5310 SDValue Res = DAG.getNode(Opc, SL, VT, LHS, MHS, RHS);
5311 if (Res.getOpcode() != Opc)
5312 return SDValue(); // Op got folded away.
5313 if (!N0.hasOneUse())
5314 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res));
5315 return Res;
5316 }
5317 case ISD::FMAXNUM:
5318 case ISD::FMINNUM:
5319 case ISD::FMAXNUM_IEEE:
5320 case ISD::FMINNUM_IEEE:
5321 case ISD::FMINIMUM:
5322 case ISD::FMAXIMUM:
5323 case ISD::FMINIMUMNUM:
5324 case ISD::FMAXIMUMNUM:
5325 case AMDGPUISD::FMAX_LEGACY:
5326 case AMDGPUISD::FMIN_LEGACY: {
5327 // fneg (fmaxnum x, y) -> fminnum (fneg x), (fneg y)
5328 // fneg (fminnum x, y) -> fmaxnum (fneg x), (fneg y)
5329 // fneg (fmax_legacy x, y) -> fmin_legacy (fneg x), (fneg y)
5330 // fneg (fmin_legacy x, y) -> fmax_legacy (fneg x), (fneg y)
5331
5332 SDValue LHS = N0.getOperand(0);
5333 SDValue RHS = N0.getOperand(1);
5334
5335 // 0 doesn't have a negated inline immediate.
5336 // TODO: This constant check should be generalized to other operations.
5338 return SDValue();
5339
5340 // Swapping min<->max flips which operand a signed zero tie selects.
5341 if ((Opc == AMDGPUISD::FMIN_LEGACY || Opc == AMDGPUISD::FMAX_LEGACY) &&
5342 !canIgnoreLegacyMinMaxTies(DAG, N0->getFlags(), LHS, RHS))
5343 return SDValue();
5344
5345 SDValue NegLHS = DAG.getNode(ISD::FNEG, SL, VT, LHS);
5346 SDValue NegRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
5347 unsigned Opposite = inverseMinMax(Opc);
5348
5349 SDValue Res = DAG.getNode(Opposite, SL, VT, NegLHS, NegRHS, N0->getFlags());
5350 if (Res.getOpcode() != Opposite)
5351 return SDValue(); // Op got folded away.
5352 if (!N0.hasOneUse())
5353 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res));
5354 return Res;
5355 }
5356 case AMDGPUISD::FMED3: {
5357 // med3 sorts a NaN input as smaller than everything regardless of its sign,
5358 // so negating all operands does not sign-flip the median when an input may
5359 // be NaN.
5360 if (!N0->getFlags().hasNoNaNs())
5361 return SDValue();
5362
5363 SDValue Ops[3];
5364 for (unsigned I = 0; I < 3; ++I)
5365 Ops[I] = DAG.getNode(ISD::FNEG, SL, VT, N0->getOperand(I), N0->getFlags());
5366
5367 SDValue Res = DAG.getNode(AMDGPUISD::FMED3, SL, VT, Ops, N0->getFlags());
5368 if (Res.getOpcode() != AMDGPUISD::FMED3)
5369 return SDValue(); // Op got folded away.
5370
5371 if (!N0.hasOneUse()) {
5372 SDValue Neg = DAG.getNode(ISD::FNEG, SL, VT, Res);
5373 DAG.ReplaceAllUsesWith(N0, Neg);
5374
5375 for (SDNode *U : Neg->users())
5376 DCI.AddToWorklist(U);
5377 }
5378
5379 return Res;
5380 }
5381 case ISD::FP_EXTEND:
5382 case ISD::FTRUNC:
5383 case ISD::FRINT:
5384 case ISD::FNEARBYINT: // XXX - Should fround be handled?
5385 case ISD::FROUNDEVEN:
5386 case ISD::FSIN:
5387 case ISD::FCANONICALIZE:
5388 case AMDGPUISD::RCP:
5389 case AMDGPUISD::RCP_LEGACY:
5390 case AMDGPUISD::RCP_IFLAG:
5391 case AMDGPUISD::SIN_HW: {
5392 SDValue CvtSrc = N0.getOperand(0);
5393 if (CvtSrc.getOpcode() == ISD::FNEG) {
5394 // (fneg (fp_extend (fneg x))) -> (fp_extend x)
5395 // (fneg (rcp (fneg x))) -> (rcp x)
5396 return DAG.getNode(Opc, SL, VT, CvtSrc.getOperand(0));
5397 }
5398
5399 if (!N0.hasOneUse())
5400 return SDValue();
5401
5402 // (fneg (fp_extend x)) -> (fp_extend (fneg x))
5403 // (fneg (rcp x)) -> (rcp (fneg x))
5404 SDValue Neg = DAG.getNode(ISD::FNEG, SL, CvtSrc.getValueType(), CvtSrc);
5405 return DAG.getNode(Opc, SL, VT, Neg, N0->getFlags());
5406 }
5407 case ISD::FP_ROUND: {
5408 SDValue CvtSrc = N0.getOperand(0);
5409
5410 if (CvtSrc.getOpcode() == ISD::FNEG) {
5411 // (fneg (fp_round (fneg x))) -> (fp_round x)
5412 return DAG.getNode(ISD::FP_ROUND, SL, VT,
5413 CvtSrc.getOperand(0), N0.getOperand(1));
5414 }
5415
5416 if (!N0.hasOneUse())
5417 return SDValue();
5418
5419 // (fneg (fp_round x)) -> (fp_round (fneg x))
5420 SDValue Neg = DAG.getNode(ISD::FNEG, SL, CvtSrc.getValueType(), CvtSrc);
5421 return DAG.getNode(ISD::FP_ROUND, SL, VT, Neg, N0.getOperand(1));
5422 }
5423 case ISD::FP16_TO_FP: {
5424 // v_cvt_f32_f16 supports source modifiers on pre-VI targets without legal
5425 // f16, but legalization of f16 fneg ends up pulling it out of the source.
5426 // Put the fneg back as a legal source operation that can be matched later.
5427 SDLoc SL(N);
5428
5429 SDValue Src = N0.getOperand(0);
5430 EVT SrcVT = Src.getValueType();
5431
5432 // fneg (fp16_to_fp x) -> fp16_to_fp (xor x, 0x8000)
5433 SDValue IntFNeg = DAG.getNode(ISD::XOR, SL, SrcVT, Src,
5434 DAG.getConstant(0x8000, SL, SrcVT));
5435 return DAG.getNode(ISD::FP16_TO_FP, SL, N->getValueType(0), IntFNeg);
5436 }
5437 case ISD::SELECT: {
5438 // fneg (select c, a, b) -> select c, (fneg a), (fneg b)
5439 // TODO: Invert conditions of foldFreeOpFromSelect
5440 return SDValue();
5441 }
5442 case ISD::BITCAST: {
5443 SDLoc SL(N);
5444 SDValue BCSrc = N0.getOperand(0);
5445 if (BCSrc.getOpcode() == ISD::BUILD_VECTOR) {
5446 SDValue HighBits = BCSrc.getOperand(BCSrc.getNumOperands() - 1);
5447 if (VT != MVT::f64 || HighBits.getValueType().getSizeInBits() != 32 ||
5448 !fnegFoldsIntoOp(HighBits.getNode()))
5449 return SDValue();
5450
5451 // f64 fneg only really needs to operate on the high half of of the
5452 // register, so try to force it to an f32 operation to help make use of
5453 // source modifiers.
5454 //
5455 //
5456 // fneg (f64 (bitcast (build_vector x, y))) ->
5457 // f64 (bitcast (build_vector (bitcast i32:x to f32),
5458 // (fneg (bitcast i32:y to f32)))
5459
5460 SDValue CastHi = DAG.getNode(ISD::BITCAST, SL, MVT::f32, HighBits);
5461 SDValue NegHi = DAG.getNode(ISD::FNEG, SL, MVT::f32, CastHi);
5462 SDValue CastBack =
5463 DAG.getNode(ISD::BITCAST, SL, HighBits.getValueType(), NegHi);
5464
5466 Ops.back() = CastBack;
5467 DCI.AddToWorklist(NegHi.getNode());
5468 SDValue Build =
5469 DAG.getNode(ISD::BUILD_VECTOR, SL, BCSrc.getValueType(), Ops);
5470 SDValue Result = DAG.getNode(ISD::BITCAST, SL, VT, Build);
5471
5472 if (!N0.hasOneUse())
5473 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Result));
5474 return Result;
5475 }
5476
5477 if (BCSrc.getOpcode() == ISD::SELECT && VT == MVT::f32 &&
5478 BCSrc.hasOneUse()) {
5479 // fneg (bitcast (f32 (select cond, i32:lhs, i32:rhs))) ->
5480 // select cond, (bitcast i32:lhs to f32), (bitcast i32:rhs to f32)
5481
5482 // TODO: Cast back result for multiple uses is beneficial in some cases.
5483
5484 SDValue LHS =
5485 DAG.getNode(ISD::BITCAST, SL, MVT::f32, BCSrc.getOperand(1));
5486 SDValue RHS =
5487 DAG.getNode(ISD::BITCAST, SL, MVT::f32, BCSrc.getOperand(2));
5488
5489 SDValue NegLHS = DAG.getNode(ISD::FNEG, SL, MVT::f32, LHS);
5490 SDValue NegRHS = DAG.getNode(ISD::FNEG, SL, MVT::f32, RHS);
5491
5492 return DAG.getNode(ISD::SELECT, SL, MVT::f32, BCSrc.getOperand(0), NegLHS,
5493 NegRHS);
5494 }
5495
5496 return SDValue();
5497 }
5498 default:
5499 return SDValue();
5500 }
5501}
5502
5504 DAGCombinerInfo &DCI) const {
5505 SelectionDAG &DAG = DCI.DAG;
5506 SDValue N0 = N->getOperand(0);
5507
5508 if (!N0.hasOneUse())
5509 return SDValue();
5510
5511 switch (N0.getOpcode()) {
5512 case ISD::FP16_TO_FP: {
5513 assert(!isTypeLegal(MVT::f16) && "should only see if f16 is illegal");
5514 SDLoc SL(N);
5515 SDValue Src = N0.getOperand(0);
5516 EVT SrcVT = Src.getValueType();
5517
5518 // fabs (fp16_to_fp x) -> fp16_to_fp (and x, 0x7fff)
5519 SDValue IntFAbs = DAG.getNode(ISD::AND, SL, SrcVT, Src,
5520 DAG.getConstant(0x7fff, SL, SrcVT));
5521 return DAG.getNode(ISD::FP16_TO_FP, SL, N->getValueType(0), IntFAbs);
5522 }
5523 case ISD::FP_ROUND: {
5524 SDLoc SL(N);
5525 SDValue CvtSrc = N0.getOperand(0);
5526
5527 // fabs (fp_round x) -> fp_round (fabs x)
5528 SDValue Abs = DAG.getNode(ISD::FABS, SL, CvtSrc.getValueType(), CvtSrc,
5529 N->getFlags());
5530 return DAG.getNode(ISD::FP_ROUND, SL, N->getValueType(0), Abs,
5531 N0.getOperand(1), N0->getFlags());
5532 }
5533 default:
5534 return SDValue();
5535 }
5536}
5537
5539 DAGCombinerInfo &DCI) const {
5540 const auto *CFP = dyn_cast<ConstantFPSDNode>(N->getOperand(0));
5541 if (!CFP)
5542 return SDValue();
5543
5544 std::optional<APFloat> Result = AMDGPU::evaluateRcp(CFP->getValueAPF());
5545 if (!Result)
5546 return SDValue();
5547
5548 return DCI.DAG.getConstantFP(*Result, SDLoc(N), N->getValueType(0));
5549}
5550
5552 if (!Subtarget->isGCN())
5553 return false;
5554
5557 auto &ST = DAG.getSubtarget<GCNSubtarget>();
5558 const auto *TII = ST.getInstrInfo();
5559
5560 if (!ST.hasVMovB64Inst() || (!SDConstant && !SDFPConstant))
5561 return false;
5562
5563 if (ST.has64BitLiterals())
5564 return true;
5565
5566 if (SDConstant) {
5567 const APInt &APVal = SDConstant->getAPIntValue();
5568 return isUInt<32>(APVal.getZExtValue()) || TII->isInlineConstant(APVal);
5569 }
5570
5571 APInt Val = SDFPConstant->getValueAPF().bitcastToAPInt();
5572 return isUInt<32>(Val.getZExtValue()) || TII->isInlineConstant(Val);
5573}
5574
5576 DAGCombinerInfo &DCI) const {
5577 SelectionDAG &DAG = DCI.DAG;
5578 SDLoc DL(N);
5579
5580 switch(N->getOpcode()) {
5581 default:
5582 break;
5583 case ISD::BITCAST: {
5584 EVT DestVT = N->getValueType(0);
5585
5586 // Push casts through vector builds. This helps avoid emitting a large
5587 // number of copies when materializing floating point vector constants.
5588 //
5589 // vNt1 bitcast (vNt0 (build_vector t0:x, t0:y)) =>
5590 // vnt1 = build_vector (t1 (bitcast t0:x)), (t1 (bitcast t0:y))
5591 if (DestVT.isVector()) {
5592 SDValue Src = N->getOperand(0);
5593 if (Src.getOpcode() == ISD::BUILD_VECTOR &&
5596 EVT SrcVT = Src.getValueType();
5597 unsigned NElts = DestVT.getVectorNumElements();
5598
5599 if (SrcVT.getVectorNumElements() == NElts) {
5600 EVT DestEltVT = DestVT.getVectorElementType();
5601
5602 SmallVector<SDValue, 8> CastedElts;
5603 SDLoc SL(N);
5604 for (unsigned I = 0, E = SrcVT.getVectorNumElements(); I != E; ++I) {
5605 SDValue Elt = Src.getOperand(I);
5606 CastedElts.push_back(DAG.getNode(ISD::BITCAST, DL, DestEltVT, Elt));
5607 }
5608
5609 return DAG.getBuildVector(DestVT, SL, CastedElts);
5610 }
5611 }
5612 }
5613
5614 if (DestVT.getSizeInBits() != 64 || !DestVT.isVector())
5615 break;
5616
5617 // Fold bitcasts of constants.
5618 //
5619 // v2i32 (bitcast i64:k) -> build_vector lo_32(k), hi_32(k)
5620 // TODO: Generalize and move to DAGCombiner
5621 SDValue Src = N->getOperand(0);
5623 SDLoc SL(N);
5624 if (isInt64ImmLegal(C, DAG))
5625 break;
5626 uint64_t CVal = C->getZExtValue();
5627 SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32,
5628 DAG.getConstant(Lo_32(CVal), SL, MVT::i32),
5629 DAG.getConstant(Hi_32(CVal), SL, MVT::i32));
5630 return DAG.getNode(ISD::BITCAST, SL, DestVT, BV);
5631 }
5632
5634 const APInt &Val = C->getValueAPF().bitcastToAPInt();
5635 SDLoc SL(N);
5636 if (isInt64ImmLegal(C, DAG))
5637 break;
5638 uint64_t CVal = Val.getZExtValue();
5639 SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32,
5640 DAG.getConstant(Lo_32(CVal), SL, MVT::i32),
5641 DAG.getConstant(Hi_32(CVal), SL, MVT::i32));
5642
5643 return DAG.getNode(ISD::BITCAST, SL, DestVT, Vec);
5644 }
5645
5646 break;
5647 }
5648 case ISD::SHL:
5649 case ISD::SRA:
5650 case ISD::SRL: {
5651 // Range metadata can be invalidated when loads are converted to legal types
5652 // (e.g. v2i64 -> v4i32).
5653 // Try to convert vector shl/sra/srl before type legalization so that range
5654 // metadata can be utilized.
5655 if (!(N->getValueType(0).isVector() &&
5658 break;
5659 if (N->getOpcode() == ISD::SHL)
5660 return performShlCombine(N, DCI);
5661 if (N->getOpcode() == ISD::SRA)
5662 return performSraCombine(N, DCI);
5663 return performSrlCombine(N, DCI);
5664 }
5665 case ISD::TRUNCATE:
5666 return performTruncateCombine(N, DCI);
5667 case ISD::MUL:
5668 return performMulCombine(N, DCI);
5669 case AMDGPUISD::MUL_U24:
5670 case AMDGPUISD::MUL_I24: {
5671 if (SDValue Simplified = simplifyMul24(N, DCI))
5672 return Simplified;
5673 break;
5674 }
5675 case AMDGPUISD::MULHI_I24:
5676 case AMDGPUISD::MULHI_U24:
5677 return simplifyMul24(N, DCI);
5678 case ISD::SMUL_LOHI:
5679 case ISD::UMUL_LOHI:
5680 return performMulLoHiCombine(N, DCI);
5681 case ISD::MULHS:
5682 return performMulhsCombine(N, DCI);
5683 case ISD::MULHU:
5684 return performMulhuCombine(N, DCI);
5685 case ISD::SELECT:
5686 return performSelectCombine(N, DCI);
5687 case ISD::FNEG:
5688 return performFNegCombine(N, DCI);
5689 case ISD::FABS:
5690 return performFAbsCombine(N, DCI);
5691 case AMDGPUISD::BFE_I32:
5692 case AMDGPUISD::BFE_U32: {
5693 assert(N->getValueType(0) == MVT::i32 &&
5694 "BFE_I32/BFE_U32 is a 32-bit operation");
5695 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(N->getOperand(2));
5696 if (!Width)
5697 break;
5698
5699 uint32_t WidthVal = Width->getZExtValue() & 0x1f;
5700 if (WidthVal == 0)
5701 return DAG.getConstant(0, DL, MVT::i32);
5702
5704 if (!Offset)
5705 break;
5706
5707 SDValue BitsFrom = N->getOperand(0);
5708 uint32_t OffsetVal = Offset->getZExtValue() & 0x1f;
5709
5710 bool Signed = N->getOpcode() == AMDGPUISD::BFE_I32;
5711
5712 if (OffsetVal == 0) {
5713 // This is already sign / zero extended, so try to fold away extra BFEs.
5714 EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), WidthVal);
5715 if (Signed) {
5716 if (DAG.ComputeNumSignBits(BitsFrom) >= 32 - WidthVal + 1)
5717 return BitsFrom;
5718
5719 // This is a sign_extend_inreg. Replace it to take advantage of existing
5720 // DAG Combines. If not eliminated, we will match back to BFE during
5721 // selection.
5722
5723 // TODO: The sext_inreg of extended types ends, although we can could
5724 // handle them in a single BFE.
5725 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32, BitsFrom,
5726 DAG.getValueType(SmallVT));
5727 }
5728
5729 if (DAG.MaskedValueIsZero(BitsFrom,
5730 APInt::getHighBitsSet(32, 32 - WidthVal)))
5731 return BitsFrom;
5732
5733 return DAG.getZeroExtendInReg(BitsFrom, DL, SmallVT);
5734 }
5735
5736 if (ConstantSDNode *CVal = dyn_cast<ConstantSDNode>(BitsFrom)) {
5737 if (Signed) {
5738 return constantFoldBFE<int32_t>(DAG,
5739 CVal->getSExtValue(),
5740 OffsetVal,
5741 WidthVal,
5742 DL);
5743 }
5744
5745 return constantFoldBFE<uint32_t>(DAG,
5746 CVal->getZExtValue(),
5747 OffsetVal,
5748 WidthVal,
5749 DL);
5750 }
5751
5752 if ((OffsetVal + WidthVal) >= 32 &&
5753 !(OffsetVal == 16 && WidthVal == 16 && Subtarget->hasSDWA())) {
5754 SDValue ShiftVal = DAG.getConstant(OffsetVal, DL, MVT::i32);
5755 return DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, MVT::i32,
5756 BitsFrom, ShiftVal);
5757 }
5758
5759 if (BitsFrom.hasOneUse()) {
5760 APInt Demanded = APInt::getBitsSet(32,
5761 OffsetVal,
5762 OffsetVal + WidthVal);
5763
5766 !DCI.isBeforeLegalizeOps());
5768 if (TLI.ShrinkDemandedConstant(BitsFrom, Demanded, TLO) ||
5769 TLI.SimplifyDemandedBits(BitsFrom, Demanded, Known, TLO)) {
5770 DCI.CommitTargetLoweringOpt(TLO);
5771 }
5772 }
5773
5774 break;
5775 }
5776 case ISD::LOAD:
5777 return performLoadCombine(N, DCI);
5778 case ISD::STORE:
5779 return performStoreCombine(N, DCI);
5780 case AMDGPUISD::RCP:
5781 case AMDGPUISD::RCP_IFLAG:
5782 return performRcpCombine(N, DCI);
5783 case ISD::AssertZext:
5784 case ISD::AssertSext:
5785 return performAssertSZExtCombine(N, DCI);
5787 return performIntrinsicWOChainCombine(N, DCI);
5788 case AMDGPUISD::FMAD_FTZ: {
5789 SDValue N0 = N->getOperand(0);
5790 SDValue N1 = N->getOperand(1);
5791 SDValue N2 = N->getOperand(2);
5792 EVT VT = N->getValueType(0);
5793
5794 // FMAD_FTZ is a FMAD + flush denormals to zero.
5795 // We flush the inputs, the intermediate step, and the output.
5799 if (N0CFP && N1CFP && N2CFP) {
5800 const auto FTZ = [](const APFloat &V) {
5801 if (V.isDenormal()) {
5802 APFloat Zero(V.getSemantics(), 0);
5803 return V.isNegative() ? -Zero : Zero;
5804 }
5805 return V;
5806 };
5807
5808 APFloat V0 = FTZ(N0CFP->getValueAPF());
5809 APFloat V1 = FTZ(N1CFP->getValueAPF());
5810 APFloat V2 = FTZ(N2CFP->getValueAPF());
5811 V0.multiply(V1, APFloat::rmNearestTiesToEven);
5812 V0 = FTZ(V0);
5813 V0.add(V2, APFloat::rmNearestTiesToEven);
5814 return DAG.getConstantFP(FTZ(V0), DL, VT);
5815 }
5816 break;
5817 }
5818 }
5819 return SDValue();
5820}
5821
5823 SDValue Op, const APInt &OriginalDemandedBits,
5824 const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
5825 unsigned Depth) const {
5826 switch (Op.getOpcode()) {
5828 switch (Op.getConstantOperandVal(0)) {
5829 case Intrinsic::amdgcn_readfirstlane:
5830 case Intrinsic::amdgcn_readlane:
5831 case Intrinsic::amdgcn_wwm: {
5832 if (SimplifyDemandedBits(Op.getOperand(1), OriginalDemandedBits,
5833 OriginalDemandedElts, Known, TLO, Depth + 1))
5834 return true;
5835 break;
5836 }
5837 case Intrinsic::amdgcn_set_inactive:
5838 case Intrinsic::amdgcn_set_inactive_chain_arg: {
5839 // The result is operand 1 in active lanes and operand 2 in inactive
5840 // lanes, so the known bits are the intersection of both operands.
5841 KnownBits KnownValue, KnownInactive;
5842 if (SimplifyDemandedBits(Op.getOperand(1), OriginalDemandedBits,
5843 OriginalDemandedElts, KnownValue, TLO,
5844 Depth + 1))
5845 return true;
5846 if (SimplifyDemandedBits(Op.getOperand(2), OriginalDemandedBits,
5847 OriginalDemandedElts, KnownInactive, TLO,
5848 Depth + 1))
5849 return true;
5850 Known = KnownValue.intersectWith(KnownInactive);
5851 break;
5852 }
5853 default:
5854 break;
5855 }
5856 break;
5857 }
5858 default:
5859 break;
5860 }
5861
5862 return false;
5863}
5864
5865//===----------------------------------------------------------------------===//
5866// Helper functions
5867//===----------------------------------------------------------------------===//
5868
5870 const TargetRegisterClass *RC,
5871 Register Reg, EVT VT,
5872 const SDLoc &SL,
5873 bool RawReg) const {
5876 Register VReg;
5877
5878 if (!MRI.isLiveIn(Reg)) {
5879 VReg = MRI.createVirtualRegister(RC);
5880 MRI.addLiveIn(Reg, VReg);
5881 } else {
5882 VReg = MRI.getLiveInVirtReg(Reg);
5883 }
5884
5885 if (RawReg)
5886 return DAG.getRegister(VReg, VT);
5887
5888 return DAG.getCopyFromReg(DAG.getEntryNode(), SL, VReg, VT);
5889}
5890
5891// This may be called multiple times, and nothing prevents creating multiple
5892// objects at the same offset. See if we already defined this object.
5894 int64_t Offset) {
5895 for (int I = MFI.getObjectIndexBegin(); I < 0; ++I) {
5896 if (MFI.getObjectOffset(I) == Offset) {
5897 assert(MFI.getObjectSize(I) == Size);
5898 return I;
5899 }
5900 }
5901
5902 return MFI.CreateFixedObject(Size, Offset, true);
5903}
5904
5906 EVT VT,
5907 const SDLoc &SL,
5908 int64_t Offset) const {
5910 MachineFrameInfo &MFI = MF.getFrameInfo();
5911 int FI = getOrCreateFixedStackObject(MFI, VT.getStoreSize(), Offset);
5912
5913 auto SrcPtrInfo = MachinePointerInfo::getStack(MF, Offset);
5914 SDValue Ptr = DAG.getFrameIndex(FI, MVT::i32);
5915
5916 return DAG.getLoad(VT, SL, DAG.getEntryNode(), Ptr, SrcPtrInfo, Align(4),
5919}
5920
5922 const SDLoc &SL,
5923 SDValue Chain,
5924 SDValue ArgVal,
5925 int64_t Offset) const {
5929
5930 SDValue Ptr = DAG.getConstant(Offset, SL, MVT::i32);
5931 // Stores to the argument stack area are relative to the stack pointer.
5932 SDValue SP =
5933 DAG.getCopyFromReg(Chain, SL, Info->getStackPtrOffsetReg(), MVT::i32);
5934 Ptr = DAG.getNode(ISD::ADD, SL, MVT::i32, SP, Ptr);
5935 SDValue Store = DAG.getStore(Chain, SL, ArgVal, Ptr, DstInfo, Align(4),
5937 return Store;
5938}
5939
5941 const TargetRegisterClass *RC,
5942 EVT VT, const SDLoc &SL,
5943 const ArgDescriptor &Arg) const {
5944 assert(Arg && "Attempting to load missing argument");
5945
5946 SDValue V = Arg.isRegister() ?
5947 CreateLiveInRegister(DAG, RC, Arg.getRegister(), VT, SL) :
5948 loadStackInputValue(DAG, VT, SL, Arg.getStackOffset());
5949
5950 if (!Arg.isMasked())
5951 return V;
5952
5953 unsigned Mask = Arg.getMask();
5954 unsigned Shift = llvm::countr_zero<unsigned>(Mask);
5955 V = DAG.getNode(ISD::SRL, SL, VT, V,
5956 DAG.getShiftAmountConstant(Shift, VT, SL));
5957 return DAG.getNode(ISD::AND, SL, VT, V,
5958 DAG.getConstant(Mask >> Shift, SL, VT));
5959}
5960
5962 uint64_t ExplicitKernArgSize, const ImplicitParameter Param) const {
5963 unsigned ExplicitArgOffset = Subtarget->getExplicitKernelArgOffset();
5964 const Align Alignment = Subtarget->getAlignmentForImplicitArgPtr();
5965 uint64_t ArgOffset =
5966 alignTo(ExplicitKernArgSize, Alignment) + ExplicitArgOffset;
5967 switch (Param) {
5968 case FIRST_IMPLICIT:
5969 return ArgOffset;
5970 case PRIVATE_BASE:
5972 case SHARED_BASE:
5973 return ArgOffset + AMDGPU::ImplicitArg::SHARED_BASE_OFFSET;
5974 case QUEUE_PTR:
5975 return ArgOffset + AMDGPU::ImplicitArg::QUEUE_PTR_OFFSET;
5976 }
5977 llvm_unreachable("unexpected implicit parameter type");
5978}
5979
5986
5988 SelectionDAG &DAG, int Enabled,
5989 int &RefinementSteps,
5990 bool &UseOneConstNR,
5991 bool Reciprocal) const {
5992 EVT VT = Operand.getValueType();
5993
5994 if (VT == MVT::f32) {
5995 RefinementSteps = 0;
5996 return DAG.getNode(AMDGPUISD::RSQ, SDLoc(Operand), VT, Operand);
5997 }
5998
5999 // TODO: There is also f64 rsq instruction, but the documentation is less
6000 // clear on its precision.
6001
6002 return SDValue();
6003}
6004
6006 SelectionDAG &DAG, int Enabled,
6007 int &RefinementSteps) const {
6008 EVT VT = Operand.getValueType();
6009
6010 if (VT == MVT::f32) {
6011 // Reciprocal, < 1 ulp error.
6012 //
6013 // This reciprocal approximation converges to < 0.5 ulp error with one
6014 // newton rhapson performed with two fused multiple adds (FMAs).
6015
6016 RefinementSteps = 0;
6017 return DAG.getNode(AMDGPUISD::RCP, SDLoc(Operand), VT, Operand);
6018 }
6019
6020 // TODO: There is also f64 rcp instruction, but the documentation is less
6021 // clear on its precision.
6022
6023 return SDValue();
6024}
6025
6026static unsigned workitemIntrinsicDim(unsigned ID) {
6027 switch (ID) {
6028 case Intrinsic::amdgcn_workitem_id_x:
6029 return 0;
6030 case Intrinsic::amdgcn_workitem_id_y:
6031 return 1;
6032 case Intrinsic::amdgcn_workitem_id_z:
6033 return 2;
6034 default:
6035 llvm_unreachable("not a workitem intrinsic");
6036 }
6037}
6038
6040 const SDValue Op, KnownBits &Known,
6041 const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const {
6042
6043 Known.resetAll(); // Don't know anything.
6044
6045 unsigned Opc = Op.getOpcode();
6046
6047 switch (Opc) {
6048 default:
6049 break;
6050 case AMDGPUISD::CARRY:
6051 case AMDGPUISD::BORROW: {
6052 Known.Zero = APInt::getHighBitsSet(32, 31);
6053 break;
6054 }
6055
6056 case AMDGPUISD::BFE_I32:
6057 case AMDGPUISD::BFE_U32: {
6058 ConstantSDNode *CWidth = dyn_cast<ConstantSDNode>(Op.getOperand(2));
6059 if (!CWidth)
6060 return;
6061
6062 uint32_t Width = CWidth->getZExtValue() & 0x1f;
6063
6064 if (Opc == AMDGPUISD::BFE_U32)
6065 Known.Zero = APInt::getHighBitsSet(32, 32 - Width);
6066
6067 break;
6068 }
6069 case AMDGPUISD::FP_TO_FP16: {
6070 unsigned BitWidth = Known.getBitWidth();
6071
6072 // High bits are zero.
6074 break;
6075 }
6076 case AMDGPUISD::MUL_U24:
6077 case AMDGPUISD::MUL_I24: {
6078 KnownBits RHSKnown = DAG.computeKnownBits(Op.getOperand(1), Depth + 1);
6079
6080 // TODO: SelfMultiply can be poison, but not undef.
6081 bool SelfMultiply = Op.getOperand(0) == Op.getOperand(1);
6082
6083 // For a self-multiply both operands are the same value, so RHSKnown also
6084 // describes the LHS; avoid the duplicated query.
6085 KnownBits LHSKnown =
6086 SelfMultiply ? RHSKnown
6087 : DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
6088
6089 if (SelfMultiply)
6090 SelfMultiply &= DAG.isGuaranteedNotToBeUndefOrPoison(
6091 Op.getOperand(0), DemandedElts, UndefPoisonKind::UndefOrPoison,
6092 Depth + 1);
6093
6094 Known = computeKnownBitsForMul24(LHSKnown, RHSKnown,
6095 Op.getScalarValueSizeInBits(),
6096 Opc == AMDGPUISD::MUL_I24, SelfMultiply);
6097 break;
6098 }
6099 case AMDGPUISD::PERM: {
6100 ConstantSDNode *CMask = dyn_cast<ConstantSDNode>(Op.getOperand(2));
6101 if (!CMask)
6102 return;
6103
6104 KnownBits LHSKnown = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
6105 KnownBits RHSKnown = DAG.computeKnownBits(Op.getOperand(1), Depth + 1);
6106 unsigned Sel = CMask->getZExtValue();
6107
6108 for (unsigned I = 0; I < 32; I += 8) {
6109 unsigned SelBits = Sel & 0xff;
6110 if (SelBits < 4) {
6111 SelBits *= 8;
6112 Known.One |= ((RHSKnown.One.getZExtValue() >> SelBits) & 0xff) << I;
6113 Known.Zero |= ((RHSKnown.Zero.getZExtValue() >> SelBits) & 0xff) << I;
6114 } else if (SelBits < 7) {
6115 SelBits = (SelBits & 3) * 8;
6116 Known.One |= ((LHSKnown.One.getZExtValue() >> SelBits) & 0xff) << I;
6117 Known.Zero |= ((LHSKnown.Zero.getZExtValue() >> SelBits) & 0xff) << I;
6118 } else if (SelBits == 0x0c) {
6119 Known.Zero |= 0xFFull << I;
6120 } else if (SelBits > 0x0c) {
6121 Known.One |= 0xFFull << I;
6122 }
6123 Sel >>= 8;
6124 }
6125 break;
6126 }
6127 case AMDGPUISD::BUFFER_LOAD_UBYTE: {
6128 Known.Zero.setHighBits(24);
6129 break;
6130 }
6131 case AMDGPUISD::BUFFER_LOAD_USHORT: {
6132 Known.Zero.setHighBits(16);
6133 break;
6134 }
6135 case AMDGPUISD::LDS: {
6136 auto *GA = cast<GlobalAddressSDNode>(Op.getOperand(0).getNode());
6137 Align Alignment = GA->getGlobal()->getPointerAlignment(DAG.getDataLayout());
6138
6139 Known.Zero.setHighBits(16);
6140 Known.Zero.setLowBits(Log2(Alignment));
6141 break;
6142 }
6143 case AMDGPUISD::SMIN3:
6144 case AMDGPUISD::SMAX3:
6145 case AMDGPUISD::SMED3:
6146 case AMDGPUISD::UMIN3:
6147 case AMDGPUISD::UMAX3:
6148 case AMDGPUISD::UMED3: {
6149 KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(2), Depth + 1);
6150 if (Known2.isUnknown())
6151 break;
6152
6153 KnownBits Known1 = DAG.computeKnownBits(Op.getOperand(1), Depth + 1);
6154 if (Known1.isUnknown())
6155 break;
6156
6157 KnownBits Known0 = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
6158 if (Known0.isUnknown())
6159 break;
6160
6161 // TODO: Handle LeadZero/LeadOne from UMIN/UMAX handling.
6162 Known.Zero = Known0.Zero & Known1.Zero & Known2.Zero;
6163 Known.One = Known0.One & Known1.One & Known2.One;
6164 break;
6165 }
6167 unsigned IID = Op.getConstantOperandVal(0);
6168 switch (IID) {
6169 case Intrinsic::amdgcn_workitem_id_x:
6170 case Intrinsic::amdgcn_workitem_id_y:
6171 case Intrinsic::amdgcn_workitem_id_z: {
6172 unsigned MaxValue = Subtarget->getMaxWorkitemID(
6174 Known.Zero.setHighBits(llvm::countl_zero(MaxValue));
6175 break;
6176 }
6177 case Intrinsic::amdgcn_readfirstlane:
6178 case Intrinsic::amdgcn_readlane:
6179 // Result is the data operand's value from some lane.
6180 Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
6181 break;
6182 default:
6183 break;
6184 }
6185 }
6186 }
6187}
6188
6190 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
6191 unsigned Depth) const {
6192 switch (Op.getOpcode()) {
6193 case AMDGPUISD::BFE_I32: {
6194 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(Op.getOperand(2));
6195 if (!Width)
6196 return 1;
6197
6198 unsigned SignBits = 32 - (Width->getZExtValue() & 0x1f) + 1;
6199 if (!isNullConstant(Op.getOperand(1)))
6200 return SignBits;
6201
6202 // TODO: Could probably figure something out with non-0 offsets.
6203 unsigned Op0SignBits = DAG.ComputeNumSignBits(Op.getOperand(0), Depth + 1);
6204 return std::max(SignBits, Op0SignBits);
6205 }
6206
6207 case AMDGPUISD::BFE_U32: {
6208 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(Op.getOperand(2));
6209 return Width ? 32 - (Width->getZExtValue() & 0x1f) : 1;
6210 }
6211
6212 case AMDGPUISD::CARRY:
6213 case AMDGPUISD::BORROW:
6214 return 31;
6215 case AMDGPUISD::BUFFER_LOAD_BYTE:
6216 return 25;
6217 case AMDGPUISD::BUFFER_LOAD_SHORT:
6218 return 17;
6219 case AMDGPUISD::BUFFER_LOAD_UBYTE:
6220 return 24;
6221 case AMDGPUISD::BUFFER_LOAD_USHORT:
6222 return 16;
6223 case AMDGPUISD::FP_TO_FP16:
6224 return 16;
6225 case AMDGPUISD::SMIN3:
6226 case AMDGPUISD::SMAX3:
6227 case AMDGPUISD::SMED3:
6228 case AMDGPUISD::UMIN3:
6229 case AMDGPUISD::UMAX3:
6230 case AMDGPUISD::UMED3: {
6231 unsigned Tmp2 = DAG.ComputeNumSignBits(Op.getOperand(2), Depth + 1);
6232 if (Tmp2 == 1)
6233 return 1; // Early out.
6234
6235 unsigned Tmp1 = DAG.ComputeNumSignBits(Op.getOperand(1), Depth + 1);
6236 if (Tmp1 == 1)
6237 return 1; // Early out.
6238
6239 unsigned Tmp0 = DAG.ComputeNumSignBits(Op.getOperand(0), Depth + 1);
6240 if (Tmp0 == 1)
6241 return 1; // Early out.
6242
6243 return std::min({Tmp0, Tmp1, Tmp2});
6244 }
6245 default:
6246 return 1;
6247 }
6248}
6249
6251 GISelValueTracking &Analysis, Register R, const APInt &DemandedElts,
6252 const MachineRegisterInfo &MRI, unsigned Depth) const {
6253 const MachineInstr *MI = MRI.getVRegDef(R);
6254 if (!MI)
6255 return 1;
6256
6257 // TODO: Check range metadata on MMO.
6258 switch (MI->getOpcode()) {
6259 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE:
6260 return 25;
6261 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT:
6262 return 17;
6263 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE:
6264 return 24;
6265 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT:
6266 return 16;
6267 case AMDGPU::G_AMDGPU_SMED3:
6268 case AMDGPU::G_AMDGPU_UMED3: {
6269 auto [Dst, Src0, Src1, Src2] = MI->getFirst4Regs();
6270 unsigned Tmp2 = Analysis.computeNumSignBits(Src2, DemandedElts, Depth + 1);
6271 if (Tmp2 == 1)
6272 return 1;
6273 unsigned Tmp1 = Analysis.computeNumSignBits(Src1, DemandedElts, Depth + 1);
6274 if (Tmp1 == 1)
6275 return 1;
6276 unsigned Tmp0 = Analysis.computeNumSignBits(Src0, DemandedElts, Depth + 1);
6277 if (Tmp0 == 1)
6278 return 1;
6279 return std::min({Tmp0, Tmp1, Tmp2});
6280 }
6281 default:
6282 return 1;
6283 }
6284}
6285
6287 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
6288 UndefPoisonKind Kind, bool ConsiderFlags, unsigned Depth) const {
6289 unsigned Opcode = Op.getOpcode();
6290 switch (Opcode) {
6291 case AMDGPUISD::BFE_I32:
6292 case AMDGPUISD::BFE_U32:
6293 return false;
6294 }
6296 Op, DemandedElts, DAG, Kind, ConsiderFlags, Depth);
6297}
6298
6300 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool SNaN,
6301 unsigned Depth) const {
6302 unsigned Opcode = Op.getOpcode();
6303 switch (Opcode) {
6304 case AMDGPUISD::FMIN_LEGACY:
6305 case AMDGPUISD::FMAX_LEGACY:
6306 return DAG.isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) &&
6307 DAG.isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1);
6308 case AMDGPUISD::FMUL_LEGACY:
6309 case AMDGPUISD::CVT_PKRTZ_F16_F32: {
6310 if (SNaN)
6311 return true;
6312 return DAG.isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) &&
6313 DAG.isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1);
6314 }
6315 case AMDGPUISD::FMED3:
6316 case AMDGPUISD::FMIN3:
6317 case AMDGPUISD::FMAX3:
6318 case AMDGPUISD::FMINIMUM3:
6319 case AMDGPUISD::FMAXIMUM3:
6320 case AMDGPUISD::FMAD_FTZ: {
6321 if (SNaN)
6322 return true;
6323 return DAG.isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) &&
6324 DAG.isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) &&
6325 DAG.isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1);
6326 }
6327 case AMDGPUISD::CVT_F32_UBYTE0:
6328 case AMDGPUISD::CVT_F32_UBYTE1:
6329 case AMDGPUISD::CVT_F32_UBYTE2:
6330 case AMDGPUISD::CVT_F32_UBYTE3:
6331 return true;
6332
6333 case AMDGPUISD::RCP:
6334 case AMDGPUISD::RSQ:
6335 case AMDGPUISD::RCP_LEGACY:
6336 case AMDGPUISD::RSQ_CLAMP: {
6337 if (SNaN)
6338 return true;
6339
6340 // TODO: Need is known positive check.
6341 return false;
6342 }
6343 case ISD::FLDEXP:
6344 case AMDGPUISD::FRACT: {
6345 if (SNaN)
6346 return true;
6347 return DAG.isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
6348 }
6349 case AMDGPUISD::DIV_SCALE:
6350 case AMDGPUISD::DIV_FMAS:
6351 case AMDGPUISD::DIV_FIXUP:
6352 // TODO: Refine on operands.
6353 return SNaN;
6354 case AMDGPUISD::SIN_HW:
6355 case AMDGPUISD::COS_HW: {
6356 // TODO: Need check for infinity
6357 return SNaN;
6358 }
6360 unsigned IntrinsicID = Op.getConstantOperandVal(0);
6361 // TODO: Handle more intrinsics
6362 switch (IntrinsicID) {
6363 case Intrinsic::amdgcn_cubeid:
6364 case Intrinsic::amdgcn_cvt_off_f32_i4:
6365 return true;
6366
6367 case Intrinsic::amdgcn_frexp_mant: {
6368 if (SNaN)
6369 return true;
6370 return DAG.isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1);
6371 }
6372 case Intrinsic::amdgcn_cvt_pkrtz: {
6373 if (SNaN)
6374 return true;
6375 return DAG.isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) &&
6376 DAG.isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1);
6377 }
6378 case Intrinsic::amdgcn_rcp:
6379 case Intrinsic::amdgcn_rsq:
6380 case Intrinsic::amdgcn_rcp_legacy:
6381 case Intrinsic::amdgcn_rsq_legacy:
6382 case Intrinsic::amdgcn_rsq_clamp:
6383 case Intrinsic::amdgcn_tanh: {
6384 if (SNaN)
6385 return true;
6386
6387 // TODO: Need is known positive check.
6388 return false;
6389 }
6390 case Intrinsic::amdgcn_trig_preop:
6391 case Intrinsic::amdgcn_fdot2:
6392 // TODO: Refine on operand
6393 return SNaN;
6394 case Intrinsic::amdgcn_fma_legacy:
6395 if (SNaN)
6396 return true;
6397 return DAG.isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) &&
6398 DAG.isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1) &&
6399 DAG.isKnownNeverNaN(Op.getOperand(3), SNaN, Depth + 1);
6400 default:
6401 return false;
6402 }
6403 }
6404 default:
6405 return false;
6406 }
6407}
6408
6410 Register N0, Register N1) const {
6411 return MRI.hasOneNonDBGUse(N0); // FIXME: handle regbanks
6412}
return SDValue()
const TargetLowering & TLI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
const AArch64InstrInfo & TII
AMDGPU address space definition.
static LLVM_READONLY bool hasSourceMods(const MachineInstr &MI)
static bool isInv2Pi(const APFloat &APF)
static LLVM_READONLY bool opMustUseVOP3Encoding(const MachineInstr &MI, const MachineRegisterInfo &MRI)
returns true if the operation will definitely need to use a 64-bit encoding, and thus will use a VOP3...
static unsigned inverseMinMax(unsigned Opc)
uint64_t Imm
static SDValue extractF64Exponent(SDValue Hi, const SDLoc &SL, SelectionDAG &DAG)
static unsigned workitemIntrinsicDim(unsigned ID)
static int getOrCreateFixedStackObject(MachineFrameInfo &MFI, unsigned Size, int64_t Offset)
static SDValue constantFoldBFE(SelectionDAG &DAG, IntTy Src0, uint32_t Offset, uint32_t Width, const SDLoc &DL)
static SDValue getMad(SelectionDAG &DAG, const SDLoc &SL, EVT VT, SDValue X, SDValue Y, SDValue C, SDNodeFlags Flags=SDNodeFlags())
static SDValue getAddOneOp(const SDNode *V)
If V is an add of a constant 1, returns the other operand.
static bool canIgnoreLegacyMinMaxTies(const SelectionDAG &DAG, SDNodeFlags Flags, SDValue LHS, SDValue RHS)
static LLVM_READONLY bool selectSupportsSourceMods(const SDNode *N)
Return true if v_cndmask_b32 will support fabs/fneg source modifiers for the type for ISD::SELECT.
static SDValue getMul24(SelectionDAG &DAG, const SDLoc &SL, SDValue N0, SDValue N1, unsigned Size, bool Signed)
static bool fnegFoldsIntoOp(const SDNode *N)
static bool isI24(SDValue Op, SelectionDAG &DAG)
static bool isCttzOpc(unsigned Opc)
static bool isU24(SDValue Op, SelectionDAG &DAG)
static bool valueIsKnownNeverF32Denorm(SDValue Src)
Return true if it's known that Src can never be an f32 denormal value.
static SDValue distributeOpThroughSelect(TargetLowering::DAGCombinerInfo &DCI, unsigned Op, const SDLoc &SL, SDValue Cond, SDValue N1, SDValue N2)
static SDValue peekFNeg(SDValue Val)
static SDValue simplifyMul24(SDNode *Node24, TargetLowering::DAGCombinerInfo &DCI)
static bool isCtlzOpc(unsigned Opc)
static LLVM_READNONE bool fnegFoldsIntoOpcode(unsigned Opc)
static bool hasVolatileUser(SDNode *Val)
Interface definition of the TargetLowering class that is common to all AMD GPUs.
Contains the definition of a TargetInstrInfo class that is common to all AMD GPUs.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis Results
#define X(NUM, ENUM, NAME)
Definition ELF.h:665
block Block Frequency Analysis
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
#define LLVM_READNONE
Definition Compiler.h:327
#define LLVM_READONLY
Definition Compiler.h:334
Provides analysis for querying information about KnownBits during GISel passes.
static MaybeAlign getAlign(Value *Ptr)
IRTranslator LLVM IR MI
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
#define G(x, y, z)
Definition MD5.cpp:55
#define T
#define P(N)
MachineBasicBlock * Folded
SmallVector< MachineOperand, 4 > Cond
int64_t C1
RISCVCC::CondCode CC
MachineRegisterInfo & MRI
#define CH(x, y, z)
Definition SHA256.cpp:34
Func MI getDebugLoc()))
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
TTI::PartialReductionExtendKind ExtendKind
Value * RHS
Value * LHS
BinaryOperator * Mul
static CCAssignFn * CCAssignFnForCall(CallingConv::ID CC, bool IsVarArg)
static CCAssignFn * CCAssignFnForReturn(CallingConv::ID CC, bool IsVarArg)
unsigned allocateBarrierGlobal(const DataLayout &DL, const GlobalVariable &GV)
static std::optional< uint32_t > get32BitAbsoluteAddress(const GlobalValue &GV, unsigned AS)
unsigned allocateLDSGlobal(const DataLayout &DL, const GlobalVariable &GV)
static unsigned numBitsSigned(SDValue Op, SelectionDAG &DAG)
unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const override
This method can be implemented by targets that want to expose additional information about sign bits ...
SDValue performMulhuCombine(SDNode *N, DAGCombinerInfo &DCI) const
EVT getTypeForExtReturn(LLVMContext &Context, EVT VT, ISD::NodeType ExtendKind) const override
Return the type that should be used to zero or sign extend a zeroext/signext integer return value.
SDValue SplitVectorLoad(SDValue Op, SelectionDAG &DAG) const
Split a vector load into 2 loads of half the vector.
SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const
SDValue performLoadCombine(SDNode *N, DAGCombinerInfo &DCI) const
void analyzeFormalArgumentsCompute(CCState &State, const SmallVectorImpl< ISD::InputArg > &Ins) const
The SelectionDAGBuilder will automatically promote function arguments with illegal types.
SDValue LowerF64ToF16Safe(SDValue Src, const SDLoc &DL, SelectionDAG &DAG) const
SDValue LowerFROUND(SDValue Op, SelectionDAG &DAG) const
SDValue storeStackInputValue(SelectionDAG &DAG, const SDLoc &SL, SDValue Chain, SDValue ArgVal, int64_t Offset) const
bool storeOfVectorConstantIsCheap(bool IsZero, EVT MemVT, unsigned NumElem, unsigned AS) const override
Return true if it is expected to be cheaper to do a store of vector constant with the given size and ...
SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
bool shouldCombineMemoryType(EVT VT) const
SDValue splitBinaryBitConstantOpImpl(DAGCombinerInfo &DCI, const SDLoc &SL, unsigned Opc, SDValue LHS, uint32_t ValLo, uint32_t ValHi) const
Split the 64-bit value LHS into two 32-bit components, and perform the binary operation Opc to it wit...
SDValue lowerUnhandledCall(CallLoweringInfo &CLI, SmallVectorImpl< SDValue > &InVals, StringRef Reason) const
virtual SDValue LowerGlobalAddress(AMDGPUMachineFunctionInfo *MFI, SDValue Op, SelectionDAG &DAG) const
SDValue performAssertSZExtCombine(SDNode *N, DAGCombinerInfo &DCI) const
bool isTruncateFree(EVT Src, EVT Dest) const override
bool aggressivelyPreferBuildVectorSources(EVT VecVT) const override
SDValue LowerFCEIL(SDValue Op, SelectionDAG &DAG) const
TargetLowering::NegatibleCost getConstantNegateCost(const ConstantFPSDNode *C) const
SDValue LowerFLOGUnsafe(SDValue Op, const SDLoc &SL, SelectionDAG &DAG, bool IsLog10, SDNodeFlags Flags) const
SDValue combineFMinMaxLegacy(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, SDValue True, SDValue False, SDValue CC, SDNodeFlags Flags, DAGCombinerInfo &DCI) const
Flags must be the select flags, not the compare (SELECT_CC flags come from the fcmp and say nothing a...
SDValue performMulhsCombine(SDNode *N, DAGCombinerInfo &DCI) const
SDValue lowerFEXPUnsafeImpl(SDValue Op, const SDLoc &SL, SelectionDAG &DAG, SDNodeFlags Flags, bool IsExp10) const
bool isSDNodeAlwaysUniform(const SDNode *N) const override
bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to move this shift by a constant amount through its operand,...
SDValue performShlCombine(SDNode *N, DAGCombinerInfo &DCI) const
bool isCheapToSpeculateCtlz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic ctlz.
SDValue LowerSDIVREM(SDValue Op, SelectionDAG &DAG) const
bool isFNegFree(EVT VT) const override
Return true if an fneg operation is free to the point where it is never worthwhile to replace it with...
FPImmAction getFPImmAction(const APFloat &Imm, EVT VT, bool ForCodeSize) const override
Returns how the legalizer should materialize the specified FP immediate.
SDValue LowerINT_TO_FP64(SDValue Op, SelectionDAG &DAG, bool Signed) const
unsigned computeNumSignBitsForTargetInstr(GISelValueTracking &Analysis, Register R, const APInt &DemandedElts, const MachineRegisterInfo &MRI, unsigned Depth=0) const override
This method can be implemented by targets that want to expose additional information about sign bits ...
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
SDValue LowerFP_TO_FP16(SDValue Op, SelectionDAG &DAG) const
SDValue addTokenForArgument(SDValue Chain, SelectionDAG &DAG, MachineFrameInfo &MFI, int ClobberedFI) const
bool isReassocProfitable(MachineRegisterInfo &MRI, Register N0, Register N1) const override
bool isKnownNeverNaNForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool SNaN=false, unsigned Depth=0) const override
If SNaN is false,.
static bool needsDenormHandlingF32(const SelectionDAG &DAG, SDValue Src, SDNodeFlags Flags)
SDValue lowerFPOW(SDValue Op, SelectionDAG &DAG) const
uint32_t getImplicitParameterOffset(const MachineFunction &MF, const ImplicitParameter Param) const
Helper function that returns the byte offset of the given type of implicit parameter.
SDValue lowerFEXPF64(SDValue Op, SelectionDAG &DAG) const
SDValue LowerFFLOOR(SDValue Op, SelectionDAG &DAG) const
SDValue performSelectCombine(SDNode *N, DAGCombinerInfo &DCI) const
SDValue performFNegCombine(SDNode *N, DAGCombinerInfo &DCI) const
SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const
bool isConstantCostlierToNegate(SDValue N) const
static KnownBits computeKnownBitsForMul24(const KnownBits &LHSKnown, const KnownBits &RHSKnown, unsigned BitWidth, bool IsSigned, bool SelfMultiply)
SDValue loadInputValue(SelectionDAG &DAG, const TargetRegisterClass *RC, EVT VT, const SDLoc &SL, const ArgDescriptor &Arg) const
SDValue lowerFEXP10Unsafe(SDValue Op, const SDLoc &SL, SelectionDAG &DAG, SDNodeFlags Flags) const
Emit approx-funcs appropriate lowering for exp10.
bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtType, EVT ExtVT, std::optional< unsigned > ByteOffset) const override
Return true if it is profitable to reduce a load to a smaller type.
SDValue LowerUINT_TO_FP(SDValue Op, SelectionDAG &DAG) const
bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, UndefPoisonKind Kind, bool ConsiderFlags, unsigned Depth) const override
Return true if Op can create undef or poison from non-undef & non-poison operands.
bool isCheapToSpeculateCttz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic cttz.
SDValue performCtlz_CttzCombine(const SDLoc &SL, SDValue Cond, SDValue LHS, SDValue RHS, DAGCombinerInfo &DCI) const
SDValue performSraCombine(SDNode *N, DAGCombinerInfo &DCI) const
bool isSelectSupported(SelectSupportKind) const override
bool isZExtFree(Type *Src, Type *Dest) const override
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
SDValue lowerFEXP2(SDValue Op, SelectionDAG &DAG) const
SDValue LowerCall(CallLoweringInfo &CLI, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower calls into the specified DAG.
SDValue performSrlCombine(SDNode *N, DAGCombinerInfo &DCI) const
SDValue lowerFEXP(SDValue Op, SelectionDAG &DAG) const
SDValue getIsFinite(SelectionDAG &DAG, SDValue Op, SDNodeFlags Flags) const
bool isLoadBitCastBeneficial(EVT, EVT, const SelectionDAG &DAG, const MachineMemOperand &MMO) const final
Return true if the following transform is beneficial: fold (conv (load x)) -> (load (conv*)x) On arch...
std::pair< SDValue, SDValue > splitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HighVT, SelectionDAG &DAG) const
Split a vector value into two parts of types LoVT and HiVT.
AMDGPUTargetLowering(const TargetMachine &TM, const TargetSubtargetInfo &STI, const AMDGPUSubtarget &AMDGPUSTI)
SDValue LowerFLOGCommon(SDValue Op, SelectionDAG &DAG) const
SDValue foldFreeOpFromSelect(TargetLowering::DAGCombinerInfo &DCI, SDValue N) const
SDValue LowerINT_TO_FP32(SDValue Op, SelectionDAG &DAG, bool Signed) const
bool isFAbsFree(EVT VT) const override
Return true if an fabs operation is free to the point where it is never worthwhile to replace it with...
bool isInt64ImmLegal(SDNode *Val, SelectionDAG &DAG) const
Check whether value Val can be supported by v_mov_b64, for the current target.
SDValue loadStackInputValue(SelectionDAG &DAG, EVT VT, const SDLoc &SL, int64_t Offset) const
Similar to CreateLiveInRegister, except value maybe loaded from a stack slot rather than passed in a ...
SDValue LowerFLOG2(SDValue Op, SelectionDAG &DAG) const
static EVT getEquivalentMemType(LLVMContext &Context, EVT VT)
SDValue LowerCTLS(SDValue Op, SelectionDAG &DAG) const
Split a vector store into multiple scalar stores.
SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, int &RefinementSteps, bool &UseOneConstNR, bool Reciprocal) const override
Hooks for building estimates in place of slower divisions and square roots.
SDValue performTruncateCombine(SDNode *N, DAGCombinerInfo &DCI) const
SDValue LowerSINT_TO_FP(SDValue Op, SelectionDAG &DAG) const
static SDValue stripBitcast(SDValue Val)
SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const
SDValue CreateLiveInRegister(SelectionDAG &DAG, const TargetRegisterClass *RC, Register Reg, EVT VT, const SDLoc &SL, bool RawReg=false) const
Helper function that adds Reg to the LiveIn list of the DAG's MachineFunction.
SDValue SplitVectorStore(SDValue Op, SelectionDAG &DAG) const
Split a vector store into 2 stores of half the vector.
SDValue LowerCTLZ_CTTZ(SDValue Op, SelectionDAG &DAG) const
SDValue getNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOperations, bool ForCodeSize, NegatibleCost &Cost, unsigned Depth) const override
Return the newly negated expression if the cost is not expensive and set the cost in Cost to indicate...
std::pair< SDValue, SDValue > split64BitValue(SDValue Op, SelectionDAG &DAG) const
Return 64-bit value Op as two 32-bit integers.
SDValue performMulCombine(SDNode *N, DAGCombinerInfo &DCI) const
SDValue combineFMinMaxLegacyImpl(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, SDValue True, SDValue False, SDValue CC, SDNodeFlags Flags, DAGCombinerInfo &DCI) const
SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, int &RefinementSteps) const override
Return a reciprocal estimate value for the input operand.
SDValue LowerFNEARBYINT(SDValue Op, SelectionDAG &DAG) const
SDValue LowerSIGN_EXTEND_INREG(SDValue Op, SelectionDAG &DAG) const
static CCAssignFn * CCAssignFnForReturn(CallingConv::ID CC, bool IsVarArg)
std::pair< SDValue, SDValue > getScaledLogInput(SelectionDAG &DAG, const SDLoc SL, SDValue Op, SDNodeFlags Flags) const
If denormal handling is required return the scaled input to FLOG2, and the check for denormal range.
static CCAssignFn * CCAssignFnForCall(CallingConv::ID CC, bool IsVarArg)
Selects the correct CCAssignFn for a given CallingConvention value.
bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &OriginalDemandedBits, const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const override
Attempt to simplify any target nodes based on the demanded bits/elts, returning true on success.
static bool allUsesHaveSourceMods(const SDNode *N, unsigned CostThreshold=4)
SDValue LowerFROUNDEVEN(SDValue Op, SelectionDAG &DAG) const
static unsigned numBitsUnsigned(SDValue Op, SelectionDAG &DAG)
SDValue lowerFEXPUnsafe(SDValue Op, const SDLoc &SL, SelectionDAG &DAG, SDNodeFlags Flags) const
SDValue LowerFTRUNC(SDValue Op, SelectionDAG &DAG) const
SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const
static bool allowApproxFunc(const SelectionDAG &DAG, SDNodeFlags Flags)
bool ShouldShrinkFPConstant(EVT VT) const override
If true, then instruction selection should seek to shrink the FP constant of the specified type to a ...
SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, const SmallVectorImpl< SDValue > &OutVals, const SDLoc &DL, SelectionDAG &DAG) const override
This hook must be implemented to lower outgoing return values, described by the Outs array,...
SDValue performStoreCombine(SDNode *N, DAGCombinerInfo &DCI) const
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
This callback is invoked when a node result type is illegal for the target, and the operation was reg...
SDValue performRcpCombine(SDNode *N, DAGCombinerInfo &DCI) const
SDValue lowerCTLZResults(SDValue Op, SelectionDAG &DAG) const
SDValue LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG) const
SDValue performFAbsCombine(SDNode *N, DAGCombinerInfo &DCI) const
SDValue LowerFP_TO_INT64(SDValue Op, SelectionDAG &DAG, bool Signed) const
static bool shouldFoldFNegIntoSrc(SDNode *FNeg, SDValue FNegSrc)
bool isNarrowingProfitable(SDNode *N, EVT SrcVT, EVT DestVT) const override
Return true if it's profitable to narrow operations of type SrcVT to DestVT.
SDValue LowerFRINT(SDValue Op, SelectionDAG &DAG) const
SDValue performIntrinsicWOChainCombine(SDNode *N, DAGCombinerInfo &DCI) const
SDValue LowerUDIVREM(SDValue Op, SelectionDAG &DAG) const
SDValue performMulLoHiCombine(SDNode *N, DAGCombinerInfo &DCI) const
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
SDValue lowerINT_TO_FPImpl(SDValue Op, SelectionDAG &DAG, bool Signed) const
void LowerUDIVREM64(SDValue Op, SelectionDAG &DAG, SmallVectorImpl< SDValue > &Results) const
SDValue WidenOrSplitVectorLoad(SDValue Op, SelectionDAG &DAG) const
Widen a suitably aligned v3 load.
SDValue LowerDIVREMToFloat(SDValue Op, SelectionDAG &DAG, bool sign) const
std::pair< EVT, EVT > getSplitDestVTs(const EVT &VT, SelectionDAG &DAG) const
Split a vector type into two parts.
SDValue getHiHalf64(SDValue Op, SelectionDAG &DAG) const
SDValue LowerINT_TO_FP16(SDValue Op, SelectionDAG &DAG, EVT FP16Ty) const
unsigned getVectorIdxWidth(const DataLayout &) const override
Returns the type to be used for the index operand vector operations.
static const fltSemantics & IEEEsingle()
Definition APFloat.h:304
static const fltSemantics & IEEEdouble()
Definition APFloat.h:305
static constexpr roundingMode rmNearestTiesToEven
Definition APFloat.h:361
static const fltSemantics & IEEEhalf()
Definition APFloat.h:302
static APFloat getQNaN(const fltSemantics &Sem, bool Negative=false, const APInt *payload=nullptr)
Factory for QNaN values.
Definition APFloat.h:1224
bool bitwiseIsEqual(const APFloat &RHS) const
Definition APFloat.h:1548
static APFloat getSmallestNormalized(const fltSemantics &Sem, bool Negative=false)
Returns the smallest (by magnitude) normalized finite number in the given semantics.
Definition APFloat.h:1262
APInt bitcastToAPInt() const
Definition APFloat.h:1475
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
Definition APFloat.h:1202
Class for arbitrary precision integers.
Definition APInt.h:78
uint64_t getZExtValue() const
Get zero extended value.
Definition APInt.h:1560
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
Definition APInt.h:202
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
Definition APInt.h:254
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
Definition APInt.h:205
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
Definition APInt.h:215
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
Definition APInt.h:302
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
Definition APInt.h:292
This class represents an incoming formal argument to a Function.
Definition Argument.h:32
const BlockAddress * getBlockAddress() const
CCState - This class holds information needed while lowering arguments and return values.
static CCValAssign getCustomMem(unsigned ValNo, MVT ValVT, int64_t Offset, MVT LocVT, LocInfo HTP)
const APFloat & getValueAPF() const
bool isNegative() const
Return true if the value is negative.
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
Diagnostic information for unsupported feature in backend.
const DataLayout & getDataLayout() const
Get the data layout of the module this function belongs to.
Definition Function.cpp:356
iterator_range< arg_iterator > args()
Definition Function.h:886
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition Function.h:273
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition Function.cpp:352
bool signBitIsZeroOrNaN() const
Return true if the sign bit must be 0, ignoring the sign of nans.
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:76
LLVM_ABI void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
Machine Value Type.
static auto integer_fixedlen_vector_valuetypes()
uint64_t getScalarSizeInBits() const
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
static auto integer_valuetypes()
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
LLVM_ABI int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
int getObjectIndexBegin() const
Return the minimum frame object index.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
DenormalMode getDenormalMode(const fltSemantics &FPType) const
Returns the denormal handling type for the default rounding mode of the function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Representation of each machine instruction.
A description of a memory reference used in the backend.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOInvariant
The memory access always returns the same value (or traps).
Flags getFlags() const
Return the raw flags of the source value,.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
This is an abstract virtual class for memory operations.
unsigned getAddressSpace() const
Return the address space for the associated pointer.
Align getAlign() const
bool isSimple() const
Returns true if the memory operation is neither atomic or volatile.
MachineMemOperand * getMemOperand() const
Return the unique MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getChain() const
bool isInvariant() const
EVT getMemoryVT() const
Return the type of the in-memory value.
Wrapper class representing virtual and physical registers.
Definition Register.h:20
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
const DebugLoc & getDebugLoc() const
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
SDNodeFlags getFlags() const
SDVTList getVTList() const
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
iterator_range< user_iterator > users()
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node, in exactly one operand.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
unsigned getOpcode() const
unsigned getNumOperands() const
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
SIModeRegisterDefaults getMode() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
LLVM_ABI bool isKnownNeverLogicalZero(SDValue Op, const APInt &DemandedElts, unsigned Depth=0) const
Test whether the given floating point SDValue (or all elements of it, if it is a vector) is known to ...
SDValue getExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT, unsigned Opcode)
Convert Op, which must be of integer type, to the integer type VT, by either any/sign/zero-extending ...
LLVM_ABI unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
const TargetSubtargetInfo & getSubtarget() const
LLVM_ABI SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
LLVM_ABI SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
LLVM_ABI SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
LLVM_ABI SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
LLVM_ABI void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
LLVM_ABI SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
LLVM_ABI SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
LLVM_ABI SDValue getRegister(Register Reg, EVT VT)
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false, SDNodeFlags Flags={})
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
LLVM_ABI SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
const TargetLowering & getTargetLoweringInfo() const
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
LLVM_ABI SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
LLVM_ABI SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, Register Reg, EVT VT)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
LLVM_ABI SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
LLVM_ABI SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Helper function to build ISD::STORE nodes.
LLVM_ABI SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
LLVM_ABI void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
LLVM_ABI SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
LLVM_ABI SDValue getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
bool isConstantValueOfAnyType(SDValue N) const
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
LLVM_ABI SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
LLVM_ABI SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Loads are not normal binary operators: their result type is not determined by their operands,...
LLVM_ABI KnownFPClass computeKnownFPClass(SDValue Op, FPClassTest InterestedClasses, unsigned Depth=0) const
Determine floating-point class information about Op.
LLVM_ABI bool isGuaranteedNotToBeUndefOrPoison(SDValue Op, UndefPoisonKind Kind=UndefPoisonKind::UndefOrPoison, unsigned Depth=0) const
Return true if this function can prove that Op is never poison and, Kind can be used to track poison ...
LLVM_ABI SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI SDValue getValueType(EVT)
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
LLVM_ABI bool isKnownNeverNaN(SDValue Op, const APInt &DemandedElts, bool SNaN=false, unsigned Depth=0) const
Test whether the given SDValue (or all elements of it, if it is a vector) is known to never be NaN in...
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
LLVM_ABI unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, unsigned TargetFlags=0)
LLVM_ABI SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getPOISON(EVT VT)
Return a POISON node. POISON does not have a useful SDLoc.
LLVM_ABI SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
LLVM_ABI KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
LLVM_ABI SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
LLVM_ABI bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
LLVM_ABI SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
LLVM_ABI std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getValue() const
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
void setMaxDivRemBitWidthSupported(unsigned SizeInBits)
Set the size in bits of the maximum div/rem the backend supports.
bool PredictableSelectIsExpensive
Tells the code generator that select is more expensive than a branch if the branch is usually predict...
virtual bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT, std::optional< unsigned > ByteOffset=std::nullopt) const
Return true if it is profitable to reduce a load to a smaller type.
unsigned MaxStoresPerMemcpyOptSize
Likewise for functions with the OptSize attribute.
const TargetMachine & getTargetMachine() const
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
unsigned MaxGluedStoresPerMemcpy
Specify max number of store instructions to glue in inlined memcpy.
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
void addBypassSlowDiv(unsigned int SlowBitWidth, unsigned int FastBitWidth)
Tells the code generator which bitwidths to bypass.
void setMaxLargeFPConvertBitWidthSupported(unsigned SizeInBits)
Set the size in bits of the maximum fp to/from int conversion the backend supports.
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
SelectSupportKind
Enum that describes what type of support for selects the target has.
virtual bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *=nullptr) const
Determine if the target supports unaligned memory accesses.
unsigned MaxStoresPerMemsetOptSize
Likewise for functions with the OptSize attribute.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const
Returns the type for the shift amount of a shift opcode.
unsigned MaxStoresPerMemmove
Specify maximum number of store instructions per memmove call.
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
unsigned MaxStoresPerMemmoveOptSize
Likewise for functions with the OptSize attribute.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
void setSupportsUnalignedAtomics(bool UnalignedSupported)
Sets whether unaligned atomic operations are supported.
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
unsigned MaxStoresPerMemset
Specify maximum number of store instructions per memset call.
FPImmAction
How the legalizer should materialize a floating-point immediate.
@ ConstantPool
Load the immediate from the constant pool.
@ Legal
The target can instruction select the immediate natively.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
void setMinCmpXchgSizeInBits(unsigned SizeInBits)
Sets the minimum cmpxchg or ll/sc size supported by the backend.
void AddPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
If Opc/OrigVT is specified as being promoted, the promotion code defaults to trying a larger integer/...
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
unsigned GatherAllAliasesMaxDepth
Depth that GatherAllAliases should continue looking for chain dependencies when trying to find a more...
NegatibleCost
Enum that specifies when a float negation is beneficial.
bool allowsMemoryAccessForAlignment(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
This function returns true if the memory access is aligned or if the target allows this specific unal...
unsigned MaxStoresPerMemcpy
Specify maximum number of store instructions per memcpy call.
void setSchedulingPreference(Sched::Preference Pref)
Specify the target scheduling preference.
void setJumpIsExpensive(bool isExpensive=true)
Tells the code generator not to expand logic operations on comparison predicates into separate sequen...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue scalarizeVectorStore(StoreSDNode *ST, SelectionDAG &DAG) const
SDValue expandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG) const
Expands an unaligned store to 2 half-size stores for integer values, and possibly more for vectors.
std::pair< SDValue, SDValue > expandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG) const
Expands an unaligned load to 2 half-size loads for an integer, and possibly more for vectors.
virtual SDValue getNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOps, bool OptForSize, NegatibleCost &Cost, unsigned Depth=0) const
Return the newly negated expression if the cost is not expensive and set the cost in Cost to indicate...
std::pair< SDValue, SDValue > scalarizeVectorLoad(LoadSDNode *LD, SelectionDAG &DAG) const
Turn load of vector type into a load of the individual elements.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
TargetLowering(const TargetLowering &)=delete
virtual bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, UndefPoisonKind Kind, bool ConsiderFlags, unsigned Depth) const
Return true if Op can create undef or poison from non-undef & non-poison operands.
Primary interface to the complete machine description for the target machine.
TargetSubtargetInfo - Generic base class for all target subtargets.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition TypeSize.h:352
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Definition Type.h:354
LLVM Value Representation.
Definition Value.h:75
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:314
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ CONSTANT_ADDRESS_32BIT
Address space for 32-bit constant memory.
@ BARRIER
Address space for modeling barrier IDs as addresses.
@ REGION_ADDRESS
Address space for region memory. (GDS)
@ LOCAL_ADDRESS
Address space for local memory.
@ CONSTANT_ADDRESS
Address space for constant memory (VTX2).
@ GLOBAL_ADDRESS
Address space for global memory (RAT0, VTX0).
bool isIntrinsicAlwaysUniform(unsigned IntrID)
TargetExtType * isNamedBarrier(const GlobalVariable &GV)
std::optional< APFloat > evaluateRcp(const APFloat &Val)
Evaluate the constant-folded result of v_rcp for Val, accounting for the hardware's denormal flushing...
bool isUniformMMO(const MachineMemOperand *MMO)
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition CallingConv.h:24
@ AMDGPU_CS
Used for Mesa/AMDPAL compute shaders.
@ AMDGPU_VS
Used for Mesa vertex shaders, or AMDPAL last shader stage before rasterization (vertex shader if tess...
@ AMDGPU_KERNEL
Used for AMDGPU code object kernels.
@ AMDGPU_Gfx
Used for AMD graphics targets.
@ AMDGPU_CS_ChainPreserve
Used on AMDGPUs to give the middle-end more control over argument placement.
@ AMDGPU_HS
Used for Mesa/AMDPAL hull shaders (= tessellation control shaders).
@ AMDGPU_GS
Used for Mesa/AMDPAL geometry shaders.
@ AMDGPU_CS_Chain
Used on AMDGPUs to give the middle-end more control over argument placement.
@ AMDGPU_PS
Used for Mesa/AMDPAL pixel shaders.
@ Cold
Attempts to make code in the caller as efficient as possible under the assumption that the call is no...
Definition CallingConv.h:47
@ SPIR_KERNEL
Used for SPIR kernel functions.
@ Fast
Attempts to make calls as fast as possible (e.g.
Definition CallingConv.h:41
@ AMDGPU_ES
Used for AMDPAL shader stage before geometry shader if geometry is in use.
@ AMDGPU_LS
Used for AMDPAL vertex shader if tessellation is in use.
@ C
The default llvm calling convention, compatible with C.
Definition CallingConv.h:34
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition ISDOpcodes.h:43
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition ISDOpcodes.h:851
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition ISDOpcodes.h:277
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition ISDOpcodes.h:605
@ BSWAP
Byte Swap and Counting operators.
Definition ISDOpcodes.h:811
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
Definition ISDOpcodes.h:527
@ ADD
Simple integer binary arithmetic operators.
Definition ISDOpcodes.h:266
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition ISDOpcodes.h:885
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition ISDOpcodes.h:523
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition ISDOpcodes.h:912
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition ISDOpcodes.h:589
@ FADD
Simple binary floating point operators.
Definition ISDOpcodes.h:420
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition ISDOpcodes.h:771
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition ISDOpcodes.h:282
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition ISDOpcodes.h:256
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ CTLZ_ZERO_POISON
Definition ISDOpcodes.h:820
@ SIGN_EXTEND
Conversion operators.
Definition ISDOpcodes.h:876
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BRIND
BRIND - Indirect branch.
@ BR_JT
BR_JT - Jumptable branch.
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition ISDOpcodes.h:546
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
Definition ISDOpcodes.h:553
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition ISDOpcodes.h:828
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
Definition ISDOpcodes.h:249
@ CTLS
Count leading redundant sign bits.
Definition ISDOpcodes.h:824
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition ISDOpcodes.h:720
@ STRICT_FP16_TO_FP
@ SHL
Shift and rotation operations.
Definition ISDOpcodes.h:793
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition ISDOpcodes.h:659
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition ISDOpcodes.h:619
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Same as FMINNUM/FMAXNUM, except that a signaling NaN operand deterministi...
@ EntryToken
EntryToken - This is the marker used to indicate the start of a region.
Definition ISDOpcodes.h:50
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition ISDOpcodes.h:581
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
Definition ISDOpcodes.h:226
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition ISDOpcodes.h:882
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition ISDOpcodes.h:843
@ FMINNUM
FMINNUM/FMAXNUM - Same semantics as the llvm.minnum/maxnum intrinsics.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition ISDOpcodes.h:920
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition ISDOpcodes.h:751
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition ISDOpcodes.h:837
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
Definition ISDOpcodes.h:331
@ INLINEASM_BR
INLINEASM_BR - Branching version of inline asm. Used by asm-goto.
@ FMINIMUM
FMINIMUM/FMAXIMUM - Same semantics as the llvm.minimum/maximum intrinsics.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition ISDOpcodes.h:958
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition ISDOpcodes.h:763
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition ISDOpcodes.h:207
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition ISDOpcodes.h:570
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition ISDOpcodes.h:55
@ CTTZ_ZERO_POISON
Bit counting operators with a poisoned result for zero inputs.
Definition ISDOpcodes.h:819
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition ISDOpcodes.h:991
@ ADDRSPACECAST
ADDRSPACECAST - This operator converts between pointers of different address spaces.
@ INLINEASM
INLINEASM - Represents an inline asm block.
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition ISDOpcodes.h:977
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition ISDOpcodes.h:888
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition ISDOpcodes.h:64
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition ISDOpcodes.h:539
@ FMINIMUMNUM
FMINIMUMNUM/FMAXIMUMNUM - Same semantics as the llvm.minimumnum/maximumnum intrinsics.
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition ISDOpcodes.h:215
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition ISDOpcodes.h:561
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
LLVM_ABI CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
constexpr double ln2
constexpr double ln10
constexpr float log2ef
Definition MathExtras.h:52
constexpr double log2e
This is an optimization pass for GlobalISel generic memory operations.
@ Offset
Definition DWP.cpp:577
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
InstructionCost Cost
LLVM_ABI bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
@ Known
Known to have no common set bits.
LLVM_ABI void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< EVT > *MemVTs=nullptr, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
Definition Analysis.cpp:121
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
bool CCAssignFn(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, Type *OrigTy, CCState &State)
CCAssignFn - This function assigns a location for Val, updating State to reflect the change.
@ Load
The value being inserted comes from a load (InsertElement only).
@ Store
The extracted value is stored (ExtractElement only).
SDValue peekFPSignOps(SDValue Val)
Strip fabs/fneg/fcopysign from a value to get the underlying source.
LLVM_ABI ConstantFPSDNode * isConstOrConstSplatFP(SDValue N, bool AllowUndefs=false)
Returns the SDNode if it is a constant splat BuildVector or constant float.
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
Definition MathExtras.h:380
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition bit.h:204
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
Definition bit.h:263
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
constexpr uint32_t Hi_32(uint64_t Value)
Return the high 32 bits of a 64 bit value.
Definition MathExtras.h:151
constexpr uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition Alignment.h:149
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
Definition MathExtras.h:190
constexpr uint32_t Lo_32(uint64_t Value)
Return the low 32 bits of a 64 bit value.
Definition MathExtras.h:156
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
CombineLevel
Definition DAGCombine.h:15
@ AfterLegalizeDAG
Definition DAGCombine.h:19
@ BeforeLegalizeTypes
Definition DAGCombine.h:16
@ AfterLegalizeTypes
Definition DAGCombine.h:17
To bit_cast(const From &from) noexcept
Definition bit.h:90
@ Mul
Product of integers.
@ Add
Sum of integers.
@ Fast
Assign the register banks as fast as possible (default).
DWARFExpression::Operation Op
LLVM_ABI ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
constexpr unsigned BitWidth
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
LLVM_ABI bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
UndefPoisonKind
Enumeration to track whether we are interested in Undef, Poison, or both.
Definition UndefPoison.h:20
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition Alignment.h:206
APFloat neg(APFloat X)
Returns the negated value of the argument.
Definition APFloat.h:1742
unsigned Log2(Align A)
Returns the log2 of the alignment.
Definition Alignment.h:202
LLVM_ABI bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
MCRegisterClass TargetRegisterClass
Definition FastISel.h:58
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
Definition Error.cpp:177
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:881
#define N
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
MCRegister getRegister() const
unsigned getStackOffset() const
DenormalModeKind Input
Denormal treatment kind for floating point instruction inputs in the default floating-point environme...
@ PreserveSign
The sign of a flushed-to-zero number is preserved in the sign of 0.
constexpr bool inputsAreZero() const
Return true if input denormals must be implicitly treated as 0.
static constexpr DenormalMode getPreserveSign()
Extended Value Type.
Definition ValueTypes.h:35
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition ValueTypes.h:433
EVT getPow2VectorType(LLVMContext &Context) const
Widens the length of the given vector EVT up to the nearest power of 2 and returns that type.
Definition ValueTypes.h:523
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition ValueTypes.h:160
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition ValueTypes.h:70
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
Definition ValueTypes.h:144
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
Definition ValueTypes.h:322
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition ValueTypes.h:170
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition ValueTypes.h:411
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition ValueTypes.h:281
uint64_t getScalarSizeInBits() const
Definition ValueTypes.h:423
EVT getHalfSizedIntegerVT(LLVMContext &Context) const
Finds the smallest simple value type that is greater than or equal to half the width of this EVT.
Definition ValueTypes.h:468
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
Definition ValueTypes.h:516
TypeSize getStoreSizeInBits() const
Return the number of bits overwritten by a store of the specified value type.
Definition ValueTypes.h:450
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition ValueTypes.h:354
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition ValueTypes.h:61
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
Definition ValueTypes.h:419
EVT getRoundIntegerType(LLVMContext &Context) const
Rounds the bit-width of the given integer EVT up to the nearest power of two (and at least to eight),...
Definition ValueTypes.h:457
bool isVector() const
Return true if this is a vector value type.
Definition ValueTypes.h:191
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition ValueTypes.h:361
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition ValueTypes.h:330
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition ValueTypes.h:366
bool isExtended() const
Test if the given EVT is extended (as opposed to being simple).
Definition ValueTypes.h:165
EVT changeElementType(LLVMContext &Context, EVT EltVT) const
Return a VT for a type whose attributes match ourselves with the exception of the element type that i...
Definition ValueTypes.h:136
LLVM_ABI const fltSemantics & getFltSemantics() const
Returns an APFloat semantics tag appropriate for the value type.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition ValueTypes.h:374
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition ValueTypes.h:346
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition ValueTypes.h:175
InputArg - This struct carries flags and type information about a single incoming (formal) argument o...
MVT VT
Legalized type of this argument part.
bool isUnknown() const
Returns true if we don't know any bits.
Definition KnownBits.h:64
KnownBits trunc(unsigned BitWidth) const
Return known bits for a truncation of the value we're tracking.
Definition KnownBits.h:165
KnownBits zext(unsigned BitWidth) const
Return known bits for a zero extension of the value we're tracking.
Definition KnownBits.h:176
unsigned countMaxActiveBits() const
Returns the maximum number of bits needed to represent all possible unsigned values with these known ...
Definition KnownBits.h:310
KnownBits intersectWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for both this and RHS.
Definition KnownBits.h:325
KnownBits sext(unsigned BitWidth) const
Return known bits for a sign extension of the value we're tracking.
Definition KnownBits.h:184
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
Definition KnownBits.h:262
bool isNegative() const
Returns true if this value is known to be negative.
Definition KnownBits.h:103
static LLVM_ABI KnownBits mul(const KnownBits &LHS, const KnownBits &RHS, bool NoUndefSelfMultiply=false)
Compute known bits resulting from multiplying LHS and RHS.
Matching combinators.
This class contains a discriminated union of information about pointers in memory operands,...
LLVM_ABI bool isDereferenceable(unsigned Size, LLVMContext &C, const DataLayout &DL) const
Return true if memory region [V, V+Offset+Size) is known to be dereferenceable.
static LLVM_ABI MachinePointerInfo getStack(MachineFunction &MF, int64_t Offset, uint8_t ID=0)
Stack pointer relative access.
MachinePointerInfo getWithOffset(int64_t O) const
These are IR-level optimization flags that may be propagated to SDNodes.
void setAllowContract(bool b)
bool hasNoSignedZeros() const
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
DenormalMode FP32Denormals
If this is set, neither input or output denormals are flushed for most f32 instructions.
This structure contains all information that is necessary for lowering calls.
SmallVector< ISD::InputArg, 32 > Ins
LLVM_ABI void AddToWorklist(SDNode *N)
LLVM_ABI SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
LLVM_ABI void CommitTargetLoweringOpt(const TargetLoweringOpt &TLO)
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...